fbcrawl-colly 0.2.6 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -2
- data/Dockerfile +14 -0
- data/Gemfile.lock +8 -7
- data/README.md +1 -1
- data/fbcrawl-colly.gemspec +2 -6
- data/fbcrawl.proto +76 -3
- data/{fbcolly → fbcrawl}/fbcolly.go +63 -56
- data/fbcrawl/pb/fbcrawl.pb.go +1647 -0
- data/fbcrawl/pb/fbcrawl_grpc.pb.go +380 -0
- data/go.mod +4 -1
- data/go.sum +10 -0
- data/lib/fbcrawl-colly.rb +1 -4
- data/lib/fbcrawl_colly.rb +5 -0
- data/lib/fbcrawl_colly/client.rb +46 -0
- data/lib/fbcrawl_colly/version.rb +1 -1
- data/lib/pb/fbcrawl_pb.rb +117 -0
- data/lib/pb/fbcrawl_services_pb.rb +31 -0
- data/main.go +74 -77
- metadata +14 -26
- data/ext/fbcrawl_colly/.gitignore +0 -2
- data/ext/fbcrawl_colly/Makefile +0 -6
- data/ext/fbcrawl_colly/extconf.rb +0 -6
- data/lib/fbcrawl_colly/colly.rb +0 -57
- data/lib/fbcrawl_colly/ffi.rb +0 -18
@@ -0,0 +1,380 @@
|
|
1
|
+
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
2
|
+
|
3
|
+
package pb
|
4
|
+
|
5
|
+
import (
|
6
|
+
context "context"
|
7
|
+
grpc "google.golang.org/grpc"
|
8
|
+
codes "google.golang.org/grpc/codes"
|
9
|
+
status "google.golang.org/grpc/status"
|
10
|
+
)
|
11
|
+
|
12
|
+
// This is a compile-time assertion to ensure that this generated file
|
13
|
+
// is compatible with the grpc package it is being compiled against.
|
14
|
+
const _ = grpc.SupportPackageIsVersion6
|
15
|
+
|
16
|
+
// GrpcClient is the client API for Grpc service.
|
17
|
+
//
|
18
|
+
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
19
|
+
type GrpcClient interface {
|
20
|
+
// Sends a greeting
|
21
|
+
Init(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Pointer, error)
|
22
|
+
FreeColly(ctx context.Context, in *Pointer, opts ...grpc.CallOption) (*Empty, error)
|
23
|
+
Login(ctx context.Context, in *LoginRequest, opts ...grpc.CallOption) (*LoginResponse, error)
|
24
|
+
LoginWithCookies(ctx context.Context, in *LoginWithCookiesRequest, opts ...grpc.CallOption) (*Empty, error)
|
25
|
+
FetchGroupInfo(ctx context.Context, in *FetchGroupInfoRequest, opts ...grpc.CallOption) (*FacebookGroup, error)
|
26
|
+
FetchGroupFeed(ctx context.Context, in *FetchGroupFeedRequest, opts ...grpc.CallOption) (*FacebookPostList, error)
|
27
|
+
FetchPost(ctx context.Context, in *FetchPostRequest, opts ...grpc.CallOption) (*FacebookPost, error)
|
28
|
+
FetchContentImages(ctx context.Context, in *FetchContentImagesRequest, opts ...grpc.CallOption) (*FacebookImageList, error)
|
29
|
+
FetchImageUrl(ctx context.Context, in *FetchImageUrlRequest, opts ...grpc.CallOption) (*FacebookImage, error)
|
30
|
+
}
|
31
|
+
|
32
|
+
type grpcClient struct {
|
33
|
+
cc grpc.ClientConnInterface
|
34
|
+
}
|
35
|
+
|
36
|
+
func NewGrpcClient(cc grpc.ClientConnInterface) GrpcClient {
|
37
|
+
return &grpcClient{cc}
|
38
|
+
}
|
39
|
+
|
40
|
+
func (c *grpcClient) Init(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Pointer, error) {
|
41
|
+
out := new(Pointer)
|
42
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/Init", in, out, opts...)
|
43
|
+
if err != nil {
|
44
|
+
return nil, err
|
45
|
+
}
|
46
|
+
return out, nil
|
47
|
+
}
|
48
|
+
|
49
|
+
func (c *grpcClient) FreeColly(ctx context.Context, in *Pointer, opts ...grpc.CallOption) (*Empty, error) {
|
50
|
+
out := new(Empty)
|
51
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/FreeColly", in, out, opts...)
|
52
|
+
if err != nil {
|
53
|
+
return nil, err
|
54
|
+
}
|
55
|
+
return out, nil
|
56
|
+
}
|
57
|
+
|
58
|
+
func (c *grpcClient) Login(ctx context.Context, in *LoginRequest, opts ...grpc.CallOption) (*LoginResponse, error) {
|
59
|
+
out := new(LoginResponse)
|
60
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/Login", in, out, opts...)
|
61
|
+
if err != nil {
|
62
|
+
return nil, err
|
63
|
+
}
|
64
|
+
return out, nil
|
65
|
+
}
|
66
|
+
|
67
|
+
func (c *grpcClient) LoginWithCookies(ctx context.Context, in *LoginWithCookiesRequest, opts ...grpc.CallOption) (*Empty, error) {
|
68
|
+
out := new(Empty)
|
69
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/LoginWithCookies", in, out, opts...)
|
70
|
+
if err != nil {
|
71
|
+
return nil, err
|
72
|
+
}
|
73
|
+
return out, nil
|
74
|
+
}
|
75
|
+
|
76
|
+
func (c *grpcClient) FetchGroupInfo(ctx context.Context, in *FetchGroupInfoRequest, opts ...grpc.CallOption) (*FacebookGroup, error) {
|
77
|
+
out := new(FacebookGroup)
|
78
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/FetchGroupInfo", in, out, opts...)
|
79
|
+
if err != nil {
|
80
|
+
return nil, err
|
81
|
+
}
|
82
|
+
return out, nil
|
83
|
+
}
|
84
|
+
|
85
|
+
func (c *grpcClient) FetchGroupFeed(ctx context.Context, in *FetchGroupFeedRequest, opts ...grpc.CallOption) (*FacebookPostList, error) {
|
86
|
+
out := new(FacebookPostList)
|
87
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/FetchGroupFeed", in, out, opts...)
|
88
|
+
if err != nil {
|
89
|
+
return nil, err
|
90
|
+
}
|
91
|
+
return out, nil
|
92
|
+
}
|
93
|
+
|
94
|
+
func (c *grpcClient) FetchPost(ctx context.Context, in *FetchPostRequest, opts ...grpc.CallOption) (*FacebookPost, error) {
|
95
|
+
out := new(FacebookPost)
|
96
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/FetchPost", in, out, opts...)
|
97
|
+
if err != nil {
|
98
|
+
return nil, err
|
99
|
+
}
|
100
|
+
return out, nil
|
101
|
+
}
|
102
|
+
|
103
|
+
func (c *grpcClient) FetchContentImages(ctx context.Context, in *FetchContentImagesRequest, opts ...grpc.CallOption) (*FacebookImageList, error) {
|
104
|
+
out := new(FacebookImageList)
|
105
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/FetchContentImages", in, out, opts...)
|
106
|
+
if err != nil {
|
107
|
+
return nil, err
|
108
|
+
}
|
109
|
+
return out, nil
|
110
|
+
}
|
111
|
+
|
112
|
+
func (c *grpcClient) FetchImageUrl(ctx context.Context, in *FetchImageUrlRequest, opts ...grpc.CallOption) (*FacebookImage, error) {
|
113
|
+
out := new(FacebookImage)
|
114
|
+
err := c.cc.Invoke(ctx, "/fbcrawl_colly.Grpc/FetchImageUrl", in, out, opts...)
|
115
|
+
if err != nil {
|
116
|
+
return nil, err
|
117
|
+
}
|
118
|
+
return out, nil
|
119
|
+
}
|
120
|
+
|
121
|
+
// GrpcServer is the server API for Grpc service.
|
122
|
+
// All implementations must embed UnimplementedGrpcServer
|
123
|
+
// for forward compatibility
|
124
|
+
type GrpcServer interface {
|
125
|
+
// Sends a greeting
|
126
|
+
Init(context.Context, *Empty) (*Pointer, error)
|
127
|
+
FreeColly(context.Context, *Pointer) (*Empty, error)
|
128
|
+
Login(context.Context, *LoginRequest) (*LoginResponse, error)
|
129
|
+
LoginWithCookies(context.Context, *LoginWithCookiesRequest) (*Empty, error)
|
130
|
+
FetchGroupInfo(context.Context, *FetchGroupInfoRequest) (*FacebookGroup, error)
|
131
|
+
FetchGroupFeed(context.Context, *FetchGroupFeedRequest) (*FacebookPostList, error)
|
132
|
+
FetchPost(context.Context, *FetchPostRequest) (*FacebookPost, error)
|
133
|
+
FetchContentImages(context.Context, *FetchContentImagesRequest) (*FacebookImageList, error)
|
134
|
+
FetchImageUrl(context.Context, *FetchImageUrlRequest) (*FacebookImage, error)
|
135
|
+
mustEmbedUnimplementedGrpcServer()
|
136
|
+
}
|
137
|
+
|
138
|
+
// UnimplementedGrpcServer must be embedded to have forward compatible implementations.
|
139
|
+
type UnimplementedGrpcServer struct {
|
140
|
+
}
|
141
|
+
|
142
|
+
func (*UnimplementedGrpcServer) Init(context.Context, *Empty) (*Pointer, error) {
|
143
|
+
return nil, status.Errorf(codes.Unimplemented, "method Init not implemented")
|
144
|
+
}
|
145
|
+
func (*UnimplementedGrpcServer) FreeColly(context.Context, *Pointer) (*Empty, error) {
|
146
|
+
return nil, status.Errorf(codes.Unimplemented, "method FreeColly not implemented")
|
147
|
+
}
|
148
|
+
func (*UnimplementedGrpcServer) Login(context.Context, *LoginRequest) (*LoginResponse, error) {
|
149
|
+
return nil, status.Errorf(codes.Unimplemented, "method Login not implemented")
|
150
|
+
}
|
151
|
+
func (*UnimplementedGrpcServer) LoginWithCookies(context.Context, *LoginWithCookiesRequest) (*Empty, error) {
|
152
|
+
return nil, status.Errorf(codes.Unimplemented, "method LoginWithCookies not implemented")
|
153
|
+
}
|
154
|
+
func (*UnimplementedGrpcServer) FetchGroupInfo(context.Context, *FetchGroupInfoRequest) (*FacebookGroup, error) {
|
155
|
+
return nil, status.Errorf(codes.Unimplemented, "method FetchGroupInfo not implemented")
|
156
|
+
}
|
157
|
+
func (*UnimplementedGrpcServer) FetchGroupFeed(context.Context, *FetchGroupFeedRequest) (*FacebookPostList, error) {
|
158
|
+
return nil, status.Errorf(codes.Unimplemented, "method FetchGroupFeed not implemented")
|
159
|
+
}
|
160
|
+
func (*UnimplementedGrpcServer) FetchPost(context.Context, *FetchPostRequest) (*FacebookPost, error) {
|
161
|
+
return nil, status.Errorf(codes.Unimplemented, "method FetchPost not implemented")
|
162
|
+
}
|
163
|
+
func (*UnimplementedGrpcServer) FetchContentImages(context.Context, *FetchContentImagesRequest) (*FacebookImageList, error) {
|
164
|
+
return nil, status.Errorf(codes.Unimplemented, "method FetchContentImages not implemented")
|
165
|
+
}
|
166
|
+
func (*UnimplementedGrpcServer) FetchImageUrl(context.Context, *FetchImageUrlRequest) (*FacebookImage, error) {
|
167
|
+
return nil, status.Errorf(codes.Unimplemented, "method FetchImageUrl not implemented")
|
168
|
+
}
|
169
|
+
func (*UnimplementedGrpcServer) mustEmbedUnimplementedGrpcServer() {}
|
170
|
+
|
171
|
+
func RegisterGrpcServer(s *grpc.Server, srv GrpcServer) {
|
172
|
+
s.RegisterService(&_Grpc_serviceDesc, srv)
|
173
|
+
}
|
174
|
+
|
175
|
+
func _Grpc_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
176
|
+
in := new(Empty)
|
177
|
+
if err := dec(in); err != nil {
|
178
|
+
return nil, err
|
179
|
+
}
|
180
|
+
if interceptor == nil {
|
181
|
+
return srv.(GrpcServer).Init(ctx, in)
|
182
|
+
}
|
183
|
+
info := &grpc.UnaryServerInfo{
|
184
|
+
Server: srv,
|
185
|
+
FullMethod: "/fbcrawl_colly.Grpc/Init",
|
186
|
+
}
|
187
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
188
|
+
return srv.(GrpcServer).Init(ctx, req.(*Empty))
|
189
|
+
}
|
190
|
+
return interceptor(ctx, in, info, handler)
|
191
|
+
}
|
192
|
+
|
193
|
+
func _Grpc_FreeColly_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
194
|
+
in := new(Pointer)
|
195
|
+
if err := dec(in); err != nil {
|
196
|
+
return nil, err
|
197
|
+
}
|
198
|
+
if interceptor == nil {
|
199
|
+
return srv.(GrpcServer).FreeColly(ctx, in)
|
200
|
+
}
|
201
|
+
info := &grpc.UnaryServerInfo{
|
202
|
+
Server: srv,
|
203
|
+
FullMethod: "/fbcrawl_colly.Grpc/FreeColly",
|
204
|
+
}
|
205
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
206
|
+
return srv.(GrpcServer).FreeColly(ctx, req.(*Pointer))
|
207
|
+
}
|
208
|
+
return interceptor(ctx, in, info, handler)
|
209
|
+
}
|
210
|
+
|
211
|
+
func _Grpc_Login_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
212
|
+
in := new(LoginRequest)
|
213
|
+
if err := dec(in); err != nil {
|
214
|
+
return nil, err
|
215
|
+
}
|
216
|
+
if interceptor == nil {
|
217
|
+
return srv.(GrpcServer).Login(ctx, in)
|
218
|
+
}
|
219
|
+
info := &grpc.UnaryServerInfo{
|
220
|
+
Server: srv,
|
221
|
+
FullMethod: "/fbcrawl_colly.Grpc/Login",
|
222
|
+
}
|
223
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
224
|
+
return srv.(GrpcServer).Login(ctx, req.(*LoginRequest))
|
225
|
+
}
|
226
|
+
return interceptor(ctx, in, info, handler)
|
227
|
+
}
|
228
|
+
|
229
|
+
func _Grpc_LoginWithCookies_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
230
|
+
in := new(LoginWithCookiesRequest)
|
231
|
+
if err := dec(in); err != nil {
|
232
|
+
return nil, err
|
233
|
+
}
|
234
|
+
if interceptor == nil {
|
235
|
+
return srv.(GrpcServer).LoginWithCookies(ctx, in)
|
236
|
+
}
|
237
|
+
info := &grpc.UnaryServerInfo{
|
238
|
+
Server: srv,
|
239
|
+
FullMethod: "/fbcrawl_colly.Grpc/LoginWithCookies",
|
240
|
+
}
|
241
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
242
|
+
return srv.(GrpcServer).LoginWithCookies(ctx, req.(*LoginWithCookiesRequest))
|
243
|
+
}
|
244
|
+
return interceptor(ctx, in, info, handler)
|
245
|
+
}
|
246
|
+
|
247
|
+
func _Grpc_FetchGroupInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
248
|
+
in := new(FetchGroupInfoRequest)
|
249
|
+
if err := dec(in); err != nil {
|
250
|
+
return nil, err
|
251
|
+
}
|
252
|
+
if interceptor == nil {
|
253
|
+
return srv.(GrpcServer).FetchGroupInfo(ctx, in)
|
254
|
+
}
|
255
|
+
info := &grpc.UnaryServerInfo{
|
256
|
+
Server: srv,
|
257
|
+
FullMethod: "/fbcrawl_colly.Grpc/FetchGroupInfo",
|
258
|
+
}
|
259
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
260
|
+
return srv.(GrpcServer).FetchGroupInfo(ctx, req.(*FetchGroupInfoRequest))
|
261
|
+
}
|
262
|
+
return interceptor(ctx, in, info, handler)
|
263
|
+
}
|
264
|
+
|
265
|
+
func _Grpc_FetchGroupFeed_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
266
|
+
in := new(FetchGroupFeedRequest)
|
267
|
+
if err := dec(in); err != nil {
|
268
|
+
return nil, err
|
269
|
+
}
|
270
|
+
if interceptor == nil {
|
271
|
+
return srv.(GrpcServer).FetchGroupFeed(ctx, in)
|
272
|
+
}
|
273
|
+
info := &grpc.UnaryServerInfo{
|
274
|
+
Server: srv,
|
275
|
+
FullMethod: "/fbcrawl_colly.Grpc/FetchGroupFeed",
|
276
|
+
}
|
277
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
278
|
+
return srv.(GrpcServer).FetchGroupFeed(ctx, req.(*FetchGroupFeedRequest))
|
279
|
+
}
|
280
|
+
return interceptor(ctx, in, info, handler)
|
281
|
+
}
|
282
|
+
|
283
|
+
func _Grpc_FetchPost_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
284
|
+
in := new(FetchPostRequest)
|
285
|
+
if err := dec(in); err != nil {
|
286
|
+
return nil, err
|
287
|
+
}
|
288
|
+
if interceptor == nil {
|
289
|
+
return srv.(GrpcServer).FetchPost(ctx, in)
|
290
|
+
}
|
291
|
+
info := &grpc.UnaryServerInfo{
|
292
|
+
Server: srv,
|
293
|
+
FullMethod: "/fbcrawl_colly.Grpc/FetchPost",
|
294
|
+
}
|
295
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
296
|
+
return srv.(GrpcServer).FetchPost(ctx, req.(*FetchPostRequest))
|
297
|
+
}
|
298
|
+
return interceptor(ctx, in, info, handler)
|
299
|
+
}
|
300
|
+
|
301
|
+
func _Grpc_FetchContentImages_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
302
|
+
in := new(FetchContentImagesRequest)
|
303
|
+
if err := dec(in); err != nil {
|
304
|
+
return nil, err
|
305
|
+
}
|
306
|
+
if interceptor == nil {
|
307
|
+
return srv.(GrpcServer).FetchContentImages(ctx, in)
|
308
|
+
}
|
309
|
+
info := &grpc.UnaryServerInfo{
|
310
|
+
Server: srv,
|
311
|
+
FullMethod: "/fbcrawl_colly.Grpc/FetchContentImages",
|
312
|
+
}
|
313
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
314
|
+
return srv.(GrpcServer).FetchContentImages(ctx, req.(*FetchContentImagesRequest))
|
315
|
+
}
|
316
|
+
return interceptor(ctx, in, info, handler)
|
317
|
+
}
|
318
|
+
|
319
|
+
func _Grpc_FetchImageUrl_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
320
|
+
in := new(FetchImageUrlRequest)
|
321
|
+
if err := dec(in); err != nil {
|
322
|
+
return nil, err
|
323
|
+
}
|
324
|
+
if interceptor == nil {
|
325
|
+
return srv.(GrpcServer).FetchImageUrl(ctx, in)
|
326
|
+
}
|
327
|
+
info := &grpc.UnaryServerInfo{
|
328
|
+
Server: srv,
|
329
|
+
FullMethod: "/fbcrawl_colly.Grpc/FetchImageUrl",
|
330
|
+
}
|
331
|
+
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
332
|
+
return srv.(GrpcServer).FetchImageUrl(ctx, req.(*FetchImageUrlRequest))
|
333
|
+
}
|
334
|
+
return interceptor(ctx, in, info, handler)
|
335
|
+
}
|
336
|
+
|
337
|
+
var _Grpc_serviceDesc = grpc.ServiceDesc{
|
338
|
+
ServiceName: "fbcrawl_colly.Grpc",
|
339
|
+
HandlerType: (*GrpcServer)(nil),
|
340
|
+
Methods: []grpc.MethodDesc{
|
341
|
+
{
|
342
|
+
MethodName: "Init",
|
343
|
+
Handler: _Grpc_Init_Handler,
|
344
|
+
},
|
345
|
+
{
|
346
|
+
MethodName: "FreeColly",
|
347
|
+
Handler: _Grpc_FreeColly_Handler,
|
348
|
+
},
|
349
|
+
{
|
350
|
+
MethodName: "Login",
|
351
|
+
Handler: _Grpc_Login_Handler,
|
352
|
+
},
|
353
|
+
{
|
354
|
+
MethodName: "LoginWithCookies",
|
355
|
+
Handler: _Grpc_LoginWithCookies_Handler,
|
356
|
+
},
|
357
|
+
{
|
358
|
+
MethodName: "FetchGroupInfo",
|
359
|
+
Handler: _Grpc_FetchGroupInfo_Handler,
|
360
|
+
},
|
361
|
+
{
|
362
|
+
MethodName: "FetchGroupFeed",
|
363
|
+
Handler: _Grpc_FetchGroupFeed_Handler,
|
364
|
+
},
|
365
|
+
{
|
366
|
+
MethodName: "FetchPost",
|
367
|
+
Handler: _Grpc_FetchPost_Handler,
|
368
|
+
},
|
369
|
+
{
|
370
|
+
MethodName: "FetchContentImages",
|
371
|
+
Handler: _Grpc_FetchContentImages_Handler,
|
372
|
+
},
|
373
|
+
{
|
374
|
+
MethodName: "FetchImageUrl",
|
375
|
+
Handler: _Grpc_FetchImageUrl_Handler,
|
376
|
+
},
|
377
|
+
},
|
378
|
+
Streams: []grpc.StreamDesc{},
|
379
|
+
Metadata: "fbcrawl.proto",
|
380
|
+
}
|
data/go.mod
CHANGED
@@ -4,12 +4,15 @@ go 1.14
|
|
4
4
|
|
5
5
|
require (
|
6
6
|
github.com/PuerkitoBio/goquery v1.5.1
|
7
|
-
github.com/andybalholm/cascadia v1.2.0 // indirect
|
8
7
|
github.com/gocolly/colly/v2 v2.1.0
|
9
8
|
github.com/golang/protobuf v1.4.2
|
10
9
|
github.com/google/logger v1.1.0
|
10
|
+
github.com/hashicorp/golang-lru v0.5.4
|
11
11
|
github.com/olebedev/when v0.0.0-20190311101825-c3b538a97254
|
12
12
|
github.com/thoas/go-funk v0.7.0
|
13
|
+
github.com/xlzd/gotp v0.0.0-20181030022105-c8557ba2c119
|
13
14
|
golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
|
15
|
+
google.golang.org/grpc v1.27.0
|
16
|
+
google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200814200710-a3740e5ed326 // indirect
|
14
17
|
google.golang.org/protobuf v1.25.0
|
15
18
|
)
|
data/go.sum
CHANGED
@@ -27,6 +27,7 @@ github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
|
|
27
27
|
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
28
28
|
github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=
|
29
29
|
github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=
|
30
|
+
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
30
31
|
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
31
32
|
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
32
33
|
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
@@ -50,6 +51,8 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
|
|
50
51
|
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
51
52
|
github.com/google/logger v1.1.0 h1:saB74Etb4EAJNH3z74CVbCKk75hld/8T0CsXKetWCwM=
|
52
53
|
github.com/google/logger v1.1.0/go.mod h1:w7O8nrRr0xufejBlQMI83MXqRusvREoJdaAxV+CoAB4=
|
54
|
+
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
55
|
+
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
53
56
|
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
|
54
57
|
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
55
58
|
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
@@ -71,6 +74,8 @@ github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++Ja
|
|
71
74
|
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
72
75
|
github.com/thoas/go-funk v0.7.0 h1:GmirKrs6j6zJbhJIficOsz2aAI7700KsU/5YrdHRM1Y=
|
73
76
|
github.com/thoas/go-funk v0.7.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
|
77
|
+
github.com/xlzd/gotp v0.0.0-20181030022105-c8557ba2c119 h1:YyPWX3jLOtYKulBR6AScGIs74lLrJcgeKRwcbAuQOG4=
|
78
|
+
github.com/xlzd/gotp v0.0.0-20181030022105-c8557ba2c119/go.mod h1:/nuTSlK+okRfR/vnIPqR89fFKonnWPiZymN5ydRJkX8=
|
74
79
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
75
80
|
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
76
81
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
@@ -116,10 +121,15 @@ google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuh
|
|
116
121
|
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
117
122
|
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
118
123
|
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
124
|
+
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
|
119
125
|
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
120
126
|
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
121
127
|
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
128
|
+
google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
|
122
129
|
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
130
|
+
google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
|
131
|
+
google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200814200710-a3740e5ed326 h1:9xXKDULLQoff9XAN3whAGrh4vrLLvydDbmFGWofOWI0=
|
132
|
+
google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200814200710-a3740e5ed326/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
|
123
133
|
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
124
134
|
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
125
135
|
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
data/lib/fbcrawl-colly.rb
CHANGED