leggy 0.1.3 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +7 -0
  3. data/Gemfile +4 -0
  4. data/README.md +195 -2
  5. data/Rakefile +4 -0
  6. data/leggy.gemspec +2 -2
  7. data/lib/leggy.rb +14 -5
  8. data/lib/leggy/app.rb +1 -2
  9. data/lib/leggy/crawl.rb +1 -2
  10. data/lib/leggy/crawl_options.rb +1 -2
  11. data/lib/leggy/error.rb +4 -0
  12. data/lib/leggy/error_handler.rb +30 -0
  13. data/lib/leggy/exceptions.rb +30 -0
  14. data/lib/leggy/mapping/app.rb +1 -6
  15. data/lib/leggy/mapping/crawl.rb +1 -1
  16. data/lib/leggy/mapping/crawl_options.rb +1 -1
  17. data/lib/leggy/mapping/url.rb +1 -1
  18. data/lib/leggy/mapping/user.rb +1 -1
  19. data/lib/leggy/resource/app.rb +3 -6
  20. data/lib/leggy/resource/crawl.rb +9 -12
  21. data/lib/leggy/resource/result.rb +20 -0
  22. data/lib/leggy/resource/url.rb +3 -6
  23. data/lib/leggy/resource/user.rb +1 -4
  24. data/lib/leggy/url.rb +1 -2
  25. data/lib/leggy/user.rb +1 -2
  26. data/lib/leggy/version.rb +1 -1
  27. data/spec/cassettes/{leggy.yml → leggy_app.yml} +106 -822
  28. data/spec/cassettes/leggy_crawl_all.yml +39 -0
  29. data/spec/cassettes/leggy_crawl_cancel.yml +147 -0
  30. data/spec/cassettes/leggy_crawl_start.yml +147 -0
  31. data/spec/cassettes/leggy_crawl_status.yml +183 -0
  32. data/spec/cassettes/leggy_exception.yml +38 -0
  33. data/spec/cassettes/leggy_result.yml +147 -0
  34. data/spec/cassettes/leggy_url.yml +373 -0
  35. data/spec/cassettes/leggy_user.yml +38 -0
  36. data/spec/leggy/exceptions_spec.rb +25 -0
  37. data/spec/leggy/leggy_spec.rb +1 -162
  38. data/spec/leggy/resource/app_spec.rb +64 -0
  39. data/spec/leggy/resource/crawl_spec.rb +88 -0
  40. data/spec/leggy/resource/result_spec.rb +46 -0
  41. data/spec/leggy/resource/url_spec.rb +59 -0
  42. data/spec/leggy/resource/user_spec.rb +22 -0
  43. data/spec/spec_helper.rb +37 -31
  44. data/spec/support/helpers.rb +26 -0
  45. data/spec/support/vcr.rb +12 -0
  46. metadata +48 -11
data/spec/cassettes/leggy_exception.yml ADDED
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: https://invalid:@api.80legs.com/v2/crawls
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Content-Type:
11
+ - application/json
12
+ User-Agent:
13
+ - Faraday v0.9.2
14
+ Accept-Encoding:
15
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
16
+ Accept:
17
+ - "*/*"
18
+ response:
19
+ status:
20
+ code: 401
21
+ message: Unauthorized
22
+ headers:
23
+ Date:
24
+ - Sat, 12 Dec 2015 19:26:12 GMT
25
+ Server:
26
+ - Cowboy
27
+ Www-Authenticate:
28
+ - Basic realm="Datafiniti"
29
+ Content-Length:
30
+ - '0'
31
+ Connection:
32
+ - keep-alive
33
+ body:
34
+ encoding: UTF-8
35
+ string: ''
36
+ http_version:
37
+ recorded_at: Sat, 12 Dec 2015 19:26:20 GMT
38
+ recorded_with: VCR 3.0.0
data/spec/cassettes/leggy_result.yml ADDED
@@ -0,0 +1,147 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: put
5
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_crawl_list
6
+ body:
7
+ encoding: UTF-8
8
+ string: |
9
+ [
10
+ "http://example.com",
11
+ "https://example.org"
12
+ ]
13
+ headers:
14
+ Content-Type:
15
+ - application/octet-stream
16
+ User-Agent:
17
+ - Faraday v0.9.2
18
+ Accept-Encoding:
19
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
20
+ Accept:
21
+ - "*/*"
22
+ response:
23
+ status:
24
+ code: 204
25
+ message: No Content
26
+ headers:
27
+ Content-Length:
28
+ - '0'
29
+ Content-Type:
30
+ - application/octet-stream
31
+ Date:
32
+ - Sat, 12 Dec 2015 18:20:24 GMT
33
+ Server:
34
+ - Cowboy
35
+ Connection:
36
+ - keep-alive
37
+ body:
38
+ encoding: UTF-8
39
+ string: ''
40
+ http_version:
41
+ recorded_at: Sat, 12 Dec 2015 18:20:30 GMT
42
+ - request:
43
+ method: post
44
+ uri: https://<TOKEN>:@api.80legs.com/v2/crawls/testing_crawl_results
45
+ body:
46
+ encoding: UTF-8
47
+ string: '{"app":"HeaderData.js","urllist":"sample_crawl_list","max_depth":1,"max_urls":10}'
48
+ headers:
49
+ Content-Type:
50
+ - application/json
51
+ User-Agent:
52
+ - Faraday v0.9.2
53
+ Accept-Encoding:
54
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
55
+ Accept:
56
+ - "*/*"
57
+ response:
58
+ status:
59
+ code: 204
60
+ message: No Content
61
+ headers:
62
+ Content-Length:
63
+ - '0'
64
+ Content-Type:
65
+ - application/json
66
+ Date:
67
+ - Sat, 12 Dec 2015 18:20:25 GMT
68
+ Server:
69
+ - Cowboy
70
+ Connection:
71
+ - keep-alive
72
+ body:
73
+ encoding: UTF-8
74
+ string: ''
75
+ http_version:
76
+ recorded_at: Sat, 12 Dec 2015 18:20:31 GMT
77
+ - request:
78
+ method: get
79
+ uri: https://<TOKEN>:@api.80legs.com/v2/results/testing_crawl_results
80
+ body:
81
+ encoding: US-ASCII
82
+ string: ''
83
+ headers:
84
+ Content-Type:
85
+ - application/json
86
+ User-Agent:
87
+ - Faraday v0.9.2
88
+ Accept-Encoding:
89
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
90
+ Accept:
91
+ - "*/*"
92
+ response:
93
+ status:
94
+ code: 200
95
+ message: OK
96
+ headers:
97
+ Content-Type:
98
+ - application/json
99
+ Date:
100
+ - Sat, 12 Dec 2015 18:24:22 GMT
101
+ Server:
102
+ - Cowboy
103
+ Content-Length:
104
+ - '196'
105
+ Connection:
106
+ - keep-alive
107
+ body:
108
+ encoding: UTF-8
109
+ string: '["http://datafiniti-voltron-results.s3.amazonaws.com/<TOKEN>/167627_1.txt?AWSAccessKeyId=AKIAIKFPIVMDUTHAUULA&Signature=kh2ZrkJvi1YgPGbdziibTdYDmII%3D&Expires=1450499467"]'
110
+ http_version:
111
+ recorded_at: Sat, 12 Dec 2015 18:24:27 GMT
112
+ - request:
113
+ method: delete
114
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_crawl_list
115
+ body:
116
+ encoding: US-ASCII
117
+ string: ''
118
+ headers:
119
+ Content-Type:
120
+ - application/json
121
+ User-Agent:
122
+ - Faraday v0.9.2
123
+ Accept-Encoding:
124
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
125
+ Accept:
126
+ - "*/*"
127
+ response:
128
+ status:
129
+ code: 204
130
+ message: No Content
131
+ headers:
132
+ Content-Length:
133
+ - '0'
134
+ Content-Type:
135
+ - application/octet-stream
136
+ Date:
137
+ - Sat, 12 Dec 2015 18:24:22 GMT
138
+ Server:
139
+ - Cowboy
140
+ Connection:
141
+ - keep-alive
142
+ body:
143
+ encoding: UTF-8
144
+ string: ''
145
+ http_version:
146
+ recorded_at: Sat, 12 Dec 2015 18:24:28 GMT
147
+ recorded_with: VCR 3.0.0
data/spec/cassettes/leggy_url.yml ADDED
@@ -0,0 +1,373 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: put
5
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_all
6
+ body:
7
+ encoding: UTF-8
8
+ string: |
9
+ [
10
+ "http://example.com",
11
+ "https://example.org"
12
+ ]
13
+ headers:
14
+ Content-Type:
15
+ - application/octet-stream
16
+ User-Agent:
17
+ - Faraday v0.9.2
18
+ Accept-Encoding:
19
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
20
+ Accept:
21
+ - "*/*"
22
+ response:
23
+ status:
24
+ code: 204
25
+ message: No Content
26
+ headers:
27
+ Content-Length:
28
+ - '0'
29
+ Content-Type:
30
+ - application/octet-stream
31
+ Date:
32
+ - Sat, 12 Dec 2015 06:04:24 GMT
33
+ Server:
34
+ - Cowboy
35
+ Connection:
36
+ - keep-alive
37
+ body:
38
+ encoding: UTF-8
39
+ string: ''
40
+ http_version:
41
+ recorded_at: Sat, 12 Dec 2015 06:04:29 GMT
42
+ - request:
43
+ method: get
44
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists
45
+ body:
46
+ encoding: US-ASCII
47
+ string: ''
48
+ headers:
49
+ Content-Type:
50
+ - application/json
51
+ User-Agent:
52
+ - Faraday v0.9.2
53
+ Accept-Encoding:
54
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
55
+ Accept:
56
+ - "*/*"
57
+ response:
58
+ status:
59
+ code: 200
60
+ message: OK
61
+ headers:
62
+ Content-Type:
63
+ - application/json
64
+ Date:
65
+ - Sat, 12 Dec 2015 06:04:25 GMT
66
+ Server:
67
+ - Cowboy
68
+ Content-Length:
69
+ - '1981'
70
+ Connection:
71
+ - keep-alive
72
+ body:
73
+ encoding: UTF-8
74
+ string: '[{"location":"urllists/<TOKEN>/temp_one","name":"temp_one","user":"<TOKEN>","date_created":"2015-1-26"},{"location":"urllists/<TOKEN>/successful_url_list_upload","name":"successful_url_list_upload","user":"<TOKEN>","date_created":"2015-3-5"},{"location":"urllists/<TOKEN>/sample_all","name":"sample_all","user":"<TOKEN>","date_created":"2015-12-12"},{"location":"urllists/<TOKEN>/federal_gov_domains","name":"federal_gov_domains","user":"<TOKEN>","date_created":"2015-11-11"},{"location":"urllists/<TOKEN>/energy-ca-gov","name":"energy-ca-gov","user":"<TOKEN>","date_created":"2015-1-22"},{"location":"urllists/<TOKEN>/DSIRE","name":"DSIRE","user":"<TOKEN>","date_created":"2015-2-26"},{"location":"urllists/<TOKEN>/delete_existing_test","name":"delete_existing_test","user":"<TOKEN>","date_created":"2015-3-5"},{"location":"urllists/<TOKEN>/crawler_test","name":"crawler_test","user":"<TOKEN>","date_created":"2014-12-1"},{"location":"urllists/<TOKEN>/cpuc-ca-gov","name":"cpuc-ca-gov","user":"<TOKEN>","date_created":"2015-2-8"},{"location":"urllists/<TOKEN>/civvic_crawler_test_urls","name":"civvic_crawler_test_urls","user":"<TOKEN>","date_created":"2014-12-1"},{"location":"urllists/<TOKEN>/caiso-com","name":"caiso-com","user":"<TOKEN>","date_created":"2015-2-8"},{"location":"urllists/<TOKEN>/.example.com","name":".example.com","user":"<TOKEN>","date_created":"2014-12-1"}]'
75
+ http_version:
76
+ recorded_at: Sat, 12 Dec 2015 06:04:29 GMT
77
+ - request:
78
+ method: delete
79
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_all
80
+ body:
81
+ encoding: US-ASCII
82
+ string: ''
83
+ headers:
84
+ Content-Type:
85
+ - application/json
86
+ User-Agent:
87
+ - Faraday v0.9.2
88
+ Accept-Encoding:
89
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
90
+ Accept:
91
+ - "*/*"
92
+ response:
93
+ status:
94
+ code: 204
95
+ message: No Content
96
+ headers:
97
+ Content-Length:
98
+ - '0'
99
+ Content-Type:
100
+ - application/octet-stream
101
+ Date:
102
+ - Sat, 12 Dec 2015 06:04:25 GMT
103
+ Server:
104
+ - Cowboy
105
+ Connection:
106
+ - keep-alive
107
+ body:
108
+ encoding: UTF-8
109
+ string: ''
110
+ http_version:
111
+ recorded_at: Sat, 12 Dec 2015 06:04:30 GMT
112
+ - request:
113
+ method: put
114
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample
115
+ body:
116
+ encoding: UTF-8
117
+ string: |
118
+ [
119
+ "http://example.com",
120
+ "https://example.org"
121
+ ]
122
+ headers:
123
+ Content-Type:
124
+ - application/octet-stream
125
+ User-Agent:
126
+ - Faraday v0.9.2
127
+ Accept-Encoding:
128
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
+ Accept:
130
+ - "*/*"
131
+ response:
132
+ status:
133
+ code: 204
134
+ message: No Content
135
+ headers:
136
+ Content-Length:
137
+ - '0'
138
+ Content-Type:
139
+ - application/octet-stream
140
+ Date:
141
+ - Sat, 12 Dec 2015 06:04:26 GMT
142
+ Server:
143
+ - Cowboy
144
+ Connection:
145
+ - keep-alive
146
+ body:
147
+ encoding: UTF-8
148
+ string: ''
149
+ http_version:
150
+ recorded_at: Sat, 12 Dec 2015 06:04:30 GMT
151
+ - request:
152
+ method: delete
153
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample
154
+ body:
155
+ encoding: US-ASCII
156
+ string: ''
157
+ headers:
158
+ Content-Type:
159
+ - application/json
160
+ User-Agent:
161
+ - Faraday v0.9.2
162
+ Accept-Encoding:
163
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
164
+ Accept:
165
+ - "*/*"
166
+ response:
167
+ status:
168
+ code: 204
169
+ message: No Content
170
+ headers:
171
+ Content-Length:
172
+ - '0'
173
+ Content-Type:
174
+ - application/octet-stream
175
+ Date:
176
+ - Sat, 12 Dec 2015 06:04:26 GMT
177
+ Server:
178
+ - Cowboy
179
+ Connection:
180
+ - keep-alive
181
+ body:
182
+ encoding: UTF-8
183
+ string: ''
184
+ http_version:
185
+ recorded_at: Sat, 12 Dec 2015 06:04:31 GMT
186
+ - request:
187
+ method: put
188
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
189
+ body:
190
+ encoding: UTF-8
191
+ string: |
192
+ [
193
+ "http://example.com",
194
+ "https://example.org"
195
+ ]
196
+ headers:
197
+ Content-Type:
198
+ - application/octet-stream
199
+ User-Agent:
200
+ - Faraday v0.9.2
201
+ Accept-Encoding:
202
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
203
+ Accept:
204
+ - "*/*"
205
+ response:
206
+ status:
207
+ code: 204
208
+ message: No Content
209
+ headers:
210
+ Content-Length:
211
+ - '0'
212
+ Content-Type:
213
+ - application/octet-stream
214
+ Date:
215
+ - Sat, 12 Dec 2015 06:04:27 GMT
216
+ Server:
217
+ - Cowboy
218
+ Connection:
219
+ - keep-alive
220
+ body:
221
+ encoding: UTF-8
222
+ string: ''
223
+ http_version:
224
+ recorded_at: Sat, 12 Dec 2015 06:04:31 GMT
225
+ - request:
226
+ method: get
227
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
228
+ body:
229
+ encoding: US-ASCII
230
+ string: ''
231
+ headers:
232
+ Content-Type:
233
+ - application/json
234
+ User-Agent:
235
+ - Faraday v0.9.2
236
+ Accept-Encoding:
237
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
238
+ Accept:
239
+ - "*/*"
240
+ response:
241
+ status:
242
+ code: 200
243
+ message: OK
244
+ headers:
245
+ Content-Type:
246
+ - application/octet-stream
247
+ Date:
248
+ - Sat, 12 Dec 2015 06:04:27 GMT
249
+ Server:
250
+ - Cowboy
251
+ Content-Length:
252
+ - '52'
253
+ Connection:
254
+ - keep-alive
255
+ body:
256
+ encoding: UTF-8
257
+ string: |
258
+ [
259
+ "http://example.com",
260
+ "https://example.org"
261
+ ]
262
+ http_version:
263
+ recorded_at: Sat, 12 Dec 2015 06:04:32 GMT
264
+ - request:
265
+ method: delete
266
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
267
+ body:
268
+ encoding: US-ASCII
269
+ string: ''
270
+ headers:
271
+ Content-Type:
272
+ - application/json
273
+ User-Agent:
274
+ - Faraday v0.9.2
275
+ Accept-Encoding:
276
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
277
+ Accept:
278
+ - "*/*"
279
+ response:
280
+ status:
281
+ code: 204
282
+ message: No Content
283
+ headers:
284
+ Content-Length:
285
+ - '0'
286
+ Content-Type:
287
+ - application/octet-stream
288
+ Date:
289
+ - Sat, 12 Dec 2015 06:04:28 GMT
290
+ Server:
291
+ - Cowboy
292
+ Connection:
293
+ - keep-alive
294
+ body:
295
+ encoding: UTF-8
296
+ string: ''
297
+ http_version:
298
+ recorded_at: Sat, 12 Dec 2015 06:04:33 GMT
299
+ - request:
300
+ method: put
301
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_delete
302
+ body:
303
+ encoding: UTF-8
304
+ string: |
305
+ [
306
+ "http://example.com",
307
+ "https://example.org"
308
+ ]
309
+ headers:
310
+ Content-Type:
311
+ - application/octet-stream
312
+ User-Agent:
313
+ - Faraday v0.9.2
314
+ Accept-Encoding:
315
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
316
+ Accept:
317
+ - "*/*"
318
+ response:
319
+ status:
320
+ code: 204
321
+ message: No Content
322
+ headers:
323
+ Content-Length:
324
+ - '0'
325
+ Content-Type:
326
+ - application/octet-stream
327
+ Date:
328
+ - Sat, 12 Dec 2015 06:04:28 GMT
329
+ Server:
330
+ - Cowboy
331
+ Connection:
332
+ - keep-alive
333
+ body:
334
+ encoding: UTF-8
335
+ string: ''
336
+ http_version:
337
+ recorded_at: Sat, 12 Dec 2015 06:04:33 GMT
338
+ - request:
339
+ method: delete
340
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_delete
341
+ body:
342
+ encoding: US-ASCII
343
+ string: ''
344
+ headers:
345
+ Content-Type:
346
+ - application/json
347
+ User-Agent:
348
+ - Faraday v0.9.2
349
+ Accept-Encoding:
350
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
351
+ Accept:
352
+ - "*/*"
353
+ response:
354
+ status:
355
+ code: 204
356
+ message: No Content
357
+ headers:
358
+ Content-Length:
359
+ - '0'
360
+ Content-Type:
361
+ - application/octet-stream
362
+ Date:
363
+ - Sat, 12 Dec 2015 06:04:29 GMT
364
+ Server:
365
+ - Cowboy
366
+ Connection:
367
+ - keep-alive
368
+ body:
369
+ encoding: UTF-8
370
+ string: ''
371
+ http_version:
372
+ recorded_at: Sat, 12 Dec 2015 06:04:34 GMT
373
+ recorded_with: VCR 3.0.0