leggy 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +7 -0
  3. data/Gemfile +4 -0
  4. data/README.md +195 -2
  5. data/Rakefile +4 -0
  6. data/leggy.gemspec +2 -2
  7. data/lib/leggy.rb +14 -5
  8. data/lib/leggy/app.rb +1 -2
  9. data/lib/leggy/crawl.rb +1 -2
  10. data/lib/leggy/crawl_options.rb +1 -2
  11. data/lib/leggy/error.rb +4 -0
  12. data/lib/leggy/error_handler.rb +30 -0
  13. data/lib/leggy/exceptions.rb +30 -0
  14. data/lib/leggy/mapping/app.rb +1 -6
  15. data/lib/leggy/mapping/crawl.rb +1 -1
  16. data/lib/leggy/mapping/crawl_options.rb +1 -1
  17. data/lib/leggy/mapping/url.rb +1 -1
  18. data/lib/leggy/mapping/user.rb +1 -1
  19. data/lib/leggy/resource/app.rb +3 -6
  20. data/lib/leggy/resource/crawl.rb +9 -12
  21. data/lib/leggy/resource/result.rb +20 -0
  22. data/lib/leggy/resource/url.rb +3 -6
  23. data/lib/leggy/resource/user.rb +1 -4
  24. data/lib/leggy/url.rb +1 -2
  25. data/lib/leggy/user.rb +1 -2
  26. data/lib/leggy/version.rb +1 -1
  27. data/spec/cassettes/{leggy.yml → leggy_app.yml} +106 -822
  28. data/spec/cassettes/leggy_crawl_all.yml +39 -0
  29. data/spec/cassettes/leggy_crawl_cancel.yml +147 -0
  30. data/spec/cassettes/leggy_crawl_start.yml +147 -0
  31. data/spec/cassettes/leggy_crawl_status.yml +183 -0
  32. data/spec/cassettes/leggy_exception.yml +38 -0
  33. data/spec/cassettes/leggy_result.yml +147 -0
  34. data/spec/cassettes/leggy_url.yml +373 -0
  35. data/spec/cassettes/leggy_user.yml +38 -0
  36. data/spec/leggy/exceptions_spec.rb +25 -0
  37. data/spec/leggy/leggy_spec.rb +1 -162
  38. data/spec/leggy/resource/app_spec.rb +64 -0
  39. data/spec/leggy/resource/crawl_spec.rb +88 -0
  40. data/spec/leggy/resource/result_spec.rb +46 -0
  41. data/spec/leggy/resource/url_spec.rb +59 -0
  42. data/spec/leggy/resource/user_spec.rb +22 -0
  43. data/spec/spec_helper.rb +37 -31
  44. data/spec/support/helpers.rb +26 -0
  45. data/spec/support/vcr.rb +12 -0
  46. metadata +48 -11
data/spec/cassettes/leggy_exception.yml ADDED
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: https://invalid:@api.80legs.com/v2/crawls
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Content-Type:
11
+ - application/json
12
+ User-Agent:
13
+ - Faraday v0.9.2
14
+ Accept-Encoding:
15
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
16
+ Accept:
17
+ - "*/*"
18
+ response:
19
+ status:
20
+ code: 401
21
+ message: Unauthorized
22
+ headers:
23
+ Date:
24
+ - Sat, 12 Dec 2015 19:26:12 GMT
25
+ Server:
26
+ - Cowboy
27
+ Www-Authenticate:
28
+ - Basic realm="Datafiniti"
29
+ Content-Length:
30
+ - '0'
31
+ Connection:
32
+ - keep-alive
33
+ body:
34
+ encoding: UTF-8
35
+ string: ''
36
+ http_version:
37
+ recorded_at: Sat, 12 Dec 2015 19:26:20 GMT
38
+ recorded_with: VCR 3.0.0
data/spec/cassettes/leggy_result.yml ADDED
@@ -0,0 +1,147 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: put
5
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_crawl_list
6
+ body:
7
+ encoding: UTF-8
8
+ string: |
9
+ [
10
+ "http://example.com",
11
+ "https://example.org"
12
+ ]
13
+ headers:
14
+ Content-Type:
15
+ - application/octet-stream
16
+ User-Agent:
17
+ - Faraday v0.9.2
18
+ Accept-Encoding:
19
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
20
+ Accept:
21
+ - "*/*"
22
+ response:
23
+ status:
24
+ code: 204
25
+ message: No Content
26
+ headers:
27
+ Content-Length:
28
+ - '0'
29
+ Content-Type:
30
+ - application/octet-stream
31
+ Date:
32
+ - Sat, 12 Dec 2015 18:20:24 GMT
33
+ Server:
34
+ - Cowboy
35
+ Connection:
36
+ - keep-alive
37
+ body:
38
+ encoding: UTF-8
39
+ string: ''
40
+ http_version:
41
+ recorded_at: Sat, 12 Dec 2015 18:20:30 GMT
42
+ - request:
43
+ method: post
44
+ uri: https://<TOKEN>:@api.80legs.com/v2/crawls/testing_crawl_results
45
+ body:
46
+ encoding: UTF-8
47
+ string: '{"app":"HeaderData.js","urllist":"sample_crawl_list","max_depth":1,"max_urls":10}'
48
+ headers:
49
+ Content-Type:
50
+ - application/json
51
+ User-Agent:
52
+ - Faraday v0.9.2
53
+ Accept-Encoding:
54
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
55
+ Accept:
56
+ - "*/*"
57
+ response:
58
+ status:
59
+ code: 204
60
+ message: No Content
61
+ headers:
62
+ Content-Length:
63
+ - '0'
64
+ Content-Type:
65
+ - application/json
66
+ Date:
67
+ - Sat, 12 Dec 2015 18:20:25 GMT
68
+ Server:
69
+ - Cowboy
70
+ Connection:
71
+ - keep-alive
72
+ body:
73
+ encoding: UTF-8
74
+ string: ''
75
+ http_version:
76
+ recorded_at: Sat, 12 Dec 2015 18:20:31 GMT
77
+ - request:
78
+ method: get
79
+ uri: https://<TOKEN>:@api.80legs.com/v2/results/testing_crawl_results
80
+ body:
81
+ encoding: US-ASCII
82
+ string: ''
83
+ headers:
84
+ Content-Type:
85
+ - application/json
86
+ User-Agent:
87
+ - Faraday v0.9.2
88
+ Accept-Encoding:
89
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
90
+ Accept:
91
+ - "*/*"
92
+ response:
93
+ status:
94
+ code: 200
95
+ message: OK
96
+ headers:
97
+ Content-Type:
98
+ - application/json
99
+ Date:
100
+ - Sat, 12 Dec 2015 18:24:22 GMT
101
+ Server:
102
+ - Cowboy
103
+ Content-Length:
104
+ - '196'
105
+ Connection:
106
+ - keep-alive
107
+ body:
108
+ encoding: UTF-8
109
+ string: '["http://datafiniti-voltron-results.s3.amazonaws.com/<TOKEN>/167627_1.txt?AWSAccessKeyId=AKIAIKFPIVMDUTHAUULA&Signature=kh2ZrkJvi1YgPGbdziibTdYDmII%3D&Expires=1450499467"]'
110
+ http_version:
111
+ recorded_at: Sat, 12 Dec 2015 18:24:27 GMT
112
+ - request:
113
+ method: delete
114
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_crawl_list
115
+ body:
116
+ encoding: US-ASCII
117
+ string: ''
118
+ headers:
119
+ Content-Type:
120
+ - application/json
121
+ User-Agent:
122
+ - Faraday v0.9.2
123
+ Accept-Encoding:
124
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
125
+ Accept:
126
+ - "*/*"
127
+ response:
128
+ status:
129
+ code: 204
130
+ message: No Content
131
+ headers:
132
+ Content-Length:
133
+ - '0'
134
+ Content-Type:
135
+ - application/octet-stream
136
+ Date:
137
+ - Sat, 12 Dec 2015 18:24:22 GMT
138
+ Server:
139
+ - Cowboy
140
+ Connection:
141
+ - keep-alive
142
+ body:
143
+ encoding: UTF-8
144
+ string: ''
145
+ http_version:
146
+ recorded_at: Sat, 12 Dec 2015 18:24:28 GMT
147
+ recorded_with: VCR 3.0.0
data/spec/cassettes/leggy_url.yml ADDED
@@ -0,0 +1,373 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: put
5
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_all
6
+ body:
7
+ encoding: UTF-8
8
+ string: |
9
+ [
10
+ "http://example.com",
11
+ "https://example.org"
12
+ ]
13
+ headers:
14
+ Content-Type:
15
+ - application/octet-stream
16
+ User-Agent:
17
+ - Faraday v0.9.2
18
+ Accept-Encoding:
19
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
20
+ Accept:
21
+ - "*/*"
22
+ response:
23
+ status:
24
+ code: 204
25
+ message: No Content
26
+ headers:
27
+ Content-Length:
28
+ - '0'
29
+ Content-Type:
30
+ - application/octet-stream
31
+ Date:
32
+ - Sat, 12 Dec 2015 06:04:24 GMT
33
+ Server:
34
+ - Cowboy
35
+ Connection:
36
+ - keep-alive
37
+ body:
38
+ encoding: UTF-8
39
+ string: ''
40
+ http_version:
41
+ recorded_at: Sat, 12 Dec 2015 06:04:29 GMT
42
+ - request:
43
+ method: get
44
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists
45
+ body:
46
+ encoding: US-ASCII
47
+ string: ''
48
+ headers:
49
+ Content-Type:
50
+ - application/json
51
+ User-Agent:
52
+ - Faraday v0.9.2
53
+ Accept-Encoding:
54
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
55
+ Accept:
56
+ - "*/*"
57
+ response:
58
+ status:
59
+ code: 200
60
+ message: OK
61
+ headers:
62
+ Content-Type:
63
+ - application/json
64
+ Date:
65
+ - Sat, 12 Dec 2015 06:04:25 GMT
66
+ Server:
67
+ - Cowboy
68
+ Content-Length:
69
+ - '1981'
70
+ Connection:
71
+ - keep-alive
72
+ body:
73
+ encoding: UTF-8
74
+ string: '[{"location":"urllists/<TOKEN>/temp_one","name":"temp_one","user":"<TOKEN>","date_created":"2015-1-26"},{"location":"urllists/<TOKEN>/successful_url_list_upload","name":"successful_url_list_upload","user":"<TOKEN>","date_created":"2015-3-5"},{"location":"urllists/<TOKEN>/sample_all","name":"sample_all","user":"<TOKEN>","date_created":"2015-12-12"},{"location":"urllists/<TOKEN>/federal_gov_domains","name":"federal_gov_domains","user":"<TOKEN>","date_created":"2015-11-11"},{"location":"urllists/<TOKEN>/energy-ca-gov","name":"energy-ca-gov","user":"<TOKEN>","date_created":"2015-1-22"},{"location":"urllists/<TOKEN>/DSIRE","name":"DSIRE","user":"<TOKEN>","date_created":"2015-2-26"},{"location":"urllists/<TOKEN>/delete_existing_test","name":"delete_existing_test","user":"<TOKEN>","date_created":"2015-3-5"},{"location":"urllists/<TOKEN>/crawler_test","name":"crawler_test","user":"<TOKEN>","date_created":"2014-12-1"},{"location":"urllists/<TOKEN>/cpuc-ca-gov","name":"cpuc-ca-gov","user":"<TOKEN>","date_created":"2015-2-8"},{"location":"urllists/<TOKEN>/civvic_crawler_test_urls","name":"civvic_crawler_test_urls","user":"<TOKEN>","date_created":"2014-12-1"},{"location":"urllists/<TOKEN>/caiso-com","name":"caiso-com","user":"<TOKEN>","date_created":"2015-2-8"},{"location":"urllists/<TOKEN>/.example.com","name":".example.com","user":"<TOKEN>","date_created":"2014-12-1"}]'
75
+ http_version:
76
+ recorded_at: Sat, 12 Dec 2015 06:04:29 GMT
77
+ - request:
78
+ method: delete
79
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_all
80
+ body:
81
+ encoding: US-ASCII
82
+ string: ''
83
+ headers:
84
+ Content-Type:
85
+ - application/json
86
+ User-Agent:
87
+ - Faraday v0.9.2
88
+ Accept-Encoding:
89
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
90
+ Accept:
91
+ - "*/*"
92
+ response:
93
+ status:
94
+ code: 204
95
+ message: No Content
96
+ headers:
97
+ Content-Length:
98
+ - '0'
99
+ Content-Type:
100
+ - application/octet-stream
101
+ Date:
102
+ - Sat, 12 Dec 2015 06:04:25 GMT
103
+ Server:
104
+ - Cowboy
105
+ Connection:
106
+ - keep-alive
107
+ body:
108
+ encoding: UTF-8
109
+ string: ''
110
+ http_version:
111
+ recorded_at: Sat, 12 Dec 2015 06:04:30 GMT
112
+ - request:
113
+ method: put
114
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample
115
+ body:
116
+ encoding: UTF-8
117
+ string: |
118
+ [
119
+ "http://example.com",
120
+ "https://example.org"
121
+ ]
122
+ headers:
123
+ Content-Type:
124
+ - application/octet-stream
125
+ User-Agent:
126
+ - Faraday v0.9.2
127
+ Accept-Encoding:
128
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
+ Accept:
130
+ - "*/*"
131
+ response:
132
+ status:
133
+ code: 204
134
+ message: No Content
135
+ headers:
136
+ Content-Length:
137
+ - '0'
138
+ Content-Type:
139
+ - application/octet-stream
140
+ Date:
141
+ - Sat, 12 Dec 2015 06:04:26 GMT
142
+ Server:
143
+ - Cowboy
144
+ Connection:
145
+ - keep-alive
146
+ body:
147
+ encoding: UTF-8
148
+ string: ''
149
+ http_version:
150
+ recorded_at: Sat, 12 Dec 2015 06:04:30 GMT
151
+ - request:
152
+ method: delete
153
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample
154
+ body:
155
+ encoding: US-ASCII
156
+ string: ''
157
+ headers:
158
+ Content-Type:
159
+ - application/json
160
+ User-Agent:
161
+ - Faraday v0.9.2
162
+ Accept-Encoding:
163
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
164
+ Accept:
165
+ - "*/*"
166
+ response:
167
+ status:
168
+ code: 204
169
+ message: No Content
170
+ headers:
171
+ Content-Length:
172
+ - '0'
173
+ Content-Type:
174
+ - application/octet-stream
175
+ Date:
176
+ - Sat, 12 Dec 2015 06:04:26 GMT
177
+ Server:
178
+ - Cowboy
179
+ Connection:
180
+ - keep-alive
181
+ body:
182
+ encoding: UTF-8
183
+ string: ''
184
+ http_version:
185
+ recorded_at: Sat, 12 Dec 2015 06:04:31 GMT
186
+ - request:
187
+ method: put
188
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
189
+ body:
190
+ encoding: UTF-8
191
+ string: |
192
+ [
193
+ "http://example.com",
194
+ "https://example.org"
195
+ ]
196
+ headers:
197
+ Content-Type:
198
+ - application/octet-stream
199
+ User-Agent:
200
+ - Faraday v0.9.2
201
+ Accept-Encoding:
202
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
203
+ Accept:
204
+ - "*/*"
205
+ response:
206
+ status:
207
+ code: 204
208
+ message: No Content
209
+ headers:
210
+ Content-Length:
211
+ - '0'
212
+ Content-Type:
213
+ - application/octet-stream
214
+ Date:
215
+ - Sat, 12 Dec 2015 06:04:27 GMT
216
+ Server:
217
+ - Cowboy
218
+ Connection:
219
+ - keep-alive
220
+ body:
221
+ encoding: UTF-8
222
+ string: ''
223
+ http_version:
224
+ recorded_at: Sat, 12 Dec 2015 06:04:31 GMT
225
+ - request:
226
+ method: get
227
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
228
+ body:
229
+ encoding: US-ASCII
230
+ string: ''
231
+ headers:
232
+ Content-Type:
233
+ - application/json
234
+ User-Agent:
235
+ - Faraday v0.9.2
236
+ Accept-Encoding:
237
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
238
+ Accept:
239
+ - "*/*"
240
+ response:
241
+ status:
242
+ code: 200
243
+ message: OK
244
+ headers:
245
+ Content-Type:
246
+ - application/octet-stream
247
+ Date:
248
+ - Sat, 12 Dec 2015 06:04:27 GMT
249
+ Server:
250
+ - Cowboy
251
+ Content-Length:
252
+ - '52'
253
+ Connection:
254
+ - keep-alive
255
+ body:
256
+ encoding: UTF-8
257
+ string: |
258
+ [
259
+ "http://example.com",
260
+ "https://example.org"
261
+ ]
262
+ http_version:
263
+ recorded_at: Sat, 12 Dec 2015 06:04:32 GMT
264
+ - request:
265
+ method: delete
266
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
267
+ body:
268
+ encoding: US-ASCII
269
+ string: ''
270
+ headers:
271
+ Content-Type:
272
+ - application/json
273
+ User-Agent:
274
+ - Faraday v0.9.2
275
+ Accept-Encoding:
276
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
277
+ Accept:
278
+ - "*/*"
279
+ response:
280
+ status:
281
+ code: 204
282
+ message: No Content
283
+ headers:
284
+ Content-Length:
285
+ - '0'
286
+ Content-Type:
287
+ - application/octet-stream
288
+ Date:
289
+ - Sat, 12 Dec 2015 06:04:28 GMT
290
+ Server:
291
+ - Cowboy
292
+ Connection:
293
+ - keep-alive
294
+ body:
295
+ encoding: UTF-8
296
+ string: ''
297
+ http_version:
298
+ recorded_at: Sat, 12 Dec 2015 06:04:33 GMT
299
+ - request:
300
+ method: put
301
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_delete
302
+ body:
303
+ encoding: UTF-8
304
+ string: |
305
+ [
306
+ "http://example.com",
307
+ "https://example.org"
308
+ ]
309
+ headers:
310
+ Content-Type:
311
+ - application/octet-stream
312
+ User-Agent:
313
+ - Faraday v0.9.2
314
+ Accept-Encoding:
315
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
316
+ Accept:
317
+ - "*/*"
318
+ response:
319
+ status:
320
+ code: 204
321
+ message: No Content
322
+ headers:
323
+ Content-Length:
324
+ - '0'
325
+ Content-Type:
326
+ - application/octet-stream
327
+ Date:
328
+ - Sat, 12 Dec 2015 06:04:28 GMT
329
+ Server:
330
+ - Cowboy
331
+ Connection:
332
+ - keep-alive
333
+ body:
334
+ encoding: UTF-8
335
+ string: ''
336
+ http_version:
337
+ recorded_at: Sat, 12 Dec 2015 06:04:33 GMT
338
+ - request:
339
+ method: delete
340
+ uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_delete
341
+ body:
342
+ encoding: US-ASCII
343
+ string: ''
344
+ headers:
345
+ Content-Type:
346
+ - application/json
347
+ User-Agent:
348
+ - Faraday v0.9.2
349
+ Accept-Encoding:
350
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
351
+ Accept:
352
+ - "*/*"
353
+ response:
354
+ status:
355
+ code: 204
356
+ message: No Content
357
+ headers:
358
+ Content-Length:
359
+ - '0'
360
+ Content-Type:
361
+ - application/octet-stream
362
+ Date:
363
+ - Sat, 12 Dec 2015 06:04:29 GMT
364
+ Server:
365
+ - Cowboy
366
+ Connection:
367
+ - keep-alive
368
+ body:
369
+ encoding: UTF-8
370
+ string: ''
371
+ http_version:
372
+ recorded_at: Sat, 12 Dec 2015 06:04:34 GMT
373
+ recorded_with: VCR 3.0.0