scrapinghub-client 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +6 -0
  5. data/Gemfile +5 -0
  6. data/LICENSE +21 -0
  7. data/README.md +46 -0
  8. data/Rakefile +22 -0
  9. data/lib/scrapinghub-client.rb +3 -0
  10. data/lib/scrapinghub/jobs.rb +175 -0
  11. data/lib/scrapinghub/version.rb +3 -0
  12. data/scrapinghub.gemspec +26 -0
  13. data/spec/fixtures/vcr_cassettes/jobs/delete/bad_auth.yml +38 -0
  14. data/spec/fixtures/vcr_cassettes/jobs/delete/job/invalid.yml +38 -0
  15. data/spec/fixtures/vcr_cassettes/jobs/delete/job/multiple.yml +38 -0
  16. data/spec/fixtures/vcr_cassettes/jobs/delete/job/single.yml +38 -0
  17. data/spec/fixtures/vcr_cassettes/jobs/delete/project/invalid.yml +39 -0
  18. data/spec/fixtures/vcr_cassettes/jobs/delete/project/valid.yml +38 -0
  19. data/spec/fixtures/vcr_cassettes/jobs/list/bad_auth.yml +38 -0
  20. data/spec/fixtures/vcr_cassettes/jobs/list/count/3.yml +53 -0
  21. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/invalid.yml +38 -0
  22. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/multiple.yml +53 -0
  23. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/single.yml +48 -0
  24. data/spec/fixtures/vcr_cassettes/jobs/list/job/invalid.yml +38 -0
  25. data/spec/fixtures/vcr_cassettes/jobs/list/job/multiple.yml +48 -0
  26. data/spec/fixtures/vcr_cassettes/jobs/list/job/single.yml +43 -0
  27. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/invalid.yml +81 -0
  28. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/multiple.yml +67 -0
  29. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/single.yml +72 -0
  30. data/spec/fixtures/vcr_cassettes/jobs/list/project/invalid.yml +39 -0
  31. data/spec/fixtures/vcr_cassettes/jobs/list/project/valid.yml +81 -0
  32. data/spec/fixtures/vcr_cassettes/jobs/list/spider/invalid.yml +38 -0
  33. data/spec/fixtures/vcr_cassettes/jobs/list/spider/valid.yml +62 -0
  34. data/spec/fixtures/vcr_cassettes/jobs/list/state/finished.yml +81 -0
  35. data/spec/fixtures/vcr_cassettes/jobs/list/state/pending.yml +38 -0
  36. data/spec/fixtures/vcr_cassettes/jobs/schedule/bad_auth.yml +38 -0
  37. data/spec/fixtures/vcr_cassettes/jobs/schedule/project/invalid.yml +39 -0
  38. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/add_tag.yml +38 -0
  39. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/already-running.yml +39 -0
  40. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/extra.yml +38 -0
  41. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/minimal.yml +38 -0
  42. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/priority.yml +38 -0
  43. data/spec/fixtures/vcr_cassettes/jobs/stop/bad_auth.yml +38 -0
  44. data/spec/fixtures/vcr_cassettes/jobs/stop/job/already-stopped.yml +38 -0
  45. data/spec/fixtures/vcr_cassettes/jobs/stop/job/invalid.yml +38 -0
  46. data/spec/fixtures/vcr_cassettes/jobs/stop/job/valid.yml +38 -0
  47. data/spec/fixtures/vcr_cassettes/jobs/stop/project/invalid.yml +39 -0
  48. data/spec/fixtures/vcr_cassettes/jobs/update/bad_auth.yml +38 -0
  49. data/spec/fixtures/vcr_cassettes/jobs/update/has_tag.yml +38 -0
  50. data/spec/fixtures/vcr_cassettes/jobs/update/job.yml +38 -0
  51. data/spec/fixtures/vcr_cassettes/jobs/update/lacks_tag.yml +38 -0
  52. data/spec/fixtures/vcr_cassettes/jobs/update/no-query-filters.yml +38 -0
  53. data/spec/fixtures/vcr_cassettes/jobs/update/no-update-params.yml +38 -0
  54. data/spec/fixtures/vcr_cassettes/jobs/update/project/invalid.yml +39 -0
  55. data/spec/fixtures/vcr_cassettes/jobs/update/spider.yml +38 -0
  56. data/spec/fixtures/vcr_cassettes/jobs/update/state.yml +38 -0
  57. data/spec/integration/jobs_spec.rb +567 -0
  58. data/spec/spec_helper.rb +22 -0
  59. data/spec/unit/jobs_spec.rb +188 -0
  60. data/spec/unit/scrapinghub_spec.rb +8 -0
  61. metadata +200 -0
@@ -0,0 +1,81 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?lacks_tag=baz&project=1
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 18:07:46 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 9, "total": 9, "jobs": [{"responses_received":
36
+ 5323, "items_scraped": 1030, "logs": 167, "tags": ["foo", "bar"], "version":
37
+ "1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-23T01:26:37",
38
+ "priority": 2, "state": "finished", "close_reason": "cancel_timeout", "spider_type":
39
+ "manual", "spider_args": {}, "started_time": "2015-07-22T23:02:13", "id":
40
+ "1/1/6", "errors_count": 4, "elapsed": 4601972}, {"responses_received":
41
+ 6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
42
+ "cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
43
+ "priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
44
+ {}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
45
+ 3, "elapsed": 242886016, "logs": 179}, {"responses_received": 365486, "items_scraped":
46
+ 4037, "close_reason": "cancel_timeout", "logs": 33631, "tags": ["foo"], "version":
47
+ "1434570463", "spider": "firearms_for_sale_crawl", "updated_time": "2015-07-21T02:43:25",
48
+ "priority": 2, "state": "finished", "spider_args": {}, "started_time": "2015-06-27T18:36:06",
49
+ "id": "1/5/1", "errors_count": 0, "elapsed": 4374159, "spider_type": "manual"},
50
+ {"responses_received": 276710, "items_scraped": 63362, "close_reason": "finished",
51
+ "logs": 7356, "tags": [], "version": "1434570463", "spider": "palmetto_state_armory_crawl",
52
+ "priority": 2, "updated_time": "2015-07-02T20:49:25", "state": "finished",
53
+ "spider_type": "manual", "spider_args": {"": ""}, "started_time": "2015-06-27T18:35:17",
54
+ "id": "1/3/2", "errors_count": 0, "elapsed": 1977501051}, {"logs": 2189,
55
+ "spider": "cheaper_than_dirt_crawl", "spider_args": {}, "id": "1/2/1",
56
+ "priority": 2, "items_scraped": 81852, "state": "finished", "version": "1434570463",
57
+ "spider_type": "manual", "errors_count": 0, "responses_received": 86259, "tags":
58
+ [], "elapsed": 2901598784, "close_reason": "finished", "started_time": "2015-06-20T15:59:43",
59
+ "updated_time": "2015-06-22T04:07:48"}, {"responses_received": 1194, "items_scraped":
60
+ 1066, "version": "1434570463", "tags": ["bar"], "spider": "palmetto_state_armory_crawl",
61
+ "updated_time": "2015-06-21T20:07:12", "priority": 2, "state": "finished",
62
+ "close_reason": "cancel_timeout", "spider_args": {}, "errors_count": 0, "started_time":
63
+ "2015-06-21T19:02:41", "id": "1/3/1", "logs": 88, "elapsed": 4580569,
64
+ "spider_type": "manual"}, {"logs": 21, "spider": "atlantic_firearms_crawl",
65
+ "spider_args": {}, "id": "1/1/3", "priority": 2, "items_scraped": 25,
66
+ "state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
67
+ 0, "responses_received": 114, "tags": [], "elapsed": 3274222342, "close_reason":
68
+ "cancel_timeout", "started_time": "2015-06-17T20:29:46", "updated_time": "2015-06-17T20:32:47"},
69
+ {"responses_received": 0, "items_scraped": 0, "close_reason": "no_reason",
70
+ "logs": 13, "tags": [], "started_time": "2015-06-17T20:27:58", "version":
71
+ "1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06",
72
+ "priority": 2, "state": "finished", "spider_args": {}, "errors_count": 2,
73
+ "elapsed": 3274779960, "id": "1/1/2", "spider_type": "manual"}, {"responses_received":
74
+ 0, "items_scraped": 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29",
75
+ "version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
76
+ "2015-06-17T20:17:40", "priority": 2, "state": "finished", "close_reason":
77
+ "no_reason", "spider_args": {}, "errors_count": 2, "elapsed": 3275406782,
78
+ "id": "1/1/1", "spider_type": "manual"}]}'
79
+ http_version:
80
+ recorded_at: Sat, 25 Jul 2015 18:07:47 GMT
81
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,67 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?lacks_tag=bar&project=1
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 18:07:46 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 6, "total": 6, "jobs": [{"responses_received":
36
+ 6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
37
+ "cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
38
+ "priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
39
+ {}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
40
+ 3, "elapsed": 242885676, "logs": 179}, {"responses_received": 276710, "items_scraped":
41
+ 63362, "updated_time": "2015-07-02T20:49:25", "logs": 7356, "tags": [], "version":
42
+ "1434570463", "spider": "palmetto_state_armory_crawl", "priority": 2, "state":
43
+ "finished", "close_reason": "finished", "spider_type": "manual", "spider_args":
44
+ {"": ""}, "started_time": "2015-06-27T18:35:17", "id": "1/3/2", "errors_count":
45
+ 0, "elapsed": 1977500711}, {"logs": 2189, "spider": "cheaper_than_dirt_crawl",
46
+ "spider_args": {}, "id": "1/2/1", "priority": 2, "items_scraped": 81852,
47
+ "state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
48
+ 0, "responses_received": 86259, "tags": [], "elapsed": 2901598444, "close_reason":
49
+ "finished", "started_time": "2015-06-20T15:59:43", "updated_time": "2015-06-22T04:07:48"},
50
+ {"responses_received": 114, "items_scraped": 25, "close_reason": "cancel_timeout",
51
+ "logs": 21, "tags": [], "version": "1434570463", "spider": "atlantic_firearms_crawl",
52
+ "updated_time": "2015-06-17T20:32:47", "priority": 2, "state": "finished",
53
+ "spider_args": {}, "started_time": "2015-06-17T20:29:46", "id": "1/1/3",
54
+ "errors_count": 0, "elapsed": 3274222002, "spider_type": "manual"}, {"responses_received":
55
+ 0, "items_scraped": 0, "close_reason": "no_reason", "logs": 13, "tags": [],
56
+ "started_time": "2015-06-17T20:27:58", "version": "1434570463", "spider":
57
+ "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06", "priority":
58
+ 2, "state": "finished", "spider_args": {}, "errors_count": 2, "elapsed": 3274779620,
59
+ "id": "1/1/2", "spider_type": "manual"}, {"responses_received": 0, "items_scraped":
60
+ 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29", "version":
61
+ "1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:17:40",
62
+ "priority": 2, "state": "finished", "close_reason": "no_reason", "spider_type":
63
+ "manual", "spider_args": {}, "errors_count": 2, "elapsed": 3275406442, "id":
64
+ "1/1/1"}]}'
65
+ http_version:
66
+ recorded_at: Sat, 25 Jul 2015 18:07:46 GMT
67
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,72 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?lacks_tag=foo&project=1
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 18:07:46 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 7, "total": 7, "jobs": [{"responses_received":
36
+ 6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
37
+ "cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
38
+ "priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
39
+ {}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
40
+ 3, "elapsed": 242885333, "logs": 179}, {"responses_received": 276710, "items_scraped":
41
+ 63362, "updated_time": "2015-07-02T20:49:25", "logs": 7356, "tags": [], "version":
42
+ "1434570463", "spider": "palmetto_state_armory_crawl", "priority": 2, "state":
43
+ "finished", "close_reason": "finished", "spider_type": "manual", "spider_args":
44
+ {"": ""}, "started_time": "2015-06-27T18:35:17", "id": "1/3/2", "errors_count":
45
+ 0, "elapsed": 1977500368}, {"logs": 2189, "spider": "cheaper_than_dirt_crawl",
46
+ "spider_args": {}, "id": "1/2/1", "priority": 2, "items_scraped": 81852,
47
+ "state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
48
+ 0, "responses_received": 86259, "tags": [], "elapsed": 2901598101, "close_reason":
49
+ "finished", "started_time": "2015-06-20T15:59:43", "updated_time": "2015-06-22T04:07:48"},
50
+ {"responses_received": 1194, "items_scraped": 1066, "close_reason": "cancel_timeout",
51
+ "version": "1434570463", "tags": ["bar"], "spider": "palmetto_state_armory_crawl",
52
+ "updated_time": "2015-06-21T20:07:12", "priority": 2, "state": "finished",
53
+ "spider_type": "manual", "spider_args": {}, "started_time": "2015-06-21T19:02:41",
54
+ "id": "1/3/1", "errors_count": 0, "elapsed": 4579886, "logs": 88}, {"responses_received":
55
+ 114, "items_scraped": 25, "close_reason": "cancel_timeout", "logs": 21, "tags":
56
+ [], "version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
57
+ "2015-06-17T20:32:47", "priority": 2, "state": "finished", "spider_args":
58
+ {}, "started_time": "2015-06-17T20:29:46", "id": "1/1/3", "errors_count":
59
+ 0, "elapsed": 3274221659, "spider_type": "manual"}, {"responses_received":
60
+ 0, "items_scraped": 0, "close_reason": "no_reason", "logs": 13, "tags": [],
61
+ "started_time": "2015-06-17T20:27:58", "version": "1434570463", "spider":
62
+ "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06", "priority":
63
+ 2, "state": "finished", "spider_args": {}, "errors_count": 2, "elapsed": 3274779277,
64
+ "id": "1/1/2", "spider_type": "manual"}, {"responses_received": 0, "items_scraped":
65
+ 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29", "version":
66
+ "1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:17:40",
67
+ "priority": 2, "state": "finished", "close_reason": "no_reason", "spider_type":
68
+ "manual", "spider_args": {}, "errors_count": 2, "elapsed": 3275406099, "id":
69
+ "1/1/1"}]}'
70
+ http_version:
71
+ recorded_at: Sat, 25 Jul 2015 18:07:46 GMT
72
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,39 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=2
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: User 'abevoelker' doesn't have access to project 2
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 17:46:37 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '89'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "User ''abevoelker'' doesn''t have
36
+ access to project 2"}'
37
+ http_version:
38
+ recorded_at: Sat, 25 Jul 2015 17:46:38 GMT
39
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,81 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=1
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 17:38:48 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 9, "total": 9, "jobs": [{"responses_received":
36
+ 5323, "items_scraped": 1030, "logs": 167, "tags": ["foo", "bar"], "version":
37
+ "1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-23T01:26:37",
38
+ "priority": 2, "state": "finished", "close_reason": "cancel_timeout", "spider_type":
39
+ "manual", "spider_args": {}, "started_time": "2015-07-22T23:02:13", "id":
40
+ "1/1/6", "errors_count": 4, "elapsed": 2863730}, {"responses_received":
41
+ 6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
42
+ "cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
43
+ "priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
44
+ {}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
45
+ 3, "elapsed": 241147774, "logs": 179}, {"responses_received": 365486, "items_scraped":
46
+ 4037, "close_reason": "cancel_timeout", "logs": 33631, "tags": ["foo"], "version":
47
+ "1434570463", "spider": "firearms_for_sale_crawl", "updated_time": "2015-07-21T02:43:25",
48
+ "priority": 2, "state": "finished", "spider_args": {}, "started_time": "2015-06-27T18:36:06",
49
+ "id": "1/5/1", "errors_count": 0, "elapsed": 2635917, "spider_type": "manual"},
50
+ {"responses_received": 276710, "items_scraped": 63362, "close_reason": "finished",
51
+ "logs": 7356, "tags": [], "version": "1434570463", "spider": "palmetto_state_armory_crawl",
52
+ "priority": 2, "updated_time": "2015-07-02T20:49:25", "state": "finished",
53
+ "spider_type": "manual", "spider_args": {"": ""}, "started_time": "2015-06-27T18:35:17",
54
+ "id": "1/3/2", "errors_count": 0, "elapsed": 1975762809}, {"logs": 2189,
55
+ "spider": "cheaper_than_dirt_crawl", "spider_args": {}, "id": "1/2/1",
56
+ "priority": 2, "items_scraped": 81852, "state": "finished", "version": "1434570463",
57
+ "spider_type": "manual", "errors_count": 0, "responses_received": 86259, "tags":
58
+ [], "elapsed": 2899860542, "close_reason": "finished", "started_time": "2015-06-20T15:59:43",
59
+ "updated_time": "2015-06-22T04:07:48"}, {"responses_received": 1194, "items_scraped":
60
+ 1066, "version": "1434570463", "tags": ["bar"], "spider": "palmetto_state_armory_crawl",
61
+ "updated_time": "2015-06-21T20:07:12", "priority": 2, "state": "finished",
62
+ "close_reason": "cancel_timeout", "spider_args": {}, "errors_count": 0, "started_time":
63
+ "2015-06-21T19:02:41", "id": "1/3/1", "logs": 88, "elapsed": 2842327,
64
+ "spider_type": "manual"}, {"logs": 21, "spider": "atlantic_firearms_crawl",
65
+ "spider_args": {}, "id": "1/1/3", "priority": 2, "items_scraped": 25,
66
+ "state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
67
+ 0, "responses_received": 114, "tags": [], "elapsed": 3272484100, "close_reason":
68
+ "cancel_timeout", "started_time": "2015-06-17T20:29:46", "updated_time": "2015-06-17T20:32:47"},
69
+ {"responses_received": 0, "items_scraped": 0, "close_reason": "no_reason",
70
+ "logs": 13, "tags": [], "started_time": "2015-06-17T20:27:58", "version":
71
+ "1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06",
72
+ "priority": 2, "state": "finished", "spider_args": {}, "errors_count": 2,
73
+ "elapsed": 3273041718, "id": "1/1/2", "spider_type": "manual"}, {"responses_received":
74
+ 0, "items_scraped": 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29",
75
+ "version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
76
+ "2015-06-17T20:17:40", "priority": 2, "state": "finished", "close_reason":
77
+ "no_reason", "spider_args": {}, "errors_count": 2, "elapsed": 3273668540,
78
+ "id": "1/1/1", "spider_type": "manual"}]}'
79
+ http_version:
80
+ recorded_at: Sat, 25 Jul 2015 17:38:48 GMT
81
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=1&spider=bar
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 17:58:00 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 0, "total": 0, "jobs": []}'
36
+ http_version:
37
+ recorded_at: Sat, 25 Jul 2015 17:58:00 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,62 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=1&spider=atlantic_firearms_crawl
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 17:57:17 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 5, "total": 5, "jobs": [{"responses_received":
36
+ 5323, "items_scraped": 1030, "close_reason": "cancel_timeout", "logs": 167,
37
+ "tags": ["foo", "bar"], "version": "1434570463", "spider": "atlantic_firearms_crawl",
38
+ "updated_time": "2015-07-23T01:26:37", "priority": 2, "state": "finished",
39
+ "spider_type": "manual", "spider_args": {}, "started_time": "2015-07-22T23:02:13",
40
+ "id": "1/1/6", "errors_count": 4, "elapsed": 3972688}, {"logs": 179, "spider":
41
+ "atlantic_firearms_crawl", "spider_args": {}, "id": "1/1/4", "priority":
42
+ 2, "items_scraped": 1122, "state": "finished", "version": "1434570463", "spider_type":
43
+ "manual", "errors_count": 3, "responses_received": 6243, "tags": [], "elapsed":
44
+ 242256732, "close_reason": "cancel_timeout", "started_time": "2015-07-22T20:03:31",
45
+ "updated_time": "2015-07-22T22:35:04"}, {"logs": 21, "spider": "atlantic_firearms_crawl",
46
+ "spider_args": {}, "id": "1/1/3", "priority": 2, "items_scraped": 25,
47
+ "state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
48
+ 0, "responses_received": 114, "tags": [], "elapsed": 3273593058, "close_reason":
49
+ "cancel_timeout", "started_time": "2015-06-17T20:29:46", "updated_time": "2015-06-17T20:32:47"},
50
+ {"spider": "atlantic_firearms_crawl", "items_scraped": 0, "updated_time":
51
+ "2015-06-17T20:28:06", "responses_received": 0, "logs": 13, "tags": [], "started_time":
52
+ "2015-06-17T20:27:58", "close_reason": "no_reason", "priority": 2, "state":
53
+ "finished", "version": "1434570463", "spider_type": "manual", "spider_args":
54
+ {}, "errors_count": 2, "elapsed": 3274150676, "id": "1/1/2"}, {"responses_received":
55
+ 0, "items_scraped": 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29",
56
+ "version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
57
+ "2015-06-17T20:17:40", "priority": 2, "state": "finished", "close_reason":
58
+ "no_reason", "spider_type": "manual", "spider_args": {}, "errors_count": 2,
59
+ "elapsed": 3274777498, "id": "1/1/1"}]}'
60
+ http_version:
61
+ recorded_at: Sat, 25 Jul 2015 17:57:17 GMT
62
+ recorded_with: VCR 2.9.3