scrapinghub-client 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.rspec +3 -0
- data/.travis.yml +6 -0
- data/Gemfile +5 -0
- data/LICENSE +21 -0
- data/README.md +46 -0
- data/Rakefile +22 -0
- data/lib/scrapinghub-client.rb +3 -0
- data/lib/scrapinghub/jobs.rb +175 -0
- data/lib/scrapinghub/version.rb +3 -0
- data/scrapinghub.gemspec +26 -0
- data/spec/fixtures/vcr_cassettes/jobs/delete/bad_auth.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/delete/job/invalid.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/delete/job/multiple.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/delete/job/single.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/delete/project/invalid.yml +39 -0
- data/spec/fixtures/vcr_cassettes/jobs/delete/project/valid.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/bad_auth.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/count/3.yml +53 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/invalid.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/multiple.yml +53 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/single.yml +48 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/job/invalid.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/job/multiple.yml +48 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/job/single.yml +43 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/invalid.yml +81 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/multiple.yml +67 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/single.yml +72 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/project/invalid.yml +39 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/project/valid.yml +81 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/spider/invalid.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/spider/valid.yml +62 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/state/finished.yml +81 -0
- data/spec/fixtures/vcr_cassettes/jobs/list/state/pending.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/schedule/bad_auth.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/schedule/project/invalid.yml +39 -0
- data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/add_tag.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/already-running.yml +39 -0
- data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/extra.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/minimal.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/priority.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/stop/bad_auth.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/stop/job/already-stopped.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/stop/job/invalid.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/stop/job/valid.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/stop/project/invalid.yml +39 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/bad_auth.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/has_tag.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/job.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/lacks_tag.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/no-query-filters.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/no-update-params.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/project/invalid.yml +39 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/spider.yml +38 -0
- data/spec/fixtures/vcr_cassettes/jobs/update/state.yml +38 -0
- data/spec/integration/jobs_spec.rb +567 -0
- data/spec/spec_helper.rb +22 -0
- data/spec/unit/jobs_spec.rb +188 -0
- data/spec/unit/scrapinghub_spec.rb +8 -0
- metadata +200 -0
@@ -0,0 +1,81 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?lacks_tag=baz&project=1
|
6
|
+
body:
|
7
|
+
encoding: US-ASCII
|
8
|
+
string: ''
|
9
|
+
headers:
|
10
|
+
Accept-Encoding:
|
11
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
12
|
+
Accept:
|
13
|
+
- "*/*"
|
14
|
+
User-Agent:
|
15
|
+
- Ruby
|
16
|
+
response:
|
17
|
+
status:
|
18
|
+
code: 200
|
19
|
+
message: OK
|
20
|
+
headers:
|
21
|
+
Server:
|
22
|
+
- nginx/1.8.0
|
23
|
+
Date:
|
24
|
+
- Sat, 25 Jul 2015 18:07:46 GMT
|
25
|
+
Content-Type:
|
26
|
+
- application/json
|
27
|
+
Transfer-Encoding:
|
28
|
+
- chunked
|
29
|
+
Connection:
|
30
|
+
- keep-alive
|
31
|
+
Vary:
|
32
|
+
- Cookie
|
33
|
+
body:
|
34
|
+
encoding: ASCII-8BIT
|
35
|
+
string: '{"status": "ok", "count": 9, "total": 9, "jobs": [{"responses_received":
|
36
|
+
5323, "items_scraped": 1030, "logs": 167, "tags": ["foo", "bar"], "version":
|
37
|
+
"1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-23T01:26:37",
|
38
|
+
"priority": 2, "state": "finished", "close_reason": "cancel_timeout", "spider_type":
|
39
|
+
"manual", "spider_args": {}, "started_time": "2015-07-22T23:02:13", "id":
|
40
|
+
"1/1/6", "errors_count": 4, "elapsed": 4601972}, {"responses_received":
|
41
|
+
6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
|
42
|
+
"cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
|
43
|
+
"priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
|
44
|
+
{}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
|
45
|
+
3, "elapsed": 242886016, "logs": 179}, {"responses_received": 365486, "items_scraped":
|
46
|
+
4037, "close_reason": "cancel_timeout", "logs": 33631, "tags": ["foo"], "version":
|
47
|
+
"1434570463", "spider": "firearms_for_sale_crawl", "updated_time": "2015-07-21T02:43:25",
|
48
|
+
"priority": 2, "state": "finished", "spider_args": {}, "started_time": "2015-06-27T18:36:06",
|
49
|
+
"id": "1/5/1", "errors_count": 0, "elapsed": 4374159, "spider_type": "manual"},
|
50
|
+
{"responses_received": 276710, "items_scraped": 63362, "close_reason": "finished",
|
51
|
+
"logs": 7356, "tags": [], "version": "1434570463", "spider": "palmetto_state_armory_crawl",
|
52
|
+
"priority": 2, "updated_time": "2015-07-02T20:49:25", "state": "finished",
|
53
|
+
"spider_type": "manual", "spider_args": {"": ""}, "started_time": "2015-06-27T18:35:17",
|
54
|
+
"id": "1/3/2", "errors_count": 0, "elapsed": 1977501051}, {"logs": 2189,
|
55
|
+
"spider": "cheaper_than_dirt_crawl", "spider_args": {}, "id": "1/2/1",
|
56
|
+
"priority": 2, "items_scraped": 81852, "state": "finished", "version": "1434570463",
|
57
|
+
"spider_type": "manual", "errors_count": 0, "responses_received": 86259, "tags":
|
58
|
+
[], "elapsed": 2901598784, "close_reason": "finished", "started_time": "2015-06-20T15:59:43",
|
59
|
+
"updated_time": "2015-06-22T04:07:48"}, {"responses_received": 1194, "items_scraped":
|
60
|
+
1066, "version": "1434570463", "tags": ["bar"], "spider": "palmetto_state_armory_crawl",
|
61
|
+
"updated_time": "2015-06-21T20:07:12", "priority": 2, "state": "finished",
|
62
|
+
"close_reason": "cancel_timeout", "spider_args": {}, "errors_count": 0, "started_time":
|
63
|
+
"2015-06-21T19:02:41", "id": "1/3/1", "logs": 88, "elapsed": 4580569,
|
64
|
+
"spider_type": "manual"}, {"logs": 21, "spider": "atlantic_firearms_crawl",
|
65
|
+
"spider_args": {}, "id": "1/1/3", "priority": 2, "items_scraped": 25,
|
66
|
+
"state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
|
67
|
+
0, "responses_received": 114, "tags": [], "elapsed": 3274222342, "close_reason":
|
68
|
+
"cancel_timeout", "started_time": "2015-06-17T20:29:46", "updated_time": "2015-06-17T20:32:47"},
|
69
|
+
{"responses_received": 0, "items_scraped": 0, "close_reason": "no_reason",
|
70
|
+
"logs": 13, "tags": [], "started_time": "2015-06-17T20:27:58", "version":
|
71
|
+
"1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06",
|
72
|
+
"priority": 2, "state": "finished", "spider_args": {}, "errors_count": 2,
|
73
|
+
"elapsed": 3274779960, "id": "1/1/2", "spider_type": "manual"}, {"responses_received":
|
74
|
+
0, "items_scraped": 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29",
|
75
|
+
"version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
|
76
|
+
"2015-06-17T20:17:40", "priority": 2, "state": "finished", "close_reason":
|
77
|
+
"no_reason", "spider_args": {}, "errors_count": 2, "elapsed": 3275406782,
|
78
|
+
"id": "1/1/1", "spider_type": "manual"}]}'
|
79
|
+
http_version:
|
80
|
+
recorded_at: Sat, 25 Jul 2015 18:07:47 GMT
|
81
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,67 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?lacks_tag=bar&project=1
|
6
|
+
body:
|
7
|
+
encoding: US-ASCII
|
8
|
+
string: ''
|
9
|
+
headers:
|
10
|
+
Accept-Encoding:
|
11
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
12
|
+
Accept:
|
13
|
+
- "*/*"
|
14
|
+
User-Agent:
|
15
|
+
- Ruby
|
16
|
+
response:
|
17
|
+
status:
|
18
|
+
code: 200
|
19
|
+
message: OK
|
20
|
+
headers:
|
21
|
+
Server:
|
22
|
+
- nginx/1.8.0
|
23
|
+
Date:
|
24
|
+
- Sat, 25 Jul 2015 18:07:46 GMT
|
25
|
+
Content-Type:
|
26
|
+
- application/json
|
27
|
+
Transfer-Encoding:
|
28
|
+
- chunked
|
29
|
+
Connection:
|
30
|
+
- keep-alive
|
31
|
+
Vary:
|
32
|
+
- Cookie
|
33
|
+
body:
|
34
|
+
encoding: ASCII-8BIT
|
35
|
+
string: '{"status": "ok", "count": 6, "total": 6, "jobs": [{"responses_received":
|
36
|
+
6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
|
37
|
+
"cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
|
38
|
+
"priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
|
39
|
+
{}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
|
40
|
+
3, "elapsed": 242885676, "logs": 179}, {"responses_received": 276710, "items_scraped":
|
41
|
+
63362, "updated_time": "2015-07-02T20:49:25", "logs": 7356, "tags": [], "version":
|
42
|
+
"1434570463", "spider": "palmetto_state_armory_crawl", "priority": 2, "state":
|
43
|
+
"finished", "close_reason": "finished", "spider_type": "manual", "spider_args":
|
44
|
+
{"": ""}, "started_time": "2015-06-27T18:35:17", "id": "1/3/2", "errors_count":
|
45
|
+
0, "elapsed": 1977500711}, {"logs": 2189, "spider": "cheaper_than_dirt_crawl",
|
46
|
+
"spider_args": {}, "id": "1/2/1", "priority": 2, "items_scraped": 81852,
|
47
|
+
"state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
|
48
|
+
0, "responses_received": 86259, "tags": [], "elapsed": 2901598444, "close_reason":
|
49
|
+
"finished", "started_time": "2015-06-20T15:59:43", "updated_time": "2015-06-22T04:07:48"},
|
50
|
+
{"responses_received": 114, "items_scraped": 25, "close_reason": "cancel_timeout",
|
51
|
+
"logs": 21, "tags": [], "version": "1434570463", "spider": "atlantic_firearms_crawl",
|
52
|
+
"updated_time": "2015-06-17T20:32:47", "priority": 2, "state": "finished",
|
53
|
+
"spider_args": {}, "started_time": "2015-06-17T20:29:46", "id": "1/1/3",
|
54
|
+
"errors_count": 0, "elapsed": 3274222002, "spider_type": "manual"}, {"responses_received":
|
55
|
+
0, "items_scraped": 0, "close_reason": "no_reason", "logs": 13, "tags": [],
|
56
|
+
"started_time": "2015-06-17T20:27:58", "version": "1434570463", "spider":
|
57
|
+
"atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06", "priority":
|
58
|
+
2, "state": "finished", "spider_args": {}, "errors_count": 2, "elapsed": 3274779620,
|
59
|
+
"id": "1/1/2", "spider_type": "manual"}, {"responses_received": 0, "items_scraped":
|
60
|
+
0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29", "version":
|
61
|
+
"1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:17:40",
|
62
|
+
"priority": 2, "state": "finished", "close_reason": "no_reason", "spider_type":
|
63
|
+
"manual", "spider_args": {}, "errors_count": 2, "elapsed": 3275406442, "id":
|
64
|
+
"1/1/1"}]}'
|
65
|
+
http_version:
|
66
|
+
recorded_at: Sat, 25 Jul 2015 18:07:46 GMT
|
67
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,72 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?lacks_tag=foo&project=1
|
6
|
+
body:
|
7
|
+
encoding: US-ASCII
|
8
|
+
string: ''
|
9
|
+
headers:
|
10
|
+
Accept-Encoding:
|
11
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
12
|
+
Accept:
|
13
|
+
- "*/*"
|
14
|
+
User-Agent:
|
15
|
+
- Ruby
|
16
|
+
response:
|
17
|
+
status:
|
18
|
+
code: 200
|
19
|
+
message: OK
|
20
|
+
headers:
|
21
|
+
Server:
|
22
|
+
- nginx/1.8.0
|
23
|
+
Date:
|
24
|
+
- Sat, 25 Jul 2015 18:07:46 GMT
|
25
|
+
Content-Type:
|
26
|
+
- application/json
|
27
|
+
Transfer-Encoding:
|
28
|
+
- chunked
|
29
|
+
Connection:
|
30
|
+
- keep-alive
|
31
|
+
Vary:
|
32
|
+
- Cookie
|
33
|
+
body:
|
34
|
+
encoding: ASCII-8BIT
|
35
|
+
string: '{"status": "ok", "count": 7, "total": 7, "jobs": [{"responses_received":
|
36
|
+
6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
|
37
|
+
"cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
|
38
|
+
"priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
|
39
|
+
{}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
|
40
|
+
3, "elapsed": 242885333, "logs": 179}, {"responses_received": 276710, "items_scraped":
|
41
|
+
63362, "updated_time": "2015-07-02T20:49:25", "logs": 7356, "tags": [], "version":
|
42
|
+
"1434570463", "spider": "palmetto_state_armory_crawl", "priority": 2, "state":
|
43
|
+
"finished", "close_reason": "finished", "spider_type": "manual", "spider_args":
|
44
|
+
{"": ""}, "started_time": "2015-06-27T18:35:17", "id": "1/3/2", "errors_count":
|
45
|
+
0, "elapsed": 1977500368}, {"logs": 2189, "spider": "cheaper_than_dirt_crawl",
|
46
|
+
"spider_args": {}, "id": "1/2/1", "priority": 2, "items_scraped": 81852,
|
47
|
+
"state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
|
48
|
+
0, "responses_received": 86259, "tags": [], "elapsed": 2901598101, "close_reason":
|
49
|
+
"finished", "started_time": "2015-06-20T15:59:43", "updated_time": "2015-06-22T04:07:48"},
|
50
|
+
{"responses_received": 1194, "items_scraped": 1066, "close_reason": "cancel_timeout",
|
51
|
+
"version": "1434570463", "tags": ["bar"], "spider": "palmetto_state_armory_crawl",
|
52
|
+
"updated_time": "2015-06-21T20:07:12", "priority": 2, "state": "finished",
|
53
|
+
"spider_type": "manual", "spider_args": {}, "started_time": "2015-06-21T19:02:41",
|
54
|
+
"id": "1/3/1", "errors_count": 0, "elapsed": 4579886, "logs": 88}, {"responses_received":
|
55
|
+
114, "items_scraped": 25, "close_reason": "cancel_timeout", "logs": 21, "tags":
|
56
|
+
[], "version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
|
57
|
+
"2015-06-17T20:32:47", "priority": 2, "state": "finished", "spider_args":
|
58
|
+
{}, "started_time": "2015-06-17T20:29:46", "id": "1/1/3", "errors_count":
|
59
|
+
0, "elapsed": 3274221659, "spider_type": "manual"}, {"responses_received":
|
60
|
+
0, "items_scraped": 0, "close_reason": "no_reason", "logs": 13, "tags": [],
|
61
|
+
"started_time": "2015-06-17T20:27:58", "version": "1434570463", "spider":
|
62
|
+
"atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06", "priority":
|
63
|
+
2, "state": "finished", "spider_args": {}, "errors_count": 2, "elapsed": 3274779277,
|
64
|
+
"id": "1/1/2", "spider_type": "manual"}, {"responses_received": 0, "items_scraped":
|
65
|
+
0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29", "version":
|
66
|
+
"1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:17:40",
|
67
|
+
"priority": 2, "state": "finished", "close_reason": "no_reason", "spider_type":
|
68
|
+
"manual", "spider_args": {}, "errors_count": 2, "elapsed": 3275406099, "id":
|
69
|
+
"1/1/1"}]}'
|
70
|
+
http_version:
|
71
|
+
recorded_at: Sat, 25 Jul 2015 18:07:46 GMT
|
72
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,39 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=2
|
6
|
+
body:
|
7
|
+
encoding: US-ASCII
|
8
|
+
string: ''
|
9
|
+
headers:
|
10
|
+
Accept-Encoding:
|
11
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
12
|
+
Accept:
|
13
|
+
- "*/*"
|
14
|
+
User-Agent:
|
15
|
+
- Ruby
|
16
|
+
response:
|
17
|
+
status:
|
18
|
+
code: 400
|
19
|
+
message: User 'abevoelker' doesn't have access to project 2
|
20
|
+
headers:
|
21
|
+
Server:
|
22
|
+
- nginx/1.8.0
|
23
|
+
Date:
|
24
|
+
- Sat, 25 Jul 2015 17:46:37 GMT
|
25
|
+
Content-Type:
|
26
|
+
- application/json
|
27
|
+
Content-Length:
|
28
|
+
- '89'
|
29
|
+
Connection:
|
30
|
+
- keep-alive
|
31
|
+
Vary:
|
32
|
+
- Cookie
|
33
|
+
body:
|
34
|
+
encoding: UTF-8
|
35
|
+
string: '{"status": "badrequest", "message": "User ''abevoelker'' doesn''t have
|
36
|
+
access to project 2"}'
|
37
|
+
http_version:
|
38
|
+
recorded_at: Sat, 25 Jul 2015 17:46:38 GMT
|
39
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,81 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=1
|
6
|
+
body:
|
7
|
+
encoding: US-ASCII
|
8
|
+
string: ''
|
9
|
+
headers:
|
10
|
+
Accept-Encoding:
|
11
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
12
|
+
Accept:
|
13
|
+
- "*/*"
|
14
|
+
User-Agent:
|
15
|
+
- Ruby
|
16
|
+
response:
|
17
|
+
status:
|
18
|
+
code: 200
|
19
|
+
message: OK
|
20
|
+
headers:
|
21
|
+
Server:
|
22
|
+
- nginx/1.8.0
|
23
|
+
Date:
|
24
|
+
- Sat, 25 Jul 2015 17:38:48 GMT
|
25
|
+
Content-Type:
|
26
|
+
- application/json
|
27
|
+
Transfer-Encoding:
|
28
|
+
- chunked
|
29
|
+
Connection:
|
30
|
+
- keep-alive
|
31
|
+
Vary:
|
32
|
+
- Cookie
|
33
|
+
body:
|
34
|
+
encoding: ASCII-8BIT
|
35
|
+
string: '{"status": "ok", "count": 9, "total": 9, "jobs": [{"responses_received":
|
36
|
+
5323, "items_scraped": 1030, "logs": 167, "tags": ["foo", "bar"], "version":
|
37
|
+
"1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-23T01:26:37",
|
38
|
+
"priority": 2, "state": "finished", "close_reason": "cancel_timeout", "spider_type":
|
39
|
+
"manual", "spider_args": {}, "started_time": "2015-07-22T23:02:13", "id":
|
40
|
+
"1/1/6", "errors_count": 4, "elapsed": 2863730}, {"responses_received":
|
41
|
+
6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
|
42
|
+
"cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
|
43
|
+
"priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
|
44
|
+
{}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
|
45
|
+
3, "elapsed": 241147774, "logs": 179}, {"responses_received": 365486, "items_scraped":
|
46
|
+
4037, "close_reason": "cancel_timeout", "logs": 33631, "tags": ["foo"], "version":
|
47
|
+
"1434570463", "spider": "firearms_for_sale_crawl", "updated_time": "2015-07-21T02:43:25",
|
48
|
+
"priority": 2, "state": "finished", "spider_args": {}, "started_time": "2015-06-27T18:36:06",
|
49
|
+
"id": "1/5/1", "errors_count": 0, "elapsed": 2635917, "spider_type": "manual"},
|
50
|
+
{"responses_received": 276710, "items_scraped": 63362, "close_reason": "finished",
|
51
|
+
"logs": 7356, "tags": [], "version": "1434570463", "spider": "palmetto_state_armory_crawl",
|
52
|
+
"priority": 2, "updated_time": "2015-07-02T20:49:25", "state": "finished",
|
53
|
+
"spider_type": "manual", "spider_args": {"": ""}, "started_time": "2015-06-27T18:35:17",
|
54
|
+
"id": "1/3/2", "errors_count": 0, "elapsed": 1975762809}, {"logs": 2189,
|
55
|
+
"spider": "cheaper_than_dirt_crawl", "spider_args": {}, "id": "1/2/1",
|
56
|
+
"priority": 2, "items_scraped": 81852, "state": "finished", "version": "1434570463",
|
57
|
+
"spider_type": "manual", "errors_count": 0, "responses_received": 86259, "tags":
|
58
|
+
[], "elapsed": 2899860542, "close_reason": "finished", "started_time": "2015-06-20T15:59:43",
|
59
|
+
"updated_time": "2015-06-22T04:07:48"}, {"responses_received": 1194, "items_scraped":
|
60
|
+
1066, "version": "1434570463", "tags": ["bar"], "spider": "palmetto_state_armory_crawl",
|
61
|
+
"updated_time": "2015-06-21T20:07:12", "priority": 2, "state": "finished",
|
62
|
+
"close_reason": "cancel_timeout", "spider_args": {}, "errors_count": 0, "started_time":
|
63
|
+
"2015-06-21T19:02:41", "id": "1/3/1", "logs": 88, "elapsed": 2842327,
|
64
|
+
"spider_type": "manual"}, {"logs": 21, "spider": "atlantic_firearms_crawl",
|
65
|
+
"spider_args": {}, "id": "1/1/3", "priority": 2, "items_scraped": 25,
|
66
|
+
"state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
|
67
|
+
0, "responses_received": 114, "tags": [], "elapsed": 3272484100, "close_reason":
|
68
|
+
"cancel_timeout", "started_time": "2015-06-17T20:29:46", "updated_time": "2015-06-17T20:32:47"},
|
69
|
+
{"responses_received": 0, "items_scraped": 0, "close_reason": "no_reason",
|
70
|
+
"logs": 13, "tags": [], "started_time": "2015-06-17T20:27:58", "version":
|
71
|
+
"1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06",
|
72
|
+
"priority": 2, "state": "finished", "spider_args": {}, "errors_count": 2,
|
73
|
+
"elapsed": 3273041718, "id": "1/1/2", "spider_type": "manual"}, {"responses_received":
|
74
|
+
0, "items_scraped": 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29",
|
75
|
+
"version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
|
76
|
+
"2015-06-17T20:17:40", "priority": 2, "state": "finished", "close_reason":
|
77
|
+
"no_reason", "spider_args": {}, "errors_count": 2, "elapsed": 3273668540,
|
78
|
+
"id": "1/1/1", "spider_type": "manual"}]}'
|
79
|
+
http_version:
|
80
|
+
recorded_at: Sat, 25 Jul 2015 17:38:48 GMT
|
81
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,38 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=1&spider=bar
|
6
|
+
body:
|
7
|
+
encoding: US-ASCII
|
8
|
+
string: ''
|
9
|
+
headers:
|
10
|
+
Accept-Encoding:
|
11
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
12
|
+
Accept:
|
13
|
+
- "*/*"
|
14
|
+
User-Agent:
|
15
|
+
- Ruby
|
16
|
+
response:
|
17
|
+
status:
|
18
|
+
code: 200
|
19
|
+
message: OK
|
20
|
+
headers:
|
21
|
+
Server:
|
22
|
+
- nginx/1.8.0
|
23
|
+
Date:
|
24
|
+
- Sat, 25 Jul 2015 17:58:00 GMT
|
25
|
+
Content-Type:
|
26
|
+
- application/json
|
27
|
+
Transfer-Encoding:
|
28
|
+
- chunked
|
29
|
+
Connection:
|
30
|
+
- keep-alive
|
31
|
+
Vary:
|
32
|
+
- Cookie
|
33
|
+
body:
|
34
|
+
encoding: ASCII-8BIT
|
35
|
+
string: '{"status": "ok", "count": 0, "total": 0, "jobs": []}'
|
36
|
+
http_version:
|
37
|
+
recorded_at: Sat, 25 Jul 2015 17:58:00 GMT
|
38
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,62 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=1&spider=atlantic_firearms_crawl
|
6
|
+
body:
|
7
|
+
encoding: US-ASCII
|
8
|
+
string: ''
|
9
|
+
headers:
|
10
|
+
Accept-Encoding:
|
11
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
12
|
+
Accept:
|
13
|
+
- "*/*"
|
14
|
+
User-Agent:
|
15
|
+
- Ruby
|
16
|
+
response:
|
17
|
+
status:
|
18
|
+
code: 200
|
19
|
+
message: OK
|
20
|
+
headers:
|
21
|
+
Server:
|
22
|
+
- nginx/1.8.0
|
23
|
+
Date:
|
24
|
+
- Sat, 25 Jul 2015 17:57:17 GMT
|
25
|
+
Content-Type:
|
26
|
+
- application/json
|
27
|
+
Transfer-Encoding:
|
28
|
+
- chunked
|
29
|
+
Connection:
|
30
|
+
- keep-alive
|
31
|
+
Vary:
|
32
|
+
- Cookie
|
33
|
+
body:
|
34
|
+
encoding: ASCII-8BIT
|
35
|
+
string: '{"status": "ok", "count": 5, "total": 5, "jobs": [{"responses_received":
|
36
|
+
5323, "items_scraped": 1030, "close_reason": "cancel_timeout", "logs": 167,
|
37
|
+
"tags": ["foo", "bar"], "version": "1434570463", "spider": "atlantic_firearms_crawl",
|
38
|
+
"updated_time": "2015-07-23T01:26:37", "priority": 2, "state": "finished",
|
39
|
+
"spider_type": "manual", "spider_args": {}, "started_time": "2015-07-22T23:02:13",
|
40
|
+
"id": "1/1/6", "errors_count": 4, "elapsed": 3972688}, {"logs": 179, "spider":
|
41
|
+
"atlantic_firearms_crawl", "spider_args": {}, "id": "1/1/4", "priority":
|
42
|
+
2, "items_scraped": 1122, "state": "finished", "version": "1434570463", "spider_type":
|
43
|
+
"manual", "errors_count": 3, "responses_received": 6243, "tags": [], "elapsed":
|
44
|
+
242256732, "close_reason": "cancel_timeout", "started_time": "2015-07-22T20:03:31",
|
45
|
+
"updated_time": "2015-07-22T22:35:04"}, {"logs": 21, "spider": "atlantic_firearms_crawl",
|
46
|
+
"spider_args": {}, "id": "1/1/3", "priority": 2, "items_scraped": 25,
|
47
|
+
"state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
|
48
|
+
0, "responses_received": 114, "tags": [], "elapsed": 3273593058, "close_reason":
|
49
|
+
"cancel_timeout", "started_time": "2015-06-17T20:29:46", "updated_time": "2015-06-17T20:32:47"},
|
50
|
+
{"spider": "atlantic_firearms_crawl", "items_scraped": 0, "updated_time":
|
51
|
+
"2015-06-17T20:28:06", "responses_received": 0, "logs": 13, "tags": [], "started_time":
|
52
|
+
"2015-06-17T20:27:58", "close_reason": "no_reason", "priority": 2, "state":
|
53
|
+
"finished", "version": "1434570463", "spider_type": "manual", "spider_args":
|
54
|
+
{}, "errors_count": 2, "elapsed": 3274150676, "id": "1/1/2"}, {"responses_received":
|
55
|
+
0, "items_scraped": 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29",
|
56
|
+
"version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
|
57
|
+
"2015-06-17T20:17:40", "priority": 2, "state": "finished", "close_reason":
|
58
|
+
"no_reason", "spider_type": "manual", "spider_args": {}, "errors_count": 2,
|
59
|
+
"elapsed": 3274777498, "id": "1/1/1"}]}'
|
60
|
+
http_version:
|
61
|
+
recorded_at: Sat, 25 Jul 2015 17:57:17 GMT
|
62
|
+
recorded_with: VCR 2.9.3
|