scrapinghub-client 0.0.1

data/spec/fixtures/vcr_cassettes/jobs/list/state/finished.yml

@@ -0,0 +1,81 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=1&state=finished
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sat, 25 Jul 2015 17:59:13 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "ok", "count": 9, "total": 9, "jobs": [{"responses_received":
+        5323, "items_scraped": 1030, "logs": 167, "tags": ["foo", "bar"], "version":
+        "1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-23T01:26:37",
+        "priority": 2, "state": "finished", "close_reason": "cancel_timeout", "spider_type":
+        "manual", "spider_args": {}, "started_time": "2015-07-22T23:02:13", "id":
+        "1/1/6", "errors_count": 4, "elapsed": 4088408}, {"responses_received":
+        6243, "items_scraped": 1122, "version": "1434570463", "tags": [], "close_reason":
+        "cancel_timeout", "spider": "atlantic_firearms_crawl", "updated_time": "2015-07-22T22:35:04",
+        "priority": 2, "state": "finished", "spider_type": "manual", "spider_args":
+        {}, "started_time": "2015-07-22T20:03:31", "id": "1/1/4", "errors_count":
+        3, "elapsed": 242372452, "logs": 179}, {"responses_received": 365486, "items_scraped":
+        4037, "close_reason": "cancel_timeout", "logs": 33631, "tags": ["foo"], "version":
+        "1434570463", "spider": "firearms_for_sale_crawl", "updated_time": "2015-07-21T02:43:25",
+        "priority": 2, "state": "finished", "spider_args": {}, "started_time": "2015-06-27T18:36:06",
+        "id": "1/5/1", "errors_count": 0, "elapsed": 3860595, "spider_type": "manual"},
+        {"responses_received": 276710, "items_scraped": 63362, "close_reason": "finished",
+        "logs": 7356, "tags": [], "version": "1434570463", "spider": "palmetto_state_armory_crawl",
+        "priority": 2, "updated_time": "2015-07-02T20:49:25", "state": "finished",
+        "spider_type": "manual", "spider_args": {"": ""}, "started_time": "2015-06-27T18:35:17",
+        "id": "1/3/2", "errors_count": 0, "elapsed": 1976987487}, {"logs": 2189,
+        "spider": "cheaper_than_dirt_crawl", "spider_args": {}, "id": "1/2/1",
+        "priority": 2, "items_scraped": 81852, "state": "finished", "version": "1434570463",
+        "spider_type": "manual", "errors_count": 0, "responses_received": 86259, "tags":
+        [], "elapsed": 2901085220, "close_reason": "finished", "started_time": "2015-06-20T15:59:43",
+        "updated_time": "2015-06-22T04:07:48"}, {"responses_received": 1194, "items_scraped":
+        1066, "version": "1434570463", "tags": ["bar"], "spider": "palmetto_state_armory_crawl",
+        "updated_time": "2015-06-21T20:07:12", "priority": 2, "state": "finished",
+        "close_reason": "cancel_timeout", "spider_args": {}, "errors_count": 0, "started_time":
+        "2015-06-21T19:02:41", "id": "1/3/1", "logs": 88, "elapsed": 4067005,
+        "spider_type": "manual"}, {"logs": 21, "spider": "atlantic_firearms_crawl",
+        "spider_args": {}, "id": "1/1/3", "priority": 2, "items_scraped": 25,
+        "state": "finished", "version": "1434570463", "spider_type": "manual", "errors_count":
+        0, "responses_received": 114, "tags": [], "elapsed": 3273708778, "close_reason":
+        "cancel_timeout", "started_time": "2015-06-17T20:29:46", "updated_time": "2015-06-17T20:32:47"},
+        {"responses_received": 0, "items_scraped": 0, "close_reason": "no_reason",
+        "logs": 13, "tags": [], "started_time": "2015-06-17T20:27:58", "version":
+        "1434570463", "spider": "atlantic_firearms_crawl", "updated_time": "2015-06-17T20:28:06",
+        "priority": 2, "state": "finished", "spider_args": {}, "errors_count": 2,
+        "elapsed": 3274266396, "id": "1/1/2", "spider_type": "manual"}, {"responses_received":
+        0, "items_scraped": 0, "logs": 13, "tags": [], "started_time": "2015-06-17T20:17:29",
+        "version": "1434570463", "spider": "atlantic_firearms_crawl", "updated_time":
+        "2015-06-17T20:17:40", "priority": 2, "state": "finished", "close_reason":
+        "no_reason", "spider_args": {}, "errors_count": 2, "elapsed": 3274893218,
+        "id": "1/1/1", "spider_type": "manual"}]}'
+    http_version: 
+  recorded_at: Sat, 25 Jul 2015 17:59:13 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/list/state/pending.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://XXX:@dash.scrapinghub.com/api/jobs/list.json?project=1&state=pending
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sat, 25 Jul 2015 18:00:23 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "ok", "count": 0, "total": 0, "jobs": []}'
+    http_version: 
+  recorded_at: Sat, 25 Jul 2015 18:00:24 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/schedule/bad_auth.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/schedule.json
+    body:
+      encoding: UTF-8
+      string: project=1&spider=atlantic_firearms_crawl
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 403
+      message: FORBIDDEN
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 01:10:47 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "error", "message": "Authentication failed"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 01:10:48 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/schedule/project/invalid.yml

@@ -0,0 +1,39 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/schedule.json
+    body:
+      encoding: UTF-8
+      string: project=2&spider=atlantic_firearms_crawl
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 400
+      message: User 'abevoelker' doesn't have access to project 2
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 01:11:42 GMT
+      Content-Type:
+      - application/json
+      Content-Length:
+      - '89'
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: UTF-8
+      string: '{"status": "badrequest", "message": "User ''abevoelker'' doesn''t have
+        access to project 2"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 01:11:42 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/add_tag.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/schedule.json
+    body:
+      encoding: UTF-8
+      string: add_tag=foo&project=1&spider=atlantic_firearms_crawl
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 01:51:54 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "ok", "jobid": "1/1/14"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 01:51:55 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/already-running.yml

@@ -0,0 +1,39 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/schedule.json
+    body:
+      encoding: UTF-8
+      string: project=1&spider=atlantic_firearms_crawl
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 400
+      message: BAD REQUEST
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 01:37:49 GMT
+      Content-Type:
+      - application/json
+      Content-Length:
+      - '84'
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: UTF-8
+      string: '{"status": "error", "message": "Spider ''atlantic_firearms_crawl''
+        already scheduled"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 01:37:49 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/extra.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/schedule.json
+    body:
+      encoding: UTF-8
+      string: DOWNLOAD_DELAY=0.5&project=1&spider=atlantic_firearms_crawl
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 01:58:16 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "ok", "jobid": "1/1/17"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 01:58:17 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/minimal.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/schedule.json
+    body:
+      encoding: UTF-8
+      string: project=1&spider=atlantic_firearms_crawl
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 01:31:53 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "ok", "jobid": "1/1/11"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 01:31:53 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/priority.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/schedule.json
+    body:
+      encoding: UTF-8
+      string: priority=4&project=1&spider=atlantic_firearms_crawl
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 01:55:59 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "ok", "jobid": "1/1/16"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 01:55:59 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/stop/bad_auth.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/jobs/stop.json
+    body:
+      encoding: UTF-8
+      string: job=1%2F1%2F9&project=1
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 403
+      message: FORBIDDEN
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 00:06:28 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "error", "message": "Authentication failed"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 00:06:28 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/stop/job/already-stopped.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/jobs/stop.json
+    body:
+      encoding: UTF-8
+      string: job=1%2F1%2F6&project=1
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sun, 26 Jul 2015 00:26:15 GMT
+      Content-Type:
+      - application/json
+      Content-Length:
+      - '16'
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: UTF-8
+      string: '{"status": "ok"}'
+    http_version: 
+  recorded_at: Sun, 26 Jul 2015 00:26:16 GMT
+recorded_with: VCR 2.9.3