scrapinghub-client 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +6 -0
  5. data/Gemfile +5 -0
  6. data/LICENSE +21 -0
  7. data/README.md +46 -0
  8. data/Rakefile +22 -0
  9. data/lib/scrapinghub-client.rb +3 -0
  10. data/lib/scrapinghub/jobs.rb +175 -0
  11. data/lib/scrapinghub/version.rb +3 -0
  12. data/scrapinghub.gemspec +26 -0
  13. data/spec/fixtures/vcr_cassettes/jobs/delete/bad_auth.yml +38 -0
  14. data/spec/fixtures/vcr_cassettes/jobs/delete/job/invalid.yml +38 -0
  15. data/spec/fixtures/vcr_cassettes/jobs/delete/job/multiple.yml +38 -0
  16. data/spec/fixtures/vcr_cassettes/jobs/delete/job/single.yml +38 -0
  17. data/spec/fixtures/vcr_cassettes/jobs/delete/project/invalid.yml +39 -0
  18. data/spec/fixtures/vcr_cassettes/jobs/delete/project/valid.yml +38 -0
  19. data/spec/fixtures/vcr_cassettes/jobs/list/bad_auth.yml +38 -0
  20. data/spec/fixtures/vcr_cassettes/jobs/list/count/3.yml +53 -0
  21. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/invalid.yml +38 -0
  22. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/multiple.yml +53 -0
  23. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/single.yml +48 -0
  24. data/spec/fixtures/vcr_cassettes/jobs/list/job/invalid.yml +38 -0
  25. data/spec/fixtures/vcr_cassettes/jobs/list/job/multiple.yml +48 -0
  26. data/spec/fixtures/vcr_cassettes/jobs/list/job/single.yml +43 -0
  27. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/invalid.yml +81 -0
  28. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/multiple.yml +67 -0
  29. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/single.yml +72 -0
  30. data/spec/fixtures/vcr_cassettes/jobs/list/project/invalid.yml +39 -0
  31. data/spec/fixtures/vcr_cassettes/jobs/list/project/valid.yml +81 -0
  32. data/spec/fixtures/vcr_cassettes/jobs/list/spider/invalid.yml +38 -0
  33. data/spec/fixtures/vcr_cassettes/jobs/list/spider/valid.yml +62 -0
  34. data/spec/fixtures/vcr_cassettes/jobs/list/state/finished.yml +81 -0
  35. data/spec/fixtures/vcr_cassettes/jobs/list/state/pending.yml +38 -0
  36. data/spec/fixtures/vcr_cassettes/jobs/schedule/bad_auth.yml +38 -0
  37. data/spec/fixtures/vcr_cassettes/jobs/schedule/project/invalid.yml +39 -0
  38. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/add_tag.yml +38 -0
  39. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/already-running.yml +39 -0
  40. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/extra.yml +38 -0
  41. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/minimal.yml +38 -0
  42. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/priority.yml +38 -0
  43. data/spec/fixtures/vcr_cassettes/jobs/stop/bad_auth.yml +38 -0
  44. data/spec/fixtures/vcr_cassettes/jobs/stop/job/already-stopped.yml +38 -0
  45. data/spec/fixtures/vcr_cassettes/jobs/stop/job/invalid.yml +38 -0
  46. data/spec/fixtures/vcr_cassettes/jobs/stop/job/valid.yml +38 -0
  47. data/spec/fixtures/vcr_cassettes/jobs/stop/project/invalid.yml +39 -0
  48. data/spec/fixtures/vcr_cassettes/jobs/update/bad_auth.yml +38 -0
  49. data/spec/fixtures/vcr_cassettes/jobs/update/has_tag.yml +38 -0
  50. data/spec/fixtures/vcr_cassettes/jobs/update/job.yml +38 -0
  51. data/spec/fixtures/vcr_cassettes/jobs/update/lacks_tag.yml +38 -0
  52. data/spec/fixtures/vcr_cassettes/jobs/update/no-query-filters.yml +38 -0
  53. data/spec/fixtures/vcr_cassettes/jobs/update/no-update-params.yml +38 -0
  54. data/spec/fixtures/vcr_cassettes/jobs/update/project/invalid.yml +39 -0
  55. data/spec/fixtures/vcr_cassettes/jobs/update/spider.yml +38 -0
  56. data/spec/fixtures/vcr_cassettes/jobs/update/state.yml +38 -0
  57. data/spec/integration/jobs_spec.rb +567 -0
  58. data/spec/spec_helper.rb +22 -0
  59. data/spec/unit/jobs_spec.rb +188 -0
  60. data/spec/unit/scrapinghub_spec.rb +8 -0
  61. metadata +200 -0
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/stop.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F123%2F123&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 00:23:54 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '16'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "ok"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 00:23:54 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/stop.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F1%2F9&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 00:00:17 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '16'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "ok"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 00:00:17 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,39 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/stop.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F1%2F9&project=2
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: User 'abevoelker' doesn't have access to project 2
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 23:50:30 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '89'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "User ''abevoelker'' doesn''t have
36
+ access to project 2"}'
37
+ http_version:
38
+ recorded_at: Sat, 25 Jul 2015 23:50:30 GMT
39
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F1%2F18&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 403
19
+ message: FORBIDDEN
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:11:29 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "error", "message": "Authentication failed"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:11:30 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: has_tag=bar&project=1&remove_tag=bar
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:53:09 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 4}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:53:10 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: add_tag=baz&job=1%2F3%2F7&job=1%2F1%2F18&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:46:42 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 2}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:46:43 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: add_tag=bar&lacks_tag=bar&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:54:32 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 18}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:54:33 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: No query filters provided
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:08:06 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '64'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "No query filters provided"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:08:07 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F1%2F18&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: No update modifiers provided
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:04:46 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '67'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "No update modifiers provided"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:04:47 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,39 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: project=2
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: User 'abevoelker' doesn't have access to project 2
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:01:09 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '89'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "User ''abevoelker'' doesn''t have
36
+ access to project 2"}'
37
+ http_version:
38
+ recorded_at: Sun, 26 Jul 2015 03:01:09 GMT
39
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: add_tag=foo&project=1&spider=atlantic_firearms_crawl
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:38:46 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 11}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:38:46 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: add_tag=bar&project=1&state=running
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:44:28 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 2}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:44:29 GMT
38
+ recorded_with: VCR 2.9.3