scrapinghub-client 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +6 -0
  5. data/Gemfile +5 -0
  6. data/LICENSE +21 -0
  7. data/README.md +46 -0
  8. data/Rakefile +22 -0
  9. data/lib/scrapinghub-client.rb +3 -0
  10. data/lib/scrapinghub/jobs.rb +175 -0
  11. data/lib/scrapinghub/version.rb +3 -0
  12. data/scrapinghub.gemspec +26 -0
  13. data/spec/fixtures/vcr_cassettes/jobs/delete/bad_auth.yml +38 -0
  14. data/spec/fixtures/vcr_cassettes/jobs/delete/job/invalid.yml +38 -0
  15. data/spec/fixtures/vcr_cassettes/jobs/delete/job/multiple.yml +38 -0
  16. data/spec/fixtures/vcr_cassettes/jobs/delete/job/single.yml +38 -0
  17. data/spec/fixtures/vcr_cassettes/jobs/delete/project/invalid.yml +39 -0
  18. data/spec/fixtures/vcr_cassettes/jobs/delete/project/valid.yml +38 -0
  19. data/spec/fixtures/vcr_cassettes/jobs/list/bad_auth.yml +38 -0
  20. data/spec/fixtures/vcr_cassettes/jobs/list/count/3.yml +53 -0
  21. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/invalid.yml +38 -0
  22. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/multiple.yml +53 -0
  23. data/spec/fixtures/vcr_cassettes/jobs/list/has_tag/single.yml +48 -0
  24. data/spec/fixtures/vcr_cassettes/jobs/list/job/invalid.yml +38 -0
  25. data/spec/fixtures/vcr_cassettes/jobs/list/job/multiple.yml +48 -0
  26. data/spec/fixtures/vcr_cassettes/jobs/list/job/single.yml +43 -0
  27. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/invalid.yml +81 -0
  28. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/multiple.yml +67 -0
  29. data/spec/fixtures/vcr_cassettes/jobs/list/lacks_tag/single.yml +72 -0
  30. data/spec/fixtures/vcr_cassettes/jobs/list/project/invalid.yml +39 -0
  31. data/spec/fixtures/vcr_cassettes/jobs/list/project/valid.yml +81 -0
  32. data/spec/fixtures/vcr_cassettes/jobs/list/spider/invalid.yml +38 -0
  33. data/spec/fixtures/vcr_cassettes/jobs/list/spider/valid.yml +62 -0
  34. data/spec/fixtures/vcr_cassettes/jobs/list/state/finished.yml +81 -0
  35. data/spec/fixtures/vcr_cassettes/jobs/list/state/pending.yml +38 -0
  36. data/spec/fixtures/vcr_cassettes/jobs/schedule/bad_auth.yml +38 -0
  37. data/spec/fixtures/vcr_cassettes/jobs/schedule/project/invalid.yml +39 -0
  38. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/add_tag.yml +38 -0
  39. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/already-running.yml +39 -0
  40. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/extra.yml +38 -0
  41. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/minimal.yml +38 -0
  42. data/spec/fixtures/vcr_cassettes/jobs/schedule/spider/priority.yml +38 -0
  43. data/spec/fixtures/vcr_cassettes/jobs/stop/bad_auth.yml +38 -0
  44. data/spec/fixtures/vcr_cassettes/jobs/stop/job/already-stopped.yml +38 -0
  45. data/spec/fixtures/vcr_cassettes/jobs/stop/job/invalid.yml +38 -0
  46. data/spec/fixtures/vcr_cassettes/jobs/stop/job/valid.yml +38 -0
  47. data/spec/fixtures/vcr_cassettes/jobs/stop/project/invalid.yml +39 -0
  48. data/spec/fixtures/vcr_cassettes/jobs/update/bad_auth.yml +38 -0
  49. data/spec/fixtures/vcr_cassettes/jobs/update/has_tag.yml +38 -0
  50. data/spec/fixtures/vcr_cassettes/jobs/update/job.yml +38 -0
  51. data/spec/fixtures/vcr_cassettes/jobs/update/lacks_tag.yml +38 -0
  52. data/spec/fixtures/vcr_cassettes/jobs/update/no-query-filters.yml +38 -0
  53. data/spec/fixtures/vcr_cassettes/jobs/update/no-update-params.yml +38 -0
  54. data/spec/fixtures/vcr_cassettes/jobs/update/project/invalid.yml +39 -0
  55. data/spec/fixtures/vcr_cassettes/jobs/update/spider.yml +38 -0
  56. data/spec/fixtures/vcr_cassettes/jobs/update/state.yml +38 -0
  57. data/spec/integration/jobs_spec.rb +567 -0
  58. data/spec/spec_helper.rb +22 -0
  59. data/spec/unit/jobs_spec.rb +188 -0
  60. data/spec/unit/scrapinghub_spec.rb +8 -0
  61. metadata +200 -0
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/stop.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F123%2F123&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 00:23:54 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '16'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "ok"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 00:23:54 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/stop.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F1%2F9&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 00:00:17 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '16'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "ok"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 00:00:17 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,39 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/stop.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F1%2F9&project=2
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: User 'abevoelker' doesn't have access to project 2
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sat, 25 Jul 2015 23:50:30 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '89'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "User ''abevoelker'' doesn''t have
36
+ access to project 2"}'
37
+ http_version:
38
+ recorded_at: Sat, 25 Jul 2015 23:50:30 GMT
39
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F1%2F18&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 403
19
+ message: FORBIDDEN
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:11:29 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "error", "message": "Authentication failed"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:11:30 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: has_tag=bar&project=1&remove_tag=bar
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:53:09 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 4}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:53:10 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: add_tag=baz&job=1%2F3%2F7&job=1%2F1%2F18&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:46:42 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 2}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:46:43 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: add_tag=bar&lacks_tag=bar&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:54:32 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 18}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:54:33 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: No query filters provided
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:08:06 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '64'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "No query filters provided"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:08:07 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: job=1%2F1%2F18&project=1
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: No update modifiers provided
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:04:46 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '67'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "No update modifiers provided"}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:04:47 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,39 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: project=2
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 400
19
+ message: User 'abevoelker' doesn't have access to project 2
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:01:09 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Content-Length:
28
+ - '89'
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: UTF-8
35
+ string: '{"status": "badrequest", "message": "User ''abevoelker'' doesn''t have
36
+ access to project 2"}'
37
+ http_version:
38
+ recorded_at: Sun, 26 Jul 2015 03:01:09 GMT
39
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: add_tag=foo&project=1&spider=atlantic_firearms_crawl
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:38:46 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 11}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:38:46 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: post
5
+ uri: http://XXX:@dash.scrapinghub.com/api/jobs/update.json
6
+ body:
7
+ encoding: UTF-8
8
+ string: add_tag=bar&project=1&state=running
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - "*/*"
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Server:
22
+ - nginx/1.8.0
23
+ Date:
24
+ - Sun, 26 Jul 2015 03:44:28 GMT
25
+ Content-Type:
26
+ - application/json
27
+ Transfer-Encoding:
28
+ - chunked
29
+ Connection:
30
+ - keep-alive
31
+ Vary:
32
+ - Cookie
33
+ body:
34
+ encoding: ASCII-8BIT
35
+ string: '{"status": "ok", "count": 2}'
36
+ http_version:
37
+ recorded_at: Sun, 26 Jul 2015 03:44:29 GMT
38
+ recorded_with: VCR 2.9.3