rest-scrapyd 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -0
- data/lib/rest-scrapyd.rb +5 -0
- data/test/test_api.rb +59 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5a432b7b410654bb0c2b236d10b38f9f897e4961
|
4
|
+
data.tar.gz: 6a4ffb540f02a85f136b642157f04462ee79c4fb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 47838ab472ed6cff33d4be2586dfaa90b2fa354c2e009c4753ec2bc26d39c047bf92bb0d2af1552e6e1f47b699f5071e561179b6c801dbfb3f901256b9e2aff0
|
7
|
+
data.tar.gz: 8e342c482a30ff15aa14cea0137ab17120b94c8fd1308c3c65400d1490247db6b850a82d2478e7064cd4e1d25a02563f97db4f3d1556bb5600a565ec74b28272
|
data/README.md
CHANGED
@@ -48,6 +48,9 @@ r.listversions
|
|
48
48
|
r = RestScrapyd.new project: "project1"
|
49
49
|
r.schedule("spider1", "123456-master")
|
50
50
|
# => "1234567890abcdef1234567890abcdef"
|
51
|
+
|
52
|
+
# http basic is also possible, when running scrapyd behind a reverse proxy
|
53
|
+
r = RestScrapyd.new site: "https://example.com:6843/", username: "deploy", password: "s3cret"
|
51
54
|
```
|
52
55
|
|
53
56
|
For more information, see the [RestScrapyd](http://rubydoc.info/github/wvengen/rest-scrapyd/RestScrapyd.html)
|
data/lib/rest-scrapyd.rb
CHANGED
@@ -8,6 +8,7 @@ RestScrapyd = RC::Builder.client do
|
|
8
8
|
|
9
9
|
use RC::DefaultSite , 'http://localhost:6800/'
|
10
10
|
use RC::DefaultHeaders, {'Accept' => 'application/json'}
|
11
|
+
use RC::AuthBasic , nil, nil
|
11
12
|
|
12
13
|
use RC::CommonLogger , nil
|
13
14
|
use RC::ErrorHandler , lambda {|env| RuntimeError.new(env[RC::RESPONSE_BODY]['message']) }
|
@@ -17,7 +18,11 @@ RestScrapyd = RC::Builder.client do
|
|
17
18
|
use RC::Cache , nil, 600
|
18
19
|
end
|
19
20
|
|
21
|
+
|
20
22
|
# Don't define a `RestScrapyd::Client` and include it in the class to avoid yardoc choking
|
23
|
+
# (plus two newlines to avoid this getting into the documentation)
|
24
|
+
|
25
|
+
|
21
26
|
class RestScrapyd
|
22
27
|
include RestCore
|
23
28
|
|
data/test/test_api.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
|
2
|
+
require 'rest-scrapyd'
|
3
|
+
require 'rest-core/test'
|
4
|
+
|
5
|
+
Pork::API.describe RestScrapyd do
|
6
|
+
after do
|
7
|
+
WebMock.reset!
|
8
|
+
end
|
9
|
+
|
10
|
+
def new
|
11
|
+
RestScrapyd.new(project: 'default')
|
12
|
+
end
|
13
|
+
|
14
|
+
would 'schedule' do
|
15
|
+
stub_request(:post, 'http://localhost:6800/schedule.json').
|
16
|
+
to_return(:body => '{"status": "ok", "jobid": "1234Q"}')
|
17
|
+
new.schedule('spider', 'version').should.eq("1234Q")
|
18
|
+
end
|
19
|
+
|
20
|
+
would 'cancel' do
|
21
|
+
stub_request(:post, 'http://localhost:6800/cancel.json').
|
22
|
+
to_return(:body => '{"status": "ok", "prevstate": "running"}')
|
23
|
+
new.cancel('4321Q').should.eq("running")
|
24
|
+
end
|
25
|
+
|
26
|
+
would 'listprojects' do
|
27
|
+
stub_request(:get, 'http://localhost:6800/listprojects.json').
|
28
|
+
to_return(:body => '{"status": "ok", "projects": ["myproject", "otherproject"]}')
|
29
|
+
new.listprojects.should.eq(["myproject", "otherproject"])
|
30
|
+
end
|
31
|
+
|
32
|
+
would 'listversions' do
|
33
|
+
stub_request(:get, 'http://localhost:6800/listversions.json?project=default').
|
34
|
+
to_return(:body => '{"status": "ok", "versions": ["r5", "r1"]}')
|
35
|
+
new.listversions.should.eq(["r5", "r1"])
|
36
|
+
end
|
37
|
+
|
38
|
+
would 'listspiders' do
|
39
|
+
stub_request(:get, 'http://localhost:6800/listspiders.json?project=default').
|
40
|
+
to_return(:body => '{"status": "ok", "spiders": ["spider1", "spider2", "spider3"]}')
|
41
|
+
new.listspiders.should.eq(["spider1", "spider2", "spider3"])
|
42
|
+
end
|
43
|
+
|
44
|
+
would 'listjobs' do
|
45
|
+
stub_request(:get, 'http://localhost:6800/listjobs.json?project=default').
|
46
|
+
to_return(:body => '{"status": "ok",
|
47
|
+
"pending": [{"id": "12A", "spider": "spider1"}],
|
48
|
+
"running": [{"id": "12B", "spider": "spider2", "start_time": "2012-09-12 10:14:03.594664"}],
|
49
|
+
"finished": [{"id": "12C", "spider": "spider3", "start_time": "2012-09-12 10:14:03.594664", "end_time": "2012-09-12 10:24:03.594664"}]}')
|
50
|
+
new.listjobs.keys.should.eq(["pending", "running", "finished"])
|
51
|
+
end
|
52
|
+
|
53
|
+
would 'error code' do
|
54
|
+
stub_request(:get, 'http://localhost:6800/listprojects.json').
|
55
|
+
to_return(:body => '{"status": "error", "message": "whoops"}')
|
56
|
+
lambda { new.listprojects }.should.raise(RuntimeError)
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-scrapyd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- wvengen
|
@@ -35,6 +35,7 @@ files:
|
|
35
35
|
- LICENSE
|
36
36
|
- README.md
|
37
37
|
- lib/rest-scrapyd.rb
|
38
|
+
- test/test_api.rb
|
38
39
|
homepage: https://github.com/wvengen/rest-scrapyd
|
39
40
|
licenses:
|
40
41
|
- MIT
|
@@ -60,4 +61,5 @@ rubygems_version: 2.4.3
|
|
60
61
|
signing_key:
|
61
62
|
specification_version: 4
|
62
63
|
summary: REST client for the Scrapyd API
|
63
|
-
test_files:
|
64
|
+
test_files:
|
65
|
+
- test/test_api.rb
|