diffbot_simple 0.0.4 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +42 -20
- data/lib/diffbot_simple/symbolize.rb +11 -8
- data/lib/diffbot_simple/v2/analyze.rb +1 -6
- data/lib/diffbot_simple/v2/api_helper.rb +8 -2
- data/lib/diffbot_simple/v2/article.rb +1 -1
- data/lib/diffbot_simple/v2/bulk.rb +11 -0
- data/lib/diffbot_simple/v2/bulk_api.rb +8 -0
- data/lib/diffbot_simple/v2/client.rb +13 -6
- data/lib/diffbot_simple/v2/crawl.rb +53 -0
- data/lib/diffbot_simple/v2/crawlbot_api.rb +22 -0
- data/lib/diffbot_simple/v2/custom.rb +1 -5
- data/lib/diffbot_simple/v2/image.rb +0 -4
- data/lib/diffbot_simple/v2/product.rb +0 -4
- data/lib/diffbot_simple/version.rb +1 -1
- data/lib/diffbot_simple.rb +4 -1
- data/spec/serialize_test_data.json +38 -0
- data/spec/symbolize_spec.rb +17 -0
- data/spec/{analyze_spec.rb → v2/analyze_spec.rb} +4 -4
- data/spec/v2/article_spec.rb +12 -12
- data/spec/v2/bulk_api_spec.rb +54 -0
- data/spec/v2/bulk_spec.rb +56 -0
- data/spec/v2/client_spec.rb +35 -4
- data/spec/v2/crawl_spec.rb +66 -0
- data/spec/v2/crawlbot_api_spec.rb +54 -0
- data/spec/v2/custom_spec.rb +4 -4
- data/spec/{image_spec.rb → v2/image_spec.rb} +4 -4
- data/spec/{product_spec.rb → v2/product_spec.rb} +4 -4
- metadata +24 -11
- data/lib/diffbot_simple/v2/crawlbot.rb +0 -75
- data/spec/v2/crawlbot_spec.rb +0 -113
data/spec/v2/crawlbot_spec.rb
DELETED
@@ -1,113 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
module DiffbotSimple::V2
|
3
|
-
describe Crawlbot do
|
4
|
-
let(:client) { Client.new token: token }
|
5
|
-
let(:custom) { client.custom name: "my_custom_api" }
|
6
|
-
let(:single_crawl_response_body) {{body: '{"jobs":[{"foo":"bar"}]}'}}
|
7
|
-
let(:name) { "crawl_name"}
|
8
|
-
let(:subject) { client.crawlbot }
|
9
|
-
context "when retreiving all crawls" do
|
10
|
-
let(:all) { stubbed_request;subject.all; }
|
11
|
-
let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: {token: token}).to_return( single_crawl_response_body) }
|
12
|
-
it "should make a request to /crawl with the token as argument" do
|
13
|
-
all
|
14
|
-
expect(stubbed_request).to have_been_requested
|
15
|
-
end
|
16
|
-
it "should return an crawl array " do
|
17
|
-
expect(all).to eql([{ foo: 'bar' }])
|
18
|
-
end
|
19
|
-
end
|
20
|
-
context "when asking for a named crawl" do
|
21
|
-
let(:named_crawl) { stubbed_request; subject.single_crawl name: name, onlyProcessIfNew: 0, apiUrl: custom }
|
22
|
-
let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, onlyProcessIfNew: 0, apiUrl: custom.to_crawl_api_url}).to_return single_crawl_response_body() }
|
23
|
-
it "should make a request to /crawl with the token and name as arguments" do
|
24
|
-
named_crawl
|
25
|
-
expect(stubbed_request).to have_been_requested
|
26
|
-
end
|
27
|
-
it "should return an crawl hash" do
|
28
|
-
expect(named_crawl).to eql({ foo: 'bar' })
|
29
|
-
end
|
30
|
-
end
|
31
|
-
context "when deleting a named crawl" do
|
32
|
-
let(:delete) { stubbed_request; subject.delete name: name }
|
33
|
-
let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, delete: 1 }).to_return(body: '{"response":"Successfully deleted job." }') }
|
34
|
-
it "should make the request to delete it" do
|
35
|
-
delete
|
36
|
-
expect(stubbed_request).to have_been_requested
|
37
|
-
end
|
38
|
-
it "should return the faked response" do
|
39
|
-
expect(delete).to eql({response: "Successfully deleted job."})
|
40
|
-
end
|
41
|
-
end
|
42
|
-
context "when pausing or unpausing a named crawl" do
|
43
|
-
let(:pause) { stubbed_pause_request; subject.pause name: name }
|
44
|
-
let(:unpause) { stubbed_unpause_request; subject.unpause name: name }
|
45
|
-
let(:stubbed_pause_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, pause: 1 }).to_return(single_crawl_response_body) }
|
46
|
-
let(:stubbed_unpause_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, pause: 0 }).to_return(single_crawl_response_body) }
|
47
|
-
it "should make the request to pause it" do
|
48
|
-
pause
|
49
|
-
expect(stubbed_pause_request).to have_been_requested
|
50
|
-
end
|
51
|
-
it "should make the request to unpause it" do
|
52
|
-
unpause
|
53
|
-
expect(stubbed_unpause_request).to have_been_requested
|
54
|
-
end
|
55
|
-
end
|
56
|
-
context "when restarting a named crawl" do
|
57
|
-
let(:restart) { stubbed_request; subject.restart name: name }
|
58
|
-
let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, restart: 1 }).to_return(single_crawl_response_body) }
|
59
|
-
it "should make the request to restart it" do
|
60
|
-
restart
|
61
|
-
expect(stubbed_request).to have_been_requested
|
62
|
-
end
|
63
|
-
end
|
64
|
-
context "when requesting a crawls result" do
|
65
|
-
let(:result) { stubbed_crawl_request;stubbed_result_request; subject.result name: name }
|
66
|
-
let(:test_download_url) { "http://google.com" }
|
67
|
-
let(:stubbed_crawl_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token}).to_return(body: "{\"jobs\":[{\"downloadJson\":\"#{test_download_url}\"}]}") }
|
68
|
-
let(:stubbed_result_request) { stub_request(:get, test_download_url).to_return(body: "[{'f':'b'}]") }
|
69
|
-
it "should make the two requests to get the results" do
|
70
|
-
result
|
71
|
-
expect(stubbed_crawl_request).to have_been_requested
|
72
|
-
expect(stubbed_result_request).to have_been_requested
|
73
|
-
end
|
74
|
-
end
|
75
|
-
describe "if diffbots response is an error" do
|
76
|
-
let(:error_from_diffbot) { { error: "Your token has exceeded the allowed number of calls, or has otherwise been throttled for API abuse.", errorCode: 429 }.to_json }
|
77
|
-
let(:stubbed_request) { stub_request(:get, /#{base_url}\/crawl*/).to_return(body: error_from_diffbot) }
|
78
|
-
shared_examples_for "an error" do
|
79
|
-
it "and raise an DiffbotError" do
|
80
|
-
expect{raiser}.to raise_error DiffbotError
|
81
|
-
end
|
82
|
-
end
|
83
|
-
context "on all" do
|
84
|
-
let(:raiser) { stubbed_request;subject.all; }
|
85
|
-
it_should_behave_like "an error"
|
86
|
-
end
|
87
|
-
context "on single_crawl" do
|
88
|
-
let(:raiser) { stubbed_request;subject.single_crawl name: name; }
|
89
|
-
it_should_behave_like "an error"
|
90
|
-
end
|
91
|
-
context "on delete" do
|
92
|
-
let(:raiser) { stubbed_request;subject.delete name: name; }
|
93
|
-
it_should_behave_like "an error"
|
94
|
-
end
|
95
|
-
context "on pause" do
|
96
|
-
let(:raiser) { stubbed_request;subject.pause name: name; }
|
97
|
-
it_should_behave_like "an error"
|
98
|
-
end
|
99
|
-
context "on unpause" do
|
100
|
-
let(:raiser) { stubbed_request;subject.unpause name: name; }
|
101
|
-
it_should_behave_like "an error"
|
102
|
-
end
|
103
|
-
context "on restart" do
|
104
|
-
let(:raiser) { stubbed_request;subject.restart name: name; }
|
105
|
-
it_should_behave_like "an error"
|
106
|
-
end
|
107
|
-
context "on result" do
|
108
|
-
let(:raiser) { stubbed_request;subject.result name: name; }
|
109
|
-
it_should_behave_like "an error"
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|