diffbot_simple 0.0.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +42 -20
- data/lib/diffbot_simple/symbolize.rb +11 -8
- data/lib/diffbot_simple/v2/analyze.rb +1 -6
- data/lib/diffbot_simple/v2/api_helper.rb +8 -2
- data/lib/diffbot_simple/v2/article.rb +1 -1
- data/lib/diffbot_simple/v2/bulk.rb +11 -0
- data/lib/diffbot_simple/v2/bulk_api.rb +8 -0
- data/lib/diffbot_simple/v2/client.rb +13 -6
- data/lib/diffbot_simple/v2/crawl.rb +53 -0
- data/lib/diffbot_simple/v2/crawlbot_api.rb +22 -0
- data/lib/diffbot_simple/v2/custom.rb +1 -5
- data/lib/diffbot_simple/v2/image.rb +0 -4
- data/lib/diffbot_simple/v2/product.rb +0 -4
- data/lib/diffbot_simple/version.rb +1 -1
- data/lib/diffbot_simple.rb +4 -1
- data/spec/serialize_test_data.json +38 -0
- data/spec/symbolize_spec.rb +17 -0
- data/spec/{analyze_spec.rb → v2/analyze_spec.rb} +4 -4
- data/spec/v2/article_spec.rb +12 -12
- data/spec/v2/bulk_api_spec.rb +54 -0
- data/spec/v2/bulk_spec.rb +56 -0
- data/spec/v2/client_spec.rb +35 -4
- data/spec/v2/crawl_spec.rb +66 -0
- data/spec/v2/crawlbot_api_spec.rb +54 -0
- data/spec/v2/custom_spec.rb +4 -4
- data/spec/{image_spec.rb → v2/image_spec.rb} +4 -4
- data/spec/{product_spec.rb → v2/product_spec.rb} +4 -4
- metadata +24 -11
- data/lib/diffbot_simple/v2/crawlbot.rb +0 -75
- data/spec/v2/crawlbot_spec.rb +0 -113
data/spec/v2/crawlbot_spec.rb
DELETED
@@ -1,113 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
module DiffbotSimple::V2
|
3
|
-
describe Crawlbot do
|
4
|
-
let(:client) { Client.new token: token }
|
5
|
-
let(:custom) { client.custom name: "my_custom_api" }
|
6
|
-
let(:single_crawl_response_body) {{body: '{"jobs":[{"foo":"bar"}]}'}}
|
7
|
-
let(:name) { "crawl_name"}
|
8
|
-
let(:subject) { client.crawlbot }
|
9
|
-
context "when retreiving all crawls" do
|
10
|
-
let(:all) { stubbed_request;subject.all; }
|
11
|
-
let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: {token: token}).to_return( single_crawl_response_body) }
|
12
|
-
it "should make a request to /crawl with the token as argument" do
|
13
|
-
all
|
14
|
-
expect(stubbed_request).to have_been_requested
|
15
|
-
end
|
16
|
-
it "should return an crawl array " do
|
17
|
-
expect(all).to eql([{ foo: 'bar' }])
|
18
|
-
end
|
19
|
-
end
|
20
|
-
context "when asking for a named crawl" do
|
21
|
-
let(:named_crawl) { stubbed_request; subject.single_crawl name: name, onlyProcessIfNew: 0, apiUrl: custom }
|
22
|
-
let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, onlyProcessIfNew: 0, apiUrl: custom.to_crawl_api_url}).to_return single_crawl_response_body() }
|
23
|
-
it "should make a request to /crawl with the token and name as arguments" do
|
24
|
-
named_crawl
|
25
|
-
expect(stubbed_request).to have_been_requested
|
26
|
-
end
|
27
|
-
it "should return an crawl hash" do
|
28
|
-
expect(named_crawl).to eql({ foo: 'bar' })
|
29
|
-
end
|
30
|
-
end
|
31
|
-
context "when deleting a named crawl" do
|
32
|
-
let(:delete) { stubbed_request; subject.delete name: name }
|
33
|
-
let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, delete: 1 }).to_return(body: '{"response":"Successfully deleted job." }') }
|
34
|
-
it "should make the request to delete it" do
|
35
|
-
delete
|
36
|
-
expect(stubbed_request).to have_been_requested
|
37
|
-
end
|
38
|
-
it "should return the faked response" do
|
39
|
-
expect(delete).to eql({response: "Successfully deleted job."})
|
40
|
-
end
|
41
|
-
end
|
42
|
-
context "when pausing or unpausing a named crawl" do
|
43
|
-
let(:pause) { stubbed_pause_request; subject.pause name: name }
|
44
|
-
let(:unpause) { stubbed_unpause_request; subject.unpause name: name }
|
45
|
-
let(:stubbed_pause_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, pause: 1 }).to_return(single_crawl_response_body) }
|
46
|
-
let(:stubbed_unpause_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, pause: 0 }).to_return(single_crawl_response_body) }
|
47
|
-
it "should make the request to pause it" do
|
48
|
-
pause
|
49
|
-
expect(stubbed_pause_request).to have_been_requested
|
50
|
-
end
|
51
|
-
it "should make the request to unpause it" do
|
52
|
-
unpause
|
53
|
-
expect(stubbed_unpause_request).to have_been_requested
|
54
|
-
end
|
55
|
-
end
|
56
|
-
context "when restarting a named crawl" do
|
57
|
-
let(:restart) { stubbed_request; subject.restart name: name }
|
58
|
-
let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, restart: 1 }).to_return(single_crawl_response_body) }
|
59
|
-
it "should make the request to restart it" do
|
60
|
-
restart
|
61
|
-
expect(stubbed_request).to have_been_requested
|
62
|
-
end
|
63
|
-
end
|
64
|
-
context "when requesting a crawls result" do
|
65
|
-
let(:result) { stubbed_crawl_request;stubbed_result_request; subject.result name: name }
|
66
|
-
let(:test_download_url) { "http://google.com" }
|
67
|
-
let(:stubbed_crawl_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token}).to_return(body: "{\"jobs\":[{\"downloadJson\":\"#{test_download_url}\"}]}") }
|
68
|
-
let(:stubbed_result_request) { stub_request(:get, test_download_url).to_return(body: "[{'f':'b'}]") }
|
69
|
-
it "should make the two requests to get the results" do
|
70
|
-
result
|
71
|
-
expect(stubbed_crawl_request).to have_been_requested
|
72
|
-
expect(stubbed_result_request).to have_been_requested
|
73
|
-
end
|
74
|
-
end
|
75
|
-
describe "if diffbots response is an error" do
|
76
|
-
let(:error_from_diffbot) { { error: "Your token has exceeded the allowed number of calls, or has otherwise been throttled for API abuse.", errorCode: 429 }.to_json }
|
77
|
-
let(:stubbed_request) { stub_request(:get, /#{base_url}\/crawl*/).to_return(body: error_from_diffbot) }
|
78
|
-
shared_examples_for "an error" do
|
79
|
-
it "and raise an DiffbotError" do
|
80
|
-
expect{raiser}.to raise_error DiffbotError
|
81
|
-
end
|
82
|
-
end
|
83
|
-
context "on all" do
|
84
|
-
let(:raiser) { stubbed_request;subject.all; }
|
85
|
-
it_should_behave_like "an error"
|
86
|
-
end
|
87
|
-
context "on single_crawl" do
|
88
|
-
let(:raiser) { stubbed_request;subject.single_crawl name: name; }
|
89
|
-
it_should_behave_like "an error"
|
90
|
-
end
|
91
|
-
context "on delete" do
|
92
|
-
let(:raiser) { stubbed_request;subject.delete name: name; }
|
93
|
-
it_should_behave_like "an error"
|
94
|
-
end
|
95
|
-
context "on pause" do
|
96
|
-
let(:raiser) { stubbed_request;subject.pause name: name; }
|
97
|
-
it_should_behave_like "an error"
|
98
|
-
end
|
99
|
-
context "on unpause" do
|
100
|
-
let(:raiser) { stubbed_request;subject.unpause name: name; }
|
101
|
-
it_should_behave_like "an error"
|
102
|
-
end
|
103
|
-
context "on restart" do
|
104
|
-
let(:raiser) { stubbed_request;subject.restart name: name; }
|
105
|
-
it_should_behave_like "an error"
|
106
|
-
end
|
107
|
-
context "on result" do
|
108
|
-
let(:raiser) { stubbed_request;subject.result name: name; }
|
109
|
-
it_should_behave_like "an error"
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|