wayfarer 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yaml +1 -1
- data/Gemfile.lock +20 -15
- data/docs/cookbook/user_agent.md +1 -1
- data/docs/guides/browser_automation/capybara.md +64 -1
- data/docs/guides/browser_automation/custom_adapters.md +100 -0
- data/docs/guides/browser_automation/ferrum.md +3 -3
- data/docs/guides/browser_automation/selenium.md +7 -5
- data/docs/guides/callbacks.md +117 -10
- data/docs/guides/configuration.md +16 -10
- data/docs/guides/error_handling.md +9 -5
- data/docs/guides/networking.md +77 -3
- data/docs/index.md +9 -1
- data/docs/reference/api/base.md +4 -4
- data/docs/reference/configuration_keys.md +42 -0
- data/docs/reference/environment_variables.md +25 -27
- data/lib/wayfarer/base.rb +7 -17
- data/lib/wayfarer/callbacks.rb +71 -0
- data/lib/wayfarer/cli/base.rb +5 -1
- data/lib/wayfarer/cli/job.rb +7 -3
- data/lib/wayfarer/cli/route.rb +2 -2
- data/lib/wayfarer/cli/route_printer.rb +7 -7
- data/lib/wayfarer/config/capybara.rb +10 -0
- data/lib/wayfarer/config/ferrum.rb +11 -0
- data/lib/wayfarer/config/networking.rb +26 -0
- data/lib/wayfarer/config/redis.rb +14 -0
- data/lib/wayfarer/config/root.rb +11 -0
- data/lib/wayfarer/config/selenium.rb +21 -0
- data/lib/wayfarer/config/strconv.rb +45 -0
- data/lib/wayfarer/config/struct.rb +72 -0
- data/lib/wayfarer/gc.rb +3 -7
- data/lib/wayfarer/middleware/fetch.rb +7 -3
- data/lib/wayfarer/middleware/router.rb +2 -2
- data/lib/wayfarer/middleware/worker.rb +12 -9
- data/lib/wayfarer/networking/capybara.rb +28 -0
- data/lib/wayfarer/networking/context.rb +36 -0
- data/lib/wayfarer/networking/ferrum.rb +17 -52
- data/lib/wayfarer/networking/http.rb +34 -0
- data/lib/wayfarer/networking/pool.rb +15 -10
- data/lib/wayfarer/networking/result.rb +1 -1
- data/lib/wayfarer/networking/selenium.rb +20 -47
- data/lib/wayfarer/networking/strategy.rb +38 -0
- data/lib/wayfarer/page.rb +2 -3
- data/lib/wayfarer/redis/pool.rb +3 -1
- data/lib/wayfarer/routing/dsl.rb +8 -8
- data/lib/wayfarer/routing/matchers/custom.rb +23 -0
- data/lib/wayfarer/routing/matchers/host.rb +19 -0
- data/lib/wayfarer/routing/matchers/path.rb +48 -0
- data/lib/wayfarer/routing/matchers/query.rb +63 -0
- data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
- data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
- data/lib/wayfarer/routing/matchers/url.rb +17 -0
- data/lib/wayfarer/routing/route.rb +1 -1
- data/lib/wayfarer.rb +9 -9
- data/spec/base_spec.rb +14 -0
- data/spec/callbacks_spec.rb +102 -0
- data/spec/cli/job_spec.rb +6 -6
- data/spec/config/capybara_spec.rb +18 -0
- data/spec/config/ferrum_spec.rb +24 -0
- data/spec/config/networking_spec.rb +73 -0
- data/spec/config/redis_spec.rb +32 -0
- data/spec/config/root_spec.rb +31 -0
- data/spec/config/selenium_spec.rb +56 -0
- data/spec/config/strconv_spec.rb +58 -0
- data/spec/config/struct_spec.rb +66 -0
- data/spec/gc_spec.rb +8 -6
- data/spec/middleware/fetch_spec.rb +20 -8
- data/spec/middleware/router_spec.rb +7 -0
- data/spec/middleware/worker_spec.rb +64 -27
- data/spec/networking/capybara_spec.rb +12 -0
- data/spec/networking/context_spec.rb +127 -0
- data/spec/networking/ferrum_spec.rb +6 -22
- data/spec/networking/http_spec.rb +12 -0
- data/spec/networking/pool_spec.rb +37 -12
- data/spec/networking/selenium_spec.rb +6 -22
- data/spec/networking/strategy.rb +170 -0
- data/spec/redis/pool_spec.rb +1 -1
- data/spec/routing/dsl_spec.rb +10 -10
- data/spec/routing/integration_spec.rb +22 -22
- data/spec/routing/{custom_matcher_spec.rb → matchers/custom_spec.rb} +4 -4
- data/spec/routing/{host_matcher_spec.rb → matchers/host_spec.rb} +6 -6
- data/spec/routing/{path_matcher_spec.rb → matchers/path_spec.rb} +6 -6
- data/spec/routing/{query_matcher_spec.rb → matchers/query_spec.rb} +15 -15
- data/spec/routing/{scheme_matcher_spec.rb → matchers/scheme_spec.rb} +4 -4
- data/spec/routing/{suffix_matcher_spec.rb → matchers/suffix_spec.rb} +4 -4
- data/spec/routing/{uri_matcher_spec.rb → matchers/uri_spec.rb} +4 -4
- data/spec/routing/path_finder_spec.rb +1 -1
- data/spec/routing/root_route_spec.rb +2 -2
- data/spec/routing/route_spec.rb +2 -2
- data/spec/spec_helpers.rb +13 -5
- data/spec/wayfarer_spec.rb +1 -1
- data/wayfarer.gemspec +8 -7
- metadata +74 -33
- data/lib/wayfarer/config.rb +0 -67
- data/lib/wayfarer/networking/healer.rb +0 -21
- data/lib/wayfarer/networking/net_http.rb +0 -52
- data/lib/wayfarer/routing/custom_matcher.rb +0 -21
- data/lib/wayfarer/routing/host_matcher.rb +0 -23
- data/lib/wayfarer/routing/path_matcher.rb +0 -46
- data/lib/wayfarer/routing/query_matcher.rb +0 -67
- data/lib/wayfarer/routing/scheme_matcher.rb +0 -21
- data/lib/wayfarer/routing/suffix_matcher.rb +0 -21
- data/lib/wayfarer/routing/url_matcher.rb +0 -21
- data/spec/config_spec.rb +0 -144
- data/spec/networking/adapter.rb +0 -135
- data/spec/networking/healer_spec.rb +0 -46
- data/spec/networking/net_http_spec.rb +0 -37
data/spec/config_spec.rb
DELETED
@@ -1,144 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "spec_helpers"
|
4
|
-
|
5
|
-
describe Wayfarer::Config do
|
6
|
-
describe "::defaults" do
|
7
|
-
it "returns the default config" do
|
8
|
-
config = Wayfarer::Config.default
|
9
|
-
expect(config.adapter).to be(:net_http)
|
10
|
-
expect(config.adapter_pool_size).to be(3)
|
11
|
-
expect(config.adapter_pool_timeout).to eq(ConnectionPool::DEFAULTS[:timeout])
|
12
|
-
expect(config.ferrum_options).to eq({})
|
13
|
-
expect(config.selenium_argv).to eq(%i[chrome])
|
14
|
-
expect(config.redis_url).to eq("redis://localhost:6379")
|
15
|
-
expect(config.redis_factory).not_to be(nil)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
describe "::from_environment" do
|
20
|
-
let(:env) { {} }
|
21
|
-
subject(:config) { Wayfarer::Config.from_environment(env) }
|
22
|
-
|
23
|
-
describe "#adapter" do
|
24
|
-
context "by default" do
|
25
|
-
it "returns :net_http" do
|
26
|
-
expect(config.adapter).to be(:net_http)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
context "with env var set" do
|
31
|
-
before { env["WAYFARER_ADAPTER"] = "ferrum" }
|
32
|
-
|
33
|
-
it "symbolizes the env var" do
|
34
|
-
expect(config.adapter).to be(:ferrum)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
describe "#adapter_pool_size" do
|
40
|
-
context "by default" do
|
41
|
-
it "returns the default config value" do
|
42
|
-
expect(config.adapter_pool_size).to be(3)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
context "with env var set" do
|
47
|
-
before { env["WAYFARER_POOL_SIZE"] = "1234" }
|
48
|
-
|
49
|
-
it "parses the env var to a number" do
|
50
|
-
expect(config.adapter_pool_size).to be(1234)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe "#adapter_pool_timeout" do
|
56
|
-
context "by default" do
|
57
|
-
it "returns the default config value" do
|
58
|
-
expect(config.adapter_pool_timeout).to be(ConnectionPool::DEFAULTS[:timeout])
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
context "with env var set" do
|
63
|
-
let(:env) { { "WAYFARER_POOL_TIMEOUT" => "1000" } }
|
64
|
-
|
65
|
-
it "parses the env var to a number" do
|
66
|
-
expect(config.adapter_pool_timeout).to be(1000)
|
67
|
-
end
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
describe "#ferrum_options" do
|
72
|
-
context "by default" do
|
73
|
-
it "is empty" do
|
74
|
-
expect(config.ferrum_options).to eq({})
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
context "with env var set" do
|
79
|
-
before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
|
80
|
-
|
81
|
-
it "parses the env var to a Hash" do
|
82
|
-
expect(config.ferrum_options).to eq(url: "http://chrome:3000", headless: false)
|
83
|
-
end
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
|
-
describe "#selenium_argv" do
|
88
|
-
context "by default" do
|
89
|
-
it "is [:chrome]" do
|
90
|
-
expect(config.selenium_argv).to eq(%i[chrome])
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
context "with env var set" do
|
95
|
-
before { env["WAYFARER_SELENIUM_ARGV"] = "firefox" }
|
96
|
-
|
97
|
-
it "parses the env var to an array" do
|
98
|
-
expect(config.selenium_argv).to eq(%w[firefox])
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
102
|
-
|
103
|
-
describe "#redis_url" do
|
104
|
-
context "by default" do
|
105
|
-
it "is redis://localhost:6379" do
|
106
|
-
expect(config.redis_url).to eq("redis://localhost:6379")
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
|
-
context "with env var set" do
|
111
|
-
before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
|
112
|
-
|
113
|
-
it "reads the env var" do
|
114
|
-
expect(config.redis_url).to eq("redis://redis:6379")
|
115
|
-
end
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
describe "#http_headers" do
|
120
|
-
context "by default" do
|
121
|
-
it "is {}" do
|
122
|
-
expect(config.http_headers).to eq({})
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
context "with env var set" do
|
127
|
-
before { env["WAYFARER_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
|
128
|
-
|
129
|
-
it "reads the env var" do
|
130
|
-
expect(config.http_headers).to eq("user-agent": "foo",
|
131
|
-
authorization: "bar")
|
132
|
-
end
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
136
|
-
|
137
|
-
describe "#redis_factory" do
|
138
|
-
let(:config) { Wayfarer::Config.default }
|
139
|
-
|
140
|
-
it "instantiates Redis clients" do
|
141
|
-
expect(config.redis_factory.call).to be_a(Redis)
|
142
|
-
end
|
143
|
-
end
|
144
|
-
end
|
data/spec/networking/adapter.rb
DELETED
@@ -1,135 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "spec_helpers"
|
4
|
-
|
5
|
-
RSpec.shared_examples "Network adapter" do |options|
|
6
|
-
describe "::renew_on" do
|
7
|
-
it "returns an Array" do
|
8
|
-
expect(adapter.class.renew_on).to be_an(Array)
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
describe "#fetch" do
|
13
|
-
it "returns Success" do
|
14
|
-
url = test_app_path("/hello_world")
|
15
|
-
result = adapter.fetch(url)
|
16
|
-
expect(result).to be_a(Wayfarer::Networking::Result::Success)
|
17
|
-
expect(result.page).to be_a(Wayfarer::Page)
|
18
|
-
end
|
19
|
-
|
20
|
-
it "sets the URL" do
|
21
|
-
url = test_app_path("/status_code/404")
|
22
|
-
page = adapter.fetch(url).page
|
23
|
-
expect(page.url).to eq test_app_path("/status_code/404")
|
24
|
-
end
|
25
|
-
|
26
|
-
it "sets the response body" do
|
27
|
-
url = test_app_path("/hello_world")
|
28
|
-
page = adapter.fetch(url).page
|
29
|
-
expect(page.body).to match(/Hello world!/)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
describe "#live" do
|
34
|
-
context "when automating a browser" do
|
35
|
-
before { skip unless options[:browser] }
|
36
|
-
|
37
|
-
it "returns a live page" do
|
38
|
-
adapter.fetch(test_app_path("/hello_world"))
|
39
|
-
|
40
|
-
url = test_app_path("/body/foobar")
|
41
|
-
case adapter
|
42
|
-
when Wayfarer::Networking::Selenium then adapter.browser.navigate.to(url)
|
43
|
-
when Wayfarer::Networking::Ferrum then adapter.browser.goto(url)
|
44
|
-
end
|
45
|
-
|
46
|
-
page = adapter.live(nil)
|
47
|
-
expect(page.url).to eq(url)
|
48
|
-
expect(page.body).to match(/foobar/)
|
49
|
-
# Response headers and status code cannot be tested here, because
|
50
|
-
# Selenium does not support them.
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
context "when using plain HTTP" do
|
55
|
-
before { skip if options[:browser] }
|
56
|
-
|
57
|
-
it "returns the same page" do
|
58
|
-
page = build(:page)
|
59
|
-
expect(adapter.live(page)).to be(page)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
describe "#body" do
|
65
|
-
it "returns the body" do
|
66
|
-
url = test_app_path("/hello_world")
|
67
|
-
adapter.fetch(url).page
|
68
|
-
expect(adapter.body).to match(/Hello world!/)
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
describe "#free" do
|
73
|
-
it "frees the browser" do
|
74
|
-
skip unless options[:browser]
|
75
|
-
|
76
|
-
expect {
|
77
|
-
adapter.free
|
78
|
-
}.to change { adapter.browser }.to(nil)
|
79
|
-
end
|
80
|
-
|
81
|
-
it "frees the Capybara session" do
|
82
|
-
skip unless options[:capybara]
|
83
|
-
|
84
|
-
adapter.capybara
|
85
|
-
|
86
|
-
expect {
|
87
|
-
adapter.free
|
88
|
-
}.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
describe "#renew" do
|
93
|
-
it "does not raise" do
|
94
|
-
expect { adapter.renew }.not_to raise_error
|
95
|
-
end
|
96
|
-
|
97
|
-
it "renews the browser" do
|
98
|
-
skip unless options[:browser]
|
99
|
-
|
100
|
-
expect {
|
101
|
-
adapter.renew
|
102
|
-
}.to(change { adapter.browser })
|
103
|
-
end
|
104
|
-
|
105
|
-
it "frees the Capybara session" do
|
106
|
-
skip unless options[:capybara]
|
107
|
-
|
108
|
-
adapter.capybara
|
109
|
-
|
110
|
-
expect {
|
111
|
-
adapter.renew
|
112
|
-
}.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
|
113
|
-
end
|
114
|
-
end
|
115
|
-
|
116
|
-
describe "#capybara" do
|
117
|
-
before { skip unless options[:capybara] }
|
118
|
-
|
119
|
-
it "returns a Capybara driver" do
|
120
|
-
expect(adapter.capybara).to be_a(Capybara::Session)
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
|
-
describe "HTTP headers" do
|
125
|
-
before { Wayfarer.config.http_headers = { "User-Agent" => "Foobar" } }
|
126
|
-
|
127
|
-
it "uses configured HTTP headers" do
|
128
|
-
skip unless options[:request_headers]
|
129
|
-
|
130
|
-
url = test_app_path("/headers/HTTP_USER_AGENT")
|
131
|
-
page = adapter.fetch(url).page
|
132
|
-
expect(page.body).to match(/Foobar/)
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
@@ -1,46 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "spec_helpers"
|
4
|
-
|
5
|
-
describe Wayfarer::Networking::Healer do
|
6
|
-
let(:error) { Class.new(StandardError) }
|
7
|
-
let(:renew_on) { [error] }
|
8
|
-
|
9
|
-
let(:adapter) do
|
10
|
-
double.tap do |adapter|
|
11
|
-
allow(adapter.class).to receive(:renew_on).and_return([error])
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
subject(:healer) do
|
16
|
-
Wayfarer::Networking::Healer.new(adapter)
|
17
|
-
end
|
18
|
-
|
19
|
-
describe "#fetch" do
|
20
|
-
context "when proxied call raises renewing error" do
|
21
|
-
before do
|
22
|
-
allow(adapter).to receive(:fetch).and_raise(error)
|
23
|
-
end
|
24
|
-
|
25
|
-
it "renews its adapter and re-raises" do
|
26
|
-
expect(adapter).to receive(:renew)
|
27
|
-
expect {
|
28
|
-
healer.fetch(URI("https://example.com"))
|
29
|
-
}.to raise_error(error)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
context "when proxied call raises non-renewing error" do
|
34
|
-
before do
|
35
|
-
allow(adapter).to receive(:fetch).and_raise(StandardError.new)
|
36
|
-
end
|
37
|
-
|
38
|
-
it "does not renew its adapter and intercept" do
|
39
|
-
expect(adapter).not_to receive(:renew)
|
40
|
-
expect {
|
41
|
-
healer.fetch(URI("https://example.com"))
|
42
|
-
}.to raise_error(StandardError)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
@@ -1,37 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "spec_helpers"
|
4
|
-
require_relative "adapter"
|
5
|
-
|
6
|
-
describe Wayfarer::Networking::NetHTTP do
|
7
|
-
subject(:adapter) { Wayfarer::Networking::NetHTTP.send(:new) }
|
8
|
-
|
9
|
-
after { adapter.free }
|
10
|
-
|
11
|
-
include_examples "Network adapter", browser: false,
|
12
|
-
capybara: false,
|
13
|
-
request_headers: true
|
14
|
-
|
15
|
-
describe "#fetch" do
|
16
|
-
it "sets the status code" do
|
17
|
-
url = test_app_path("/status_code/404")
|
18
|
-
page = adapter.fetch(url).page
|
19
|
-
expect(page.status_code).to be 404
|
20
|
-
end
|
21
|
-
|
22
|
-
it "sets response headers" do
|
23
|
-
url = test_app_path("/hello_world")
|
24
|
-
page = adapter.fetch(url).page
|
25
|
-
expect(page.headers["hello"]).to eq ["world"]
|
26
|
-
end
|
27
|
-
|
28
|
-
context "when encountering a redirect" do
|
29
|
-
it "returns a Redirect" do
|
30
|
-
url = test_app_path("/redirect?times=3")
|
31
|
-
result = adapter.fetch(url)
|
32
|
-
expect(result).to be_a Wayfarer::Networking::Result::Redirect
|
33
|
-
expect(result.redirect_url).to eq test_app_path("/redirect?times=2")
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|