wayfarer 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yaml +1 -1
- data/Gemfile.lock +20 -15
- data/docs/cookbook/user_agent.md +1 -1
- data/docs/guides/browser_automation/capybara.md +64 -1
- data/docs/guides/browser_automation/custom_adapters.md +100 -0
- data/docs/guides/browser_automation/ferrum.md +3 -3
- data/docs/guides/browser_automation/selenium.md +7 -5
- data/docs/guides/callbacks.md +117 -10
- data/docs/guides/configuration.md +16 -10
- data/docs/guides/error_handling.md +9 -5
- data/docs/guides/networking.md +77 -3
- data/docs/index.md +9 -1
- data/docs/reference/api/base.md +4 -4
- data/docs/reference/configuration_keys.md +42 -0
- data/docs/reference/environment_variables.md +25 -27
- data/lib/wayfarer/base.rb +7 -17
- data/lib/wayfarer/callbacks.rb +71 -0
- data/lib/wayfarer/cli/base.rb +5 -1
- data/lib/wayfarer/cli/job.rb +7 -3
- data/lib/wayfarer/cli/route.rb +2 -2
- data/lib/wayfarer/cli/route_printer.rb +7 -7
- data/lib/wayfarer/config/capybara.rb +10 -0
- data/lib/wayfarer/config/ferrum.rb +11 -0
- data/lib/wayfarer/config/networking.rb +26 -0
- data/lib/wayfarer/config/redis.rb +14 -0
- data/lib/wayfarer/config/root.rb +11 -0
- data/lib/wayfarer/config/selenium.rb +21 -0
- data/lib/wayfarer/config/strconv.rb +45 -0
- data/lib/wayfarer/config/struct.rb +72 -0
- data/lib/wayfarer/gc.rb +3 -7
- data/lib/wayfarer/middleware/fetch.rb +7 -3
- data/lib/wayfarer/middleware/router.rb +2 -2
- data/lib/wayfarer/middleware/worker.rb +12 -9
- data/lib/wayfarer/networking/capybara.rb +28 -0
- data/lib/wayfarer/networking/context.rb +36 -0
- data/lib/wayfarer/networking/ferrum.rb +17 -52
- data/lib/wayfarer/networking/http.rb +34 -0
- data/lib/wayfarer/networking/pool.rb +15 -10
- data/lib/wayfarer/networking/result.rb +1 -1
- data/lib/wayfarer/networking/selenium.rb +20 -47
- data/lib/wayfarer/networking/strategy.rb +38 -0
- data/lib/wayfarer/page.rb +2 -3
- data/lib/wayfarer/redis/pool.rb +3 -1
- data/lib/wayfarer/routing/dsl.rb +8 -8
- data/lib/wayfarer/routing/matchers/custom.rb +23 -0
- data/lib/wayfarer/routing/matchers/host.rb +19 -0
- data/lib/wayfarer/routing/matchers/path.rb +48 -0
- data/lib/wayfarer/routing/matchers/query.rb +63 -0
- data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
- data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
- data/lib/wayfarer/routing/matchers/url.rb +17 -0
- data/lib/wayfarer/routing/route.rb +1 -1
- data/lib/wayfarer.rb +9 -9
- data/spec/base_spec.rb +14 -0
- data/spec/callbacks_spec.rb +102 -0
- data/spec/cli/job_spec.rb +6 -6
- data/spec/config/capybara_spec.rb +18 -0
- data/spec/config/ferrum_spec.rb +24 -0
- data/spec/config/networking_spec.rb +73 -0
- data/spec/config/redis_spec.rb +32 -0
- data/spec/config/root_spec.rb +31 -0
- data/spec/config/selenium_spec.rb +56 -0
- data/spec/config/strconv_spec.rb +58 -0
- data/spec/config/struct_spec.rb +66 -0
- data/spec/gc_spec.rb +8 -6
- data/spec/middleware/fetch_spec.rb +20 -8
- data/spec/middleware/router_spec.rb +7 -0
- data/spec/middleware/worker_spec.rb +64 -27
- data/spec/networking/capybara_spec.rb +12 -0
- data/spec/networking/context_spec.rb +127 -0
- data/spec/networking/ferrum_spec.rb +6 -22
- data/spec/networking/http_spec.rb +12 -0
- data/spec/networking/pool_spec.rb +37 -12
- data/spec/networking/selenium_spec.rb +6 -22
- data/spec/networking/strategy.rb +170 -0
- data/spec/redis/pool_spec.rb +1 -1
- data/spec/routing/dsl_spec.rb +10 -10
- data/spec/routing/integration_spec.rb +22 -22
- data/spec/routing/{custom_matcher_spec.rb → matchers/custom_spec.rb} +4 -4
- data/spec/routing/{host_matcher_spec.rb → matchers/host_spec.rb} +6 -6
- data/spec/routing/{path_matcher_spec.rb → matchers/path_spec.rb} +6 -6
- data/spec/routing/{query_matcher_spec.rb → matchers/query_spec.rb} +15 -15
- data/spec/routing/{scheme_matcher_spec.rb → matchers/scheme_spec.rb} +4 -4
- data/spec/routing/{suffix_matcher_spec.rb → matchers/suffix_spec.rb} +4 -4
- data/spec/routing/{uri_matcher_spec.rb → matchers/uri_spec.rb} +4 -4
- data/spec/routing/path_finder_spec.rb +1 -1
- data/spec/routing/root_route_spec.rb +2 -2
- data/spec/routing/route_spec.rb +2 -2
- data/spec/spec_helpers.rb +13 -5
- data/spec/wayfarer_spec.rb +1 -1
- data/wayfarer.gemspec +8 -7
- metadata +74 -33
- data/lib/wayfarer/config.rb +0 -67
- data/lib/wayfarer/networking/healer.rb +0 -21
- data/lib/wayfarer/networking/net_http.rb +0 -52
- data/lib/wayfarer/routing/custom_matcher.rb +0 -21
- data/lib/wayfarer/routing/host_matcher.rb +0 -23
- data/lib/wayfarer/routing/path_matcher.rb +0 -46
- data/lib/wayfarer/routing/query_matcher.rb +0 -67
- data/lib/wayfarer/routing/scheme_matcher.rb +0 -21
- data/lib/wayfarer/routing/suffix_matcher.rb +0 -21
- data/lib/wayfarer/routing/url_matcher.rb +0 -21
- data/spec/config_spec.rb +0 -144
- data/spec/networking/adapter.rb +0 -135
- data/spec/networking/healer_spec.rb +0 -46
- data/spec/networking/net_http_spec.rb +0 -37
data/spec/config_spec.rb
DELETED
@@ -1,144 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "spec_helpers"
|
4
|
-
|
5
|
-
describe Wayfarer::Config do
|
6
|
-
describe "::defaults" do
|
7
|
-
it "returns the default config" do
|
8
|
-
config = Wayfarer::Config.default
|
9
|
-
expect(config.adapter).to be(:net_http)
|
10
|
-
expect(config.adapter_pool_size).to be(3)
|
11
|
-
expect(config.adapter_pool_timeout).to eq(ConnectionPool::DEFAULTS[:timeout])
|
12
|
-
expect(config.ferrum_options).to eq({})
|
13
|
-
expect(config.selenium_argv).to eq(%i[chrome])
|
14
|
-
expect(config.redis_url).to eq("redis://localhost:6379")
|
15
|
-
expect(config.redis_factory).not_to be(nil)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
describe "::from_environment" do
|
20
|
-
let(:env) { {} }
|
21
|
-
subject(:config) { Wayfarer::Config.from_environment(env) }
|
22
|
-
|
23
|
-
describe "#adapter" do
|
24
|
-
context "by default" do
|
25
|
-
it "returns :net_http" do
|
26
|
-
expect(config.adapter).to be(:net_http)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
context "with env var set" do
|
31
|
-
before { env["WAYFARER_ADAPTER"] = "ferrum" }
|
32
|
-
|
33
|
-
it "symbolizes the env var" do
|
34
|
-
expect(config.adapter).to be(:ferrum)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
describe "#adapter_pool_size" do
|
40
|
-
context "by default" do
|
41
|
-
it "returns the default config value" do
|
42
|
-
expect(config.adapter_pool_size).to be(3)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
context "with env var set" do
|
47
|
-
before { env["WAYFARER_POOL_SIZE"] = "1234" }
|
48
|
-
|
49
|
-
it "parses the env var to a number" do
|
50
|
-
expect(config.adapter_pool_size).to be(1234)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe "#adapter_pool_timeout" do
|
56
|
-
context "by default" do
|
57
|
-
it "returns the default config value" do
|
58
|
-
expect(config.adapter_pool_timeout).to be(ConnectionPool::DEFAULTS[:timeout])
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
context "with env var set" do
|
63
|
-
let(:env) { { "WAYFARER_POOL_TIMEOUT" => "1000" } }
|
64
|
-
|
65
|
-
it "parses the env var to a number" do
|
66
|
-
expect(config.adapter_pool_timeout).to be(1000)
|
67
|
-
end
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
describe "#ferrum_options" do
|
72
|
-
context "by default" do
|
73
|
-
it "is empty" do
|
74
|
-
expect(config.ferrum_options).to eq({})
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
context "with env var set" do
|
79
|
-
before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
|
80
|
-
|
81
|
-
it "parses the env var to a Hash" do
|
82
|
-
expect(config.ferrum_options).to eq(url: "http://chrome:3000", headless: false)
|
83
|
-
end
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
|
-
describe "#selenium_argv" do
|
88
|
-
context "by default" do
|
89
|
-
it "is [:chrome]" do
|
90
|
-
expect(config.selenium_argv).to eq(%i[chrome])
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
context "with env var set" do
|
95
|
-
before { env["WAYFARER_SELENIUM_ARGV"] = "firefox" }
|
96
|
-
|
97
|
-
it "parses the env var to an array" do
|
98
|
-
expect(config.selenium_argv).to eq(%w[firefox])
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
102
|
-
|
103
|
-
describe "#redis_url" do
|
104
|
-
context "by default" do
|
105
|
-
it "is redis://localhost:6379" do
|
106
|
-
expect(config.redis_url).to eq("redis://localhost:6379")
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
|
-
context "with env var set" do
|
111
|
-
before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
|
112
|
-
|
113
|
-
it "reads the env var" do
|
114
|
-
expect(config.redis_url).to eq("redis://redis:6379")
|
115
|
-
end
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
describe "#http_headers" do
|
120
|
-
context "by default" do
|
121
|
-
it "is {}" do
|
122
|
-
expect(config.http_headers).to eq({})
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
context "with env var set" do
|
127
|
-
before { env["WAYFARER_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
|
128
|
-
|
129
|
-
it "reads the env var" do
|
130
|
-
expect(config.http_headers).to eq("user-agent": "foo",
|
131
|
-
authorization: "bar")
|
132
|
-
end
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
136
|
-
|
137
|
-
describe "#redis_factory" do
|
138
|
-
let(:config) { Wayfarer::Config.default }
|
139
|
-
|
140
|
-
it "instantiates Redis clients" do
|
141
|
-
expect(config.redis_factory.call).to be_a(Redis)
|
142
|
-
end
|
143
|
-
end
|
144
|
-
end
|
data/spec/networking/adapter.rb
DELETED
@@ -1,135 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "spec_helpers"
|
4
|
-
|
5
|
-
RSpec.shared_examples "Network adapter" do |options|
|
6
|
-
describe "::renew_on" do
|
7
|
-
it "returns an Array" do
|
8
|
-
expect(adapter.class.renew_on).to be_an(Array)
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
describe "#fetch" do
|
13
|
-
it "returns Success" do
|
14
|
-
url = test_app_path("/hello_world")
|
15
|
-
result = adapter.fetch(url)
|
16
|
-
expect(result).to be_a(Wayfarer::Networking::Result::Success)
|
17
|
-
expect(result.page).to be_a(Wayfarer::Page)
|
18
|
-
end
|
19
|
-
|
20
|
-
it "sets the URL" do
|
21
|
-
url = test_app_path("/status_code/404")
|
22
|
-
page = adapter.fetch(url).page
|
23
|
-
expect(page.url).to eq test_app_path("/status_code/404")
|
24
|
-
end
|
25
|
-
|
26
|
-
it "sets the response body" do
|
27
|
-
url = test_app_path("/hello_world")
|
28
|
-
page = adapter.fetch(url).page
|
29
|
-
expect(page.body).to match(/Hello world!/)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
describe "#live" do
|
34
|
-
context "when automating a browser" do
|
35
|
-
before { skip unless options[:browser] }
|
36
|
-
|
37
|
-
it "returns a live page" do
|
38
|
-
adapter.fetch(test_app_path("/hello_world"))
|
39
|
-
|
40
|
-
url = test_app_path("/body/foobar")
|
41
|
-
case adapter
|
42
|
-
when Wayfarer::Networking::Selenium then adapter.browser.navigate.to(url)
|
43
|
-
when Wayfarer::Networking::Ferrum then adapter.browser.goto(url)
|
44
|
-
end
|
45
|
-
|
46
|
-
page = adapter.live(nil)
|
47
|
-
expect(page.url).to eq(url)
|
48
|
-
expect(page.body).to match(/foobar/)
|
49
|
-
# Response headers and status code cannot be tested here, because
|
50
|
-
# Selenium does not support them.
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
context "when using plain HTTP" do
|
55
|
-
before { skip if options[:browser] }
|
56
|
-
|
57
|
-
it "returns the same page" do
|
58
|
-
page = build(:page)
|
59
|
-
expect(adapter.live(page)).to be(page)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
describe "#body" do
|
65
|
-
it "returns the body" do
|
66
|
-
url = test_app_path("/hello_world")
|
67
|
-
adapter.fetch(url).page
|
68
|
-
expect(adapter.body).to match(/Hello world!/)
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
describe "#free" do
|
73
|
-
it "frees the browser" do
|
74
|
-
skip unless options[:browser]
|
75
|
-
|
76
|
-
expect {
|
77
|
-
adapter.free
|
78
|
-
}.to change { adapter.browser }.to(nil)
|
79
|
-
end
|
80
|
-
|
81
|
-
it "frees the Capybara session" do
|
82
|
-
skip unless options[:capybara]
|
83
|
-
|
84
|
-
adapter.capybara
|
85
|
-
|
86
|
-
expect {
|
87
|
-
adapter.free
|
88
|
-
}.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
describe "#renew" do
|
93
|
-
it "does not raise" do
|
94
|
-
expect { adapter.renew }.not_to raise_error
|
95
|
-
end
|
96
|
-
|
97
|
-
it "renews the browser" do
|
98
|
-
skip unless options[:browser]
|
99
|
-
|
100
|
-
expect {
|
101
|
-
adapter.renew
|
102
|
-
}.to(change { adapter.browser })
|
103
|
-
end
|
104
|
-
|
105
|
-
it "frees the Capybara session" do
|
106
|
-
skip unless options[:capybara]
|
107
|
-
|
108
|
-
adapter.capybara
|
109
|
-
|
110
|
-
expect {
|
111
|
-
adapter.renew
|
112
|
-
}.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
|
113
|
-
end
|
114
|
-
end
|
115
|
-
|
116
|
-
describe "#capybara" do
|
117
|
-
before { skip unless options[:capybara] }
|
118
|
-
|
119
|
-
it "returns a Capybara driver" do
|
120
|
-
expect(adapter.capybara).to be_a(Capybara::Session)
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
|
-
describe "HTTP headers" do
|
125
|
-
before { Wayfarer.config.http_headers = { "User-Agent" => "Foobar" } }
|
126
|
-
|
127
|
-
it "uses configured HTTP headers" do
|
128
|
-
skip unless options[:request_headers]
|
129
|
-
|
130
|
-
url = test_app_path("/headers/HTTP_USER_AGENT")
|
131
|
-
page = adapter.fetch(url).page
|
132
|
-
expect(page.body).to match(/Foobar/)
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
@@ -1,46 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "spec_helpers"
|
4
|
-
|
5
|
-
describe Wayfarer::Networking::Healer do
|
6
|
-
let(:error) { Class.new(StandardError) }
|
7
|
-
let(:renew_on) { [error] }
|
8
|
-
|
9
|
-
let(:adapter) do
|
10
|
-
double.tap do |adapter|
|
11
|
-
allow(adapter.class).to receive(:renew_on).and_return([error])
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
subject(:healer) do
|
16
|
-
Wayfarer::Networking::Healer.new(adapter)
|
17
|
-
end
|
18
|
-
|
19
|
-
describe "#fetch" do
|
20
|
-
context "when proxied call raises renewing error" do
|
21
|
-
before do
|
22
|
-
allow(adapter).to receive(:fetch).and_raise(error)
|
23
|
-
end
|
24
|
-
|
25
|
-
it "renews its adapter and re-raises" do
|
26
|
-
expect(adapter).to receive(:renew)
|
27
|
-
expect {
|
28
|
-
healer.fetch(URI("https://example.com"))
|
29
|
-
}.to raise_error(error)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
context "when proxied call raises non-renewing error" do
|
34
|
-
before do
|
35
|
-
allow(adapter).to receive(:fetch).and_raise(StandardError.new)
|
36
|
-
end
|
37
|
-
|
38
|
-
it "does not renew its adapter and intercept" do
|
39
|
-
expect(adapter).not_to receive(:renew)
|
40
|
-
expect {
|
41
|
-
healer.fetch(URI("https://example.com"))
|
42
|
-
}.to raise_error(StandardError)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
@@ -1,37 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "spec_helpers"
|
4
|
-
require_relative "adapter"
|
5
|
-
|
6
|
-
describe Wayfarer::Networking::NetHTTP do
|
7
|
-
subject(:adapter) { Wayfarer::Networking::NetHTTP.send(:new) }
|
8
|
-
|
9
|
-
after { adapter.free }
|
10
|
-
|
11
|
-
include_examples "Network adapter", browser: false,
|
12
|
-
capybara: false,
|
13
|
-
request_headers: true
|
14
|
-
|
15
|
-
describe "#fetch" do
|
16
|
-
it "sets the status code" do
|
17
|
-
url = test_app_path("/status_code/404")
|
18
|
-
page = adapter.fetch(url).page
|
19
|
-
expect(page.status_code).to be 404
|
20
|
-
end
|
21
|
-
|
22
|
-
it "sets response headers" do
|
23
|
-
url = test_app_path("/hello_world")
|
24
|
-
page = adapter.fetch(url).page
|
25
|
-
expect(page.headers["hello"]).to eq ["world"]
|
26
|
-
end
|
27
|
-
|
28
|
-
context "when encountering a redirect" do
|
29
|
-
it "returns a Redirect" do
|
30
|
-
url = test_app_path("/redirect?times=3")
|
31
|
-
result = adapter.fetch(url)
|
32
|
-
expect(result).to be_a Wayfarer::Networking::Result::Redirect
|
33
|
-
expect(result.redirect_url).to eq test_app_path("/redirect?times=2")
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|