wayfarer 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yaml +1 -1
  3. data/Gemfile.lock +20 -15
  4. data/docs/cookbook/user_agent.md +1 -1
  5. data/docs/guides/browser_automation/capybara.md +64 -1
  6. data/docs/guides/browser_automation/custom_adapters.md +100 -0
  7. data/docs/guides/browser_automation/ferrum.md +3 -3
  8. data/docs/guides/browser_automation/selenium.md +7 -5
  9. data/docs/guides/callbacks.md +117 -10
  10. data/docs/guides/configuration.md +16 -10
  11. data/docs/guides/error_handling.md +9 -5
  12. data/docs/guides/networking.md +77 -3
  13. data/docs/index.md +9 -1
  14. data/docs/reference/api/base.md +4 -4
  15. data/docs/reference/configuration_keys.md +42 -0
  16. data/docs/reference/environment_variables.md +25 -27
  17. data/lib/wayfarer/base.rb +7 -17
  18. data/lib/wayfarer/callbacks.rb +71 -0
  19. data/lib/wayfarer/cli/base.rb +5 -1
  20. data/lib/wayfarer/cli/job.rb +7 -3
  21. data/lib/wayfarer/cli/route.rb +2 -2
  22. data/lib/wayfarer/cli/route_printer.rb +7 -7
  23. data/lib/wayfarer/config/capybara.rb +10 -0
  24. data/lib/wayfarer/config/ferrum.rb +11 -0
  25. data/lib/wayfarer/config/networking.rb +26 -0
  26. data/lib/wayfarer/config/redis.rb +14 -0
  27. data/lib/wayfarer/config/root.rb +11 -0
  28. data/lib/wayfarer/config/selenium.rb +21 -0
  29. data/lib/wayfarer/config/strconv.rb +45 -0
  30. data/lib/wayfarer/config/struct.rb +72 -0
  31. data/lib/wayfarer/gc.rb +3 -7
  32. data/lib/wayfarer/middleware/fetch.rb +7 -3
  33. data/lib/wayfarer/middleware/router.rb +2 -2
  34. data/lib/wayfarer/middleware/worker.rb +12 -9
  35. data/lib/wayfarer/networking/capybara.rb +28 -0
  36. data/lib/wayfarer/networking/context.rb +36 -0
  37. data/lib/wayfarer/networking/ferrum.rb +17 -52
  38. data/lib/wayfarer/networking/http.rb +34 -0
  39. data/lib/wayfarer/networking/pool.rb +15 -10
  40. data/lib/wayfarer/networking/result.rb +1 -1
  41. data/lib/wayfarer/networking/selenium.rb +20 -47
  42. data/lib/wayfarer/networking/strategy.rb +38 -0
  43. data/lib/wayfarer/page.rb +2 -3
  44. data/lib/wayfarer/redis/pool.rb +3 -1
  45. data/lib/wayfarer/routing/dsl.rb +8 -8
  46. data/lib/wayfarer/routing/matchers/custom.rb +23 -0
  47. data/lib/wayfarer/routing/matchers/host.rb +19 -0
  48. data/lib/wayfarer/routing/matchers/path.rb +48 -0
  49. data/lib/wayfarer/routing/matchers/query.rb +63 -0
  50. data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
  51. data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
  52. data/lib/wayfarer/routing/matchers/url.rb +17 -0
  53. data/lib/wayfarer/routing/route.rb +1 -1
  54. data/lib/wayfarer.rb +9 -9
  55. data/spec/base_spec.rb +14 -0
  56. data/spec/callbacks_spec.rb +102 -0
  57. data/spec/cli/job_spec.rb +6 -6
  58. data/spec/config/capybara_spec.rb +18 -0
  59. data/spec/config/ferrum_spec.rb +24 -0
  60. data/spec/config/networking_spec.rb +73 -0
  61. data/spec/config/redis_spec.rb +32 -0
  62. data/spec/config/root_spec.rb +31 -0
  63. data/spec/config/selenium_spec.rb +56 -0
  64. data/spec/config/strconv_spec.rb +58 -0
  65. data/spec/config/struct_spec.rb +66 -0
  66. data/spec/gc_spec.rb +8 -6
  67. data/spec/middleware/fetch_spec.rb +20 -8
  68. data/spec/middleware/router_spec.rb +7 -0
  69. data/spec/middleware/worker_spec.rb +64 -27
  70. data/spec/networking/capybara_spec.rb +12 -0
  71. data/spec/networking/context_spec.rb +127 -0
  72. data/spec/networking/ferrum_spec.rb +6 -22
  73. data/spec/networking/http_spec.rb +12 -0
  74. data/spec/networking/pool_spec.rb +37 -12
  75. data/spec/networking/selenium_spec.rb +6 -22
  76. data/spec/networking/strategy.rb +170 -0
  77. data/spec/redis/pool_spec.rb +1 -1
  78. data/spec/routing/dsl_spec.rb +10 -10
  79. data/spec/routing/integration_spec.rb +22 -22
  80. data/spec/routing/{custom_matcher_spec.rb → matchers/custom_spec.rb} +4 -4
  81. data/spec/routing/{host_matcher_spec.rb → matchers/host_spec.rb} +6 -6
  82. data/spec/routing/{path_matcher_spec.rb → matchers/path_spec.rb} +6 -6
  83. data/spec/routing/{query_matcher_spec.rb → matchers/query_spec.rb} +15 -15
  84. data/spec/routing/{scheme_matcher_spec.rb → matchers/scheme_spec.rb} +4 -4
  85. data/spec/routing/{suffix_matcher_spec.rb → matchers/suffix_spec.rb} +4 -4
  86. data/spec/routing/{uri_matcher_spec.rb → matchers/uri_spec.rb} +4 -4
  87. data/spec/routing/path_finder_spec.rb +1 -1
  88. data/spec/routing/root_route_spec.rb +2 -2
  89. data/spec/routing/route_spec.rb +2 -2
  90. data/spec/spec_helpers.rb +13 -5
  91. data/spec/wayfarer_spec.rb +1 -1
  92. data/wayfarer.gemspec +8 -7
  93. metadata +74 -33
  94. data/lib/wayfarer/config.rb +0 -67
  95. data/lib/wayfarer/networking/healer.rb +0 -21
  96. data/lib/wayfarer/networking/net_http.rb +0 -52
  97. data/lib/wayfarer/routing/custom_matcher.rb +0 -21
  98. data/lib/wayfarer/routing/host_matcher.rb +0 -23
  99. data/lib/wayfarer/routing/path_matcher.rb +0 -46
  100. data/lib/wayfarer/routing/query_matcher.rb +0 -67
  101. data/lib/wayfarer/routing/scheme_matcher.rb +0 -21
  102. data/lib/wayfarer/routing/suffix_matcher.rb +0 -21
  103. data/lib/wayfarer/routing/url_matcher.rb +0 -21
  104. data/spec/config_spec.rb +0 -144
  105. data/spec/networking/adapter.rb +0 -135
  106. data/spec/networking/healer_spec.rb +0 -46
  107. data/spec/networking/net_http_spec.rb +0 -37
data/spec/config_spec.rb DELETED
@@ -1,144 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "spec_helpers"
4
-
5
- describe Wayfarer::Config do
6
- describe "::defaults" do
7
- it "returns the default config" do
8
- config = Wayfarer::Config.default
9
- expect(config.adapter).to be(:net_http)
10
- expect(config.adapter_pool_size).to be(3)
11
- expect(config.adapter_pool_timeout).to eq(ConnectionPool::DEFAULTS[:timeout])
12
- expect(config.ferrum_options).to eq({})
13
- expect(config.selenium_argv).to eq(%i[chrome])
14
- expect(config.redis_url).to eq("redis://localhost:6379")
15
- expect(config.redis_factory).not_to be(nil)
16
- end
17
- end
18
-
19
- describe "::from_environment" do
20
- let(:env) { {} }
21
- subject(:config) { Wayfarer::Config.from_environment(env) }
22
-
23
- describe "#adapter" do
24
- context "by default" do
25
- it "returns :net_http" do
26
- expect(config.adapter).to be(:net_http)
27
- end
28
- end
29
-
30
- context "with env var set" do
31
- before { env["WAYFARER_ADAPTER"] = "ferrum" }
32
-
33
- it "symbolizes the env var" do
34
- expect(config.adapter).to be(:ferrum)
35
- end
36
- end
37
- end
38
-
39
- describe "#adapter_pool_size" do
40
- context "by default" do
41
- it "returns the default config value" do
42
- expect(config.adapter_pool_size).to be(3)
43
- end
44
- end
45
-
46
- context "with env var set" do
47
- before { env["WAYFARER_POOL_SIZE"] = "1234" }
48
-
49
- it "parses the env var to a number" do
50
- expect(config.adapter_pool_size).to be(1234)
51
- end
52
- end
53
- end
54
-
55
- describe "#adapter_pool_timeout" do
56
- context "by default" do
57
- it "returns the default config value" do
58
- expect(config.adapter_pool_timeout).to be(ConnectionPool::DEFAULTS[:timeout])
59
- end
60
- end
61
-
62
- context "with env var set" do
63
- let(:env) { { "WAYFARER_POOL_TIMEOUT" => "1000" } }
64
-
65
- it "parses the env var to a number" do
66
- expect(config.adapter_pool_timeout).to be(1000)
67
- end
68
- end
69
- end
70
-
71
- describe "#ferrum_options" do
72
- context "by default" do
73
- it "is empty" do
74
- expect(config.ferrum_options).to eq({})
75
- end
76
- end
77
-
78
- context "with env var set" do
79
- before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
80
-
81
- it "parses the env var to a Hash" do
82
- expect(config.ferrum_options).to eq(url: "http://chrome:3000", headless: false)
83
- end
84
- end
85
- end
86
-
87
- describe "#selenium_argv" do
88
- context "by default" do
89
- it "is [:chrome]" do
90
- expect(config.selenium_argv).to eq(%i[chrome])
91
- end
92
- end
93
-
94
- context "with env var set" do
95
- before { env["WAYFARER_SELENIUM_ARGV"] = "firefox" }
96
-
97
- it "parses the env var to an array" do
98
- expect(config.selenium_argv).to eq(%w[firefox])
99
- end
100
- end
101
- end
102
-
103
- describe "#redis_url" do
104
- context "by default" do
105
- it "is redis://localhost:6379" do
106
- expect(config.redis_url).to eq("redis://localhost:6379")
107
- end
108
- end
109
-
110
- context "with env var set" do
111
- before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
112
-
113
- it "reads the env var" do
114
- expect(config.redis_url).to eq("redis://redis:6379")
115
- end
116
- end
117
- end
118
-
119
- describe "#http_headers" do
120
- context "by default" do
121
- it "is {}" do
122
- expect(config.http_headers).to eq({})
123
- end
124
- end
125
-
126
- context "with env var set" do
127
- before { env["WAYFARER_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
128
-
129
- it "reads the env var" do
130
- expect(config.http_headers).to eq("user-agent": "foo",
131
- authorization: "bar")
132
- end
133
- end
134
- end
135
- end
136
-
137
- describe "#redis_factory" do
138
- let(:config) { Wayfarer::Config.default }
139
-
140
- it "instantiates Redis clients" do
141
- expect(config.redis_factory.call).to be_a(Redis)
142
- end
143
- end
144
- end
@@ -1,135 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "spec_helpers"
4
-
5
- RSpec.shared_examples "Network adapter" do |options|
6
- describe "::renew_on" do
7
- it "returns an Array" do
8
- expect(adapter.class.renew_on).to be_an(Array)
9
- end
10
- end
11
-
12
- describe "#fetch" do
13
- it "returns Success" do
14
- url = test_app_path("/hello_world")
15
- result = adapter.fetch(url)
16
- expect(result).to be_a(Wayfarer::Networking::Result::Success)
17
- expect(result.page).to be_a(Wayfarer::Page)
18
- end
19
-
20
- it "sets the URL" do
21
- url = test_app_path("/status_code/404")
22
- page = adapter.fetch(url).page
23
- expect(page.url).to eq test_app_path("/status_code/404")
24
- end
25
-
26
- it "sets the response body" do
27
- url = test_app_path("/hello_world")
28
- page = adapter.fetch(url).page
29
- expect(page.body).to match(/Hello world!/)
30
- end
31
- end
32
-
33
- describe "#live" do
34
- context "when automating a browser" do
35
- before { skip unless options[:browser] }
36
-
37
- it "returns a live page" do
38
- adapter.fetch(test_app_path("/hello_world"))
39
-
40
- url = test_app_path("/body/foobar")
41
- case adapter
42
- when Wayfarer::Networking::Selenium then adapter.browser.navigate.to(url)
43
- when Wayfarer::Networking::Ferrum then adapter.browser.goto(url)
44
- end
45
-
46
- page = adapter.live(nil)
47
- expect(page.url).to eq(url)
48
- expect(page.body).to match(/foobar/)
49
- # Response headers and status code cannot be tested here, because
50
- # Selenium does not support them.
51
- end
52
- end
53
-
54
- context "when using plain HTTP" do
55
- before { skip if options[:browser] }
56
-
57
- it "returns the same page" do
58
- page = build(:page)
59
- expect(adapter.live(page)).to be(page)
60
- end
61
- end
62
- end
63
-
64
- describe "#body" do
65
- it "returns the body" do
66
- url = test_app_path("/hello_world")
67
- adapter.fetch(url).page
68
- expect(adapter.body).to match(/Hello world!/)
69
- end
70
- end
71
-
72
- describe "#free" do
73
- it "frees the browser" do
74
- skip unless options[:browser]
75
-
76
- expect {
77
- adapter.free
78
- }.to change { adapter.browser }.to(nil)
79
- end
80
-
81
- it "frees the Capybara session" do
82
- skip unless options[:capybara]
83
-
84
- adapter.capybara
85
-
86
- expect {
87
- adapter.free
88
- }.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
89
- end
90
- end
91
-
92
- describe "#renew" do
93
- it "does not raise" do
94
- expect { adapter.renew }.not_to raise_error
95
- end
96
-
97
- it "renews the browser" do
98
- skip unless options[:browser]
99
-
100
- expect {
101
- adapter.renew
102
- }.to(change { adapter.browser })
103
- end
104
-
105
- it "frees the Capybara session" do
106
- skip unless options[:capybara]
107
-
108
- adapter.capybara
109
-
110
- expect {
111
- adapter.renew
112
- }.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
113
- end
114
- end
115
-
116
- describe "#capybara" do
117
- before { skip unless options[:capybara] }
118
-
119
- it "returns a Capybara driver" do
120
- expect(adapter.capybara).to be_a(Capybara::Session)
121
- end
122
- end
123
-
124
- describe "HTTP headers" do
125
- before { Wayfarer.config.http_headers = { "User-Agent" => "Foobar" } }
126
-
127
- it "uses configured HTTP headers" do
128
- skip unless options[:request_headers]
129
-
130
- url = test_app_path("/headers/HTTP_USER_AGENT")
131
- page = adapter.fetch(url).page
132
- expect(page.body).to match(/Foobar/)
133
- end
134
- end
135
- end
@@ -1,46 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "spec_helpers"
4
-
5
- describe Wayfarer::Networking::Healer do
6
- let(:error) { Class.new(StandardError) }
7
- let(:renew_on) { [error] }
8
-
9
- let(:adapter) do
10
- double.tap do |adapter|
11
- allow(adapter.class).to receive(:renew_on).and_return([error])
12
- end
13
- end
14
-
15
- subject(:healer) do
16
- Wayfarer::Networking::Healer.new(adapter)
17
- end
18
-
19
- describe "#fetch" do
20
- context "when proxied call raises renewing error" do
21
- before do
22
- allow(adapter).to receive(:fetch).and_raise(error)
23
- end
24
-
25
- it "renews its adapter and re-raises" do
26
- expect(adapter).to receive(:renew)
27
- expect {
28
- healer.fetch(URI("https://example.com"))
29
- }.to raise_error(error)
30
- end
31
- end
32
-
33
- context "when proxied call raises non-renewing error" do
34
- before do
35
- allow(adapter).to receive(:fetch).and_raise(StandardError.new)
36
- end
37
-
38
- it "does not renew its adapter and intercept" do
39
- expect(adapter).not_to receive(:renew)
40
- expect {
41
- healer.fetch(URI("https://example.com"))
42
- }.to raise_error(StandardError)
43
- end
44
- end
45
- end
46
- end
@@ -1,37 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "spec_helpers"
4
- require_relative "adapter"
5
-
6
- describe Wayfarer::Networking::NetHTTP do
7
- subject(:adapter) { Wayfarer::Networking::NetHTTP.send(:new) }
8
-
9
- after { adapter.free }
10
-
11
- include_examples "Network adapter", browser: false,
12
- capybara: false,
13
- request_headers: true
14
-
15
- describe "#fetch" do
16
- it "sets the status code" do
17
- url = test_app_path("/status_code/404")
18
- page = adapter.fetch(url).page
19
- expect(page.status_code).to be 404
20
- end
21
-
22
- it "sets response headers" do
23
- url = test_app_path("/hello_world")
24
- page = adapter.fetch(url).page
25
- expect(page.headers["hello"]).to eq ["world"]
26
- end
27
-
28
- context "when encountering a redirect" do
29
- it "returns a Redirect" do
30
- url = test_app_path("/redirect?times=3")
31
- result = adapter.fetch(url)
32
- expect(result).to be_a Wayfarer::Networking::Result::Redirect
33
- expect(result.redirect_url).to eq test_app_path("/redirect?times=2")
34
- end
35
- end
36
- end
37
- end