wayfarer 0.4.0 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yaml +1 -1
  3. data/Gemfile.lock +20 -15
  4. data/docs/cookbook/user_agent.md +1 -1
  5. data/docs/guides/browser_automation/capybara.md +64 -1
  6. data/docs/guides/browser_automation/custom_adapters.md +100 -0
  7. data/docs/guides/browser_automation/ferrum.md +3 -3
  8. data/docs/guides/browser_automation/selenium.md +7 -5
  9. data/docs/guides/callbacks.md +117 -10
  10. data/docs/guides/configuration.md +16 -10
  11. data/docs/guides/error_handling.md +9 -5
  12. data/docs/guides/networking.md +77 -3
  13. data/docs/index.md +9 -1
  14. data/docs/reference/api/base.md +4 -4
  15. data/docs/reference/configuration_keys.md +42 -0
  16. data/docs/reference/environment_variables.md +25 -27
  17. data/lib/wayfarer/base.rb +7 -17
  18. data/lib/wayfarer/callbacks.rb +71 -0
  19. data/lib/wayfarer/cli/base.rb +5 -1
  20. data/lib/wayfarer/cli/job.rb +7 -3
  21. data/lib/wayfarer/cli/route.rb +2 -2
  22. data/lib/wayfarer/cli/route_printer.rb +7 -7
  23. data/lib/wayfarer/config/capybara.rb +10 -0
  24. data/lib/wayfarer/config/ferrum.rb +11 -0
  25. data/lib/wayfarer/config/networking.rb +26 -0
  26. data/lib/wayfarer/config/redis.rb +14 -0
  27. data/lib/wayfarer/config/root.rb +11 -0
  28. data/lib/wayfarer/config/selenium.rb +21 -0
  29. data/lib/wayfarer/config/strconv.rb +45 -0
  30. data/lib/wayfarer/config/struct.rb +72 -0
  31. data/lib/wayfarer/gc.rb +3 -7
  32. data/lib/wayfarer/middleware/fetch.rb +7 -3
  33. data/lib/wayfarer/middleware/router.rb +2 -2
  34. data/lib/wayfarer/middleware/worker.rb +12 -9
  35. data/lib/wayfarer/networking/capybara.rb +28 -0
  36. data/lib/wayfarer/networking/context.rb +36 -0
  37. data/lib/wayfarer/networking/ferrum.rb +17 -52
  38. data/lib/wayfarer/networking/http.rb +34 -0
  39. data/lib/wayfarer/networking/pool.rb +15 -10
  40. data/lib/wayfarer/networking/result.rb +1 -1
  41. data/lib/wayfarer/networking/selenium.rb +20 -47
  42. data/lib/wayfarer/networking/strategy.rb +38 -0
  43. data/lib/wayfarer/page.rb +2 -3
  44. data/lib/wayfarer/redis/pool.rb +3 -1
  45. data/lib/wayfarer/routing/dsl.rb +8 -8
  46. data/lib/wayfarer/routing/matchers/custom.rb +23 -0
  47. data/lib/wayfarer/routing/matchers/host.rb +19 -0
  48. data/lib/wayfarer/routing/matchers/path.rb +48 -0
  49. data/lib/wayfarer/routing/matchers/query.rb +63 -0
  50. data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
  51. data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
  52. data/lib/wayfarer/routing/matchers/url.rb +17 -0
  53. data/lib/wayfarer/routing/route.rb +1 -1
  54. data/lib/wayfarer.rb +9 -9
  55. data/spec/base_spec.rb +14 -0
  56. data/spec/callbacks_spec.rb +102 -0
  57. data/spec/cli/job_spec.rb +6 -6
  58. data/spec/config/capybara_spec.rb +18 -0
  59. data/spec/config/ferrum_spec.rb +24 -0
  60. data/spec/config/networking_spec.rb +73 -0
  61. data/spec/config/redis_spec.rb +32 -0
  62. data/spec/config/root_spec.rb +31 -0
  63. data/spec/config/selenium_spec.rb +56 -0
  64. data/spec/config/strconv_spec.rb +58 -0
  65. data/spec/config/struct_spec.rb +66 -0
  66. data/spec/gc_spec.rb +8 -6
  67. data/spec/middleware/fetch_spec.rb +20 -8
  68. data/spec/middleware/router_spec.rb +7 -0
  69. data/spec/middleware/worker_spec.rb +64 -27
  70. data/spec/networking/capybara_spec.rb +12 -0
  71. data/spec/networking/context_spec.rb +127 -0
  72. data/spec/networking/ferrum_spec.rb +6 -22
  73. data/spec/networking/http_spec.rb +12 -0
  74. data/spec/networking/pool_spec.rb +37 -12
  75. data/spec/networking/selenium_spec.rb +6 -22
  76. data/spec/networking/strategy.rb +170 -0
  77. data/spec/redis/pool_spec.rb +1 -1
  78. data/spec/routing/dsl_spec.rb +10 -10
  79. data/spec/routing/integration_spec.rb +22 -22
  80. data/spec/routing/{custom_matcher_spec.rb → matchers/custom_spec.rb} +4 -4
  81. data/spec/routing/{host_matcher_spec.rb → matchers/host_spec.rb} +6 -6
  82. data/spec/routing/{path_matcher_spec.rb → matchers/path_spec.rb} +6 -6
  83. data/spec/routing/{query_matcher_spec.rb → matchers/query_spec.rb} +15 -15
  84. data/spec/routing/{scheme_matcher_spec.rb → matchers/scheme_spec.rb} +4 -4
  85. data/spec/routing/{suffix_matcher_spec.rb → matchers/suffix_spec.rb} +4 -4
  86. data/spec/routing/{uri_matcher_spec.rb → matchers/uri_spec.rb} +4 -4
  87. data/spec/routing/path_finder_spec.rb +1 -1
  88. data/spec/routing/root_route_spec.rb +2 -2
  89. data/spec/routing/route_spec.rb +2 -2
  90. data/spec/spec_helpers.rb +13 -5
  91. data/spec/wayfarer_spec.rb +1 -1
  92. data/wayfarer.gemspec +8 -7
  93. metadata +74 -33
  94. data/lib/wayfarer/config.rb +0 -67
  95. data/lib/wayfarer/networking/healer.rb +0 -21
  96. data/lib/wayfarer/networking/net_http.rb +0 -52
  97. data/lib/wayfarer/routing/custom_matcher.rb +0 -21
  98. data/lib/wayfarer/routing/host_matcher.rb +0 -23
  99. data/lib/wayfarer/routing/path_matcher.rb +0 -46
  100. data/lib/wayfarer/routing/query_matcher.rb +0 -67
  101. data/lib/wayfarer/routing/scheme_matcher.rb +0 -21
  102. data/lib/wayfarer/routing/suffix_matcher.rb +0 -21
  103. data/lib/wayfarer/routing/url_matcher.rb +0 -21
  104. data/spec/config_spec.rb +0 -144
  105. data/spec/networking/adapter.rb +0 -135
  106. data/spec/networking/healer_spec.rb +0 -46
  107. data/spec/networking/net_http_spec.rb +0 -37
data/spec/config_spec.rb DELETED
@@ -1,144 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "spec_helpers"
4
-
5
- describe Wayfarer::Config do
6
- describe "::defaults" do
7
- it "returns the default config" do
8
- config = Wayfarer::Config.default
9
- expect(config.adapter).to be(:net_http)
10
- expect(config.adapter_pool_size).to be(3)
11
- expect(config.adapter_pool_timeout).to eq(ConnectionPool::DEFAULTS[:timeout])
12
- expect(config.ferrum_options).to eq({})
13
- expect(config.selenium_argv).to eq(%i[chrome])
14
- expect(config.redis_url).to eq("redis://localhost:6379")
15
- expect(config.redis_factory).not_to be(nil)
16
- end
17
- end
18
-
19
- describe "::from_environment" do
20
- let(:env) { {} }
21
- subject(:config) { Wayfarer::Config.from_environment(env) }
22
-
23
- describe "#adapter" do
24
- context "by default" do
25
- it "returns :net_http" do
26
- expect(config.adapter).to be(:net_http)
27
- end
28
- end
29
-
30
- context "with env var set" do
31
- before { env["WAYFARER_ADAPTER"] = "ferrum" }
32
-
33
- it "symbolizes the env var" do
34
- expect(config.adapter).to be(:ferrum)
35
- end
36
- end
37
- end
38
-
39
- describe "#adapter_pool_size" do
40
- context "by default" do
41
- it "returns the default config value" do
42
- expect(config.adapter_pool_size).to be(3)
43
- end
44
- end
45
-
46
- context "with env var set" do
47
- before { env["WAYFARER_POOL_SIZE"] = "1234" }
48
-
49
- it "parses the env var to a number" do
50
- expect(config.adapter_pool_size).to be(1234)
51
- end
52
- end
53
- end
54
-
55
- describe "#adapter_pool_timeout" do
56
- context "by default" do
57
- it "returns the default config value" do
58
- expect(config.adapter_pool_timeout).to be(ConnectionPool::DEFAULTS[:timeout])
59
- end
60
- end
61
-
62
- context "with env var set" do
63
- let(:env) { { "WAYFARER_POOL_TIMEOUT" => "1000" } }
64
-
65
- it "parses the env var to a number" do
66
- expect(config.adapter_pool_timeout).to be(1000)
67
- end
68
- end
69
- end
70
-
71
- describe "#ferrum_options" do
72
- context "by default" do
73
- it "is empty" do
74
- expect(config.ferrum_options).to eq({})
75
- end
76
- end
77
-
78
- context "with env var set" do
79
- before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
80
-
81
- it "parses the env var to a Hash" do
82
- expect(config.ferrum_options).to eq(url: "http://chrome:3000", headless: false)
83
- end
84
- end
85
- end
86
-
87
- describe "#selenium_argv" do
88
- context "by default" do
89
- it "is [:chrome]" do
90
- expect(config.selenium_argv).to eq(%i[chrome])
91
- end
92
- end
93
-
94
- context "with env var set" do
95
- before { env["WAYFARER_SELENIUM_ARGV"] = "firefox" }
96
-
97
- it "parses the env var to an array" do
98
- expect(config.selenium_argv).to eq(%w[firefox])
99
- end
100
- end
101
- end
102
-
103
- describe "#redis_url" do
104
- context "by default" do
105
- it "is redis://localhost:6379" do
106
- expect(config.redis_url).to eq("redis://localhost:6379")
107
- end
108
- end
109
-
110
- context "with env var set" do
111
- before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
112
-
113
- it "reads the env var" do
114
- expect(config.redis_url).to eq("redis://redis:6379")
115
- end
116
- end
117
- end
118
-
119
- describe "#http_headers" do
120
- context "by default" do
121
- it "is {}" do
122
- expect(config.http_headers).to eq({})
123
- end
124
- end
125
-
126
- context "with env var set" do
127
- before { env["WAYFARER_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
128
-
129
- it "reads the env var" do
130
- expect(config.http_headers).to eq("user-agent": "foo",
131
- authorization: "bar")
132
- end
133
- end
134
- end
135
- end
136
-
137
- describe "#redis_factory" do
138
- let(:config) { Wayfarer::Config.default }
139
-
140
- it "instantiates Redis clients" do
141
- expect(config.redis_factory.call).to be_a(Redis)
142
- end
143
- end
144
- end
@@ -1,135 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "spec_helpers"
4
-
5
- RSpec.shared_examples "Network adapter" do |options|
6
- describe "::renew_on" do
7
- it "returns an Array" do
8
- expect(adapter.class.renew_on).to be_an(Array)
9
- end
10
- end
11
-
12
- describe "#fetch" do
13
- it "returns Success" do
14
- url = test_app_path("/hello_world")
15
- result = adapter.fetch(url)
16
- expect(result).to be_a(Wayfarer::Networking::Result::Success)
17
- expect(result.page).to be_a(Wayfarer::Page)
18
- end
19
-
20
- it "sets the URL" do
21
- url = test_app_path("/status_code/404")
22
- page = adapter.fetch(url).page
23
- expect(page.url).to eq test_app_path("/status_code/404")
24
- end
25
-
26
- it "sets the response body" do
27
- url = test_app_path("/hello_world")
28
- page = adapter.fetch(url).page
29
- expect(page.body).to match(/Hello world!/)
30
- end
31
- end
32
-
33
- describe "#live" do
34
- context "when automating a browser" do
35
- before { skip unless options[:browser] }
36
-
37
- it "returns a live page" do
38
- adapter.fetch(test_app_path("/hello_world"))
39
-
40
- url = test_app_path("/body/foobar")
41
- case adapter
42
- when Wayfarer::Networking::Selenium then adapter.browser.navigate.to(url)
43
- when Wayfarer::Networking::Ferrum then adapter.browser.goto(url)
44
- end
45
-
46
- page = adapter.live(nil)
47
- expect(page.url).to eq(url)
48
- expect(page.body).to match(/foobar/)
49
- # Response headers and status code cannot be tested here, because
50
- # Selenium does not support them.
51
- end
52
- end
53
-
54
- context "when using plain HTTP" do
55
- before { skip if options[:browser] }
56
-
57
- it "returns the same page" do
58
- page = build(:page)
59
- expect(adapter.live(page)).to be(page)
60
- end
61
- end
62
- end
63
-
64
- describe "#body" do
65
- it "returns the body" do
66
- url = test_app_path("/hello_world")
67
- adapter.fetch(url).page
68
- expect(adapter.body).to match(/Hello world!/)
69
- end
70
- end
71
-
72
- describe "#free" do
73
- it "frees the browser" do
74
- skip unless options[:browser]
75
-
76
- expect {
77
- adapter.free
78
- }.to change { adapter.browser }.to(nil)
79
- end
80
-
81
- it "frees the Capybara session" do
82
- skip unless options[:capybara]
83
-
84
- adapter.capybara
85
-
86
- expect {
87
- adapter.free
88
- }.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
89
- end
90
- end
91
-
92
- describe "#renew" do
93
- it "does not raise" do
94
- expect { adapter.renew }.not_to raise_error
95
- end
96
-
97
- it "renews the browser" do
98
- skip unless options[:browser]
99
-
100
- expect {
101
- adapter.renew
102
- }.to(change { adapter.browser })
103
- end
104
-
105
- it "frees the Capybara session" do
106
- skip unless options[:capybara]
107
-
108
- adapter.capybara
109
-
110
- expect {
111
- adapter.renew
112
- }.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
113
- end
114
- end
115
-
116
- describe "#capybara" do
117
- before { skip unless options[:capybara] }
118
-
119
- it "returns a Capybara driver" do
120
- expect(adapter.capybara).to be_a(Capybara::Session)
121
- end
122
- end
123
-
124
- describe "HTTP headers" do
125
- before { Wayfarer.config.http_headers = { "User-Agent" => "Foobar" } }
126
-
127
- it "uses configured HTTP headers" do
128
- skip unless options[:request_headers]
129
-
130
- url = test_app_path("/headers/HTTP_USER_AGENT")
131
- page = adapter.fetch(url).page
132
- expect(page.body).to match(/Foobar/)
133
- end
134
- end
135
- end
@@ -1,46 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "spec_helpers"
4
-
5
- describe Wayfarer::Networking::Healer do
6
- let(:error) { Class.new(StandardError) }
7
- let(:renew_on) { [error] }
8
-
9
- let(:adapter) do
10
- double.tap do |adapter|
11
- allow(adapter.class).to receive(:renew_on).and_return([error])
12
- end
13
- end
14
-
15
- subject(:healer) do
16
- Wayfarer::Networking::Healer.new(adapter)
17
- end
18
-
19
- describe "#fetch" do
20
- context "when proxied call raises renewing error" do
21
- before do
22
- allow(adapter).to receive(:fetch).and_raise(error)
23
- end
24
-
25
- it "renews its adapter and re-raises" do
26
- expect(adapter).to receive(:renew)
27
- expect {
28
- healer.fetch(URI("https://example.com"))
29
- }.to raise_error(error)
30
- end
31
- end
32
-
33
- context "when proxied call raises non-renewing error" do
34
- before do
35
- allow(adapter).to receive(:fetch).and_raise(StandardError.new)
36
- end
37
-
38
- it "does not renew its adapter and intercept" do
39
- expect(adapter).not_to receive(:renew)
40
- expect {
41
- healer.fetch(URI("https://example.com"))
42
- }.to raise_error(StandardError)
43
- end
44
- end
45
- end
46
- end
@@ -1,37 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "spec_helpers"
4
- require_relative "adapter"
5
-
6
- describe Wayfarer::Networking::NetHTTP do
7
- subject(:adapter) { Wayfarer::Networking::NetHTTP.send(:new) }
8
-
9
- after { adapter.free }
10
-
11
- include_examples "Network adapter", browser: false,
12
- capybara: false,
13
- request_headers: true
14
-
15
- describe "#fetch" do
16
- it "sets the status code" do
17
- url = test_app_path("/status_code/404")
18
- page = adapter.fetch(url).page
19
- expect(page.status_code).to be 404
20
- end
21
-
22
- it "sets response headers" do
23
- url = test_app_path("/hello_world")
24
- page = adapter.fetch(url).page
25
- expect(page.headers["hello"]).to eq ["world"]
26
- end
27
-
28
- context "when encountering a redirect" do
29
- it "returns a Redirect" do
30
- url = test_app_path("/redirect?times=3")
31
- result = adapter.fetch(url)
32
- expect(result).to be_a Wayfarer::Networking::Result::Redirect
33
- expect(result.redirect_url).to eq test_app_path("/redirect?times=2")
34
- end
35
- end
36
- end
37
- end