wayfarer 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.env +17 -0
- data/.github/workflows/lint.yaml +27 -0
- data/.github/workflows/release.yaml +30 -0
- data/.github/workflows/tests.yaml +21 -0
- data/.gitignore +5 -1
- data/.rubocop.yml +36 -0
- data/.vale.ini +8 -0
- data/.yardopts +1 -3
- data/Dockerfile +6 -4
- data/Gemfile +24 -0
- data/Gemfile.lock +274 -164
- data/Rakefile +7 -51
- data/bin/wayfarer +1 -1
- data/docker-compose.yml +23 -13
- data/docs/cookbook/consent_screen.md +2 -2
- data/docs/cookbook/executing_javascript.md +3 -3
- data/docs/cookbook/navigation.md +12 -12
- data/docs/cookbook/querying_html.md +3 -3
- data/docs/cookbook/screenshots.md +2 -2
- data/docs/guides/callbacks.md +25 -125
- data/docs/guides/cli.md +71 -0
- data/docs/guides/configuration.md +10 -35
- data/docs/guides/development.md +67 -0
- data/docs/guides/handlers.md +60 -0
- data/docs/guides/index.md +1 -0
- data/docs/guides/jobs.md +142 -31
- data/docs/guides/navigation.md +1 -1
- data/docs/guides/networking/capybara.md +13 -22
- data/docs/guides/networking/custom_adapters.md +103 -41
- data/docs/guides/networking/ferrum.md +4 -4
- data/docs/guides/networking/http.md +9 -13
- data/docs/guides/networking/selenium.md +10 -11
- data/docs/guides/pages.md +78 -10
- data/docs/guides/redis.md +10 -0
- data/docs/guides/routing.md +156 -0
- data/docs/guides/tasks.md +53 -9
- data/docs/guides/tutorial.md +66 -0
- data/docs/guides/user_agents.md +115 -0
- data/docs/index.md +17 -40
- data/lib/wayfarer/base.rb +125 -46
- data/lib/wayfarer/batch_completion.rb +60 -0
- data/lib/wayfarer/callbacks.rb +22 -48
- data/lib/wayfarer/cli/route_printer.rb +85 -89
- data/lib/wayfarer/cli.rb +103 -0
- data/lib/wayfarer/gc.rb +18 -6
- data/lib/wayfarer/handler.rb +15 -7
- data/lib/wayfarer/kv.rb +28 -0
- data/lib/wayfarer/logging.rb +38 -0
- data/lib/wayfarer/middleware/base.rb +2 -0
- data/lib/wayfarer/middleware/batch_completion.rb +19 -0
- data/lib/wayfarer/middleware/chain.rb +7 -1
- data/lib/wayfarer/middleware/content_type.rb +59 -0
- data/lib/wayfarer/middleware/controller.rb +19 -15
- data/lib/wayfarer/middleware/dedup.rb +22 -13
- data/lib/wayfarer/middleware/dispatch.rb +17 -4
- data/lib/wayfarer/middleware/normalize.rb +7 -14
- data/lib/wayfarer/middleware/redis.rb +15 -0
- data/lib/wayfarer/middleware/router.rb +33 -35
- data/lib/wayfarer/middleware/stage.rb +5 -5
- data/lib/wayfarer/middleware/uri_parser.rb +31 -0
- data/lib/wayfarer/middleware/user_agent.rb +49 -0
- data/lib/wayfarer/networking/capybara.rb +1 -1
- data/lib/wayfarer/networking/context.rb +14 -3
- data/lib/wayfarer/networking/ferrum.rb +1 -4
- data/lib/wayfarer/networking/follow.rb +14 -7
- data/lib/wayfarer/networking/http.rb +1 -1
- data/lib/wayfarer/networking/pool.rb +23 -13
- data/lib/wayfarer/networking/selenium.rb +15 -7
- data/lib/wayfarer/networking/strategy.rb +2 -2
- data/lib/wayfarer/page.rb +34 -14
- data/lib/wayfarer/parsing/xml.rb +6 -6
- data/lib/wayfarer/parsing.rb +21 -0
- data/lib/wayfarer/redis/barrier.rb +26 -21
- data/lib/wayfarer/redis/counter.rb +18 -9
- data/lib/wayfarer/redis/pool.rb +1 -1
- data/lib/wayfarer/redis/resettable.rb +19 -0
- data/lib/wayfarer/routing/dsl.rb +166 -30
- data/lib/wayfarer/routing/hash_stack.rb +33 -0
- data/lib/wayfarer/routing/matchers/custom.rb +8 -5
- data/lib/wayfarer/routing/matchers/{suffix.rb → empty_params.rb} +2 -6
- data/lib/wayfarer/routing/matchers/host.rb +15 -9
- data/lib/wayfarer/routing/matchers/path.rb +11 -31
- data/lib/wayfarer/routing/matchers/query.rb +41 -17
- data/lib/wayfarer/routing/matchers/result.rb +12 -0
- data/lib/wayfarer/routing/matchers/scheme.rb +13 -5
- data/lib/wayfarer/routing/matchers/url.rb +13 -5
- data/lib/wayfarer/routing/path_consumer.rb +130 -0
- data/lib/wayfarer/routing/path_finder.rb +151 -23
- data/lib/wayfarer/routing/result.rb +1 -1
- data/lib/wayfarer/routing/root_route.rb +17 -1
- data/lib/wayfarer/routing/route.rb +66 -19
- data/lib/wayfarer/routing/serializable.rb +28 -0
- data/lib/wayfarer/routing/sub_route.rb +53 -0
- data/lib/wayfarer/routing/target_route.rb +17 -1
- data/lib/wayfarer/stringify.rb +21 -30
- data/lib/wayfarer/task.rb +9 -17
- data/lib/wayfarer/uri/normalization.rb +120 -0
- data/lib/wayfarer.rb +72 -5
- data/mise.toml +2 -0
- data/mkdocs.yml +44 -8
- data/rake/docs.rake +26 -0
- data/rake/lint.rake +9 -0
- data/rake/release.rake +23 -0
- data/rake/tests.rake +32 -0
- data/requirements.txt +1 -1
- data/spec/factories/job.rb +8 -0
- data/spec/factories/middleware.rb +2 -2
- data/spec/factories/path_finder.rb +11 -0
- data/spec/factories/redis.rb +19 -0
- data/spec/factories/task.rb +46 -2
- data/spec/spec_helpers.rb +55 -51
- data/spec/support/active_job_helpers.rb +8 -0
- data/spec/support/integration_helpers.rb +21 -0
- data/spec/support/redis_helpers.rb +9 -0
- data/spec/support/test_app.rb +66 -37
- data/spec/wayfarer/base_spec.rb +200 -0
- data/spec/wayfarer/batch_completion_spec.rb +142 -0
- data/spec/wayfarer/cli/job_spec.rb +88 -0
- data/spec/wayfarer/cli/routing_spec.rb +322 -0
- data/spec/{cli → wayfarer/cli}/version_spec.rb +1 -1
- data/spec/wayfarer/gc_spec.rb +29 -0
- data/spec/wayfarer/handler_spec.rb +9 -0
- data/spec/wayfarer/integration/callbacks_spec.rb +200 -0
- data/spec/wayfarer/integration/content_type_spec.rb +37 -0
- data/spec/wayfarer/integration/custom_routing_spec.rb +51 -0
- data/spec/wayfarer/integration/gc_spec.rb +40 -0
- data/spec/wayfarer/integration/handler_spec.rb +65 -0
- data/spec/wayfarer/integration/page_spec.rb +79 -0
- data/spec/wayfarer/integration/params_spec.rb +64 -0
- data/spec/wayfarer/integration/parsing_spec.rb +99 -0
- data/spec/wayfarer/integration/retry_spec.rb +112 -0
- data/spec/wayfarer/integration/stage_spec.rb +58 -0
- data/spec/wayfarer/middleware/batch_completion_spec.rb +33 -0
- data/spec/{middleware → wayfarer/middleware}/chain_spec.rb +24 -19
- data/spec/wayfarer/middleware/content_type_spec.rb +83 -0
- data/spec/{middleware → wayfarer/middleware}/controller_spec.rb +24 -22
- data/spec/wayfarer/middleware/dedup_spec.rb +66 -0
- data/spec/wayfarer/middleware/normalize_spec.rb +32 -0
- data/spec/wayfarer/middleware/router_spec.rb +102 -0
- data/spec/wayfarer/middleware/stage_spec.rb +63 -0
- data/spec/wayfarer/middleware/uri_parser_spec.rb +63 -0
- data/spec/wayfarer/middleware/user_agent_spec.rb +158 -0
- data/spec/wayfarer/networking/capybara_spec.rb +13 -0
- data/spec/{networking → wayfarer/networking}/context_spec.rb +46 -38
- data/spec/wayfarer/networking/ferrum_spec.rb +13 -0
- data/spec/{networking → wayfarer/networking}/follow_spec.rb +11 -6
- data/spec/wayfarer/networking/http_spec.rb +12 -0
- data/spec/{networking → wayfarer/networking}/pool_spec.rb +16 -14
- data/spec/wayfarer/networking/selenium_spec.rb +12 -0
- data/spec/{networking → wayfarer/networking}/strategy.rb +33 -54
- data/spec/wayfarer/page_spec.rb +69 -0
- data/spec/{parsing → wayfarer/parsing}/json_spec.rb +1 -1
- data/spec/wayfarer/parsing/xml_parse_spec.rb +25 -0
- data/spec/wayfarer/redis/barrier_spec.rb +39 -0
- data/spec/wayfarer/redis/counter_spec.rb +34 -0
- data/spec/{redis → wayfarer/redis}/pool_spec.rb +4 -3
- data/spec/{routing → wayfarer/routing}/dsl_spec.rb +12 -22
- data/spec/wayfarer/routing/hash_stack_spec.rb +63 -0
- data/spec/wayfarer/routing/integration_spec.rb +101 -0
- data/spec/wayfarer/routing/matchers/custom_spec.rb +39 -0
- data/spec/wayfarer/routing/matchers/host_spec.rb +56 -0
- data/spec/wayfarer/routing/matchers/matcher.rb +17 -0
- data/spec/wayfarer/routing/matchers/path_spec.rb +43 -0
- data/spec/wayfarer/routing/matchers/query_spec.rb +123 -0
- data/spec/wayfarer/routing/matchers/scheme_spec.rb +45 -0
- data/spec/wayfarer/routing/matchers/url_spec.rb +33 -0
- data/spec/wayfarer/routing/path_consumer_spec.rb +123 -0
- data/spec/wayfarer/routing/path_finder_spec.rb +409 -0
- data/spec/wayfarer/routing/root_route_spec.rb +51 -0
- data/spec/wayfarer/routing/route_spec.rb +74 -0
- data/spec/wayfarer/routing/sub_route_spec.rb +103 -0
- data/spec/wayfarer/task_spec.rb +13 -0
- data/spec/wayfarer/uri/normalization_spec.rb +98 -0
- data/spec/wayfarer_spec.rb +2 -2
- data/wayfarer.gemspec +18 -28
- metadata +797 -265
- data/.github/workflows/ci.yaml +0 -32
- data/.rbenv-gemsets +0 -1
- data/.ruby-version +0 -1
- data/RELEASING.md +0 -17
- data/docs/cookbook/user_agent.md +0 -7
- data/docs/guides/error_handling.md +0 -53
- data/docs/guides/networking.md +0 -94
- data/docs/guides/performance.md +0 -130
- data/docs/guides/reliability.md +0 -41
- data/docs/guides/routing/steering.md +0 -30
- data/docs/reference/api/base.md +0 -48
- data/docs/reference/cli.md +0 -61
- data/docs/reference/configuration_keys.md +0 -43
- data/docs/reference/environment_variables.md +0 -83
- data/lib/wayfarer/cli/base.rb +0 -45
- data/lib/wayfarer/cli/generate.rb +0 -17
- data/lib/wayfarer/cli/job.rb +0 -56
- data/lib/wayfarer/cli/route.rb +0 -29
- data/lib/wayfarer/cli/runner.rb +0 -34
- data/lib/wayfarer/cli/templates/Gemfile.tt +0 -5
- data/lib/wayfarer/cli/templates/job.rb.tt +0 -10
- data/lib/wayfarer/config/capybara.rb +0 -10
- data/lib/wayfarer/config/ferrum.rb +0 -11
- data/lib/wayfarer/config/networking.rb +0 -29
- data/lib/wayfarer/config/redis.rb +0 -14
- data/lib/wayfarer/config/root.rb +0 -11
- data/lib/wayfarer/config/selenium.rb +0 -21
- data/lib/wayfarer/config/strconv.rb +0 -45
- data/lib/wayfarer/config/struct.rb +0 -72
- data/lib/wayfarer/middleware/fetch.rb +0 -56
- data/lib/wayfarer/redis/connection.rb +0 -13
- data/lib/wayfarer/redis/version.rb +0 -19
- data/lib/wayfarer/routing/router.rb +0 -28
- data/spec/base_spec.rb +0 -224
- data/spec/callbacks_spec.rb +0 -102
- data/spec/cli/generate_spec.rb +0 -39
- data/spec/cli/job_spec.rb +0 -78
- data/spec/config/capybara_spec.rb +0 -18
- data/spec/config/ferrum_spec.rb +0 -24
- data/spec/config/networking_spec.rb +0 -73
- data/spec/config/redis_spec.rb +0 -32
- data/spec/config/root_spec.rb +0 -31
- data/spec/config/selenium_spec.rb +0 -56
- data/spec/config/strconv_spec.rb +0 -58
- data/spec/config/struct_spec.rb +0 -66
- data/spec/fixtures/dummy_job.rb +0 -7
- data/spec/gc_spec.rb +0 -59
- data/spec/handler_spec.rb +0 -11
- data/spec/integration/callbacks_spec.rb +0 -85
- data/spec/integration/page_spec.rb +0 -62
- data/spec/integration/params_spec.rb +0 -56
- data/spec/integration/stage_spec.rb +0 -51
- data/spec/integration/steering_spec.rb +0 -57
- data/spec/middleware/dedup_spec.rb +0 -88
- data/spec/middleware/dispatch_spec.rb +0 -43
- data/spec/middleware/fetch_spec.rb +0 -155
- data/spec/middleware/normalize_spec.rb +0 -29
- data/spec/middleware/router_spec.rb +0 -105
- data/spec/middleware/stage_spec.rb +0 -62
- data/spec/networking/capybara_spec.rb +0 -12
- data/spec/networking/ferrum_spec.rb +0 -12
- data/spec/networking/http_spec.rb +0 -12
- data/spec/networking/selenium_spec.rb +0 -12
- data/spec/page_spec.rb +0 -47
- data/spec/parsing/xml_spec.rb +0 -25
- data/spec/redis/barrier_spec.rb +0 -78
- data/spec/redis/counter_spec.rb +0 -32
- data/spec/redis/version_spec.rb +0 -13
- data/spec/routing/integration_spec.rb +0 -110
- data/spec/routing/matchers/custom_spec.rb +0 -31
- data/spec/routing/matchers/host_spec.rb +0 -49
- data/spec/routing/matchers/path_spec.rb +0 -43
- data/spec/routing/matchers/query_spec.rb +0 -137
- data/spec/routing/matchers/scheme_spec.rb +0 -25
- data/spec/routing/matchers/suffix_spec.rb +0 -41
- data/spec/routing/matchers/uri_spec.rb +0 -27
- data/spec/routing/path_finder_spec.rb +0 -33
- data/spec/routing/root_route_spec.rb +0 -29
- data/spec/routing/route_spec.rb +0 -43
- data/spec/routing/router_spec.rb +0 -24
- data/spec/task_spec.rb +0 -34
- data/spec/{stringify_spec.rb → wayfarer/stringify_spec.rb} +2 -2
@@ -0,0 +1,103 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Routing::SubRoute do
|
6
|
+
subject(:result) { root.invoke(task) }
|
7
|
+
|
8
|
+
let(:root) { Wayfarer::Routing::RootRoute.new }
|
9
|
+
let(:task) { build(:task, :uri, url: url) }
|
10
|
+
|
11
|
+
describe "subrouting" do
|
12
|
+
let(:url) { "http://example.com" }
|
13
|
+
|
14
|
+
before do
|
15
|
+
root.custom do |sub_root, uri|
|
16
|
+
if uri.to_s == "http://example.com"
|
17
|
+
sub_root.to :foo
|
18
|
+
else
|
19
|
+
sub_root.to :bar
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
it { is_expected.to be_a(Wayfarer::Routing::Result::Match) }
|
25
|
+
|
26
|
+
describe "action" do
|
27
|
+
subject { result.action }
|
28
|
+
|
29
|
+
it { is_expected.to be(:foo) }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "nested path parameters" do
|
34
|
+
let(:url) { "http://example.com/hello/world" }
|
35
|
+
|
36
|
+
before do
|
37
|
+
root.host "example.com", path: ":foo" do
|
38
|
+
custom do |sub_root, uri|
|
39
|
+
sub_root.path ":bar", to: uri.host.split(".").first.reverse
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
it { is_expected.to be_a(Wayfarer::Routing::Result::Match) }
|
45
|
+
|
46
|
+
describe "action" do
|
47
|
+
subject { result.action }
|
48
|
+
|
49
|
+
it { is_expected.to eq("elpmaxe") }
|
50
|
+
end
|
51
|
+
|
52
|
+
describe "params" do
|
53
|
+
subject { result.params }
|
54
|
+
|
55
|
+
it { is_expected.to eq("foo" => "hello", "bar" => "world") }
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe "overridden target" do
|
60
|
+
let(:url) { "http://example.com" }
|
61
|
+
|
62
|
+
before do
|
63
|
+
root.to(:root).custom do |sub_route, uri|
|
64
|
+
if uri.to_s == "http://example.com"
|
65
|
+
sub_route.to :foo
|
66
|
+
else
|
67
|
+
sub_route.to :bar
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
it { is_expected.to be_a(Wayfarer::Routing::Result::Match) }
|
73
|
+
|
74
|
+
describe "action" do
|
75
|
+
subject { result.action }
|
76
|
+
|
77
|
+
it { is_expected.to be(:foo) }
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
describe "nested custom routing" do
|
82
|
+
let(:url) { "http://example.com" }
|
83
|
+
|
84
|
+
before do
|
85
|
+
root.custom do |sub_root_a, uri|
|
86
|
+
sub_root_a.custom do |sub_root_b, _uri|
|
87
|
+
case uri.host
|
88
|
+
when "example.com" then sub_root_b.to(:example)
|
89
|
+
when "w3c.org" then sub_root_b.to(:w3c)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
it { is_expected.to be_a(Wayfarer::Routing::Result::Match) }
|
96
|
+
|
97
|
+
describe "action" do
|
98
|
+
subject { result.action }
|
99
|
+
|
100
|
+
it { is_expected.to be(:example) }
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Task do
|
6
|
+
subject(:task) { build(:task) }
|
7
|
+
|
8
|
+
describe "#[], #[]=" do
|
9
|
+
specify do
|
10
|
+
expect { task[:foo] = "bar" }.to change { task[:foo] }.from(nil).to("bar")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
RSpec.describe Wayfarer::URI::Normalization do
|
6
|
+
using RSpec::Parameterized::TableSyntax
|
7
|
+
|
8
|
+
let(:default_tracking_params) { Wayfarer.config.dig(:normalization, :tracking_params) }
|
9
|
+
let(:default_schemes) { Wayfarer.config.dig(:normalization, :schemes) }
|
10
|
+
|
11
|
+
# rubocop:disable Layout/LineLength, Layout/ExtraSpacing
|
12
|
+
context "when the URI is valid and absolute" do
|
13
|
+
where(
|
14
|
+
:input, :remove_www, :remove_trailing_slash, :remove_fragment, :remove_tracking_parameters, :order_query_parameters, :output
|
15
|
+
) do
|
16
|
+
"https://www.example.com" | true | true | true | true | true | "https://example.com"
|
17
|
+
"https://example.com/path/" | false | true | false | false | false | "https://example.com/path"
|
18
|
+
"https://example.com/page#section" | false | false | true | false | false | "https://example.com/page"
|
19
|
+
"https://example.com?fbclid=123" | false | false | false | true | false | "https://example.com"
|
20
|
+
"https://www.example.com/?b=2&a=1" | true | true | false | false | true | "https://example.com?a=1&b=2"
|
21
|
+
"https://example.com/p/#f" | false | true | true | false | false | "https://example.com/p"
|
22
|
+
"https://www.example.com/path/?z=9&fbclid=1#frag" | false | false | false | false | false | "https://www.example.com/path/?z=9&fbclid=1#frag"
|
23
|
+
"https://www.example.com/p/?z=9&a=1&utm_source=c#f" | true | true | true | true | true | "https://example.com/p?a=1&z=9"
|
24
|
+
"HTTP://WWW.Example.COM/A/Path?b=2&a=1" | true | true | true | true | true | "http://example.com/A/Path?a=1&b=2"
|
25
|
+
"https://blog.example.com/" | true | true | true | true | true | "https://blog.example.com"
|
26
|
+
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&utm_campaign=test&feature=share" | true | true | true | true | true | "https://youtube.com/watch?feature=share&v=dQw4w9WgXcQ"
|
27
|
+
"https://example.com/search?q=ruby&is_awesome=&sort=asc" | true | true | true | true | true | "https://example.com/search?q=ruby&sort=asc"
|
28
|
+
"https://example.com/a%20path/?q=a%26b" | true | true | true | true | true | "https://example.com/a%20path?q=a%26b"
|
29
|
+
"https://example.com/path?a=1" | true | true | true | true | true | "https://example.com/path?a=1"
|
30
|
+
"https://example.com/" | true | true | true | true | true | "https://example.com"
|
31
|
+
"http://example.com:8080" | true | true | true | true | true | "http://example.com:8080"
|
32
|
+
"https://user:password@example.com" | true | true | true | true | true | "https://user:password@example.com"
|
33
|
+
"https://example.com/Path/To/Resource" | true | true | true | true | true | "https://example.com/Path/To/Resource"
|
34
|
+
"https://example.com" | true | true | true | true | true | "https://example.com"
|
35
|
+
"https://example.com/../../etc/passwd" | true | true | true | true | true | "https://example.com/etc/passwd"
|
36
|
+
"https://www.microsоft.com" | true | true | true | true | true | "https://xn--microsft-sbh.com"
|
37
|
+
"https://example.com/a%252fb" | true | true | true | true | true | "https://example.com/a%252fb"
|
38
|
+
"https://example.com/path%00/something" | true | true | true | true | true | "https://example.com/path%00/something"
|
39
|
+
"https://www.example.com/path/?z=9&a=1#frag" | false | false | false | false | false | "https://www.example.com/path/?z=9&a=1#frag"
|
40
|
+
"https://www.example.com/path/?z=9&a=1#frag" | true | false | false | false | false | "https://example.com/path/?z=9&a=1#frag"
|
41
|
+
"https://www.example.com/path/?z=9&a=1#frag" | false | true | false | false | false | "https://www.example.com/path?z=9&a=1#frag"
|
42
|
+
"https://www.example.com/path/?z=9&a=1#frag" | false | false | true | false | false | "https://www.example.com/path/?z=9&a=1"
|
43
|
+
"https://www.example.com/path/?z=9&fbclid=123#frag" | false | false | false | true | false | "https://www.example.com/path/?z=9#frag"
|
44
|
+
"https://www.example.com/path/?z=9&a=1#frag" | false | false | false | false | true | "https://www.example.com/path/?a=1&z=9#frag"
|
45
|
+
"https://www.example.com/path/?z=9&fbclid=1&a=1#frag" | true | true | true | true | true | "https://example.com/path?a=1&z=9"
|
46
|
+
"https://www.example.com/" | false | false | false | false | false | "https://www.example.com/"
|
47
|
+
"https://example.com?b=2&a=1" | true | true | true | false | true | "https://example.com?a=1&b=2"
|
48
|
+
"https://example.com?utm_source=test&a=1" | false | true | true | false | true | "https://example.com?a=1&utm_source=test"
|
49
|
+
"https://example.com?utm_source=test&a=1" | false | true | true | true | false | "https://example.com?a=1"
|
50
|
+
"https://www.example.com/trailing/" | false | true | true | true | true | "https://www.example.com/trailing"
|
51
|
+
"https://www.example.com/trailing/" | false | false | true | true | true | "https://www.example.com/trailing/"
|
52
|
+
end
|
53
|
+
# rubocop:enable Layout/LineLength, Layout/ExtraSpacing
|
54
|
+
|
55
|
+
with_them do
|
56
|
+
subject { described_class.canonical!(Addressable::URI.parse(input)).to_s }
|
57
|
+
|
58
|
+
before do
|
59
|
+
Wayfarer.config.fetch(:normalization).merge!(
|
60
|
+
remove_www: remove_www,
|
61
|
+
remove_trailing_slash: remove_trailing_slash,
|
62
|
+
remove_fragment: remove_fragment,
|
63
|
+
remove_tracking_parameters: remove_tracking_parameters,
|
64
|
+
order_query_parameters: order_query_parameters,
|
65
|
+
tracking_params: default_tracking_params,
|
66
|
+
schemes: default_schemes
|
67
|
+
)
|
68
|
+
end
|
69
|
+
|
70
|
+
it { is_expected.to eq(output) }
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
context "with invalid URLs" do
|
75
|
+
before do
|
76
|
+
# Consistent configuration setting
|
77
|
+
Wayfarer.config.fetch(:normalization).merge!(
|
78
|
+
remove_www: true,
|
79
|
+
remove_trailing_slash: true,
|
80
|
+
remove_fragment: true,
|
81
|
+
remove_tracking_parameters: true,
|
82
|
+
order_query_parameters: true,
|
83
|
+
tracking_params: default_tracking_params,
|
84
|
+
schemes: default_schemes
|
85
|
+
)
|
86
|
+
end
|
87
|
+
|
88
|
+
specify do
|
89
|
+
expect { described_class.canonical!(Addressable::URI.parse("")) }
|
90
|
+
.to raise_error(Wayfarer::URI::Normalization::RelativeURIError)
|
91
|
+
end
|
92
|
+
|
93
|
+
specify do
|
94
|
+
expect { described_class.canonical!(Addressable::URI.parse("/path/to/resource")) }
|
95
|
+
.to raise_error(Wayfarer::URI::Normalization::RelativeURIError)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
data/spec/wayfarer_spec.rb
CHANGED
@@ -4,10 +4,10 @@ require "spec_helpers"
|
|
4
4
|
|
5
5
|
describe Wayfarer do
|
6
6
|
describe "::VERSION" do
|
7
|
-
it("is present") { expect(defined? Wayfarer::VERSION).not_to
|
7
|
+
it("is present") { expect(defined? Wayfarer::VERSION).not_to be_nil }
|
8
8
|
end
|
9
9
|
|
10
10
|
describe "::config" do
|
11
|
-
|
11
|
+
specify { expect(described_class.config).to be_a(Hash) }
|
12
12
|
end
|
13
13
|
end
|
data/wayfarer.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = "wayfarer"
|
5
|
-
s.version = "0.4.
|
5
|
+
s.version = "0.4.8"
|
6
6
|
s.license = "MIT"
|
7
7
|
|
8
8
|
s.homepage = "http://github.com/bauerd/wayfarer"
|
@@ -14,38 +14,28 @@ Gem::Specification.new do |s|
|
|
14
14
|
s.email = "mail@bauerd.com"
|
15
15
|
|
16
16
|
s.files = `git ls-files`.split("\n")
|
17
|
-
s.test_files = `git ls-files --
|
17
|
+
s.test_files = `git ls-files -- spec`.split("\n")
|
18
18
|
|
19
19
|
s.require_paths = ["lib"]
|
20
20
|
|
21
21
|
s.executables << "wayfarer"
|
22
22
|
|
23
|
-
s.required_ruby_version = "~>
|
23
|
+
s.required_ruby_version = "~> 3.4"
|
24
24
|
|
25
|
-
s.add_runtime_dependency "activejob", ">=
|
25
|
+
s.add_runtime_dependency "activejob", ">= 7.1"
|
26
26
|
s.add_runtime_dependency "addressable", "~> 2.8"
|
27
|
-
s.add_runtime_dependency "capybara", "~> 3.
|
28
|
-
s.add_runtime_dependency "connection_pool", "~> 2.
|
29
|
-
s.add_runtime_dependency "
|
30
|
-
s.add_runtime_dependency "
|
31
|
-
s.add_runtime_dependency "
|
32
|
-
s.add_runtime_dependency "
|
33
|
-
s.add_runtime_dependency "mustermann", "~>
|
34
|
-
s.add_runtime_dependency "
|
35
|
-
s.add_runtime_dependency "
|
36
|
-
s.add_runtime_dependency "
|
37
|
-
s.add_runtime_dependency "
|
38
|
-
s.add_runtime_dependency "
|
39
|
-
s.add_runtime_dependency "
|
40
|
-
s.add_runtime_dependency "
|
41
|
-
|
42
|
-
s.add_development_dependency "cuprite", "~> 0.13"
|
43
|
-
s.add_development_dependency "factory_bot", "~> 6.0"
|
44
|
-
s.add_development_dependency "faker", "~> 1.7"
|
45
|
-
s.add_development_dependency "pry", "~> 0.10"
|
46
|
-
s.add_development_dependency "rake", "~> 10.4"
|
47
|
-
s.add_development_dependency "rspec", "~> 3.4"
|
48
|
-
s.add_development_dependency "rubocop", "~> 0.49"
|
49
|
-
s.add_development_dependency "sinatra", "~> 1.4"
|
50
|
-
s.add_development_dependency "yard", "~> 0.9"
|
27
|
+
s.add_runtime_dependency "capybara", "~> 3.4"
|
28
|
+
s.add_runtime_dependency "connection_pool", "~> 2.5"
|
29
|
+
s.add_runtime_dependency "ferrum", "~> 0.17"
|
30
|
+
s.add_runtime_dependency "metainspector", "~> 5.16"
|
31
|
+
s.add_runtime_dependency "mime-types", "~> 3.7"
|
32
|
+
s.add_runtime_dependency "mock_redis", "~> 0.52"
|
33
|
+
s.add_runtime_dependency "mustermann", "~> 3.0"
|
34
|
+
s.add_runtime_dependency "net-http-persistent", "~> 4.0" # updated
|
35
|
+
s.add_runtime_dependency "nokogiri", "~> 1.18"
|
36
|
+
s.add_runtime_dependency "redis", "~> 5.4"
|
37
|
+
s.add_runtime_dependency "selenium-webdriver", "~> 4.35" # updated
|
38
|
+
s.add_runtime_dependency "thor", "~> 1.0" # problematic
|
39
|
+
s.add_runtime_dependency "zeitwerk", "~> 2.7" # updated
|
40
|
+
s.add_runtime_dependency "sorted_set", "~> 1.0"
|
51
41
|
end
|