wayfarer 0.4.6 → 0.4.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/lint.yaml +25 -0
- data/.github/workflows/release.yaml +29 -0
- data/.github/workflows/tests.yaml +30 -0
- data/.gitignore +4 -0
- data/.rubocop.yml +5 -0
- data/.vale.ini +5 -0
- data/.yardopts +1 -3
- data/Dockerfile +5 -4
- data/Gemfile +3 -0
- data/Gemfile.lock +107 -102
- data/Rakefile +5 -56
- data/bin/wayfarer +1 -1
- data/docker-compose.yml +20 -9
- data/docs/cookbook/consent_screen.md +2 -2
- data/docs/cookbook/executing_javascript.md +3 -3
- data/docs/cookbook/navigation.md +12 -12
- data/docs/cookbook/querying_html.md +3 -3
- data/docs/cookbook/screenshots.md +2 -2
- data/docs/cookbook/user_agent.md +1 -1
- data/docs/design.md +36 -0
- data/docs/guides/callbacks.md +24 -126
- data/docs/guides/configuration.md +8 -8
- data/docs/guides/handlers.md +60 -0
- data/docs/guides/index.md +1 -0
- data/docs/guides/jobs/error_handling.md +40 -0
- data/docs/guides/jobs.md +99 -31
- data/docs/guides/navigation.md +1 -1
- data/docs/guides/networking/capybara.md +13 -22
- data/docs/guides/networking/custom_adapters.md +82 -41
- data/docs/guides/networking/ferrum.md +4 -4
- data/docs/guides/networking/http.md +9 -13
- data/docs/guides/networking/selenium.md +10 -11
- data/docs/guides/pages.md +76 -10
- data/docs/guides/redis.md +10 -0
- data/docs/guides/routing.md +74 -0
- data/docs/guides/tasks.md +33 -9
- data/docs/guides/tutorial.md +60 -0
- data/docs/guides/user_agents.md +113 -0
- data/docs/index.md +17 -40
- data/docs/reference/cli.md +35 -25
- data/docs/reference/configuration.md +36 -0
- data/lib/wayfarer/base.rb +124 -46
- data/lib/wayfarer/batch_completion.rb +56 -0
- data/lib/wayfarer/callbacks.rb +22 -48
- data/lib/wayfarer/cli/route_printer.rb +71 -57
- data/lib/wayfarer/cli.rb +121 -0
- data/lib/wayfarer/gc.rb +13 -6
- data/lib/wayfarer/handler.rb +15 -7
- data/lib/wayfarer/logging.rb +38 -0
- data/lib/wayfarer/middleware/base.rb +2 -0
- data/lib/wayfarer/middleware/batch_completion.rb +19 -0
- data/lib/wayfarer/middleware/content_type.rb +54 -0
- data/lib/wayfarer/middleware/controller.rb +19 -15
- data/lib/wayfarer/middleware/dedup.rb +16 -13
- data/lib/wayfarer/middleware/dispatch.rb +12 -4
- data/lib/wayfarer/middleware/normalize.rb +12 -11
- data/lib/wayfarer/middleware/redis.rb +15 -0
- data/lib/wayfarer/middleware/router.rb +33 -35
- data/lib/wayfarer/middleware/stage.rb +5 -5
- data/lib/wayfarer/middleware/uri_parser.rb +30 -0
- data/lib/wayfarer/middleware/user_agent.rb +49 -0
- data/lib/wayfarer/networking/capybara.rb +1 -1
- data/lib/wayfarer/networking/context.rb +2 -2
- data/lib/wayfarer/networking/ferrum.rb +2 -2
- data/lib/wayfarer/networking/follow.rb +12 -6
- data/lib/wayfarer/networking/http.rb +1 -1
- data/lib/wayfarer/networking/pool.rb +17 -12
- data/lib/wayfarer/networking/selenium.rb +3 -3
- data/lib/wayfarer/networking/strategy.rb +2 -2
- data/lib/wayfarer/page.rb +36 -14
- data/lib/wayfarer/parsing/xml.rb +6 -6
- data/lib/wayfarer/parsing.rb +24 -0
- data/lib/wayfarer/redis/barrier.rb +13 -21
- data/lib/wayfarer/redis/counter.rb +19 -9
- data/lib/wayfarer/redis/pool.rb +1 -1
- data/lib/wayfarer/redis/resettable.rb +19 -0
- data/lib/wayfarer/routing/dsl.rb +1 -0
- data/lib/wayfarer/routing/matchers/path.rb +4 -2
- data/lib/wayfarer/routing/root_route.rb +5 -1
- data/lib/wayfarer/routing/route.rb +4 -14
- data/lib/wayfarer/stringify.rb +22 -30
- data/lib/wayfarer/task.rb +12 -18
- data/lib/wayfarer.rb +28 -1
- data/mkdocs.yml +52 -7
- data/rake/docs.rake +26 -0
- data/rake/lint.rake +105 -0
- data/rake/release.rake +29 -0
- data/rake/tests.rake +28 -0
- data/requirements.txt +1 -1
- data/spec/base_spec.rb +140 -160
- data/spec/batch_completion_spec.rb +104 -0
- data/spec/cli/job_spec.rb +19 -23
- data/spec/cli/routing_spec.rb +101 -0
- data/spec/cli/version_spec.rb +1 -1
- data/spec/factories/task.rb +7 -1
- data/spec/fixtures/dummy_job.rb +5 -3
- data/spec/gc_spec.rb +8 -50
- data/spec/handler_spec.rb +1 -1
- data/spec/integration/callbacks_spec.rb +157 -45
- data/spec/integration/content_type_spec.rb +145 -0
- data/spec/integration/gc_spec.rb +44 -0
- data/spec/integration/handler_spec.rb +66 -0
- data/spec/integration/page_spec.rb +44 -29
- data/spec/integration/params_spec.rb +33 -25
- data/spec/integration/parsing_spec.rb +125 -0
- data/spec/integration/routing_spec.rb +18 -0
- data/spec/integration/stage_spec.rb +27 -20
- data/spec/middleware/batch_completion_spec.rb +34 -0
- data/spec/middleware/chain_spec.rb +8 -8
- data/spec/middleware/content_type_spec.rb +86 -0
- data/spec/middleware/controller_spec.rb +5 -5
- data/spec/middleware/dedup_spec.rb +38 -55
- data/spec/middleware/dispatch_spec.rb +23 -7
- data/spec/middleware/normalize_spec.rb +44 -13
- data/spec/middleware/router_spec.rb +29 -30
- data/spec/middleware/stage_spec.rb +8 -8
- data/spec/middleware/uri_parser_spec.rb +53 -0
- data/spec/middleware/{fetch_spec.rb → user_agent_spec.rb} +28 -27
- data/spec/networking/context_spec.rb +1 -1
- data/spec/networking/follow_spec.rb +2 -2
- data/spec/networking/pool_spec.rb +5 -5
- data/spec/networking/strategy.rb +2 -2
- data/spec/page_spec.rb +42 -20
- data/spec/parsing/xml_spec.rb +11 -12
- data/spec/redis/barrier_spec.rb +8 -48
- data/spec/redis/counter_spec.rb +13 -1
- data/spec/redis/pool_spec.rb +1 -1
- data/spec/spec_helpers.rb +27 -16
- data/spec/support/test_app.rb +8 -0
- data/spec/task_spec.rb +3 -24
- data/spec/wayfarer_spec.rb +1 -1
- data/wayfarer.gemspec +4 -3
- metadata +61 -51
- data/.github/workflows/ci.yaml +0 -32
- data/docs/guides/error_handling.md +0 -53
- data/docs/guides/networking.md +0 -94
- data/docs/guides/performance.md +0 -130
- data/docs/guides/reliability.md +0 -41
- data/docs/guides/routing/steering.md +0 -30
- data/docs/reference/api/base.md +0 -48
- data/docs/reference/configuration_keys.md +0 -43
- data/docs/reference/environment_variables.md +0 -83
- data/lib/wayfarer/cli/base.rb +0 -45
- data/lib/wayfarer/cli/generate.rb +0 -17
- data/lib/wayfarer/cli/job.rb +0 -56
- data/lib/wayfarer/cli/route.rb +0 -29
- data/lib/wayfarer/cli/runner.rb +0 -34
- data/lib/wayfarer/cli/templates/Gemfile.tt +0 -5
- data/lib/wayfarer/cli/templates/job.rb.tt +0 -10
- data/lib/wayfarer/config/capybara.rb +0 -10
- data/lib/wayfarer/config/ferrum.rb +0 -11
- data/lib/wayfarer/config/networking.rb +0 -29
- data/lib/wayfarer/config/redis.rb +0 -14
- data/lib/wayfarer/config/root.rb +0 -11
- data/lib/wayfarer/config/selenium.rb +0 -21
- data/lib/wayfarer/config/strconv.rb +0 -45
- data/lib/wayfarer/config/struct.rb +0 -72
- data/lib/wayfarer/middleware/fetch.rb +0 -56
- data/lib/wayfarer/redis/connection.rb +0 -13
- data/lib/wayfarer/redis/version.rb +0 -19
- data/lib/wayfarer/routing/router.rb +0 -28
- data/spec/callbacks_spec.rb +0 -102
- data/spec/cli/generate_spec.rb +0 -39
- data/spec/config/capybara_spec.rb +0 -18
- data/spec/config/ferrum_spec.rb +0 -24
- data/spec/config/networking_spec.rb +0 -73
- data/spec/config/redis_spec.rb +0 -32
- data/spec/config/root_spec.rb +0 -31
- data/spec/config/selenium_spec.rb +0 -56
- data/spec/config/strconv_spec.rb +0 -58
- data/spec/config/struct_spec.rb +0 -66
- data/spec/integration/steering_spec.rb +0 -57
- data/spec/redis/version_spec.rb +0 -13
- data/spec/routing/router_spec.rb +0 -24
data/spec/networking/strategy.rb
CHANGED
@@ -76,7 +76,7 @@ RSpec.shared_examples "Network strategy" do |options|
|
|
76
76
|
|
77
77
|
describe "HTTP headers" do
|
78
78
|
before do
|
79
|
-
Wayfarer.config
|
79
|
+
Wayfarer.config[:network][:http_headers] = { "User-Agent" => "Foobar" }
|
80
80
|
context.renew
|
81
81
|
end
|
82
82
|
|
@@ -158,7 +158,7 @@ RSpec.shared_examples "Network strategy" do |options|
|
|
158
158
|
|
159
159
|
describe "Client timeout" do
|
160
160
|
before do
|
161
|
-
Wayfarer.config
|
161
|
+
Wayfarer.config[:selenium][:client_timeout] = 42
|
162
162
|
context.renew
|
163
163
|
end
|
164
164
|
|
data/spec/page_spec.rb
CHANGED
@@ -3,45 +3,67 @@
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
5
|
describe Wayfarer::Page do
|
6
|
+
describe "#mime_type" do
|
7
|
+
let(:headers) { { "Content-Type" => "text/html; charset=UTF-8" } }
|
8
|
+
|
9
|
+
subject { build(:page, headers: headers).mime_type }
|
10
|
+
|
11
|
+
it { is_expected.to eq("text/html") }
|
12
|
+
end
|
13
|
+
|
6
14
|
describe "#doc" do
|
7
15
|
context "when Content-Type is absent" do
|
8
|
-
subject
|
16
|
+
subject { build(:page, :html, headers: {}).doc }
|
9
17
|
|
10
|
-
it
|
11
|
-
expect(page.doc).to be_a(Nokogiri::HTML::Document)
|
12
|
-
end
|
18
|
+
it { is_expected.to be(nil) }
|
13
19
|
end
|
14
20
|
|
15
21
|
context "when Content-Type is HTML" do
|
16
|
-
subject(:page) { build(:page, :html) }
|
22
|
+
subject(:page) { build(:page, :html).doc }
|
17
23
|
|
18
|
-
it
|
19
|
-
expect(page.doc).to be_a(Nokogiri::HTML::Document)
|
20
|
-
end
|
24
|
+
it { is_expected.to be_a(Nokogiri::HTML::Document) }
|
21
25
|
end
|
22
26
|
|
23
27
|
context "when Content-Type is XML" do
|
24
|
-
subject(:page) { build(:page, :xml) }
|
28
|
+
subject(:page) { build(:page, :xml).doc }
|
25
29
|
|
26
|
-
it
|
27
|
-
expect(page.doc).to be_a(Nokogiri::XML::Document)
|
28
|
-
end
|
30
|
+
it { is_expected.to be_a(Nokogiri::XML::Document) }
|
29
31
|
end
|
30
32
|
|
31
33
|
context "when Content-Type is JSON" do
|
32
|
-
subject(:page) { build(:page, :json) }
|
34
|
+
subject(:page) { build(:page, :json).doc }
|
33
35
|
|
34
|
-
it
|
35
|
-
|
36
|
-
|
36
|
+
it { is_expected.to be_a(Hash) }
|
37
|
+
end
|
38
|
+
|
39
|
+
context "with Content-Type parameters" do
|
40
|
+
subject(:page) { build(:page, headers: { "content-type" => "text/html; charset=UTF-8" }).doc }
|
41
|
+
|
42
|
+
it { is_expected.to be_a(Nokogiri::HTML::Document) }
|
43
|
+
end
|
44
|
+
|
45
|
+
context "when Content-Type is unsupported" do
|
46
|
+
subject(:page) { build(:page, headers: { "Content-Type" => "foo/bar" }).doc }
|
47
|
+
|
48
|
+
it { is_expected.to be_nil }
|
37
49
|
end
|
38
50
|
end
|
39
51
|
|
40
|
-
describe "#
|
41
|
-
subject(:
|
52
|
+
describe "#headers" do
|
53
|
+
subject(:headers) { build(:page, headers: { "Content-Type" => "text/html" }).headers }
|
54
|
+
|
55
|
+
it "downcases keys" do
|
56
|
+
expect(headers).to eq("content-type" => "text/html")
|
57
|
+
end
|
42
58
|
|
43
|
-
it "
|
44
|
-
expect(
|
59
|
+
it "is case-sensitive" do
|
60
|
+
expect(headers["Content-Type"]).to be_nil
|
45
61
|
end
|
46
62
|
end
|
63
|
+
|
64
|
+
describe "#meta" do
|
65
|
+
subject(:page) { build(:page).meta }
|
66
|
+
|
67
|
+
it { is_expected.to be_a(MetaInspector::Document) }
|
68
|
+
end
|
47
69
|
end
|
data/spec/parsing/xml_spec.rb
CHANGED
@@ -2,24 +2,23 @@
|
|
2
2
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
|
-
describe Wayfarer::Parsing::XML do
|
6
|
-
|
5
|
+
describe Wayfarer::Parsing::XML, ".parse" do
|
6
|
+
let(:xml) { "<span>Foobar</span>" }
|
7
|
+
subject(:doc) { Wayfarer::Parsing::XML.parse(xml, variant) }
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
html_str = "<span>Foobar</span>"
|
11
|
-
doc = parser.parse_html(html_str)
|
9
|
+
context "HTML" do
|
10
|
+
let(:variant) { :html }
|
12
11
|
|
13
|
-
|
12
|
+
specify do
|
13
|
+
expect(doc).to be_a(Nokogiri::HTML::Document)
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
xml_str = "<barqux>Foobar</barqux>"
|
20
|
-
doc = parser.parse_xml(xml_str)
|
17
|
+
context "XML" do
|
18
|
+
let(:variant) { :xml }
|
21
19
|
|
22
|
-
|
20
|
+
specify do
|
21
|
+
expect(doc).to be_a(Nokogiri::XML::Document)
|
23
22
|
end
|
24
23
|
end
|
25
24
|
end
|
data/spec/redis/barrier_spec.rb
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
5
|
describe Wayfarer::Redis::Barrier, redis: true do
|
6
|
-
let(:
|
7
|
-
subject(:barrier) { Wayfarer::Redis::Barrier.new(
|
6
|
+
let(:task) { build(:task, :redis_pool) }
|
7
|
+
subject(:barrier) { Wayfarer::Redis::Barrier.new(task) }
|
8
8
|
|
9
9
|
describe "#redis_key" do
|
10
10
|
it "returns the expected Redis key" do
|
@@ -14,65 +14,25 @@ describe Wayfarer::Redis::Barrier, redis: true do
|
|
14
14
|
|
15
15
|
describe "#reset!" do
|
16
16
|
it "resets seen URLs" do
|
17
|
-
barrier.
|
17
|
+
barrier.check!(task.url)
|
18
18
|
barrier.reset!
|
19
|
-
expect(barrier.
|
19
|
+
expect(barrier.check!(task.url)).to be(false)
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
describe "#
|
23
|
+
describe "#check!" do
|
24
24
|
context "with seen URL" do
|
25
|
-
before { barrier.
|
25
|
+
before { barrier.check!(task.url) }
|
26
26
|
|
27
27
|
it "returns true" do
|
28
|
-
expect(barrier.
|
28
|
+
expect(barrier.check!(task.url)).to be(true)
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
32
32
|
context "with unseen URL" do
|
33
33
|
it "returns false" do
|
34
|
-
expect(barrier.
|
34
|
+
expect(barrier.check!(task.url)).to be(false)
|
35
35
|
end
|
36
36
|
end
|
37
37
|
end
|
38
|
-
|
39
|
-
describe "#peek" do
|
40
|
-
let(:seen_urls) { %w[https://yahoo.com https://google.com] }
|
41
|
-
let(:unseen_urls) { %w[https://w3c.org https://nasa.gov] }
|
42
|
-
|
43
|
-
before do
|
44
|
-
allow(Wayfarer::Redis::Version).to receive(:determine).and_return(redis_version)
|
45
|
-
|
46
|
-
seen_urls.each do |url|
|
47
|
-
barrier.seen?(url)
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
context "Redis >= 6.2.0" do
|
52
|
-
let(:redis_version) { [6, 2, 5] }
|
53
|
-
|
54
|
-
it "returns membership" do
|
55
|
-
urls = [*seen_urls, *unseen_urls]
|
56
|
-
expect(barrier.peek(urls)).to eq([true, true, false, false])
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
context "Redis < 6.2.0" do
|
61
|
-
let(:redis_version) { [2, 6, 10] }
|
62
|
-
|
63
|
-
it "returns membership" do
|
64
|
-
urls = [*seen_urls, *unseen_urls]
|
65
|
-
expect(barrier.peek(urls)).to eq([true, true, false, false])
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
describe "#unsee" do
|
71
|
-
it "removes a seen URL" do
|
72
|
-
barrier.seen?(url)
|
73
|
-
expect {
|
74
|
-
barrier.unsee(url)
|
75
|
-
}.to change { barrier.seen?(url) }.to(false)
|
76
|
-
end
|
77
|
-
end
|
78
38
|
end
|
data/spec/redis/counter_spec.rb
CHANGED
@@ -3,7 +3,9 @@
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
5
|
describe Wayfarer::Redis::Counter, redis: true do
|
6
|
-
|
6
|
+
let(:callback) { -> {} }
|
7
|
+
let(:task) { build(:task, :redis_pool) }
|
8
|
+
subject(:counter) { Wayfarer::Redis::Counter.new(task, &callback) }
|
7
9
|
|
8
10
|
describe "#redis_key" do
|
9
11
|
it "returns the expected Redis key" do
|
@@ -21,6 +23,16 @@ describe Wayfarer::Redis::Counter, redis: true do
|
|
21
23
|
it "decrements" do
|
22
24
|
expect { counter.decrement }.to change { counter.value }.by(-1)
|
23
25
|
end
|
26
|
+
|
27
|
+
context "when reaching zero" do
|
28
|
+
let(:callback) { -> { @callback_called = true } }
|
29
|
+
|
30
|
+
before { counter.increment }
|
31
|
+
|
32
|
+
it "runs callback" do
|
33
|
+
expect { counter.decrement }.to change { @callback_called }.to(true).from(nil)
|
34
|
+
end
|
35
|
+
end
|
24
36
|
end
|
25
37
|
|
26
38
|
describe "#reset!" do
|
data/spec/redis/pool_spec.rb
CHANGED
data/spec/spec_helpers.rb
CHANGED
@@ -33,12 +33,12 @@ module SpecHelpers
|
|
33
33
|
FileUtils.mkdir_p(Pathname.new(path).dirname)
|
34
34
|
IO.write(path, contents)
|
35
35
|
end
|
36
|
+
end
|
36
37
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
end
|
38
|
+
module ActiveJobTestHelper
|
39
|
+
# perform_enqueued_jobs calls assert_equal
|
40
|
+
def assert_equal(expected, actual, _)
|
41
|
+
expect(actual).to eq(expected)
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
@@ -46,9 +46,13 @@ RSpec.configure do |config|
|
|
46
46
|
config.include(FactoryBot::Syntax::Methods)
|
47
47
|
config.include(ActiveJob::TestHelper)
|
48
48
|
config.include(SpecHelpers)
|
49
|
+
config.include(ActiveJobTestHelper)
|
49
50
|
|
50
51
|
config.before(:suite) do
|
52
|
+
Wayfarer::Logging.logger = Logger.new($stdout)
|
53
|
+
|
51
54
|
FactoryBot.find_definitions
|
55
|
+
FactoryBot::SyntaxRunner.include SpecHelpers
|
52
56
|
|
53
57
|
mutex = Mutex.new
|
54
58
|
cvar = ConditionVariable.new
|
@@ -67,10 +71,8 @@ RSpec.configure do |config|
|
|
67
71
|
end
|
68
72
|
|
69
73
|
config.before do
|
70
|
-
Wayfarer.config = Wayfarer::
|
71
|
-
Wayfarer.config
|
72
|
-
|
73
|
-
::Redis.new(url: redis_host).flushall
|
74
|
+
Wayfarer.config = Wayfarer::DEFAULT_CONFIG.clone
|
75
|
+
Wayfarer.config[:redis][:url] = redis_host
|
74
76
|
|
75
77
|
ActiveJob::Base.queue_adapter = :test
|
76
78
|
ActiveJob::Base.logger = Logger.new(nil)
|
@@ -81,17 +83,21 @@ RSpec.configure do |config|
|
|
81
83
|
# For unknown reasons, Wayfarer::Networking::Ferrum#renew can take a very
|
82
84
|
# long time to instantiate a new ::Ferrum::Browser and requires the 60s
|
83
85
|
# timeout. TODO: Figure out what's going on
|
84
|
-
Wayfarer.config
|
85
|
-
Wayfarer.config
|
86
|
-
Wayfarer.config
|
87
|
-
Wayfarer.config
|
86
|
+
Wayfarer.config[:ferrum][:options] = { url: "http://chrome:3000", timeout: 120 }
|
87
|
+
Wayfarer.config[:selenium][:driver] = :firefox
|
88
|
+
Wayfarer.config[:selenium][:options] = { url: "http://firefox:4444" }
|
89
|
+
Wayfarer.config[:selenium][:client_timeout] = 10
|
88
90
|
end
|
89
91
|
|
90
92
|
# TODO: Undo side-effect
|
91
93
|
Capybara.register_driver(:cuprite) do |app|
|
92
|
-
Capybara::Cuprite::Driver.new(app, Wayfarer.config
|
94
|
+
Capybara::Cuprite::Driver.new(app, Wayfarer.config[:ferrum][:options])
|
93
95
|
end
|
94
|
-
Wayfarer.config
|
96
|
+
Wayfarer.config[:capybara][:driver] = :cuprite
|
97
|
+
end
|
98
|
+
|
99
|
+
config.before(:each, redis: true) do
|
100
|
+
::Redis.new(url: redis_host).flushall
|
95
101
|
end
|
96
102
|
|
97
103
|
config.around(cli: true) do |example|
|
@@ -99,7 +105,12 @@ RSpec.configure do |config|
|
|
99
105
|
tmp = Dir.mktmpdir
|
100
106
|
Dir.chdir(tmp)
|
101
107
|
example.run
|
108
|
+
ensure
|
102
109
|
Dir.chdir(origin)
|
103
|
-
FileUtils.rm_r(tmp)
|
110
|
+
# FileUtils.rm_r(tmp)
|
111
|
+
end
|
112
|
+
|
113
|
+
config.before(:each, cli: true) do
|
114
|
+
Object.send(:remove_const, :DummyJob) if defined?(DummyJob)
|
104
115
|
end
|
105
116
|
end
|
data/spec/support/test_app.rb
CHANGED
@@ -12,6 +12,14 @@ class TestApp < Sinatra::Base
|
|
12
12
|
request.env[params[:header]]
|
13
13
|
end
|
14
14
|
|
15
|
+
get "/response_header/:key/:val" do
|
16
|
+
headers params[:key] => params[:val]
|
17
|
+
end
|
18
|
+
|
19
|
+
get "/response_header/Content-Type/*" do
|
20
|
+
headers "Content-Type" => params[:splat].first
|
21
|
+
end
|
22
|
+
|
15
23
|
get "/hello_world" do
|
16
24
|
headers "hello" => "world"
|
17
25
|
"Hello world!"
|
data/spec/task_spec.rb
CHANGED
@@ -5,30 +5,9 @@ require "spec_helpers"
|
|
5
5
|
describe Wayfarer::Task do
|
6
6
|
subject(:task) { build(:task) }
|
7
7
|
|
8
|
-
describe "#
|
9
|
-
|
10
|
-
expect
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
describe "#barrier" do
|
15
|
-
it "returns its batch barrier" do
|
16
|
-
expect(task.barrier).to be_a(Wayfarer::Redis::Barrier)
|
17
|
-
expect(task.barrier.batch).to eq(task.batch)
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
describe "#counter" do
|
22
|
-
it "returns its batch counter" do
|
23
|
-
expect(task.counter).to be_a(Wayfarer::Redis::Counter)
|
24
|
-
expect(task.counter.batch).to eq(task.batch)
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
describe "#gc" do
|
29
|
-
it "returns a garbage collector" do
|
30
|
-
expect(task.gc).to be_a(Wayfarer::GC)
|
31
|
-
expect(task.gc.task).to be(task)
|
8
|
+
describe "#[], #[]=" do
|
9
|
+
specify do
|
10
|
+
expect { task[:foo] = "bar" }.to change { task[:foo] }.from(nil).to("bar")
|
32
11
|
end
|
33
12
|
end
|
34
13
|
end
|
data/spec/wayfarer_spec.rb
CHANGED
data/wayfarer.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = "wayfarer"
|
5
|
-
s.version = "0.4.
|
5
|
+
s.version = "0.4.7"
|
6
6
|
s.license = "MIT"
|
7
7
|
|
8
8
|
s.homepage = "http://github.com/bauerd/wayfarer"
|
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
|
|
22
22
|
|
23
23
|
s.required_ruby_version = "~> 2.7"
|
24
24
|
|
25
|
-
s.add_runtime_dependency "activejob", ">=
|
25
|
+
s.add_runtime_dependency "activejob", ">= 7.1"
|
26
26
|
s.add_runtime_dependency "addressable", "~> 2.8"
|
27
27
|
s.add_runtime_dependency "capybara", "~> 3.0"
|
28
28
|
s.add_runtime_dependency "connection_pool", "~> 2.2"
|
@@ -30,14 +30,15 @@ Gem::Specification.new do |s|
|
|
30
30
|
s.add_runtime_dependency "ferrum", "~> 0.9"
|
31
31
|
s.add_runtime_dependency "metainspector", "~> 5.0"
|
32
32
|
s.add_runtime_dependency "mime-types", "~> 3.0"
|
33
|
-
s.add_runtime_dependency "mustermann", "~> 1.1"
|
34
33
|
s.add_runtime_dependency "mock_redis", "~> 0.29"
|
34
|
+
s.add_runtime_dependency "mustermann", "~> 1.1"
|
35
35
|
s.add_runtime_dependency "net-http-persistent", "~> 3.0"
|
36
36
|
s.add_runtime_dependency "nokogiri", "~> 1.11"
|
37
37
|
s.add_runtime_dependency "normalize_url", "~> 0.0.6"
|
38
38
|
s.add_runtime_dependency "redis", "~> 4.4", "< 4.5" # redis 4.5.x broke SMISMEMBER
|
39
39
|
s.add_runtime_dependency "selenium-webdriver", "~> 3.4"
|
40
40
|
s.add_runtime_dependency "thor", "~> 1.0"
|
41
|
+
s.add_runtime_dependency "zeitwerk", "~> 2.4"
|
41
42
|
|
42
43
|
s.add_development_dependency "cuprite", "~> 0.13"
|
43
44
|
s.add_development_dependency "factory_bot", "~> 6.0"
|