wayfarer 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +14 -10
- data/docs/cookbook/batch_routing.md +22 -0
- data/docs/cookbook/consent_screen.md +36 -0
- data/docs/cookbook/executing_javascript.md +41 -0
- data/docs/cookbook/querying_html.md +3 -3
- data/docs/cookbook/screenshots.md +2 -2
- data/docs/guides/browser_automation/capybara.md +6 -3
- data/docs/guides/browser_automation/ferrum.md +3 -1
- data/docs/guides/browser_automation/selenium.md +4 -2
- data/docs/guides/callbacks.md +5 -5
- data/docs/guides/debugging.md +17 -0
- data/docs/guides/error_handling.md +22 -26
- data/docs/guides/jobs.md +44 -18
- data/docs/guides/navigation.md +73 -0
- data/docs/guides/pages.md +4 -4
- data/docs/guides/performance.md +108 -0
- data/docs/guides/reliability.md +41 -0
- data/docs/guides/routing/steering.md +30 -0
- data/docs/guides/tasks.md +9 -33
- data/docs/reference/api/base.md +13 -127
- data/docs/reference/api/route.md +1 -1
- data/docs/reference/cli.md +0 -78
- data/docs/reference/configuration_keys.md +1 -1
- data/lib/wayfarer/cli/job.rb +1 -3
- data/lib/wayfarer/cli/route.rb +4 -2
- data/lib/wayfarer/cli/templates/job.rb.tt +3 -1
- data/lib/wayfarer/config/networking.rb +1 -1
- data/lib/wayfarer/config/struct.rb +1 -1
- data/lib/wayfarer/middleware/fetch.rb +15 -4
- data/lib/wayfarer/middleware/router.rb +34 -2
- data/lib/wayfarer/middleware/worker.rb +4 -24
- data/lib/wayfarer/networking/pool.rb +9 -8
- data/lib/wayfarer/page.rb +1 -1
- data/lib/wayfarer/routing/matchers/custom.rb +2 -0
- data/lib/wayfarer/routing/matchers/path.rb +1 -0
- data/lib/wayfarer/routing/route.rb +6 -0
- data/lib/wayfarer/routing/router.rb +27 -0
- data/lib/wayfarer/stringify.rb +13 -7
- data/lib/wayfarer.rb +3 -1
- data/spec/callbacks_spec.rb +2 -2
- data/spec/config/networking_spec.rb +2 -2
- data/spec/factories/{queue/middleware.rb → middleware.rb} +3 -3
- data/spec/factories/{queue/page.rb → page.rb} +3 -3
- data/spec/factories/{queue/task.rb → task.rb} +0 -0
- data/spec/fixtures/dummy_job.rb +1 -1
- data/spec/middleware/chain_spec.rb +17 -17
- data/spec/middleware/fetch_spec.rb +27 -11
- data/spec/middleware/router_spec.rb +34 -7
- data/spec/middleware/worker_spec.rb +3 -13
- data/spec/routing/router_spec.rb +24 -0
- data/wayfarer.gemspec +1 -1
- metadata +16 -8
- data/spec/factories/queue/chain.rb +0 -11
@@ -6,44 +6,24 @@ module Wayfarer
|
|
6
6
|
def self.included(base)
|
7
7
|
base.include(Wayfarer::Redis::Connection)
|
8
8
|
base.include(Wayfarer::Middleware::Stage::API)
|
9
|
+
base.include(Wayfarer::Middleware::Router::API)
|
10
|
+
base.include(Wayfarer::Middleware::Fetch::API)
|
9
11
|
base.include(Wayfarer::Callbacks)
|
10
12
|
base.include(InstanceMethods)
|
11
|
-
base.extend(ClassMethods)
|
12
|
-
end
|
13
|
-
|
14
|
-
module ClassMethods
|
15
|
-
def route
|
16
|
-
@route ||= Wayfarer::Routing::RootRoute.new.tap do |route|
|
17
|
-
Docile.dsl_eval(route) { yield if block_given? }
|
18
|
-
end
|
19
|
-
end
|
20
13
|
end
|
21
14
|
|
22
15
|
module InstanceMethods
|
23
|
-
extend Forwardable
|
24
|
-
|
25
|
-
delegate %i[params agent] => "task.metadata"
|
26
|
-
|
27
16
|
def call(task)
|
28
17
|
run_callbacks :action do
|
29
18
|
public_send(task.metadata.action)
|
30
|
-
yield if block_given? # TODO: Should be excluded from callback block
|
31
19
|
end
|
20
|
+
|
21
|
+
yield if block_given?
|
32
22
|
end
|
33
23
|
|
34
24
|
def chain
|
35
25
|
Wayfarer::Middleware::Chain.new([*Wayfarer.middleware, self])
|
36
26
|
end
|
37
|
-
|
38
|
-
def browser
|
39
|
-
agent.instance
|
40
|
-
end
|
41
|
-
|
42
|
-
def page(live: false)
|
43
|
-
return task.metadata.page unless live
|
44
|
-
|
45
|
-
task.metadata.page = agent.live&.page || task.metadata.page
|
46
|
-
end
|
47
27
|
end
|
48
28
|
end
|
49
29
|
end
|
@@ -4,22 +4,23 @@ module Wayfarer
|
|
4
4
|
module Networking
|
5
5
|
class Pool
|
6
6
|
include Singleton
|
7
|
-
extend Forwardable
|
8
7
|
|
9
8
|
cattr_accessor :registry, default: { http: HTTP,
|
10
9
|
ferrum: Ferrum,
|
11
10
|
selenium: Selenium,
|
12
11
|
capybara: Capybara }
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
timeout: Wayfarer.config.network.pool_timeout,
|
19
|
-
&method(:context))
|
13
|
+
def pool
|
14
|
+
@pool ||= ConnectionPool.new(size: Wayfarer.config.network.pool_size,
|
15
|
+
timeout: Wayfarer.config.network.pool_timeout,
|
16
|
+
&method(:context))
|
20
17
|
end
|
21
18
|
|
22
|
-
|
19
|
+
def with(&block)
|
20
|
+
pool.with(&block)
|
21
|
+
rescue ConnectionPool::TimeoutError => e
|
22
|
+
raise Wayfarer::UserAgentTimeoutError, e
|
23
|
+
end
|
23
24
|
|
24
25
|
def free
|
25
26
|
pool.shutdown(&:renew)
|
data/lib/wayfarer/page.rb
CHANGED
@@ -20,7 +20,7 @@ module Wayfarer
|
|
20
20
|
# If no Content-Type field is present, assume HTML/XML
|
21
21
|
return @doc = Wayfarer::Parsing::XML.parse_html(body) unless headers["content-type"]
|
22
22
|
|
23
|
-
content_type = headers["content-type"]
|
23
|
+
content_type = headers["content-type"]
|
24
24
|
sub_type = MIME::Types[content_type].first.sub_type
|
25
25
|
|
26
26
|
@doc = case sub_type
|
@@ -4,6 +4,7 @@ module Wayfarer
|
|
4
4
|
module Routing
|
5
5
|
class Route
|
6
6
|
include DSL
|
7
|
+
include Stringify
|
7
8
|
|
8
9
|
attr_reader :children
|
9
10
|
|
@@ -12,6 +13,11 @@ module Wayfarer
|
|
12
13
|
:action,
|
13
14
|
:path_offset
|
14
15
|
|
16
|
+
stringify :matcher,
|
17
|
+
:action,
|
18
|
+
:path_offset,
|
19
|
+
:children
|
20
|
+
|
15
21
|
def initialize(matcher = Matchers::Custom.new { children.any? }, path_offset = "/")
|
16
22
|
@matcher = matcher
|
17
23
|
@children = []
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module Routing
|
5
|
+
class Router
|
6
|
+
ArgumentCountError = Class.new(StandardError)
|
7
|
+
|
8
|
+
extend Forwardable
|
9
|
+
|
10
|
+
attr_reader :root
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@blocks = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def draw(&block)
|
17
|
+
@blocks.push(block)
|
18
|
+
end
|
19
|
+
|
20
|
+
def invoke(url, arguments)
|
21
|
+
@root = Wayfarer::Routing::RootRoute.new
|
22
|
+
@blocks.each { |block| Docile.dsl_eval(@root, *arguments, &block) }
|
23
|
+
root.invoke(url)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/wayfarer/stringify.rb
CHANGED
@@ -1,31 +1,38 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# 346325
|
4
|
+
|
3
5
|
module Wayfarer
|
4
6
|
module Stringify
|
5
7
|
def self.included(base)
|
6
8
|
base.include(InstanceMethods)
|
7
9
|
base.extend(ClassMethods)
|
10
|
+
base.instance_eval do
|
11
|
+
cattr_accessor :stringified_attributes do
|
12
|
+
[]
|
13
|
+
end
|
14
|
+
end
|
8
15
|
end
|
9
16
|
|
10
17
|
module ClassMethods
|
11
18
|
def stringify(*variables)
|
12
19
|
stringified_attributes.concat(variables)
|
13
20
|
end
|
14
|
-
|
15
|
-
def stringified_attributes
|
16
|
-
@stringified_attributes ||= []
|
17
|
-
end
|
18
21
|
end
|
19
22
|
|
20
23
|
module InstanceMethods
|
21
24
|
def to_s
|
22
|
-
|
25
|
+
if self.class.stringified_attributes.any?
|
26
|
+
"#<#{class_name} #{attributes.join(', ')}>"
|
27
|
+
else
|
28
|
+
"#<#{class_name}>"
|
29
|
+
end
|
23
30
|
end
|
24
31
|
|
25
32
|
alias inspect to_s
|
26
33
|
|
27
34
|
def class_name
|
28
|
-
self.class.name
|
35
|
+
self.class.name
|
29
36
|
end
|
30
37
|
|
31
38
|
def attributes
|
@@ -34,7 +41,6 @@ module Wayfarer
|
|
34
41
|
.map { |attr| [attr, public_send(attr)] }
|
35
42
|
.to_h
|
36
43
|
.map { |k, v| [k, "=", v.inspect].join }
|
37
|
-
.join(", ")
|
38
44
|
end
|
39
45
|
end
|
40
46
|
end
|
data/lib/wayfarer.rb
CHANGED
@@ -39,7 +39,7 @@ module Wayfarer
|
|
39
39
|
module VERSION
|
40
40
|
MAJOR = 0
|
41
41
|
MINOR = 4
|
42
|
-
TINY =
|
42
|
+
TINY = 2
|
43
43
|
STRING = [MAJOR, MINOR, TINY].join(".")
|
44
44
|
end
|
45
45
|
|
@@ -50,6 +50,8 @@ module Wayfarer
|
|
50
50
|
Wayfarer::Middleware::Normalize.new,
|
51
51
|
Wayfarer::Middleware::Router.new,
|
52
52
|
Wayfarer::Middleware::Fetch.new]
|
53
|
+
|
54
|
+
UserAgentTimeoutError = Class.new(StandardError)
|
53
55
|
end
|
54
56
|
|
55
57
|
loader.eager_load
|
data/spec/callbacks_spec.rb
CHANGED
@@ -8,8 +8,8 @@ describe Wayfarer::Callbacks do
|
|
8
8
|
|
9
9
|
let(:klass) do
|
10
10
|
Class.new(Wayfarer::Base) do
|
11
|
-
route
|
12
|
-
route
|
11
|
+
route { host "alpha.com", to: :alpha }
|
12
|
+
route { host "beta.com", to: :beta }
|
13
13
|
|
14
14
|
before_fetch do |job|
|
15
15
|
Spy.before_fetch(self, job)
|
@@ -2,12 +2,12 @@
|
|
2
2
|
|
3
3
|
FactoryBot.define do
|
4
4
|
factory :middleware, class: OpenStruct do
|
5
|
-
|
5
|
+
receiver { -> {} }
|
6
6
|
|
7
7
|
initialize_with do
|
8
|
-
new(
|
8
|
+
new(receiver: receiver).tap do |middleware|
|
9
9
|
middleware.define_singleton_method(:call) do |task, &block|
|
10
|
-
|
10
|
+
receiver.call(task, &block)
|
11
11
|
end
|
12
12
|
end
|
13
13
|
end
|
@@ -8,7 +8,7 @@ FactoryBot.define do
|
|
8
8
|
body { "" }
|
9
9
|
|
10
10
|
trait :html do
|
11
|
-
headers { { "Content-Type" =>
|
11
|
+
headers { { "Content-Type" => "text/html" } }
|
12
12
|
body do
|
13
13
|
<<~HTML
|
14
14
|
<!doctype html>
|
@@ -24,7 +24,7 @@ FactoryBot.define do
|
|
24
24
|
end
|
25
25
|
|
26
26
|
trait :xml do
|
27
|
-
headers { { "Content-Type" =>
|
27
|
+
headers { { "Content-Type" => "application/xml" } }
|
28
28
|
body do
|
29
29
|
<<~XML
|
30
30
|
<menu id="file" value="File">
|
@@ -39,7 +39,7 @@ FactoryBot.define do
|
|
39
39
|
end
|
40
40
|
|
41
41
|
trait :json do
|
42
|
-
headers { { "Content-Type" =>
|
42
|
+
headers { { "Content-Type" => "application/json" } }
|
43
43
|
body do
|
44
44
|
<<~JSON
|
45
45
|
{
|
File without changes
|
data/spec/fixtures/dummy_job.rb
CHANGED
@@ -4,7 +4,7 @@ require "spec_helpers"
|
|
4
4
|
|
5
5
|
describe Wayfarer::Middleware::Chain do
|
6
6
|
let(:task) { build(:task) }
|
7
|
-
subject(:chain) {
|
7
|
+
subject(:chain) { described_class.new(middlewares) }
|
8
8
|
|
9
9
|
describe "#call" do
|
10
10
|
context "without middlewares" do
|
@@ -51,13 +51,13 @@ describe Wayfarer::Middleware::Chain do
|
|
51
51
|
|
52
52
|
describe "Return value" do
|
53
53
|
let(:middlewares) do
|
54
|
-
[build(:middleware,
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
build(:middleware,
|
59
|
-
|
60
|
-
|
54
|
+
[build(:middleware, receiver: lambda do |task, &block|
|
55
|
+
task.metadata.foobar = 42
|
56
|
+
block.call
|
57
|
+
end),
|
58
|
+
build(:middleware, receiver: lambda do |task|
|
59
|
+
task.metadata.foobar *= 1337
|
60
|
+
end)]
|
61
61
|
end
|
62
62
|
|
63
63
|
it "is the last middleware's return value" do
|
@@ -67,18 +67,18 @@ describe Wayfarer::Middleware::Chain do
|
|
67
67
|
|
68
68
|
describe "Metadata" do
|
69
69
|
let(:first) do
|
70
|
-
build(:middleware,
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
70
|
+
build(:middleware, receiver: lambda do |task, &block|
|
71
|
+
task.metadata.foobar = 42
|
72
|
+
block.call
|
73
|
+
task.metadata.barqux = 1337
|
74
|
+
end)
|
75
75
|
end
|
76
76
|
|
77
77
|
let(:last) do
|
78
|
-
build(:middleware,
|
79
|
-
|
80
|
-
|
81
|
-
|
78
|
+
build(:middleware, receiver: lambda do |task|
|
79
|
+
raise unless task.metadata.foobar == 42
|
80
|
+
raise if task.metadata.barqux
|
81
|
+
end)
|
82
82
|
end
|
83
83
|
|
84
84
|
let(:middlewares) { [first, last] }
|
@@ -6,12 +6,14 @@ describe Wayfarer::Middleware::Fetch do
|
|
6
6
|
let(:klass) { Class.new(Wayfarer::Base) }
|
7
7
|
let(:job) { klass.new }
|
8
8
|
let(:task) { build(:task) }
|
9
|
-
let(:
|
9
|
+
let(:context) { double(instance: instance) }
|
10
|
+
let(:instance) { spy }
|
10
11
|
subject(:fetch) { Wayfarer::Middleware::Fetch.new }
|
11
12
|
|
12
13
|
before do
|
13
14
|
task.job = job
|
14
|
-
allow(fetch.pool).to receive(:with).and_yield(
|
15
|
+
allow(fetch.pool).to receive(:with).and_yield(context)
|
16
|
+
allow(context).to receive(:fetch)
|
15
17
|
end
|
16
18
|
|
17
19
|
describe "#call" do
|
@@ -21,14 +23,8 @@ describe Wayfarer::Middleware::Fetch do
|
|
21
23
|
}.to change { fetch.task }.to(task)
|
22
24
|
end
|
23
25
|
|
24
|
-
it "assigns the agent" do
|
25
|
-
expect {
|
26
|
-
fetch.call(task)
|
27
|
-
}.to change { task.metadata.agent }.to(agent)
|
28
|
-
end
|
29
|
-
|
30
26
|
it "retrieves the page" do
|
31
|
-
expect(
|
27
|
+
expect(context).to receive(:fetch).with(task.url)
|
32
28
|
fetch.call(task)
|
33
29
|
end
|
34
30
|
|
@@ -48,7 +44,7 @@ describe Wayfarer::Middleware::Fetch do
|
|
48
44
|
|
49
45
|
before do
|
50
46
|
task.metadata.staged_urls = Set.new
|
51
|
-
allow(
|
47
|
+
allow(context).to receive(:fetch).with(task.url).and_return(result)
|
52
48
|
end
|
53
49
|
|
54
50
|
it "stages the redirect URL" do
|
@@ -67,7 +63,7 @@ describe Wayfarer::Middleware::Fetch do
|
|
67
63
|
let(:result) { Wayfarer::Networking::Result::Success.new(page) }
|
68
64
|
|
69
65
|
before do
|
70
|
-
allow(
|
66
|
+
allow(context).to receive(:fetch).with(task.url).and_return(result)
|
71
67
|
end
|
72
68
|
|
73
69
|
it "assigns the page" do
|
@@ -76,9 +72,29 @@ describe Wayfarer::Middleware::Fetch do
|
|
76
72
|
}.to change { task.metadata.page }.to(page)
|
77
73
|
end
|
78
74
|
|
75
|
+
it "assigns the instance" do
|
76
|
+
expect {
|
77
|
+
fetch.call(task)
|
78
|
+
}.to change { task.metadata.agent }.to(instance)
|
79
|
+
end
|
80
|
+
|
79
81
|
it "yields" do
|
80
82
|
expect { |spy| fetch.call(task, &spy) }.to yield_control
|
81
83
|
end
|
82
84
|
end
|
83
85
|
end
|
86
|
+
|
87
|
+
describe described_class::API do
|
88
|
+
let(:klass) do
|
89
|
+
Class.new(Wayfarer::Base) do
|
90
|
+
route do |host, path|
|
91
|
+
to :index, host: host, path: path
|
92
|
+
end
|
93
|
+
|
94
|
+
steer do |task|
|
95
|
+
[task.batch, "/foobar"]
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
84
100
|
end
|
@@ -4,17 +4,22 @@ require "spec_helpers"
|
|
4
4
|
|
5
5
|
describe Wayfarer::Middleware::Router do
|
6
6
|
let(:task) { build(:task, url: url) }
|
7
|
-
let(:klass) do
|
8
|
-
Class.new(Wayfarer::Base) do
|
9
|
-
route.to :index, host: "example.com", path: "/foobar"
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
7
|
subject(:router) { Wayfarer::Middleware::Router.new }
|
14
8
|
|
15
|
-
before
|
9
|
+
before do
|
10
|
+
task.job = klass.new
|
11
|
+
allow(task.job).to receive(:arguments).and_return([task])
|
12
|
+
end
|
16
13
|
|
17
14
|
describe "#call" do
|
15
|
+
let(:klass) do
|
16
|
+
Class.new(Wayfarer::Base) do
|
17
|
+
route do
|
18
|
+
to :index, host: "example.com", path: "/foobar"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
18
23
|
context "when URL matches a route" do
|
19
24
|
let(:url) { "https://example.com/foobar" }
|
20
25
|
|
@@ -50,4 +55,26 @@ describe Wayfarer::Middleware::Router do
|
|
50
55
|
end
|
51
56
|
end
|
52
57
|
end
|
58
|
+
|
59
|
+
describe described_class::API do
|
60
|
+
let(:klass) do
|
61
|
+
Class.new(Wayfarer::Base) do
|
62
|
+
route do |host, path|
|
63
|
+
to :index, host: host, path: path
|
64
|
+
end
|
65
|
+
|
66
|
+
steer do |task|
|
67
|
+
[task.batch, "/foobar"]
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
let(:task) { build(:task, batch: "example.com", url: "https://example.com/foobar") }
|
73
|
+
|
74
|
+
it "passes arguments" do
|
75
|
+
router.call(task)
|
76
|
+
expect(task.metadata.action).to be(:index)
|
77
|
+
expect(task.metadata.params).to eq({})
|
78
|
+
end
|
79
|
+
end
|
53
80
|
end
|
@@ -18,9 +18,9 @@ describe Wayfarer::Middleware::Worker do
|
|
18
18
|
allow_any_instance_of(job).to receive(:arguments).and_return([task])
|
19
19
|
end
|
20
20
|
|
21
|
-
describe "::
|
22
|
-
it "returns a
|
23
|
-
expect(job.
|
21
|
+
describe "::router" do
|
22
|
+
it "returns a router" do
|
23
|
+
expect(job.router).to be_a(Wayfarer::Routing::Router)
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
@@ -67,16 +67,6 @@ describe Wayfarer::Middleware::Worker do
|
|
67
67
|
end
|
68
68
|
end
|
69
69
|
|
70
|
-
describe "#browser" do
|
71
|
-
it "returns the context's instance" do
|
72
|
-
worker = job.new
|
73
|
-
context = spy
|
74
|
-
expect(worker.task.metadata).to receive(:agent).and_return(context)
|
75
|
-
expect(context).to receive(:instance)
|
76
|
-
worker.browser
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
70
|
describe "#page" do
|
81
71
|
let(:worker) { job.new }
|
82
72
|
let(:page) { Object.new }
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Routing::Router do
|
6
|
+
subject(:router) do
|
7
|
+
described_class.new.tap do |router|
|
8
|
+
router.draw do |alpha, beta|
|
9
|
+
path alpha, to: :alpha
|
10
|
+
path beta, to: :beta
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
describe "#invoke" do
|
16
|
+
let(:url) { Addressable::URI.parse("https://example.com/alpha") }
|
17
|
+
|
18
|
+
it "evaluates routes" do
|
19
|
+
result = router.invoke(url, %w[/alpha /beta])
|
20
|
+
expect(result).to be_a(Wayfarer::Routing::Result::Match)
|
21
|
+
expect(result.action).to be(:alpha)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/wayfarer.gemspec
CHANGED
metadata
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: wayfarer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dominic Bauer
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
date: 2021-09-26 00:00:00.000000000 Z
|
@@ -388,6 +388,9 @@ files:
|
|
388
388
|
- Rakefile
|
389
389
|
- bin/wayfarer
|
390
390
|
- docker-compose.yml
|
391
|
+
- docs/cookbook/batch_routing.md
|
392
|
+
- docs/cookbook/consent_screen.md
|
393
|
+
- docs/cookbook/executing_javascript.md
|
391
394
|
- docs/cookbook/querying_html.md
|
392
395
|
- docs/cookbook/screenshots.md
|
393
396
|
- docs/cookbook/user_agent.md
|
@@ -397,11 +400,15 @@ files:
|
|
397
400
|
- docs/guides/browser_automation/selenium.md
|
398
401
|
- docs/guides/callbacks.md
|
399
402
|
- docs/guides/configuration.md
|
403
|
+
- docs/guides/debugging.md
|
400
404
|
- docs/guides/error_handling.md
|
401
405
|
- docs/guides/jobs.md
|
406
|
+
- docs/guides/navigation.md
|
402
407
|
- docs/guides/networking.md
|
403
408
|
- docs/guides/pages.md
|
404
409
|
- docs/guides/performance.md
|
410
|
+
- docs/guides/reliability.md
|
411
|
+
- docs/guides/routing/steering.md
|
405
412
|
- docs/guides/tasks.md
|
406
413
|
- docs/index.md
|
407
414
|
- docs/reference/api/base.md
|
@@ -464,6 +471,7 @@ files:
|
|
464
471
|
- lib/wayfarer/routing/result.rb
|
465
472
|
- lib/wayfarer/routing/root_route.rb
|
466
473
|
- lib/wayfarer/routing/route.rb
|
474
|
+
- lib/wayfarer/routing/router.rb
|
467
475
|
- lib/wayfarer/routing/target_route.rb
|
468
476
|
- lib/wayfarer/serializer.rb
|
469
477
|
- lib/wayfarer/stringify.rb
|
@@ -483,10 +491,9 @@ files:
|
|
483
491
|
- spec/config/selenium_spec.rb
|
484
492
|
- spec/config/strconv_spec.rb
|
485
493
|
- spec/config/struct_spec.rb
|
486
|
-
- spec/factories/
|
487
|
-
- spec/factories/
|
488
|
-
- spec/factories/
|
489
|
-
- spec/factories/queue/task.rb
|
494
|
+
- spec/factories/middleware.rb
|
495
|
+
- spec/factories/page.rb
|
496
|
+
- spec/factories/task.rb
|
490
497
|
- spec/fixtures/dummy_job.rb
|
491
498
|
- spec/gc_spec.rb
|
492
499
|
- spec/middleware/chain_spec.rb
|
@@ -522,6 +529,7 @@ files:
|
|
522
529
|
- spec/routing/path_finder_spec.rb
|
523
530
|
- spec/routing/root_route_spec.rb
|
524
531
|
- spec/routing/route_spec.rb
|
532
|
+
- spec/routing/router_spec.rb
|
525
533
|
- spec/spec_helpers.rb
|
526
534
|
- spec/stringify_spec.rb
|
527
535
|
- spec/support/static/finders.html
|
@@ -539,7 +547,7 @@ homepage: http://github.com/bauerd/wayfarer
|
|
539
547
|
licenses:
|
540
548
|
- MIT
|
541
549
|
metadata: {}
|
542
|
-
post_install_message:
|
550
|
+
post_install_message:
|
543
551
|
rdoc_options: []
|
544
552
|
require_paths:
|
545
553
|
- lib
|
@@ -555,7 +563,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
555
563
|
version: '0'
|
556
564
|
requirements: []
|
557
565
|
rubygems_version: 3.1.6
|
558
|
-
signing_key:
|
566
|
+
signing_key:
|
559
567
|
specification_version: 4
|
560
568
|
summary: Versatile web crawling with Ruby
|
561
569
|
test_files: []
|