crabfarm 0.3.5 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/crabfarm/base_navigator.rb +6 -2
- data/lib/crabfarm/base_reducer.rb +3 -18
- data/lib/crabfarm/cli.rb +34 -8
- data/lib/crabfarm/context_factory.rb +2 -2
- data/lib/crabfarm/engines/async_state_manager.rb +1 -1
- data/lib/crabfarm/engines/sync_state_manager.rb +4 -2
- data/lib/crabfarm/modes/console.rb +53 -20
- data/lib/crabfarm/modes/recorder/memento.rb +81 -0
- data/lib/crabfarm/modes/recorder/snapshot.rb +56 -0
- data/lib/crabfarm/modes/server.rb +1 -1
- data/lib/crabfarm/modes/shared/interactive_decorator.rb +71 -0
- data/lib/crabfarm/modes/shared/snapshot_decorator.rb +46 -0
- data/lib/crabfarm/rspec.rb +2 -4
- data/lib/crabfarm/templates/dot_crabfarm.erb +1 -1
- data/lib/crabfarm/templates/navigator.rb.erb +3 -11
- data/lib/crabfarm/templates/reducer.rb.erb +1 -2
- data/lib/crabfarm/transition_service.rb +35 -0
- data/lib/crabfarm/version.rb +1 -1
- metadata +6 -3
- data/lib/crabfarm/modes/recorder.rb +0 -81
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 034f0f2656c2a21910734636aa34f9d7ba22fc02
|
4
|
+
data.tar.gz: f16ff3f1a110bdf38c4d58aea37e7effae1b8ccb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5b5563a6dc4ae2837a28b76eb10bbacb871db117450f065221aae73f314cd708f4ace2b0b06ca71dd0289dcc8e785879f1bd1742c0600b31687328c6b223e6c8
|
7
|
+
data.tar.gz: 2db01f23a912d304dd98046ddbc16406685c41327dbe17db21b5415ee27f355a8905fb6410380656047a670700fdf43f26974ee66175f47128708f864f595e0f
|
@@ -12,7 +12,7 @@ module Crabfarm
|
|
12
12
|
def_delegators '@context', :http
|
13
13
|
def_delegators '@context.store', :get, :fetch
|
14
14
|
|
15
|
-
def initialize(_context,
|
15
|
+
def initialize(_context, _params)
|
16
16
|
@context = _context
|
17
17
|
@params = _params
|
18
18
|
end
|
@@ -60,10 +60,14 @@ module Crabfarm
|
|
60
60
|
end
|
61
61
|
|
62
62
|
reducer = _reducer_class.new _target, @params.merge(_options)
|
63
|
-
reducer
|
63
|
+
execute_reducer reducer
|
64
64
|
reducer
|
65
65
|
end
|
66
66
|
|
67
|
+
def execute_reducer(_reducer)
|
68
|
+
_reducer.run
|
69
|
+
end
|
70
|
+
|
67
71
|
def start_forked_navigation(_name, _value, _block, _mutex)
|
68
72
|
Thread.new {
|
69
73
|
fork = ForkedNavigator.new @context, self, _name, _mutex
|
@@ -4,7 +4,7 @@ module Crabfarm
|
|
4
4
|
class BaseReducer < Delegator
|
5
5
|
include Assertion::Fields
|
6
6
|
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :raw_document, :document, :params
|
8
8
|
|
9
9
|
def self.use_parser(_parser_name)
|
10
10
|
@parser_name = _parser_name
|
@@ -26,8 +26,8 @@ module Crabfarm
|
|
26
26
|
def initialize(_target, _params)
|
27
27
|
reset_fields
|
28
28
|
|
29
|
-
@
|
30
|
-
@document = parser.parse @
|
29
|
+
@raw_document = parser.preprocess_parsing_target _target
|
30
|
+
@document = parser.parse @raw_document
|
31
31
|
@params = _params
|
32
32
|
|
33
33
|
super @document
|
@@ -37,21 +37,6 @@ module Crabfarm
|
|
37
37
|
raise NotImplementedError.new
|
38
38
|
end
|
39
39
|
|
40
|
-
def take_snapshot(_name=nil)
|
41
|
-
file_path = self.class.snapshot_path _name
|
42
|
-
|
43
|
-
dir_path = file_path.split(File::SEPARATOR)[0...-1]
|
44
|
-
FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
|
45
|
-
|
46
|
-
File.write file_path, @parsed_data
|
47
|
-
file_path
|
48
|
-
end
|
49
|
-
|
50
|
-
def take_snapshot_and_fail(_name=nil)
|
51
|
-
file_path = take_snapshot _name
|
52
|
-
raise ArgumentError.new "New snapshot for #{self.class.to_s} generated in '#{file_path}'"
|
53
|
-
end
|
54
|
-
|
55
40
|
def as_json(_options=nil)
|
56
41
|
field_hash
|
57
42
|
end
|
data/lib/crabfarm/cli.rb
CHANGED
@@ -114,7 +114,7 @@ module Crabfarm
|
|
114
114
|
end
|
115
115
|
|
116
116
|
c.desc "Generates a new crabfarm struct"
|
117
|
-
c.command :
|
117
|
+
c.command :struct do |sub|
|
118
118
|
sub.action do |global_options,options,args|
|
119
119
|
next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
|
120
120
|
|
@@ -124,16 +124,42 @@ module Crabfarm
|
|
124
124
|
end
|
125
125
|
end
|
126
126
|
|
127
|
-
desc "
|
127
|
+
desc "Crawler fixture recorders"
|
128
128
|
command [:record, :r] do |c|
|
129
|
-
c.desc "Run recorder in playback mode"
|
130
|
-
c.switch [:p, :playback], :default_value => false
|
131
129
|
|
132
|
-
c.
|
133
|
-
|
130
|
+
c.desc "Perform a memento recording for use in tests"
|
131
|
+
c.command :memento do |memento|
|
132
|
+
|
133
|
+
memento.desc "Run recorder in playback mode"
|
134
|
+
memento.switch [:p, :playback], :default_value => false
|
135
|
+
|
136
|
+
memento.action do |global_options, options, args|
|
137
|
+
next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
|
138
|
+
|
139
|
+
require "crabfarm/modes/recorder/memento"
|
140
|
+
Crabfarm::Modes::Recorder::Memento.start args[0], options[:playback]
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
144
|
+
|
145
|
+
c.desc "Take snapshots from a navigator run"
|
146
|
+
c.command [:snapshot, :snap] do |snap|
|
147
|
+
|
148
|
+
snap.desc "Navigate on a memento"
|
149
|
+
snap.flag [:m, :memento]
|
150
|
+
|
151
|
+
snap.desc "Navigation query string, if not given navigator is run in interactive mode"
|
152
|
+
snap.flag [:q, :query]
|
153
|
+
|
154
|
+
snap.action do |global_options, options, args|
|
155
|
+
next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
|
156
|
+
|
157
|
+
ContextFactory.with_context options[:memento] do |context|
|
158
|
+
require "crabfarm/modes/recorder/snapshot"
|
159
|
+
Crabfarm::Modes::Recorder::Snapshot.start context, args[0], options[:query]
|
160
|
+
end
|
161
|
+
end
|
134
162
|
|
135
|
-
require "crabfarm/modes/recorder"
|
136
|
-
Crabfarm::Modes::Recorder.start args[0], options[:playback]
|
137
163
|
end
|
138
164
|
end
|
139
165
|
|
@@ -21,8 +21,8 @@ module Crabfarm
|
|
21
21
|
|
22
22
|
def load_crabtrap_context(_memento)
|
23
23
|
require 'crabfarm/crabtrap_context'
|
24
|
-
require 'crabfarm/modes/recorder'
|
25
|
-
m_path = Modes::Recorder.memento_path _memento
|
24
|
+
require 'crabfarm/modes/recorder/memento'
|
25
|
+
m_path = Modes::Recorder::Memento.memento_path _memento
|
26
26
|
raise ResourceNotFoundError.new "Could not find memento '#{_name}'" unless File.exists? m_path
|
27
27
|
Crabfarm::CrabtrapContext.new :replay, m_path
|
28
28
|
end
|
@@ -5,12 +5,14 @@ module Crabfarm
|
|
5
5
|
module Engines
|
6
6
|
class SyncStateManager
|
7
7
|
|
8
|
+
attr_reader :context
|
9
|
+
|
8
10
|
def initialize(_context)
|
9
11
|
@context = _context
|
10
12
|
@lock = Mutex.new
|
11
13
|
end
|
12
14
|
|
13
|
-
def reload
|
15
|
+
def reload
|
14
16
|
@lock.synchronize {
|
15
17
|
ActiveSupport::Dependencies.clear
|
16
18
|
@context.reset
|
@@ -23,7 +25,7 @@ module Crabfarm
|
|
23
25
|
}
|
24
26
|
end
|
25
27
|
|
26
|
-
def
|
28
|
+
def navigate(_name, _params={})
|
27
29
|
@lock.synchronize {
|
28
30
|
output = { name: _name, params: _params }
|
29
31
|
|
@@ -6,65 +6,98 @@ require 'crabfarm/engines/sync_state_manager'
|
|
6
6
|
|
7
7
|
module Crabfarm
|
8
8
|
module Modes
|
9
|
-
|
9
|
+
module Console
|
10
10
|
|
11
|
-
|
11
|
+
module Colors
|
12
|
+
NOTICE = '555555'
|
13
|
+
QUESTION = '555555'
|
14
|
+
WARNING = :yellow
|
15
|
+
ERROR = :red
|
16
|
+
RESULT = '00FF00'
|
17
|
+
end
|
18
|
+
|
19
|
+
class ConsoleDsl
|
20
|
+
|
21
|
+
def initialize(_manager)
|
22
|
+
@manager = _manager
|
23
|
+
end
|
12
24
|
|
13
25
|
def reload!
|
14
|
-
puts "Reloading crawler source".color
|
15
|
-
|
26
|
+
puts "Reloading crawler source".color Colors::NOTICE
|
27
|
+
@manager.reload
|
28
|
+
nil
|
16
29
|
end
|
17
30
|
|
18
31
|
def reset
|
19
|
-
puts "Resetting crawling context".color
|
20
|
-
|
32
|
+
puts "Resetting crawling context".color Colors::NOTICE
|
33
|
+
@manager.reset
|
34
|
+
nil
|
21
35
|
end
|
22
36
|
|
23
37
|
def navigate(_name=nil, _params={})
|
24
38
|
if _name.nil?
|
25
|
-
puts "Must provide a navigator name".color
|
39
|
+
puts "Must provide a navigator name".color Colors::ERROR
|
26
40
|
return
|
27
41
|
end
|
28
42
|
|
29
43
|
begin
|
30
|
-
puts "Navigating
|
31
|
-
output =
|
44
|
+
puts "Navigating...".color Colors::NOTICE
|
45
|
+
output = @manager.navigate _name, _params
|
46
|
+
puts JSON.pretty_generate(output.doc).gsub(/(^|\\n)/, ' ').color Colors::RESULT
|
47
|
+
puts "Completed in #{output.elapsed.real} s".color Colors::NOTICE
|
48
|
+
|
49
|
+
rescue Exception => e
|
50
|
+
puts "#{e.to_s}".color Colors::ERROR
|
51
|
+
puts e.backtrace
|
52
|
+
end
|
53
|
+
end
|
32
54
|
|
33
|
-
|
34
|
-
|
35
|
-
puts "
|
55
|
+
def snap(_name=nil, _params={})
|
56
|
+
if _name.nil?
|
57
|
+
puts "Must provide a navigator name".color Colors::ERROR
|
58
|
+
return
|
59
|
+
end
|
60
|
+
|
61
|
+
begin
|
62
|
+
puts "Navigating, waiting to hit a reducer...".color Colors::NOTICE
|
63
|
+
require 'crabfarm/modes/shared/snapshot_decorator'
|
64
|
+
TransitionService.with_navigator_decorator Shared::SnapshotDecorator do
|
65
|
+
@manager.navigate _name, _params
|
66
|
+
end
|
67
|
+
puts "Navigation completed".color Colors::NOTICE
|
36
68
|
|
37
69
|
rescue Exception => e
|
38
|
-
puts "#{e.to_s}".color
|
70
|
+
puts "#{e.to_s}".color Colors::ERROR
|
39
71
|
puts e.backtrace
|
40
72
|
end
|
41
73
|
end
|
42
74
|
|
43
75
|
def help
|
44
76
|
puts "Ejem..."
|
77
|
+
nil
|
45
78
|
end
|
46
79
|
|
47
|
-
alias :
|
48
|
-
alias :r :reset
|
80
|
+
alias :nav :navigate
|
49
81
|
end
|
50
82
|
|
51
83
|
def self.process_input(_context)
|
52
|
-
dsl = ConsoleDsl.new _context
|
84
|
+
dsl = ConsoleDsl.new Engines::SyncStateManager.new _context
|
53
85
|
|
54
86
|
loop do
|
55
87
|
begin
|
56
|
-
dsl.instance_eval Readline.readline("> ", true)
|
88
|
+
output = dsl.instance_eval Readline.readline("> ", true)
|
89
|
+
puts output.inspect unless output.nil?
|
57
90
|
rescue SyntaxError => se
|
58
|
-
puts "Syntax error: #{se.message}".color(
|
91
|
+
puts "Syntax error: #{se.message}".color(Colors::ERROR)
|
59
92
|
rescue SystemExit, Interrupt
|
60
93
|
break
|
61
94
|
rescue => e
|
62
|
-
puts "#{e.to_s}".color(
|
95
|
+
puts "#{e.to_s}".color(Colors::ERROR)
|
63
96
|
puts e.backtrace
|
64
97
|
end
|
65
98
|
end
|
66
99
|
|
67
|
-
puts "Exiting".color(
|
100
|
+
puts "Exiting".color(Colors::NOTICE)
|
68
101
|
end
|
69
102
|
|
70
103
|
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'rainbow'
|
2
|
+
require 'rainbow/ext/string'
|
3
|
+
require 'crabfarm/support/webdriver_factory'
|
4
|
+
require 'crabfarm/crabtrap_runner'
|
5
|
+
|
6
|
+
module Crabfarm
|
7
|
+
module Modes
|
8
|
+
module Recorder
|
9
|
+
module Memento
|
10
|
+
extend self
|
11
|
+
|
12
|
+
def memento_path(_name)
|
13
|
+
File.join(GlobalState.mementos_path, _name + '.json.gz')
|
14
|
+
end
|
15
|
+
|
16
|
+
def start(_target, _replay=false)
|
17
|
+
return puts "Must provide a recording target" unless _target.is_a? String
|
18
|
+
|
19
|
+
target_path = memento_path _target
|
20
|
+
return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
|
21
|
+
|
22
|
+
start_crabtrap _replay, target_path
|
23
|
+
|
24
|
+
begin
|
25
|
+
driver = build_driver Crabfarm.config.recorder_driver
|
26
|
+
return puts "Invalid recorder_driver name '#{Crabfarm.config.recorder_driver}'" if driver.nil?
|
27
|
+
|
28
|
+
begin
|
29
|
+
puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
|
30
|
+
loop do
|
31
|
+
driver.current_url
|
32
|
+
sleep 1.0
|
33
|
+
end
|
34
|
+
rescue Exception => e
|
35
|
+
# noop
|
36
|
+
end
|
37
|
+
|
38
|
+
puts "Releasing crawling context".color(:green)
|
39
|
+
driver.quit rescue nil
|
40
|
+
ensure
|
41
|
+
crabtrap.stop
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def start_crabtrap(_replay, _target_path)
|
48
|
+
crabtrap_config = Crabfarm.config.crabtrap_config
|
49
|
+
crabtrap_config[:mode] = _replay ? :replay : :capture
|
50
|
+
crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
|
51
|
+
crabtrap_config[:bucket_path] = _target_path
|
52
|
+
|
53
|
+
@crabtrap = CrabtrapRunner.new crabtrap_config
|
54
|
+
@crabtrap.start
|
55
|
+
end
|
56
|
+
|
57
|
+
def crabtrap
|
58
|
+
@crabtrap
|
59
|
+
end
|
60
|
+
|
61
|
+
def build_driver(_name)
|
62
|
+
case _name.to_sym
|
63
|
+
when :firefox
|
64
|
+
Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
|
65
|
+
when :chrome
|
66
|
+
Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
|
67
|
+
else return nil end
|
68
|
+
end
|
69
|
+
|
70
|
+
def driver_config
|
71
|
+
{
|
72
|
+
proxy: "127.0.0.1:#{crabtrap.port}",
|
73
|
+
window_width: Crabfarm.config.webdriver_window_width,
|
74
|
+
window_height: Crabfarm.config.webdriver_window_height
|
75
|
+
}
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'rainbow'
|
2
|
+
require 'rainbow/ext/string'
|
3
|
+
require 'crabfarm/modes/console'
|
4
|
+
require 'crabfarm/modes/shared/snapshot_decorator'
|
5
|
+
require 'crabfarm/modes/shared/interactive_decorator'
|
6
|
+
|
7
|
+
module Crabfarm
|
8
|
+
module Modes
|
9
|
+
module Recorder
|
10
|
+
module Snapshot
|
11
|
+
extend self
|
12
|
+
|
13
|
+
def start(_context, _navigator, _query=nil)
|
14
|
+
return puts "Must provide a navigator name" unless _navigator.is_a? String
|
15
|
+
|
16
|
+
begin
|
17
|
+
puts "Navigating, waiting to hit a reducer...".color(Console::Colors::NOTICE)
|
18
|
+
service.with_navigator_decorator Shared::SnapshotDecorator do
|
19
|
+
if _query.nil?
|
20
|
+
service.with_navigator_decorator Shared::InteractiveDecorator do
|
21
|
+
service.transition _context, _navigator
|
22
|
+
end
|
23
|
+
else
|
24
|
+
_query = parse_query_string _query
|
25
|
+
service.transition _context, _navigator, _query
|
26
|
+
end
|
27
|
+
end
|
28
|
+
puts "Navigation completed".color(Console::Colors::NOTICE)
|
29
|
+
rescue Exception => e
|
30
|
+
puts "#{e.to_s}".color Console::Colors::ERROR
|
31
|
+
puts e.backtrace
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def service
|
38
|
+
TransitionService
|
39
|
+
end
|
40
|
+
|
41
|
+
def parse_query_string(_string)
|
42
|
+
result = {}
|
43
|
+
|
44
|
+
parts = _string.split '&'
|
45
|
+
parts.each do |part|
|
46
|
+
key, val = part.split '='
|
47
|
+
result[key.to_sym] = Shared::InteractiveDecorator::InteractiveHash.parse_input val
|
48
|
+
end
|
49
|
+
|
50
|
+
result
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'inquirer'
|
2
|
+
require 'crabfarm/modes/console'
|
3
|
+
|
4
|
+
module Crabfarm
|
5
|
+
module Modes
|
6
|
+
module Shared
|
7
|
+
module InteractiveDecorator
|
8
|
+
|
9
|
+
module Colors
|
10
|
+
include Crabfarm::Modes::Console::Colors
|
11
|
+
end
|
12
|
+
|
13
|
+
class InteractiveHash < Hash
|
14
|
+
|
15
|
+
def self.parse_input(_val)
|
16
|
+
case _val
|
17
|
+
when 'true' then true
|
18
|
+
when 'false' then false
|
19
|
+
when 'nil' then nil
|
20
|
+
when /^\d+$/ then _val.to_i
|
21
|
+
when /^\d*\.\d+$/ then _val.to_f
|
22
|
+
when /^:[^\s]+$/ then _val.to_sym
|
23
|
+
when /^\'.*?\'$/ then _val[1...-1]
|
24
|
+
when /^\".*?\"$/ then _val[1...-1]
|
25
|
+
else _val end
|
26
|
+
end
|
27
|
+
|
28
|
+
['[]', :fetch, :has_key?, :key?, :include?].each do |method|
|
29
|
+
define_method method do |*args|
|
30
|
+
cache_value args[0]
|
31
|
+
super(*args)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def merge(*args)
|
36
|
+
InteractiveHash.new.merge! super
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def cache
|
42
|
+
@cache ||= self.to_h
|
43
|
+
end
|
44
|
+
|
45
|
+
def cache_value(_key)
|
46
|
+
unless cache.key? _key
|
47
|
+
value = cache[_key] = Ask.input("Value for '#{_key}'? (blank to skip)".color Colors::QUESTION).strip
|
48
|
+
self[_key] = parse_input value unless value.empty?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def parse_input(*_args)
|
53
|
+
self.class.parse_input(*_args)
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.decorate(_navigator)
|
59
|
+
|
60
|
+
def _navigator.params
|
61
|
+
@i_params ||= InteractiveHash.new.merge! @params
|
62
|
+
end
|
63
|
+
|
64
|
+
_navigator
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'inquirer'
|
2
|
+
require 'crabfarm/modes/console'
|
3
|
+
|
4
|
+
module Crabfarm
|
5
|
+
module Modes
|
6
|
+
module Shared
|
7
|
+
module SnapshotDecorator
|
8
|
+
|
9
|
+
module Colors
|
10
|
+
include Crabfarm::Modes::Console::Colors
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.decorate(_navigator)
|
14
|
+
|
15
|
+
def _navigator.execute_reducer(_reducer)
|
16
|
+
loop do
|
17
|
+
name = Ask.input "-- Name for #{_reducer.class.to_s} snapshot (blank to skip)".color Colors::QUESTION
|
18
|
+
if name.empty?
|
19
|
+
puts "-- Skipping snapshot".color Colors::WARNING
|
20
|
+
break
|
21
|
+
else
|
22
|
+
file_path = _reducer.class.snapshot_path name
|
23
|
+
|
24
|
+
if File.exist? file_path
|
25
|
+
puts "-- Could not save snapshot, file already exist!".color Colors::ERROR
|
26
|
+
else
|
27
|
+
dir_path = file_path.split(File::SEPARATOR)[0...-1]
|
28
|
+
FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
|
29
|
+
File.write file_path, _reducer.raw_document
|
30
|
+
puts "-- Snapshot written to #{file_path}".color Colors::RESULT
|
31
|
+
break
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
_reducer.run
|
37
|
+
end
|
38
|
+
|
39
|
+
_navigator
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/crabfarm/rspec.rb
CHANGED
@@ -14,7 +14,7 @@ module Crabfarm
|
|
14
14
|
end
|
15
15
|
|
16
16
|
snapshot_path = described_class.snapshot_path _snapshot
|
17
|
-
raise Error.new "Snapshot does not exist #{
|
17
|
+
raise Error.new "Snapshot does not exist #{snapshot_path}" unless File.exist? snapshot_path
|
18
18
|
|
19
19
|
data = File.read snapshot_path
|
20
20
|
reducer = described_class.new data, _options
|
@@ -43,8 +43,6 @@ module Crabfarm
|
|
43
43
|
@state ||= navigate
|
44
44
|
end
|
45
45
|
|
46
|
-
alias :navigator :state
|
47
|
-
|
48
46
|
def last_state
|
49
47
|
@last_state
|
50
48
|
end
|
@@ -54,7 +52,7 @@ module Crabfarm
|
|
54
52
|
end
|
55
53
|
|
56
54
|
def browser(_session_id=nil)
|
57
|
-
@context.pool.
|
55
|
+
@context.pool.driver(_session_id)
|
58
56
|
end
|
59
57
|
|
60
58
|
end
|
@@ -1,18 +1,10 @@
|
|
1
1
|
class <%= navigator_class %> < Crabfarm::BaseNavigator
|
2
2
|
|
3
3
|
def run
|
4
|
-
<%
|
5
|
-
|
6
|
-
|
7
|
-
<% else %>
|
8
|
-
browser.goto '<%= navigator_url %>'
|
9
|
-
<% end %>
|
4
|
+
<% unless navigator_url.nil? %>browser.goto '<%= navigator_url %>'
|
5
|
+
|
6
|
+
<% end %># Add some navigation code here.
|
10
7
|
|
11
|
-
# Call the homonymous reducer over the entire document and output it by default
|
12
|
-
# You can call other reducers by using the `reduce` method, like this:
|
13
|
-
#
|
14
|
-
# reduce browser.search('td').first, using: MyOtherReducer
|
15
|
-
#
|
16
8
|
reduce_with_defaults
|
17
9
|
end
|
18
10
|
|
@@ -1,6 +1,19 @@
|
|
1
1
|
module Crabfarm
|
2
2
|
class TransitionService
|
3
3
|
|
4
|
+
def self.with_navigator_decorator(_decorator)
|
5
|
+
@decorator = DecoratorChain.new @decorator, _decorator
|
6
|
+
begin
|
7
|
+
yield
|
8
|
+
ensure
|
9
|
+
@decorator = @decorator.base
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.current_decorator
|
14
|
+
@decorator
|
15
|
+
end
|
16
|
+
|
4
17
|
def self.transition(_context, _name, _params={})
|
5
18
|
self.new(_context).transition(_name, _params)
|
6
19
|
end
|
@@ -18,6 +31,7 @@ module Crabfarm
|
|
18
31
|
|
19
32
|
@context.prepare
|
20
33
|
@navigator = navigator_class.new @context, _params
|
34
|
+
@navigator = current_decorator.decorate @navigator unless current_decorator.nil?
|
21
35
|
|
22
36
|
@document = @navigator.run
|
23
37
|
@document = @document.as_json if @document.respond_to? :as_json
|
@@ -27,10 +41,31 @@ module Crabfarm
|
|
27
41
|
|
28
42
|
private
|
29
43
|
|
44
|
+
def current_decorator
|
45
|
+
self.class.current_decorator
|
46
|
+
end
|
47
|
+
|
30
48
|
def load_class_from_uri(_uri)
|
31
49
|
class_name = Utils::Naming.decode_crabfarm_uri _uri
|
32
50
|
class_name.constantize
|
33
51
|
end
|
34
52
|
|
53
|
+
class DecoratorChain
|
54
|
+
|
55
|
+
attr_reader :base, :new
|
56
|
+
|
57
|
+
def initialize(_base, _new)
|
58
|
+
@base = _base
|
59
|
+
@new = _new
|
60
|
+
end
|
61
|
+
|
62
|
+
def decorate(_navigator)
|
63
|
+
_navigator = @new.decorate _navigator
|
64
|
+
return _navigator if @base.nil?
|
65
|
+
@base.decorate _navigator
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
|
35
70
|
end
|
36
71
|
end
|
data/lib/crabfarm/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crabfarm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ignacio Baixas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-04-
|
11
|
+
date: 2015-04-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -412,8 +412,11 @@ files:
|
|
412
412
|
- lib/crabfarm/modes/console.rb
|
413
413
|
- lib/crabfarm/modes/generator.rb
|
414
414
|
- lib/crabfarm/modes/publisher.rb
|
415
|
-
- lib/crabfarm/modes/recorder.rb
|
415
|
+
- lib/crabfarm/modes/recorder/memento.rb
|
416
|
+
- lib/crabfarm/modes/recorder/snapshot.rb
|
416
417
|
- lib/crabfarm/modes/server.rb
|
418
|
+
- lib/crabfarm/modes/shared/interactive_decorator.rb
|
419
|
+
- lib/crabfarm/modes/shared/snapshot_decorator.rb
|
417
420
|
- lib/crabfarm/phantom_runner.rb
|
418
421
|
- lib/crabfarm/rspec.rb
|
419
422
|
- lib/crabfarm/state_store.rb
|
@@ -1,81 +0,0 @@
|
|
1
|
-
require 'rainbow'
|
2
|
-
require 'rainbow/ext/string'
|
3
|
-
require 'crabfarm/support/webdriver_factory'
|
4
|
-
require 'crabfarm/crabtrap_runner'
|
5
|
-
|
6
|
-
module Crabfarm
|
7
|
-
module Modes
|
8
|
-
module Recorder
|
9
|
-
extend self
|
10
|
-
|
11
|
-
def memento_path(_name)
|
12
|
-
File.join(GlobalState.mementos_path, _name + '.json.gz')
|
13
|
-
end
|
14
|
-
|
15
|
-
def start(_target, _replay=false)
|
16
|
-
return puts "Must provide a recording target" unless _target.is_a? String
|
17
|
-
|
18
|
-
target_path = memento_path _target
|
19
|
-
return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
|
20
|
-
|
21
|
-
start_crabtrap _replay, target_path
|
22
|
-
|
23
|
-
begin
|
24
|
-
driver = build_driver Crabfarm.config.recorder_driver
|
25
|
-
return puts "Invalid recorder_driver name '#{Crabfarm.config.recorder_driver}'" if driver.nil?
|
26
|
-
|
27
|
-
begin
|
28
|
-
puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
|
29
|
-
loop do
|
30
|
-
driver.current_url
|
31
|
-
sleep 1.0
|
32
|
-
end
|
33
|
-
rescue Exception => e
|
34
|
-
# noop
|
35
|
-
end
|
36
|
-
|
37
|
-
puts "Releasing crawling context".color(:green)
|
38
|
-
driver.quit rescue nil
|
39
|
-
ensure
|
40
|
-
crabtrap.stop
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
private
|
45
|
-
|
46
|
-
def start_crabtrap(_replay, _target_path)
|
47
|
-
crabtrap_config = Crabfarm.config.crabtrap_config
|
48
|
-
crabtrap_config[:mode] = _replay ? :replay : :capture
|
49
|
-
crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
|
50
|
-
crabtrap_config[:bucket_path] = _target_path
|
51
|
-
|
52
|
-
@crabtrap = CrabtrapRunner.new crabtrap_config
|
53
|
-
@crabtrap.start
|
54
|
-
end
|
55
|
-
|
56
|
-
def crabtrap
|
57
|
-
@crabtrap
|
58
|
-
end
|
59
|
-
|
60
|
-
def build_driver(_name)
|
61
|
-
case _name.to_sym
|
62
|
-
when :firefox
|
63
|
-
Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
|
64
|
-
when :chrome
|
65
|
-
Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
|
66
|
-
else return nil end
|
67
|
-
end
|
68
|
-
|
69
|
-
def driver_config
|
70
|
-
{
|
71
|
-
proxy: "127.0.0.1:#{crabtrap.port}",
|
72
|
-
window_width: Crabfarm.config.webdriver_window_width,
|
73
|
-
window_height: Crabfarm.config.webdriver_window_height
|
74
|
-
}
|
75
|
-
end
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|