crabfarm 0.3.5 → 0.3.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/crabfarm/base_navigator.rb +6 -2
- data/lib/crabfarm/base_reducer.rb +3 -18
- data/lib/crabfarm/cli.rb +34 -8
- data/lib/crabfarm/context_factory.rb +2 -2
- data/lib/crabfarm/engines/async_state_manager.rb +1 -1
- data/lib/crabfarm/engines/sync_state_manager.rb +4 -2
- data/lib/crabfarm/modes/console.rb +53 -20
- data/lib/crabfarm/modes/recorder/memento.rb +81 -0
- data/lib/crabfarm/modes/recorder/snapshot.rb +56 -0
- data/lib/crabfarm/modes/server.rb +1 -1
- data/lib/crabfarm/modes/shared/interactive_decorator.rb +71 -0
- data/lib/crabfarm/modes/shared/snapshot_decorator.rb +46 -0
- data/lib/crabfarm/rspec.rb +2 -4
- data/lib/crabfarm/templates/dot_crabfarm.erb +1 -1
- data/lib/crabfarm/templates/navigator.rb.erb +3 -11
- data/lib/crabfarm/templates/reducer.rb.erb +1 -2
- data/lib/crabfarm/transition_service.rb +35 -0
- data/lib/crabfarm/version.rb +1 -1
- metadata +6 -3
- data/lib/crabfarm/modes/recorder.rb +0 -81
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 034f0f2656c2a21910734636aa34f9d7ba22fc02
|
4
|
+
data.tar.gz: f16ff3f1a110bdf38c4d58aea37e7effae1b8ccb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5b5563a6dc4ae2837a28b76eb10bbacb871db117450f065221aae73f314cd708f4ace2b0b06ca71dd0289dcc8e785879f1bd1742c0600b31687328c6b223e6c8
|
7
|
+
data.tar.gz: 2db01f23a912d304dd98046ddbc16406685c41327dbe17db21b5415ee27f355a8905fb6410380656047a670700fdf43f26974ee66175f47128708f864f595e0f
|
@@ -12,7 +12,7 @@ module Crabfarm
|
|
12
12
|
def_delegators '@context', :http
|
13
13
|
def_delegators '@context.store', :get, :fetch
|
14
14
|
|
15
|
-
def initialize(_context,
|
15
|
+
def initialize(_context, _params)
|
16
16
|
@context = _context
|
17
17
|
@params = _params
|
18
18
|
end
|
@@ -60,10 +60,14 @@ module Crabfarm
|
|
60
60
|
end
|
61
61
|
|
62
62
|
reducer = _reducer_class.new _target, @params.merge(_options)
|
63
|
-
reducer
|
63
|
+
execute_reducer reducer
|
64
64
|
reducer
|
65
65
|
end
|
66
66
|
|
67
|
+
def execute_reducer(_reducer)
|
68
|
+
_reducer.run
|
69
|
+
end
|
70
|
+
|
67
71
|
def start_forked_navigation(_name, _value, _block, _mutex)
|
68
72
|
Thread.new {
|
69
73
|
fork = ForkedNavigator.new @context, self, _name, _mutex
|
@@ -4,7 +4,7 @@ module Crabfarm
|
|
4
4
|
class BaseReducer < Delegator
|
5
5
|
include Assertion::Fields
|
6
6
|
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :raw_document, :document, :params
|
8
8
|
|
9
9
|
def self.use_parser(_parser_name)
|
10
10
|
@parser_name = _parser_name
|
@@ -26,8 +26,8 @@ module Crabfarm
|
|
26
26
|
def initialize(_target, _params)
|
27
27
|
reset_fields
|
28
28
|
|
29
|
-
@
|
30
|
-
@document = parser.parse @
|
29
|
+
@raw_document = parser.preprocess_parsing_target _target
|
30
|
+
@document = parser.parse @raw_document
|
31
31
|
@params = _params
|
32
32
|
|
33
33
|
super @document
|
@@ -37,21 +37,6 @@ module Crabfarm
|
|
37
37
|
raise NotImplementedError.new
|
38
38
|
end
|
39
39
|
|
40
|
-
def take_snapshot(_name=nil)
|
41
|
-
file_path = self.class.snapshot_path _name
|
42
|
-
|
43
|
-
dir_path = file_path.split(File::SEPARATOR)[0...-1]
|
44
|
-
FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
|
45
|
-
|
46
|
-
File.write file_path, @parsed_data
|
47
|
-
file_path
|
48
|
-
end
|
49
|
-
|
50
|
-
def take_snapshot_and_fail(_name=nil)
|
51
|
-
file_path = take_snapshot _name
|
52
|
-
raise ArgumentError.new "New snapshot for #{self.class.to_s} generated in '#{file_path}'"
|
53
|
-
end
|
54
|
-
|
55
40
|
def as_json(_options=nil)
|
56
41
|
field_hash
|
57
42
|
end
|
data/lib/crabfarm/cli.rb
CHANGED
@@ -114,7 +114,7 @@ module Crabfarm
|
|
114
114
|
end
|
115
115
|
|
116
116
|
c.desc "Generates a new crabfarm struct"
|
117
|
-
c.command :
|
117
|
+
c.command :struct do |sub|
|
118
118
|
sub.action do |global_options,options,args|
|
119
119
|
next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
|
120
120
|
|
@@ -124,16 +124,42 @@ module Crabfarm
|
|
124
124
|
end
|
125
125
|
end
|
126
126
|
|
127
|
-
desc "
|
127
|
+
desc "Crawler fixture recorders"
|
128
128
|
command [:record, :r] do |c|
|
129
|
-
c.desc "Run recorder in playback mode"
|
130
|
-
c.switch [:p, :playback], :default_value => false
|
131
129
|
|
132
|
-
c.
|
133
|
-
|
130
|
+
c.desc "Perform a memento recording for use in tests"
|
131
|
+
c.command :memento do |memento|
|
132
|
+
|
133
|
+
memento.desc "Run recorder in playback mode"
|
134
|
+
memento.switch [:p, :playback], :default_value => false
|
135
|
+
|
136
|
+
memento.action do |global_options, options, args|
|
137
|
+
next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
|
138
|
+
|
139
|
+
require "crabfarm/modes/recorder/memento"
|
140
|
+
Crabfarm::Modes::Recorder::Memento.start args[0], options[:playback]
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
144
|
+
|
145
|
+
c.desc "Take snapshots from a navigator run"
|
146
|
+
c.command [:snapshot, :snap] do |snap|
|
147
|
+
|
148
|
+
snap.desc "Navigate on a memento"
|
149
|
+
snap.flag [:m, :memento]
|
150
|
+
|
151
|
+
snap.desc "Navigation query string, if not given navigator is run in interactive mode"
|
152
|
+
snap.flag [:q, :query]
|
153
|
+
|
154
|
+
snap.action do |global_options, options, args|
|
155
|
+
next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
|
156
|
+
|
157
|
+
ContextFactory.with_context options[:memento] do |context|
|
158
|
+
require "crabfarm/modes/recorder/snapshot"
|
159
|
+
Crabfarm::Modes::Recorder::Snapshot.start context, args[0], options[:query]
|
160
|
+
end
|
161
|
+
end
|
134
162
|
|
135
|
-
require "crabfarm/modes/recorder"
|
136
|
-
Crabfarm::Modes::Recorder.start args[0], options[:playback]
|
137
163
|
end
|
138
164
|
end
|
139
165
|
|
@@ -21,8 +21,8 @@ module Crabfarm
|
|
21
21
|
|
22
22
|
def load_crabtrap_context(_memento)
|
23
23
|
require 'crabfarm/crabtrap_context'
|
24
|
-
require 'crabfarm/modes/recorder'
|
25
|
-
m_path = Modes::Recorder.memento_path _memento
|
24
|
+
require 'crabfarm/modes/recorder/memento'
|
25
|
+
m_path = Modes::Recorder::Memento.memento_path _memento
|
26
26
|
raise ResourceNotFoundError.new "Could not find memento '#{_name}'" unless File.exists? m_path
|
27
27
|
Crabfarm::CrabtrapContext.new :replay, m_path
|
28
28
|
end
|
@@ -5,12 +5,14 @@ module Crabfarm
|
|
5
5
|
module Engines
|
6
6
|
class SyncStateManager
|
7
7
|
|
8
|
+
attr_reader :context
|
9
|
+
|
8
10
|
def initialize(_context)
|
9
11
|
@context = _context
|
10
12
|
@lock = Mutex.new
|
11
13
|
end
|
12
14
|
|
13
|
-
def reload
|
15
|
+
def reload
|
14
16
|
@lock.synchronize {
|
15
17
|
ActiveSupport::Dependencies.clear
|
16
18
|
@context.reset
|
@@ -23,7 +25,7 @@ module Crabfarm
|
|
23
25
|
}
|
24
26
|
end
|
25
27
|
|
26
|
-
def
|
28
|
+
def navigate(_name, _params={})
|
27
29
|
@lock.synchronize {
|
28
30
|
output = { name: _name, params: _params }
|
29
31
|
|
@@ -6,65 +6,98 @@ require 'crabfarm/engines/sync_state_manager'
|
|
6
6
|
|
7
7
|
module Crabfarm
|
8
8
|
module Modes
|
9
|
-
|
9
|
+
module Console
|
10
10
|
|
11
|
-
|
11
|
+
module Colors
|
12
|
+
NOTICE = '555555'
|
13
|
+
QUESTION = '555555'
|
14
|
+
WARNING = :yellow
|
15
|
+
ERROR = :red
|
16
|
+
RESULT = '00FF00'
|
17
|
+
end
|
18
|
+
|
19
|
+
class ConsoleDsl
|
20
|
+
|
21
|
+
def initialize(_manager)
|
22
|
+
@manager = _manager
|
23
|
+
end
|
12
24
|
|
13
25
|
def reload!
|
14
|
-
puts "Reloading crawler source".color
|
15
|
-
|
26
|
+
puts "Reloading crawler source".color Colors::NOTICE
|
27
|
+
@manager.reload
|
28
|
+
nil
|
16
29
|
end
|
17
30
|
|
18
31
|
def reset
|
19
|
-
puts "Resetting crawling context".color
|
20
|
-
|
32
|
+
puts "Resetting crawling context".color Colors::NOTICE
|
33
|
+
@manager.reset
|
34
|
+
nil
|
21
35
|
end
|
22
36
|
|
23
37
|
def navigate(_name=nil, _params={})
|
24
38
|
if _name.nil?
|
25
|
-
puts "Must provide a navigator name".color
|
39
|
+
puts "Must provide a navigator name".color Colors::ERROR
|
26
40
|
return
|
27
41
|
end
|
28
42
|
|
29
43
|
begin
|
30
|
-
puts "Navigating
|
31
|
-
output =
|
44
|
+
puts "Navigating...".color Colors::NOTICE
|
45
|
+
output = @manager.navigate _name, _params
|
46
|
+
puts JSON.pretty_generate(output.doc).gsub(/(^|\\n)/, ' ').color Colors::RESULT
|
47
|
+
puts "Completed in #{output.elapsed.real} s".color Colors::NOTICE
|
48
|
+
|
49
|
+
rescue Exception => e
|
50
|
+
puts "#{e.to_s}".color Colors::ERROR
|
51
|
+
puts e.backtrace
|
52
|
+
end
|
53
|
+
end
|
32
54
|
|
33
|
-
|
34
|
-
|
35
|
-
puts "
|
55
|
+
def snap(_name=nil, _params={})
|
56
|
+
if _name.nil?
|
57
|
+
puts "Must provide a navigator name".color Colors::ERROR
|
58
|
+
return
|
59
|
+
end
|
60
|
+
|
61
|
+
begin
|
62
|
+
puts "Navigating, waiting to hit a reducer...".color Colors::NOTICE
|
63
|
+
require 'crabfarm/modes/shared/snapshot_decorator'
|
64
|
+
TransitionService.with_navigator_decorator Shared::SnapshotDecorator do
|
65
|
+
@manager.navigate _name, _params
|
66
|
+
end
|
67
|
+
puts "Navigation completed".color Colors::NOTICE
|
36
68
|
|
37
69
|
rescue Exception => e
|
38
|
-
puts "#{e.to_s}".color
|
70
|
+
puts "#{e.to_s}".color Colors::ERROR
|
39
71
|
puts e.backtrace
|
40
72
|
end
|
41
73
|
end
|
42
74
|
|
43
75
|
def help
|
44
76
|
puts "Ejem..."
|
77
|
+
nil
|
45
78
|
end
|
46
79
|
|
47
|
-
alias :
|
48
|
-
alias :r :reset
|
80
|
+
alias :nav :navigate
|
49
81
|
end
|
50
82
|
|
51
83
|
def self.process_input(_context)
|
52
|
-
dsl = ConsoleDsl.new _context
|
84
|
+
dsl = ConsoleDsl.new Engines::SyncStateManager.new _context
|
53
85
|
|
54
86
|
loop do
|
55
87
|
begin
|
56
|
-
dsl.instance_eval Readline.readline("> ", true)
|
88
|
+
output = dsl.instance_eval Readline.readline("> ", true)
|
89
|
+
puts output.inspect unless output.nil?
|
57
90
|
rescue SyntaxError => se
|
58
|
-
puts "Syntax error: #{se.message}".color(
|
91
|
+
puts "Syntax error: #{se.message}".color(Colors::ERROR)
|
59
92
|
rescue SystemExit, Interrupt
|
60
93
|
break
|
61
94
|
rescue => e
|
62
|
-
puts "#{e.to_s}".color(
|
95
|
+
puts "#{e.to_s}".color(Colors::ERROR)
|
63
96
|
puts e.backtrace
|
64
97
|
end
|
65
98
|
end
|
66
99
|
|
67
|
-
puts "Exiting".color(
|
100
|
+
puts "Exiting".color(Colors::NOTICE)
|
68
101
|
end
|
69
102
|
|
70
103
|
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'rainbow'
|
2
|
+
require 'rainbow/ext/string'
|
3
|
+
require 'crabfarm/support/webdriver_factory'
|
4
|
+
require 'crabfarm/crabtrap_runner'
|
5
|
+
|
6
|
+
module Crabfarm
|
7
|
+
module Modes
|
8
|
+
module Recorder
|
9
|
+
module Memento
|
10
|
+
extend self
|
11
|
+
|
12
|
+
def memento_path(_name)
|
13
|
+
File.join(GlobalState.mementos_path, _name + '.json.gz')
|
14
|
+
end
|
15
|
+
|
16
|
+
def start(_target, _replay=false)
|
17
|
+
return puts "Must provide a recording target" unless _target.is_a? String
|
18
|
+
|
19
|
+
target_path = memento_path _target
|
20
|
+
return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
|
21
|
+
|
22
|
+
start_crabtrap _replay, target_path
|
23
|
+
|
24
|
+
begin
|
25
|
+
driver = build_driver Crabfarm.config.recorder_driver
|
26
|
+
return puts "Invalid recorder_driver name '#{Crabfarm.config.recorder_driver}'" if driver.nil?
|
27
|
+
|
28
|
+
begin
|
29
|
+
puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
|
30
|
+
loop do
|
31
|
+
driver.current_url
|
32
|
+
sleep 1.0
|
33
|
+
end
|
34
|
+
rescue Exception => e
|
35
|
+
# noop
|
36
|
+
end
|
37
|
+
|
38
|
+
puts "Releasing crawling context".color(:green)
|
39
|
+
driver.quit rescue nil
|
40
|
+
ensure
|
41
|
+
crabtrap.stop
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def start_crabtrap(_replay, _target_path)
|
48
|
+
crabtrap_config = Crabfarm.config.crabtrap_config
|
49
|
+
crabtrap_config[:mode] = _replay ? :replay : :capture
|
50
|
+
crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
|
51
|
+
crabtrap_config[:bucket_path] = _target_path
|
52
|
+
|
53
|
+
@crabtrap = CrabtrapRunner.new crabtrap_config
|
54
|
+
@crabtrap.start
|
55
|
+
end
|
56
|
+
|
57
|
+
def crabtrap
|
58
|
+
@crabtrap
|
59
|
+
end
|
60
|
+
|
61
|
+
def build_driver(_name)
|
62
|
+
case _name.to_sym
|
63
|
+
when :firefox
|
64
|
+
Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
|
65
|
+
when :chrome
|
66
|
+
Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
|
67
|
+
else return nil end
|
68
|
+
end
|
69
|
+
|
70
|
+
def driver_config
|
71
|
+
{
|
72
|
+
proxy: "127.0.0.1:#{crabtrap.port}",
|
73
|
+
window_width: Crabfarm.config.webdriver_window_width,
|
74
|
+
window_height: Crabfarm.config.webdriver_window_height
|
75
|
+
}
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'rainbow'
|
2
|
+
require 'rainbow/ext/string'
|
3
|
+
require 'crabfarm/modes/console'
|
4
|
+
require 'crabfarm/modes/shared/snapshot_decorator'
|
5
|
+
require 'crabfarm/modes/shared/interactive_decorator'
|
6
|
+
|
7
|
+
module Crabfarm
|
8
|
+
module Modes
|
9
|
+
module Recorder
|
10
|
+
module Snapshot
|
11
|
+
extend self
|
12
|
+
|
13
|
+
def start(_context, _navigator, _query=nil)
|
14
|
+
return puts "Must provide a navigator name" unless _navigator.is_a? String
|
15
|
+
|
16
|
+
begin
|
17
|
+
puts "Navigating, waiting to hit a reducer...".color(Console::Colors::NOTICE)
|
18
|
+
service.with_navigator_decorator Shared::SnapshotDecorator do
|
19
|
+
if _query.nil?
|
20
|
+
service.with_navigator_decorator Shared::InteractiveDecorator do
|
21
|
+
service.transition _context, _navigator
|
22
|
+
end
|
23
|
+
else
|
24
|
+
_query = parse_query_string _query
|
25
|
+
service.transition _context, _navigator, _query
|
26
|
+
end
|
27
|
+
end
|
28
|
+
puts "Navigation completed".color(Console::Colors::NOTICE)
|
29
|
+
rescue Exception => e
|
30
|
+
puts "#{e.to_s}".color Console::Colors::ERROR
|
31
|
+
puts e.backtrace
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def service
|
38
|
+
TransitionService
|
39
|
+
end
|
40
|
+
|
41
|
+
def parse_query_string(_string)
|
42
|
+
result = {}
|
43
|
+
|
44
|
+
parts = _string.split '&'
|
45
|
+
parts.each do |part|
|
46
|
+
key, val = part.split '='
|
47
|
+
result[key.to_sym] = Shared::InteractiveDecorator::InteractiveHash.parse_input val
|
48
|
+
end
|
49
|
+
|
50
|
+
result
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'inquirer'
|
2
|
+
require 'crabfarm/modes/console'
|
3
|
+
|
4
|
+
module Crabfarm
|
5
|
+
module Modes
|
6
|
+
module Shared
|
7
|
+
module InteractiveDecorator
|
8
|
+
|
9
|
+
module Colors
|
10
|
+
include Crabfarm::Modes::Console::Colors
|
11
|
+
end
|
12
|
+
|
13
|
+
class InteractiveHash < Hash
|
14
|
+
|
15
|
+
def self.parse_input(_val)
|
16
|
+
case _val
|
17
|
+
when 'true' then true
|
18
|
+
when 'false' then false
|
19
|
+
when 'nil' then nil
|
20
|
+
when /^\d+$/ then _val.to_i
|
21
|
+
when /^\d*\.\d+$/ then _val.to_f
|
22
|
+
when /^:[^\s]+$/ then _val.to_sym
|
23
|
+
when /^\'.*?\'$/ then _val[1...-1]
|
24
|
+
when /^\".*?\"$/ then _val[1...-1]
|
25
|
+
else _val end
|
26
|
+
end
|
27
|
+
|
28
|
+
['[]', :fetch, :has_key?, :key?, :include?].each do |method|
|
29
|
+
define_method method do |*args|
|
30
|
+
cache_value args[0]
|
31
|
+
super(*args)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def merge(*args)
|
36
|
+
InteractiveHash.new.merge! super
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def cache
|
42
|
+
@cache ||= self.to_h
|
43
|
+
end
|
44
|
+
|
45
|
+
def cache_value(_key)
|
46
|
+
unless cache.key? _key
|
47
|
+
value = cache[_key] = Ask.input("Value for '#{_key}'? (blank to skip)".color Colors::QUESTION).strip
|
48
|
+
self[_key] = parse_input value unless value.empty?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def parse_input(*_args)
|
53
|
+
self.class.parse_input(*_args)
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.decorate(_navigator)
|
59
|
+
|
60
|
+
def _navigator.params
|
61
|
+
@i_params ||= InteractiveHash.new.merge! @params
|
62
|
+
end
|
63
|
+
|
64
|
+
_navigator
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'inquirer'
|
2
|
+
require 'crabfarm/modes/console'
|
3
|
+
|
4
|
+
module Crabfarm
|
5
|
+
module Modes
|
6
|
+
module Shared
|
7
|
+
module SnapshotDecorator
|
8
|
+
|
9
|
+
module Colors
|
10
|
+
include Crabfarm::Modes::Console::Colors
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.decorate(_navigator)
|
14
|
+
|
15
|
+
def _navigator.execute_reducer(_reducer)
|
16
|
+
loop do
|
17
|
+
name = Ask.input "-- Name for #{_reducer.class.to_s} snapshot (blank to skip)".color Colors::QUESTION
|
18
|
+
if name.empty?
|
19
|
+
puts "-- Skipping snapshot".color Colors::WARNING
|
20
|
+
break
|
21
|
+
else
|
22
|
+
file_path = _reducer.class.snapshot_path name
|
23
|
+
|
24
|
+
if File.exist? file_path
|
25
|
+
puts "-- Could not save snapshot, file already exist!".color Colors::ERROR
|
26
|
+
else
|
27
|
+
dir_path = file_path.split(File::SEPARATOR)[0...-1]
|
28
|
+
FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
|
29
|
+
File.write file_path, _reducer.raw_document
|
30
|
+
puts "-- Snapshot written to #{file_path}".color Colors::RESULT
|
31
|
+
break
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
_reducer.run
|
37
|
+
end
|
38
|
+
|
39
|
+
_navigator
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/crabfarm/rspec.rb
CHANGED
@@ -14,7 +14,7 @@ module Crabfarm
|
|
14
14
|
end
|
15
15
|
|
16
16
|
snapshot_path = described_class.snapshot_path _snapshot
|
17
|
-
raise Error.new "Snapshot does not exist #{
|
17
|
+
raise Error.new "Snapshot does not exist #{snapshot_path}" unless File.exist? snapshot_path
|
18
18
|
|
19
19
|
data = File.read snapshot_path
|
20
20
|
reducer = described_class.new data, _options
|
@@ -43,8 +43,6 @@ module Crabfarm
|
|
43
43
|
@state ||= navigate
|
44
44
|
end
|
45
45
|
|
46
|
-
alias :navigator :state
|
47
|
-
|
48
46
|
def last_state
|
49
47
|
@last_state
|
50
48
|
end
|
@@ -54,7 +52,7 @@ module Crabfarm
|
|
54
52
|
end
|
55
53
|
|
56
54
|
def browser(_session_id=nil)
|
57
|
-
@context.pool.
|
55
|
+
@context.pool.driver(_session_id)
|
58
56
|
end
|
59
57
|
|
60
58
|
end
|
@@ -1,18 +1,10 @@
|
|
1
1
|
class <%= navigator_class %> < Crabfarm::BaseNavigator
|
2
2
|
|
3
3
|
def run
|
4
|
-
<%
|
5
|
-
|
6
|
-
|
7
|
-
<% else %>
|
8
|
-
browser.goto '<%= navigator_url %>'
|
9
|
-
<% end %>
|
4
|
+
<% unless navigator_url.nil? %>browser.goto '<%= navigator_url %>'
|
5
|
+
|
6
|
+
<% end %># Add some navigation code here.
|
10
7
|
|
11
|
-
# Call the homonymous reducer over the entire document and output it by default
|
12
|
-
# You can call other reducers by using the `reduce` method, like this:
|
13
|
-
#
|
14
|
-
# reduce browser.search('td').first, using: MyOtherReducer
|
15
|
-
#
|
16
8
|
reduce_with_defaults
|
17
9
|
end
|
18
10
|
|
@@ -1,6 +1,19 @@
|
|
1
1
|
module Crabfarm
|
2
2
|
class TransitionService
|
3
3
|
|
4
|
+
def self.with_navigator_decorator(_decorator)
|
5
|
+
@decorator = DecoratorChain.new @decorator, _decorator
|
6
|
+
begin
|
7
|
+
yield
|
8
|
+
ensure
|
9
|
+
@decorator = @decorator.base
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.current_decorator
|
14
|
+
@decorator
|
15
|
+
end
|
16
|
+
|
4
17
|
def self.transition(_context, _name, _params={})
|
5
18
|
self.new(_context).transition(_name, _params)
|
6
19
|
end
|
@@ -18,6 +31,7 @@ module Crabfarm
|
|
18
31
|
|
19
32
|
@context.prepare
|
20
33
|
@navigator = navigator_class.new @context, _params
|
34
|
+
@navigator = current_decorator.decorate @navigator unless current_decorator.nil?
|
21
35
|
|
22
36
|
@document = @navigator.run
|
23
37
|
@document = @document.as_json if @document.respond_to? :as_json
|
@@ -27,10 +41,31 @@ module Crabfarm
|
|
27
41
|
|
28
42
|
private
|
29
43
|
|
44
|
+
def current_decorator
|
45
|
+
self.class.current_decorator
|
46
|
+
end
|
47
|
+
|
30
48
|
def load_class_from_uri(_uri)
|
31
49
|
class_name = Utils::Naming.decode_crabfarm_uri _uri
|
32
50
|
class_name.constantize
|
33
51
|
end
|
34
52
|
|
53
|
+
class DecoratorChain
|
54
|
+
|
55
|
+
attr_reader :base, :new
|
56
|
+
|
57
|
+
def initialize(_base, _new)
|
58
|
+
@base = _base
|
59
|
+
@new = _new
|
60
|
+
end
|
61
|
+
|
62
|
+
def decorate(_navigator)
|
63
|
+
_navigator = @new.decorate _navigator
|
64
|
+
return _navigator if @base.nil?
|
65
|
+
@base.decorate _navigator
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
|
35
70
|
end
|
36
71
|
end
|
data/lib/crabfarm/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crabfarm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ignacio Baixas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-04-
|
11
|
+
date: 2015-04-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -412,8 +412,11 @@ files:
|
|
412
412
|
- lib/crabfarm/modes/console.rb
|
413
413
|
- lib/crabfarm/modes/generator.rb
|
414
414
|
- lib/crabfarm/modes/publisher.rb
|
415
|
-
- lib/crabfarm/modes/recorder.rb
|
415
|
+
- lib/crabfarm/modes/recorder/memento.rb
|
416
|
+
- lib/crabfarm/modes/recorder/snapshot.rb
|
416
417
|
- lib/crabfarm/modes/server.rb
|
418
|
+
- lib/crabfarm/modes/shared/interactive_decorator.rb
|
419
|
+
- lib/crabfarm/modes/shared/snapshot_decorator.rb
|
417
420
|
- lib/crabfarm/phantom_runner.rb
|
418
421
|
- lib/crabfarm/rspec.rb
|
419
422
|
- lib/crabfarm/state_store.rb
|
@@ -1,81 +0,0 @@
|
|
1
|
-
require 'rainbow'
|
2
|
-
require 'rainbow/ext/string'
|
3
|
-
require 'crabfarm/support/webdriver_factory'
|
4
|
-
require 'crabfarm/crabtrap_runner'
|
5
|
-
|
6
|
-
module Crabfarm
|
7
|
-
module Modes
|
8
|
-
module Recorder
|
9
|
-
extend self
|
10
|
-
|
11
|
-
def memento_path(_name)
|
12
|
-
File.join(GlobalState.mementos_path, _name + '.json.gz')
|
13
|
-
end
|
14
|
-
|
15
|
-
def start(_target, _replay=false)
|
16
|
-
return puts "Must provide a recording target" unless _target.is_a? String
|
17
|
-
|
18
|
-
target_path = memento_path _target
|
19
|
-
return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
|
20
|
-
|
21
|
-
start_crabtrap _replay, target_path
|
22
|
-
|
23
|
-
begin
|
24
|
-
driver = build_driver Crabfarm.config.recorder_driver
|
25
|
-
return puts "Invalid recorder_driver name '#{Crabfarm.config.recorder_driver}'" if driver.nil?
|
26
|
-
|
27
|
-
begin
|
28
|
-
puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
|
29
|
-
loop do
|
30
|
-
driver.current_url
|
31
|
-
sleep 1.0
|
32
|
-
end
|
33
|
-
rescue Exception => e
|
34
|
-
# noop
|
35
|
-
end
|
36
|
-
|
37
|
-
puts "Releasing crawling context".color(:green)
|
38
|
-
driver.quit rescue nil
|
39
|
-
ensure
|
40
|
-
crabtrap.stop
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
private
|
45
|
-
|
46
|
-
def start_crabtrap(_replay, _target_path)
|
47
|
-
crabtrap_config = Crabfarm.config.crabtrap_config
|
48
|
-
crabtrap_config[:mode] = _replay ? :replay : :capture
|
49
|
-
crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
|
50
|
-
crabtrap_config[:bucket_path] = _target_path
|
51
|
-
|
52
|
-
@crabtrap = CrabtrapRunner.new crabtrap_config
|
53
|
-
@crabtrap.start
|
54
|
-
end
|
55
|
-
|
56
|
-
def crabtrap
|
57
|
-
@crabtrap
|
58
|
-
end
|
59
|
-
|
60
|
-
def build_driver(_name)
|
61
|
-
case _name.to_sym
|
62
|
-
when :firefox
|
63
|
-
Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
|
64
|
-
when :chrome
|
65
|
-
Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
|
66
|
-
else return nil end
|
67
|
-
end
|
68
|
-
|
69
|
-
def driver_config
|
70
|
-
{
|
71
|
-
proxy: "127.0.0.1:#{crabtrap.port}",
|
72
|
-
window_width: Crabfarm.config.webdriver_window_width,
|
73
|
-
window_height: Crabfarm.config.webdriver_window_height
|
74
|
-
}
|
75
|
-
end
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|