crabfarm 0.3.5 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 5732944441ee66f0cfb3f4545c383cbd5b022549
4
- data.tar.gz: 34fab2b972d79c63f852234d72c6580f4f9d9628
3
+ metadata.gz: 034f0f2656c2a21910734636aa34f9d7ba22fc02
4
+ data.tar.gz: f16ff3f1a110bdf38c4d58aea37e7effae1b8ccb
5
5
  SHA512:
6
- metadata.gz: bc67baf038a0f87955a21d8a9a1baf851a2c1a32fe3f3861c9529e7ddf46aee7ffe5e16855443dca5ca5fcc3a839d56788c232efc673fb46cdc933b57f4afcbe
7
- data.tar.gz: b35b59b5b5d5412c4a6290f00f636dd6e27ef6977f6de94f69940b8dde05d4b6c5f2d3e6580c32479514924c03f94c26a9eda43d2b46ca0789b688dea9516a3c
6
+ metadata.gz: 5b5563a6dc4ae2837a28b76eb10bbacb871db117450f065221aae73f314cd708f4ace2b0b06ca71dd0289dcc8e785879f1bd1742c0600b31687328c6b223e6c8
7
+ data.tar.gz: 2db01f23a912d304dd98046ddbc16406685c41327dbe17db21b5415ee27f355a8905fb6410380656047a670700fdf43f26974ee66175f47128708f864f595e0f
@@ -12,7 +12,7 @@ module Crabfarm
12
12
  def_delegators '@context', :http
13
13
  def_delegators '@context.store', :get, :fetch
14
14
 
15
- def initialize(_context, _params)
15
+ def initialize(_context, _params)
16
16
  @context = _context
17
17
  @params = _params
18
18
  end
@@ -60,10 +60,14 @@ module Crabfarm
60
60
  end
61
61
 
62
62
  reducer = _reducer_class.new _target, @params.merge(_options)
63
- reducer.run
63
+ execute_reducer reducer
64
64
  reducer
65
65
  end
66
66
 
67
+ def execute_reducer(_reducer)
68
+ _reducer.run
69
+ end
70
+
67
71
  def start_forked_navigation(_name, _value, _block, _mutex)
68
72
  Thread.new {
69
73
  fork = ForkedNavigator.new @context, self, _name, _mutex
@@ -4,7 +4,7 @@ module Crabfarm
4
4
  class BaseReducer < Delegator
5
5
  include Assertion::Fields
6
6
 
7
- attr_reader :params, :document
7
+ attr_reader :raw_document, :document, :params
8
8
 
9
9
  def self.use_parser(_parser_name)
10
10
  @parser_name = _parser_name
@@ -26,8 +26,8 @@ module Crabfarm
26
26
  def initialize(_target, _params)
27
27
  reset_fields
28
28
 
29
- @parsed_data = parser.preprocess_parsing_target _target
30
- @document = parser.parse @parsed_data
29
+ @raw_document = parser.preprocess_parsing_target _target
30
+ @document = parser.parse @raw_document
31
31
  @params = _params
32
32
 
33
33
  super @document
@@ -37,21 +37,6 @@ module Crabfarm
37
37
  raise NotImplementedError.new
38
38
  end
39
39
 
40
- def take_snapshot(_name=nil)
41
- file_path = self.class.snapshot_path _name
42
-
43
- dir_path = file_path.split(File::SEPARATOR)[0...-1]
44
- FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
45
-
46
- File.write file_path, @parsed_data
47
- file_path
48
- end
49
-
50
- def take_snapshot_and_fail(_name=nil)
51
- file_path = take_snapshot _name
52
- raise ArgumentError.new "New snapshot for #{self.class.to_s} generated in '#{file_path}'"
53
- end
54
-
55
40
  def as_json(_options=nil)
56
41
  field_hash
57
42
  end
data/lib/crabfarm/cli.rb CHANGED
@@ -114,7 +114,7 @@ module Crabfarm
114
114
  end
115
115
 
116
116
  c.desc "Generates a new crabfarm struct"
117
- c.command :reducer do |sub|
117
+ c.command :struct do |sub|
118
118
  sub.action do |global_options,options,args|
119
119
  next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
120
120
 
@@ -124,16 +124,42 @@ module Crabfarm
124
124
  end
125
125
  end
126
126
 
127
- desc "Perform a memento recording for use in tests"
127
+ desc "Crawler fixture recorders"
128
128
  command [:record, :r] do |c|
129
- c.desc "Run recorder in playback mode"
130
- c.switch [:p, :playback], :default_value => false
131
129
 
132
- c.action do |global_options, options, args|
133
- next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
130
+ c.desc "Perform a memento recording for use in tests"
131
+ c.command :memento do |memento|
132
+
133
+ memento.desc "Run recorder in playback mode"
134
+ memento.switch [:p, :playback], :default_value => false
135
+
136
+ memento.action do |global_options, options, args|
137
+ next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
138
+
139
+ require "crabfarm/modes/recorder/memento"
140
+ Crabfarm::Modes::Recorder::Memento.start args[0], options[:playback]
141
+ end
142
+
143
+ end
144
+
145
+ c.desc "Take snapshots from a navigator run"
146
+ c.command [:snapshot, :snap] do |snap|
147
+
148
+ snap.desc "Navigate on a memento"
149
+ snap.flag [:m, :memento]
150
+
151
+ snap.desc "Navigation query string, if not given navigator is run in interactive mode"
152
+ snap.flag [:q, :query]
153
+
154
+ snap.action do |global_options, options, args|
155
+ next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
156
+
157
+ ContextFactory.with_context options[:memento] do |context|
158
+ require "crabfarm/modes/recorder/snapshot"
159
+ Crabfarm::Modes::Recorder::Snapshot.start context, args[0], options[:query]
160
+ end
161
+ end
134
162
 
135
- require "crabfarm/modes/recorder"
136
- Crabfarm::Modes::Recorder.start args[0], options[:playback]
137
163
  end
138
164
  end
139
165
 
@@ -21,8 +21,8 @@ module Crabfarm
21
21
 
22
22
  def load_crabtrap_context(_memento)
23
23
  require 'crabfarm/crabtrap_context'
24
- require 'crabfarm/modes/recorder'
25
- m_path = Modes::Recorder.memento_path _memento
24
+ require 'crabfarm/modes/recorder/memento'
25
+ m_path = Modes::Recorder::Memento.memento_path _memento
26
26
  raise ResourceNotFoundError.new "Could not find memento '#{_name}'" unless File.exists? m_path
27
27
  Crabfarm::CrabtrapContext.new :replay, m_path
28
28
  end
@@ -38,7 +38,7 @@ module Crabfarm
38
38
  start
39
39
  end
40
40
 
41
- def transition(_name, _params={}, _wait=nil)
41
+ def navigate(_name, _params={}, _wait=nil)
42
42
  @lock.synchronize {
43
43
  if @fatal
44
44
  raise CrawlerError.new @fatal
@@ -5,12 +5,14 @@ module Crabfarm
5
5
  module Engines
6
6
  class SyncStateManager
7
7
 
8
+ attr_reader :context
9
+
8
10
  def initialize(_context)
9
11
  @context = _context
10
12
  @lock = Mutex.new
11
13
  end
12
14
 
13
- def reload!
15
+ def reload
14
16
  @lock.synchronize {
15
17
  ActiveSupport::Dependencies.clear
16
18
  @context.reset
@@ -23,7 +25,7 @@ module Crabfarm
23
25
  }
24
26
  end
25
27
 
26
- def transition(_name, _params={})
28
+ def navigate(_name, _params={})
27
29
  @lock.synchronize {
28
30
  output = { name: _name, params: _params }
29
31
 
@@ -6,65 +6,98 @@ require 'crabfarm/engines/sync_state_manager'
6
6
 
7
7
  module Crabfarm
8
8
  module Modes
9
- class Console
9
+ module Console
10
10
 
11
- class ConsoleDsl < Engines::SyncStateManager
11
+ module Colors
12
+ NOTICE = '555555'
13
+ QUESTION = '555555'
14
+ WARNING = :yellow
15
+ ERROR = :red
16
+ RESULT = '00FF00'
17
+ end
18
+
19
+ class ConsoleDsl
20
+
21
+ def initialize(_manager)
22
+ @manager = _manager
23
+ end
12
24
 
13
25
  def reload!
14
- puts "Reloading crawler source".color(:green)
15
- super
26
+ puts "Reloading crawler source".color Colors::NOTICE
27
+ @manager.reload
28
+ nil
16
29
  end
17
30
 
18
31
  def reset
19
- puts "Resetting crawling context".color(:green)
20
- super
32
+ puts "Resetting crawling context".color Colors::NOTICE
33
+ @manager.reset
34
+ nil
21
35
  end
22
36
 
23
37
  def navigate(_name=nil, _params={})
24
38
  if _name.nil?
25
- puts "Must provide a navigator name".color(:red)
39
+ puts "Must provide a navigator name".color Colors::ERROR
26
40
  return
27
41
  end
28
42
 
29
43
  begin
30
- puts "Navigating to #{_name.to_s.camelize} state"
31
- output = super
44
+ puts "Navigating...".color Colors::NOTICE
45
+ output = @manager.navigate _name, _params
46
+ puts JSON.pretty_generate(output.doc).gsub(/(^|\\n)/, ' ').color Colors::RESULT
47
+ puts "Completed in #{output.elapsed.real} s".color Colors::NOTICE
48
+
49
+ rescue Exception => e
50
+ puts "#{e.to_s}".color Colors::ERROR
51
+ puts e.backtrace
52
+ end
53
+ end
32
54
 
33
- puts "State changed, generated document:"
34
- puts JSON.pretty_generate(output.doc).color(:green).gsub(/(^|\\n)/, ' ')
35
- puts "Completed in #{output.elapsed.real} s"
55
+ def snap(_name=nil, _params={})
56
+ if _name.nil?
57
+ puts "Must provide a navigator name".color Colors::ERROR
58
+ return
59
+ end
60
+
61
+ begin
62
+ puts "Navigating, waiting to hit a reducer...".color Colors::NOTICE
63
+ require 'crabfarm/modes/shared/snapshot_decorator'
64
+ TransitionService.with_navigator_decorator Shared::SnapshotDecorator do
65
+ @manager.navigate _name, _params
66
+ end
67
+ puts "Navigation completed".color Colors::NOTICE
36
68
 
37
69
  rescue Exception => e
38
- puts "#{e.to_s}".color(:red)
70
+ puts "#{e.to_s}".color Colors::ERROR
39
71
  puts e.backtrace
40
72
  end
41
73
  end
42
74
 
43
75
  def help
44
76
  puts "Ejem..."
77
+ nil
45
78
  end
46
79
 
47
- alias :n :navigate
48
- alias :r :reset
80
+ alias :nav :navigate
49
81
  end
50
82
 
51
83
  def self.process_input(_context)
52
- dsl = ConsoleDsl.new _context
84
+ dsl = ConsoleDsl.new Engines::SyncStateManager.new _context
53
85
 
54
86
  loop do
55
87
  begin
56
- dsl.instance_eval Readline.readline("> ", true)
88
+ output = dsl.instance_eval Readline.readline("> ", true)
89
+ puts output.inspect unless output.nil?
57
90
  rescue SyntaxError => se
58
- puts "Syntax error: #{se.message}".color(:red)
91
+ puts "Syntax error: #{se.message}".color(Colors::ERROR)
59
92
  rescue SystemExit, Interrupt
60
93
  break
61
94
  rescue => e
62
- puts "#{e.to_s}".color(:red)
95
+ puts "#{e.to_s}".color(Colors::ERROR)
63
96
  puts e.backtrace
64
97
  end
65
98
  end
66
99
 
67
- puts "Exiting".color(:green)
100
+ puts "Exiting".color(Colors::NOTICE)
68
101
  end
69
102
 
70
103
  end
@@ -0,0 +1,81 @@
1
+ require 'rainbow'
2
+ require 'rainbow/ext/string'
3
+ require 'crabfarm/support/webdriver_factory'
4
+ require 'crabfarm/crabtrap_runner'
5
+
6
+ module Crabfarm
7
+ module Modes
8
+ module Recorder
9
+ module Memento
10
+ extend self
11
+
12
+ def memento_path(_name)
13
+ File.join(GlobalState.mementos_path, _name + '.json.gz')
14
+ end
15
+
16
+ def start(_target, _replay=false)
17
+ return puts "Must provide a recording target" unless _target.is_a? String
18
+
19
+ target_path = memento_path _target
20
+ return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
21
+
22
+ start_crabtrap _replay, target_path
23
+
24
+ begin
25
+ driver = build_driver Crabfarm.config.recorder_driver
26
+ return puts "Invalid recorder_driver name '#{Crabfarm.config.recorder_driver}'" if driver.nil?
27
+
28
+ begin
29
+ puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
30
+ loop do
31
+ driver.current_url
32
+ sleep 1.0
33
+ end
34
+ rescue Exception => e
35
+ # noop
36
+ end
37
+
38
+ puts "Releasing crawling context".color(:green)
39
+ driver.quit rescue nil
40
+ ensure
41
+ crabtrap.stop
42
+ end
43
+ end
44
+
45
+ private
46
+
47
+ def start_crabtrap(_replay, _target_path)
48
+ crabtrap_config = Crabfarm.config.crabtrap_config
49
+ crabtrap_config[:mode] = _replay ? :replay : :capture
50
+ crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
51
+ crabtrap_config[:bucket_path] = _target_path
52
+
53
+ @crabtrap = CrabtrapRunner.new crabtrap_config
54
+ @crabtrap.start
55
+ end
56
+
57
+ def crabtrap
58
+ @crabtrap
59
+ end
60
+
61
+ def build_driver(_name)
62
+ case _name.to_sym
63
+ when :firefox
64
+ Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
65
+ when :chrome
66
+ Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
67
+ else return nil end
68
+ end
69
+
70
+ def driver_config
71
+ {
72
+ proxy: "127.0.0.1:#{crabtrap.port}",
73
+ window_width: Crabfarm.config.webdriver_window_width,
74
+ window_height: Crabfarm.config.webdriver_window_height
75
+ }
76
+ end
77
+
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,56 @@
1
+ require 'rainbow'
2
+ require 'rainbow/ext/string'
3
+ require 'crabfarm/modes/console'
4
+ require 'crabfarm/modes/shared/snapshot_decorator'
5
+ require 'crabfarm/modes/shared/interactive_decorator'
6
+
7
+ module Crabfarm
8
+ module Modes
9
+ module Recorder
10
+ module Snapshot
11
+ extend self
12
+
13
+ def start(_context, _navigator, _query=nil)
14
+ return puts "Must provide a navigator name" unless _navigator.is_a? String
15
+
16
+ begin
17
+ puts "Navigating, waiting to hit a reducer...".color(Console::Colors::NOTICE)
18
+ service.with_navigator_decorator Shared::SnapshotDecorator do
19
+ if _query.nil?
20
+ service.with_navigator_decorator Shared::InteractiveDecorator do
21
+ service.transition _context, _navigator
22
+ end
23
+ else
24
+ _query = parse_query_string _query
25
+ service.transition _context, _navigator, _query
26
+ end
27
+ end
28
+ puts "Navigation completed".color(Console::Colors::NOTICE)
29
+ rescue Exception => e
30
+ puts "#{e.to_s}".color Console::Colors::ERROR
31
+ puts e.backtrace
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def service
38
+ TransitionService
39
+ end
40
+
41
+ def parse_query_string(_string)
42
+ result = {}
43
+
44
+ parts = _string.split '&'
45
+ parts.each do |part|
46
+ key, val = part.split '='
47
+ result[key.to_sym] = Shared::InteractiveDecorator::InteractiveHash.parse_input val
48
+ end
49
+
50
+ result
51
+ end
52
+
53
+ end
54
+ end
55
+ end
56
+ end
@@ -54,7 +54,7 @@ module Crabfarm
54
54
  optional :wait, type: Float
55
55
  end
56
56
  put :state do
57
- print_state evaluator.transition params[:name], params[:params], wait
57
+ print_state evaluator.navigate params[:name], params[:params], wait
58
58
  end
59
59
  end
60
60
 
@@ -0,0 +1,71 @@
1
+ require 'inquirer'
2
+ require 'crabfarm/modes/console'
3
+
4
+ module Crabfarm
5
+ module Modes
6
+ module Shared
7
+ module InteractiveDecorator
8
+
9
+ module Colors
10
+ include Crabfarm::Modes::Console::Colors
11
+ end
12
+
13
+ class InteractiveHash < Hash
14
+
15
+ def self.parse_input(_val)
16
+ case _val
17
+ when 'true' then true
18
+ when 'false' then false
19
+ when 'nil' then nil
20
+ when /^\d+$/ then _val.to_i
21
+ when /^\d*\.\d+$/ then _val.to_f
22
+ when /^:[^\s]+$/ then _val.to_sym
23
+ when /^\'.*?\'$/ then _val[1...-1]
24
+ when /^\".*?\"$/ then _val[1...-1]
25
+ else _val end
26
+ end
27
+
28
+ ['[]', :fetch, :has_key?, :key?, :include?].each do |method|
29
+ define_method method do |*args|
30
+ cache_value args[0]
31
+ super(*args)
32
+ end
33
+ end
34
+
35
+ def merge(*args)
36
+ InteractiveHash.new.merge! super
37
+ end
38
+
39
+ private
40
+
41
+ def cache
42
+ @cache ||= self.to_h
43
+ end
44
+
45
+ def cache_value(_key)
46
+ unless cache.key? _key
47
+ value = cache[_key] = Ask.input("Value for '#{_key}'? (blank to skip)".color Colors::QUESTION).strip
48
+ self[_key] = parse_input value unless value.empty?
49
+ end
50
+ end
51
+
52
+ def parse_input(*_args)
53
+ self.class.parse_input(*_args)
54
+ end
55
+
56
+ end
57
+
58
+ def self.decorate(_navigator)
59
+
60
+ def _navigator.params
61
+ @i_params ||= InteractiveHash.new.merge! @params
62
+ end
63
+
64
+ _navigator
65
+ end
66
+
67
+ end
68
+
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,46 @@
1
+ require 'inquirer'
2
+ require 'crabfarm/modes/console'
3
+
4
+ module Crabfarm
5
+ module Modes
6
+ module Shared
7
+ module SnapshotDecorator
8
+
9
+ module Colors
10
+ include Crabfarm::Modes::Console::Colors
11
+ end
12
+
13
+ def self.decorate(_navigator)
14
+
15
+ def _navigator.execute_reducer(_reducer)
16
+ loop do
17
+ name = Ask.input "-- Name for #{_reducer.class.to_s} snapshot (blank to skip)".color Colors::QUESTION
18
+ if name.empty?
19
+ puts "-- Skipping snapshot".color Colors::WARNING
20
+ break
21
+ else
22
+ file_path = _reducer.class.snapshot_path name
23
+
24
+ if File.exist? file_path
25
+ puts "-- Could not save snapshot, file already exist!".color Colors::ERROR
26
+ else
27
+ dir_path = file_path.split(File::SEPARATOR)[0...-1]
28
+ FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
29
+ File.write file_path, _reducer.raw_document
30
+ puts "-- Snapshot written to #{file_path}".color Colors::RESULT
31
+ break
32
+ end
33
+ end
34
+ end
35
+
36
+ _reducer.run
37
+ end
38
+
39
+ _navigator
40
+ end
41
+
42
+ end
43
+
44
+ end
45
+ end
46
+ end
@@ -14,7 +14,7 @@ module Crabfarm
14
14
  end
15
15
 
16
16
  snapshot_path = described_class.snapshot_path _snapshot
17
- raise Error.new "Snapshot does not exist #{_snapshot}" unless File.exist? snapshot_path
17
+ raise Error.new "Snapshot does not exist #{snapshot_path}" unless File.exist? snapshot_path
18
18
 
19
19
  data = File.read snapshot_path
20
20
  reducer = described_class.new data, _options
@@ -43,8 +43,6 @@ module Crabfarm
43
43
  @state ||= navigate
44
44
  end
45
45
 
46
- alias :navigator :state
47
-
48
46
  def last_state
49
47
  @last_state
50
48
  end
@@ -54,7 +52,7 @@ module Crabfarm
54
52
  end
55
53
 
56
54
  def browser(_session_id=nil)
57
- @context.pool.browser(_session_id)
55
+ @context.pool.driver(_session_id)
58
56
  end
59
57
 
60
58
  end
@@ -6,4 +6,4 @@ files:
6
6
  - Gemfile.lock
7
7
  - boot.rb
8
8
  - app/**/*.*
9
- - bin/**/*.*
9
+ - bin/*
@@ -1,18 +1,10 @@
1
1
  class <%= navigator_class %> < Crabfarm::BaseNavigator
2
2
 
3
3
  def run
4
- <% if navigator_url.nil? %>
5
- # replace the following by your navigation code:
6
- raise NotImplementedError.new 'You must provide some navigation code for <%= navigator_class %>'
7
- <% else %>
8
- browser.goto '<%= navigator_url %>'
9
- <% end %>
4
+ <% unless navigator_url.nil? %>browser.goto '<%= navigator_url %>'
5
+
6
+ <% end %># Add some navigation code here.
10
7
 
11
- # Call the homonymous reducer over the entire document and output it by default
12
- # You can call other reducers by using the `reduce` method, like this:
13
- #
14
- # reduce browser.search('td').first, using: MyOtherReducer
15
- #
16
8
  reduce_with_defaults
17
9
  end
18
10
 
@@ -1,8 +1,7 @@
1
1
  class <%= reducer_class %> < Crabfarm::BaseReducer
2
2
 
3
3
  def run
4
- # You can replace the following line after running the owner navigator specs once.
5
- take_snapshot_and_fail
4
+ # Put some data extraction code here.
6
5
  end
7
6
 
8
7
  end
@@ -1,6 +1,19 @@
1
1
  module Crabfarm
2
2
  class TransitionService
3
3
 
4
+ def self.with_navigator_decorator(_decorator)
5
+ @decorator = DecoratorChain.new @decorator, _decorator
6
+ begin
7
+ yield
8
+ ensure
9
+ @decorator = @decorator.base
10
+ end
11
+ end
12
+
13
+ def self.current_decorator
14
+ @decorator
15
+ end
16
+
4
17
  def self.transition(_context, _name, _params={})
5
18
  self.new(_context).transition(_name, _params)
6
19
  end
@@ -18,6 +31,7 @@ module Crabfarm
18
31
 
19
32
  @context.prepare
20
33
  @navigator = navigator_class.new @context, _params
34
+ @navigator = current_decorator.decorate @navigator unless current_decorator.nil?
21
35
 
22
36
  @document = @navigator.run
23
37
  @document = @document.as_json if @document.respond_to? :as_json
@@ -27,10 +41,31 @@ module Crabfarm
27
41
 
28
42
  private
29
43
 
44
+ def current_decorator
45
+ self.class.current_decorator
46
+ end
47
+
30
48
  def load_class_from_uri(_uri)
31
49
  class_name = Utils::Naming.decode_crabfarm_uri _uri
32
50
  class_name.constantize
33
51
  end
34
52
 
53
+ class DecoratorChain
54
+
55
+ attr_reader :base, :new
56
+
57
+ def initialize(_base, _new)
58
+ @base = _base
59
+ @new = _new
60
+ end
61
+
62
+ def decorate(_navigator)
63
+ _navigator = @new.decorate _navigator
64
+ return _navigator if @base.nil?
65
+ @base.decorate _navigator
66
+ end
67
+
68
+ end
69
+
35
70
  end
36
71
  end
@@ -1,3 +1,3 @@
1
1
  module Crabfarm
2
- VERSION = "0.3.5"
2
+ VERSION = "0.3.6"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crabfarm
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 0.3.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ignacio Baixas
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-15 00:00:00.000000000 Z
11
+ date: 2015-04-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -412,8 +412,11 @@ files:
412
412
  - lib/crabfarm/modes/console.rb
413
413
  - lib/crabfarm/modes/generator.rb
414
414
  - lib/crabfarm/modes/publisher.rb
415
- - lib/crabfarm/modes/recorder.rb
415
+ - lib/crabfarm/modes/recorder/memento.rb
416
+ - lib/crabfarm/modes/recorder/snapshot.rb
416
417
  - lib/crabfarm/modes/server.rb
418
+ - lib/crabfarm/modes/shared/interactive_decorator.rb
419
+ - lib/crabfarm/modes/shared/snapshot_decorator.rb
417
420
  - lib/crabfarm/phantom_runner.rb
418
421
  - lib/crabfarm/rspec.rb
419
422
  - lib/crabfarm/state_store.rb
@@ -1,81 +0,0 @@
1
- require 'rainbow'
2
- require 'rainbow/ext/string'
3
- require 'crabfarm/support/webdriver_factory'
4
- require 'crabfarm/crabtrap_runner'
5
-
6
- module Crabfarm
7
- module Modes
8
- module Recorder
9
- extend self
10
-
11
- def memento_path(_name)
12
- File.join(GlobalState.mementos_path, _name + '.json.gz')
13
- end
14
-
15
- def start(_target, _replay=false)
16
- return puts "Must provide a recording target" unless _target.is_a? String
17
-
18
- target_path = memento_path _target
19
- return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
20
-
21
- start_crabtrap _replay, target_path
22
-
23
- begin
24
- driver = build_driver Crabfarm.config.recorder_driver
25
- return puts "Invalid recorder_driver name '#{Crabfarm.config.recorder_driver}'" if driver.nil?
26
-
27
- begin
28
- puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
29
- loop do
30
- driver.current_url
31
- sleep 1.0
32
- end
33
- rescue Exception => e
34
- # noop
35
- end
36
-
37
- puts "Releasing crawling context".color(:green)
38
- driver.quit rescue nil
39
- ensure
40
- crabtrap.stop
41
- end
42
- end
43
-
44
- private
45
-
46
- def start_crabtrap(_replay, _target_path)
47
- crabtrap_config = Crabfarm.config.crabtrap_config
48
- crabtrap_config[:mode] = _replay ? :replay : :capture
49
- crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
50
- crabtrap_config[:bucket_path] = _target_path
51
-
52
- @crabtrap = CrabtrapRunner.new crabtrap_config
53
- @crabtrap.start
54
- end
55
-
56
- def crabtrap
57
- @crabtrap
58
- end
59
-
60
- def build_driver(_name)
61
- case _name.to_sym
62
- when :firefox
63
- Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
64
- when :chrome
65
- Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
66
- else return nil end
67
- end
68
-
69
- def driver_config
70
- {
71
- proxy: "127.0.0.1:#{crabtrap.port}",
72
- window_width: Crabfarm.config.webdriver_window_width,
73
- window_height: Crabfarm.config.webdriver_window_height
74
- }
75
- end
76
-
77
-
78
-
79
- end
80
- end
81
- end