crabfarm 0.3.5 → 0.3.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 5732944441ee66f0cfb3f4545c383cbd5b022549
4
- data.tar.gz: 34fab2b972d79c63f852234d72c6580f4f9d9628
3
+ metadata.gz: 034f0f2656c2a21910734636aa34f9d7ba22fc02
4
+ data.tar.gz: f16ff3f1a110bdf38c4d58aea37e7effae1b8ccb
5
5
  SHA512:
6
- metadata.gz: bc67baf038a0f87955a21d8a9a1baf851a2c1a32fe3f3861c9529e7ddf46aee7ffe5e16855443dca5ca5fcc3a839d56788c232efc673fb46cdc933b57f4afcbe
7
- data.tar.gz: b35b59b5b5d5412c4a6290f00f636dd6e27ef6977f6de94f69940b8dde05d4b6c5f2d3e6580c32479514924c03f94c26a9eda43d2b46ca0789b688dea9516a3c
6
+ metadata.gz: 5b5563a6dc4ae2837a28b76eb10bbacb871db117450f065221aae73f314cd708f4ace2b0b06ca71dd0289dcc8e785879f1bd1742c0600b31687328c6b223e6c8
7
+ data.tar.gz: 2db01f23a912d304dd98046ddbc16406685c41327dbe17db21b5415ee27f355a8905fb6410380656047a670700fdf43f26974ee66175f47128708f864f595e0f
@@ -12,7 +12,7 @@ module Crabfarm
12
12
  def_delegators '@context', :http
13
13
  def_delegators '@context.store', :get, :fetch
14
14
 
15
- def initialize(_context, _params)
15
+ def initialize(_context, _params)
16
16
  @context = _context
17
17
  @params = _params
18
18
  end
@@ -60,10 +60,14 @@ module Crabfarm
60
60
  end
61
61
 
62
62
  reducer = _reducer_class.new _target, @params.merge(_options)
63
- reducer.run
63
+ execute_reducer reducer
64
64
  reducer
65
65
  end
66
66
 
67
+ def execute_reducer(_reducer)
68
+ _reducer.run
69
+ end
70
+
67
71
  def start_forked_navigation(_name, _value, _block, _mutex)
68
72
  Thread.new {
69
73
  fork = ForkedNavigator.new @context, self, _name, _mutex
@@ -4,7 +4,7 @@ module Crabfarm
4
4
  class BaseReducer < Delegator
5
5
  include Assertion::Fields
6
6
 
7
- attr_reader :params, :document
7
+ attr_reader :raw_document, :document, :params
8
8
 
9
9
  def self.use_parser(_parser_name)
10
10
  @parser_name = _parser_name
@@ -26,8 +26,8 @@ module Crabfarm
26
26
  def initialize(_target, _params)
27
27
  reset_fields
28
28
 
29
- @parsed_data = parser.preprocess_parsing_target _target
30
- @document = parser.parse @parsed_data
29
+ @raw_document = parser.preprocess_parsing_target _target
30
+ @document = parser.parse @raw_document
31
31
  @params = _params
32
32
 
33
33
  super @document
@@ -37,21 +37,6 @@ module Crabfarm
37
37
  raise NotImplementedError.new
38
38
  end
39
39
 
40
- def take_snapshot(_name=nil)
41
- file_path = self.class.snapshot_path _name
42
-
43
- dir_path = file_path.split(File::SEPARATOR)[0...-1]
44
- FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
45
-
46
- File.write file_path, @parsed_data
47
- file_path
48
- end
49
-
50
- def take_snapshot_and_fail(_name=nil)
51
- file_path = take_snapshot _name
52
- raise ArgumentError.new "New snapshot for #{self.class.to_s} generated in '#{file_path}'"
53
- end
54
-
55
40
  def as_json(_options=nil)
56
41
  field_hash
57
42
  end
data/lib/crabfarm/cli.rb CHANGED
@@ -114,7 +114,7 @@ module Crabfarm
114
114
  end
115
115
 
116
116
  c.desc "Generates a new crabfarm struct"
117
- c.command :reducer do |sub|
117
+ c.command :struct do |sub|
118
118
  sub.action do |global_options,options,args|
119
119
  next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
120
120
 
@@ -124,16 +124,42 @@ module Crabfarm
124
124
  end
125
125
  end
126
126
 
127
- desc "Perform a memento recording for use in tests"
127
+ desc "Crawler fixture recorders"
128
128
  command [:record, :r] do |c|
129
- c.desc "Run recorder in playback mode"
130
- c.switch [:p, :playback], :default_value => false
131
129
 
132
- c.action do |global_options, options, args|
133
- next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
130
+ c.desc "Perform a memento recording for use in tests"
131
+ c.command :memento do |memento|
132
+
133
+ memento.desc "Run recorder in playback mode"
134
+ memento.switch [:p, :playback], :default_value => false
135
+
136
+ memento.action do |global_options, options, args|
137
+ next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
138
+
139
+ require "crabfarm/modes/recorder/memento"
140
+ Crabfarm::Modes::Recorder::Memento.start args[0], options[:playback]
141
+ end
142
+
143
+ end
144
+
145
+ c.desc "Take snapshots from a navigator run"
146
+ c.command [:snapshot, :snap] do |snap|
147
+
148
+ snap.desc "Navigate on a memento"
149
+ snap.flag [:m, :memento]
150
+
151
+ snap.desc "Navigation query string, if not given navigator is run in interactive mode"
152
+ snap.flag [:q, :query]
153
+
154
+ snap.action do |global_options, options, args|
155
+ next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
156
+
157
+ ContextFactory.with_context options[:memento] do |context|
158
+ require "crabfarm/modes/recorder/snapshot"
159
+ Crabfarm::Modes::Recorder::Snapshot.start context, args[0], options[:query]
160
+ end
161
+ end
134
162
 
135
- require "crabfarm/modes/recorder"
136
- Crabfarm::Modes::Recorder.start args[0], options[:playback]
137
163
  end
138
164
  end
139
165
 
@@ -21,8 +21,8 @@ module Crabfarm
21
21
 
22
22
  def load_crabtrap_context(_memento)
23
23
  require 'crabfarm/crabtrap_context'
24
- require 'crabfarm/modes/recorder'
25
- m_path = Modes::Recorder.memento_path _memento
24
+ require 'crabfarm/modes/recorder/memento'
25
+ m_path = Modes::Recorder::Memento.memento_path _memento
26
26
  raise ResourceNotFoundError.new "Could not find memento '#{_name}'" unless File.exists? m_path
27
27
  Crabfarm::CrabtrapContext.new :replay, m_path
28
28
  end
@@ -38,7 +38,7 @@ module Crabfarm
38
38
  start
39
39
  end
40
40
 
41
- def transition(_name, _params={}, _wait=nil)
41
+ def navigate(_name, _params={}, _wait=nil)
42
42
  @lock.synchronize {
43
43
  if @fatal
44
44
  raise CrawlerError.new @fatal
@@ -5,12 +5,14 @@ module Crabfarm
5
5
  module Engines
6
6
  class SyncStateManager
7
7
 
8
+ attr_reader :context
9
+
8
10
  def initialize(_context)
9
11
  @context = _context
10
12
  @lock = Mutex.new
11
13
  end
12
14
 
13
- def reload!
15
+ def reload
14
16
  @lock.synchronize {
15
17
  ActiveSupport::Dependencies.clear
16
18
  @context.reset
@@ -23,7 +25,7 @@ module Crabfarm
23
25
  }
24
26
  end
25
27
 
26
- def transition(_name, _params={})
28
+ def navigate(_name, _params={})
27
29
  @lock.synchronize {
28
30
  output = { name: _name, params: _params }
29
31
 
@@ -6,65 +6,98 @@ require 'crabfarm/engines/sync_state_manager'
6
6
 
7
7
  module Crabfarm
8
8
  module Modes
9
- class Console
9
+ module Console
10
10
 
11
- class ConsoleDsl < Engines::SyncStateManager
11
+ module Colors
12
+ NOTICE = '555555'
13
+ QUESTION = '555555'
14
+ WARNING = :yellow
15
+ ERROR = :red
16
+ RESULT = '00FF00'
17
+ end
18
+
19
+ class ConsoleDsl
20
+
21
+ def initialize(_manager)
22
+ @manager = _manager
23
+ end
12
24
 
13
25
  def reload!
14
- puts "Reloading crawler source".color(:green)
15
- super
26
+ puts "Reloading crawler source".color Colors::NOTICE
27
+ @manager.reload
28
+ nil
16
29
  end
17
30
 
18
31
  def reset
19
- puts "Resetting crawling context".color(:green)
20
- super
32
+ puts "Resetting crawling context".color Colors::NOTICE
33
+ @manager.reset
34
+ nil
21
35
  end
22
36
 
23
37
  def navigate(_name=nil, _params={})
24
38
  if _name.nil?
25
- puts "Must provide a navigator name".color(:red)
39
+ puts "Must provide a navigator name".color Colors::ERROR
26
40
  return
27
41
  end
28
42
 
29
43
  begin
30
- puts "Navigating to #{_name.to_s.camelize} state"
31
- output = super
44
+ puts "Navigating...".color Colors::NOTICE
45
+ output = @manager.navigate _name, _params
46
+ puts JSON.pretty_generate(output.doc).gsub(/(^|\\n)/, ' ').color Colors::RESULT
47
+ puts "Completed in #{output.elapsed.real} s".color Colors::NOTICE
48
+
49
+ rescue Exception => e
50
+ puts "#{e.to_s}".color Colors::ERROR
51
+ puts e.backtrace
52
+ end
53
+ end
32
54
 
33
- puts "State changed, generated document:"
34
- puts JSON.pretty_generate(output.doc).color(:green).gsub(/(^|\\n)/, ' ')
35
- puts "Completed in #{output.elapsed.real} s"
55
+ def snap(_name=nil, _params={})
56
+ if _name.nil?
57
+ puts "Must provide a navigator name".color Colors::ERROR
58
+ return
59
+ end
60
+
61
+ begin
62
+ puts "Navigating, waiting to hit a reducer...".color Colors::NOTICE
63
+ require 'crabfarm/modes/shared/snapshot_decorator'
64
+ TransitionService.with_navigator_decorator Shared::SnapshotDecorator do
65
+ @manager.navigate _name, _params
66
+ end
67
+ puts "Navigation completed".color Colors::NOTICE
36
68
 
37
69
  rescue Exception => e
38
- puts "#{e.to_s}".color(:red)
70
+ puts "#{e.to_s}".color Colors::ERROR
39
71
  puts e.backtrace
40
72
  end
41
73
  end
42
74
 
43
75
  def help
44
76
  puts "Ejem..."
77
+ nil
45
78
  end
46
79
 
47
- alias :n :navigate
48
- alias :r :reset
80
+ alias :nav :navigate
49
81
  end
50
82
 
51
83
  def self.process_input(_context)
52
- dsl = ConsoleDsl.new _context
84
+ dsl = ConsoleDsl.new Engines::SyncStateManager.new _context
53
85
 
54
86
  loop do
55
87
  begin
56
- dsl.instance_eval Readline.readline("> ", true)
88
+ output = dsl.instance_eval Readline.readline("> ", true)
89
+ puts output.inspect unless output.nil?
57
90
  rescue SyntaxError => se
58
- puts "Syntax error: #{se.message}".color(:red)
91
+ puts "Syntax error: #{se.message}".color(Colors::ERROR)
59
92
  rescue SystemExit, Interrupt
60
93
  break
61
94
  rescue => e
62
- puts "#{e.to_s}".color(:red)
95
+ puts "#{e.to_s}".color(Colors::ERROR)
63
96
  puts e.backtrace
64
97
  end
65
98
  end
66
99
 
67
- puts "Exiting".color(:green)
100
+ puts "Exiting".color(Colors::NOTICE)
68
101
  end
69
102
 
70
103
  end
@@ -0,0 +1,81 @@
1
+ require 'rainbow'
2
+ require 'rainbow/ext/string'
3
+ require 'crabfarm/support/webdriver_factory'
4
+ require 'crabfarm/crabtrap_runner'
5
+
6
+ module Crabfarm
7
+ module Modes
8
+ module Recorder
9
+ module Memento
10
+ extend self
11
+
12
+ def memento_path(_name)
13
+ File.join(GlobalState.mementos_path, _name + '.json.gz')
14
+ end
15
+
16
+ def start(_target, _replay=false)
17
+ return puts "Must provide a recording target" unless _target.is_a? String
18
+
19
+ target_path = memento_path _target
20
+ return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
21
+
22
+ start_crabtrap _replay, target_path
23
+
24
+ begin
25
+ driver = build_driver Crabfarm.config.recorder_driver
26
+ return puts "Invalid recorder_driver name '#{Crabfarm.config.recorder_driver}'" if driver.nil?
27
+
28
+ begin
29
+ puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
30
+ loop do
31
+ driver.current_url
32
+ sleep 1.0
33
+ end
34
+ rescue Exception => e
35
+ # noop
36
+ end
37
+
38
+ puts "Releasing crawling context".color(:green)
39
+ driver.quit rescue nil
40
+ ensure
41
+ crabtrap.stop
42
+ end
43
+ end
44
+
45
+ private
46
+
47
+ def start_crabtrap(_replay, _target_path)
48
+ crabtrap_config = Crabfarm.config.crabtrap_config
49
+ crabtrap_config[:mode] = _replay ? :replay : :capture
50
+ crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
51
+ crabtrap_config[:bucket_path] = _target_path
52
+
53
+ @crabtrap = CrabtrapRunner.new crabtrap_config
54
+ @crabtrap.start
55
+ end
56
+
57
+ def crabtrap
58
+ @crabtrap
59
+ end
60
+
61
+ def build_driver(_name)
62
+ case _name.to_sym
63
+ when :firefox
64
+ Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
65
+ when :chrome
66
+ Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
67
+ else return nil end
68
+ end
69
+
70
+ def driver_config
71
+ {
72
+ proxy: "127.0.0.1:#{crabtrap.port}",
73
+ window_width: Crabfarm.config.webdriver_window_width,
74
+ window_height: Crabfarm.config.webdriver_window_height
75
+ }
76
+ end
77
+
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,56 @@
1
+ require 'rainbow'
2
+ require 'rainbow/ext/string'
3
+ require 'crabfarm/modes/console'
4
+ require 'crabfarm/modes/shared/snapshot_decorator'
5
+ require 'crabfarm/modes/shared/interactive_decorator'
6
+
7
+ module Crabfarm
8
+ module Modes
9
+ module Recorder
10
+ module Snapshot
11
+ extend self
12
+
13
+ def start(_context, _navigator, _query=nil)
14
+ return puts "Must provide a navigator name" unless _navigator.is_a? String
15
+
16
+ begin
17
+ puts "Navigating, waiting to hit a reducer...".color(Console::Colors::NOTICE)
18
+ service.with_navigator_decorator Shared::SnapshotDecorator do
19
+ if _query.nil?
20
+ service.with_navigator_decorator Shared::InteractiveDecorator do
21
+ service.transition _context, _navigator
22
+ end
23
+ else
24
+ _query = parse_query_string _query
25
+ service.transition _context, _navigator, _query
26
+ end
27
+ end
28
+ puts "Navigation completed".color(Console::Colors::NOTICE)
29
+ rescue Exception => e
30
+ puts "#{e.to_s}".color Console::Colors::ERROR
31
+ puts e.backtrace
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def service
38
+ TransitionService
39
+ end
40
+
41
+ def parse_query_string(_string)
42
+ result = {}
43
+
44
+ parts = _string.split '&'
45
+ parts.each do |part|
46
+ key, val = part.split '='
47
+ result[key.to_sym] = Shared::InteractiveDecorator::InteractiveHash.parse_input val
48
+ end
49
+
50
+ result
51
+ end
52
+
53
+ end
54
+ end
55
+ end
56
+ end
@@ -54,7 +54,7 @@ module Crabfarm
54
54
  optional :wait, type: Float
55
55
  end
56
56
  put :state do
57
- print_state evaluator.transition params[:name], params[:params], wait
57
+ print_state evaluator.navigate params[:name], params[:params], wait
58
58
  end
59
59
  end
60
60
 
@@ -0,0 +1,71 @@
1
+ require 'inquirer'
2
+ require 'crabfarm/modes/console'
3
+
4
+ module Crabfarm
5
+ module Modes
6
+ module Shared
7
+ module InteractiveDecorator
8
+
9
+ module Colors
10
+ include Crabfarm::Modes::Console::Colors
11
+ end
12
+
13
+ class InteractiveHash < Hash
14
+
15
+ def self.parse_input(_val)
16
+ case _val
17
+ when 'true' then true
18
+ when 'false' then false
19
+ when 'nil' then nil
20
+ when /^\d+$/ then _val.to_i
21
+ when /^\d*\.\d+$/ then _val.to_f
22
+ when /^:[^\s]+$/ then _val.to_sym
23
+ when /^\'.*?\'$/ then _val[1...-1]
24
+ when /^\".*?\"$/ then _val[1...-1]
25
+ else _val end
26
+ end
27
+
28
+ ['[]', :fetch, :has_key?, :key?, :include?].each do |method|
29
+ define_method method do |*args|
30
+ cache_value args[0]
31
+ super(*args)
32
+ end
33
+ end
34
+
35
+ def merge(*args)
36
+ InteractiveHash.new.merge! super
37
+ end
38
+
39
+ private
40
+
41
+ def cache
42
+ @cache ||= self.to_h
43
+ end
44
+
45
+ def cache_value(_key)
46
+ unless cache.key? _key
47
+ value = cache[_key] = Ask.input("Value for '#{_key}'? (blank to skip)".color Colors::QUESTION).strip
48
+ self[_key] = parse_input value unless value.empty?
49
+ end
50
+ end
51
+
52
+ def parse_input(*_args)
53
+ self.class.parse_input(*_args)
54
+ end
55
+
56
+ end
57
+
58
+ def self.decorate(_navigator)
59
+
60
+ def _navigator.params
61
+ @i_params ||= InteractiveHash.new.merge! @params
62
+ end
63
+
64
+ _navigator
65
+ end
66
+
67
+ end
68
+
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,46 @@
1
+ require 'inquirer'
2
+ require 'crabfarm/modes/console'
3
+
4
+ module Crabfarm
5
+ module Modes
6
+ module Shared
7
+ module SnapshotDecorator
8
+
9
+ module Colors
10
+ include Crabfarm::Modes::Console::Colors
11
+ end
12
+
13
+ def self.decorate(_navigator)
14
+
15
+ def _navigator.execute_reducer(_reducer)
16
+ loop do
17
+ name = Ask.input "-- Name for #{_reducer.class.to_s} snapshot (blank to skip)".color Colors::QUESTION
18
+ if name.empty?
19
+ puts "-- Skipping snapshot".color Colors::WARNING
20
+ break
21
+ else
22
+ file_path = _reducer.class.snapshot_path name
23
+
24
+ if File.exist? file_path
25
+ puts "-- Could not save snapshot, file already exist!".color Colors::ERROR
26
+ else
27
+ dir_path = file_path.split(File::SEPARATOR)[0...-1]
28
+ FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
29
+ File.write file_path, _reducer.raw_document
30
+ puts "-- Snapshot written to #{file_path}".color Colors::RESULT
31
+ break
32
+ end
33
+ end
34
+ end
35
+
36
+ _reducer.run
37
+ end
38
+
39
+ _navigator
40
+ end
41
+
42
+ end
43
+
44
+ end
45
+ end
46
+ end
@@ -14,7 +14,7 @@ module Crabfarm
14
14
  end
15
15
 
16
16
  snapshot_path = described_class.snapshot_path _snapshot
17
- raise Error.new "Snapshot does not exist #{_snapshot}" unless File.exist? snapshot_path
17
+ raise Error.new "Snapshot does not exist #{snapshot_path}" unless File.exist? snapshot_path
18
18
 
19
19
  data = File.read snapshot_path
20
20
  reducer = described_class.new data, _options
@@ -43,8 +43,6 @@ module Crabfarm
43
43
  @state ||= navigate
44
44
  end
45
45
 
46
- alias :navigator :state
47
-
48
46
  def last_state
49
47
  @last_state
50
48
  end
@@ -54,7 +52,7 @@ module Crabfarm
54
52
  end
55
53
 
56
54
  def browser(_session_id=nil)
57
- @context.pool.browser(_session_id)
55
+ @context.pool.driver(_session_id)
58
56
  end
59
57
 
60
58
  end
@@ -6,4 +6,4 @@ files:
6
6
  - Gemfile.lock
7
7
  - boot.rb
8
8
  - app/**/*.*
9
- - bin/**/*.*
9
+ - bin/*
@@ -1,18 +1,10 @@
1
1
  class <%= navigator_class %> < Crabfarm::BaseNavigator
2
2
 
3
3
  def run
4
- <% if navigator_url.nil? %>
5
- # replace the following by your navigation code:
6
- raise NotImplementedError.new 'You must provide some navigation code for <%= navigator_class %>'
7
- <% else %>
8
- browser.goto '<%= navigator_url %>'
9
- <% end %>
4
+ <% unless navigator_url.nil? %>browser.goto '<%= navigator_url %>'
5
+
6
+ <% end %># Add some navigation code here.
10
7
 
11
- # Call the homonymous reducer over the entire document and output it by default
12
- # You can call other reducers by using the `reduce` method, like this:
13
- #
14
- # reduce browser.search('td').first, using: MyOtherReducer
15
- #
16
8
  reduce_with_defaults
17
9
  end
18
10
 
@@ -1,8 +1,7 @@
1
1
  class <%= reducer_class %> < Crabfarm::BaseReducer
2
2
 
3
3
  def run
4
- # You can replace the following line after running the owner navigator specs once.
5
- take_snapshot_and_fail
4
+ # Put some data extraction code here.
6
5
  end
7
6
 
8
7
  end
@@ -1,6 +1,19 @@
1
1
  module Crabfarm
2
2
  class TransitionService
3
3
 
4
+ def self.with_navigator_decorator(_decorator)
5
+ @decorator = DecoratorChain.new @decorator, _decorator
6
+ begin
7
+ yield
8
+ ensure
9
+ @decorator = @decorator.base
10
+ end
11
+ end
12
+
13
+ def self.current_decorator
14
+ @decorator
15
+ end
16
+
4
17
  def self.transition(_context, _name, _params={})
5
18
  self.new(_context).transition(_name, _params)
6
19
  end
@@ -18,6 +31,7 @@ module Crabfarm
18
31
 
19
32
  @context.prepare
20
33
  @navigator = navigator_class.new @context, _params
34
+ @navigator = current_decorator.decorate @navigator unless current_decorator.nil?
21
35
 
22
36
  @document = @navigator.run
23
37
  @document = @document.as_json if @document.respond_to? :as_json
@@ -27,10 +41,31 @@ module Crabfarm
27
41
 
28
42
  private
29
43
 
44
+ def current_decorator
45
+ self.class.current_decorator
46
+ end
47
+
30
48
  def load_class_from_uri(_uri)
31
49
  class_name = Utils::Naming.decode_crabfarm_uri _uri
32
50
  class_name.constantize
33
51
  end
34
52
 
53
+ class DecoratorChain
54
+
55
+ attr_reader :base, :new
56
+
57
+ def initialize(_base, _new)
58
+ @base = _base
59
+ @new = _new
60
+ end
61
+
62
+ def decorate(_navigator)
63
+ _navigator = @new.decorate _navigator
64
+ return _navigator if @base.nil?
65
+ @base.decorate _navigator
66
+ end
67
+
68
+ end
69
+
35
70
  end
36
71
  end
@@ -1,3 +1,3 @@
1
1
  module Crabfarm
2
- VERSION = "0.3.5"
2
+ VERSION = "0.3.6"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crabfarm
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 0.3.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ignacio Baixas
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-15 00:00:00.000000000 Z
11
+ date: 2015-04-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -412,8 +412,11 @@ files:
412
412
  - lib/crabfarm/modes/console.rb
413
413
  - lib/crabfarm/modes/generator.rb
414
414
  - lib/crabfarm/modes/publisher.rb
415
- - lib/crabfarm/modes/recorder.rb
415
+ - lib/crabfarm/modes/recorder/memento.rb
416
+ - lib/crabfarm/modes/recorder/snapshot.rb
416
417
  - lib/crabfarm/modes/server.rb
418
+ - lib/crabfarm/modes/shared/interactive_decorator.rb
419
+ - lib/crabfarm/modes/shared/snapshot_decorator.rb
417
420
  - lib/crabfarm/phantom_runner.rb
418
421
  - lib/crabfarm/rspec.rb
419
422
  - lib/crabfarm/state_store.rb
@@ -1,81 +0,0 @@
1
- require 'rainbow'
2
- require 'rainbow/ext/string'
3
- require 'crabfarm/support/webdriver_factory'
4
- require 'crabfarm/crabtrap_runner'
5
-
6
- module Crabfarm
7
- module Modes
8
- module Recorder
9
- extend self
10
-
11
- def memento_path(_name)
12
- File.join(GlobalState.mementos_path, _name + '.json.gz')
13
- end
14
-
15
- def start(_target, _replay=false)
16
- return puts "Must provide a recording target" unless _target.is_a? String
17
-
18
- target_path = memento_path _target
19
- return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
20
-
21
- start_crabtrap _replay, target_path
22
-
23
- begin
24
- driver = build_driver Crabfarm.config.recorder_driver
25
- return puts "Invalid recorder_driver name '#{Crabfarm.config.recorder_driver}'" if driver.nil?
26
-
27
- begin
28
- puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
29
- loop do
30
- driver.current_url
31
- sleep 1.0
32
- end
33
- rescue Exception => e
34
- # noop
35
- end
36
-
37
- puts "Releasing crawling context".color(:green)
38
- driver.quit rescue nil
39
- ensure
40
- crabtrap.stop
41
- end
42
- end
43
-
44
- private
45
-
46
- def start_crabtrap(_replay, _target_path)
47
- crabtrap_config = Crabfarm.config.crabtrap_config
48
- crabtrap_config[:mode] = _replay ? :replay : :capture
49
- crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
50
- crabtrap_config[:bucket_path] = _target_path
51
-
52
- @crabtrap = CrabtrapRunner.new crabtrap_config
53
- @crabtrap.start
54
- end
55
-
56
- def crabtrap
57
- @crabtrap
58
- end
59
-
60
- def build_driver(_name)
61
- case _name.to_sym
62
- when :firefox
63
- Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
64
- when :chrome
65
- Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
66
- else return nil end
67
- end
68
-
69
- def driver_config
70
- {
71
- proxy: "127.0.0.1:#{crabtrap.port}",
72
- window_width: Crabfarm.config.webdriver_window_width,
73
- window_height: Crabfarm.config.webdriver_window_height
74
- }
75
- end
76
-
77
-
78
-
79
- end
80
- end
81
- end