crabfarm 0.3.7 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/crabfarm.rb +28 -0
- data/lib/crabfarm/adapters/browser/abstract_webdriver.rb +17 -3
- data/lib/crabfarm/adapters/browser/phantom_js.rb +1 -1
- data/lib/crabfarm/adapters/driver_wrapper/watir.rb +13 -0
- data/lib/crabfarm/base.rb +9 -0
- data/lib/crabfarm/base_navigator.rb +5 -1
- data/lib/crabfarm/base_reducer.rb +5 -1
- data/lib/crabfarm/base_struct.rb +7 -1
- data/lib/crabfarm/cli.rb +10 -0
- data/lib/crabfarm/configuration.rb +1 -1
- data/lib/crabfarm/crabtrap_context.rb +1 -1
- data/lib/crabfarm/crabtrap_runner.rb +10 -0
- data/lib/crabfarm/driver_pool.rb +2 -2
- data/lib/crabfarm/dsl/surfer/search_context.rb +4 -0
- data/lib/crabfarm/http_client.rb +4 -1
- data/lib/crabfarm/live/controller.rb +127 -0
- data/lib/crabfarm/live/helpers.rb +42 -0
- data/lib/crabfarm/live/interactable.rb +46 -0
- data/lib/crabfarm/live/manager.rb +90 -0
- data/lib/crabfarm/live/navigator_runner.rb +68 -0
- data/lib/crabfarm/live/reducer_runner.rb +22 -0
- data/lib/crabfarm/live/watcher.rb +67 -0
- data/lib/crabfarm/modes/live.rb +32 -0
- data/lib/crabfarm/modes/recorder/memento.rb +3 -2
- data/lib/crabfarm/version.rb +1 -1
- metadata +39 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 808bde80c9b692ad1a374508c17848f8ccaef0a3
|
4
|
+
data.tar.gz: e5d400990eefba3b89b3945f3126bd2de2d34973
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3ff92d32566d149d45fe91660548448ca66f350a55603696cc21b2cc9e72568e539ebb503c4466cf0ecba4cb37e1e60497e988033b9bd56aae3a79cf5841e5b1
|
7
|
+
data.tar.gz: 0442673d0120a6883c493ebc93fe92d2d95ae02e86422503bb8bc91cb50cd7d422b77de3bb1136b84bafdef7c3b415f147e7ac8ca288af862bb4c690eab63f8a
|
data/lib/crabfarm.rb
CHANGED
@@ -26,6 +26,12 @@ module Crabfarm
|
|
26
26
|
|
27
27
|
@@config = Configuration.new
|
28
28
|
@@logger = nil
|
29
|
+
@@live = nil
|
30
|
+
@@debug = false
|
31
|
+
|
32
|
+
def self.root
|
33
|
+
File.dirname __dir__
|
34
|
+
end
|
29
35
|
|
30
36
|
def self.config
|
31
37
|
@@config
|
@@ -43,6 +49,28 @@ module Crabfarm
|
|
43
49
|
@@config.instance_eval File.read _path
|
44
50
|
end
|
45
51
|
|
52
|
+
def self.install_live_backend!
|
53
|
+
require "crabfarm/live/manager"
|
54
|
+
@@live = Live::Manager.new
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.live
|
58
|
+
@@live
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.live?
|
62
|
+
not @@live.nil?
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.enable_debugging!
|
66
|
+
require 'pry-byebug'
|
67
|
+
@@debug = true
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.debug?
|
71
|
+
@@debug
|
72
|
+
end
|
73
|
+
|
46
74
|
module Strategies
|
47
75
|
# bundled browser adapters
|
48
76
|
register :browser, :phantomjs, 'Crabfarm::Adapters::Browser::PhantomJs', dependencies: ['selenium-webdriver']
|
@@ -19,11 +19,17 @@ module Crabfarm
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def build_driver(_session_id)
|
22
|
-
|
22
|
+
wrap_driver(if Crabfarm.live?
|
23
|
+
build_live_instance _session_id
|
24
|
+
else
|
25
|
+
build_webdriver_instance
|
26
|
+
end)
|
23
27
|
end
|
24
28
|
|
25
|
-
def release_driver(_driver)
|
26
|
-
|
29
|
+
def release_driver(_session_id, _driver)
|
30
|
+
unless Crabfarm.live? and _session_id == :default_driver
|
31
|
+
_driver.driver.quit rescue nil
|
32
|
+
end
|
27
33
|
end
|
28
34
|
|
29
35
|
private
|
@@ -38,6 +44,14 @@ module Crabfarm
|
|
38
44
|
raise NotImplementedError.new
|
39
45
|
end
|
40
46
|
|
47
|
+
def build_live_instance(_session_id)
|
48
|
+
if _session_id == :default_driver
|
49
|
+
Crabfarm.live.primary_driver
|
50
|
+
else
|
51
|
+
Crabfarm.live.generate_support_driver
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
41
55
|
def load_driver_config
|
42
56
|
{
|
43
57
|
capabilities: Crabfarm.config.webdriver_capabilities,
|
@@ -2,18 +2,31 @@ class Watir::Browser
|
|
2
2
|
def to_html
|
3
3
|
html
|
4
4
|
end
|
5
|
+
|
6
|
+
def webdriver_elements
|
7
|
+
[]
|
8
|
+
end
|
5
9
|
end
|
6
10
|
|
7
11
|
class Watir::Element
|
8
12
|
def to_html
|
9
13
|
html
|
10
14
|
end
|
15
|
+
|
16
|
+
def webdriver_elements
|
17
|
+
if @element then [@element] else [] end
|
18
|
+
end
|
19
|
+
|
11
20
|
end
|
12
21
|
|
13
22
|
class Watir::ElementCollection
|
14
23
|
def to_html
|
15
24
|
self.map(&:html).join
|
16
25
|
end
|
26
|
+
|
27
|
+
def webdriver_elements
|
28
|
+
elements
|
29
|
+
end
|
17
30
|
end
|
18
31
|
|
19
32
|
module Crabfarm
|
@@ -1,10 +1,14 @@
|
|
1
1
|
require 'thwait'
|
2
|
+
require 'crabfarm/base'
|
3
|
+
require 'crabfarm/assertion/context'
|
4
|
+
require 'crabfarm/live/interactable'
|
2
5
|
require 'crabfarm/forked_navigator'
|
3
|
-
require "crabfarm/assertion/context"
|
4
6
|
|
5
7
|
module Crabfarm
|
6
8
|
class BaseNavigator
|
9
|
+
include Base
|
7
10
|
include Assertion::Context
|
11
|
+
include Live::Interactable
|
8
12
|
extend Forwardable
|
9
13
|
|
10
14
|
attr_reader :params
|
@@ -1,8 +1,12 @@
|
|
1
|
-
require
|
1
|
+
require 'crabfarm/base'
|
2
|
+
require 'crabfarm/assertion/fields'
|
3
|
+
require 'crabfarm/live/interactable'
|
2
4
|
|
3
5
|
module Crabfarm
|
4
6
|
class BaseReducer < Delegator
|
7
|
+
include Base
|
5
8
|
include Assertion::Fields
|
9
|
+
include Live::Interactable
|
6
10
|
|
7
11
|
attr_reader :raw_document, :document, :params
|
8
12
|
|
data/lib/crabfarm/base_struct.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
-
require
|
1
|
+
require 'crabfarm/base'
|
2
|
+
require 'crabfarm/assertion/context'
|
2
3
|
|
3
4
|
module Crabfarm
|
4
5
|
class BaseStruct
|
6
|
+
include Base
|
5
7
|
include Assertion::Fields
|
6
8
|
|
7
9
|
def initialize(_values={})
|
@@ -13,5 +15,9 @@ module Crabfarm
|
|
13
15
|
field_hash
|
14
16
|
end
|
15
17
|
|
18
|
+
def to_json(_options={})
|
19
|
+
field_hash.to_json(_options)
|
20
|
+
end
|
21
|
+
|
16
22
|
end
|
17
23
|
end
|
data/lib/crabfarm/cli.rb
CHANGED
@@ -27,6 +27,16 @@ module Crabfarm
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
+
desc "Starts the crawler in live mode"
|
31
|
+
command [:live, :i] do |c|
|
32
|
+
c.action do |global_options,options,args|
|
33
|
+
next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
|
34
|
+
|
35
|
+
require "crabfarm/modes/live"
|
36
|
+
Crabfarm::Modes::Live.start_watch
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
30
40
|
desc "Starts the crawler in server mode"
|
31
41
|
command [:server, :s] do |c|
|
32
42
|
c.desc "Set the server host, defaults to 0.0.0.0"
|
@@ -38,6 +38,15 @@ module Crabfarm
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
41
|
+
def kill
|
42
|
+
unless @process.nil?
|
43
|
+
logger.info "Killing crabtrap (PID: #{@process.pid})"
|
44
|
+
@process.stop 0
|
45
|
+
@process = nil
|
46
|
+
logger.info "Crabtrap stopped"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
41
50
|
private
|
42
51
|
|
43
52
|
def spawn_crabtrap
|
@@ -59,6 +68,7 @@ module Crabfarm
|
|
59
68
|
cmd << mode.to_s
|
60
69
|
cmd << @config[:bucket_path] if mode != :pass
|
61
70
|
cmd << "--port=#{port}"
|
71
|
+
cmd << "--virtual=#{@config[:virtual]}" if @config.key? :virtual
|
62
72
|
cmd
|
63
73
|
end
|
64
74
|
|
data/lib/crabfarm/driver_pool.rb
CHANGED
@@ -15,12 +15,12 @@ module Crabfarm
|
|
15
15
|
|
16
16
|
def reset(_session_id=nil)
|
17
17
|
if _session_id.nil?
|
18
|
-
@drivers.
|
18
|
+
@drivers.each { |k, d| @factory.release_driver(k, d) }
|
19
19
|
@drivers = Hash.new
|
20
20
|
else
|
21
21
|
_session_id = _session_id.to_sym
|
22
22
|
driver = @drivers.delete _session_id
|
23
|
-
@factory.release_driver driver unless driver.nil?
|
23
|
+
@factory.release_driver(_session_id, driver) unless driver.nil?
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
data/lib/crabfarm/http_client.rb
CHANGED
@@ -25,7 +25,10 @@ module Crabfarm
|
|
25
25
|
attr_reader :proxy_addr, :proxy_port
|
26
26
|
|
27
27
|
def initialize(_proxy=nil)
|
28
|
-
if
|
28
|
+
if Crabfarm.live?
|
29
|
+
@proxy_addr = '127.0.0.1'
|
30
|
+
@proxy_port = Crabfarm.live.proxy_port
|
31
|
+
elsif _proxy.nil?
|
29
32
|
@proxy_addr = nil
|
30
33
|
@proxy_port = nil
|
31
34
|
else
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'benchmark'
|
2
|
+
require 'rainbow'
|
3
|
+
require 'rainbow/ext/string'
|
4
|
+
require 'crabfarm/modes/console'
|
5
|
+
|
6
|
+
require 'crabfarm/support/webdriver_factory'
|
7
|
+
require 'crabfarm/crabtrap_runner'
|
8
|
+
|
9
|
+
require 'crabfarm/live/helpers.rb'
|
10
|
+
require 'crabfarm/live/navigator_runner.rb'
|
11
|
+
require 'crabfarm/live/reducer_runner.rb'
|
12
|
+
|
13
|
+
module Crabfarm
|
14
|
+
module Live
|
15
|
+
class Controller
|
16
|
+
|
17
|
+
Colors = Crabfarm::Modes::Console::Colors
|
18
|
+
|
19
|
+
class LiveWarning < StandardError; end
|
20
|
+
|
21
|
+
def initialize(_manager)
|
22
|
+
@manager = _manager
|
23
|
+
end
|
24
|
+
|
25
|
+
def execute_live(_class)
|
26
|
+
begin
|
27
|
+
runner = build_runner_for _class
|
28
|
+
prepare_session_for runner
|
29
|
+
elapsed = Benchmark.measure { runner.execute }
|
30
|
+
display_feedback runner, elapsed
|
31
|
+
rescue LiveWarning => exc
|
32
|
+
display_warning_feedback exc
|
33
|
+
rescue Exception => exc
|
34
|
+
display_error_feedback exc
|
35
|
+
ensure
|
36
|
+
clean_up_session
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def build_runner_for(_class)
|
43
|
+
raise LiveWarning.new "'#{_class.to_s} is not Interactable" unless _class < Interactable
|
44
|
+
|
45
|
+
puts "Launching #{_class.to_s}".color Colors::NOTICE
|
46
|
+
|
47
|
+
runner = if _class.live_delegate
|
48
|
+
build_runner_for _class.live_delegate
|
49
|
+
else
|
50
|
+
if _class < BaseNavigator
|
51
|
+
NavigatorRunner
|
52
|
+
elsif _class < BaseReducer
|
53
|
+
ReducerRunner
|
54
|
+
else
|
55
|
+
raise LiveWarning.new "Don't know how to run #{_class.to_s}, you should provide a navigator or reducer as delegate."
|
56
|
+
end.new _class
|
57
|
+
end
|
58
|
+
|
59
|
+
runner.dsl.instance_eval(&_class.live_setup) if _class.live_setup
|
60
|
+
runner
|
61
|
+
end
|
62
|
+
|
63
|
+
def prepare_session_for(_runner)
|
64
|
+
@manager.stop_crabtrap
|
65
|
+
if _runner.memento
|
66
|
+
@manager.start_crabtrap :replay, memento_path(_runner.memento)
|
67
|
+
else
|
68
|
+
@manager.start_crabtrap :pass
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def clean_up_session
|
73
|
+
# leave crabtrap running for debugging purposes.
|
74
|
+
end
|
75
|
+
|
76
|
+
def display_feedback(_runner, _elapsed)
|
77
|
+
load_web_ui
|
78
|
+
|
79
|
+
@manager.primary_driver.execute_script(
|
80
|
+
"window.crabfarm.showResults(arguments[0], arguments[1]);",
|
81
|
+
_runner.output,
|
82
|
+
_elapsed.real
|
83
|
+
);
|
84
|
+
|
85
|
+
puts _runner.output.to_s.color Colors::RESULT
|
86
|
+
puts "Completed in #{_elapsed.real} s".color Colors::NOTICE
|
87
|
+
end
|
88
|
+
|
89
|
+
def display_error_feedback(_exc)
|
90
|
+
load_web_ui
|
91
|
+
|
92
|
+
@manager.primary_driver.execute_script(
|
93
|
+
"window.crabfarm.showError(arguments[0], arguments[1]);",
|
94
|
+
"#{_exc.class.to_s}: #{_exc.to_s}",
|
95
|
+
_exc.backtrace.join("\n")
|
96
|
+
);
|
97
|
+
|
98
|
+
puts "#{_exc.class.to_s}: #{_exc.to_s}".color Colors::ERROR
|
99
|
+
puts _exc.backtrace
|
100
|
+
end
|
101
|
+
|
102
|
+
def display_warning_feedback(_exc)
|
103
|
+
load_web_ui
|
104
|
+
puts "Warning: #{_exc.to_s}".color Colors::WARNING
|
105
|
+
end
|
106
|
+
|
107
|
+
def load_web_ui
|
108
|
+
Helpers.inject_style @manager.primary_driver, 'https://www.crabtrap.io/selectorgadget_combined.css'
|
109
|
+
Helpers.inject_style @manager.primary_driver, 'https://www.crabtrap.io/tools.css'
|
110
|
+
Helpers.inject_script @manager.primary_driver, 'https://www.crabtrap.io/selectorgadget_combined.js'
|
111
|
+
Helpers.inject_script @manager.primary_driver, 'https://www.crabtrap.io/tools.js'
|
112
|
+
wait_for_injection
|
113
|
+
end
|
114
|
+
|
115
|
+
def wait_for_injection
|
116
|
+
while @manager.primary_driver.execute_script "return (typeof window.crabfarm === 'undefined');"
|
117
|
+
sleep 1.0
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def memento_path(_name)
|
122
|
+
File.join(GlobalState.mementos_path, _name + '.json.gz')
|
123
|
+
end
|
124
|
+
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Crabfarm
|
2
|
+
module Live
|
3
|
+
module Helpers
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def inject_script(_driver, _path)
|
7
|
+
_driver.execute_script("
|
8
|
+
(function() {
|
9
|
+
var script = document.createElement('script');
|
10
|
+
script.async = false;
|
11
|
+
script.src = '#{_path}';
|
12
|
+
document.head.appendChild(script);
|
13
|
+
})();
|
14
|
+
")
|
15
|
+
end
|
16
|
+
|
17
|
+
def inject_style(_driver, _path)
|
18
|
+
_driver.execute_script("
|
19
|
+
(function() {
|
20
|
+
var link = document.createElement('link');
|
21
|
+
link.setAttribute('rel','stylesheet');
|
22
|
+
link.setAttribute('type','text/css');
|
23
|
+
link.setAttribute('href','#{_path}');
|
24
|
+
link.setAttribute('media','all');
|
25
|
+
document.head.appendChild(link);
|
26
|
+
})();
|
27
|
+
")
|
28
|
+
end
|
29
|
+
|
30
|
+
def set_style(_elements, _style)
|
31
|
+
return if _elements.size == 0
|
32
|
+
# Not sure about using a bridge method directly here...
|
33
|
+
_elements.first.send(:bridge).executeScript("
|
34
|
+
for(var i = 0, l = arguments[0].length; i < l; i++) {
|
35
|
+
arguments[0][i].setAttribute('style', arguments[1]);
|
36
|
+
}
|
37
|
+
", _elements, _style)
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'crabfarm/live/helpers'
|
2
|
+
|
3
|
+
module Crabfarm
|
4
|
+
module Live
|
5
|
+
module Interactable
|
6
|
+
|
7
|
+
def self.included(klass)
|
8
|
+
klass.extend ClassMethods
|
9
|
+
end
|
10
|
+
|
11
|
+
module ClassMethods
|
12
|
+
|
13
|
+
def live(_options={}, &_setup)
|
14
|
+
@delegate = _options[:delegate]
|
15
|
+
@setup = _setup
|
16
|
+
end
|
17
|
+
|
18
|
+
def live_delegate
|
19
|
+
@delegate
|
20
|
+
end
|
21
|
+
|
22
|
+
def live_setup
|
23
|
+
@setup
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
# Tooling
|
29
|
+
|
30
|
+
def highlight(_elements)
|
31
|
+
if Crabfarm.live?
|
32
|
+
if _elements.respond_to? :webdriver_elements
|
33
|
+
_elements = _elements.webdriver_elements
|
34
|
+
end
|
35
|
+
|
36
|
+
if _elements.is_a? String
|
37
|
+
_elements = Crabfarm.live.primary_driver.find_elements(css: _elements)
|
38
|
+
end
|
39
|
+
|
40
|
+
Helpers.set_style _elements, "border: 3px solid yellow;"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'crabfarm/support/webdriver_factory'
|
2
|
+
require 'crabfarm/crabtrap_runner'
|
3
|
+
|
4
|
+
module Crabfarm
|
5
|
+
module Live
|
6
|
+
class Manager
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@port = Utils::PortDiscovery.find_available_port
|
10
|
+
@driver = Crabfarm.config.recorder_driver
|
11
|
+
end
|
12
|
+
|
13
|
+
def proxy_port
|
14
|
+
@port
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
load_primary_driver
|
19
|
+
end
|
20
|
+
|
21
|
+
def stop
|
22
|
+
stop_crabtrap
|
23
|
+
release_primary_driver
|
24
|
+
end
|
25
|
+
|
26
|
+
def primary_driver
|
27
|
+
@driver
|
28
|
+
end
|
29
|
+
|
30
|
+
def generate_support_driver
|
31
|
+
# TODO: improve on this mechanics, maybe use a frame in the same driver for this
|
32
|
+
build_driver
|
33
|
+
end
|
34
|
+
|
35
|
+
def start_crabtrap(_mode, _memento_path=nil)
|
36
|
+
|
37
|
+
raise ConfigurationError.new "No memento found at #{_memento_path}" unless File.exists? _memento_path
|
38
|
+
|
39
|
+
if @crabtrap.nil?
|
40
|
+
options = {
|
41
|
+
mode: _mode,
|
42
|
+
bucket_path: _memento_path,
|
43
|
+
port: @port,
|
44
|
+
virtual: File.expand_path('./assets/live-tools', Crabfarm.root)
|
45
|
+
}
|
46
|
+
|
47
|
+
@crabtrap = CrabtrapRunner.new config.crabtrap_config.merge(options)
|
48
|
+
@crabtrap.start
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def stop_crabtrap
|
53
|
+
unless @crabtrap.nil?
|
54
|
+
@crabtrap.kill
|
55
|
+
@crabtrap = nil
|
56
|
+
else nil end
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
def load_primary_driver
|
62
|
+
@driver = build_driver
|
63
|
+
end
|
64
|
+
|
65
|
+
def release_primary_driver
|
66
|
+
@driver.quit rescue nil unless @driver.nil?
|
67
|
+
end
|
68
|
+
|
69
|
+
def build_driver
|
70
|
+
case @driver
|
71
|
+
when :firefox
|
72
|
+
Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
|
73
|
+
when :chrome
|
74
|
+
Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
|
75
|
+
else return nil end
|
76
|
+
end
|
77
|
+
|
78
|
+
def driver_config
|
79
|
+
{
|
80
|
+
proxy: "127.0.0.1:#{@port}"
|
81
|
+
}
|
82
|
+
end
|
83
|
+
|
84
|
+
def config
|
85
|
+
Crabfarm.config
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Crabfarm
|
4
|
+
module Live
|
5
|
+
class NavigatorRunner
|
6
|
+
|
7
|
+
class Dsl
|
8
|
+
extend Forwardable
|
9
|
+
|
10
|
+
def initialize(_runner)
|
11
|
+
@runner = _runner
|
12
|
+
end
|
13
|
+
|
14
|
+
def_delegators :@runner, :use_memento, :use_params, :clear_params, :navigate_to
|
15
|
+
end
|
16
|
+
|
17
|
+
def initialize(_target)
|
18
|
+
@target = _target
|
19
|
+
@params = {}
|
20
|
+
end
|
21
|
+
|
22
|
+
def dsl
|
23
|
+
@dsl ||= Dsl.new self
|
24
|
+
end
|
25
|
+
|
26
|
+
def memento
|
27
|
+
if @memento.nil? then memento_for(@target) else @memento end
|
28
|
+
end
|
29
|
+
|
30
|
+
def use_memento(_memento)
|
31
|
+
@memento = _memento
|
32
|
+
end
|
33
|
+
|
34
|
+
def use_params(_params={})
|
35
|
+
@params = @params.merge _params
|
36
|
+
end
|
37
|
+
|
38
|
+
def clear_params
|
39
|
+
@params = {}
|
40
|
+
end
|
41
|
+
|
42
|
+
def navigate_to(_navigator, _params={})
|
43
|
+
end
|
44
|
+
|
45
|
+
def execute
|
46
|
+
context = Crabfarm::Context.new
|
47
|
+
begin
|
48
|
+
# TODO: execute prerequisites
|
49
|
+
@transition = TransitionService.transition context, @target, @params
|
50
|
+
ensure
|
51
|
+
context.release
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def output
|
56
|
+
return nil if @transition.document.nil?
|
57
|
+
JSON.pretty_generate(@transition.document).gsub(/(^|\\n)/, ' ')
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def memento_for(_class)
|
63
|
+
Utils::Naming.route_from_constant(_class.to_s).join(File::SEPARATOR)
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Crabfarm
|
2
|
+
module Live
|
3
|
+
class ReducerRunner < NavigatorRunner
|
4
|
+
|
5
|
+
def initialize(_target)
|
6
|
+
# use navigator runner for now.
|
7
|
+
super navigator_from_reducer _target
|
8
|
+
end
|
9
|
+
|
10
|
+
def navigator_from_reducer _reducer
|
11
|
+
m = _reducer.to_s.match(/^(.*?)Reducer$/)
|
12
|
+
if m
|
13
|
+
navigator = m[1].constantize rescue nil
|
14
|
+
return navigator if navigator and navigator < BaseNavigator
|
15
|
+
end
|
16
|
+
|
17
|
+
raise Controller::LiveWarning.new "Could not find a matching navigator for reducer #{_reducer.to_s}"
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'listen'
|
2
|
+
|
3
|
+
module Crabfarm
|
4
|
+
module Live
|
5
|
+
class Watcher
|
6
|
+
|
7
|
+
PATH_RGX = /^\/[^\/]+\/(.*?)\.rb$/i
|
8
|
+
|
9
|
+
def initialize(_controller)
|
10
|
+
@controller = _controller
|
11
|
+
@candidates = nil
|
12
|
+
end
|
13
|
+
|
14
|
+
def watch(_sleep)
|
15
|
+
begin
|
16
|
+
start_listener
|
17
|
+
loop do
|
18
|
+
execute_pending
|
19
|
+
sleep _sleep
|
20
|
+
end
|
21
|
+
ensure
|
22
|
+
stop_listener
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def start_listener
|
29
|
+
base_path = File.join CF_PATH, 'app'
|
30
|
+
@listener = Listen.to(base_path) do |modified, added, removed|
|
31
|
+
@candidates = (added + modified).map do |path|
|
32
|
+
class_from_path path[base_path.length..-1]
|
33
|
+
end.reject &:nil?
|
34
|
+
end
|
35
|
+
@listener.start
|
36
|
+
end
|
37
|
+
|
38
|
+
def stop_listener
|
39
|
+
@listener.stop if @listener
|
40
|
+
end
|
41
|
+
|
42
|
+
def execute_pending
|
43
|
+
unless @candidates.nil?
|
44
|
+
ActiveSupport::Dependencies.clear
|
45
|
+
@candidates.each do |class_name|
|
46
|
+
target = class_name.constantize rescue nil
|
47
|
+
if target and target < Crabfarm::Live::Interactable
|
48
|
+
@controller.execute_live target
|
49
|
+
break
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
@candidates = nil
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def class_from_path(_filename)
|
58
|
+
_filename = _filename.gsub File::SEPARATOR, '/'
|
59
|
+
m = _filename.match PATH_RGX
|
60
|
+
return nil if m.nil?
|
61
|
+
Utils::Naming.decode_crabfarm_uri m[1]
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'crabfarm/live/watcher'
|
2
|
+
require 'crabfarm/live/controller'
|
3
|
+
|
4
|
+
module Crabfarm
|
5
|
+
module Modes
|
6
|
+
module Live
|
7
|
+
extend self
|
8
|
+
|
9
|
+
def start_watch
|
10
|
+
begin
|
11
|
+
Crabfarm.enable_debugging!
|
12
|
+
Crabfarm.install_live_backend!
|
13
|
+
Crabfarm.live.start
|
14
|
+
|
15
|
+
controller = Crabfarm::Live::Controller.new Crabfarm.live
|
16
|
+
watcher = Crabfarm::Live::Watcher.new controller
|
17
|
+
watcher.watch 0.2
|
18
|
+
|
19
|
+
rescue SystemExit, Interrupt
|
20
|
+
# nothing
|
21
|
+
rescue Exception => e
|
22
|
+
puts "Fatal error: #{e.to_s}".color Console::Colors::ERROR
|
23
|
+
puts e.backtrace
|
24
|
+
ensure
|
25
|
+
puts 'Exiting'
|
26
|
+
Crabfarm.live.stop rescue nil
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -28,11 +28,12 @@ module Crabfarm
|
|
28
28
|
begin
|
29
29
|
puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
|
30
30
|
loop do
|
31
|
-
driver.
|
31
|
+
driver.window_handle
|
32
32
|
sleep 1.0
|
33
33
|
end
|
34
34
|
rescue Exception => e
|
35
|
-
|
35
|
+
puts e
|
36
|
+
puts e.backtrace
|
36
37
|
end
|
37
38
|
|
38
39
|
puts "Releasing crawling context".color(:green)
|
data/lib/crabfarm/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crabfarm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ignacio Baixas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -128,6 +128,34 @@ dependencies:
|
|
128
128
|
- - ~>
|
129
129
|
- !ruby/object:Gem::Version
|
130
130
|
version: 0.5.5
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
name: listen
|
133
|
+
requirement: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ~>
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '2.7'
|
138
|
+
type: :runtime
|
139
|
+
prerelease: false
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - ~>
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '2.7'
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
name: pry-byebug
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
148
|
+
requirements:
|
149
|
+
- - '>='
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '0'
|
152
|
+
type: :runtime
|
153
|
+
prerelease: false
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - '>='
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
131
159
|
- !ruby/object:Gem::Dependency
|
132
160
|
name: selenium-webdriver
|
133
161
|
requirement: !ruby/object:Gem::Requirement
|
@@ -390,6 +418,7 @@ files:
|
|
390
418
|
- lib/crabfarm/assertion/parsers.rb
|
391
419
|
- lib/crabfarm/assertion/validations.rb
|
392
420
|
- lib/crabfarm/assertion/wrapper.rb
|
421
|
+
- lib/crabfarm/base.rb
|
393
422
|
- lib/crabfarm/base_navigator.rb
|
394
423
|
- lib/crabfarm/base_reducer.rb
|
395
424
|
- lib/crabfarm/base_struct.rb
|
@@ -409,8 +438,16 @@ files:
|
|
409
438
|
- lib/crabfarm/forked_navigator.rb
|
410
439
|
- lib/crabfarm/global_state.rb
|
411
440
|
- lib/crabfarm/http_client.rb
|
441
|
+
- lib/crabfarm/live/controller.rb
|
442
|
+
- lib/crabfarm/live/helpers.rb
|
443
|
+
- lib/crabfarm/live/interactable.rb
|
444
|
+
- lib/crabfarm/live/manager.rb
|
445
|
+
- lib/crabfarm/live/navigator_runner.rb
|
446
|
+
- lib/crabfarm/live/reducer_runner.rb
|
447
|
+
- lib/crabfarm/live/watcher.rb
|
412
448
|
- lib/crabfarm/modes/console.rb
|
413
449
|
- lib/crabfarm/modes/generator.rb
|
450
|
+
- lib/crabfarm/modes/live.rb
|
414
451
|
- lib/crabfarm/modes/publisher.rb
|
415
452
|
- lib/crabfarm/modes/recorder/memento.rb
|
416
453
|
- lib/crabfarm/modes/recorder/snapshot.rb
|