crabfarm 0.3.7 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/crabfarm.rb +28 -0
- data/lib/crabfarm/adapters/browser/abstract_webdriver.rb +17 -3
- data/lib/crabfarm/adapters/browser/phantom_js.rb +1 -1
- data/lib/crabfarm/adapters/driver_wrapper/watir.rb +13 -0
- data/lib/crabfarm/base.rb +9 -0
- data/lib/crabfarm/base_navigator.rb +5 -1
- data/lib/crabfarm/base_reducer.rb +5 -1
- data/lib/crabfarm/base_struct.rb +7 -1
- data/lib/crabfarm/cli.rb +10 -0
- data/lib/crabfarm/configuration.rb +1 -1
- data/lib/crabfarm/crabtrap_context.rb +1 -1
- data/lib/crabfarm/crabtrap_runner.rb +10 -0
- data/lib/crabfarm/driver_pool.rb +2 -2
- data/lib/crabfarm/dsl/surfer/search_context.rb +4 -0
- data/lib/crabfarm/http_client.rb +4 -1
- data/lib/crabfarm/live/controller.rb +127 -0
- data/lib/crabfarm/live/helpers.rb +42 -0
- data/lib/crabfarm/live/interactable.rb +46 -0
- data/lib/crabfarm/live/manager.rb +90 -0
- data/lib/crabfarm/live/navigator_runner.rb +68 -0
- data/lib/crabfarm/live/reducer_runner.rb +22 -0
- data/lib/crabfarm/live/watcher.rb +67 -0
- data/lib/crabfarm/modes/live.rb +32 -0
- data/lib/crabfarm/modes/recorder/memento.rb +3 -2
- data/lib/crabfarm/version.rb +1 -1
- metadata +39 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 808bde80c9b692ad1a374508c17848f8ccaef0a3
|
4
|
+
data.tar.gz: e5d400990eefba3b89b3945f3126bd2de2d34973
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3ff92d32566d149d45fe91660548448ca66f350a55603696cc21b2cc9e72568e539ebb503c4466cf0ecba4cb37e1e60497e988033b9bd56aae3a79cf5841e5b1
|
7
|
+
data.tar.gz: 0442673d0120a6883c493ebc93fe92d2d95ae02e86422503bb8bc91cb50cd7d422b77de3bb1136b84bafdef7c3b415f147e7ac8ca288af862bb4c690eab63f8a
|
data/lib/crabfarm.rb
CHANGED
@@ -26,6 +26,12 @@ module Crabfarm
|
|
26
26
|
|
27
27
|
@@config = Configuration.new
|
28
28
|
@@logger = nil
|
29
|
+
@@live = nil
|
30
|
+
@@debug = false
|
31
|
+
|
32
|
+
def self.root
|
33
|
+
File.dirname __dir__
|
34
|
+
end
|
29
35
|
|
30
36
|
def self.config
|
31
37
|
@@config
|
@@ -43,6 +49,28 @@ module Crabfarm
|
|
43
49
|
@@config.instance_eval File.read _path
|
44
50
|
end
|
45
51
|
|
52
|
+
def self.install_live_backend!
|
53
|
+
require "crabfarm/live/manager"
|
54
|
+
@@live = Live::Manager.new
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.live
|
58
|
+
@@live
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.live?
|
62
|
+
not @@live.nil?
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.enable_debugging!
|
66
|
+
require 'pry-byebug'
|
67
|
+
@@debug = true
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.debug?
|
71
|
+
@@debug
|
72
|
+
end
|
73
|
+
|
46
74
|
module Strategies
|
47
75
|
# bundled browser adapters
|
48
76
|
register :browser, :phantomjs, 'Crabfarm::Adapters::Browser::PhantomJs', dependencies: ['selenium-webdriver']
|
@@ -19,11 +19,17 @@ module Crabfarm
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def build_driver(_session_id)
|
22
|
-
|
22
|
+
wrap_driver(if Crabfarm.live?
|
23
|
+
build_live_instance _session_id
|
24
|
+
else
|
25
|
+
build_webdriver_instance
|
26
|
+
end)
|
23
27
|
end
|
24
28
|
|
25
|
-
def release_driver(_driver)
|
26
|
-
|
29
|
+
def release_driver(_session_id, _driver)
|
30
|
+
unless Crabfarm.live? and _session_id == :default_driver
|
31
|
+
_driver.driver.quit rescue nil
|
32
|
+
end
|
27
33
|
end
|
28
34
|
|
29
35
|
private
|
@@ -38,6 +44,14 @@ module Crabfarm
|
|
38
44
|
raise NotImplementedError.new
|
39
45
|
end
|
40
46
|
|
47
|
+
def build_live_instance(_session_id)
|
48
|
+
if _session_id == :default_driver
|
49
|
+
Crabfarm.live.primary_driver
|
50
|
+
else
|
51
|
+
Crabfarm.live.generate_support_driver
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
41
55
|
def load_driver_config
|
42
56
|
{
|
43
57
|
capabilities: Crabfarm.config.webdriver_capabilities,
|
@@ -2,18 +2,31 @@ class Watir::Browser
|
|
2
2
|
def to_html
|
3
3
|
html
|
4
4
|
end
|
5
|
+
|
6
|
+
def webdriver_elements
|
7
|
+
[]
|
8
|
+
end
|
5
9
|
end
|
6
10
|
|
7
11
|
class Watir::Element
|
8
12
|
def to_html
|
9
13
|
html
|
10
14
|
end
|
15
|
+
|
16
|
+
def webdriver_elements
|
17
|
+
if @element then [@element] else [] end
|
18
|
+
end
|
19
|
+
|
11
20
|
end
|
12
21
|
|
13
22
|
class Watir::ElementCollection
|
14
23
|
def to_html
|
15
24
|
self.map(&:html).join
|
16
25
|
end
|
26
|
+
|
27
|
+
def webdriver_elements
|
28
|
+
elements
|
29
|
+
end
|
17
30
|
end
|
18
31
|
|
19
32
|
module Crabfarm
|
@@ -1,10 +1,14 @@
|
|
1
1
|
require 'thwait'
|
2
|
+
require 'crabfarm/base'
|
3
|
+
require 'crabfarm/assertion/context'
|
4
|
+
require 'crabfarm/live/interactable'
|
2
5
|
require 'crabfarm/forked_navigator'
|
3
|
-
require "crabfarm/assertion/context"
|
4
6
|
|
5
7
|
module Crabfarm
|
6
8
|
class BaseNavigator
|
9
|
+
include Base
|
7
10
|
include Assertion::Context
|
11
|
+
include Live::Interactable
|
8
12
|
extend Forwardable
|
9
13
|
|
10
14
|
attr_reader :params
|
@@ -1,8 +1,12 @@
|
|
1
|
-
require
|
1
|
+
require 'crabfarm/base'
|
2
|
+
require 'crabfarm/assertion/fields'
|
3
|
+
require 'crabfarm/live/interactable'
|
2
4
|
|
3
5
|
module Crabfarm
|
4
6
|
class BaseReducer < Delegator
|
7
|
+
include Base
|
5
8
|
include Assertion::Fields
|
9
|
+
include Live::Interactable
|
6
10
|
|
7
11
|
attr_reader :raw_document, :document, :params
|
8
12
|
|
data/lib/crabfarm/base_struct.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
-
require
|
1
|
+
require 'crabfarm/base'
|
2
|
+
require 'crabfarm/assertion/context'
|
2
3
|
|
3
4
|
module Crabfarm
|
4
5
|
class BaseStruct
|
6
|
+
include Base
|
5
7
|
include Assertion::Fields
|
6
8
|
|
7
9
|
def initialize(_values={})
|
@@ -13,5 +15,9 @@ module Crabfarm
|
|
13
15
|
field_hash
|
14
16
|
end
|
15
17
|
|
18
|
+
def to_json(_options={})
|
19
|
+
field_hash.to_json(_options)
|
20
|
+
end
|
21
|
+
|
16
22
|
end
|
17
23
|
end
|
data/lib/crabfarm/cli.rb
CHANGED
@@ -27,6 +27,16 @@ module Crabfarm
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
+
desc "Starts the crawler in live mode"
|
31
|
+
command [:live, :i] do |c|
|
32
|
+
c.action do |global_options,options,args|
|
33
|
+
next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
|
34
|
+
|
35
|
+
require "crabfarm/modes/live"
|
36
|
+
Crabfarm::Modes::Live.start_watch
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
30
40
|
desc "Starts the crawler in server mode"
|
31
41
|
command [:server, :s] do |c|
|
32
42
|
c.desc "Set the server host, defaults to 0.0.0.0"
|
@@ -38,6 +38,15 @@ module Crabfarm
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
41
|
+
def kill
|
42
|
+
unless @process.nil?
|
43
|
+
logger.info "Killing crabtrap (PID: #{@process.pid})"
|
44
|
+
@process.stop 0
|
45
|
+
@process = nil
|
46
|
+
logger.info "Crabtrap stopped"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
41
50
|
private
|
42
51
|
|
43
52
|
def spawn_crabtrap
|
@@ -59,6 +68,7 @@ module Crabfarm
|
|
59
68
|
cmd << mode.to_s
|
60
69
|
cmd << @config[:bucket_path] if mode != :pass
|
61
70
|
cmd << "--port=#{port}"
|
71
|
+
cmd << "--virtual=#{@config[:virtual]}" if @config.key? :virtual
|
62
72
|
cmd
|
63
73
|
end
|
64
74
|
|
data/lib/crabfarm/driver_pool.rb
CHANGED
@@ -15,12 +15,12 @@ module Crabfarm
|
|
15
15
|
|
16
16
|
def reset(_session_id=nil)
|
17
17
|
if _session_id.nil?
|
18
|
-
@drivers.
|
18
|
+
@drivers.each { |k, d| @factory.release_driver(k, d) }
|
19
19
|
@drivers = Hash.new
|
20
20
|
else
|
21
21
|
_session_id = _session_id.to_sym
|
22
22
|
driver = @drivers.delete _session_id
|
23
|
-
@factory.release_driver driver unless driver.nil?
|
23
|
+
@factory.release_driver(_session_id, driver) unless driver.nil?
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
data/lib/crabfarm/http_client.rb
CHANGED
@@ -25,7 +25,10 @@ module Crabfarm
|
|
25
25
|
attr_reader :proxy_addr, :proxy_port
|
26
26
|
|
27
27
|
def initialize(_proxy=nil)
|
28
|
-
if
|
28
|
+
if Crabfarm.live?
|
29
|
+
@proxy_addr = '127.0.0.1'
|
30
|
+
@proxy_port = Crabfarm.live.proxy_port
|
31
|
+
elsif _proxy.nil?
|
29
32
|
@proxy_addr = nil
|
30
33
|
@proxy_port = nil
|
31
34
|
else
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'benchmark'
|
2
|
+
require 'rainbow'
|
3
|
+
require 'rainbow/ext/string'
|
4
|
+
require 'crabfarm/modes/console'
|
5
|
+
|
6
|
+
require 'crabfarm/support/webdriver_factory'
|
7
|
+
require 'crabfarm/crabtrap_runner'
|
8
|
+
|
9
|
+
require 'crabfarm/live/helpers.rb'
|
10
|
+
require 'crabfarm/live/navigator_runner.rb'
|
11
|
+
require 'crabfarm/live/reducer_runner.rb'
|
12
|
+
|
13
|
+
module Crabfarm
|
14
|
+
module Live
|
15
|
+
class Controller
|
16
|
+
|
17
|
+
Colors = Crabfarm::Modes::Console::Colors
|
18
|
+
|
19
|
+
class LiveWarning < StandardError; end
|
20
|
+
|
21
|
+
def initialize(_manager)
|
22
|
+
@manager = _manager
|
23
|
+
end
|
24
|
+
|
25
|
+
def execute_live(_class)
|
26
|
+
begin
|
27
|
+
runner = build_runner_for _class
|
28
|
+
prepare_session_for runner
|
29
|
+
elapsed = Benchmark.measure { runner.execute }
|
30
|
+
display_feedback runner, elapsed
|
31
|
+
rescue LiveWarning => exc
|
32
|
+
display_warning_feedback exc
|
33
|
+
rescue Exception => exc
|
34
|
+
display_error_feedback exc
|
35
|
+
ensure
|
36
|
+
clean_up_session
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def build_runner_for(_class)
|
43
|
+
raise LiveWarning.new "'#{_class.to_s} is not Interactable" unless _class < Interactable
|
44
|
+
|
45
|
+
puts "Launching #{_class.to_s}".color Colors::NOTICE
|
46
|
+
|
47
|
+
runner = if _class.live_delegate
|
48
|
+
build_runner_for _class.live_delegate
|
49
|
+
else
|
50
|
+
if _class < BaseNavigator
|
51
|
+
NavigatorRunner
|
52
|
+
elsif _class < BaseReducer
|
53
|
+
ReducerRunner
|
54
|
+
else
|
55
|
+
raise LiveWarning.new "Don't know how to run #{_class.to_s}, you should provide a navigator or reducer as delegate."
|
56
|
+
end.new _class
|
57
|
+
end
|
58
|
+
|
59
|
+
runner.dsl.instance_eval(&_class.live_setup) if _class.live_setup
|
60
|
+
runner
|
61
|
+
end
|
62
|
+
|
63
|
+
def prepare_session_for(_runner)
|
64
|
+
@manager.stop_crabtrap
|
65
|
+
if _runner.memento
|
66
|
+
@manager.start_crabtrap :replay, memento_path(_runner.memento)
|
67
|
+
else
|
68
|
+
@manager.start_crabtrap :pass
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def clean_up_session
|
73
|
+
# leave crabtrap running for debugging purposes.
|
74
|
+
end
|
75
|
+
|
76
|
+
def display_feedback(_runner, _elapsed)
|
77
|
+
load_web_ui
|
78
|
+
|
79
|
+
@manager.primary_driver.execute_script(
|
80
|
+
"window.crabfarm.showResults(arguments[0], arguments[1]);",
|
81
|
+
_runner.output,
|
82
|
+
_elapsed.real
|
83
|
+
);
|
84
|
+
|
85
|
+
puts _runner.output.to_s.color Colors::RESULT
|
86
|
+
puts "Completed in #{_elapsed.real} s".color Colors::NOTICE
|
87
|
+
end
|
88
|
+
|
89
|
+
def display_error_feedback(_exc)
|
90
|
+
load_web_ui
|
91
|
+
|
92
|
+
@manager.primary_driver.execute_script(
|
93
|
+
"window.crabfarm.showError(arguments[0], arguments[1]);",
|
94
|
+
"#{_exc.class.to_s}: #{_exc.to_s}",
|
95
|
+
_exc.backtrace.join("\n")
|
96
|
+
);
|
97
|
+
|
98
|
+
puts "#{_exc.class.to_s}: #{_exc.to_s}".color Colors::ERROR
|
99
|
+
puts _exc.backtrace
|
100
|
+
end
|
101
|
+
|
102
|
+
def display_warning_feedback(_exc)
|
103
|
+
load_web_ui
|
104
|
+
puts "Warning: #{_exc.to_s}".color Colors::WARNING
|
105
|
+
end
|
106
|
+
|
107
|
+
def load_web_ui
|
108
|
+
Helpers.inject_style @manager.primary_driver, 'https://www.crabtrap.io/selectorgadget_combined.css'
|
109
|
+
Helpers.inject_style @manager.primary_driver, 'https://www.crabtrap.io/tools.css'
|
110
|
+
Helpers.inject_script @manager.primary_driver, 'https://www.crabtrap.io/selectorgadget_combined.js'
|
111
|
+
Helpers.inject_script @manager.primary_driver, 'https://www.crabtrap.io/tools.js'
|
112
|
+
wait_for_injection
|
113
|
+
end
|
114
|
+
|
115
|
+
def wait_for_injection
|
116
|
+
while @manager.primary_driver.execute_script "return (typeof window.crabfarm === 'undefined');"
|
117
|
+
sleep 1.0
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def memento_path(_name)
|
122
|
+
File.join(GlobalState.mementos_path, _name + '.json.gz')
|
123
|
+
end
|
124
|
+
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Crabfarm
|
2
|
+
module Live
|
3
|
+
module Helpers
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def inject_script(_driver, _path)
|
7
|
+
_driver.execute_script("
|
8
|
+
(function() {
|
9
|
+
var script = document.createElement('script');
|
10
|
+
script.async = false;
|
11
|
+
script.src = '#{_path}';
|
12
|
+
document.head.appendChild(script);
|
13
|
+
})();
|
14
|
+
")
|
15
|
+
end
|
16
|
+
|
17
|
+
def inject_style(_driver, _path)
|
18
|
+
_driver.execute_script("
|
19
|
+
(function() {
|
20
|
+
var link = document.createElement('link');
|
21
|
+
link.setAttribute('rel','stylesheet');
|
22
|
+
link.setAttribute('type','text/css');
|
23
|
+
link.setAttribute('href','#{_path}');
|
24
|
+
link.setAttribute('media','all');
|
25
|
+
document.head.appendChild(link);
|
26
|
+
})();
|
27
|
+
")
|
28
|
+
end
|
29
|
+
|
30
|
+
def set_style(_elements, _style)
|
31
|
+
return if _elements.size == 0
|
32
|
+
# Not sure about using a bridge method directly here...
|
33
|
+
_elements.first.send(:bridge).executeScript("
|
34
|
+
for(var i = 0, l = arguments[0].length; i < l; i++) {
|
35
|
+
arguments[0][i].setAttribute('style', arguments[1]);
|
36
|
+
}
|
37
|
+
", _elements, _style)
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'crabfarm/live/helpers'
|
2
|
+
|
3
|
+
module Crabfarm
|
4
|
+
module Live
|
5
|
+
module Interactable
|
6
|
+
|
7
|
+
def self.included(klass)
|
8
|
+
klass.extend ClassMethods
|
9
|
+
end
|
10
|
+
|
11
|
+
module ClassMethods
|
12
|
+
|
13
|
+
def live(_options={}, &_setup)
|
14
|
+
@delegate = _options[:delegate]
|
15
|
+
@setup = _setup
|
16
|
+
end
|
17
|
+
|
18
|
+
def live_delegate
|
19
|
+
@delegate
|
20
|
+
end
|
21
|
+
|
22
|
+
def live_setup
|
23
|
+
@setup
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
# Tooling
|
29
|
+
|
30
|
+
def highlight(_elements)
|
31
|
+
if Crabfarm.live?
|
32
|
+
if _elements.respond_to? :webdriver_elements
|
33
|
+
_elements = _elements.webdriver_elements
|
34
|
+
end
|
35
|
+
|
36
|
+
if _elements.is_a? String
|
37
|
+
_elements = Crabfarm.live.primary_driver.find_elements(css: _elements)
|
38
|
+
end
|
39
|
+
|
40
|
+
Helpers.set_style _elements, "border: 3px solid yellow;"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'crabfarm/support/webdriver_factory'
|
2
|
+
require 'crabfarm/crabtrap_runner'
|
3
|
+
|
4
|
+
module Crabfarm
|
5
|
+
module Live
|
6
|
+
class Manager
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@port = Utils::PortDiscovery.find_available_port
|
10
|
+
@driver = Crabfarm.config.recorder_driver
|
11
|
+
end
|
12
|
+
|
13
|
+
def proxy_port
|
14
|
+
@port
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
load_primary_driver
|
19
|
+
end
|
20
|
+
|
21
|
+
def stop
|
22
|
+
stop_crabtrap
|
23
|
+
release_primary_driver
|
24
|
+
end
|
25
|
+
|
26
|
+
def primary_driver
|
27
|
+
@driver
|
28
|
+
end
|
29
|
+
|
30
|
+
def generate_support_driver
|
31
|
+
# TODO: improve on this mechanics, maybe use a frame in the same driver for this
|
32
|
+
build_driver
|
33
|
+
end
|
34
|
+
|
35
|
+
def start_crabtrap(_mode, _memento_path=nil)
|
36
|
+
|
37
|
+
raise ConfigurationError.new "No memento found at #{_memento_path}" unless File.exists? _memento_path
|
38
|
+
|
39
|
+
if @crabtrap.nil?
|
40
|
+
options = {
|
41
|
+
mode: _mode,
|
42
|
+
bucket_path: _memento_path,
|
43
|
+
port: @port,
|
44
|
+
virtual: File.expand_path('./assets/live-tools', Crabfarm.root)
|
45
|
+
}
|
46
|
+
|
47
|
+
@crabtrap = CrabtrapRunner.new config.crabtrap_config.merge(options)
|
48
|
+
@crabtrap.start
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def stop_crabtrap
|
53
|
+
unless @crabtrap.nil?
|
54
|
+
@crabtrap.kill
|
55
|
+
@crabtrap = nil
|
56
|
+
else nil end
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
def load_primary_driver
|
62
|
+
@driver = build_driver
|
63
|
+
end
|
64
|
+
|
65
|
+
def release_primary_driver
|
66
|
+
@driver.quit rescue nil unless @driver.nil?
|
67
|
+
end
|
68
|
+
|
69
|
+
def build_driver
|
70
|
+
case @driver
|
71
|
+
when :firefox
|
72
|
+
Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
|
73
|
+
when :chrome
|
74
|
+
Crabfarm::Support::WebdriverFactory.build_chrome_driver driver_config
|
75
|
+
else return nil end
|
76
|
+
end
|
77
|
+
|
78
|
+
def driver_config
|
79
|
+
{
|
80
|
+
proxy: "127.0.0.1:#{@port}"
|
81
|
+
}
|
82
|
+
end
|
83
|
+
|
84
|
+
def config
|
85
|
+
Crabfarm.config
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Crabfarm
|
4
|
+
module Live
|
5
|
+
class NavigatorRunner
|
6
|
+
|
7
|
+
class Dsl
|
8
|
+
extend Forwardable
|
9
|
+
|
10
|
+
def initialize(_runner)
|
11
|
+
@runner = _runner
|
12
|
+
end
|
13
|
+
|
14
|
+
def_delegators :@runner, :use_memento, :use_params, :clear_params, :navigate_to
|
15
|
+
end
|
16
|
+
|
17
|
+
def initialize(_target)
|
18
|
+
@target = _target
|
19
|
+
@params = {}
|
20
|
+
end
|
21
|
+
|
22
|
+
def dsl
|
23
|
+
@dsl ||= Dsl.new self
|
24
|
+
end
|
25
|
+
|
26
|
+
def memento
|
27
|
+
if @memento.nil? then memento_for(@target) else @memento end
|
28
|
+
end
|
29
|
+
|
30
|
+
def use_memento(_memento)
|
31
|
+
@memento = _memento
|
32
|
+
end
|
33
|
+
|
34
|
+
def use_params(_params={})
|
35
|
+
@params = @params.merge _params
|
36
|
+
end
|
37
|
+
|
38
|
+
def clear_params
|
39
|
+
@params = {}
|
40
|
+
end
|
41
|
+
|
42
|
+
def navigate_to(_navigator, _params={})
|
43
|
+
end
|
44
|
+
|
45
|
+
def execute
|
46
|
+
context = Crabfarm::Context.new
|
47
|
+
begin
|
48
|
+
# TODO: execute prerequisites
|
49
|
+
@transition = TransitionService.transition context, @target, @params
|
50
|
+
ensure
|
51
|
+
context.release
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def output
|
56
|
+
return nil if @transition.document.nil?
|
57
|
+
JSON.pretty_generate(@transition.document).gsub(/(^|\\n)/, ' ')
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def memento_for(_class)
|
63
|
+
Utils::Naming.route_from_constant(_class.to_s).join(File::SEPARATOR)
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Crabfarm
|
2
|
+
module Live
|
3
|
+
class ReducerRunner < NavigatorRunner
|
4
|
+
|
5
|
+
def initialize(_target)
|
6
|
+
# use navigator runner for now.
|
7
|
+
super navigator_from_reducer _target
|
8
|
+
end
|
9
|
+
|
10
|
+
def navigator_from_reducer _reducer
|
11
|
+
m = _reducer.to_s.match(/^(.*?)Reducer$/)
|
12
|
+
if m
|
13
|
+
navigator = m[1].constantize rescue nil
|
14
|
+
return navigator if navigator and navigator < BaseNavigator
|
15
|
+
end
|
16
|
+
|
17
|
+
raise Controller::LiveWarning.new "Could not find a matching navigator for reducer #{_reducer.to_s}"
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'listen'
|
2
|
+
|
3
|
+
module Crabfarm
|
4
|
+
module Live
|
5
|
+
class Watcher
|
6
|
+
|
7
|
+
PATH_RGX = /^\/[^\/]+\/(.*?)\.rb$/i
|
8
|
+
|
9
|
+
def initialize(_controller)
|
10
|
+
@controller = _controller
|
11
|
+
@candidates = nil
|
12
|
+
end
|
13
|
+
|
14
|
+
def watch(_sleep)
|
15
|
+
begin
|
16
|
+
start_listener
|
17
|
+
loop do
|
18
|
+
execute_pending
|
19
|
+
sleep _sleep
|
20
|
+
end
|
21
|
+
ensure
|
22
|
+
stop_listener
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def start_listener
|
29
|
+
base_path = File.join CF_PATH, 'app'
|
30
|
+
@listener = Listen.to(base_path) do |modified, added, removed|
|
31
|
+
@candidates = (added + modified).map do |path|
|
32
|
+
class_from_path path[base_path.length..-1]
|
33
|
+
end.reject &:nil?
|
34
|
+
end
|
35
|
+
@listener.start
|
36
|
+
end
|
37
|
+
|
38
|
+
def stop_listener
|
39
|
+
@listener.stop if @listener
|
40
|
+
end
|
41
|
+
|
42
|
+
def execute_pending
|
43
|
+
unless @candidates.nil?
|
44
|
+
ActiveSupport::Dependencies.clear
|
45
|
+
@candidates.each do |class_name|
|
46
|
+
target = class_name.constantize rescue nil
|
47
|
+
if target and target < Crabfarm::Live::Interactable
|
48
|
+
@controller.execute_live target
|
49
|
+
break
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
@candidates = nil
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def class_from_path(_filename)
|
58
|
+
_filename = _filename.gsub File::SEPARATOR, '/'
|
59
|
+
m = _filename.match PATH_RGX
|
60
|
+
return nil if m.nil?
|
61
|
+
Utils::Naming.decode_crabfarm_uri m[1]
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'crabfarm/live/watcher'
|
2
|
+
require 'crabfarm/live/controller'
|
3
|
+
|
4
|
+
module Crabfarm
|
5
|
+
module Modes
|
6
|
+
module Live
|
7
|
+
extend self
|
8
|
+
|
9
|
+
def start_watch
|
10
|
+
begin
|
11
|
+
Crabfarm.enable_debugging!
|
12
|
+
Crabfarm.install_live_backend!
|
13
|
+
Crabfarm.live.start
|
14
|
+
|
15
|
+
controller = Crabfarm::Live::Controller.new Crabfarm.live
|
16
|
+
watcher = Crabfarm::Live::Watcher.new controller
|
17
|
+
watcher.watch 0.2
|
18
|
+
|
19
|
+
rescue SystemExit, Interrupt
|
20
|
+
# nothing
|
21
|
+
rescue Exception => e
|
22
|
+
puts "Fatal error: #{e.to_s}".color Console::Colors::ERROR
|
23
|
+
puts e.backtrace
|
24
|
+
ensure
|
25
|
+
puts 'Exiting'
|
26
|
+
Crabfarm.live.stop rescue nil
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -28,11 +28,12 @@ module Crabfarm
|
|
28
28
|
begin
|
29
29
|
puts "Press Ctrl-C or close browser to stop #{_replay ? 'playback' : 'capturing'}."
|
30
30
|
loop do
|
31
|
-
driver.
|
31
|
+
driver.window_handle
|
32
32
|
sleep 1.0
|
33
33
|
end
|
34
34
|
rescue Exception => e
|
35
|
-
|
35
|
+
puts e
|
36
|
+
puts e.backtrace
|
36
37
|
end
|
37
38
|
|
38
39
|
puts "Releasing crawling context".color(:green)
|
data/lib/crabfarm/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crabfarm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ignacio Baixas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -128,6 +128,34 @@ dependencies:
|
|
128
128
|
- - ~>
|
129
129
|
- !ruby/object:Gem::Version
|
130
130
|
version: 0.5.5
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
name: listen
|
133
|
+
requirement: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ~>
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '2.7'
|
138
|
+
type: :runtime
|
139
|
+
prerelease: false
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - ~>
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '2.7'
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
name: pry-byebug
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
148
|
+
requirements:
|
149
|
+
- - '>='
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '0'
|
152
|
+
type: :runtime
|
153
|
+
prerelease: false
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - '>='
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
131
159
|
- !ruby/object:Gem::Dependency
|
132
160
|
name: selenium-webdriver
|
133
161
|
requirement: !ruby/object:Gem::Requirement
|
@@ -390,6 +418,7 @@ files:
|
|
390
418
|
- lib/crabfarm/assertion/parsers.rb
|
391
419
|
- lib/crabfarm/assertion/validations.rb
|
392
420
|
- lib/crabfarm/assertion/wrapper.rb
|
421
|
+
- lib/crabfarm/base.rb
|
393
422
|
- lib/crabfarm/base_navigator.rb
|
394
423
|
- lib/crabfarm/base_reducer.rb
|
395
424
|
- lib/crabfarm/base_struct.rb
|
@@ -409,8 +438,16 @@ files:
|
|
409
438
|
- lib/crabfarm/forked_navigator.rb
|
410
439
|
- lib/crabfarm/global_state.rb
|
411
440
|
- lib/crabfarm/http_client.rb
|
441
|
+
- lib/crabfarm/live/controller.rb
|
442
|
+
- lib/crabfarm/live/helpers.rb
|
443
|
+
- lib/crabfarm/live/interactable.rb
|
444
|
+
- lib/crabfarm/live/manager.rb
|
445
|
+
- lib/crabfarm/live/navigator_runner.rb
|
446
|
+
- lib/crabfarm/live/reducer_runner.rb
|
447
|
+
- lib/crabfarm/live/watcher.rb
|
412
448
|
- lib/crabfarm/modes/console.rb
|
413
449
|
- lib/crabfarm/modes/generator.rb
|
450
|
+
- lib/crabfarm/modes/live.rb
|
414
451
|
- lib/crabfarm/modes/publisher.rb
|
415
452
|
- lib/crabfarm/modes/recorder/memento.rb
|
416
453
|
- lib/crabfarm/modes/recorder/snapshot.rb
|