crabfarm 0.6.1 → 0.6.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/assets/live-tools/tools.js +0 -1
- data/lib/crabfarm/adapters/browser/abstract_webdriver.rb +13 -28
- data/lib/crabfarm/adapters/browser/base.rb +43 -0
- data/lib/crabfarm/adapters/browser/chrome.rb +5 -1
- data/lib/crabfarm/adapters/browser/firefox.rb +4 -0
- data/lib/crabfarm/adapters/browser/noop.rb +3 -10
- data/lib/crabfarm/adapters/browser/phantom_js.rb +8 -8
- data/lib/crabfarm/adapters/driver_wrapper/pincers.rb +6 -0
- data/lib/crabfarm/cli.rb +1 -7
- data/lib/crabfarm/context.rb +17 -21
- data/lib/crabfarm/driver_pool.rb +3 -2
- data/lib/crabfarm/errors.rb +2 -0
- data/lib/crabfarm/http_client.rb +1 -1
- data/lib/crabfarm/live/context.rb +32 -2
- data/lib/crabfarm/live/controller.rb +19 -16
- data/lib/crabfarm/live/interactable.rb +5 -14
- data/lib/crabfarm/live/manager.rb +85 -95
- data/lib/crabfarm/live/navigator_runner.rb +43 -40
- data/lib/crabfarm/live/navigator_runner_direct.rb +1 -2
- data/lib/crabfarm/live/navigator_runner_rspec.rb +34 -55
- data/lib/crabfarm/live/reducer_runner.rb +43 -22
- data/lib/crabfarm/live/reducer_runner_direct.rb +1 -2
- data/lib/crabfarm/live/reducer_runner_rspec.rb +0 -5
- data/lib/crabfarm/live/viewer.rb +92 -0
- data/lib/crabfarm/live/watcher.rb +6 -1
- data/lib/crabfarm/modes/live.rb +1 -1
- data/lib/crabfarm/modes/recorder/snapshot.rb +9 -25
- data/lib/crabfarm/support/webdriver_factory.rb +1 -1
- data/lib/crabfarm/templates/Gemfile.erb +1 -1
- data/lib/crabfarm/utils/rspec_runner.rb +54 -22
- data/lib/crabfarm/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 09ceb80d5125962d7f54f0b7d9f79bc568221f18
|
4
|
+
data.tar.gz: f72e8f9951b3ae981bece342c9a7364b007677e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1572b106739d6f9a7b2ae385bb640b926f05707e31daf0a99758bb308ab8020a374636fc9df7eb56cf868c3a67aaa861d9538d859438367bc7a300e440613bf5
|
7
|
+
data.tar.gz: 39049c8c5b3b5c115dfdc11c92a67bb81411aff7512c16b524c6874c66cf0638adcb9a1a369628e83f5a50d667cabb8c494325f534b94a468ea941969f9790b7
|
data/assets/live-tools/tools.js
CHANGED
@@ -1,38 +1,31 @@
|
|
1
|
+
require 'crabfarm/adapters/browser/base'
|
2
|
+
|
1
3
|
module Crabfarm
|
2
4
|
module Adapters
|
3
5
|
module Browser
|
4
|
-
class AbstractWebdriver
|
6
|
+
class AbstractWebdriver < Base
|
5
7
|
|
6
|
-
attr_accessor :config
|
8
|
+
attr_accessor :config
|
7
9
|
|
8
|
-
def initialize(_proxy=nil
|
10
|
+
def initialize(_proxy=nil)
|
9
11
|
@config = load_driver_config
|
10
12
|
@config[:proxy] = _proxy
|
11
|
-
@viewer = _viewer
|
12
13
|
end
|
13
14
|
|
14
|
-
def
|
15
|
-
|
15
|
+
def build_driver(_session_id)
|
16
|
+
wrap_driver build_webdriver_instance
|
16
17
|
end
|
17
18
|
|
18
|
-
def
|
19
|
-
|
19
|
+
def reset_driver(_wrapped)
|
20
|
+
_wrapped.driver.manage.delete_all_cookies # pincers exposes driver?
|
20
21
|
end
|
21
22
|
|
22
|
-
def
|
23
|
-
|
24
|
-
build_webdriver_instance
|
25
|
-
else
|
26
|
-
viewer.attach _session_id == :default_driver
|
27
|
-
end)
|
23
|
+
def extract_webdriver(_wrapped)
|
24
|
+
_wrapped.driver
|
28
25
|
end
|
29
26
|
|
30
|
-
def release_driver(
|
31
|
-
|
32
|
-
_wrapped.driver.quit rescue nil
|
33
|
-
else
|
34
|
-
viewer.detach _wrapped.driver
|
35
|
-
end
|
27
|
+
def release_driver(_wrapped)
|
28
|
+
_wrapped.driver.quit rescue nil
|
36
29
|
end
|
37
30
|
|
38
31
|
private
|
@@ -47,14 +40,6 @@ module Crabfarm
|
|
47
40
|
raise NotImplementedError.new
|
48
41
|
end
|
49
42
|
|
50
|
-
def start_server
|
51
|
-
# Nothing by default
|
52
|
-
end
|
53
|
-
|
54
|
-
def stop_server
|
55
|
-
# Nothing by default
|
56
|
-
end
|
57
|
-
|
58
43
|
def load_driver_config
|
59
44
|
{
|
60
45
|
capabilities: Crabfarm.config.webdriver_capabilities,
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Crabfarm
|
2
|
+
module Adapters
|
3
|
+
module Browser
|
4
|
+
class Base
|
5
|
+
|
6
|
+
def headless?
|
7
|
+
true
|
8
|
+
end
|
9
|
+
|
10
|
+
def build_driver(_session_id)
|
11
|
+
ensure_implementation :build_driver
|
12
|
+
end
|
13
|
+
|
14
|
+
def reset_driver(_driver)
|
15
|
+
nil
|
16
|
+
end
|
17
|
+
|
18
|
+
def extract_webdriver(_driver)
|
19
|
+
nil
|
20
|
+
end
|
21
|
+
|
22
|
+
def release_driver(_driver)
|
23
|
+
nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def prepare_driver_services
|
27
|
+
# Nothing by default
|
28
|
+
end
|
29
|
+
|
30
|
+
def cleanup_driver_services
|
31
|
+
# Nothing by default
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def ensure_implementation(_name)
|
37
|
+
raise NotImplementedError.new "Missing #{_name} implementation on browser adapter"
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -1,11 +1,15 @@
|
|
1
1
|
require 'crabfarm/support/webdriver_factory'
|
2
|
-
require
|
2
|
+
require 'crabfarm/adapters/browser/abstract_webdriver'
|
3
3
|
|
4
4
|
module Crabfarm
|
5
5
|
module Adapters
|
6
6
|
module Browser
|
7
7
|
class Chrome < AbstractWebdriver
|
8
8
|
|
9
|
+
def headless?
|
10
|
+
false
|
11
|
+
end
|
12
|
+
|
9
13
|
private
|
10
14
|
|
11
15
|
def build_webdriver_instance
|
@@ -1,24 +1,17 @@
|
|
1
|
+
require "crabfarm/adapters/browser/base"
|
2
|
+
|
1
3
|
module Crabfarm
|
2
4
|
module Adapters
|
3
5
|
module Browser
|
4
|
-
class Noop
|
6
|
+
class Noop < Base
|
5
7
|
|
6
8
|
def initialize(_proxy=nil)
|
7
9
|
end
|
8
10
|
|
9
|
-
def prepare_driver_services
|
10
|
-
end
|
11
|
-
|
12
|
-
def cleanup_driver_services
|
13
|
-
end
|
14
|
-
|
15
11
|
def build_driver(_session_id)
|
16
12
|
_session_id || :noop
|
17
13
|
end
|
18
14
|
|
19
|
-
def release_driver(_driver)
|
20
|
-
end
|
21
|
-
|
22
15
|
end
|
23
16
|
end
|
24
17
|
end
|
@@ -7,21 +7,21 @@ module Crabfarm
|
|
7
7
|
module Browser
|
8
8
|
class PhantomJs < AbstractWebdriver
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
def build_webdriver_instance
|
13
|
-
Support::WebdriverFactory.build_remote_driver driver_config
|
14
|
-
end
|
15
|
-
|
16
|
-
def start_server
|
10
|
+
def prepare_driver_services
|
17
11
|
@phantom = load_and_start_phantom if @phantom.nil?
|
18
12
|
end
|
19
13
|
|
20
|
-
def
|
14
|
+
def cleanup_driver_services
|
21
15
|
@phantom.stop unless @phantom.nil?
|
22
16
|
@phantom = nil
|
23
17
|
end
|
24
18
|
|
19
|
+
private
|
20
|
+
|
21
|
+
def build_webdriver_instance
|
22
|
+
Support::WebdriverFactory.build_remote_driver driver_config
|
23
|
+
end
|
24
|
+
|
25
25
|
def load_and_start_phantom
|
26
26
|
new_phantom = Support::PhantomRunner.new phantom_config
|
27
27
|
new_phantom.start
|
data/lib/crabfarm/cli.rb
CHANGED
@@ -155,18 +155,12 @@ module Crabfarm
|
|
155
155
|
c.desc "Take snapshots from a navigator run"
|
156
156
|
c.command [:snapshot, :snap] do |snap|
|
157
157
|
|
158
|
-
snap.desc "Navigate on a memento"
|
159
|
-
snap.flag [:m, :memento]
|
160
|
-
|
161
|
-
snap.desc "Navigation query string, if not given navigator is run in interactive mode"
|
162
|
-
snap.flag [:q, :query]
|
163
|
-
|
164
158
|
snap.action do |global_options, options, args|
|
165
159
|
next puts "This command can only be ran inside a crabfarm application" unless Crabfarm.inside_crawler_app?
|
166
160
|
|
167
161
|
Crabfarm.with_context options[:memento] do |context|
|
168
162
|
require "crabfarm/modes/recorder/snapshot"
|
169
|
-
Crabfarm::Modes::Recorder::Snapshot.start context, args[0]
|
163
|
+
Crabfarm::Modes::Recorder::Snapshot.start context, args[0]
|
170
164
|
end
|
171
165
|
end
|
172
166
|
|
data/lib/crabfarm/context.rb
CHANGED
@@ -32,7 +32,7 @@ module Crabfarm
|
|
32
32
|
private
|
33
33
|
|
34
34
|
def load_services
|
35
|
-
|
35
|
+
init_browser_adapter
|
36
36
|
init_driver_pool
|
37
37
|
init_http_client
|
38
38
|
end
|
@@ -45,27 +45,23 @@ module Crabfarm
|
|
45
45
|
def unload_services
|
46
46
|
release_http_client
|
47
47
|
release_driver_pool
|
48
|
-
|
48
|
+
release_browser_adapter
|
49
49
|
end
|
50
50
|
|
51
|
-
def
|
52
|
-
if @
|
53
|
-
@
|
54
|
-
@
|
51
|
+
def init_browser_adapter
|
52
|
+
if @browser_adapter.nil?
|
53
|
+
@browser_adapter = build_browser_adapter proxy
|
54
|
+
@browser_adapter.prepare_driver_services
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
58
|
-
def
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
def release_driver_factory
|
63
|
-
@factory.cleanup_driver_services unless @factory.nil?
|
64
|
-
@factory.nil?
|
58
|
+
def release_browser_adapter
|
59
|
+
@browser_adapter.cleanup_driver_services unless @browser_adapter.nil?
|
60
|
+
@browser_adapter.nil?
|
65
61
|
end
|
66
62
|
|
67
63
|
def init_driver_pool
|
68
|
-
@pool = DriverPool.new @
|
64
|
+
@pool = DriverPool.new @browser_adapter if @pool.nil?
|
69
65
|
end
|
70
66
|
|
71
67
|
def release_driver_pool
|
@@ -74,11 +70,15 @@ module Crabfarm
|
|
74
70
|
end
|
75
71
|
|
76
72
|
def init_http_client
|
77
|
-
@http = build_http_client if @http.nil?
|
73
|
+
@http = build_http_client proxy if @http.nil?
|
74
|
+
end
|
75
|
+
|
76
|
+
def build_browser_adapter(_proxy)
|
77
|
+
Strategies.load(:browser, config.browser).new _proxy
|
78
78
|
end
|
79
79
|
|
80
|
-
def build_http_client
|
81
|
-
HttpClient.new
|
80
|
+
def build_http_client(_proxy)
|
81
|
+
HttpClient.new _proxy
|
82
82
|
end
|
83
83
|
|
84
84
|
def release_http_client
|
@@ -89,10 +89,6 @@ module Crabfarm
|
|
89
89
|
config.proxy
|
90
90
|
end
|
91
91
|
|
92
|
-
def viewer
|
93
|
-
nil # no viewer provided by default
|
94
|
-
end
|
95
|
-
|
96
92
|
def config
|
97
93
|
Crabfarm.config
|
98
94
|
end
|
data/lib/crabfarm/driver_pool.rb
CHANGED
@@ -15,13 +15,14 @@ module Crabfarm
|
|
15
15
|
|
16
16
|
def reset(_session_id=nil)
|
17
17
|
if _session_id.nil?
|
18
|
-
@drivers.each { |
|
18
|
+
@drivers.values.each { |d| @factory.release_driver d }
|
19
19
|
@drivers = Hash.new
|
20
20
|
else
|
21
21
|
_session_id = _session_id.to_sym
|
22
22
|
driver = @drivers.delete _session_id
|
23
|
-
@factory.release_driver(
|
23
|
+
@factory.release_driver(driver) unless driver.nil?
|
24
24
|
end
|
25
|
+
nil
|
25
26
|
end
|
26
27
|
|
27
28
|
def release
|
data/lib/crabfarm/errors.rb
CHANGED
data/lib/crabfarm/http_client.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'crabfarm/adapters/browser/base'
|
1
2
|
require 'crabfarm/utils/webdriver'
|
2
3
|
|
3
4
|
module Crabfarm
|
@@ -9,11 +10,40 @@ module Crabfarm
|
|
9
10
|
end
|
10
11
|
|
11
12
|
def proxy
|
13
|
+
# override proxy so every context service points to crabtrap
|
12
14
|
"127.0.0.1:#{@manager.proxy_port}"
|
13
15
|
end
|
14
16
|
|
15
|
-
|
16
|
-
|
17
|
+
private
|
18
|
+
|
19
|
+
def build_browser_adapter(_proxy)
|
20
|
+
# use a special browser adapter to override primary driver
|
21
|
+
return BrowserAdapter.new @manager
|
22
|
+
end
|
23
|
+
|
24
|
+
# TODO: override build_http_client, i would like to tap into the http requests and show downloaded data in viewer
|
25
|
+
|
26
|
+
class BrowserAdapter < Crabfarm::Adapters::Browser::Base
|
27
|
+
|
28
|
+
def initialize(_manager)
|
29
|
+
@manager = _manager
|
30
|
+
end
|
31
|
+
|
32
|
+
def build_driver(_session_id)
|
33
|
+
if _session_id == :default_driver
|
34
|
+
@manager.primary_driver
|
35
|
+
else
|
36
|
+
@manager.browser_adapter.build_driver _session_id
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def release_driver(_driver)
|
41
|
+
if _driver != @manager.primary_driver
|
42
|
+
@manager.browser_adapter.release_driver _driver
|
43
|
+
end
|
44
|
+
nil
|
45
|
+
end
|
46
|
+
|
17
47
|
end
|
18
48
|
|
19
49
|
end
|
@@ -11,24 +11,36 @@ module Crabfarm
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def display_external_error(_exc)
|
14
|
-
|
14
|
+
try_reset
|
15
|
+
|
15
16
|
display_error_feedback _exc
|
16
17
|
end
|
17
18
|
|
18
19
|
def execute_live(_class)
|
20
|
+
try_reset
|
21
|
+
|
19
22
|
begin
|
20
|
-
|
21
|
-
prepare_session_for runner
|
22
|
-
runner.execute
|
23
|
+
build_runner_for(_class).execute
|
23
24
|
rescue Exception => exc
|
24
25
|
display_error_feedback exc
|
25
|
-
ensure
|
26
|
-
clean_up_session
|
27
26
|
end
|
28
27
|
end
|
29
28
|
|
30
29
|
private
|
31
30
|
|
31
|
+
def try_reset
|
32
|
+
begin
|
33
|
+
@manager.reset
|
34
|
+
rescue Exception => exc
|
35
|
+
# restart manager if reset failed
|
36
|
+
@manager.stop rescue nil
|
37
|
+
@manager.start
|
38
|
+
|
39
|
+
Utils::Console.error "Something went wrong, restarting live mode:"
|
40
|
+
Utils::Console.exception exc
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
32
44
|
def build_runner_for(_class)
|
33
45
|
raise ArgumentError.new "'#{_class.to_s} is not Interactable" unless _class < Interactable
|
34
46
|
|
@@ -50,17 +62,8 @@ module Crabfarm
|
|
50
62
|
runner
|
51
63
|
end
|
52
64
|
|
53
|
-
def prepare_session_for(_runner)
|
54
|
-
@manager.reset_driver_status
|
55
|
-
end
|
56
|
-
|
57
|
-
def clean_up_session
|
58
|
-
# leave crabtrap running for debugging purposes.
|
59
|
-
end
|
60
|
-
|
61
65
|
def display_error_feedback(_exc)
|
62
|
-
@manager.
|
63
|
-
@manager.show_dialog(
|
66
|
+
@manager.show_message(
|
64
67
|
:error,
|
65
68
|
'Crawler error!',
|
66
69
|
"#{_exc.class.to_s}: #{_exc.to_s}",
|