crabfarm 0.4.2 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/assets/live-tools/instructions.html +17 -0
- data/assets/live-tools/tools.css +16 -0
- data/assets/live-tools/tools.js +41 -47
- data/assets/live-tools/welcome.html +18 -0
- data/lib/crabfarm/adapters/browser/abstract_webdriver.rb +19 -16
- data/lib/crabfarm/adapters/browser/phantom_js.rb +11 -11
- data/lib/crabfarm/assertion/fields.rb +8 -0
- data/lib/crabfarm/base_navigator.rb +4 -11
- data/lib/crabfarm/base_reducer.rb +5 -1
- data/lib/crabfarm/cli.rb +13 -13
- data/lib/crabfarm/context.rb +15 -3
- data/lib/crabfarm/factories/context.rb +24 -0
- data/lib/crabfarm/factories/decorable.rb +71 -0
- data/lib/crabfarm/factories/navigator.rb +14 -0
- data/lib/crabfarm/factories/reducer.rb +14 -0
- data/lib/crabfarm/factories/snapshot_reducer.rb +14 -0
- data/lib/crabfarm/http_client.rb +2 -5
- data/lib/crabfarm/live/context.rb +21 -0
- data/lib/crabfarm/live/controller.rb +20 -72
- data/lib/crabfarm/live/interactable.rb +6 -2
- data/lib/crabfarm/live/manager.rb +90 -18
- data/lib/crabfarm/live/navigator_runner.rb +43 -13
- data/lib/crabfarm/live/navigator_runner_direct.rb +39 -0
- data/lib/crabfarm/live/navigator_runner_rspec.rb +111 -0
- data/lib/crabfarm/live/reducer_runner.rb +61 -10
- data/lib/crabfarm/live/reducer_runner_direct.rb +41 -0
- data/lib/crabfarm/live/reducer_runner_rspec.rb +23 -0
- data/lib/crabfarm/live/watcher.rb +21 -7
- data/lib/crabfarm/modes/console.rb +1 -1
- data/lib/crabfarm/modes/live.rb +7 -4
- data/lib/crabfarm/modes/recorder/memento.rb +1 -5
- data/lib/crabfarm/modes/recorder/snapshot.rb +2 -2
- data/lib/crabfarm/modes/shared/interactive_decorator.rb +0 -1
- data/lib/crabfarm/modes/shared/snapshot_decorator.rb +17 -22
- data/lib/crabfarm/rspec/navigator_spec_helpers.rb +56 -0
- data/lib/crabfarm/rspec/reducer_spec_helpers.rb +29 -0
- data/lib/crabfarm/rspec/reducer_spy.rb +36 -0
- data/lib/crabfarm/rspec/reducer_spy_manager.rb +38 -0
- data/lib/crabfarm/rspec.rb +22 -66
- data/lib/crabfarm/support/phantom_runner.rb +77 -0
- data/lib/crabfarm/templates/Gemfile.erb +1 -0
- data/lib/crabfarm/transition_service.rb +3 -48
- data/lib/crabfarm/utils/console.rb +77 -0
- data/lib/crabfarm/utils/naming.rb +4 -3
- data/lib/crabfarm/utils/resolve.rb +39 -0
- data/lib/crabfarm/utils/rspec_runner.rb +54 -0
- data/lib/crabfarm/{live/helpers.rb → utils/webdriver.rb} +3 -3
- data/lib/crabfarm/version.rb +1 -1
- data/lib/crabfarm.rb +24 -3
- metadata +23 -6
- data/lib/crabfarm/context_factory.rb +0 -32
- data/lib/crabfarm/global_state.rb +0 -22
- data/lib/crabfarm/phantom_runner.rb +0 -75
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'inquirer'
|
3
|
+
require 'rainbow'
|
4
|
+
require 'rainbow/ext/string'
|
5
|
+
|
6
|
+
module Crabfarm
|
7
|
+
module Utils
|
8
|
+
module Console
|
9
|
+
extend self
|
10
|
+
|
11
|
+
COLOR_INFO = '888888'
|
12
|
+
COLOR_QUESTION = '888888'
|
13
|
+
COLOR_WARNING = :yellow
|
14
|
+
COLOR_ERROR = :red
|
15
|
+
COLOR_HIGHLIGHT = '00FF00'
|
16
|
+
|
17
|
+
def system(_message)
|
18
|
+
puts _message
|
19
|
+
end
|
20
|
+
|
21
|
+
def operation(_message)
|
22
|
+
puts _message
|
23
|
+
end
|
24
|
+
|
25
|
+
def info(_message)
|
26
|
+
puts _message.color COLOR_INFO
|
27
|
+
end
|
28
|
+
|
29
|
+
def result(_message)
|
30
|
+
puts _message.color COLOR_HIGHLIGHT
|
31
|
+
end
|
32
|
+
|
33
|
+
def json_result(_data)
|
34
|
+
if _data.nil?
|
35
|
+
result 'nil'
|
36
|
+
else
|
37
|
+
result JSON.pretty_generate(_data).gsub(/(^|\\n)/, ' ')
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def warning(_message)
|
42
|
+
puts _message.color COLOR_WARNING
|
43
|
+
end
|
44
|
+
|
45
|
+
def error(_message)
|
46
|
+
puts _message.color COLOR_ERROR
|
47
|
+
end
|
48
|
+
|
49
|
+
def exception(_exc)
|
50
|
+
error "#{_exc.class.to_s}: #{_exc.to_s}"
|
51
|
+
backtrace _exc
|
52
|
+
end
|
53
|
+
|
54
|
+
def backtrace(_exc)
|
55
|
+
_exc.backtrace.each { |i| info(i) }
|
56
|
+
end
|
57
|
+
|
58
|
+
def question(_question)
|
59
|
+
Ask.input(_question.color COLOR_QUESTION)
|
60
|
+
end
|
61
|
+
|
62
|
+
def trap_errors(_context)
|
63
|
+
begin
|
64
|
+
yield
|
65
|
+
rescue SystemExit, Interrupt
|
66
|
+
raise
|
67
|
+
rescue Exception => e
|
68
|
+
error = "Error #{_context}"
|
69
|
+
warning error + ', check log for more information'
|
70
|
+
Crabfarm.logger.error error
|
71
|
+
Crabfarm.logger.error e
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -1,16 +1,17 @@
|
|
1
1
|
module Crabfarm
|
2
2
|
module Utils
|
3
3
|
module Naming
|
4
|
+
extend self
|
4
5
|
|
5
|
-
def
|
6
|
+
def is_constant_name?(_name)
|
6
7
|
/^([A-Z][A-Za-z0-9]*)(\:\:[A-Z][A-Za-z0-9]*)*$/ === _name
|
7
8
|
end
|
8
9
|
|
9
|
-
def
|
10
|
+
def route_from_constant(_class_name)
|
10
11
|
_class_name.split('::').map(&:underscore)
|
11
12
|
end
|
12
13
|
|
13
|
-
def
|
14
|
+
def decode_crabfarm_uri(_uri)
|
14
15
|
_uri.to_s.split('/').map { |p| p.gsub(/[^A-Z0-9:]+/i, '_').camelize }.join('::')
|
15
16
|
end
|
16
17
|
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Crabfarm
|
2
|
+
module Utils
|
3
|
+
module Resolve
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def navigator_class(_name)
|
7
|
+
if _name.is_a? String or _name.is_a? Symbol
|
8
|
+
(Naming.decode_crabfarm_uri(_name.to_s)).constantize
|
9
|
+
else _name end
|
10
|
+
end
|
11
|
+
|
12
|
+
def reducer_class(_name)
|
13
|
+
if _name.is_a? String or _name.is_a? Symbol
|
14
|
+
(Naming.decode_crabfarm_uri(_name.to_s) + 'Reducer').constantize
|
15
|
+
else _name end
|
16
|
+
end
|
17
|
+
|
18
|
+
def memento_path(_name)
|
19
|
+
File.join(mementos_path, _name.to_s + '.json.gz')
|
20
|
+
end
|
21
|
+
|
22
|
+
def snapshot_path(_name, _format)
|
23
|
+
_name = self.to_s.underscore if _name.nil?
|
24
|
+
File.join(snapshots_path, _name + '.' + _format)
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def mementos_path
|
30
|
+
File.join(Crabfarm.app_path, 'spec/mementos')
|
31
|
+
end
|
32
|
+
|
33
|
+
def snapshots_path
|
34
|
+
File.join(Crabfarm.app_path, 'spec/snapshots')
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
|
3
|
+
module Crabfarm
|
4
|
+
module Utils
|
5
|
+
module RSpecRunner
|
6
|
+
extend self
|
7
|
+
|
8
|
+
class SilentFormatter
|
9
|
+
::RSpec::Core::Formatters.register self
|
10
|
+
|
11
|
+
def initialize(_output)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class DummyOptions
|
16
|
+
def configure(_); end
|
17
|
+
end
|
18
|
+
|
19
|
+
def run_spec_for(_target, _filter=nil)
|
20
|
+
|
21
|
+
_target = Array(_target)
|
22
|
+
# TODO: check target exist?
|
23
|
+
|
24
|
+
first_time_config
|
25
|
+
|
26
|
+
begin
|
27
|
+
::RSpec.configuration.files_or_directories_to_run = _target
|
28
|
+
::RSpec.configuration.inclusion_filter = _filter if _filter
|
29
|
+
|
30
|
+
runner = ::RSpec::Core::Runner.new DummyOptions.new
|
31
|
+
runner.run $stderr, $stdout
|
32
|
+
|
33
|
+
::RSpec.world.example_groups.map { |g| g.filtered_examples }.flatten
|
34
|
+
ensure
|
35
|
+
::RSpec.clear_examples
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def first_time_config
|
42
|
+
unless @ready
|
43
|
+
options = ::RSpec::Core::ConfigurationOptions.new [
|
44
|
+
'-f','Crabfarm::Utils::RSpecRunner::SilentFormatter'
|
45
|
+
]
|
46
|
+
|
47
|
+
options.configure ::RSpec.configuration
|
48
|
+
@ready = true
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/lib/crabfarm/version.rb
CHANGED
data/lib/crabfarm.rb
CHANGED
@@ -6,21 +6,24 @@ require "active_support/inflector"
|
|
6
6
|
require "crabfarm/version"
|
7
7
|
require "crabfarm/errors"
|
8
8
|
require "crabfarm/configuration"
|
9
|
-
require "crabfarm/global_state"
|
10
9
|
require "crabfarm/driver_pool"
|
11
10
|
require "crabfarm/http_client"
|
12
|
-
require "crabfarm/phantom_runner"
|
13
11
|
require "crabfarm/state_store"
|
14
12
|
require "crabfarm/context"
|
15
|
-
require "crabfarm/context_factory"
|
16
13
|
require "crabfarm/transition_service"
|
17
14
|
require "crabfarm/base_navigator"
|
18
15
|
require "crabfarm/base_reducer"
|
19
16
|
require "crabfarm/base_struct"
|
20
17
|
require "crabfarm/strategies"
|
21
18
|
|
19
|
+
require "crabfarm/factories/context"
|
20
|
+
require "crabfarm/factories/navigator"
|
21
|
+
require "crabfarm/factories/reducer"
|
22
|
+
require "crabfarm/factories/snapshot_reducer"
|
23
|
+
|
22
24
|
require "crabfarm/utils/port_discovery"
|
23
25
|
require "crabfarm/utils/naming"
|
26
|
+
require "crabfarm/utils/resolve"
|
24
27
|
|
25
28
|
module Crabfarm
|
26
29
|
|
@@ -33,6 +36,14 @@ module Crabfarm
|
|
33
36
|
File.dirname __dir__
|
34
37
|
end
|
35
38
|
|
39
|
+
def self.inside_crawler_app?
|
40
|
+
defined? CF_PATH
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.app_path
|
44
|
+
CF_PATH
|
45
|
+
end
|
46
|
+
|
36
47
|
def self.config
|
37
48
|
@@config
|
38
49
|
end
|
@@ -71,6 +82,16 @@ module Crabfarm
|
|
71
82
|
@@debug
|
72
83
|
end
|
73
84
|
|
85
|
+
def self.with_context(_memento=nil)
|
86
|
+
ctx = Factories::Context.build _memento
|
87
|
+
begin
|
88
|
+
ctx.prepare
|
89
|
+
yield ctx
|
90
|
+
ensure
|
91
|
+
ctx.release
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
74
95
|
module Strategies
|
75
96
|
# bundled browser adapters
|
76
97
|
register :browser, :phantomjs, 'Crabfarm::Adapters::Browser::PhantomJs', dependencies: ['selenium-webdriver']
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crabfarm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ignacio Baixas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-05-
|
11
|
+
date: 2015-05-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -425,7 +425,6 @@ files:
|
|
425
425
|
- lib/crabfarm/cli.rb
|
426
426
|
- lib/crabfarm/configuration.rb
|
427
427
|
- lib/crabfarm/context.rb
|
428
|
-
- lib/crabfarm/context_factory.rb
|
429
428
|
- lib/crabfarm/crabtrap_context.rb
|
430
429
|
- lib/crabfarm/crabtrap_runner.rb
|
431
430
|
- lib/crabfarm/driver_pool.rb
|
@@ -435,15 +434,23 @@ files:
|
|
435
434
|
- lib/crabfarm/engines/async_state_manager.rb
|
436
435
|
- lib/crabfarm/engines/sync_state_manager.rb
|
437
436
|
- lib/crabfarm/errors.rb
|
437
|
+
- lib/crabfarm/factories/context.rb
|
438
|
+
- lib/crabfarm/factories/decorable.rb
|
439
|
+
- lib/crabfarm/factories/navigator.rb
|
440
|
+
- lib/crabfarm/factories/reducer.rb
|
441
|
+
- lib/crabfarm/factories/snapshot_reducer.rb
|
438
442
|
- lib/crabfarm/forked_navigator.rb
|
439
|
-
- lib/crabfarm/global_state.rb
|
440
443
|
- lib/crabfarm/http_client.rb
|
444
|
+
- lib/crabfarm/live/context.rb
|
441
445
|
- lib/crabfarm/live/controller.rb
|
442
|
-
- lib/crabfarm/live/helpers.rb
|
443
446
|
- lib/crabfarm/live/interactable.rb
|
444
447
|
- lib/crabfarm/live/manager.rb
|
445
448
|
- lib/crabfarm/live/navigator_runner.rb
|
449
|
+
- lib/crabfarm/live/navigator_runner_direct.rb
|
450
|
+
- lib/crabfarm/live/navigator_runner_rspec.rb
|
446
451
|
- lib/crabfarm/live/reducer_runner.rb
|
452
|
+
- lib/crabfarm/live/reducer_runner_direct.rb
|
453
|
+
- lib/crabfarm/live/reducer_runner_rspec.rb
|
447
454
|
- lib/crabfarm/live/watcher.rb
|
448
455
|
- lib/crabfarm/modes/console.rb
|
449
456
|
- lib/crabfarm/modes/generator.rb
|
@@ -454,12 +461,16 @@ files:
|
|
454
461
|
- lib/crabfarm/modes/server.rb
|
455
462
|
- lib/crabfarm/modes/shared/interactive_decorator.rb
|
456
463
|
- lib/crabfarm/modes/shared/snapshot_decorator.rb
|
457
|
-
- lib/crabfarm/
|
464
|
+
- lib/crabfarm/rspec/navigator_spec_helpers.rb
|
465
|
+
- lib/crabfarm/rspec/reducer_spec_helpers.rb
|
466
|
+
- lib/crabfarm/rspec/reducer_spy.rb
|
467
|
+
- lib/crabfarm/rspec/reducer_spy_manager.rb
|
458
468
|
- lib/crabfarm/rspec.rb
|
459
469
|
- lib/crabfarm/state_store.rb
|
460
470
|
- lib/crabfarm/strategies.rb
|
461
471
|
- lib/crabfarm/support/custom_puma.rb
|
462
472
|
- lib/crabfarm/support/gli.rb
|
473
|
+
- lib/crabfarm/support/phantom_runner.rb
|
463
474
|
- lib/crabfarm/support/webdriver_factory.rb
|
464
475
|
- lib/crabfarm/templates/boot.rb.erb
|
465
476
|
- lib/crabfarm/templates/crabfarm_bin.erb
|
@@ -476,16 +487,22 @@ files:
|
|
476
487
|
- lib/crabfarm/templates/spec_helper.rb.erb
|
477
488
|
- lib/crabfarm/templates/struct.rb.erb
|
478
489
|
- lib/crabfarm/transition_service.rb
|
490
|
+
- lib/crabfarm/utils/console.rb
|
479
491
|
- lib/crabfarm/utils/naming.rb
|
480
492
|
- lib/crabfarm/utils/port_discovery.rb
|
481
493
|
- lib/crabfarm/utils/processes.rb
|
494
|
+
- lib/crabfarm/utils/resolve.rb
|
495
|
+
- lib/crabfarm/utils/rspec_runner.rb
|
496
|
+
- lib/crabfarm/utils/webdriver.rb
|
482
497
|
- lib/crabfarm/version.rb
|
483
498
|
- lib/crabfarm.rb
|
484
499
|
- bin/crabfarm
|
500
|
+
- assets/live-tools/instructions.html
|
485
501
|
- assets/live-tools/selectorgadget_combined.css
|
486
502
|
- assets/live-tools/selectorgadget_combined.js
|
487
503
|
- assets/live-tools/tools.css
|
488
504
|
- assets/live-tools/tools.js
|
505
|
+
- assets/live-tools/welcome.html
|
489
506
|
homepage: https://github.com/platanus/crabfarm-gem
|
490
507
|
licenses:
|
491
508
|
- MIT
|
@@ -1,32 +0,0 @@
|
|
1
|
-
module Crabfarm
|
2
|
-
module ContextFactory
|
3
|
-
|
4
|
-
def with_context(_memento=nil)
|
5
|
-
ctx = build_context(_memento)
|
6
|
-
begin
|
7
|
-
ctx.prepare
|
8
|
-
yield ctx
|
9
|
-
ensure
|
10
|
-
ctx.release
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
def build_context(_memento=nil)
|
15
|
-
if _memento.nil?
|
16
|
-
Crabfarm::Context.new
|
17
|
-
else
|
18
|
-
load_crabtrap_context _memento
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def load_crabtrap_context(_memento)
|
23
|
-
require 'crabfarm/crabtrap_context'
|
24
|
-
require 'crabfarm/modes/recorder/memento'
|
25
|
-
m_path = Modes::Recorder::Memento.memento_path _memento
|
26
|
-
raise ResourceNotFoundError.new "Could not find memento '#{_name}'" unless File.exists? m_path
|
27
|
-
Crabfarm::CrabtrapContext.new :replay, m_path
|
28
|
-
end
|
29
|
-
|
30
|
-
extend self
|
31
|
-
end
|
32
|
-
end
|
@@ -1,22 +0,0 @@
|
|
1
|
-
module Crabfarm
|
2
|
-
module GlobalState
|
3
|
-
|
4
|
-
def inside_crawler_app?
|
5
|
-
defined? CF_PATH
|
6
|
-
end
|
7
|
-
|
8
|
-
def app_path
|
9
|
-
CF_PATH
|
10
|
-
end
|
11
|
-
|
12
|
-
def mementos_path
|
13
|
-
File.join(app_path, 'spec/mementos')
|
14
|
-
end
|
15
|
-
|
16
|
-
def snapshots_path
|
17
|
-
File.join app_path, 'spec/snapshots'
|
18
|
-
end
|
19
|
-
|
20
|
-
extend self
|
21
|
-
end
|
22
|
-
end
|
@@ -1,75 +0,0 @@
|
|
1
|
-
require 'timeout'
|
2
|
-
require 'crabfarm/utils/processes'
|
3
|
-
|
4
|
-
module Crabfarm
|
5
|
-
class PhantomRunner
|
6
|
-
|
7
|
-
PHANTOM_START_TM = 5 # seconds
|
8
|
-
|
9
|
-
def initialize(_config={})
|
10
|
-
@config = _config;
|
11
|
-
@process = nil
|
12
|
-
end
|
13
|
-
|
14
|
-
def port
|
15
|
-
@config[:port]
|
16
|
-
end
|
17
|
-
|
18
|
-
def start
|
19
|
-
logger.info "Starting phantomjs in port #{port}"
|
20
|
-
@process = spawn_phantomjs
|
21
|
-
logger.info "Phantomjs started (PID: #{@process.pid})"
|
22
|
-
end
|
23
|
-
|
24
|
-
def stop
|
25
|
-
unless @process.nil?
|
26
|
-
logger.info "Stopping phantomjs (PID: #{@process.pid})"
|
27
|
-
@process.stop
|
28
|
-
@process = nil
|
29
|
-
logger.info "Phantomjs stopped"
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
private
|
34
|
-
|
35
|
-
def spawn_phantomjs
|
36
|
-
proc = nil
|
37
|
-
begin
|
38
|
-
proc = Utils::Processes.start_logged_process 'phantomjs', phantomjs_cmd, logger
|
39
|
-
Timeout::timeout(PHANTOM_START_TM) { wait_for_server }
|
40
|
-
rescue ChildProcess::LaunchError
|
41
|
-
raise BinaryMissingError.new 'phantomjs', @config[:bin_path]
|
42
|
-
rescue Timeout::Error
|
43
|
-
proc.stop
|
44
|
-
raise
|
45
|
-
end
|
46
|
-
proc
|
47
|
-
end
|
48
|
-
|
49
|
-
def phantomjs_cmd
|
50
|
-
cmd = [@config[:bin_path]]
|
51
|
-
cmd << '--load-images=false' unless @config[:load_images]
|
52
|
-
cmd << "--proxy=#{@config[:proxy]}" unless @config[:proxy].nil?
|
53
|
-
cmd << "--webdriver=#{port}"
|
54
|
-
cmd << "--ssl-protocol=#{@config[:ssl]}" unless @config[:ssl].nil?
|
55
|
-
cmd << "--ignore-ssl-errors=true"
|
56
|
-
cmd << "--webdriver-loglevel=WARN"
|
57
|
-
cmd
|
58
|
-
end
|
59
|
-
|
60
|
-
def wait_for_server
|
61
|
-
loop do
|
62
|
-
begin
|
63
|
-
Net::HTTP.get_response(URI.parse("http://127.0.0.1:#{port}/status"))
|
64
|
-
break
|
65
|
-
rescue
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
def logger
|
71
|
-
Crabfarm.logger
|
72
|
-
end
|
73
|
-
|
74
|
-
end
|
75
|
-
end
|