crabfarm 0.0.3 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/crabfarm/adapters/{capybara_adapter.rb → browser/capybara.rb} +2 -2
- data/lib/crabfarm/adapters/{surfer_adapter.rb → browser/surfer.rb} +4 -2
- data/lib/crabfarm/adapters/{watir_adapter.rb → browser/watir.rb} +4 -2
- data/lib/crabfarm/adapters/output/hash.rb +11 -0
- data/lib/crabfarm/adapters/output/jbuilder.rb +13 -0
- data/lib/crabfarm/adapters/output/ostruct.rb +14 -0
- data/lib/crabfarm/base_parser.rb +15 -5
- data/lib/crabfarm/base_state.rb +44 -7
- data/lib/crabfarm/configuration.rb +4 -2
- data/lib/crabfarm/engines/safe_state_loop.rb +1 -2
- data/lib/crabfarm/forked_state.rb +27 -0
- data/lib/crabfarm/modes/console.rb +2 -2
- data/lib/crabfarm/strategies.rb +28 -0
- data/lib/crabfarm/templates/Crabfile.erb +1 -1
- data/lib/crabfarm/templates/boot.rb.erb +2 -1
- data/lib/crabfarm/version.rb +1 -1
- data/lib/crabfarm.rb +13 -4
- metadata +10 -6
- data/lib/crabfarm/adapters.rb +0 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b162bd44fc267ef4526bef924629cec5757e252a
|
4
|
+
data.tar.gz: 75dd81f5efd9266a0ee97d426965e6c44e1f63c2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: de642539e7d2b4f0fe156490090222c5cd64c68763f0d6d9c68fb2b896f5c667c30e05b05d545dff9bbfb53c3de700c22e1eddc84c813f499b98dcdcee82930f
|
7
|
+
data.tar.gz: 1fef6418933fc2f6d697214408a294fc0350ab65439a6480ae1dc8deb2df8d9f26b0c1a1dee59cf8415f68aa28e04b7de3aa76ccb7fea29b43b4c7c18fad5094
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
|
3
|
+
module Crabfarm
|
4
|
+
class OStructOutputBuilder
|
5
|
+
def self.prepare
|
6
|
+
# TODO: maybe wrap open struct in a class that automatically generate other openstruct when nested properties are accessed
|
7
|
+
OpenStruct.new
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.serialize(_output)
|
11
|
+
_output.to_h
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/crabfarm/base_parser.rb
CHANGED
@@ -1,26 +1,36 @@
|
|
1
1
|
module Crabfarm
|
2
|
-
class BaseParser
|
2
|
+
class BaseParser < Delegator
|
3
3
|
|
4
4
|
attr_reader :browser, :params
|
5
5
|
|
6
6
|
def self.browser_dsl(_dsl)
|
7
|
-
@
|
7
|
+
@browser_dsl = _dsl
|
8
8
|
end
|
9
9
|
|
10
10
|
def initialize(_module, _driver, _params)
|
11
|
-
dsl_class =
|
11
|
+
dsl_class = Strategies.load(:browser_dsl, class_browser_dsl || _module.settings.browser_dsl)
|
12
12
|
@browser = dsl_class.wrap _driver
|
13
13
|
@params = _params
|
14
|
+
|
15
|
+
super @browser
|
14
16
|
end
|
15
17
|
|
16
18
|
def parse
|
17
19
|
raise NotImplementedError.new
|
18
20
|
end
|
19
21
|
|
22
|
+
def __getobj__
|
23
|
+
@browser
|
24
|
+
end
|
25
|
+
|
26
|
+
def __setobj__(obj)
|
27
|
+
@browser = obj
|
28
|
+
end
|
29
|
+
|
20
30
|
private
|
21
31
|
|
22
|
-
def
|
23
|
-
self.class.instance_variable_get :@
|
32
|
+
def class_browser_dsl
|
33
|
+
self.class.instance_variable_get :@browser_dsl
|
24
34
|
end
|
25
35
|
end
|
26
36
|
end
|
data/lib/crabfarm/base_state.rb
CHANGED
@@ -1,14 +1,21 @@
|
|
1
|
+
require 'thwait'
|
2
|
+
require 'crabfarm/forked_state'
|
3
|
+
|
1
4
|
module Crabfarm
|
2
5
|
class BaseState
|
3
6
|
extend Forwardable
|
4
7
|
|
5
|
-
attr_reader :params
|
8
|
+
attr_reader :params, :output
|
6
9
|
|
7
10
|
def_delegators :@pool, :driver
|
8
11
|
def_delegators :@store, :get, :fetch
|
9
12
|
|
10
13
|
def self.browser_dsl(_dsl)
|
11
|
-
@
|
14
|
+
@class_browser_dsl = _dsl
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.output_builder(_builder)
|
18
|
+
@class_output_builder = _builder
|
12
19
|
end
|
13
20
|
|
14
21
|
def initialize(_module, _pool, _store, _params)
|
@@ -16,8 +23,9 @@ module Crabfarm
|
|
16
23
|
@pool = _pool
|
17
24
|
@store = _store
|
18
25
|
@params = _params
|
19
|
-
|
20
|
-
@dsl =
|
26
|
+
|
27
|
+
@dsl = Strategies.load(:browser_dsl, class_browser_dsl || @module.settings.browser_dsl)
|
28
|
+
@builder = Strategies.load(:output_builder, class_output_builder || @module.settings.output_builder)
|
21
29
|
end
|
22
30
|
|
23
31
|
def browser(_name=nil)
|
@@ -25,17 +33,46 @@ module Crabfarm
|
|
25
33
|
end
|
26
34
|
|
27
35
|
def output
|
28
|
-
@output ||=
|
36
|
+
@output ||= @builder.prepare
|
37
|
+
end
|
38
|
+
|
39
|
+
def output_as_json
|
40
|
+
@builder.serialize @output
|
29
41
|
end
|
30
42
|
|
31
43
|
def crawl
|
32
44
|
raise NotImplementedError.new
|
33
45
|
end
|
34
46
|
|
47
|
+
def fork_each(_enumerator, &_block)
|
48
|
+
session_id = 0
|
49
|
+
mutex = Mutex.new
|
50
|
+
ths = _enumerator.map do |value|
|
51
|
+
session_id += 1
|
52
|
+
start_forked_state("th_session_#{session_id}", value, _block, mutex)
|
53
|
+
end
|
54
|
+
ThreadsWait.all_waits(*ths)
|
55
|
+
end
|
56
|
+
|
35
57
|
private
|
36
58
|
|
37
|
-
def
|
38
|
-
self.class.instance_variable_get :@
|
59
|
+
def class_browser_dsl
|
60
|
+
self.class.instance_variable_get :@class_browser_dsl
|
61
|
+
end
|
62
|
+
|
63
|
+
def class_output_builder
|
64
|
+
self.class.instance_variable_get :@class_output_builder
|
65
|
+
end
|
66
|
+
|
67
|
+
def start_forked_state(_name, _value, _block, _mutex)
|
68
|
+
Thread.new {
|
69
|
+
sub_state = ForkedState.new self, _name, _mutex
|
70
|
+
begin
|
71
|
+
sub_state.instance_exec _value, &_block
|
72
|
+
ensure
|
73
|
+
sub_state.driver.reset
|
74
|
+
end
|
75
|
+
}
|
39
76
|
end
|
40
77
|
end
|
41
78
|
end
|
@@ -5,7 +5,8 @@ module Crabfarm
|
|
5
5
|
class Option < Struct.new(:name, :type, :text); end
|
6
6
|
|
7
7
|
OPTIONS = [
|
8
|
-
[:
|
8
|
+
[:browser_dsl, :string, 'Default browser dsl used by parsers and states'],
|
9
|
+
[:output_builder, :string, 'Default json output builder used by states'],
|
9
10
|
[:driver_factory, :mixed, 'Driver factory, disabled if phantom_mode is used'],
|
10
11
|
[:log_path, :string, 'Path where logs should be stored'],
|
11
12
|
|
@@ -39,7 +40,8 @@ module Crabfarm
|
|
39
40
|
|
40
41
|
def initialize
|
41
42
|
@values = {
|
42
|
-
|
43
|
+
browser_dsl: :surfer,
|
44
|
+
output_builder: :hash,
|
43
45
|
driver_factory: nil,
|
44
46
|
log_path: 'logs',
|
45
47
|
|
@@ -77,8 +77,7 @@ module Crabfarm
|
|
77
77
|
while @running
|
78
78
|
if @working
|
79
79
|
begin
|
80
|
-
|
81
|
-
@doc = last_state.output.attributes!
|
80
|
+
@doc = @context.run_state(@next_state_name, @next_state_params).output_as_json
|
82
81
|
@error = nil
|
83
82
|
rescue Exception => e
|
84
83
|
@doc = nil
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Crabfarm
|
2
|
+
class ForkedState
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
def_delegators :@state, :params, :get, :fetch
|
6
|
+
|
7
|
+
def initialize(_state, _name, _mutex)
|
8
|
+
@state = _state
|
9
|
+
@name = _name
|
10
|
+
@mutex = _mutex
|
11
|
+
end
|
12
|
+
|
13
|
+
def driver
|
14
|
+
@driver ||= @state.driver(@name)
|
15
|
+
end
|
16
|
+
|
17
|
+
def browser
|
18
|
+
@browser ||= @state.browser(@name)
|
19
|
+
end
|
20
|
+
|
21
|
+
def lock_output
|
22
|
+
@mutex.synchronize {
|
23
|
+
yield @state.output
|
24
|
+
}
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -33,8 +33,8 @@ module Crabfarm
|
|
33
33
|
end
|
34
34
|
|
35
35
|
begin
|
36
|
-
|
37
|
-
puts JSON.pretty_generate(
|
36
|
+
doc = @context.run_state(_name, _params).output_as_json
|
37
|
+
puts JSON.pretty_generate(doc).color(:green)
|
38
38
|
rescue EntityNotFoundError => e
|
39
39
|
puts "#{e.to_s}".color(:red)
|
40
40
|
rescue => e
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Crabfarm
|
2
|
+
module Strategies
|
3
|
+
|
4
|
+
class Loader
|
5
|
+
def initialize(_klass, _pkg=nil)
|
6
|
+
@klass = _klass
|
7
|
+
@pkg = _pkg
|
8
|
+
end
|
9
|
+
|
10
|
+
def load
|
11
|
+
require @pkg if @pkg
|
12
|
+
if @klass.is_a? String then Object.const_get @klass else @klass end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
@@register = {}
|
17
|
+
|
18
|
+
def self.register(_cat, _name, _klass, _pkg=nil)
|
19
|
+
@@register[_cat.to_s + _name.to_s] = Loader.new(_klass, _pkg)
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.load(_cat, _name)
|
23
|
+
full_name = _cat.to_s + _name.to_s
|
24
|
+
raise ConfigurationError.new "Invalid #{_cat} name #{_name}" unless @@register.has_key? full_name
|
25
|
+
@@register[full_name].load
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -1,2 +1,2 @@
|
|
1
|
-
|
1
|
+
set_browser_dsl :surfer
|
2
2
|
set_driver :phantomjs
|
data/lib/crabfarm/version.rb
CHANGED
data/lib/crabfarm.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require "forwardable"
|
2
|
-
require "
|
2
|
+
require "active_support/inflector"
|
3
3
|
require "selenium-webdriver"
|
4
4
|
|
5
5
|
require "crabfarm/version"
|
@@ -15,10 +15,19 @@ require "crabfarm/state_store"
|
|
15
15
|
require "crabfarm/context"
|
16
16
|
require "crabfarm/base_state"
|
17
17
|
require "crabfarm/base_parser"
|
18
|
-
require
|
19
|
-
require "crabfarm/adapters"
|
18
|
+
require "crabfarm/strategies"
|
20
19
|
require "crabfarm/loader"
|
21
20
|
|
22
21
|
module Crabfarm
|
23
|
-
|
22
|
+
module Strategies
|
23
|
+
# bundled browser dsl adapters
|
24
|
+
register :browser_dsl, :surfer, 'Crabfarm::SurferBrowserDsl', 'crabfarm/adapters/browser/surfer'
|
25
|
+
register :browser_dsl, :watir, 'Crabfarm::WatirBrowserDsl', 'crabfarm/adapters/browser/watir'
|
26
|
+
register :browser_dsl, :capybara, 'Crabfarm::CapybaraBrowserDsl', 'crabfarm/adapters/browser/capybara'
|
27
|
+
|
28
|
+
# bundled state output builders
|
29
|
+
register :output_builder, :hash, 'Crabfarm::HashOutputBuilder', 'crabfarm/adapters/output/hash'
|
30
|
+
register :output_builder, :ostruct, 'Crabfarm::OStructOutputBuilder', 'crabfarm/adapters/output/ostruct'
|
31
|
+
register :output_builder, :jbuilder, 'Crabfarm::JbuilderOutputBuilder', 'crabfarm/adapters/output/jbuilder'
|
32
|
+
end
|
24
33
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crabfarm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ignacio Baixas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-01-
|
11
|
+
date: 2015-01-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: jbuilder
|
@@ -290,10 +290,12 @@ executables:
|
|
290
290
|
extensions: []
|
291
291
|
extra_rdoc_files: []
|
292
292
|
files:
|
293
|
-
- lib/crabfarm/adapters/
|
294
|
-
- lib/crabfarm/adapters/
|
295
|
-
- lib/crabfarm/adapters/
|
296
|
-
- lib/crabfarm/adapters.rb
|
293
|
+
- lib/crabfarm/adapters/browser/capybara.rb
|
294
|
+
- lib/crabfarm/adapters/browser/surfer.rb
|
295
|
+
- lib/crabfarm/adapters/browser/watir.rb
|
296
|
+
- lib/crabfarm/adapters/output/hash.rb
|
297
|
+
- lib/crabfarm/adapters/output/jbuilder.rb
|
298
|
+
- lib/crabfarm/adapters/output/ostruct.rb
|
297
299
|
- lib/crabfarm/base_parser.rb
|
298
300
|
- lib/crabfarm/base_state.rb
|
299
301
|
- lib/crabfarm/cli.rb
|
@@ -307,6 +309,7 @@ files:
|
|
307
309
|
- lib/crabfarm/dsl/surfer.rb
|
308
310
|
- lib/crabfarm/engines/safe_state_loop.rb
|
309
311
|
- lib/crabfarm/errors.rb
|
312
|
+
- lib/crabfarm/forked_state.rb
|
310
313
|
- lib/crabfarm/loader.rb
|
311
314
|
- lib/crabfarm/modes/console.rb
|
312
315
|
- lib/crabfarm/modes/generator.rb
|
@@ -316,6 +319,7 @@ files:
|
|
316
319
|
- lib/crabfarm/phantom_runner.rb
|
317
320
|
- lib/crabfarm/rspec.rb
|
318
321
|
- lib/crabfarm/state_store.rb
|
322
|
+
- lib/crabfarm/strategies.rb
|
319
323
|
- lib/crabfarm/support/custom_puma.rb
|
320
324
|
- lib/crabfarm/support/gli.rb
|
321
325
|
- lib/crabfarm/templates/boot.rb.erb
|
data/lib/crabfarm/adapters.rb
DELETED
@@ -1,23 +0,0 @@
|
|
1
|
-
require 'crabfarm/adapters/capybara_adapter'
|
2
|
-
require 'crabfarm/adapters/surfer_adapter'
|
3
|
-
require 'crabfarm/adapters/watir_adapter'
|
4
|
-
|
5
|
-
module Crabfarm
|
6
|
-
module Adapters
|
7
|
-
@@adapters = {}
|
8
|
-
|
9
|
-
def self.register_dsl(_name, _adapter)
|
10
|
-
@@adapters[_name.to_sym] = _adapter
|
11
|
-
end
|
12
|
-
|
13
|
-
def self.load_from_dsl_name _name
|
14
|
-
raise ConfigurationError.new "Invalid dsl name #{_name}" unless @@adapters.has_key? _name.to_sym
|
15
|
-
@@adapters[_name.to_sym]
|
16
|
-
end
|
17
|
-
|
18
|
-
# bundled adapters
|
19
|
-
register_dsl :watir, WatirAdapter
|
20
|
-
register_dsl :capybara, CapybaraAdapter
|
21
|
-
register_dsl :surfer, SurferAdapter
|
22
|
-
end
|
23
|
-
end
|