abid 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +306 -4
- data/abid.gemspec +1 -0
- data/lib/Abidfile.rb +1 -80
- data/lib/abid.rb +3 -1
- data/lib/abid/application.rb +21 -17
- data/lib/abid/concurrent_extention.rb +6 -0
- data/lib/abid/concurrent_extention/ivar.rb +12 -0
- data/lib/abid/dsl_definition.rb +2 -2
- data/lib/abid/params_parser.rb +1 -1
- data/lib/abid/play.rb +19 -60
- data/lib/abid/rake_extensions/task.rb +82 -71
- data/lib/abid/state.rb +63 -8
- data/lib/abid/task.rb +101 -8
- data/lib/abid/task_manager.rb +13 -23
- data/lib/abid/tasks/core.rake +85 -0
- data/lib/abid/version.rb +1 -1
- data/lib/abid/worker.rb +26 -16
- metadata +19 -2
data/lib/abid/application.rb
CHANGED
@@ -4,12 +4,10 @@ module Abid
|
|
4
4
|
|
5
5
|
attr_reader :worker
|
6
6
|
attr_reader :config
|
7
|
-
attr_reader :futures
|
8
7
|
|
9
8
|
def initialize
|
10
9
|
super
|
11
10
|
@rakefiles = %w(abidfile Abidfile abidfile.rb Abidfile.rb) << abidfile
|
12
|
-
@futures = {}
|
13
11
|
@worker = Worker.new(self)
|
14
12
|
end
|
15
13
|
|
@@ -21,26 +19,39 @@ module Abid
|
|
21
19
|
def init(app_name = 'abid')
|
22
20
|
standard_exception_handling do
|
23
21
|
@config = IniFile.new(content: default_config)
|
24
|
-
@config.merge!(IniFile.load('config/abid.
|
22
|
+
@config.merge!(IniFile.load('config/abid.conf'))
|
25
23
|
end
|
26
24
|
|
27
25
|
super(app_name)
|
28
26
|
end
|
29
27
|
|
30
|
-
# allows the
|
28
|
+
# allows the built-in tasks to load without a abidfile
|
31
29
|
def abidfile
|
32
30
|
File.expand_path(File.join(File.dirname(__FILE__), '..', 'Abidfile.rb'))
|
33
31
|
end
|
34
32
|
|
33
|
+
# load built-in tasks
|
34
|
+
def load_rakefile
|
35
|
+
standard_exception_handling do
|
36
|
+
glob(File.expand_path('../tasks/*.rake', __FILE__)) do |name|
|
37
|
+
Rake.load_rakefile name
|
38
|
+
end
|
39
|
+
end
|
40
|
+
super
|
41
|
+
end
|
42
|
+
|
35
43
|
def run_with_threads
|
36
44
|
yield
|
37
|
-
|
45
|
+
rescue Exception => err
|
46
|
+
worker.kill
|
47
|
+
raise err
|
48
|
+
else
|
38
49
|
worker.shutdown
|
39
50
|
end
|
40
51
|
|
41
52
|
def invoke_task(task_string) # :nodoc:
|
42
53
|
name, args = parse_task_string(task_string)
|
43
|
-
self[name].async_invoke(*args).
|
54
|
+
self[name].async_invoke(*args).wait!
|
44
55
|
end
|
45
56
|
|
46
57
|
def default_config
|
@@ -61,12 +72,6 @@ module Abid
|
|
61
72
|
when '--execute-print'
|
62
73
|
# disable short option
|
63
74
|
opt.delete_at(1)
|
64
|
-
when '--dry-run'
|
65
|
-
h = opt.last
|
66
|
-
opt[-1] = lambda do |value|
|
67
|
-
h.call(value)
|
68
|
-
options.disable_state = true
|
69
|
-
end
|
70
75
|
when '--version'
|
71
76
|
opt[-1] = lambda do |_value|
|
72
77
|
puts "Abid Version: #{Abid::VERSION} (Rake Version: #{RAKEVERSION})"
|
@@ -79,14 +84,13 @@ module Abid
|
|
79
84
|
def abid_options # :nodoc:
|
80
85
|
sort_options(
|
81
86
|
[
|
82
|
-
['--
|
83
|
-
'Run the task
|
84
|
-
proc { options.
|
87
|
+
['--repair',
|
88
|
+
'Run the task in repair mode.',
|
89
|
+
proc { options.repair = true }
|
85
90
|
],
|
86
91
|
['--preview', '-p',
|
87
92
|
'Run tasks in preview mode.',
|
88
93
|
proc do
|
89
|
-
options.disable_state = true
|
90
94
|
options.preview = true
|
91
95
|
end
|
92
96
|
],
|
@@ -101,7 +105,7 @@ module Abid
|
|
101
105
|
end
|
102
106
|
|
103
107
|
def handle_options
|
104
|
-
options.rakelib =
|
108
|
+
options.rakelib = %w(rakelib tasks)
|
105
109
|
options.trace_output = $stderr
|
106
110
|
|
107
111
|
OptionParser.new do |opts|
|
data/lib/abid/dsl_definition.rb
CHANGED
@@ -8,8 +8,8 @@ module Abid
|
|
8
8
|
Rake.application.worker.define(name, thread_count)
|
9
9
|
end
|
10
10
|
|
11
|
-
def
|
12
|
-
Rake.application.
|
11
|
+
def play_base(&block)
|
12
|
+
Rake.application.play_base(&block)
|
13
13
|
end
|
14
14
|
|
15
15
|
def helpers(*extensions, &block)
|
data/lib/abid/params_parser.rb
CHANGED
data/lib/abid/play.rb
CHANGED
@@ -17,11 +17,17 @@ module Abid
|
|
17
17
|
def param(name, **param_spec)
|
18
18
|
params_spec[name] = { significant: true }.merge(param_spec)
|
19
19
|
|
20
|
-
define_method(name) { params[name] }
|
20
|
+
define_method(name) { task.params[name] }
|
21
|
+
end
|
22
|
+
|
23
|
+
def undef_param(name)
|
24
|
+
params_spec.delete(name)
|
25
|
+
undef_method(name) if method_defined?(name)
|
21
26
|
end
|
22
27
|
|
23
28
|
def hooks
|
24
29
|
@hooks ||= {
|
30
|
+
setup: [],
|
25
31
|
before: [],
|
26
32
|
after: [],
|
27
33
|
around: []
|
@@ -47,43 +53,30 @@ module Abid
|
|
47
53
|
include(*extensions) if extensions.any?
|
48
54
|
end
|
49
55
|
|
56
|
+
def setup(&block)
|
57
|
+
hooks[:setup] << block
|
58
|
+
end
|
59
|
+
|
50
60
|
def before(&block)
|
51
|
-
|
61
|
+
hooks[:before] << block
|
52
62
|
end
|
53
63
|
|
54
64
|
def after(&block)
|
55
|
-
|
65
|
+
hooks[:after] << block
|
56
66
|
end
|
57
67
|
|
58
68
|
def around(&block)
|
59
|
-
|
69
|
+
hooks[:around] << block
|
60
70
|
end
|
61
71
|
end
|
62
72
|
|
63
73
|
set :worker, :default
|
64
74
|
set :volatile, false
|
65
75
|
|
66
|
-
|
67
|
-
def_delegators :task, :application, :name, :scope
|
68
|
-
def_delegators 'self.class', :params_spec
|
69
|
-
|
70
|
-
attr_reader :prerequisites
|
71
|
-
attr_reader :params
|
72
|
-
|
73
|
-
def initialize(params)
|
74
|
-
@prerequisites = []
|
76
|
+
attr_reader :task
|
75
77
|
|
76
|
-
|
77
|
-
@
|
78
|
-
@params.freeze
|
79
|
-
end
|
80
|
-
|
81
|
-
def task
|
82
|
-
self.class.task
|
83
|
-
end
|
84
|
-
|
85
|
-
def setup
|
86
|
-
# noop
|
78
|
+
def initialize(task)
|
79
|
+
@task = task
|
87
80
|
end
|
88
81
|
|
89
82
|
def run
|
@@ -91,23 +84,7 @@ module Abid
|
|
91
84
|
end
|
92
85
|
|
93
86
|
def needs(task_name, **params)
|
94
|
-
|
95
|
-
end
|
96
|
-
|
97
|
-
def significant_params
|
98
|
-
[
|
99
|
-
name,
|
100
|
-
params.select { |p, _| params_spec[p][:significant] }
|
101
|
-
]
|
102
|
-
end
|
103
|
-
|
104
|
-
def hash
|
105
|
-
significant_params.hash
|
106
|
-
end
|
107
|
-
|
108
|
-
def eql?(other)
|
109
|
-
other.is_a?(Abid::Play) && \
|
110
|
-
significant_params.eql?(other.significant_params)
|
87
|
+
task.enhance([[task_name, params]])
|
111
88
|
end
|
112
89
|
|
113
90
|
def volatile?
|
@@ -115,25 +92,7 @@ module Abid
|
|
115
92
|
end
|
116
93
|
|
117
94
|
def preview?
|
118
|
-
application.options.preview
|
119
|
-
end
|
120
|
-
|
121
|
-
def invoke
|
122
|
-
self.class.hooks[:before].each { |blk| instance_eval(&blk) }
|
123
|
-
|
124
|
-
call_around_hooks(self.class.hooks[:around]) { run }
|
125
|
-
|
126
|
-
self.class.hooks[:after].each { |blk| instance_eval(&blk) }
|
127
|
-
end
|
128
|
-
|
129
|
-
def call_around_hooks(hooks, &body)
|
130
|
-
if hooks.empty?
|
131
|
-
body.call
|
132
|
-
else
|
133
|
-
h, *rest = hooks
|
134
|
-
instance_exec(-> { call_around_hooks(rest, &body) }, &h)
|
135
|
-
end
|
95
|
+
task.application.options.preview
|
136
96
|
end
|
137
|
-
private :call_around_hooks
|
138
97
|
end
|
139
98
|
end
|
@@ -10,7 +10,11 @@ module Abid
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def state
|
13
|
-
|
13
|
+
State.find(self)
|
14
|
+
end
|
15
|
+
|
16
|
+
def name_with_params
|
17
|
+
name
|
14
18
|
end
|
15
19
|
|
16
20
|
def async_invoke(*args)
|
@@ -20,114 +24,121 @@ module Abid
|
|
20
24
|
|
21
25
|
def async_invoke_with_call_chain(task_args, invocation_chain)
|
22
26
|
state.reload
|
23
|
-
new_chain = Rake::InvocationChain.append(self, invocation_chain)
|
24
|
-
@lock.synchronize do
|
25
|
-
if application.futures.include?(object_id)
|
26
|
-
return application.futures[object_id]
|
27
|
-
end
|
28
|
-
|
29
|
-
application.trace "** Invoke #{name}" if application.options.trace
|
30
27
|
|
31
|
-
|
32
|
-
|
33
|
-
future = async_invoke_after_prerequisites(task_args, preq_futures)
|
28
|
+
new_chain = Rake::InvocationChain.append(self, invocation_chain)
|
34
29
|
|
35
|
-
|
30
|
+
state.only_once do
|
31
|
+
if !application.options.repair && state.successed?
|
32
|
+
# skip if successed
|
33
|
+
state.ivar.try_set(false)
|
34
|
+
elsif !application.options.repair && state.failed? && !invocation_chain.empty?
|
35
|
+
# fail if not top level
|
36
|
+
fail "#{name} -- task has been failed" rescue state.ivar.try_fail($ERROR_INFO)
|
37
|
+
else
|
38
|
+
async_invoke_with_prerequisites(task_args, new_chain)
|
39
|
+
end
|
36
40
|
end
|
41
|
+
state.ivar
|
42
|
+
ensure
|
43
|
+
state.ivar.try_fail($ERROR_INFO) if $ERROR_INFO
|
37
44
|
end
|
38
45
|
|
39
|
-
def
|
40
|
-
#
|
41
|
-
|
42
|
-
|
43
|
-
|
46
|
+
def async_invoke_with_prerequisites(task_args, invocation_chain)
|
47
|
+
application.trace "** Invoke #{name_with_params}" if application.options.trace
|
48
|
+
|
49
|
+
volatiles, non_volatiles = prerequisite_tasks.partition(&:volatile?)
|
50
|
+
|
51
|
+
async_invoke_tasks(non_volatiles, task_args, invocation_chain) do |updated|
|
52
|
+
if state.successed? && !updated
|
53
|
+
application.trace "** Skip #{name_with_params}" if application.options.trace
|
54
|
+
state.ivar.try_set(false)
|
44
55
|
else
|
45
|
-
|
56
|
+
async_invoke_tasks(volatiles, task_args, invocation_chain) do
|
57
|
+
async_execute_with_session(task_args)
|
58
|
+
end
|
46
59
|
end
|
47
|
-
else
|
48
|
-
preqs = prerequisite_tasks
|
49
60
|
end
|
61
|
+
end
|
50
62
|
|
51
|
-
|
52
|
-
|
53
|
-
|
63
|
+
def async_invoke_tasks(tasks, task_args, invocation_chain, &block)
|
64
|
+
ivars = tasks.map do |t|
|
65
|
+
args = task_args.new_scope(t.arg_names)
|
66
|
+
t.async_invoke_with_call_chain(args, invocation_chain)
|
54
67
|
end
|
55
|
-
end
|
56
68
|
|
57
|
-
|
58
|
-
|
59
|
-
async_execute_with_session(task_args, false)
|
69
|
+
if ivars.empty?
|
70
|
+
block.call(false)
|
60
71
|
else
|
61
|
-
|
62
|
-
counter = Concurrent::DependencyCounter.new(preq_futures.size) do
|
72
|
+
counter = Concurrent::DependencyCounter.new(ivars.size) do
|
63
73
|
begin
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
future = async_execute_with_session(task_args, preq_updated)
|
70
|
-
|
71
|
-
future.add_observer do |_time, value, reason|
|
72
|
-
reason.nil? ? result.set(value) : result.fail(reason)
|
74
|
+
if ivars.any?(&:rejected?)
|
75
|
+
state.ivar.try_fail(ivars.find(&:rejected?).reason)
|
76
|
+
else
|
77
|
+
updated = ivars.map(&:value).any?
|
78
|
+
block.call(updated)
|
73
79
|
end
|
74
80
|
rescue Exception => err
|
75
|
-
|
81
|
+
state.ivar.try_fail(err)
|
76
82
|
end
|
77
83
|
end
|
78
|
-
|
79
|
-
result
|
84
|
+
ivars.each { |i| i.add_observer counter }
|
80
85
|
end
|
81
86
|
end
|
82
87
|
|
83
|
-
def async_execute_with_session(task_args
|
84
|
-
|
85
|
-
application.trace "** Skip #{name}" if application.options.trace
|
86
|
-
return Concurrent::IVar.new(false)
|
87
|
-
end
|
88
|
-
|
89
|
-
session_started = state.start_session
|
90
|
-
|
91
|
-
return async_wait_complete unless session_started
|
92
|
-
|
93
|
-
pool = application.worker[worker]
|
94
|
-
future = Concurrent::Future.execute(executor: pool) do
|
88
|
+
def async_execute_with_session(task_args)
|
89
|
+
async_execute_in_worker do
|
95
90
|
begin
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
91
|
+
state.session do
|
92
|
+
begin
|
93
|
+
execute(task_args) if needed?
|
94
|
+
finished = true
|
95
|
+
ensure
|
96
|
+
fail "#{name} -- thread killed" if $ERROR_INFO.nil? && !finished
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
state.ivar.try_set(true)
|
101
|
+
rescue AbidErrorTaskAlreadyRunning
|
102
|
+
async_wait_complete
|
100
103
|
end
|
101
104
|
end
|
102
|
-
ensure
|
103
|
-
# close session if error occurred outside the future
|
104
|
-
if session_started && future.nil? && $ERROR_INFO
|
105
|
-
state.close_session($ERROR_INFO)
|
106
|
-
end
|
107
105
|
end
|
108
106
|
|
109
107
|
def async_wait_complete
|
110
108
|
unless application.options.wait_external_task
|
111
|
-
err = RuntimeError.new("task #{
|
112
|
-
return
|
109
|
+
err = RuntimeError.new("task #{name_with_params} already running")
|
110
|
+
return state.ivar.try_fail(err)
|
113
111
|
end
|
114
112
|
|
115
|
-
application.trace "** Wait #{
|
113
|
+
application.trace "** Wait #{name_with_params}" if application.options.trace
|
116
114
|
|
117
|
-
|
118
|
-
Concurrent::Future.execute(executor: pool) do
|
115
|
+
async_execute_in_worker(:waiter) do
|
119
116
|
interval = application.options.wait_external_task_interval || 10
|
120
117
|
timeout = application.options.wait_external_task_timeout || 3600
|
121
118
|
timeout_tm = Time.now.to_f + timeout
|
122
119
|
|
123
120
|
loop do
|
124
121
|
state.reload
|
125
|
-
|
122
|
+
if !state.running?
|
123
|
+
state.ivar.try_set(true)
|
124
|
+
break
|
125
|
+
elsif Time.now.to_f >= timeout_tm
|
126
|
+
fail "#{name} -- timeout exceeded" rescue state.ivar.try_fail($ERROR_INFO)
|
127
|
+
break
|
128
|
+
else
|
129
|
+
sleep interval
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
126
134
|
|
127
|
-
|
128
|
-
|
135
|
+
def async_execute_in_worker(worker = nil, &block)
|
136
|
+
application.worker[worker || self.worker].post do
|
137
|
+
begin
|
138
|
+
block.call
|
139
|
+
rescue Exception => err
|
140
|
+
state.ivar.try_fail(err)
|
129
141
|
end
|
130
|
-
true
|
131
142
|
end
|
132
143
|
end
|
133
144
|
end
|