abid 0.3.0.pre.alpha.2 → 0.3.0.pre.alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/abid +1 -1
- data/lib/abid.rb +21 -6
- data/lib/abid/application.rb +17 -40
- data/lib/abid/cli.rb +8 -7
- data/lib/abid/cli/assume.rb +4 -3
- data/lib/abid/cli/list.rb +5 -4
- data/lib/abid/cli/migrate.rb +4 -5
- data/lib/abid/cli/revoke.rb +5 -4
- data/lib/abid/config.rb +3 -3
- data/lib/abid/dsl_definition.rb +3 -3
- data/lib/abid/engine.rb +13 -0
- data/lib/abid/engine/executor.rb +126 -0
- data/lib/abid/engine/process.rb +137 -0
- data/lib/abid/engine/process_manager.rb +56 -0
- data/lib/abid/engine/scheduler.rb +96 -0
- data/lib/abid/engine/waiter.rb +83 -0
- data/lib/abid/engine/worker_manager.rb +123 -0
- data/lib/abid/environment.rb +48 -0
- data/lib/abid/job.rb +49 -7
- data/lib/abid/job_manager.rb +22 -0
- data/lib/abid/mixin_task.rb +2 -2
- data/lib/abid/rake_extensions.rb +12 -4
- data/lib/abid/rake_extensions/task.rb +4 -117
- data/lib/abid/state_manager/database.rb +21 -17
- data/lib/abid/state_manager/state.rb +65 -15
- data/lib/abid/state_manager/state_proxy.rb +65 -0
- data/lib/abid/task.rb +1 -15
- data/lib/abid/version.rb +1 -1
- metadata +12 -8
- data/lib/abid/abid_module.rb +0 -19
- data/lib/abid/session.rb +0 -92
- data/lib/abid/state.rb +0 -193
- data/lib/abid/state_manager.rb +0 -17
- data/lib/abid/waiter.rb +0 -110
- data/lib/abid/worker.rb +0 -56
@@ -0,0 +1,137 @@
|
|
1
|
+
require 'concurrent/ivar'
|
2
|
+
require 'forwardable'
|
3
|
+
require 'monitor'
|
4
|
+
|
5
|
+
module Abid
|
6
|
+
module Engine
|
7
|
+
# @!visibility private
|
8
|
+
|
9
|
+
# Process object manages the task execution status.
|
10
|
+
#
|
11
|
+
# You should retrive a process object via Job#process.
|
12
|
+
# Do not create a process object by Process.new constructor.
|
13
|
+
#
|
14
|
+
# A process object has an internal status of the task execution and
|
15
|
+
# the task result (Process#result).
|
16
|
+
#
|
17
|
+
# An initial status is :unscheduled.
|
18
|
+
# When Process#prepare is called, the status gets :pending.
|
19
|
+
# When Process#execute is called and the task is posted to a thread pool,
|
20
|
+
# the status gets :running. When the task is finished, the status gets
|
21
|
+
# :complete and the result is assigned to :successed or :failed.
|
22
|
+
#
|
23
|
+
# process = Job['task_name'].process
|
24
|
+
# process.prepare
|
25
|
+
# process.start
|
26
|
+
# process.wait
|
27
|
+
# process.result #=> :successed or :failed
|
28
|
+
#
|
29
|
+
# Possible status are:
|
30
|
+
#
|
31
|
+
# <dl>
|
32
|
+
# <dt>:unscheduled</dt>
|
33
|
+
# <dd>The task is not invoked yet.</dd>
|
34
|
+
# <dt>:pending</dt>
|
35
|
+
# <dd>The task is waiting for prerequisites complete.</dd>
|
36
|
+
# <dt>:running</dt>
|
37
|
+
# <dd>The task is running.</dd>
|
38
|
+
# <dt>:complete</dt>
|
39
|
+
# <dd>The task is finished.</dd>
|
40
|
+
# </dl>
|
41
|
+
#
|
42
|
+
# Possible results are:
|
43
|
+
#
|
44
|
+
# <dl>
|
45
|
+
# <dt>:successed</dt>
|
46
|
+
# <dd>The task is successed.</dd>
|
47
|
+
# <dt>:failed</dt>
|
48
|
+
# <dd>The task is failed.</dd>
|
49
|
+
# <dt>:cancelled</dt>
|
50
|
+
# <dd>The task is not executed because of some problems.</dd>
|
51
|
+
# <dt>:skipped</dt>
|
52
|
+
# <dd>The task is not executed because already successed.</dd>
|
53
|
+
# </dl>
|
54
|
+
class Process
|
55
|
+
extend Forwardable
|
56
|
+
|
57
|
+
attr_reader :status, :error
|
58
|
+
|
59
|
+
def_delegators :@result_ivar, :add_observer, :wait, :complete?
|
60
|
+
|
61
|
+
def initialize(process_manager)
|
62
|
+
@process_manager = process_manager
|
63
|
+
@result_ivar = Concurrent::IVar.new
|
64
|
+
@status = :unscheduled
|
65
|
+
@error = nil
|
66
|
+
@mon = Monitor.new
|
67
|
+
end
|
68
|
+
|
69
|
+
%w(successed failed cancelled skipped).each do |meth|
|
70
|
+
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
71
|
+
def #{meth}?
|
72
|
+
result == :#{meth}
|
73
|
+
end
|
74
|
+
RUBY
|
75
|
+
end
|
76
|
+
|
77
|
+
def result
|
78
|
+
@result_ivar.value if @result_ivar.complete?
|
79
|
+
end
|
80
|
+
|
81
|
+
def prepare
|
82
|
+
compare_and_set_status(:pending, :unscheduled)
|
83
|
+
end
|
84
|
+
|
85
|
+
def start
|
86
|
+
compare_and_set_status(:running, :pending)
|
87
|
+
end
|
88
|
+
|
89
|
+
def finish(error = nil)
|
90
|
+
return unless compare_and_set_status(:complete, :running)
|
91
|
+
@error = error if error
|
92
|
+
@result_ivar.set(error.nil? ? :successed : :failed)
|
93
|
+
true
|
94
|
+
end
|
95
|
+
|
96
|
+
def cancel(error = nil)
|
97
|
+
return false unless compare_and_set_status(:complete, :pending)
|
98
|
+
@error = error if error
|
99
|
+
@result_ivar.set :cancelled
|
100
|
+
true
|
101
|
+
end
|
102
|
+
|
103
|
+
def skip
|
104
|
+
return false unless compare_and_set_status(:complete, :pending)
|
105
|
+
@result_ivar.set :skipped
|
106
|
+
true
|
107
|
+
end
|
108
|
+
|
109
|
+
# Force fail the task.
|
110
|
+
# @return [void]
|
111
|
+
def quit(error)
|
112
|
+
@status = :complete
|
113
|
+
@error = error
|
114
|
+
@result_ivar.try_set(:failed)
|
115
|
+
end
|
116
|
+
|
117
|
+
private
|
118
|
+
|
119
|
+
# Atomic compare and set operation.
|
120
|
+
# State is set to `next_state` only if
|
121
|
+
# `current state == expected_current`.
|
122
|
+
#
|
123
|
+
# @param [Symbol] next_state
|
124
|
+
# @param [Symbol] expected_current
|
125
|
+
#
|
126
|
+
# @return [Boolean] true if state is changed, false otherwise
|
127
|
+
def compare_and_set_status(next_state, *expected_current)
|
128
|
+
@mon.synchronize do
|
129
|
+
return unless expected_current.include? @status
|
130
|
+
@status = next_state
|
131
|
+
@process_manager.update(self)
|
132
|
+
true
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'monitor'
|
2
|
+
|
3
|
+
module Abid
|
4
|
+
module Engine
|
5
|
+
class ProcessManager
|
6
|
+
attr_reader :active_processes
|
7
|
+
|
8
|
+
# @param env [Environment] abid environment
|
9
|
+
def initialize(env)
|
10
|
+
@env = env
|
11
|
+
@active_processes = {}.compare_by_identity
|
12
|
+
@mon = Monitor.new
|
13
|
+
end
|
14
|
+
|
15
|
+
# @return [Process] new process
|
16
|
+
def create
|
17
|
+
Process.new(self)
|
18
|
+
end
|
19
|
+
|
20
|
+
# Update active process set.
|
21
|
+
# @param process [Process]
|
22
|
+
def update(process)
|
23
|
+
case process.status
|
24
|
+
when :pending, :running
|
25
|
+
add_active(process)
|
26
|
+
when :complete
|
27
|
+
delete_active(process)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# Kill all active processes
|
32
|
+
# @param error [Exception] error reason
|
33
|
+
def kill(error)
|
34
|
+
each_active { |p| p.quit(error) }
|
35
|
+
end
|
36
|
+
|
37
|
+
def active?(process)
|
38
|
+
@mon.synchronize { @active_processes.include? process }
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def add_active(process)
|
44
|
+
@mon.synchronize { @active_processes[process] ||= process }
|
45
|
+
end
|
46
|
+
|
47
|
+
def delete_active(process)
|
48
|
+
@mon.synchronize { @active_processes.delete(process) }
|
49
|
+
end
|
50
|
+
|
51
|
+
def each_active(&block)
|
52
|
+
@mon.synchronize { @active_processes.values.each(&block) }
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'concurrent/atomic/atomic_fixnum'
|
2
|
+
|
3
|
+
module Abid
|
4
|
+
module Engine
|
5
|
+
# Scheduler operates whole job flow execution.
|
6
|
+
class Scheduler
|
7
|
+
# @return [void]
|
8
|
+
def self.invoke(job, *args, invocation_chain: nil)
|
9
|
+
task_args = Rake::TaskArguments.new(job.task.arg_names, args)
|
10
|
+
invocation_chain ||= Rake::InvocationChain::EMPTY
|
11
|
+
|
12
|
+
detect_circular_dependency(job, invocation_chain)
|
13
|
+
new(job, task_args, invocation_chain).invoke
|
14
|
+
end
|
15
|
+
|
16
|
+
# @!visibility private
|
17
|
+
|
18
|
+
# Execute given block when DependencyCounter#update is called `count`
|
19
|
+
# times.
|
20
|
+
class DependencyCounter
|
21
|
+
def initialize(count, &block)
|
22
|
+
@counter = Concurrent::AtomicFixnum.new(count)
|
23
|
+
@block = block
|
24
|
+
yield if count <= 0
|
25
|
+
end
|
26
|
+
|
27
|
+
def update(*_)
|
28
|
+
@block.call if @counter.decrement.zero?
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.detect_circular_dependency(job, chain)
|
33
|
+
# raise error if job.task is a member of the chain
|
34
|
+
new_chain = Rake::InvocationChain.append(job.task, chain)
|
35
|
+
|
36
|
+
job.prerequisites.each do |preq_job|
|
37
|
+
detect_circular_dependency(preq_job, new_chain)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def initialize(job, args, invocation_chain)
|
42
|
+
@job = job
|
43
|
+
@args = args
|
44
|
+
@chain = invocation_chain.conj(@job.task)
|
45
|
+
@executor = Executor.new(job, args)
|
46
|
+
end
|
47
|
+
|
48
|
+
def invoke
|
49
|
+
return unless @executor.prepare
|
50
|
+
|
51
|
+
trace_invoke
|
52
|
+
attach_chain
|
53
|
+
invoke_prerequisites
|
54
|
+
after_prerequisites do
|
55
|
+
@executor.capture_exception do
|
56
|
+
@executor.start
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def trace_invoke
|
64
|
+
return unless @job.env.options.trace
|
65
|
+
@job.env.application.trace \
|
66
|
+
"** Invoke #{@job.task.name} #{@job.task.format_trace_flags}"
|
67
|
+
end
|
68
|
+
|
69
|
+
def attach_chain
|
70
|
+
@job.process.add_observer do
|
71
|
+
error = @job.process.error
|
72
|
+
next if error.nil?
|
73
|
+
next if @chain.nil?
|
74
|
+
|
75
|
+
error.extend(Rake::InvocationExceptionMixin) unless
|
76
|
+
error.respond_to?(:chain)
|
77
|
+
error.chain ||= @chain
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def invoke_prerequisites
|
82
|
+
@job.prerequisites.each do |preq_job|
|
83
|
+
preq_args = @args.new_scope(@job.task.arg_names)
|
84
|
+
Scheduler.new(preq_job, preq_args, @chain).invoke
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def after_prerequisites(&block)
|
89
|
+
counter = DependencyCounter.new(@job.prerequisites.size, &block)
|
90
|
+
@job.prerequisites.each do |preq_job|
|
91
|
+
preq_job.process.add_observer counter
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module Abid
|
2
|
+
module Engine
|
3
|
+
# Waits for a job to be finished which is running in external application,
|
4
|
+
# and completes the job in its own application.
|
5
|
+
#
|
6
|
+
# Waiter.new(job).wait
|
7
|
+
#
|
8
|
+
# The `job` result gets :successed or :failed when external application
|
9
|
+
# finished the job execution.
|
10
|
+
class Waiter
|
11
|
+
DEFAULT_WAIT_INTERVAL = 10
|
12
|
+
DEFAULT_WAIT_TIMEOUT = 3600
|
13
|
+
|
14
|
+
def initialize(job)
|
15
|
+
@job = job
|
16
|
+
@process = job.process
|
17
|
+
@wait_limit = Concurrent.monotonic_time + wait_timeout
|
18
|
+
end
|
19
|
+
|
20
|
+
def wait
|
21
|
+
unless @job.env.options.wait_external_task
|
22
|
+
@process.finish(AlreadyRunningError.new('job already running'))
|
23
|
+
return
|
24
|
+
end
|
25
|
+
|
26
|
+
wait_iter
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def wait_interval
|
32
|
+
@job.env.options.wait_external_task_interval ||
|
33
|
+
DEFAULT_WAIT_INTERVAL
|
34
|
+
end
|
35
|
+
|
36
|
+
def wait_timeout
|
37
|
+
@job.env.options.wait_external_task_timeout ||
|
38
|
+
DEFAULT_WAIT_TIMEOUT
|
39
|
+
end
|
40
|
+
|
41
|
+
def wait_iter
|
42
|
+
@job.env.worker_manager[:timer_set].post(wait_interval) do
|
43
|
+
capture_exception do
|
44
|
+
state = @job.state.find
|
45
|
+
|
46
|
+
check_finished(state) ||
|
47
|
+
check_timeout ||
|
48
|
+
wait_iter
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def check_finished(state)
|
54
|
+
return false if state.running?
|
55
|
+
|
56
|
+
if state.successed?
|
57
|
+
@process.finish
|
58
|
+
elsif state.failed?
|
59
|
+
@process.finish RuntimeError.new('task failed while waiting')
|
60
|
+
else
|
61
|
+
@process.finish RuntimeError.new('unexpected task state')
|
62
|
+
end
|
63
|
+
true
|
64
|
+
end
|
65
|
+
|
66
|
+
def check_timeout
|
67
|
+
return false if Concurrent.monotonic_time < @wait_limit
|
68
|
+
|
69
|
+
@process.finish RuntimeError.new('timeout')
|
70
|
+
true
|
71
|
+
end
|
72
|
+
|
73
|
+
def capture_exception
|
74
|
+
yield
|
75
|
+
rescue StandardError, ScriptError => error
|
76
|
+
@process.quit(error)
|
77
|
+
rescue Exception => exception
|
78
|
+
# TODO: exit immediately when fatal error occurs.
|
79
|
+
@process.quit(exception)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
require 'monitor'
|
3
|
+
|
4
|
+
require 'concurrent/delay'
|
5
|
+
|
6
|
+
module Abid
|
7
|
+
module Engine
|
8
|
+
# WorkerManager manges thread pools definition, creation and termination.
|
9
|
+
#
|
10
|
+
# worker_manager = Abid.global.worker_manager
|
11
|
+
#
|
12
|
+
# worker_manager.define(:main, 2)
|
13
|
+
#
|
14
|
+
# worker_manager[:main].post { :do_something }
|
15
|
+
#
|
16
|
+
# worker_manager.shutdown
|
17
|
+
#
|
18
|
+
class WorkerManager
|
19
|
+
def initialize(env)
|
20
|
+
@env = env
|
21
|
+
@workers = {}
|
22
|
+
@alive = true
|
23
|
+
@mon = Monitor.new
|
24
|
+
|
25
|
+
initialize_builtin_workers
|
26
|
+
end
|
27
|
+
|
28
|
+
# Define new worker.
|
29
|
+
#
|
30
|
+
# An actual worker is created when needed.
|
31
|
+
def define(name, num_threads)
|
32
|
+
@mon.synchronize do
|
33
|
+
check_alive!
|
34
|
+
|
35
|
+
if @workers.include?(name)
|
36
|
+
raise Error, "worker #{name} is already defined"
|
37
|
+
end
|
38
|
+
|
39
|
+
@workers[name] = Concurrent::Delay.new do
|
40
|
+
create_worker(num_threads: num_threads)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# Find or create worker
|
46
|
+
#
|
47
|
+
# @param name [String, Symbol] worker name
|
48
|
+
# @return [Concurrent::ExecutorService]
|
49
|
+
def [](name)
|
50
|
+
@mon.synchronize do
|
51
|
+
check_alive!
|
52
|
+
|
53
|
+
unless @workers.include?(name)
|
54
|
+
raise Error, "worker #{name} is not defined"
|
55
|
+
end
|
56
|
+
|
57
|
+
@workers[name].value!
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def shutdown(timeout = nil)
|
62
|
+
@mon.synchronize do
|
63
|
+
check_alive!
|
64
|
+
each_active(&:shutdown)
|
65
|
+
each_active { |worker| worker.wait_for_termination(timeout) }
|
66
|
+
|
67
|
+
result = each_active.all?(&:shutdown?)
|
68
|
+
@alive = false if result
|
69
|
+
result
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def kill
|
74
|
+
@mon.synchronize do
|
75
|
+
check_alive!
|
76
|
+
@alive = false
|
77
|
+
each_active(&:kill)
|
78
|
+
end
|
79
|
+
true
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
def initialize_builtin_workers
|
85
|
+
@workers[:default] = Concurrent::Delay.new { create_default_worker }
|
86
|
+
@workers[:waiter] = Concurrent::Delay.new { Concurrent.new_io_executor }
|
87
|
+
@workers[:timer_set] = Concurrent::Delay.new do
|
88
|
+
Concurrent::TimerSet.new(executor: self[:waiter])
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def create_worker(definition)
|
93
|
+
if definition[:num_threads] > 0
|
94
|
+
Concurrent::FixedThreadPool.new(definition[:num_threads])
|
95
|
+
else
|
96
|
+
Concurrent::CachedThreadPool.new
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def create_default_worker
|
101
|
+
create_worker(num_threads: default_num_threads)
|
102
|
+
end
|
103
|
+
|
104
|
+
def default_num_threads
|
105
|
+
if @env.options.always_multitask
|
106
|
+
@env.options.thread_pool_size ||
|
107
|
+
Rake.suggested_num_threads - 1
|
108
|
+
else
|
109
|
+
1
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def check_alive!
|
114
|
+
raise Error, 'already terminated' unless @alive
|
115
|
+
end
|
116
|
+
|
117
|
+
# Iterate on active workers
|
118
|
+
def each_active(&block)
|
119
|
+
@workers.values.select(&:fulfilled?).map(&:value).each(&block)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|