abid 0.3.0.pre.alpha.2 → 0.3.0.pre.alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/abid +1 -1
- data/lib/abid.rb +21 -6
- data/lib/abid/application.rb +17 -40
- data/lib/abid/cli.rb +8 -7
- data/lib/abid/cli/assume.rb +4 -3
- data/lib/abid/cli/list.rb +5 -4
- data/lib/abid/cli/migrate.rb +4 -5
- data/lib/abid/cli/revoke.rb +5 -4
- data/lib/abid/config.rb +3 -3
- data/lib/abid/dsl_definition.rb +3 -3
- data/lib/abid/engine.rb +13 -0
- data/lib/abid/engine/executor.rb +126 -0
- data/lib/abid/engine/process.rb +137 -0
- data/lib/abid/engine/process_manager.rb +56 -0
- data/lib/abid/engine/scheduler.rb +96 -0
- data/lib/abid/engine/waiter.rb +83 -0
- data/lib/abid/engine/worker_manager.rb +123 -0
- data/lib/abid/environment.rb +48 -0
- data/lib/abid/job.rb +49 -7
- data/lib/abid/job_manager.rb +22 -0
- data/lib/abid/mixin_task.rb +2 -2
- data/lib/abid/rake_extensions.rb +12 -4
- data/lib/abid/rake_extensions/task.rb +4 -117
- data/lib/abid/state_manager/database.rb +21 -17
- data/lib/abid/state_manager/state.rb +65 -15
- data/lib/abid/state_manager/state_proxy.rb +65 -0
- data/lib/abid/task.rb +1 -15
- data/lib/abid/version.rb +1 -1
- metadata +12 -8
- data/lib/abid/abid_module.rb +0 -19
- data/lib/abid/session.rb +0 -92
- data/lib/abid/state.rb +0 -193
- data/lib/abid/state_manager.rb +0 -17
- data/lib/abid/waiter.rb +0 -110
- data/lib/abid/worker.rb +0 -56
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'monitor'
|
2
|
+
|
3
|
+
module Abid
|
4
|
+
class Environment
|
5
|
+
def initialize
|
6
|
+
@mon = Monitor.new
|
7
|
+
end
|
8
|
+
|
9
|
+
def application
|
10
|
+
@application ||= Abid::Application.new(self)
|
11
|
+
end
|
12
|
+
attr_writer :application
|
13
|
+
|
14
|
+
def options
|
15
|
+
application.options
|
16
|
+
end
|
17
|
+
|
18
|
+
def config
|
19
|
+
@mon.synchronize do
|
20
|
+
@cofig ||= Config.new
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def job_manager
|
25
|
+
@mon.synchronize do
|
26
|
+
@job_manager ||= JobManager.new(self)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def process_manager
|
31
|
+
@mon.synchronize do
|
32
|
+
@process_manager ||= Engine::ProcessManager.new(self)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def worker_manager
|
37
|
+
@mon.synchronize do
|
38
|
+
@worker_manager ||= Engine::WorkerManager.new(self)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def db
|
43
|
+
@mon.synchronize do
|
44
|
+
@db ||= StateManager::Database.new(self)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/abid/job.rb
CHANGED
@@ -1,13 +1,27 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
require 'monitor'
|
3
|
+
|
1
4
|
module Abid
|
2
|
-
# Job
|
5
|
+
# Job is an aggregation object of components around the task.
|
3
6
|
class Job
|
4
|
-
attr_reader :name, :params
|
7
|
+
attr_reader :name, :params, :env
|
8
|
+
|
9
|
+
class << self
|
10
|
+
extend Forwardable
|
11
|
+
def_delegators 'Abid.global.job_manager', :[], :find_by_task
|
12
|
+
end
|
5
13
|
|
6
|
-
#
|
7
|
-
|
8
|
-
|
14
|
+
# @!visibility private
|
15
|
+
def initialize(env, name, params)
|
16
|
+
@env = env
|
9
17
|
@name = name
|
10
18
|
@params = params.sort.to_h.freeze
|
19
|
+
@mon = Monitor.new
|
20
|
+
end
|
21
|
+
|
22
|
+
def invoke(*args)
|
23
|
+
Engine::Scheduler.invoke(self, *args)
|
24
|
+
process.wait
|
11
25
|
end
|
12
26
|
|
13
27
|
def params_str
|
@@ -18,8 +32,36 @@ module Abid
|
|
18
32
|
@digest ||= Digest::MD5.hexdigest(name + "\n" + params_str)
|
19
33
|
end
|
20
34
|
|
21
|
-
def
|
22
|
-
|
35
|
+
def task
|
36
|
+
@task ||= Abid.application[name, nil, params]
|
37
|
+
end
|
38
|
+
|
39
|
+
def state
|
40
|
+
@state ||= StateManager::StateProxy.new(self)
|
41
|
+
end
|
42
|
+
|
43
|
+
def prerequisites
|
44
|
+
task.prerequisite_tasks.map do |preq_task|
|
45
|
+
env.job_manager.find_by_task(preq_task)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def process
|
50
|
+
@mon.synchronize do
|
51
|
+
@process ||= env.process_manager.create
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def worker
|
56
|
+
env.worker_manager[task.worker]
|
57
|
+
end
|
58
|
+
|
59
|
+
def volatile?
|
60
|
+
task.volatile? || env.options.disable_state
|
61
|
+
end
|
62
|
+
|
63
|
+
def dryrun?
|
64
|
+
env.options.dryrun || env.options.preview
|
23
65
|
end
|
24
66
|
end
|
25
67
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'monitor'
|
2
|
+
|
3
|
+
module Abid
|
4
|
+
class JobManager
|
5
|
+
def initialize(env)
|
6
|
+
@env = env
|
7
|
+
@cache = {}
|
8
|
+
@mon = Monitor.new
|
9
|
+
end
|
10
|
+
|
11
|
+
def [](name, params = {})
|
12
|
+
@mon.synchronize do
|
13
|
+
key = [name, params.sort.freeze].freeze
|
14
|
+
@cache[key] ||= Job.new(@env, name, params)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def find_by_task(task)
|
19
|
+
self[task.name, task.params]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/abid/mixin_task.rb
CHANGED
@@ -17,12 +17,12 @@ module Abid
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def execute(_args = nil)
|
20
|
-
|
20
|
+
raise 'mixin is not executable'
|
21
21
|
end
|
22
22
|
|
23
23
|
class <<self
|
24
24
|
def define_mixin(*args, &block) # :nodoc:
|
25
|
-
|
25
|
+
Abid.application.define_mixin(self, *args, &block)
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
data/lib/abid/rake_extensions.rb
CHANGED
@@ -1,6 +1,14 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
1
|
+
require 'abid/rake_extensions/task'
|
2
|
+
|
3
|
+
# Delegate Rake.application to Abid.global.application
|
4
|
+
class << Rake
|
5
|
+
def application
|
6
|
+
Abid.global.application
|
7
|
+
end
|
8
|
+
|
9
|
+
def application=(app)
|
10
|
+
Abid.global.application = app
|
5
11
|
end
|
6
12
|
end
|
13
|
+
|
14
|
+
Rake::Task.include Abid::RakeExtensions::Task
|
@@ -9,18 +9,12 @@ module Abid
|
|
9
9
|
:default
|
10
10
|
end
|
11
11
|
|
12
|
-
def
|
13
|
-
|
14
|
-
session.add_observer do |_, _, reason|
|
15
|
-
if session.successed? || session.failed?
|
16
|
-
call_hooks(:after_invoke, reason)
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
12
|
+
def job
|
13
|
+
Job[name, defined?(params) ? params : {}]
|
20
14
|
end
|
21
15
|
|
22
|
-
def
|
23
|
-
|
16
|
+
def params
|
17
|
+
{}
|
24
18
|
end
|
25
19
|
|
26
20
|
def name_with_params
|
@@ -42,113 +36,6 @@ module Abid
|
|
42
36
|
def call_hooks(tag, *args)
|
43
37
|
hooks[tag].each { |h| h.call(*args) }
|
44
38
|
end
|
45
|
-
|
46
|
-
def async_invoke(*args)
|
47
|
-
task_args = Rake::TaskArguments.new(arg_names, args)
|
48
|
-
async_invoke_with_call_chain(task_args, Rake::InvocationChain::EMPTY)
|
49
|
-
end
|
50
|
-
|
51
|
-
def async_invoke_with_call_chain(task_args, invocation_chain)
|
52
|
-
session.enter do
|
53
|
-
session.capture_exception do
|
54
|
-
new_chain = Rake::InvocationChain.append(self, invocation_chain)
|
55
|
-
|
56
|
-
unless concerned?
|
57
|
-
session.skip
|
58
|
-
break
|
59
|
-
end
|
60
|
-
|
61
|
-
call_hooks(:before_invoke)
|
62
|
-
|
63
|
-
async_invoke_prerequisites(task_args, new_chain)
|
64
|
-
|
65
|
-
async_execute_after_prerequisites(task_args)
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
def async_invoke_prerequisites(task_args, invocation_chain)
|
71
|
-
prerequisite_tasks.each do |t|
|
72
|
-
args = task_args.new_scope(t.arg_names)
|
73
|
-
t.async_invoke_with_call_chain(args, invocation_chain)
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
def async_execute_after_prerequisites(task_args)
|
78
|
-
if prerequisite_tasks.empty?
|
79
|
-
async_execute(task_args)
|
80
|
-
else
|
81
|
-
counter = Concurrent::DependencyCounter.new(prerequisite_tasks.size) do
|
82
|
-
session.capture_exception do
|
83
|
-
async_execute(task_args)
|
84
|
-
end
|
85
|
-
end
|
86
|
-
prerequisite_tasks.each { |t| t.session.add_observer counter }
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
def async_execute(task_args)
|
91
|
-
failed_task = prerequisite_tasks.find do |t|
|
92
|
-
t.session.failed? || t.session.canceled?
|
93
|
-
end
|
94
|
-
if failed_task
|
95
|
-
session.cancel(failed_task.session.error)
|
96
|
-
return
|
97
|
-
end
|
98
|
-
|
99
|
-
async_post(worker) do
|
100
|
-
if !needed?
|
101
|
-
session.skip
|
102
|
-
elsif session.lock
|
103
|
-
call_hooks(:before_execute)
|
104
|
-
|
105
|
-
execute(task_args)
|
106
|
-
|
107
|
-
session.success
|
108
|
-
else
|
109
|
-
async_wait_external
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
113
|
-
|
114
|
-
def async_wait_external
|
115
|
-
unless application.options.wait_external_task
|
116
|
-
fail "task #{name_with_params} already running"
|
117
|
-
end
|
118
|
-
|
119
|
-
application.trace "** Wait #{name_with_params}" if application.options.trace
|
120
|
-
|
121
|
-
async_post(:waiter) do
|
122
|
-
interval = application.options.wait_external_task_interval || 10
|
123
|
-
timeout = application.options.wait_external_task_timeout || 3600
|
124
|
-
timeout_tm = Time.now.to_f + timeout
|
125
|
-
|
126
|
-
loop do
|
127
|
-
state.reload
|
128
|
-
if !state.running?
|
129
|
-
session.success
|
130
|
-
break
|
131
|
-
elsif Time.now.to_f >= timeout_tm
|
132
|
-
fail "#{name} -- timeout exceeded"
|
133
|
-
else
|
134
|
-
sleep interval
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
|
-
def async_post(worker_name, &block)
|
141
|
-
application.worker[worker_name].post do
|
142
|
-
session.capture_exception do
|
143
|
-
begin
|
144
|
-
block.call
|
145
|
-
finished = true
|
146
|
-
ensure
|
147
|
-
fail 'thread killed' if $ERROR_INFO.nil? && !finished
|
148
|
-
end
|
149
|
-
end
|
150
|
-
end
|
151
|
-
end
|
152
39
|
end
|
153
40
|
end
|
154
41
|
end
|
@@ -1,35 +1,39 @@
|
|
1
1
|
require 'sequel/plugins/serialization'
|
2
|
+
require 'abid/state_manager/state'
|
3
|
+
require 'abid/state_manager/state_proxy'
|
2
4
|
|
3
5
|
module Abid
|
4
6
|
module StateManager
|
5
|
-
|
7
|
+
class Database
|
6
8
|
Sequel.extension :migration
|
7
9
|
|
8
|
-
|
9
|
-
|
10
|
-
#
|
10
|
+
MIGRATIONS_PATH = File.expand_path('../../../../migrations', __FILE__)
|
11
|
+
|
12
|
+
# Creates a new database object and checks schema version.
|
11
13
|
#
|
12
14
|
# @return [Sequel::Database] database object
|
13
|
-
|
14
|
-
|
15
|
-
Sequel
|
15
|
+
# @see Sequel.connect
|
16
|
+
def self.connect(*args)
|
17
|
+
db = Sequel.connect(*args)
|
18
|
+
Sequel::Migrator.check_current(db, MIGRATIONS_PATH)
|
16
19
|
db
|
17
20
|
rescue Sequel::Migrator::NotCurrentError
|
18
21
|
raise Error, 'current schema is out of date'
|
19
22
|
end
|
20
23
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
Sequel.connect(**params)
|
24
|
+
def initialize(env)
|
25
|
+
@env = env
|
26
|
+
@mon = Monitor.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def connection
|
30
|
+
@connection ||= self.class.connect(@env.config.database)
|
29
31
|
end
|
30
32
|
|
31
|
-
def
|
32
|
-
|
33
|
+
def states
|
34
|
+
@mon.synchronize do
|
35
|
+
@states ||= Class.new(State.Model(connection[:states]))
|
36
|
+
end
|
33
37
|
end
|
34
38
|
end
|
35
39
|
end
|
@@ -1,7 +1,11 @@
|
|
1
1
|
module Abid
|
2
2
|
module StateManager
|
3
|
+
State = Class.new(Sequel::Model)
|
4
|
+
|
3
5
|
# O/R Mapper for `states` table.
|
4
|
-
|
6
|
+
#
|
7
|
+
# Use #init_by_job to initialize a state object.
|
8
|
+
class State
|
5
9
|
RUNNING = 1
|
6
10
|
SUCCESSED = 2
|
7
11
|
FAILED = 3
|
@@ -42,9 +46,55 @@ module Abid
|
|
42
46
|
).first
|
43
47
|
end
|
44
48
|
|
45
|
-
|
46
|
-
|
47
|
-
|
49
|
+
# Initialize a state by a job.
|
50
|
+
#
|
51
|
+
# @param job [Job] job
|
52
|
+
# @return [State] state object
|
53
|
+
def self.init_by_job(job)
|
54
|
+
new(
|
55
|
+
name: job.name,
|
56
|
+
params: job.params_str,
|
57
|
+
digest: job.digest
|
58
|
+
)
|
59
|
+
end
|
60
|
+
|
61
|
+
# Find or initialize a state by a job.
|
62
|
+
#
|
63
|
+
# @param job [Job] job
|
64
|
+
# @return [State] state object
|
65
|
+
def self.find_or_init_by_job(job)
|
66
|
+
find_by_job(job) || init_by_job(job)
|
67
|
+
end
|
68
|
+
|
69
|
+
# Update the state to RUNNING.
|
70
|
+
#
|
71
|
+
# @param job [Job] job
|
72
|
+
def self.start(job)
|
73
|
+
db.transaction do
|
74
|
+
state = find_or_init_by_job(job)
|
75
|
+
state.check_running!
|
76
|
+
state.state = RUNNING
|
77
|
+
state.start_time = Time.now
|
78
|
+
state.end_time = nil
|
79
|
+
state.save
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# Update the state to SUCCESSED or FAILED.
|
84
|
+
#
|
85
|
+
# If error is given, the state will be FAILED.
|
86
|
+
#
|
87
|
+
# @param job [Job] job
|
88
|
+
# @param error [Error] error object
|
89
|
+
def self.finish(job, error = nil)
|
90
|
+
db.transaction do
|
91
|
+
state = find_or_init_by_job(job)
|
92
|
+
return unless state.running?
|
93
|
+
|
94
|
+
state.state = error ? FAILED : SUCCESSED
|
95
|
+
state.end_time = Time.now
|
96
|
+
state.save
|
97
|
+
end
|
48
98
|
end
|
49
99
|
|
50
100
|
# Assume the job to be successed
|
@@ -54,11 +104,10 @@ module Abid
|
|
54
104
|
#
|
55
105
|
# @param job [Job] job
|
56
106
|
# @param force [Boolean] force update the state
|
57
|
-
# @return [
|
107
|
+
# @return [void]
|
58
108
|
def self.assume(job, force: false)
|
59
|
-
|
60
|
-
state =
|
61
|
-
|
109
|
+
db.transaction do
|
110
|
+
state = find_or_init_by_job(job)
|
62
111
|
return state if state.successed?
|
63
112
|
state.check_running! unless force
|
64
113
|
|
@@ -72,15 +121,16 @@ module Abid
|
|
72
121
|
|
73
122
|
# Delete the state.
|
74
123
|
#
|
124
|
+
# @param state_id [Integer] State ID
|
75
125
|
# @param force [Boolean] If true, delete the state even if running
|
76
126
|
# @return [void]
|
77
|
-
def revoke(force: false)
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
127
|
+
def self.revoke(state_id, force: false)
|
128
|
+
db.transaction do
|
129
|
+
state = self[state_id]
|
130
|
+
return false if state.nil?
|
131
|
+
state.check_running! unless force
|
132
|
+
state.delete
|
133
|
+
true
|
84
134
|
end
|
85
135
|
end
|
86
136
|
|