abid 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.travis.yml +4 -0
- data/CODE_OF_CONDUCT.md +13 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +89 -0
- data/Rakefile +32 -0
- data/abid.gemspec +31 -0
- data/bin/console +14 -0
- data/bin/setup +7 -0
- data/exe/abid +4 -0
- data/lib/Abidfile.rb +80 -0
- data/lib/abid.rb +28 -0
- data/lib/abid/application.rb +141 -0
- data/lib/abid/dsl_definition.rb +21 -0
- data/lib/abid/params_parser.rb +50 -0
- data/lib/abid/play.rb +139 -0
- data/lib/abid/rake_extensions.rb +6 -0
- data/lib/abid/rake_extensions/task.rb +135 -0
- data/lib/abid/state.rb +153 -0
- data/lib/abid/task.rb +30 -0
- data/lib/abid/task_manager.rb +65 -0
- data/lib/abid/version.rb +3 -0
- data/lib/abid/waiter.rb +110 -0
- data/lib/abid/worker.rb +39 -0
- data/migrations/01_create_state_table.rb +15 -0
- metadata +183 -0
@@ -0,0 +1,21 @@
|
|
1
|
+
module Abid
|
2
|
+
module DSL
|
3
|
+
def play(*args, &block)
|
4
|
+
Abid::Task.define_play(*args, &block)
|
5
|
+
end
|
6
|
+
|
7
|
+
def define_worker(name, thread_count)
|
8
|
+
Rake.application.worker.define(name, thread_count)
|
9
|
+
end
|
10
|
+
|
11
|
+
def default_play_class(&block)
|
12
|
+
Rake.application.default_play_class(&block)
|
13
|
+
end
|
14
|
+
|
15
|
+
def helpers(*extensions, &block)
|
16
|
+
Abid::Play.helpers(*extensions, &block)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
extend Abid::DSL
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module Abid
|
2
|
+
module ParamsParser
|
3
|
+
class <<self
|
4
|
+
def parse(params, specs)
|
5
|
+
specs.map do |name, spec|
|
6
|
+
if params.include?(name)
|
7
|
+
value = type_cast(params[name], spec[:type])
|
8
|
+
elsif ENV.include?(name.to_s)
|
9
|
+
value = type_cast(ENV[name.to_s], spec[:type])
|
10
|
+
elsif spec.key?(:default)
|
11
|
+
value = spec[:default]
|
12
|
+
else
|
13
|
+
fail "param #{name} is not specified"
|
14
|
+
end
|
15
|
+
|
16
|
+
[name, value]
|
17
|
+
end.to_h
|
18
|
+
end
|
19
|
+
|
20
|
+
def type_cast(value, type)
|
21
|
+
case type
|
22
|
+
when :boolean then value == 'true'
|
23
|
+
when :int then value.to_i
|
24
|
+
when :float then value.to_f
|
25
|
+
when :string then value.to_s
|
26
|
+
when :date then type_cast_date(value)
|
27
|
+
when :datetime, :time then type_cast_time(value)
|
28
|
+
when nil then value
|
29
|
+
else fail "invalid type: #{type}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def type_cast_date(value)
|
34
|
+
case value
|
35
|
+
when Date then value
|
36
|
+
when Time, DateTime then value.to_date
|
37
|
+
else Date.parse(value.to_s)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def type_cast_time(value)
|
42
|
+
case value
|
43
|
+
when Date then value.to_time
|
44
|
+
when Time, DateTime then value
|
45
|
+
else Time.parse(value.to_s)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/abid/play.rb
ADDED
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
module Abid
|
4
|
+
class Play
|
5
|
+
class << self
|
6
|
+
attr_accessor :task
|
7
|
+
|
8
|
+
def inherited(child)
|
9
|
+
params_spec.each { |k, v| child.params_spec[k] = v.dup }
|
10
|
+
hooks.each { |k, v| child.hooks[k] = v.dup }
|
11
|
+
end
|
12
|
+
|
13
|
+
def params_spec
|
14
|
+
@params_spec ||= {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def param(name, **param_spec)
|
18
|
+
params_spec[name] = { significant: true }.merge(param_spec)
|
19
|
+
|
20
|
+
define_method(name) { params[name] }
|
21
|
+
end
|
22
|
+
|
23
|
+
def hooks
|
24
|
+
@hooks ||= {
|
25
|
+
before: [],
|
26
|
+
after: [],
|
27
|
+
around: []
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def set(name, value = nil, &block)
|
32
|
+
var = :"@#{name}"
|
33
|
+
define_method(name) do
|
34
|
+
unless instance_variable_defined?(var)
|
35
|
+
if !value.nil?
|
36
|
+
instance_variable_set(var, value)
|
37
|
+
elsif block_given?
|
38
|
+
instance_variable_set(var, instance_eval(&block))
|
39
|
+
end
|
40
|
+
end
|
41
|
+
instance_variable_get(var)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def helpers(*extensions, &block)
|
46
|
+
class_eval(&block) if block_given?
|
47
|
+
include(*extensions) if extensions.any?
|
48
|
+
end
|
49
|
+
|
50
|
+
def before(&block)
|
51
|
+
(hooks[:before] ||= []) << block
|
52
|
+
end
|
53
|
+
|
54
|
+
def after(&block)
|
55
|
+
(hooks[:after] ||= []) << block
|
56
|
+
end
|
57
|
+
|
58
|
+
def around(&block)
|
59
|
+
(hooks[:around] ||= []) << block
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
set :worker, :default
|
64
|
+
set :volatile, false
|
65
|
+
|
66
|
+
extend Forwardable
|
67
|
+
def_delegators :task, :application, :name, :scope
|
68
|
+
def_delegators 'self.class', :params_spec
|
69
|
+
|
70
|
+
attr_reader :prerequisites
|
71
|
+
attr_reader :params
|
72
|
+
|
73
|
+
def initialize(params)
|
74
|
+
@prerequisites = []
|
75
|
+
|
76
|
+
@params = ParamsParser.parse(params, params_spec)
|
77
|
+
@params = @params.sort.to_h # avoid ambiguity of keys order
|
78
|
+
@params.freeze
|
79
|
+
end
|
80
|
+
|
81
|
+
def task
|
82
|
+
self.class.task
|
83
|
+
end
|
84
|
+
|
85
|
+
def setup
|
86
|
+
# noop
|
87
|
+
end
|
88
|
+
|
89
|
+
def run
|
90
|
+
# noop
|
91
|
+
end
|
92
|
+
|
93
|
+
def needs(task_name, **params)
|
94
|
+
@prerequisites |= [[task_name, params]]
|
95
|
+
end
|
96
|
+
|
97
|
+
def significant_params
|
98
|
+
[
|
99
|
+
name,
|
100
|
+
params.select { |p, _| params_spec[p][:significant] }
|
101
|
+
]
|
102
|
+
end
|
103
|
+
|
104
|
+
def hash
|
105
|
+
significant_params.hash
|
106
|
+
end
|
107
|
+
|
108
|
+
def eql?(other)
|
109
|
+
other.is_a?(Abid::Play) && \
|
110
|
+
significant_params.eql?(other.significant_params)
|
111
|
+
end
|
112
|
+
|
113
|
+
def volatile?
|
114
|
+
volatile
|
115
|
+
end
|
116
|
+
|
117
|
+
def preview?
|
118
|
+
application.options.preview
|
119
|
+
end
|
120
|
+
|
121
|
+
def invoke
|
122
|
+
self.class.hooks[:before].each { |blk| instance_eval(&blk) }
|
123
|
+
|
124
|
+
call_around_hooks(self.class.hooks[:around]) { run }
|
125
|
+
|
126
|
+
self.class.hooks[:after].each { |blk| instance_eval(&blk) }
|
127
|
+
end
|
128
|
+
|
129
|
+
def call_around_hooks(hooks, &body)
|
130
|
+
if hooks.empty?
|
131
|
+
body.call
|
132
|
+
else
|
133
|
+
h, *rest = hooks
|
134
|
+
instance_exec(-> { call_around_hooks(rest, &body) }, &h)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
private :call_around_hooks
|
138
|
+
end
|
139
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
module Abid
|
2
|
+
module RakeExtensions
|
3
|
+
module Task
|
4
|
+
def volatile?
|
5
|
+
true
|
6
|
+
end
|
7
|
+
|
8
|
+
def worker
|
9
|
+
:default
|
10
|
+
end
|
11
|
+
|
12
|
+
def state
|
13
|
+
@state ||= State.find(self)
|
14
|
+
end
|
15
|
+
|
16
|
+
def async_invoke(*args)
|
17
|
+
task_args = Rake::TaskArguments.new(arg_names, args)
|
18
|
+
async_invoke_with_call_chain(task_args, Rake::InvocationChain::EMPTY)
|
19
|
+
end
|
20
|
+
|
21
|
+
def async_invoke_with_call_chain(task_args, invocation_chain)
|
22
|
+
state.reload
|
23
|
+
new_chain = Rake::InvocationChain.append(self, invocation_chain)
|
24
|
+
@lock.synchronize do
|
25
|
+
if application.futures.include?(object_id)
|
26
|
+
return application.futures[object_id]
|
27
|
+
end
|
28
|
+
|
29
|
+
application.trace "** Invoke #{name}" if application.options.trace
|
30
|
+
|
31
|
+
preq_futures = async_invoke_prerequisites(task_args, new_chain)
|
32
|
+
|
33
|
+
future = async_invoke_after_prerequisites(task_args, preq_futures)
|
34
|
+
|
35
|
+
application.futures[object_id] = future
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def async_invoke_prerequisites(task_args, invocation_chain)
|
40
|
+
# skip if successed
|
41
|
+
if state.successed?
|
42
|
+
if !application.options.check_prerequisites
|
43
|
+
preqs = []
|
44
|
+
else
|
45
|
+
preqs = prerequisite_tasks.reject(&:volatile?)
|
46
|
+
end
|
47
|
+
else
|
48
|
+
preqs = prerequisite_tasks
|
49
|
+
end
|
50
|
+
|
51
|
+
preqs.map do |p|
|
52
|
+
preq_args = task_args.new_scope(p.arg_names)
|
53
|
+
p.async_invoke_with_call_chain(preq_args, invocation_chain)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def async_invoke_after_prerequisites(task_args, preq_futures)
|
58
|
+
if preq_futures.empty?
|
59
|
+
async_execute_with_session(task_args, false)
|
60
|
+
else
|
61
|
+
result = Concurrent::IVar.new
|
62
|
+
counter = Concurrent::DependencyCounter.new(preq_futures.size) do
|
63
|
+
begin
|
64
|
+
failed_preq = preq_futures.find(&:rejected?)
|
65
|
+
next result.fail(failed_preq.reason) if failed_preq
|
66
|
+
|
67
|
+
preq_updated = preq_futures.map(&:value!).any?
|
68
|
+
|
69
|
+
future = async_execute_with_session(task_args, preq_updated)
|
70
|
+
|
71
|
+
future.add_observer do |_time, value, reason|
|
72
|
+
reason.nil? ? result.set(value) : result.fail(reason)
|
73
|
+
end
|
74
|
+
rescue Exception => err
|
75
|
+
result.fail(err)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
preq_futures.each { |p| p.add_observer counter }
|
79
|
+
result
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def async_execute_with_session(task_args, prerequisites_updated = false)
|
84
|
+
if (state.successed? && !prerequisites_updated) || !needed?
|
85
|
+
application.trace "** Skip #{name}" if application.options.trace
|
86
|
+
return Concurrent::IVar.new(false)
|
87
|
+
end
|
88
|
+
|
89
|
+
session_started = state.start_session
|
90
|
+
|
91
|
+
return async_wait_complete unless session_started
|
92
|
+
|
93
|
+
pool = application.worker[worker]
|
94
|
+
future = Concurrent::Future.execute(executor: pool) do
|
95
|
+
begin
|
96
|
+
execute(task_args)
|
97
|
+
true
|
98
|
+
ensure
|
99
|
+
state.close_session($ERROR_INFO)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
ensure
|
103
|
+
# close session if error occurred outside the future
|
104
|
+
if session_started && future.nil? && $ERROR_INFO
|
105
|
+
state.close_session($ERROR_INFO)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def async_wait_complete
|
110
|
+
unless application.options.wait_external_task
|
111
|
+
err = RuntimeError.new("task #{name} already running")
|
112
|
+
return Concurrent::IVar.new.fail(err)
|
113
|
+
end
|
114
|
+
|
115
|
+
application.trace "** Wait #{name}" if application.options.trace
|
116
|
+
|
117
|
+
pool = application.worker[:waiter]
|
118
|
+
Concurrent::Future.execute(executor: pool) do
|
119
|
+
interval = application.options.wait_external_task_interval || 10
|
120
|
+
timeout = application.options.wait_external_task_timeout || 3600
|
121
|
+
timeout_tm = Time.now.to_f + timeout
|
122
|
+
|
123
|
+
loop do
|
124
|
+
state.reload
|
125
|
+
break unless state.running?
|
126
|
+
|
127
|
+
sleep interval
|
128
|
+
break if Time.now.to_f >= timeout_tm
|
129
|
+
end
|
130
|
+
true
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
data/lib/abid/state.rb
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
module Abid
|
2
|
+
class State
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
RUNNING = 1
|
6
|
+
SUCCESSED = 2
|
7
|
+
FAILED = 3
|
8
|
+
|
9
|
+
STATES = constants.map { |c| [const_get(c), c] }.to_h
|
10
|
+
|
11
|
+
class <<self
|
12
|
+
def find(task)
|
13
|
+
new(task)
|
14
|
+
end
|
15
|
+
|
16
|
+
def list(pattern: nil, started_before: nil, started_after: nil)
|
17
|
+
dataset = Rake.application.database[:states]
|
18
|
+
|
19
|
+
dataset = dataset.where { start_time < started_before } if started_before
|
20
|
+
dataset = dataset.where { start_time > started_after } if started_after
|
21
|
+
dataset = dataset.order(:start_time)
|
22
|
+
|
23
|
+
dataset.map do |record|
|
24
|
+
next if pattern && record[:name] !~ pattern
|
25
|
+
{
|
26
|
+
id: record[:id],
|
27
|
+
name: record[:name],
|
28
|
+
params: deserialize(record[:params]),
|
29
|
+
state: STATES[record[:state]],
|
30
|
+
start_time: record[:start_time],
|
31
|
+
end_time: record[:end_time]
|
32
|
+
}
|
33
|
+
end.compact
|
34
|
+
end
|
35
|
+
|
36
|
+
def serialize(params)
|
37
|
+
YAML.dump(params)
|
38
|
+
end
|
39
|
+
|
40
|
+
def deserialize(bytes)
|
41
|
+
YAML.load(bytes)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def_delegators 'self.class', :serialize, :deserialize
|
46
|
+
|
47
|
+
def initialize(task)
|
48
|
+
@task = task
|
49
|
+
reload
|
50
|
+
end
|
51
|
+
|
52
|
+
def database
|
53
|
+
Rake.application.database
|
54
|
+
end
|
55
|
+
|
56
|
+
def dataset
|
57
|
+
database[:states]
|
58
|
+
end
|
59
|
+
|
60
|
+
def disabled?
|
61
|
+
@task.volatile? || Rake.application.options.disable_state
|
62
|
+
end
|
63
|
+
|
64
|
+
def reload
|
65
|
+
return if disabled?
|
66
|
+
|
67
|
+
if @record
|
68
|
+
id = @record[:id]
|
69
|
+
@record = dataset.where(id: id).first
|
70
|
+
else
|
71
|
+
@record = dataset.where(digest: digest).to_a.find do |r|
|
72
|
+
[@task.name, @task.params].eql? [r[:name], deserialize(r[:params])]
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def id
|
78
|
+
@record[:id] if @record
|
79
|
+
end
|
80
|
+
|
81
|
+
def state
|
82
|
+
@record[:state] if @record
|
83
|
+
end
|
84
|
+
|
85
|
+
def running?
|
86
|
+
state == RUNNING
|
87
|
+
end
|
88
|
+
|
89
|
+
def successed?
|
90
|
+
state == SUCCESSED
|
91
|
+
end
|
92
|
+
|
93
|
+
def failed?
|
94
|
+
state == FAILED
|
95
|
+
end
|
96
|
+
|
97
|
+
def revoke
|
98
|
+
fail 'cannot revoke volatile task' if disabled?
|
99
|
+
|
100
|
+
database.transaction do
|
101
|
+
reload
|
102
|
+
fail 'task is not executed yet' if id.nil?
|
103
|
+
fail 'task is now running' if running?
|
104
|
+
dataset.where(id: id).delete
|
105
|
+
end
|
106
|
+
|
107
|
+
@record = nil
|
108
|
+
end
|
109
|
+
|
110
|
+
def start_session
|
111
|
+
return true if disabled?
|
112
|
+
|
113
|
+
database.transaction do
|
114
|
+
reload
|
115
|
+
|
116
|
+
return false if running?
|
117
|
+
|
118
|
+
new_state = {
|
119
|
+
state: RUNNING,
|
120
|
+
start_time: Time.now,
|
121
|
+
end_time: nil
|
122
|
+
}
|
123
|
+
|
124
|
+
if @record
|
125
|
+
dataset.where(id: @record[:id]).update(new_state)
|
126
|
+
@record = @record.merge(new_state)
|
127
|
+
else
|
128
|
+
id = dataset.insert(
|
129
|
+
digest: digest,
|
130
|
+
name: @task.name,
|
131
|
+
params: serialize(@task.params),
|
132
|
+
**new_state
|
133
|
+
)
|
134
|
+
@record = { id: id, **new_state }
|
135
|
+
end
|
136
|
+
|
137
|
+
true
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def close_session(error = nil)
|
142
|
+
return if disabled?
|
143
|
+
return unless @record
|
144
|
+
state = error ? FAILED : SUCCESSED
|
145
|
+
dataset.where(id: @record[:id]).update(state: state, end_time: Time.now)
|
146
|
+
reload
|
147
|
+
end
|
148
|
+
|
149
|
+
def digest
|
150
|
+
Digest::MD5.hexdigest(@task.name + "\n" + serialize(@task.params))
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|