abid 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +306 -4
- data/abid.gemspec +1 -0
- data/lib/Abidfile.rb +1 -80
- data/lib/abid.rb +3 -1
- data/lib/abid/application.rb +21 -17
- data/lib/abid/concurrent_extention.rb +6 -0
- data/lib/abid/concurrent_extention/ivar.rb +12 -0
- data/lib/abid/dsl_definition.rb +2 -2
- data/lib/abid/params_parser.rb +1 -1
- data/lib/abid/play.rb +19 -60
- data/lib/abid/rake_extensions/task.rb +82 -71
- data/lib/abid/state.rb +63 -8
- data/lib/abid/task.rb +101 -8
- data/lib/abid/task_manager.rb +13 -23
- data/lib/abid/tasks/core.rake +85 -0
- data/lib/abid/version.rb +1 -1
- data/lib/abid/worker.rb +26 -16
- metadata +19 -2
data/lib/abid/state.rb
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
module Abid
|
2
|
+
class AbidErrorTaskAlreadyRunning < StandardError; end
|
3
|
+
|
2
4
|
class State
|
3
5
|
extend Forwardable
|
6
|
+
extend MonitorMixin
|
4
7
|
|
5
8
|
RUNNING = 1
|
6
9
|
SUCCESSED = 2
|
@@ -8,9 +11,10 @@ module Abid
|
|
8
11
|
|
9
12
|
STATES = constants.map { |c| [const_get(c), c] }.to_h
|
10
13
|
|
14
|
+
@cache = {}
|
11
15
|
class <<self
|
12
16
|
def find(task)
|
13
|
-
new(task)
|
17
|
+
synchronize { @cache[task.object_id] ||= new(task) }
|
14
18
|
end
|
15
19
|
|
16
20
|
def list(pattern: nil, started_before: nil, started_after: nil)
|
@@ -33,6 +37,15 @@ module Abid
|
|
33
37
|
end.compact
|
34
38
|
end
|
35
39
|
|
40
|
+
def revoke(id)
|
41
|
+
db = Rake.application.database
|
42
|
+
db.transaction do
|
43
|
+
running = db[:states].where(id: id, state: RUNNING).count > 0
|
44
|
+
fail 'task is now running' if running
|
45
|
+
db[:states].where(id: id).delete
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
36
49
|
def serialize(params)
|
37
50
|
YAML.dump(params)
|
38
51
|
end
|
@@ -43,12 +56,24 @@ module Abid
|
|
43
56
|
end
|
44
57
|
|
45
58
|
def_delegators 'self.class', :serialize, :deserialize
|
59
|
+
attr_reader :ivar
|
46
60
|
|
47
61
|
def initialize(task)
|
48
62
|
@task = task
|
63
|
+
@record = nil
|
64
|
+
@ivar = Concurrent::IVar.new
|
65
|
+
@already_invoked = false
|
49
66
|
reload
|
50
67
|
end
|
51
68
|
|
69
|
+
def only_once(&block)
|
70
|
+
self.class.synchronize do
|
71
|
+
return if @already_invoked
|
72
|
+
@already_invoked = true
|
73
|
+
end
|
74
|
+
block.call
|
75
|
+
end
|
76
|
+
|
52
77
|
def database
|
53
78
|
Rake.application.database
|
54
79
|
end
|
@@ -61,6 +86,10 @@ module Abid
|
|
61
86
|
@task.volatile? || Rake.application.options.disable_state
|
62
87
|
end
|
63
88
|
|
89
|
+
def preview?
|
90
|
+
Rake.application.options.dryrun || Rake.application.options.preview
|
91
|
+
end
|
92
|
+
|
64
93
|
def reload
|
65
94
|
return if disabled?
|
66
95
|
|
@@ -94,26 +123,52 @@ module Abid
|
|
94
123
|
state == FAILED
|
95
124
|
end
|
96
125
|
|
97
|
-
def
|
126
|
+
def not_found?
|
127
|
+
!disabled? && @record.nil?
|
128
|
+
end
|
129
|
+
|
130
|
+
def assume
|
98
131
|
fail 'cannot revoke volatile task' if disabled?
|
99
132
|
|
100
133
|
database.transaction do
|
101
134
|
reload
|
102
|
-
fail 'task is not executed yet' if id.nil?
|
103
135
|
fail 'task is now running' if running?
|
104
|
-
|
136
|
+
|
137
|
+
new_state = {
|
138
|
+
state: SUCCESSED,
|
139
|
+
start_time: Time.now,
|
140
|
+
end_time: Time.now
|
141
|
+
}
|
142
|
+
|
143
|
+
if @record
|
144
|
+
dataset.where(id: @record[:id]).update(new_state)
|
145
|
+
@record = @record.merge(new_state)
|
146
|
+
else
|
147
|
+
id = dataset.insert(
|
148
|
+
digest: digest,
|
149
|
+
name: @task.name,
|
150
|
+
params: serialize(@task.params),
|
151
|
+
**new_state
|
152
|
+
)
|
153
|
+
@record = { id: id, **new_state }
|
154
|
+
end
|
105
155
|
end
|
156
|
+
end
|
106
157
|
|
107
|
-
|
158
|
+
def session(&block)
|
159
|
+
started = start_session
|
160
|
+
block.call
|
161
|
+
ensure
|
162
|
+
close_session($ERROR_INFO) if started
|
108
163
|
end
|
109
164
|
|
110
165
|
def start_session
|
111
|
-
return true if disabled?
|
166
|
+
return true if disabled? || preview?
|
112
167
|
|
113
168
|
database.transaction do
|
114
169
|
reload
|
115
170
|
|
116
|
-
|
171
|
+
fail AbidErrorTaskAlreadyRunning if running?
|
117
172
|
|
118
173
|
new_state = {
|
119
174
|
state: RUNNING,
|
@@ -139,7 +194,7 @@ module Abid
|
|
139
194
|
end
|
140
195
|
|
141
196
|
def close_session(error = nil)
|
142
|
-
return if disabled?
|
197
|
+
return if disabled? || preview?
|
143
198
|
return unless @record
|
144
199
|
state = error ? FAILED : SUCCESSED
|
145
200
|
dataset.where(id: @record[:id]).update(state: state, end_time: Time.now)
|
data/lib/abid/task.rb
CHANGED
@@ -2,24 +2,117 @@ module Abid
|
|
2
2
|
class Task < Rake::Task
|
3
3
|
extend Forwardable
|
4
4
|
|
5
|
-
attr_accessor :
|
6
|
-
|
5
|
+
attr_accessor :play_class_definition, :extends
|
6
|
+
attr_reader :play, :params
|
7
7
|
|
8
|
-
def_delegators :play, :
|
8
|
+
def_delegators :play, :worker, :volatile?
|
9
9
|
|
10
10
|
def initialize(task_name, app)
|
11
11
|
super(task_name, app)
|
12
|
-
@
|
13
|
-
|
12
|
+
@siblings = {}
|
13
|
+
end
|
14
|
+
|
15
|
+
def play_class
|
16
|
+
return @play_class if @play_class
|
17
|
+
|
18
|
+
klass = application.lookup_play_class(extends, scope)
|
19
|
+
@play_class = Class.new(klass, &play_class_definition).tap do |c|
|
20
|
+
c.task = self
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def bound?
|
25
|
+
!@play.nil?
|
26
|
+
end
|
27
|
+
|
28
|
+
def bind(**params)
|
29
|
+
fail 'already bound' if bound?
|
30
|
+
|
31
|
+
parsed_params = ParamsParser.parse(params, play_class.params_spec)
|
32
|
+
return @siblings[parsed_params] if @siblings.include?(parsed_params)
|
33
|
+
|
34
|
+
sorted_params = parsed_params.sort.to_h
|
35
|
+
sorted_params.freeze
|
36
|
+
|
37
|
+
@siblings[sorted_params] = dup.tap do |t|
|
38
|
+
t.instance_eval do
|
39
|
+
@prerequisites = []
|
40
|
+
@params = sorted_params
|
41
|
+
@play = play_class.new(t)
|
42
|
+
end
|
43
|
+
play_class.hooks[:setup].each { |blk| t.play.instance_eval(&blk) }
|
44
|
+
end
|
14
45
|
end
|
15
46
|
|
16
47
|
def prerequisite_tasks
|
17
|
-
fail 'no play is bound yet'
|
48
|
+
fail 'no play is bound yet' unless bound?
|
49
|
+
|
50
|
+
prerequisites.map do |pre, params|
|
51
|
+
application[pre, @scope, **self.params.merge(params)]
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# Name of task with argument list description.
|
56
|
+
def name_with_args # :nodoc:
|
57
|
+
if params_description
|
58
|
+
"#{super} #{params_description}"
|
59
|
+
else
|
60
|
+
super
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Name of task with params
|
65
|
+
def name_with_params # :nodoc:
|
66
|
+
if params_description
|
67
|
+
"#{name} #{params_description}"
|
68
|
+
else
|
69
|
+
super
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def params_description
|
74
|
+
sig_params = play_class.params_spec.select do |_, spec|
|
75
|
+
spec[:significant]
|
76
|
+
end
|
77
|
+
return if sig_params.empty?
|
78
|
+
|
79
|
+
if bound? # unbound
|
80
|
+
p = sig_params.map { |name, _| "#{name}=#{params[name]}" }
|
81
|
+
else
|
82
|
+
p = sig_params.map { |name, spec| "#{name}:#{spec[:type]}" }
|
83
|
+
end
|
84
|
+
|
85
|
+
p.join(' ')
|
86
|
+
end
|
87
|
+
|
88
|
+
# Execute the play associated with this task.
|
89
|
+
def execute(_args = nil)
|
90
|
+
fail 'no play is bound yet' unless bound?
|
91
|
+
|
92
|
+
if application.options.dryrun
|
93
|
+
application.trace "** Execute (dry run) #{name_with_params}"
|
94
|
+
return
|
95
|
+
end
|
96
|
+
if application.options.trace
|
97
|
+
application.trace "** Execute #{name_with_params}"
|
98
|
+
end
|
99
|
+
|
100
|
+
play_class.hooks[:before].each { |blk| play.instance_eval(&blk) }
|
101
|
+
|
102
|
+
call_around_hooks(play_class.hooks[:around]) { play.run }
|
103
|
+
|
104
|
+
play_class.hooks[:after].each { |blk| play.instance_eval(&blk) }
|
105
|
+
end
|
18
106
|
|
19
|
-
|
20
|
-
|
107
|
+
def call_around_hooks(hooks, &body)
|
108
|
+
if hooks.empty?
|
109
|
+
body.call
|
110
|
+
else
|
111
|
+
h, *rest = hooks
|
112
|
+
play.instance_exec(-> { call_around_hooks(rest, &body) }, &h)
|
21
113
|
end
|
22
114
|
end
|
115
|
+
private :call_around_hooks
|
23
116
|
|
24
117
|
class <<self
|
25
118
|
def define_play(*args, &block) # :nodoc:
|
data/lib/abid/task_manager.rb
CHANGED
@@ -2,50 +2,40 @@ module Abid
|
|
2
2
|
module TaskManager
|
3
3
|
def initialize
|
4
4
|
super
|
5
|
-
@plays = {}
|
6
5
|
end
|
7
6
|
|
8
7
|
def define_play(task_class, play_name, extends: nil, &block)
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
task
|
8
|
+
define_task(task_class, play_name).tap do |task|
|
9
|
+
task.extends = extends
|
10
|
+
task.play_class_definition = block
|
11
|
+
end
|
14
12
|
end
|
15
13
|
|
16
14
|
def [](task_name, scopes = nil, **params)
|
17
15
|
task = super(task_name, scopes)
|
18
16
|
|
19
|
-
if task.respond_to? :
|
20
|
-
|
17
|
+
if task.respond_to? :bind
|
18
|
+
task.bind(**params)
|
21
19
|
else
|
22
20
|
task
|
23
21
|
end
|
24
22
|
end
|
25
23
|
|
26
|
-
def
|
27
|
-
play = task.play_class.new(**params)
|
28
|
-
|
29
|
-
return @plays[play] if @plays.include?(play)
|
30
|
-
|
31
|
-
play.setup
|
32
|
-
@plays[play] = task.dup.tap { |t| t.play = play }
|
33
|
-
end
|
34
|
-
|
35
|
-
def default_play_class(&block)
|
24
|
+
def play_base(&block)
|
36
25
|
if block_given?
|
37
|
-
@
|
26
|
+
@play_base = Class.new(Abid::Play, &block)
|
38
27
|
else
|
39
|
-
@
|
28
|
+
@play_base ||= Abid::Play
|
40
29
|
end
|
41
30
|
end
|
42
31
|
|
43
32
|
def lookup_play_class(task_name, scope = nil)
|
44
33
|
if task_name.nil?
|
45
|
-
|
34
|
+
play_base
|
35
|
+
elsif task_name.is_a? Class
|
36
|
+
task_name
|
46
37
|
else
|
47
|
-
|
48
|
-
t = lookup(task_name, scope)
|
38
|
+
t = lookup(task_name.to_s, scope)
|
49
39
|
if t.respond_to? :play_class
|
50
40
|
t.play_class
|
51
41
|
elsif t.nil?
|
@@ -0,0 +1,85 @@
|
|
1
|
+
namespace :state do
|
2
|
+
task default: :list
|
3
|
+
|
4
|
+
desc 'Show play histories'
|
5
|
+
play :list, extends: Abid::Play do
|
6
|
+
set :volatile, true
|
7
|
+
|
8
|
+
param :started_after, type: :time, default: nil
|
9
|
+
param :started_before, type: :time, default: nil
|
10
|
+
|
11
|
+
def run
|
12
|
+
states = Abid::State.list(
|
13
|
+
started_after: started_after,
|
14
|
+
started_before: started_before
|
15
|
+
)
|
16
|
+
|
17
|
+
table = states.map do |state|
|
18
|
+
params = state[:params].map do |k, v|
|
19
|
+
v.to_s =~ /\s/ ? "#{k}='#{v}'" : "#{k}=#{v}"
|
20
|
+
end.join(' ')
|
21
|
+
[
|
22
|
+
state[:id].to_s,
|
23
|
+
state[:state].to_s,
|
24
|
+
state[:name],
|
25
|
+
params,
|
26
|
+
state[:start_time].to_s,
|
27
|
+
state[:end_time].to_s
|
28
|
+
]
|
29
|
+
end
|
30
|
+
|
31
|
+
header = %w(id state name params start_time end_time)
|
32
|
+
|
33
|
+
tab_width = header.each_with_index.map do |c, i|
|
34
|
+
[c.length, table.map { |row| row[i].length }.max || 0].max
|
35
|
+
end
|
36
|
+
|
37
|
+
header.each_with_index do |c, i|
|
38
|
+
print c.ljust(tab_width[i] + 2)
|
39
|
+
end
|
40
|
+
puts
|
41
|
+
|
42
|
+
header.each_with_index do |_, i|
|
43
|
+
print '-' * (tab_width[i] + 2)
|
44
|
+
end
|
45
|
+
puts
|
46
|
+
|
47
|
+
table.map do |row|
|
48
|
+
row.each_with_index do |v, i|
|
49
|
+
print v.ljust(tab_width[i] + 2)
|
50
|
+
end
|
51
|
+
puts
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
desc 'Insert play history'
|
57
|
+
task :assume, [:task] do |_t, args|
|
58
|
+
task = Rake.application[args[:task]]
|
59
|
+
state = Abid::State.find(task)
|
60
|
+
state.assume
|
61
|
+
end
|
62
|
+
|
63
|
+
desc 'Delete play history'
|
64
|
+
task :revoke do |_t, args|
|
65
|
+
args.extras.each { |id| Abid::State.revoke(id) }
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
namespace :db do
|
70
|
+
desc 'Run migrations'
|
71
|
+
task :migrate, [:version] do |_t, args|
|
72
|
+
migrations_path = File.expand_path('../../../../migrations', __FILE__)
|
73
|
+
|
74
|
+
require 'sequel'
|
75
|
+
Sequel.extension :migration
|
76
|
+
db = Rake.application.database
|
77
|
+
if args[:version]
|
78
|
+
puts "Migrating to version #{args[:version]}"
|
79
|
+
Sequel::Migrator.run(db, migrations_path, target: args[:version].to_i)
|
80
|
+
else
|
81
|
+
puts 'Migrating to latest'
|
82
|
+
Sequel::Migrator.run(db, migrations_path)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
data/lib/abid/version.rb
CHANGED
data/lib/abid/worker.rb
CHANGED
@@ -3,37 +3,47 @@ module Abid
|
|
3
3
|
def initialize(application)
|
4
4
|
@application = application
|
5
5
|
@pools = {}
|
6
|
+
@pool_definitions = {}
|
6
7
|
|
7
|
-
|
8
|
-
Rake.suggested_thread_count - 1
|
9
|
-
@pools[:default] = Concurrent::FixedThreadPool.new(
|
10
|
-
default_thread_num,
|
11
|
-
idletime: FIXNUM_MAX
|
12
|
-
)
|
13
|
-
|
8
|
+
@pool_definitions[:waiter] = nil
|
14
9
|
@pools[:waiter] = Concurrent::SimpleExecutorService.new
|
10
|
+
|
11
|
+
if application.options.always_multitask
|
12
|
+
default_thread_num = @application.options.thread_pool_size || \
|
13
|
+
Rake.suggested_thread_count - 1
|
14
|
+
else
|
15
|
+
default_thread_num = 1
|
16
|
+
end
|
17
|
+
define(:default, default_thread_num)
|
15
18
|
end
|
16
19
|
|
17
20
|
def define(name, thread_count)
|
18
21
|
name = name.to_sym
|
19
|
-
fail "worker #{name} already defined" if @
|
20
|
-
@
|
21
|
-
thread_count,
|
22
|
-
idletime: FIXNUM_MAX
|
23
|
-
)
|
22
|
+
fail "worker #{name} already defined" if @pool_definitions.include?(name)
|
23
|
+
@pool_definitions[name] = thread_count
|
24
24
|
end
|
25
25
|
|
26
26
|
def [](name)
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
end
|
27
|
+
unless @pool_definitions.include?(name)
|
28
|
+
fail "worker #{name} is not defined"
|
29
|
+
end
|
31
30
|
|
31
|
+
@pools[name] ||= Concurrent::FixedThreadPool.new(
|
32
|
+
@pool_definitions[name],
|
33
|
+
idletime: FIXNUM_MAX
|
34
|
+
)
|
35
|
+
end
|
32
36
|
def shutdown
|
33
37
|
@pools.each do |_, pool|
|
34
38
|
pool.shutdown
|
35
39
|
pool.wait_for_termination
|
36
40
|
end
|
37
41
|
end
|
42
|
+
|
43
|
+
def kill
|
44
|
+
@pools.each do |_, pool|
|
45
|
+
pool.kill
|
46
|
+
end
|
47
|
+
end
|
38
48
|
end
|
39
49
|
end
|