abid 0.3.0.pre.alpha.2 → 0.3.0.pre.alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/abid +1 -1
- data/lib/abid.rb +21 -6
- data/lib/abid/application.rb +17 -40
- data/lib/abid/cli.rb +8 -7
- data/lib/abid/cli/assume.rb +4 -3
- data/lib/abid/cli/list.rb +5 -4
- data/lib/abid/cli/migrate.rb +4 -5
- data/lib/abid/cli/revoke.rb +5 -4
- data/lib/abid/config.rb +3 -3
- data/lib/abid/dsl_definition.rb +3 -3
- data/lib/abid/engine.rb +13 -0
- data/lib/abid/engine/executor.rb +126 -0
- data/lib/abid/engine/process.rb +137 -0
- data/lib/abid/engine/process_manager.rb +56 -0
- data/lib/abid/engine/scheduler.rb +96 -0
- data/lib/abid/engine/waiter.rb +83 -0
- data/lib/abid/engine/worker_manager.rb +123 -0
- data/lib/abid/environment.rb +48 -0
- data/lib/abid/job.rb +49 -7
- data/lib/abid/job_manager.rb +22 -0
- data/lib/abid/mixin_task.rb +2 -2
- data/lib/abid/rake_extensions.rb +12 -4
- data/lib/abid/rake_extensions/task.rb +4 -117
- data/lib/abid/state_manager/database.rb +21 -17
- data/lib/abid/state_manager/state.rb +65 -15
- data/lib/abid/state_manager/state_proxy.rb +65 -0
- data/lib/abid/task.rb +1 -15
- data/lib/abid/version.rb +1 -1
- metadata +12 -8
- data/lib/abid/abid_module.rb +0 -19
- data/lib/abid/session.rb +0 -92
- data/lib/abid/state.rb +0 -193
- data/lib/abid/state_manager.rb +0 -17
- data/lib/abid/waiter.rb +0 -110
- data/lib/abid/worker.rb +0 -56
data/lib/abid/state_manager.rb
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
require 'abid/state_manager/database'
|
2
|
-
|
3
|
-
module Abid
|
4
|
-
# StateManager manages jobs execution status and history.
|
5
|
-
#
|
6
|
-
# It ensures that same task is not executed simultaneously.
|
7
|
-
# Further more, it remembers all jobs history and prevents successed jobs to
|
8
|
-
# be executed again.
|
9
|
-
module StateManager
|
10
|
-
# @return [Sequel::Database] database object
|
11
|
-
def self.database
|
12
|
-
@database ||= Database.connect
|
13
|
-
end
|
14
|
-
|
15
|
-
autoload :State, 'abid/state_manager/state'
|
16
|
-
end
|
17
|
-
end
|
data/lib/abid/waiter.rb
DELETED
@@ -1,110 +0,0 @@
|
|
1
|
-
module Abid
|
2
|
-
# non-block waiter
|
3
|
-
class Waiter
|
4
|
-
Entry = Struct.new(:ivar, :start_time, :next_time,
|
5
|
-
:interval, :timeout, :block)
|
6
|
-
|
7
|
-
def initialize
|
8
|
-
@cv = ConditionVariable.new
|
9
|
-
@mutex = Mutex.new
|
10
|
-
@queue = MultiRBTree.new
|
11
|
-
@thread = nil
|
12
|
-
@error = nil
|
13
|
-
end
|
14
|
-
|
15
|
-
def wait(interval: 5, timeout: 60, &block)
|
16
|
-
run_thread
|
17
|
-
|
18
|
-
ivar = Concurrent::IVar.new
|
19
|
-
now = Time.now.to_f
|
20
|
-
next_time = now + interval
|
21
|
-
push(Entry.new(ivar, now, next_time, interval, timeout, block))
|
22
|
-
ivar
|
23
|
-
end
|
24
|
-
|
25
|
-
def shutdown(error = nil)
|
26
|
-
error ||= RuntimeError.new('waiter is shutting down')
|
27
|
-
@mutex.synchronize do
|
28
|
-
@error = error
|
29
|
-
@queue.each { |_, e| e.ivar.fail(error) }
|
30
|
-
@queue.clear
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def empty?
|
35
|
-
@queue.empty?
|
36
|
-
end
|
37
|
-
|
38
|
-
def alive?
|
39
|
-
@thread.alive?
|
40
|
-
end
|
41
|
-
|
42
|
-
private
|
43
|
-
|
44
|
-
def push(entry)
|
45
|
-
@mutex.synchronize do
|
46
|
-
fail @error if @error
|
47
|
-
|
48
|
-
@queue[entry.next_time] = entry
|
49
|
-
@cv.signal
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
def shift
|
54
|
-
_, e = @mutex.synchronize do
|
55
|
-
@cv.wait(@mutex) while @queue.empty?
|
56
|
-
@queue.shift
|
57
|
-
end
|
58
|
-
e
|
59
|
-
end
|
60
|
-
|
61
|
-
def sleep_until_next_time(entry)
|
62
|
-
sleep_time = entry.next_time - Time.now.to_f
|
63
|
-
return true if sleep_time <= 0
|
64
|
-
|
65
|
-
@mutex.synchronize { @cv.wait(@mutex, sleep_time) }
|
66
|
-
|
67
|
-
entry.next_time <= Time.now.to_f
|
68
|
-
end
|
69
|
-
|
70
|
-
def proc_entry(entry)
|
71
|
-
unless sleep_until_next_time(entry)
|
72
|
-
# failed to wait. retry.
|
73
|
-
push(entry)
|
74
|
-
return
|
75
|
-
end
|
76
|
-
|
77
|
-
return if entry.ivar.complete? # canceled
|
78
|
-
|
79
|
-
now = Time.now.to_f
|
80
|
-
elapsed = now - entry.start_time
|
81
|
-
ret = entry.block.call(elapsed)
|
82
|
-
if ret
|
83
|
-
entry.ivar.try_set(ret)
|
84
|
-
elsif entry.timeout > 0 && entry.timeout < elapsed
|
85
|
-
fail 'timeout exceeded'
|
86
|
-
else
|
87
|
-
entry.next_time = now + entry.interval
|
88
|
-
push(entry)
|
89
|
-
end
|
90
|
-
rescue Exception => err
|
91
|
-
begin
|
92
|
-
entry.ivar.fail(err)
|
93
|
-
rescue Concurrent::MultipleAssignmentError
|
94
|
-
nil
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
def run_thread
|
99
|
-
return if @thread
|
100
|
-
|
101
|
-
@thread = Thread.new do
|
102
|
-
begin
|
103
|
-
proc_entry(shift) while @error.nil?
|
104
|
-
ensure
|
105
|
-
shutdown($ERROR_INFO) if $ERROR_INFO
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
data/lib/abid/worker.rb
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
module Abid
|
2
|
-
class Worker
|
3
|
-
def initialize(application)
|
4
|
-
@application = application
|
5
|
-
@pools = {}
|
6
|
-
@pool_definitions = {}
|
7
|
-
|
8
|
-
if application.options.always_multitask
|
9
|
-
default_thread_num = @application.options.thread_pool_size || \
|
10
|
-
Rake.suggested_thread_count - 1
|
11
|
-
else
|
12
|
-
default_thread_num = 1
|
13
|
-
end
|
14
|
-
define(:default, default_thread_num)
|
15
|
-
|
16
|
-
define(:fresh, -1)
|
17
|
-
end
|
18
|
-
|
19
|
-
def define(name, thread_count)
|
20
|
-
name = name.to_sym
|
21
|
-
fail "worker #{name} already defined" if @pool_definitions.include?(name)
|
22
|
-
@pool_definitions[name] = thread_count
|
23
|
-
end
|
24
|
-
|
25
|
-
def [](name)
|
26
|
-
return @pools[name] if @pools.include?(name)
|
27
|
-
return self[:fresh] if name == :waiter # alias
|
28
|
-
|
29
|
-
unless @pool_definitions.include?(name)
|
30
|
-
fail "worker #{name} is not defined"
|
31
|
-
end
|
32
|
-
|
33
|
-
if @pool_definitions[name] > 0
|
34
|
-
@pools[name] = Concurrent::FixedThreadPool.new(
|
35
|
-
@pool_definitions[name],
|
36
|
-
idletime: FIXNUM_MAX
|
37
|
-
)
|
38
|
-
else
|
39
|
-
@pools[name] = Concurrent::SimpleExecutorService.new
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
def shutdown
|
44
|
-
@pools.each do |_, pool|
|
45
|
-
pool.shutdown
|
46
|
-
pool.wait_for_termination
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def kill
|
51
|
-
@pools.each do |_, pool|
|
52
|
-
pool.kill
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|