abid 0.3.0.pre.alpha.2 → 0.3.0.pre.alpha.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/exe/abid +1 -1
- data/lib/abid.rb +21 -6
- data/lib/abid/application.rb +17 -40
- data/lib/abid/cli.rb +8 -7
- data/lib/abid/cli/assume.rb +4 -3
- data/lib/abid/cli/list.rb +5 -4
- data/lib/abid/cli/migrate.rb +4 -5
- data/lib/abid/cli/revoke.rb +5 -4
- data/lib/abid/config.rb +3 -3
- data/lib/abid/dsl_definition.rb +3 -3
- data/lib/abid/engine.rb +13 -0
- data/lib/abid/engine/executor.rb +126 -0
- data/lib/abid/engine/process.rb +137 -0
- data/lib/abid/engine/process_manager.rb +56 -0
- data/lib/abid/engine/scheduler.rb +96 -0
- data/lib/abid/engine/waiter.rb +83 -0
- data/lib/abid/engine/worker_manager.rb +123 -0
- data/lib/abid/environment.rb +48 -0
- data/lib/abid/job.rb +49 -7
- data/lib/abid/job_manager.rb +22 -0
- data/lib/abid/mixin_task.rb +2 -2
- data/lib/abid/rake_extensions.rb +12 -4
- data/lib/abid/rake_extensions/task.rb +4 -117
- data/lib/abid/state_manager/database.rb +21 -17
- data/lib/abid/state_manager/state.rb +65 -15
- data/lib/abid/state_manager/state_proxy.rb +65 -0
- data/lib/abid/task.rb +1 -15
- data/lib/abid/version.rb +1 -1
- metadata +12 -8
- data/lib/abid/abid_module.rb +0 -19
- data/lib/abid/session.rb +0 -92
- data/lib/abid/state.rb +0 -193
- data/lib/abid/state_manager.rb +0 -17
- data/lib/abid/waiter.rb +0 -110
- data/lib/abid/worker.rb +0 -56
data/lib/abid/state_manager.rb
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
require 'abid/state_manager/database'
|
2
|
-
|
3
|
-
module Abid
|
4
|
-
# StateManager manages jobs execution status and history.
|
5
|
-
#
|
6
|
-
# It ensures that same task is not executed simultaneously.
|
7
|
-
# Further more, it remembers all jobs history and prevents successed jobs to
|
8
|
-
# be executed again.
|
9
|
-
module StateManager
|
10
|
-
# @return [Sequel::Database] database object
|
11
|
-
def self.database
|
12
|
-
@database ||= Database.connect
|
13
|
-
end
|
14
|
-
|
15
|
-
autoload :State, 'abid/state_manager/state'
|
16
|
-
end
|
17
|
-
end
|
data/lib/abid/waiter.rb
DELETED
@@ -1,110 +0,0 @@
|
|
1
|
-
module Abid
|
2
|
-
# non-block waiter
|
3
|
-
class Waiter
|
4
|
-
Entry = Struct.new(:ivar, :start_time, :next_time,
|
5
|
-
:interval, :timeout, :block)
|
6
|
-
|
7
|
-
def initialize
|
8
|
-
@cv = ConditionVariable.new
|
9
|
-
@mutex = Mutex.new
|
10
|
-
@queue = MultiRBTree.new
|
11
|
-
@thread = nil
|
12
|
-
@error = nil
|
13
|
-
end
|
14
|
-
|
15
|
-
def wait(interval: 5, timeout: 60, &block)
|
16
|
-
run_thread
|
17
|
-
|
18
|
-
ivar = Concurrent::IVar.new
|
19
|
-
now = Time.now.to_f
|
20
|
-
next_time = now + interval
|
21
|
-
push(Entry.new(ivar, now, next_time, interval, timeout, block))
|
22
|
-
ivar
|
23
|
-
end
|
24
|
-
|
25
|
-
def shutdown(error = nil)
|
26
|
-
error ||= RuntimeError.new('waiter is shutting down')
|
27
|
-
@mutex.synchronize do
|
28
|
-
@error = error
|
29
|
-
@queue.each { |_, e| e.ivar.fail(error) }
|
30
|
-
@queue.clear
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def empty?
|
35
|
-
@queue.empty?
|
36
|
-
end
|
37
|
-
|
38
|
-
def alive?
|
39
|
-
@thread.alive?
|
40
|
-
end
|
41
|
-
|
42
|
-
private
|
43
|
-
|
44
|
-
def push(entry)
|
45
|
-
@mutex.synchronize do
|
46
|
-
fail @error if @error
|
47
|
-
|
48
|
-
@queue[entry.next_time] = entry
|
49
|
-
@cv.signal
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
def shift
|
54
|
-
_, e = @mutex.synchronize do
|
55
|
-
@cv.wait(@mutex) while @queue.empty?
|
56
|
-
@queue.shift
|
57
|
-
end
|
58
|
-
e
|
59
|
-
end
|
60
|
-
|
61
|
-
def sleep_until_next_time(entry)
|
62
|
-
sleep_time = entry.next_time - Time.now.to_f
|
63
|
-
return true if sleep_time <= 0
|
64
|
-
|
65
|
-
@mutex.synchronize { @cv.wait(@mutex, sleep_time) }
|
66
|
-
|
67
|
-
entry.next_time <= Time.now.to_f
|
68
|
-
end
|
69
|
-
|
70
|
-
def proc_entry(entry)
|
71
|
-
unless sleep_until_next_time(entry)
|
72
|
-
# failed to wait. retry.
|
73
|
-
push(entry)
|
74
|
-
return
|
75
|
-
end
|
76
|
-
|
77
|
-
return if entry.ivar.complete? # canceled
|
78
|
-
|
79
|
-
now = Time.now.to_f
|
80
|
-
elapsed = now - entry.start_time
|
81
|
-
ret = entry.block.call(elapsed)
|
82
|
-
if ret
|
83
|
-
entry.ivar.try_set(ret)
|
84
|
-
elsif entry.timeout > 0 && entry.timeout < elapsed
|
85
|
-
fail 'timeout exceeded'
|
86
|
-
else
|
87
|
-
entry.next_time = now + entry.interval
|
88
|
-
push(entry)
|
89
|
-
end
|
90
|
-
rescue Exception => err
|
91
|
-
begin
|
92
|
-
entry.ivar.fail(err)
|
93
|
-
rescue Concurrent::MultipleAssignmentError
|
94
|
-
nil
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
def run_thread
|
99
|
-
return if @thread
|
100
|
-
|
101
|
-
@thread = Thread.new do
|
102
|
-
begin
|
103
|
-
proc_entry(shift) while @error.nil?
|
104
|
-
ensure
|
105
|
-
shutdown($ERROR_INFO) if $ERROR_INFO
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
data/lib/abid/worker.rb
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
module Abid
|
2
|
-
class Worker
|
3
|
-
def initialize(application)
|
4
|
-
@application = application
|
5
|
-
@pools = {}
|
6
|
-
@pool_definitions = {}
|
7
|
-
|
8
|
-
if application.options.always_multitask
|
9
|
-
default_thread_num = @application.options.thread_pool_size || \
|
10
|
-
Rake.suggested_thread_count - 1
|
11
|
-
else
|
12
|
-
default_thread_num = 1
|
13
|
-
end
|
14
|
-
define(:default, default_thread_num)
|
15
|
-
|
16
|
-
define(:fresh, -1)
|
17
|
-
end
|
18
|
-
|
19
|
-
def define(name, thread_count)
|
20
|
-
name = name.to_sym
|
21
|
-
fail "worker #{name} already defined" if @pool_definitions.include?(name)
|
22
|
-
@pool_definitions[name] = thread_count
|
23
|
-
end
|
24
|
-
|
25
|
-
def [](name)
|
26
|
-
return @pools[name] if @pools.include?(name)
|
27
|
-
return self[:fresh] if name == :waiter # alias
|
28
|
-
|
29
|
-
unless @pool_definitions.include?(name)
|
30
|
-
fail "worker #{name} is not defined"
|
31
|
-
end
|
32
|
-
|
33
|
-
if @pool_definitions[name] > 0
|
34
|
-
@pools[name] = Concurrent::FixedThreadPool.new(
|
35
|
-
@pool_definitions[name],
|
36
|
-
idletime: FIXNUM_MAX
|
37
|
-
)
|
38
|
-
else
|
39
|
-
@pools[name] = Concurrent::SimpleExecutorService.new
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
def shutdown
|
44
|
-
@pools.each do |_, pool|
|
45
|
-
pool.shutdown
|
46
|
-
pool.wait_for_termination
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def kill
|
51
|
-
@pools.each do |_, pool|
|
52
|
-
pool.kill
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|