isono 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +202 -0
- data/NOTICE +2 -0
- data/bin/cli +122 -0
- data/isono.gemspec +47 -0
- data/lib/ext/shellwords.rb +172 -0
- data/lib/isono.rb +61 -0
- data/lib/isono/amqp_client.rb +169 -0
- data/lib/isono/daemonize.rb +96 -0
- data/lib/isono/event_delegate_context.rb +56 -0
- data/lib/isono/event_observable.rb +86 -0
- data/lib/isono/logger.rb +48 -0
- data/lib/isono/manifest.rb +161 -0
- data/lib/isono/messaging_client.rb +116 -0
- data/lib/isono/models/event_log.rb +28 -0
- data/lib/isono/models/job_state.rb +35 -0
- data/lib/isono/models/node_state.rb +70 -0
- data/lib/isono/models/resource_instance.rb +35 -0
- data/lib/isono/node.rb +158 -0
- data/lib/isono/node_modules/base.rb +65 -0
- data/lib/isono/node_modules/data_store.rb +57 -0
- data/lib/isono/node_modules/event_channel.rb +72 -0
- data/lib/isono/node_modules/event_logger.rb +39 -0
- data/lib/isono/node_modules/job_channel.rb +86 -0
- data/lib/isono/node_modules/job_collector.rb +47 -0
- data/lib/isono/node_modules/job_worker.rb +152 -0
- data/lib/isono/node_modules/node_collector.rb +87 -0
- data/lib/isono/node_modules/node_heartbeat.rb +26 -0
- data/lib/isono/node_modules/rpc_channel.rb +482 -0
- data/lib/isono/rack.rb +67 -0
- data/lib/isono/rack/builder.rb +40 -0
- data/lib/isono/rack/data_store.rb +20 -0
- data/lib/isono/rack/job.rb +74 -0
- data/lib/isono/rack/map.rb +56 -0
- data/lib/isono/rack/object_method.rb +20 -0
- data/lib/isono/rack/proc.rb +50 -0
- data/lib/isono/rack/thread_pass.rb +22 -0
- data/lib/isono/resource_manifest.rb +273 -0
- data/lib/isono/runner/agent.rb +89 -0
- data/lib/isono/runner/rpc_server.rb +198 -0
- data/lib/isono/serializer.rb +43 -0
- data/lib/isono/thread_pool.rb +169 -0
- data/lib/isono/util.rb +212 -0
- metadata +185 -0
@@ -0,0 +1,89 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'amqp'
|
5
|
+
require 'digest/sha1'
|
6
|
+
|
7
|
+
require 'isono/agent'
|
8
|
+
|
9
|
+
module Isono
|
10
|
+
module Runner
|
11
|
+
class Agent
|
12
|
+
include Daemonize
|
13
|
+
|
14
|
+
def initialize(argv)
|
15
|
+
@argv = argv.dup
|
16
|
+
|
17
|
+
@options = {
|
18
|
+
:amqp_server_uri => URI.parse('amqp://guest:guest@localhost/'),
|
19
|
+
:log_file => nil,
|
20
|
+
:pid_file => nil,
|
21
|
+
:daemonize => true
|
22
|
+
}
|
23
|
+
|
24
|
+
parser.parse! @argv
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
def parser
|
29
|
+
@parser ||= OptionParser.new do |opts|
|
30
|
+
opts.banner = "Usage: agent [options]"
|
31
|
+
|
32
|
+
opts.separator ""
|
33
|
+
opts.separator "Agent options:"
|
34
|
+
opts.on( "-i", "--id ID", "Manually specify the Agent ID" ) {|str| @options[:agent_id] = str }
|
35
|
+
opts.on( "-p", "--pid PIDFILE", "pid file path" ) {|str| @options[:pid_file] = str }
|
36
|
+
opts.on( "-s", "--server AMQP_URI", "amqp broker server to connect" ) {|str|
|
37
|
+
begin
|
38
|
+
@options[:amqp_server_uri] = URI.parse(str)
|
39
|
+
rescue URI::InvalidURIError => e
|
40
|
+
abort "#{e}"
|
41
|
+
end
|
42
|
+
}
|
43
|
+
opts.on("-X", "Run in foreground" ) { @options[:daemonize] = false }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
def run(manifest_path=nil)
|
49
|
+
#%w(QUIT INT TERM).each { |i|
|
50
|
+
%w(EXIT).each { |i|
|
51
|
+
Signal.trap(i) { Isono::Agent.stop{ remove_pidfile if @options[:daemonize]} }
|
52
|
+
}
|
53
|
+
|
54
|
+
|
55
|
+
# load manifest file
|
56
|
+
manifest = Manifest.load_file(manifest_path.nil? ? @options[:manifest_path] : manifest_path)
|
57
|
+
|
58
|
+
if @options[:node_id]
|
59
|
+
# force overwrite node_id if the command line arg was given.
|
60
|
+
manifest.node_id(@options[:node_id])
|
61
|
+
elsif manifest.node_id.nil?
|
62
|
+
# nobody specified the node_id then set the ID in the
|
63
|
+
# default manner.
|
64
|
+
manifest.node_id(default_node_id)
|
65
|
+
end
|
66
|
+
|
67
|
+
@options[:log_file] ||= "/var/log/%s.log" % [manifest.node_name]
|
68
|
+
@options[:pid_file] ||= "/var/run/%s.pid" % [manifest.node_name]
|
69
|
+
|
70
|
+
if @options[:daemonize]
|
71
|
+
daemonize(@options[:log_file])
|
72
|
+
end
|
73
|
+
|
74
|
+
EventMachine.epoll
|
75
|
+
EventMachine.run {
|
76
|
+
Isono::Agent.start(manifest, @options)
|
77
|
+
}
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
def default_node_id
|
82
|
+
# use the ip address for the default routeas key value
|
83
|
+
Digest::SHA1.hexdigest(`/sbin/ip route get 8.8.8.8`.split("\n")[0].split.last)[0, 10]
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,198 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'amqp'
|
5
|
+
require 'digest/sha1'
|
6
|
+
|
7
|
+
require 'isono'
|
8
|
+
require 'isono/amqp_client'
|
9
|
+
|
10
|
+
module Isono
|
11
|
+
module Runner
|
12
|
+
# Run a new agent which provides RPC endpoint and delayed job.
|
13
|
+
#
|
14
|
+
# @example
|
15
|
+
#
|
16
|
+
# class RpcEndpoint1
|
17
|
+
# def func1
|
18
|
+
# end
|
19
|
+
# end
|
20
|
+
#
|
21
|
+
# class RpcEndpoint2
|
22
|
+
# def func2
|
23
|
+
# end
|
24
|
+
# end
|
25
|
+
#
|
26
|
+
# Isono::Runner::RpcServer.start do
|
27
|
+
# endpoint('xxxx1', RpcEndpoint1.new)
|
28
|
+
# endpoint('xxxx2', RpcEndpoint2.new)
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# mc = MessagingClient.new
|
32
|
+
# xxxx1 = mc.rpc('xxxx1')
|
33
|
+
# xxxx1.func1
|
34
|
+
#
|
35
|
+
# xxxx2 = mc.rpc('xxxx2')
|
36
|
+
# xxxx2.func2
|
37
|
+
module RpcServer
|
38
|
+
|
39
|
+
class EndpointBuilder
|
40
|
+
module BuildMethods
|
41
|
+
# @exmaple
|
42
|
+
# job 'command1', proc {
|
43
|
+
# # do somthing.
|
44
|
+
# }, proc {
|
45
|
+
# # do somthing on job failure.
|
46
|
+
# }
|
47
|
+
# @example
|
48
|
+
# job 'command1' do
|
49
|
+
# response.fail_cb {
|
50
|
+
# # do somthing on job failure.
|
51
|
+
# }
|
52
|
+
# sleep 10
|
53
|
+
# end
|
54
|
+
def job(command, run_cb=nil, fail_cb=nil, &blk)
|
55
|
+
app = if run_cb.is_a?(Proc)
|
56
|
+
proc {
|
57
|
+
request.fail_cb do
|
58
|
+
self.instance_eval(&fail_cb)
|
59
|
+
end
|
60
|
+
|
61
|
+
self.instance_eval(&run_cb)
|
62
|
+
}
|
63
|
+
elsif blk
|
64
|
+
blk
|
65
|
+
else
|
66
|
+
raise ArgumentError, "callbacks were not set propery"
|
67
|
+
end
|
68
|
+
add(:job, command, &app)
|
69
|
+
end
|
70
|
+
|
71
|
+
def rpc(command, &blk)
|
72
|
+
add(:rpc, command, &blk)
|
73
|
+
end
|
74
|
+
|
75
|
+
def build(endpoint, node)
|
76
|
+
helper_context = self.new(node)
|
77
|
+
|
78
|
+
app_builder = lambda { |builders|
|
79
|
+
unless builders.empty?
|
80
|
+
map_app = Rack::Map.new
|
81
|
+
builders.each { |b|
|
82
|
+
b.call(map_app, helper_context)
|
83
|
+
}
|
84
|
+
map_app
|
85
|
+
end
|
86
|
+
}
|
87
|
+
|
88
|
+
app = app_builder.call(@builders[:job])
|
89
|
+
if app
|
90
|
+
NodeModules::JobChannel.new(node).register_endpoint(endpoint, Rack.build do
|
91
|
+
run app
|
92
|
+
end)
|
93
|
+
end
|
94
|
+
|
95
|
+
app = app_builder.call(@builders[:rpc])
|
96
|
+
if app
|
97
|
+
NodeModules::RpcChannel.new(node).register_endpoint(endpoint, Rack.build do
|
98
|
+
use Rack::ThreadPass
|
99
|
+
run app
|
100
|
+
end
|
101
|
+
)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
protected
|
107
|
+
def add(type, command, &blk)
|
108
|
+
@builders[type] << lambda { |rack_map, ctx|
|
109
|
+
rack_map.map(command, Rack::Proc.new(ctx, &blk))
|
110
|
+
}
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.inherited(klass)
|
115
|
+
klass.class_eval {
|
116
|
+
@builders = {:job=>[], :rpc=>[]}
|
117
|
+
extend BuildMethods
|
118
|
+
}
|
119
|
+
end
|
120
|
+
|
121
|
+
def initialize(node)
|
122
|
+
@node = node
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
|
127
|
+
DEFAULT_MANIFEST = Manifest.new(Dir.pwd) do
|
128
|
+
load_module NodeModules::EventChannel
|
129
|
+
load_module NodeModules::RpcChannel
|
130
|
+
load_module NodeModules::JobWorker
|
131
|
+
load_module NodeModules::JobChannel
|
132
|
+
end
|
133
|
+
|
134
|
+
def start(manifest=nil, &blk)
|
135
|
+
rpcsvr = Server.new(ARGV)
|
136
|
+
rpcsvr.run(manifest, &blk)
|
137
|
+
end
|
138
|
+
module_function :start
|
139
|
+
|
140
|
+
class Server
|
141
|
+
def initialize(argv)
|
142
|
+
@argv = argv.dup
|
143
|
+
|
144
|
+
@options = {
|
145
|
+
:amqp_server_uri => URI.parse('amqp://guest:guest@localhost/'),
|
146
|
+
}
|
147
|
+
|
148
|
+
parser.parse! @argv
|
149
|
+
end
|
150
|
+
|
151
|
+
def parser
|
152
|
+
@parser ||= OptionParser.new do |opts|
|
153
|
+
opts.banner = "Usage: agent [options]"
|
154
|
+
|
155
|
+
opts.separator ""
|
156
|
+
opts.separator "Agent options:"
|
157
|
+
opts.on( "-i", "--id ID", "Manually specify the Node ID" ) {|str| @options[:node_id] = str }
|
158
|
+
opts.on( "-s", "--server AMQP_URI", "amqp broker server to connect" ) {|str|
|
159
|
+
begin
|
160
|
+
@options[:amqp_server_uri] = URI.parse(str)
|
161
|
+
rescue URI::InvalidURIError => e
|
162
|
+
abort "#{e}"
|
163
|
+
end
|
164
|
+
}
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def run(manifest=nil, &blk)
|
169
|
+
%w(EXIT).each { |i|
|
170
|
+
Signal.trap(i) { Isono::Node.stop }
|
171
|
+
}
|
172
|
+
|
173
|
+
# .to_s to avoid nil -> String conversion failure.
|
174
|
+
if @options[:node_id]
|
175
|
+
manifest.node_instance_id(@options[:node_id])
|
176
|
+
elsif manifest.node_instance_id.nil?
|
177
|
+
abort("[ERROR]: manifest.node_istance_id is not set")
|
178
|
+
end
|
179
|
+
|
180
|
+
EventMachine.epoll
|
181
|
+
EventMachine.run {
|
182
|
+
@node = Isono::Node.new(manifest)
|
183
|
+
@node.connect(@options[:amqp_server_uri], @options) do
|
184
|
+
self.instance_eval(&blk) if blk
|
185
|
+
end
|
186
|
+
}
|
187
|
+
end
|
188
|
+
|
189
|
+
def endpoint(endpoint, builder)
|
190
|
+
raise TypeError unless builder.respond_to?(:build)
|
191
|
+
builder.build(endpoint, @node)
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
|
2
|
+
module Isono
|
3
|
+
# AMQP message serializer
|
4
|
+
class Serializer
|
5
|
+
def self.instance
|
6
|
+
@serializer ||= RubySerializer.new
|
7
|
+
end
|
8
|
+
|
9
|
+
def marshal(buf)
|
10
|
+
raise NotImplementedError
|
11
|
+
end
|
12
|
+
|
13
|
+
def unmarshal(buf)
|
14
|
+
raise NotImplementedError
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
|
19
|
+
class YamlSerializer < Serializer
|
20
|
+
def initialize
|
21
|
+
require 'yaml'
|
22
|
+
end
|
23
|
+
|
24
|
+
def marshal(buf)
|
25
|
+
YAML.dump(buf)
|
26
|
+
end
|
27
|
+
|
28
|
+
def unmarshal(buf)
|
29
|
+
YAML.load(buf)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class RubySerializer < Serializer
|
34
|
+
|
35
|
+
def marshal(buf)
|
36
|
+
::Marshal.dump(buf)
|
37
|
+
end
|
38
|
+
|
39
|
+
def unmarshal(buf)
|
40
|
+
::Marshal.load(buf)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'thread'
|
3
|
+
|
4
|
+
module Isono
|
5
|
+
class ThreadPool
|
6
|
+
include Logger
|
7
|
+
|
8
|
+
class WorkerTerminateError < StandardError; end
|
9
|
+
class TimeoutError < StandardError; end
|
10
|
+
|
11
|
+
def initialize(worker_num=1, name=nil, opts={})
|
12
|
+
set_instance_logger(name)
|
13
|
+
@queue = ::Queue.new
|
14
|
+
@name = name
|
15
|
+
@opts = {:stucked_queue_num=>20}.merge(opts)
|
16
|
+
@last_stuck_warn_at = Time.now
|
17
|
+
|
18
|
+
@worker_threads = {}
|
19
|
+
worker_num.times {
|
20
|
+
t = Thread.new {
|
21
|
+
loop {
|
22
|
+
begin
|
23
|
+
while op = @queue.pop
|
24
|
+
if @queue.size > @opts[:stucked_queue_num] && Time.now - @last_stuck_warn_at > 5.0
|
25
|
+
logger.warn("too many stacked workers: #{@queue.size}")
|
26
|
+
@last_stuck_warn_at = Time.now
|
27
|
+
end
|
28
|
+
op.call
|
29
|
+
end
|
30
|
+
rescue WorkerTerminateError
|
31
|
+
# someone indicated to terminate this thread
|
32
|
+
# exit from the current loop
|
33
|
+
break
|
34
|
+
rescue Exception => e
|
35
|
+
self.logger.error(e)
|
36
|
+
end
|
37
|
+
}
|
38
|
+
@worker_threads.delete(Thread.current.__id__)
|
39
|
+
logger.info("#{Thread.current} is being terminated")
|
40
|
+
}
|
41
|
+
@worker_threads[t.__id__] = t
|
42
|
+
}
|
43
|
+
end
|
44
|
+
|
45
|
+
# Pass a block to a worker thread. The job is queued until the
|
46
|
+
# worker thread found.
|
47
|
+
# @param [Bool] immediage
|
48
|
+
# @param [Proc] blk A block to be proccessed on a worker thread.
|
49
|
+
def pass(immediate=true, &blk)
|
50
|
+
if immediate && member_thread?
|
51
|
+
return blk.call
|
52
|
+
end
|
53
|
+
|
54
|
+
@queue << blk
|
55
|
+
end
|
56
|
+
|
57
|
+
# Send a block to a worker thread similar with pass(). but this
|
58
|
+
# get the caller thread waited until the block proceeded in a
|
59
|
+
# worker thread.
|
60
|
+
# @param [Bool] immediage
|
61
|
+
# @param [Float] time_out
|
62
|
+
# @param [Proc] blk
|
63
|
+
def barrier(immediate=true, time_out=nil, &blk)
|
64
|
+
if immediate && member_thread?
|
65
|
+
return blk.call
|
66
|
+
end
|
67
|
+
|
68
|
+
q = ::Queue.new
|
69
|
+
time_start = ::Time.now
|
70
|
+
|
71
|
+
self.pass {
|
72
|
+
begin
|
73
|
+
q << blk.call
|
74
|
+
rescue Exception => e
|
75
|
+
q << e
|
76
|
+
end
|
77
|
+
}
|
78
|
+
|
79
|
+
em_sig = nil
|
80
|
+
if time_out
|
81
|
+
em_sig = EventMachine.add_timer(time_out) {
|
82
|
+
q << TimeoutError.new
|
83
|
+
}
|
84
|
+
end
|
85
|
+
|
86
|
+
res = q.shift
|
87
|
+
EventMachine.cancel_timer(em_sig)
|
88
|
+
time_elapsed = ::Time.now - time_start
|
89
|
+
logger.debug("Elapsed time for #{blk}: #{time_elapsed} secs") if time_elapsed > 0.05
|
90
|
+
if res.is_a?(Exception)
|
91
|
+
raise res
|
92
|
+
end
|
93
|
+
res
|
94
|
+
|
95
|
+
end
|
96
|
+
|
97
|
+
def clear
|
98
|
+
@queue.clear
|
99
|
+
end
|
100
|
+
|
101
|
+
# Immediatly shutdown all the worker threads
|
102
|
+
def shutdown()
|
103
|
+
@worker_threads.each {|t|
|
104
|
+
t.raise WorkerTerminateError
|
105
|
+
}
|
106
|
+
end
|
107
|
+
|
108
|
+
def member_thread?(thread=Thread.current)
|
109
|
+
@worker_threads.has_key?(thread.__id__)
|
110
|
+
end
|
111
|
+
|
112
|
+
def shutdown_graceful(timeout)
|
113
|
+
term_sig_q = ::Queue.new
|
114
|
+
worker_num = @worker_threads.size
|
115
|
+
# enqueue the terminate jobs.
|
116
|
+
worker_num.times {
|
117
|
+
@queue.push proc {
|
118
|
+
term_sig_q.enq(1)
|
119
|
+
raise WorkerTerminateError
|
120
|
+
}
|
121
|
+
}
|
122
|
+
|
123
|
+
em_sig = nil
|
124
|
+
if timeout > 0.0
|
125
|
+
em_sig = EventMachine.add_timer(timeout) {
|
126
|
+
worker_num.times {
|
127
|
+
term_sig_q << TimeoutError.new
|
128
|
+
}
|
129
|
+
}
|
130
|
+
end
|
131
|
+
|
132
|
+
timeout_workers = 0
|
133
|
+
while worker_num > 0
|
134
|
+
if term_sig_q.deq.is_a?(TimeoutError)
|
135
|
+
timeout_workers += 1
|
136
|
+
end
|
137
|
+
worker_num -= 1
|
138
|
+
end
|
139
|
+
|
140
|
+
logger.error("#{timeout_workers} of worker threads timed out during the cleanup") if timeout_workers > 0
|
141
|
+
ensure
|
142
|
+
shutdown
|
143
|
+
EventMachine.cancel_timer(em_sig)
|
144
|
+
end
|
145
|
+
|
146
|
+
def graceful_shutdown2
|
147
|
+
# make new jobs push to dummy queue.
|
148
|
+
old_queue = @queue
|
149
|
+
@queue = ::Queue.new
|
150
|
+
# wait until @queue becomes empty
|
151
|
+
if !old_queue.empty?
|
152
|
+
logger.info("Waiting for #{old_queue.size} worker jobs in #{self}")
|
153
|
+
while !old_queue.empty?
|
154
|
+
sleep 1
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
@worker_threads.each {|t|
|
159
|
+
t.raise WorkerTerminateError
|
160
|
+
}
|
161
|
+
end
|
162
|
+
|
163
|
+
|
164
|
+
private
|
165
|
+
def thread_loop
|
166
|
+
end
|
167
|
+
|
168
|
+
end
|
169
|
+
end
|