isono 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. data/LICENSE +202 -0
  2. data/NOTICE +2 -0
  3. data/bin/cli +122 -0
  4. data/isono.gemspec +47 -0
  5. data/lib/ext/shellwords.rb +172 -0
  6. data/lib/isono.rb +61 -0
  7. data/lib/isono/amqp_client.rb +169 -0
  8. data/lib/isono/daemonize.rb +96 -0
  9. data/lib/isono/event_delegate_context.rb +56 -0
  10. data/lib/isono/event_observable.rb +86 -0
  11. data/lib/isono/logger.rb +48 -0
  12. data/lib/isono/manifest.rb +161 -0
  13. data/lib/isono/messaging_client.rb +116 -0
  14. data/lib/isono/models/event_log.rb +28 -0
  15. data/lib/isono/models/job_state.rb +35 -0
  16. data/lib/isono/models/node_state.rb +70 -0
  17. data/lib/isono/models/resource_instance.rb +35 -0
  18. data/lib/isono/node.rb +158 -0
  19. data/lib/isono/node_modules/base.rb +65 -0
  20. data/lib/isono/node_modules/data_store.rb +57 -0
  21. data/lib/isono/node_modules/event_channel.rb +72 -0
  22. data/lib/isono/node_modules/event_logger.rb +39 -0
  23. data/lib/isono/node_modules/job_channel.rb +86 -0
  24. data/lib/isono/node_modules/job_collector.rb +47 -0
  25. data/lib/isono/node_modules/job_worker.rb +152 -0
  26. data/lib/isono/node_modules/node_collector.rb +87 -0
  27. data/lib/isono/node_modules/node_heartbeat.rb +26 -0
  28. data/lib/isono/node_modules/rpc_channel.rb +482 -0
  29. data/lib/isono/rack.rb +67 -0
  30. data/lib/isono/rack/builder.rb +40 -0
  31. data/lib/isono/rack/data_store.rb +20 -0
  32. data/lib/isono/rack/job.rb +74 -0
  33. data/lib/isono/rack/map.rb +56 -0
  34. data/lib/isono/rack/object_method.rb +20 -0
  35. data/lib/isono/rack/proc.rb +50 -0
  36. data/lib/isono/rack/thread_pass.rb +22 -0
  37. data/lib/isono/resource_manifest.rb +273 -0
  38. data/lib/isono/runner/agent.rb +89 -0
  39. data/lib/isono/runner/rpc_server.rb +198 -0
  40. data/lib/isono/serializer.rb +43 -0
  41. data/lib/isono/thread_pool.rb +169 -0
  42. data/lib/isono/util.rb +212 -0
  43. metadata +185 -0
@@ -0,0 +1,67 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class RpcError < RuntimeError; end
6
+ class UnknownMethodError < RpcError; end
7
+ class ResponseIncompleteError < RpcError; end
8
+
9
+ class << self
10
+ def build(&blk)
11
+ Builder.new(&blk)
12
+ end
13
+ end
14
+
15
+ class Decorator
16
+ attr_reader :app
17
+
18
+ def initialize(app)
19
+ raise TypeError unless app.respond_to?(:call)
20
+ @app = app
21
+
22
+ set_instance_logger(@app.class.to_s) if self.respond_to? :set_instance_logger
23
+ end
24
+
25
+ def call(req, res)
26
+ app.call(req, res)
27
+ end
28
+ end
29
+
30
+ class Request
31
+ attr_reader :r
32
+
33
+ def initialize(request_hash)
34
+ @r = request_hash
35
+ end
36
+
37
+ def command() @r[:command]; end
38
+ alias :key :command
39
+ def args() @r[:args]; end
40
+ def sender() @r[:sender]; end
41
+ def message_id() @r[:message_id]; end
42
+ end
43
+
44
+ class Response
45
+ attr_reader :ctx
46
+
47
+ # @param [NodeModules::RpcChannel::ResponseContext] ctx
48
+ def initialize(ctx)
49
+ raise TypeError unless ctx.is_a?(NodeModules::RpcChannel::ResponseContext)
50
+ @ctx = ctx
51
+ end
52
+
53
+ def responded?
54
+ @ctx.responded?
55
+ end
56
+
57
+ def progress(msg)
58
+ @ctx.progress(msg)
59
+ end
60
+
61
+ def response(msg)
62
+ @ctx.response(msg)
63
+ end
64
+
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,40 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class Builder
6
+ def initialize(&blk)
7
+ @filters = []
8
+ @app = Map.new
9
+ instance_eval(&blk) if blk
10
+ end
11
+
12
+ def use(decorator_class, *args)
13
+ raise TypeError unless decorator_class < Decorator
14
+ @filters << lambda {|disp| decorator_class.new(disp, *args) }
15
+ end
16
+
17
+ def run(app)
18
+ raise TypeError unless app.respond_to?(:call)
19
+ @app.map('', app)
20
+ end
21
+
22
+ def map(command, app=nil, &blk)
23
+ raise ArgumentError if app && blk
24
+ if app
25
+ raise TypeError unless app.respond_to?(:call)
26
+ @app.map(command, app)
27
+ elsif blk
28
+ @app.map(command, self.class.new(&blk))
29
+ else
30
+ raise ArgumentError
31
+ end
32
+ end
33
+
34
+ def call(req, res)
35
+ raise "main app is not set" if @app.nil?
36
+ @filters.reverse.inject(@app) {|d, f| f.call(d) }.call(req, res)
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,20 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ # Runs app.call() in the thread context of DataStore's worker.
6
+ class DataStore < Decorator
7
+ def call(req, res)
8
+ NodeModules::DataStore.pass {
9
+ begin
10
+ ret = @app.call(req, res)
11
+ res.response(ret) unless res.responded?
12
+ rescue ::Exception => e
13
+ res.response(e) unless res.responded?
14
+ raise e
15
+ end
16
+ }
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,74 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class Job < Decorator
6
+ include Logger
7
+
8
+ # Response class for nothing response.
9
+ # It is used when the job request type is :submit.
10
+ class NullResponse < Response
11
+ def progress(ret)
12
+ end
13
+
14
+ def response(ret)
15
+ end
16
+ end
17
+
18
+ class JobResponse < Response
19
+ # @param [NodeModules::RpcChannel::ResponseContext] ctx
20
+ # @param [NodeModules::JobWorker::JobContext] jobctx
21
+ def initialize(ctx, jobctx)
22
+ super(ctx)
23
+ @job = jobctx
24
+ end
25
+
26
+ # Register call back which called on the job failure.
27
+ def fail_cb(&blk)
28
+ @job.fail_cb = blk
29
+ end
30
+ end
31
+
32
+ class JobRequest < Request
33
+ # @param [Hash] request_hash
34
+ # @param [NodeModules::JobWorker::JobContext] jobctx
35
+ def initialize(request_hash, jobctx)
36
+ @job = jobctx
37
+ @r = request_hash
38
+ end
39
+ end
40
+
41
+ def initialize(app, job_worker)
42
+ super(app)
43
+ @job_worker = job_worker
44
+ end
45
+
46
+ def call(req, res)
47
+ orig_res = res
48
+ case req.r[:job_request_type]
49
+ when :submit
50
+ res = NullResponse.new(res.ctx)
51
+ end
52
+
53
+ job = @job_worker.run(req.r[:parent_job_id]){
54
+ begin
55
+ @app.call(JobRequest.new(req.r, job), JobResponse.new(res.ctx, job))
56
+ res.response(nil) unless res.responded?
57
+ rescue Exception => e
58
+ res.response(e) unless res.responded?
59
+ raise e
60
+ end
61
+ }
62
+
63
+ case req.r[:job_request_type]
64
+ when :submit
65
+ orig_res.response(job.to_hash)
66
+ else
67
+ # send job context info back at the first progress message.
68
+ # following progress messages to be handled as usual.
69
+ res.progress(job.to_hash)
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,56 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+
4
+ module Isono
5
+ module Rack
6
+ class Map
7
+ def self.build(&blk)
8
+ n = self.new
9
+ blk.call(n)
10
+ n
11
+ end
12
+
13
+ def initialize(&blk)
14
+ @table = {}
15
+ instance_eval(&blk) if blk
16
+ end
17
+
18
+ # @example
19
+ # map :xxxx do
20
+ # response.response('xxxxx')
21
+ # end
22
+ # @example
23
+ # map :xxxx, A.new do
24
+ # puts self # A.new
25
+ # end
26
+ # @example
27
+ # map :xxxx, App.new
28
+ def map(command, app=nil, &blk)
29
+ command = command.to_s
30
+
31
+ if app && blk
32
+ @table[command]=Rack::Proc.new(app, &blk)
33
+ elsif app && !blk
34
+ raise TypeError unless app.respond_to?(:call)
35
+ @table[command]=app
36
+ elsif !app && blk
37
+ @table[command]=Rack::Proc.new(&blk)
38
+ else
39
+ raise ArgumentError
40
+ end
41
+ self
42
+ end
43
+
44
+ def default(&blk)
45
+ map('', blk)
46
+ end
47
+
48
+ def call(req, res)
49
+ mapped_app = @table[req.command.to_s] || @table['']
50
+ raise UnknownMethodError if mapped_app.nil?
51
+
52
+ mapped_app.call(req, res)
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,20 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class ObjectMethod
6
+ include Logger
7
+
8
+ def initialize(obj)
9
+ @obj = obj
10
+ end
11
+
12
+ def call(req, res)
13
+ m = @obj.method(req.command)
14
+ raise UnknownMethodError, "#{req.command}" if m.nil?
15
+ res.response(m.arity.abs > 0 ? m.call(*req.args) : m.call)
16
+ end
17
+
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,50 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class Proc
6
+ include Logger
7
+
8
+ THREAD_LOCAL_KEY=self.to_s
9
+
10
+ attr_accessor :context
11
+
12
+ def initialize(context=nil, opts={}, &blk)
13
+ @context = context || Object.new
14
+ @blk = blk
15
+ end
16
+
17
+ def call(req, res)
18
+ Thread.current["#{THREAD_LOCAL_KEY}/request"] = req
19
+ Thread.current["#{THREAD_LOCAL_KEY}/response"] = res
20
+ begin
21
+ # create per-request context object from original.
22
+ c = @context.dup
23
+ c.extend InjectMethods
24
+ begin
25
+ c.instance_eval(&@blk)
26
+ # send empty message back to client if the response is not handled in block.
27
+ res.response(nil) unless res.responded?
28
+ rescue ::Exception => e
29
+ res.response(e) unless res.responded?
30
+ raise e
31
+ end
32
+ ensure
33
+ Thread.current["#{THREAD_LOCAL_KEY}/request"] = nil
34
+ Thread.current["#{THREAD_LOCAL_KEY}/response"] = nil
35
+ end
36
+ end
37
+
38
+ module InjectMethods
39
+ def request
40
+ Thread.current["#{THREAD_LOCAL_KEY}/request"]
41
+ end
42
+
43
+ def response
44
+ Thread.current["#{THREAD_LOCAL_KEY}/response"]
45
+ end
46
+ end
47
+
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,22 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class ThreadPass < Decorator
6
+ include Logger
7
+
8
+ def call(req, res)
9
+ ::Thread.new {
10
+ begin
11
+ app.call(req, res)
12
+ rescue Exception => e
13
+ logger.error(e)
14
+ res.response(e) unless res.responded?
15
+ else
16
+ raise ResponseIncompleteError unless res.responded?
17
+ end
18
+ }
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,273 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ require 'statemachine'
4
+ require 'pathname'
5
+ require 'yaml'
6
+
7
+ module Isono
8
+ class ResourceManifest
9
+ include Logger
10
+
11
+ # DSL to define a new resource manifest.
12
+ class Loader
13
+ include Logger
14
+
15
+ def initialize(m)
16
+ @manifest = m
17
+ end
18
+
19
+ def description(desc)
20
+ @manifest.description = desc
21
+ end
22
+ alias :desc :description
23
+
24
+ def statemachine(&blk)
25
+ @manifest.stm = Statemachine.build(&blk)
26
+ end
27
+
28
+ def load_path(path)
29
+ @manifest.append_load_path(path)
30
+ end
31
+
32
+ def name(name)
33
+ @manifest.name = name
34
+ end
35
+
36
+ def entry_state(state, &blk)
37
+ @manifest.entry_state[state] ||= StateItem.new
38
+ EntryState.new( @manifest.entry_state[state] ).instance_eval(&blk)
39
+ end
40
+
41
+ def exit_state(state, &blk)
42
+ @manifest.entry_state[state] ||= StateItem.new
43
+ ExitState.new( @manifest.entry_state[state] ).instance_eval(&blk)
44
+ end
45
+
46
+ def plugin(klass)
47
+ logger.debug("plugin: #{klass.to_s}")
48
+ if klass.const_defined?(:ClassMethods) && klass.const_get(:ClassMethods).is_a?(Module)
49
+ self.extend(klass.const_get(:ClassMethods))
50
+ end
51
+
52
+ #if klass.respond_to? :extend_task
53
+ if klass.const_defined?(:TaskMethods) && klass.const_get(:TaskMethods).is_a?(Module)
54
+ TaskBlock.class_eval {
55
+ include klass.const_get(:TaskMethods)
56
+ }
57
+ end
58
+ end
59
+
60
+ def manifest
61
+ @manifest
62
+ end
63
+
64
+ def config(&blk)
65
+ Manifest::ConfigStructBuilder.new(@manifest.config).instance_eval &blk
66
+ end
67
+
68
+ class EntryState
69
+ def initialize(stitem)
70
+ @state_item = stitem
71
+ end
72
+
73
+ def on_event(evname, sender, &blk)
74
+ @state_item.on_event[evname] = {
75
+ :evname => evname,
76
+ :sender => sender,
77
+ :task => TaskBlock.new(blk)
78
+ }
79
+ end
80
+
81
+ def on_command(cmd, &blk)
82
+ @state_item.on_command[cmd] = {:task=> TaskBlock.new(blk)}
83
+ end
84
+
85
+ def task(&blk)
86
+ @state_item.task = TaskBlock.new(blk)
87
+ end
88
+ end
89
+
90
+ class ExitState
91
+ def initialize(stitem)
92
+ @state_item = stitem
93
+ end
94
+
95
+ def on_event(evname, sender, &blk)
96
+ @state_item.on_event[evname] = {
97
+ :evname => evname,
98
+ :sender => sender,
99
+ :task => TaskBlock.new(blk)
100
+ }
101
+ end
102
+
103
+ def on_command(cmd, &blk)
104
+ @state_item.on_command[cmd] = {:task=> TaskBlock.new(blk)}
105
+ end
106
+
107
+ def task(&blk)
108
+ @state_item.task = TaskBlock.new(blk)
109
+ end
110
+ end
111
+
112
+ end
113
+
114
+ def self.load(path)
115
+ root_path = File.dirname(path)
116
+ manifest = new(root_path)
117
+
118
+ # instance_data has to be loaded before manifest file
119
+ # evaluation.
120
+ if File.file?(manifest.instance_data_path)
121
+ manifest.instance_data = YAML.load(File.read(manifest.instance_data_path)).freeze
122
+ end
123
+
124
+ logger.info("Loading resource.manifest: #{path}")
125
+ buf = File.read(path)
126
+ Loader.new(manifest).instance_eval(buf, path)
127
+ manifest
128
+ end
129
+
130
+ attr_reader :resource_root_path, :entry_state, :exit_state, :helpers, :load_path
131
+ attr_reader :config
132
+ attr_accessor :name, :description, :stm, :state_monitor, :instance_data
133
+
134
+ def initialize(root_path)
135
+ @resource_root_path = root_path
136
+ @entry_state = {}
137
+ @exit_state = {}
138
+ @helpers = {}
139
+ @load_path = []
140
+ @config = Manifest::ConfigStruct.new
141
+
142
+ append_load_path('lib')
143
+ end
144
+
145
+ def instance_data_path
146
+ File.expand_path('instance_data.yml', @resource_root_path)
147
+ end
148
+
149
+ def append_load_path(path)
150
+ real_path = if Pathname.new(path).absolute?
151
+ path
152
+ else
153
+ File.expand_path(path, @resource_root_path)
154
+ end
155
+ unless $LOAD_PATH.member? real_path
156
+ load_path << path
157
+ $LOAD_PATH.unshift real_path
158
+ end
159
+ end
160
+
161
+ class StateItem
162
+ attr_accessor :task
163
+ attr_reader :on_event, :on_command
164
+
165
+ def initialize()
166
+ @task = nil
167
+ @on_event = {}
168
+ @on_command = {}
169
+ end
170
+ end
171
+
172
+ class TaskBlock
173
+ include Logger
174
+
175
+ def initialize(blk)
176
+ @blk = blk
177
+ end
178
+
179
+ def call(resource_instance, args=[])
180
+ raise ArgumentError unless resource_instance.is_a?(ManagerModules::ResourceInstance)
181
+ @ri = resource_instance
182
+
183
+ instance_eval &@blk
184
+ end
185
+
186
+ private
187
+ def state_monitor
188
+ manifest.state_monitor
189
+ end
190
+
191
+ def next_event(ev, *args)
192
+ manifest.stm.process_event(ev, *args)
193
+ end
194
+
195
+ def manifest
196
+ @ri.manifest
197
+ end
198
+ end
199
+
200
+ module RakeHelper
201
+ module ClassMethods
202
+ def default_rakefile(rakefile)
203
+ rakefile =
204
+ if Pathname.new(rakefile).absolute?
205
+ rakefile.dup
206
+ else
207
+ File.expand_path(rakefile, @manifest.resource_root_path)
208
+ end
209
+ raise "File does not exist: #{rakefile}" unless File.exist?(rakefile)
210
+ @manifest.helpers[:default_rakefile] = rakefile
211
+ end
212
+
213
+ def rake_bin_path(path)
214
+ @manifest.helpers[:rake_bin_path] = path
215
+ end
216
+ end
217
+
218
+ module TaskMethods
219
+ def rake(task, rakefile=nil, &blk)
220
+ rake_path = manifest.helpers[:rake_bin_path] || Gem.bin_path('rake', 'rake')
221
+ rakefile = if rakefile
222
+ rakefile
223
+ elsif manifest.helpers[:default_rakefile]
224
+ manifest.helpers[:default_rakefile]
225
+ else
226
+ raise "Rakefile is not specified."
227
+ end
228
+
229
+ cmd = Util.quote_args("%s -I%s -f %s --rakelib %s RESOURCE_MANIFEST=%s %s",
230
+ [rake_path,
231
+ File.join(Isono.home, 'lib'),
232
+ rakefile,
233
+ #File.join(Isono.home, 'tasks/load_resource_manifest.rake'),
234
+ File.join(Isono.home, 'tasks'),
235
+ File.expand_path('resource.manifest', manifest.resource_root_path),
236
+ task
237
+ ])
238
+ logger.debug(cmd)
239
+ system(cmd)
240
+ end
241
+ end
242
+ end
243
+
244
+ module MonitorHelper
245
+ module ClassMethods
246
+
247
+ def state_monitor(monitor_class, &blk)
248
+ @manifest.config.state_monitor = self.monitor(monitor_class, &blk)
249
+ end
250
+
251
+ def monitor(monitor_class, &blk)
252
+ raise ArgumentError unless monitor_class.is_a?(Class) && monitor_class < Isono::Monitors::Base
253
+ @manifest.config.monitors ||= {}
254
+
255
+ raise "duplicate registration: #{monitor_class}" if @manifest.config.monitors.has_key?(monitor_class)
256
+
257
+ m = monitor_class.new()
258
+ m.instance_eval &blk if blk
259
+ @manifest.config.monitors[monitor_class] = m
260
+ end
261
+
262
+ end
263
+
264
+ module TaskMethods
265
+ def monitor(monitor_class)
266
+ manifest.config.monitors[monitor_class] || raise("unknown monitor class: #{monitor_class.to_s}")
267
+ end
268
+ end
269
+
270
+ end
271
+
272
+ end
273
+ end