isono 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. data/LICENSE +202 -0
  2. data/NOTICE +2 -0
  3. data/bin/cli +122 -0
  4. data/isono.gemspec +47 -0
  5. data/lib/ext/shellwords.rb +172 -0
  6. data/lib/isono.rb +61 -0
  7. data/lib/isono/amqp_client.rb +169 -0
  8. data/lib/isono/daemonize.rb +96 -0
  9. data/lib/isono/event_delegate_context.rb +56 -0
  10. data/lib/isono/event_observable.rb +86 -0
  11. data/lib/isono/logger.rb +48 -0
  12. data/lib/isono/manifest.rb +161 -0
  13. data/lib/isono/messaging_client.rb +116 -0
  14. data/lib/isono/models/event_log.rb +28 -0
  15. data/lib/isono/models/job_state.rb +35 -0
  16. data/lib/isono/models/node_state.rb +70 -0
  17. data/lib/isono/models/resource_instance.rb +35 -0
  18. data/lib/isono/node.rb +158 -0
  19. data/lib/isono/node_modules/base.rb +65 -0
  20. data/lib/isono/node_modules/data_store.rb +57 -0
  21. data/lib/isono/node_modules/event_channel.rb +72 -0
  22. data/lib/isono/node_modules/event_logger.rb +39 -0
  23. data/lib/isono/node_modules/job_channel.rb +86 -0
  24. data/lib/isono/node_modules/job_collector.rb +47 -0
  25. data/lib/isono/node_modules/job_worker.rb +152 -0
  26. data/lib/isono/node_modules/node_collector.rb +87 -0
  27. data/lib/isono/node_modules/node_heartbeat.rb +26 -0
  28. data/lib/isono/node_modules/rpc_channel.rb +482 -0
  29. data/lib/isono/rack.rb +67 -0
  30. data/lib/isono/rack/builder.rb +40 -0
  31. data/lib/isono/rack/data_store.rb +20 -0
  32. data/lib/isono/rack/job.rb +74 -0
  33. data/lib/isono/rack/map.rb +56 -0
  34. data/lib/isono/rack/object_method.rb +20 -0
  35. data/lib/isono/rack/proc.rb +50 -0
  36. data/lib/isono/rack/thread_pass.rb +22 -0
  37. data/lib/isono/resource_manifest.rb +273 -0
  38. data/lib/isono/runner/agent.rb +89 -0
  39. data/lib/isono/runner/rpc_server.rb +198 -0
  40. data/lib/isono/serializer.rb +43 -0
  41. data/lib/isono/thread_pool.rb +169 -0
  42. data/lib/isono/util.rb +212 -0
  43. metadata +185 -0
@@ -0,0 +1,67 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class RpcError < RuntimeError; end
6
+ class UnknownMethodError < RpcError; end
7
+ class ResponseIncompleteError < RpcError; end
8
+
9
+ class << self
10
+ def build(&blk)
11
+ Builder.new(&blk)
12
+ end
13
+ end
14
+
15
+ class Decorator
16
+ attr_reader :app
17
+
18
+ def initialize(app)
19
+ raise TypeError unless app.respond_to?(:call)
20
+ @app = app
21
+
22
+ set_instance_logger(@app.class.to_s) if self.respond_to? :set_instance_logger
23
+ end
24
+
25
+ def call(req, res)
26
+ app.call(req, res)
27
+ end
28
+ end
29
+
30
+ class Request
31
+ attr_reader :r
32
+
33
+ def initialize(request_hash)
34
+ @r = request_hash
35
+ end
36
+
37
+ def command() @r[:command]; end
38
+ alias :key :command
39
+ def args() @r[:args]; end
40
+ def sender() @r[:sender]; end
41
+ def message_id() @r[:message_id]; end
42
+ end
43
+
44
+ class Response
45
+ attr_reader :ctx
46
+
47
+ # @param [NodeModules::RpcChannel::ResponseContext] ctx
48
+ def initialize(ctx)
49
+ raise TypeError unless ctx.is_a?(NodeModules::RpcChannel::ResponseContext)
50
+ @ctx = ctx
51
+ end
52
+
53
+ def responded?
54
+ @ctx.responded?
55
+ end
56
+
57
+ def progress(msg)
58
+ @ctx.progress(msg)
59
+ end
60
+
61
+ def response(msg)
62
+ @ctx.response(msg)
63
+ end
64
+
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,40 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class Builder
6
+ def initialize(&blk)
7
+ @filters = []
8
+ @app = Map.new
9
+ instance_eval(&blk) if blk
10
+ end
11
+
12
+ def use(decorator_class, *args)
13
+ raise TypeError unless decorator_class < Decorator
14
+ @filters << lambda {|disp| decorator_class.new(disp, *args) }
15
+ end
16
+
17
+ def run(app)
18
+ raise TypeError unless app.respond_to?(:call)
19
+ @app.map('', app)
20
+ end
21
+
22
+ def map(command, app=nil, &blk)
23
+ raise ArgumentError if app && blk
24
+ if app
25
+ raise TypeError unless app.respond_to?(:call)
26
+ @app.map(command, app)
27
+ elsif blk
28
+ @app.map(command, self.class.new(&blk))
29
+ else
30
+ raise ArgumentError
31
+ end
32
+ end
33
+
34
+ def call(req, res)
35
+ raise "main app is not set" if @app.nil?
36
+ @filters.reverse.inject(@app) {|d, f| f.call(d) }.call(req, res)
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,20 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ # Runs app.call() in the thread context of DataStore's worker.
6
+ class DataStore < Decorator
7
+ def call(req, res)
8
+ NodeModules::DataStore.pass {
9
+ begin
10
+ ret = @app.call(req, res)
11
+ res.response(ret) unless res.responded?
12
+ rescue ::Exception => e
13
+ res.response(e) unless res.responded?
14
+ raise e
15
+ end
16
+ }
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,74 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class Job < Decorator
6
+ include Logger
7
+
8
+ # Response class for nothing response.
9
+ # It is used when the job request type is :submit.
10
+ class NullResponse < Response
11
+ def progress(ret)
12
+ end
13
+
14
+ def response(ret)
15
+ end
16
+ end
17
+
18
+ class JobResponse < Response
19
+ # @param [NodeModules::RpcChannel::ResponseContext] ctx
20
+ # @param [NodeModules::JobWorker::JobContext] jobctx
21
+ def initialize(ctx, jobctx)
22
+ super(ctx)
23
+ @job = jobctx
24
+ end
25
+
26
+ # Register call back which called on the job failure.
27
+ def fail_cb(&blk)
28
+ @job.fail_cb = blk
29
+ end
30
+ end
31
+
32
+ class JobRequest < Request
33
+ # @param [Hash] request_hash
34
+ # @param [NodeModules::JobWorker::JobContext] jobctx
35
+ def initialize(request_hash, jobctx)
36
+ @job = jobctx
37
+ @r = request_hash
38
+ end
39
+ end
40
+
41
+ def initialize(app, job_worker)
42
+ super(app)
43
+ @job_worker = job_worker
44
+ end
45
+
46
+ def call(req, res)
47
+ orig_res = res
48
+ case req.r[:job_request_type]
49
+ when :submit
50
+ res = NullResponse.new(res.ctx)
51
+ end
52
+
53
+ job = @job_worker.run(req.r[:parent_job_id]){
54
+ begin
55
+ @app.call(JobRequest.new(req.r, job), JobResponse.new(res.ctx, job))
56
+ res.response(nil) unless res.responded?
57
+ rescue Exception => e
58
+ res.response(e) unless res.responded?
59
+ raise e
60
+ end
61
+ }
62
+
63
+ case req.r[:job_request_type]
64
+ when :submit
65
+ orig_res.response(job.to_hash)
66
+ else
67
+ # send job context info back at the first progress message.
68
+ # following progress messages to be handled as usual.
69
+ res.progress(job.to_hash)
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,56 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+
4
+ module Isono
5
+ module Rack
6
+ class Map
7
+ def self.build(&blk)
8
+ n = self.new
9
+ blk.call(n)
10
+ n
11
+ end
12
+
13
+ def initialize(&blk)
14
+ @table = {}
15
+ instance_eval(&blk) if blk
16
+ end
17
+
18
+ # @example
19
+ # map :xxxx do
20
+ # response.response('xxxxx')
21
+ # end
22
+ # @example
23
+ # map :xxxx, A.new do
24
+ # puts self # A.new
25
+ # end
26
+ # @example
27
+ # map :xxxx, App.new
28
+ def map(command, app=nil, &blk)
29
+ command = command.to_s
30
+
31
+ if app && blk
32
+ @table[command]=Rack::Proc.new(app, &blk)
33
+ elsif app && !blk
34
+ raise TypeError unless app.respond_to?(:call)
35
+ @table[command]=app
36
+ elsif !app && blk
37
+ @table[command]=Rack::Proc.new(&blk)
38
+ else
39
+ raise ArgumentError
40
+ end
41
+ self
42
+ end
43
+
44
+ def default(&blk)
45
+ map('', blk)
46
+ end
47
+
48
+ def call(req, res)
49
+ mapped_app = @table[req.command.to_s] || @table['']
50
+ raise UnknownMethodError if mapped_app.nil?
51
+
52
+ mapped_app.call(req, res)
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,20 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class ObjectMethod
6
+ include Logger
7
+
8
+ def initialize(obj)
9
+ @obj = obj
10
+ end
11
+
12
+ def call(req, res)
13
+ m = @obj.method(req.command)
14
+ raise UnknownMethodError, "#{req.command}" if m.nil?
15
+ res.response(m.arity.abs > 0 ? m.call(*req.args) : m.call)
16
+ end
17
+
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,50 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class Proc
6
+ include Logger
7
+
8
+ THREAD_LOCAL_KEY=self.to_s
9
+
10
+ attr_accessor :context
11
+
12
+ def initialize(context=nil, opts={}, &blk)
13
+ @context = context || Object.new
14
+ @blk = blk
15
+ end
16
+
17
+ def call(req, res)
18
+ Thread.current["#{THREAD_LOCAL_KEY}/request"] = req
19
+ Thread.current["#{THREAD_LOCAL_KEY}/response"] = res
20
+ begin
21
+ # create per-request context object from original.
22
+ c = @context.dup
23
+ c.extend InjectMethods
24
+ begin
25
+ c.instance_eval(&@blk)
26
+ # send empty message back to client if the response is not handled in block.
27
+ res.response(nil) unless res.responded?
28
+ rescue ::Exception => e
29
+ res.response(e) unless res.responded?
30
+ raise e
31
+ end
32
+ ensure
33
+ Thread.current["#{THREAD_LOCAL_KEY}/request"] = nil
34
+ Thread.current["#{THREAD_LOCAL_KEY}/response"] = nil
35
+ end
36
+ end
37
+
38
+ module InjectMethods
39
+ def request
40
+ Thread.current["#{THREAD_LOCAL_KEY}/request"]
41
+ end
42
+
43
+ def response
44
+ Thread.current["#{THREAD_LOCAL_KEY}/response"]
45
+ end
46
+ end
47
+
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,22 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Isono
4
+ module Rack
5
+ class ThreadPass < Decorator
6
+ include Logger
7
+
8
+ def call(req, res)
9
+ ::Thread.new {
10
+ begin
11
+ app.call(req, res)
12
+ rescue Exception => e
13
+ logger.error(e)
14
+ res.response(e) unless res.responded?
15
+ else
16
+ raise ResponseIncompleteError unless res.responded?
17
+ end
18
+ }
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,273 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ require 'statemachine'
4
+ require 'pathname'
5
+ require 'yaml'
6
+
7
+ module Isono
8
+ class ResourceManifest
9
+ include Logger
10
+
11
+ # DSL to define a new resource manifest.
12
+ class Loader
13
+ include Logger
14
+
15
+ def initialize(m)
16
+ @manifest = m
17
+ end
18
+
19
+ def description(desc)
20
+ @manifest.description = desc
21
+ end
22
+ alias :desc :description
23
+
24
+ def statemachine(&blk)
25
+ @manifest.stm = Statemachine.build(&blk)
26
+ end
27
+
28
+ def load_path(path)
29
+ @manifest.append_load_path(path)
30
+ end
31
+
32
+ def name(name)
33
+ @manifest.name = name
34
+ end
35
+
36
+ def entry_state(state, &blk)
37
+ @manifest.entry_state[state] ||= StateItem.new
38
+ EntryState.new( @manifest.entry_state[state] ).instance_eval(&blk)
39
+ end
40
+
41
+ def exit_state(state, &blk)
42
+ @manifest.entry_state[state] ||= StateItem.new
43
+ ExitState.new( @manifest.entry_state[state] ).instance_eval(&blk)
44
+ end
45
+
46
+ def plugin(klass)
47
+ logger.debug("plugin: #{klass.to_s}")
48
+ if klass.const_defined?(:ClassMethods) && klass.const_get(:ClassMethods).is_a?(Module)
49
+ self.extend(klass.const_get(:ClassMethods))
50
+ end
51
+
52
+ #if klass.respond_to? :extend_task
53
+ if klass.const_defined?(:TaskMethods) && klass.const_get(:TaskMethods).is_a?(Module)
54
+ TaskBlock.class_eval {
55
+ include klass.const_get(:TaskMethods)
56
+ }
57
+ end
58
+ end
59
+
60
+ def manifest
61
+ @manifest
62
+ end
63
+
64
+ def config(&blk)
65
+ Manifest::ConfigStructBuilder.new(@manifest.config).instance_eval &blk
66
+ end
67
+
68
+ class EntryState
69
+ def initialize(stitem)
70
+ @state_item = stitem
71
+ end
72
+
73
+ def on_event(evname, sender, &blk)
74
+ @state_item.on_event[evname] = {
75
+ :evname => evname,
76
+ :sender => sender,
77
+ :task => TaskBlock.new(blk)
78
+ }
79
+ end
80
+
81
+ def on_command(cmd, &blk)
82
+ @state_item.on_command[cmd] = {:task=> TaskBlock.new(blk)}
83
+ end
84
+
85
+ def task(&blk)
86
+ @state_item.task = TaskBlock.new(blk)
87
+ end
88
+ end
89
+
90
+ class ExitState
91
+ def initialize(stitem)
92
+ @state_item = stitem
93
+ end
94
+
95
+ def on_event(evname, sender, &blk)
96
+ @state_item.on_event[evname] = {
97
+ :evname => evname,
98
+ :sender => sender,
99
+ :task => TaskBlock.new(blk)
100
+ }
101
+ end
102
+
103
+ def on_command(cmd, &blk)
104
+ @state_item.on_command[cmd] = {:task=> TaskBlock.new(blk)}
105
+ end
106
+
107
+ def task(&blk)
108
+ @state_item.task = TaskBlock.new(blk)
109
+ end
110
+ end
111
+
112
+ end
113
+
114
+ def self.load(path)
115
+ root_path = File.dirname(path)
116
+ manifest = new(root_path)
117
+
118
+ # instance_data has to be loaded before manifest file
119
+ # evaluation.
120
+ if File.file?(manifest.instance_data_path)
121
+ manifest.instance_data = YAML.load(File.read(manifest.instance_data_path)).freeze
122
+ end
123
+
124
+ logger.info("Loading resource.manifest: #{path}")
125
+ buf = File.read(path)
126
+ Loader.new(manifest).instance_eval(buf, path)
127
+ manifest
128
+ end
129
+
130
+ attr_reader :resource_root_path, :entry_state, :exit_state, :helpers, :load_path
131
+ attr_reader :config
132
+ attr_accessor :name, :description, :stm, :state_monitor, :instance_data
133
+
134
+ def initialize(root_path)
135
+ @resource_root_path = root_path
136
+ @entry_state = {}
137
+ @exit_state = {}
138
+ @helpers = {}
139
+ @load_path = []
140
+ @config = Manifest::ConfigStruct.new
141
+
142
+ append_load_path('lib')
143
+ end
144
+
145
+ def instance_data_path
146
+ File.expand_path('instance_data.yml', @resource_root_path)
147
+ end
148
+
149
+ def append_load_path(path)
150
+ real_path = if Pathname.new(path).absolute?
151
+ path
152
+ else
153
+ File.expand_path(path, @resource_root_path)
154
+ end
155
+ unless $LOAD_PATH.member? real_path
156
+ load_path << path
157
+ $LOAD_PATH.unshift real_path
158
+ end
159
+ end
160
+
161
+ class StateItem
162
+ attr_accessor :task
163
+ attr_reader :on_event, :on_command
164
+
165
+ def initialize()
166
+ @task = nil
167
+ @on_event = {}
168
+ @on_command = {}
169
+ end
170
+ end
171
+
172
+ class TaskBlock
173
+ include Logger
174
+
175
+ def initialize(blk)
176
+ @blk = blk
177
+ end
178
+
179
+ def call(resource_instance, args=[])
180
+ raise ArgumentError unless resource_instance.is_a?(ManagerModules::ResourceInstance)
181
+ @ri = resource_instance
182
+
183
+ instance_eval &@blk
184
+ end
185
+
186
+ private
187
+ def state_monitor
188
+ manifest.state_monitor
189
+ end
190
+
191
+ def next_event(ev, *args)
192
+ manifest.stm.process_event(ev, *args)
193
+ end
194
+
195
+ def manifest
196
+ @ri.manifest
197
+ end
198
+ end
199
+
200
+ module RakeHelper
201
+ module ClassMethods
202
+ def default_rakefile(rakefile)
203
+ rakefile =
204
+ if Pathname.new(rakefile).absolute?
205
+ rakefile.dup
206
+ else
207
+ File.expand_path(rakefile, @manifest.resource_root_path)
208
+ end
209
+ raise "File does not exist: #{rakefile}" unless File.exist?(rakefile)
210
+ @manifest.helpers[:default_rakefile] = rakefile
211
+ end
212
+
213
+ def rake_bin_path(path)
214
+ @manifest.helpers[:rake_bin_path] = path
215
+ end
216
+ end
217
+
218
+ module TaskMethods
219
+ def rake(task, rakefile=nil, &blk)
220
+ rake_path = manifest.helpers[:rake_bin_path] || Gem.bin_path('rake', 'rake')
221
+ rakefile = if rakefile
222
+ rakefile
223
+ elsif manifest.helpers[:default_rakefile]
224
+ manifest.helpers[:default_rakefile]
225
+ else
226
+ raise "Rakefile is not specified."
227
+ end
228
+
229
+ cmd = Util.quote_args("%s -I%s -f %s --rakelib %s RESOURCE_MANIFEST=%s %s",
230
+ [rake_path,
231
+ File.join(Isono.home, 'lib'),
232
+ rakefile,
233
+ #File.join(Isono.home, 'tasks/load_resource_manifest.rake'),
234
+ File.join(Isono.home, 'tasks'),
235
+ File.expand_path('resource.manifest', manifest.resource_root_path),
236
+ task
237
+ ])
238
+ logger.debug(cmd)
239
+ system(cmd)
240
+ end
241
+ end
242
+ end
243
+
244
+ module MonitorHelper
245
+ module ClassMethods
246
+
247
+ def state_monitor(monitor_class, &blk)
248
+ @manifest.config.state_monitor = self.monitor(monitor_class, &blk)
249
+ end
250
+
251
+ def monitor(monitor_class, &blk)
252
+ raise ArgumentError unless monitor_class.is_a?(Class) && monitor_class < Isono::Monitors::Base
253
+ @manifest.config.monitors ||= {}
254
+
255
+ raise "duplicate registration: #{monitor_class}" if @manifest.config.monitors.has_key?(monitor_class)
256
+
257
+ m = monitor_class.new()
258
+ m.instance_eval &blk if blk
259
+ @manifest.config.monitors[monitor_class] = m
260
+ end
261
+
262
+ end
263
+
264
+ module TaskMethods
265
+ def monitor(monitor_class)
266
+ manifest.config.monitors[monitor_class] || raise("unknown monitor class: #{monitor_class.to_s}")
267
+ end
268
+ end
269
+
270
+ end
271
+
272
+ end
273
+ end