flor 0.0.1 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (84) hide show
  1. data/CHANGELOG.md +13 -0
  2. data/LICENSE.txt +1 -1
  3. data/Makefile +66 -0
  4. data/README.md +57 -0
  5. data/fail.txt +7 -0
  6. data/flor.gemspec +12 -9
  7. data/intercepted.txt +123 -0
  8. data/lib/flor/colours.rb +140 -0
  9. data/lib/flor/conf.rb +88 -0
  10. data/lib/flor/core/executor.rb +473 -0
  11. data/lib/flor/core/node.rb +397 -0
  12. data/lib/flor/core/procedure.rb +600 -0
  13. data/lib/flor/core/texecutor.rb +209 -0
  14. data/lib/flor/core.rb +93 -0
  15. data/lib/flor/dollar.rb +248 -0
  16. data/lib/flor/errors.rb +36 -0
  17. data/lib/flor/flor.rb +556 -0
  18. data/lib/flor/log.rb +336 -0
  19. data/lib/flor/migrations/0001_tables.rb +122 -0
  20. data/lib/flor/parser.rb +414 -0
  21. data/lib/flor/pcore/_arr.rb +49 -0
  22. data/lib/flor/pcore/_atom.rb +43 -0
  23. data/lib/flor/pcore/_att.rb +160 -0
  24. data/lib/flor/pcore/_dump.rb +60 -0
  25. data/lib/flor/pcore/_err.rb +30 -0
  26. data/lib/flor/pcore/_happly.rb +73 -0
  27. data/lib/flor/pcore/_obj.rb +65 -0
  28. data/lib/flor/pcore/_skip.rb +63 -0
  29. data/lib/flor/pcore/apply.rb +60 -0
  30. data/lib/flor/pcore/arith.rb +46 -0
  31. data/lib/flor/pcore/break.rb +71 -0
  32. data/lib/flor/pcore/cmp.rb +72 -0
  33. data/lib/flor/pcore/cond.rb +57 -0
  34. data/lib/flor/pcore/cursor.rb +223 -0
  35. data/lib/flor/pcore/define.rb +96 -0
  36. data/lib/flor/pcore/fail.rb +45 -0
  37. data/lib/flor/pcore/ife.rb +56 -0
  38. data/lib/flor/pcore/loop.rb +53 -0
  39. data/lib/flor/pcore/map.rb +75 -0
  40. data/lib/flor/pcore/match.rb +70 -0
  41. data/lib/flor/pcore/move.rb +65 -0
  42. data/lib/flor/pcore/noeval.rb +46 -0
  43. data/lib/flor/pcore/noret.rb +47 -0
  44. data/lib/flor/pcore/push.rb +69 -0
  45. data/lib/flor/pcore/sequence.rb +39 -0
  46. data/lib/flor/pcore/set.rb +76 -0
  47. data/lib/flor/pcore/stall.rb +35 -0
  48. data/lib/flor/pcore/until.rb +122 -0
  49. data/lib/flor/pcore/val.rb +40 -0
  50. data/lib/flor/punit/cancel.rb +69 -0
  51. data/lib/flor/punit/cmap.rb +76 -0
  52. data/lib/flor/punit/concurrence.rb +149 -0
  53. data/lib/flor/punit/every.rb +46 -0
  54. data/lib/flor/punit/on.rb +81 -0
  55. data/lib/flor/punit/schedule.rb +68 -0
  56. data/lib/flor/punit/signal.rb +47 -0
  57. data/lib/flor/punit/sleep.rb +53 -0
  58. data/lib/flor/punit/task.rb +109 -0
  59. data/lib/flor/punit/trace.rb +51 -0
  60. data/lib/flor/punit/trap.rb +100 -0
  61. data/lib/flor/to_string.rb +81 -0
  62. data/lib/flor/tools/env.rb +103 -0
  63. data/lib/flor/tools/repl.rb +231 -0
  64. data/lib/flor/unit/executor.rb +260 -0
  65. data/lib/flor/unit/hooker.rb +186 -0
  66. data/lib/flor/unit/journal.rb +52 -0
  67. data/lib/flor/unit/loader.rb +181 -0
  68. data/lib/flor/unit/logger.rb +181 -0
  69. data/lib/flor/unit/models/execution.rb +105 -0
  70. data/lib/flor/unit/models/pointer.rb +31 -0
  71. data/lib/flor/unit/models/timer.rb +52 -0
  72. data/lib/flor/unit/models/trace.rb +31 -0
  73. data/lib/flor/unit/models/trap.rb +130 -0
  74. data/lib/flor/unit/models.rb +106 -0
  75. data/lib/flor/unit/scheduler.rb +419 -0
  76. data/lib/flor/unit/storage.rb +633 -0
  77. data/lib/flor/unit/tasker.rb +191 -0
  78. data/lib/flor/unit/waiter.rb +146 -0
  79. data/lib/flor/unit/wlist.rb +77 -0
  80. data/lib/flor/unit.rb +50 -0
  81. data/lib/flor.rb +40 -3
  82. metadata +152 -22
  83. checksums.yaml +0 -7
  84. data/Rakefile +0 -52
@@ -0,0 +1,419 @@
1
+ #--
2
+ # Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ # of this software and associated documentation files (the "Software"), to deal
6
+ # in the Software without restriction, including without limitation the rights
7
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
+ # copies of the Software, and to permit persons to whom the Software is
9
+ # furnished to do so, subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in
12
+ # all copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
+ # THE SOFTWARE.
21
+ #
22
+ # Made in Japan.
23
+ #++
24
+
25
+
26
+ module Flor
27
+
28
+ class Scheduler
29
+
30
+ attr_reader :conf, :env
31
+ attr_reader :hooker, :storage, :loader, :tasker
32
+
33
+ attr_reader :thread_status
34
+
35
+ attr_reader :archive
36
+
37
+ def initialize(conf={}, over_conf={})
38
+
39
+ @conf = conf.is_a?(Hash) ? conf : Flor::Conf.read(conf)
40
+ @conf.merge!(Flor::Conf.read_env)
41
+ @conf.merge!(over_conf)
42
+
43
+ fail ArgumentError.new(
44
+ "invalid domain name #{@conf['domain']}"
45
+ ) if @conf['domain'] && ! Flor.potential_domain_name?(@conf['domain'])
46
+
47
+ @env = @conf['env'] ||= 'dev'
48
+
49
+ @env = (Kernel.const_get(@env) rescue @env) if @env.match(/\A[A-Z]+\z/)
50
+ # when env is "RAILS_ENV" for example...
51
+
52
+ @hooker =
53
+ (Flor::Conf.get_class(@conf, 'hooker') || Flor::Hooker).new(self)
54
+ @storage =
55
+ (Flor::Conf.get_class(@conf, 'storage') || Flor::Storage).new(self)
56
+ @loader =
57
+ (Flor::Conf.get_class(@conf, 'loader') || Flor::Loader).new(self)
58
+ @tasker =
59
+ (Flor::Conf.get_class(@conf, 'tasker') || Flor::Tasker).new(self)
60
+
61
+ @hooker.add('logger', Flor::Logger)
62
+ @hooker.add('wlist', Flor::WaitList)
63
+
64
+ @heart_rate = @conf[:sch_heart_rate] || 0.3
65
+ @reload_frequency = @conf[:sch_reload_frequency] || 60
66
+ @max_executors = @conf[:sch_max_executors] || 1
67
+
68
+ @mutex = Mutex.new
69
+
70
+ @reloaded_at = nil
71
+ @timers = []
72
+ @exids = []
73
+
74
+ @executors = []
75
+
76
+ @archive = nil # used, so far, only for testing
77
+
78
+ c = @conf['constant']
79
+ Kernel.const_set(c, self) if c
80
+ end
81
+
82
+ def storage_mutex
83
+
84
+ @storage.mutex
85
+ end
86
+
87
+ def identifier
88
+
89
+ @identifier ||= 's' + Digest::MD5.hexdigest(self.object_id.to_s)[0, 5]
90
+ end
91
+
92
+ def shutdown
93
+
94
+ @thread_status = :shutdown
95
+ @thread = nil
96
+
97
+ @executors.each(&:shutdown)
98
+
99
+ @hooker.shutdown
100
+ @storage.shutdown
101
+ @tasker.shutdown
102
+ end
103
+
104
+ def hook(*args, &block)
105
+
106
+ @hooker.add(*args, &block)
107
+ end
108
+
109
+ def on_start_exc(e)
110
+
111
+ io = StringIO.new
112
+
113
+ head, kind =
114
+ e.is_a?(StandardError) ? [ '=sch', 'error' ] : [ '!sch', 'exception' ]
115
+ thr = Thread.current
116
+
117
+ t = head[0, 2] + Time.now.to_f.to_s.split('.').last
118
+ io.puts ' /' + t + ' ' + head * 17
119
+ io.puts " |#{t} + in #{self.class}#start"
120
+ io.puts " |#{t} db: #{@storage.db.class} #{@storage.db.object_id}"
121
+ io.puts " |#{t} thread: t#{thr.object_id} #{thr.inspect}"
122
+ io.puts " |#{t} #{kind}: #{e.inspect}"
123
+ io.puts " |#{t} backtrace:"
124
+ e.backtrace.each { |l| io.puts "|#{t} #{l}" }
125
+ io.puts ' \\' + t + ' ' + (head * 17) + ' .'
126
+
127
+ io.string
128
+ end
129
+
130
+ def start
131
+
132
+ # TODO heartbeat, every x minutes, when idle, log something
133
+
134
+ fail(
135
+ "database not ready, " +
136
+ "db ver: #{@storage.db_version.inspect}, " +
137
+ "mig ver: #{@storage.migration_version}"
138
+ ) if !! @conf['sto_migration_check'] && @storage.ready?
139
+
140
+ @thread_status = :running
141
+
142
+ @thread =
143
+ if @thread
144
+
145
+ @thread.run
146
+
147
+ else
148
+
149
+ Thread.new do
150
+ #p [ :unit_scheduler, :thread, Thread.current.object_id ]
151
+ loop do
152
+
153
+ begin
154
+
155
+ t0 = Time.now
156
+
157
+ Thread.stop if @thread_status == :stop
158
+ break if @thread_status == :shutdown
159
+
160
+ reload
161
+ trigger_timers
162
+ trigger_executions
163
+
164
+ sleep [ @heart_rate - (Time.now - t0), 0 ].max
165
+
166
+ #rescue => er
167
+ rescue Exception => ex
168
+
169
+ puts on_start_exc(ex)
170
+ end
171
+ end
172
+ end
173
+ end
174
+
175
+ self
176
+ end
177
+
178
+ def stop
179
+
180
+ @thread_status = :stop
181
+ end
182
+
183
+ def running?; @thread_status == :running; end
184
+ def stopped?; ! running?; end
185
+
186
+ def join
187
+
188
+ @thread.join
189
+ end
190
+
191
+ def launch(source_or_path, opts={})
192
+
193
+ source, domain, flow_name =
194
+ if df = Flor.split_flow_name(source_or_path)
195
+ [ source_or_path,
196
+ opts[:domain] || df[0],
197
+ df[1] ]
198
+ else
199
+ [ source_or_path,
200
+ opts[:domain] || @conf['domain'] || 'domain0',
201
+ nil ]
202
+ end
203
+
204
+ fail ArgumentError.new(
205
+ "invalid domain name #{domain.inspect}"
206
+ ) unless Flor.potential_domain_name?(domain)
207
+
208
+ if flow_name
209
+
210
+ source = @loader.library(source_or_path)
211
+
212
+ # TODO variables
213
+ # TODO payload
214
+ end
215
+
216
+ fail ArgumentError.new(
217
+ "flow not found in #{Flor.truncate(source_or_path, 35).inspect}"
218
+ ) unless source # will anyway fail badly if src is a tree (array of ...)
219
+
220
+ @archive ||= {} if opts[:archive]
221
+ # all subsequent launches will be `archive: true` ...
222
+
223
+ unit = opts[:unit] || @conf['unit'] || 'u0'
224
+
225
+ Flor.print_src(source, opts) if @conf['log_src']
226
+
227
+ exid = Flor.generate_exid(domain, unit)
228
+ msg = Flor.make_launch_msg(exid, source, opts)
229
+
230
+ Flor.print_tree(msg['tree']) if @conf['log_tree']
231
+
232
+ return [ msg, opts ] if opts[:nolaunch]
233
+ # for testing purposes
234
+
235
+ queue(msg, opts)
236
+ end
237
+
238
+ def queue(message, opts={})
239
+
240
+ @storage.put_message(message)
241
+
242
+ if opts[:wait]
243
+ wait(message['exid'], opts)
244
+ else
245
+ message['exid']
246
+ end
247
+ end
248
+
249
+ def prepare_message(point, h)
250
+
251
+ msg = { 'point' => point }
252
+ [ :exid, :name, :nid, :payload ].each { |k| msg[k.to_s] = h[k] }
253
+
254
+ fail ArgumentError.new('missing :exid key') \
255
+ unless msg['exid'].is_a?(String)
256
+ fail ArgumentError.new('missing :name string key') \
257
+ if point == 'signal' && ! msg['name'].is_a?(String)
258
+
259
+ msg
260
+ end
261
+
262
+ def cancel(h)
263
+
264
+ queue(prepare_message('cancel', h), h)
265
+ end
266
+
267
+ def signal(name, h={})
268
+
269
+ h[:payload] ||= {}
270
+ h[:name] ||= name
271
+ queue(prepare_message('signal', h), h)
272
+ end
273
+
274
+ def put_timer(message)
275
+
276
+ timer = @storage.put_timer(message)
277
+
278
+ @mutex.synchronize { @timers.push(timer).sort_by!(&:ntime) }
279
+ end
280
+
281
+ def wake_up_executions(exids)
282
+
283
+ @mutex.synchronize { @exids.concat(exids).uniq! } if exids.any?
284
+ end
285
+
286
+ def notify(executor, o)
287
+
288
+ @hooker.notify(executor, o)
289
+
290
+ rescue => err
291
+ puts '-sch' * 19
292
+ puts "+ error in #{self.class}#notify"
293
+ p err
294
+ puts err.backtrace
295
+ puts ('-sch' * 19) + ' .'
296
+ end
297
+
298
+ def trap(node, tra)
299
+
300
+ @storage.put_trap(node, tra)
301
+ end
302
+
303
+ def remove_node(exid, n)
304
+
305
+ #@storage.remove_node(exid, n)
306
+ # done in Storage#put_execution
307
+
308
+ @mutex.synchronize do
309
+ @timers.reject! { |t| t.exid == exid && t.nid == n['nid'] }
310
+ end
311
+
312
+ (@archive[exid] ||= {})[n['nid']] = Flor.dup(n) if @archive
313
+ end
314
+
315
+ # # Given an exid, returns the execution, if currently executing.
316
+ # #
317
+ # def execution(exid)
318
+ #
319
+ # ex = @executors.find { |x| x.exid == exid }
320
+ # ex ? ex.execution : nil
321
+ # end
322
+
323
+ def executor(exid)
324
+
325
+ @executors.find { |x| x.exid == exid }
326
+ end
327
+
328
+ protected
329
+
330
+ # # return [ domain, tree ]
331
+ # #
332
+ # def extract_domain_and_tree(s, opts)
333
+ #
334
+ # if Flor.potential_domain_name?(s)
335
+ #
336
+ # path = [ opts[:domain], s ].compact.join('.')
337
+ # elts = path.split('.')
338
+ # flow = @loader.library(path)
339
+ #
340
+ # fail ArgumentError.new(
341
+ # "flow not found at #{path.inspect}"
342
+ # ) unless flow
343
+ #
344
+ # [ elts[0..-2].join('.'), flow ]
345
+ #
346
+ # else
347
+ #
348
+ # [ opts[:domain], s ]
349
+ # end
350
+ # end
351
+
352
+ def reload
353
+
354
+ now = Time.now
355
+
356
+ return if @reloaded_at && (now - @reloaded_at < @reload_frequency)
357
+
358
+ @mutex.synchronize do
359
+
360
+ @reloaded_at = now
361
+ @timers = load_timers
362
+ @exids = load_exids
363
+ end
364
+ end
365
+
366
+ def load_timers
367
+
368
+ return [] if @thread_status != :running
369
+ @storage.load_timers.sort_by(&:ntime)
370
+ end
371
+
372
+ def load_exids
373
+
374
+ return [] if @thread_status != :running
375
+ @storage.load_exids
376
+ end
377
+
378
+ def trigger_timers
379
+
380
+ now = Time.now.utc
381
+ to_re_add = []
382
+
383
+ loop do
384
+
385
+ timer = @timers.first
386
+ break if timer == nil || timer.ntime_t > now
387
+
388
+ t = @mutex.synchronize { @timers.shift }
389
+ r = @storage.trigger_timer(t)
390
+ to_re_add << r if r
391
+ end
392
+
393
+ if to_re_add.any?
394
+
395
+ @mutex.synchronize do
396
+ @timers.concat(to_re_add)
397
+ @timers.sort_by! { |t| t.ntime }
398
+ end
399
+ end
400
+ end
401
+
402
+ def trigger_executions
403
+
404
+ return if @exids.empty?
405
+
406
+ while exid = @mutex.synchronize { @exids.shift }
407
+
408
+ @executors = @executors.select { |e| e.alive? }
409
+ # drop done executors
410
+
411
+ break if @executors.size > @max_executors
412
+ next if @executors.find { |e| e.exid == exid }
413
+
414
+ @executors << UnitExecutor.new(self, exid).run
415
+ end
416
+ end
417
+ end
418
+ end
419
+