flor 0.0.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. data/CHANGELOG.md +13 -0
  2. data/LICENSE.txt +1 -1
  3. data/Makefile +66 -0
  4. data/README.md +57 -0
  5. data/fail.txt +7 -0
  6. data/flor.gemspec +12 -9
  7. data/intercepted.txt +123 -0
  8. data/lib/flor/colours.rb +140 -0
  9. data/lib/flor/conf.rb +88 -0
  10. data/lib/flor/core/executor.rb +473 -0
  11. data/lib/flor/core/node.rb +397 -0
  12. data/lib/flor/core/procedure.rb +600 -0
  13. data/lib/flor/core/texecutor.rb +209 -0
  14. data/lib/flor/core.rb +93 -0
  15. data/lib/flor/dollar.rb +248 -0
  16. data/lib/flor/errors.rb +36 -0
  17. data/lib/flor/flor.rb +556 -0
  18. data/lib/flor/log.rb +336 -0
  19. data/lib/flor/migrations/0001_tables.rb +122 -0
  20. data/lib/flor/parser.rb +414 -0
  21. data/lib/flor/pcore/_arr.rb +49 -0
  22. data/lib/flor/pcore/_atom.rb +43 -0
  23. data/lib/flor/pcore/_att.rb +160 -0
  24. data/lib/flor/pcore/_dump.rb +60 -0
  25. data/lib/flor/pcore/_err.rb +30 -0
  26. data/lib/flor/pcore/_happly.rb +73 -0
  27. data/lib/flor/pcore/_obj.rb +65 -0
  28. data/lib/flor/pcore/_skip.rb +63 -0
  29. data/lib/flor/pcore/apply.rb +60 -0
  30. data/lib/flor/pcore/arith.rb +46 -0
  31. data/lib/flor/pcore/break.rb +71 -0
  32. data/lib/flor/pcore/cmp.rb +72 -0
  33. data/lib/flor/pcore/cond.rb +57 -0
  34. data/lib/flor/pcore/cursor.rb +223 -0
  35. data/lib/flor/pcore/define.rb +96 -0
  36. data/lib/flor/pcore/fail.rb +45 -0
  37. data/lib/flor/pcore/ife.rb +56 -0
  38. data/lib/flor/pcore/loop.rb +53 -0
  39. data/lib/flor/pcore/map.rb +75 -0
  40. data/lib/flor/pcore/match.rb +70 -0
  41. data/lib/flor/pcore/move.rb +65 -0
  42. data/lib/flor/pcore/noeval.rb +46 -0
  43. data/lib/flor/pcore/noret.rb +47 -0
  44. data/lib/flor/pcore/push.rb +69 -0
  45. data/lib/flor/pcore/sequence.rb +39 -0
  46. data/lib/flor/pcore/set.rb +76 -0
  47. data/lib/flor/pcore/stall.rb +35 -0
  48. data/lib/flor/pcore/until.rb +122 -0
  49. data/lib/flor/pcore/val.rb +40 -0
  50. data/lib/flor/punit/cancel.rb +69 -0
  51. data/lib/flor/punit/cmap.rb +76 -0
  52. data/lib/flor/punit/concurrence.rb +149 -0
  53. data/lib/flor/punit/every.rb +46 -0
  54. data/lib/flor/punit/on.rb +81 -0
  55. data/lib/flor/punit/schedule.rb +68 -0
  56. data/lib/flor/punit/signal.rb +47 -0
  57. data/lib/flor/punit/sleep.rb +53 -0
  58. data/lib/flor/punit/task.rb +109 -0
  59. data/lib/flor/punit/trace.rb +51 -0
  60. data/lib/flor/punit/trap.rb +100 -0
  61. data/lib/flor/to_string.rb +81 -0
  62. data/lib/flor/tools/env.rb +103 -0
  63. data/lib/flor/tools/repl.rb +231 -0
  64. data/lib/flor/unit/executor.rb +260 -0
  65. data/lib/flor/unit/hooker.rb +186 -0
  66. data/lib/flor/unit/journal.rb +52 -0
  67. data/lib/flor/unit/loader.rb +181 -0
  68. data/lib/flor/unit/logger.rb +181 -0
  69. data/lib/flor/unit/models/execution.rb +105 -0
  70. data/lib/flor/unit/models/pointer.rb +31 -0
  71. data/lib/flor/unit/models/timer.rb +52 -0
  72. data/lib/flor/unit/models/trace.rb +31 -0
  73. data/lib/flor/unit/models/trap.rb +130 -0
  74. data/lib/flor/unit/models.rb +106 -0
  75. data/lib/flor/unit/scheduler.rb +419 -0
  76. data/lib/flor/unit/storage.rb +633 -0
  77. data/lib/flor/unit/tasker.rb +191 -0
  78. data/lib/flor/unit/waiter.rb +146 -0
  79. data/lib/flor/unit/wlist.rb +77 -0
  80. data/lib/flor/unit.rb +50 -0
  81. data/lib/flor.rb +40 -3
  82. metadata +152 -22
  83. checksums.yaml +0 -7
  84. data/Rakefile +0 -52
@@ -0,0 +1,419 @@
1
+ #--
2
+ # Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ # of this software and associated documentation files (the "Software"), to deal
6
+ # in the Software without restriction, including without limitation the rights
7
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
+ # copies of the Software, and to permit persons to whom the Software is
9
+ # furnished to do so, subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in
12
+ # all copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
+ # THE SOFTWARE.
21
+ #
22
+ # Made in Japan.
23
+ #++
24
+
25
+
26
+ module Flor
27
+
28
+ class Scheduler
29
+
30
+ attr_reader :conf, :env
31
+ attr_reader :hooker, :storage, :loader, :tasker
32
+
33
+ attr_reader :thread_status
34
+
35
+ attr_reader :archive
36
+
37
+ def initialize(conf={}, over_conf={})
38
+
39
+ @conf = conf.is_a?(Hash) ? conf : Flor::Conf.read(conf)
40
+ @conf.merge!(Flor::Conf.read_env)
41
+ @conf.merge!(over_conf)
42
+
43
+ fail ArgumentError.new(
44
+ "invalid domain name #{@conf['domain']}"
45
+ ) if @conf['domain'] && ! Flor.potential_domain_name?(@conf['domain'])
46
+
47
+ @env = @conf['env'] ||= 'dev'
48
+
49
+ @env = (Kernel.const_get(@env) rescue @env) if @env.match(/\A[A-Z]+\z/)
50
+ # when env is "RAILS_ENV" for example...
51
+
52
+ @hooker =
53
+ (Flor::Conf.get_class(@conf, 'hooker') || Flor::Hooker).new(self)
54
+ @storage =
55
+ (Flor::Conf.get_class(@conf, 'storage') || Flor::Storage).new(self)
56
+ @loader =
57
+ (Flor::Conf.get_class(@conf, 'loader') || Flor::Loader).new(self)
58
+ @tasker =
59
+ (Flor::Conf.get_class(@conf, 'tasker') || Flor::Tasker).new(self)
60
+
61
+ @hooker.add('logger', Flor::Logger)
62
+ @hooker.add('wlist', Flor::WaitList)
63
+
64
+ @heart_rate = @conf[:sch_heart_rate] || 0.3
65
+ @reload_frequency = @conf[:sch_reload_frequency] || 60
66
+ @max_executors = @conf[:sch_max_executors] || 1
67
+
68
+ @mutex = Mutex.new
69
+
70
+ @reloaded_at = nil
71
+ @timers = []
72
+ @exids = []
73
+
74
+ @executors = []
75
+
76
+ @archive = nil # used, so far, only for testing
77
+
78
+ c = @conf['constant']
79
+ Kernel.const_set(c, self) if c
80
+ end
81
+
82
+ def storage_mutex
83
+
84
+ @storage.mutex
85
+ end
86
+
87
+ def identifier
88
+
89
+ @identifier ||= 's' + Digest::MD5.hexdigest(self.object_id.to_s)[0, 5]
90
+ end
91
+
92
+ def shutdown
93
+
94
+ @thread_status = :shutdown
95
+ @thread = nil
96
+
97
+ @executors.each(&:shutdown)
98
+
99
+ @hooker.shutdown
100
+ @storage.shutdown
101
+ @tasker.shutdown
102
+ end
103
+
104
+ def hook(*args, &block)
105
+
106
+ @hooker.add(*args, &block)
107
+ end
108
+
109
+ def on_start_exc(e)
110
+
111
+ io = StringIO.new
112
+
113
+ head, kind =
114
+ e.is_a?(StandardError) ? [ '=sch', 'error' ] : [ '!sch', 'exception' ]
115
+ thr = Thread.current
116
+
117
+ t = head[0, 2] + Time.now.to_f.to_s.split('.').last
118
+ io.puts ' /' + t + ' ' + head * 17
119
+ io.puts " |#{t} + in #{self.class}#start"
120
+ io.puts " |#{t} db: #{@storage.db.class} #{@storage.db.object_id}"
121
+ io.puts " |#{t} thread: t#{thr.object_id} #{thr.inspect}"
122
+ io.puts " |#{t} #{kind}: #{e.inspect}"
123
+ io.puts " |#{t} backtrace:"
124
+ e.backtrace.each { |l| io.puts "|#{t} #{l}" }
125
+ io.puts ' \\' + t + ' ' + (head * 17) + ' .'
126
+
127
+ io.string
128
+ end
129
+
130
+ def start
131
+
132
+ # TODO heartbeat, every x minutes, when idle, log something
133
+
134
+ fail(
135
+ "database not ready, " +
136
+ "db ver: #{@storage.db_version.inspect}, " +
137
+ "mig ver: #{@storage.migration_version}"
138
+ ) if !! @conf['sto_migration_check'] && @storage.ready?
139
+
140
+ @thread_status = :running
141
+
142
+ @thread =
143
+ if @thread
144
+
145
+ @thread.run
146
+
147
+ else
148
+
149
+ Thread.new do
150
+ #p [ :unit_scheduler, :thread, Thread.current.object_id ]
151
+ loop do
152
+
153
+ begin
154
+
155
+ t0 = Time.now
156
+
157
+ Thread.stop if @thread_status == :stop
158
+ break if @thread_status == :shutdown
159
+
160
+ reload
161
+ trigger_timers
162
+ trigger_executions
163
+
164
+ sleep [ @heart_rate - (Time.now - t0), 0 ].max
165
+
166
+ #rescue => er
167
+ rescue Exception => ex
168
+
169
+ puts on_start_exc(ex)
170
+ end
171
+ end
172
+ end
173
+ end
174
+
175
+ self
176
+ end
177
+
178
+ def stop
179
+
180
+ @thread_status = :stop
181
+ end
182
+
183
+ def running?; @thread_status == :running; end
184
+ def stopped?; ! running?; end
185
+
186
+ def join
187
+
188
+ @thread.join
189
+ end
190
+
191
+ def launch(source_or_path, opts={})
192
+
193
+ source, domain, flow_name =
194
+ if df = Flor.split_flow_name(source_or_path)
195
+ [ source_or_path,
196
+ opts[:domain] || df[0],
197
+ df[1] ]
198
+ else
199
+ [ source_or_path,
200
+ opts[:domain] || @conf['domain'] || 'domain0',
201
+ nil ]
202
+ end
203
+
204
+ fail ArgumentError.new(
205
+ "invalid domain name #{domain.inspect}"
206
+ ) unless Flor.potential_domain_name?(domain)
207
+
208
+ if flow_name
209
+
210
+ source = @loader.library(source_or_path)
211
+
212
+ # TODO variables
213
+ # TODO payload
214
+ end
215
+
216
+ fail ArgumentError.new(
217
+ "flow not found in #{Flor.truncate(source_or_path, 35).inspect}"
218
+ ) unless source # will anyway fail badly if src is a tree (array of ...)
219
+
220
+ @archive ||= {} if opts[:archive]
221
+ # all subsequent launches will be `archive: true` ...
222
+
223
+ unit = opts[:unit] || @conf['unit'] || 'u0'
224
+
225
+ Flor.print_src(source, opts) if @conf['log_src']
226
+
227
+ exid = Flor.generate_exid(domain, unit)
228
+ msg = Flor.make_launch_msg(exid, source, opts)
229
+
230
+ Flor.print_tree(msg['tree']) if @conf['log_tree']
231
+
232
+ return [ msg, opts ] if opts[:nolaunch]
233
+ # for testing purposes
234
+
235
+ queue(msg, opts)
236
+ end
237
+
238
+ def queue(message, opts={})
239
+
240
+ @storage.put_message(message)
241
+
242
+ if opts[:wait]
243
+ wait(message['exid'], opts)
244
+ else
245
+ message['exid']
246
+ end
247
+ end
248
+
249
+ def prepare_message(point, h)
250
+
251
+ msg = { 'point' => point }
252
+ [ :exid, :name, :nid, :payload ].each { |k| msg[k.to_s] = h[k] }
253
+
254
+ fail ArgumentError.new('missing :exid key') \
255
+ unless msg['exid'].is_a?(String)
256
+ fail ArgumentError.new('missing :name string key') \
257
+ if point == 'signal' && ! msg['name'].is_a?(String)
258
+
259
+ msg
260
+ end
261
+
262
+ def cancel(h)
263
+
264
+ queue(prepare_message('cancel', h), h)
265
+ end
266
+
267
+ def signal(name, h={})
268
+
269
+ h[:payload] ||= {}
270
+ h[:name] ||= name
271
+ queue(prepare_message('signal', h), h)
272
+ end
273
+
274
+ def put_timer(message)
275
+
276
+ timer = @storage.put_timer(message)
277
+
278
+ @mutex.synchronize { @timers.push(timer).sort_by!(&:ntime) }
279
+ end
280
+
281
+ def wake_up_executions(exids)
282
+
283
+ @mutex.synchronize { @exids.concat(exids).uniq! } if exids.any?
284
+ end
285
+
286
+ def notify(executor, o)
287
+
288
+ @hooker.notify(executor, o)
289
+
290
+ rescue => err
291
+ puts '-sch' * 19
292
+ puts "+ error in #{self.class}#notify"
293
+ p err
294
+ puts err.backtrace
295
+ puts ('-sch' * 19) + ' .'
296
+ end
297
+
298
+ def trap(node, tra)
299
+
300
+ @storage.put_trap(node, tra)
301
+ end
302
+
303
+ def remove_node(exid, n)
304
+
305
+ #@storage.remove_node(exid, n)
306
+ # done in Storage#put_execution
307
+
308
+ @mutex.synchronize do
309
+ @timers.reject! { |t| t.exid == exid && t.nid == n['nid'] }
310
+ end
311
+
312
+ (@archive[exid] ||= {})[n['nid']] = Flor.dup(n) if @archive
313
+ end
314
+
315
+ # # Given an exid, returns the execution, if currently executing.
316
+ # #
317
+ # def execution(exid)
318
+ #
319
+ # ex = @executors.find { |x| x.exid == exid }
320
+ # ex ? ex.execution : nil
321
+ # end
322
+
323
+ def executor(exid)
324
+
325
+ @executors.find { |x| x.exid == exid }
326
+ end
327
+
328
+ protected
329
+
330
+ # # return [ domain, tree ]
331
+ # #
332
+ # def extract_domain_and_tree(s, opts)
333
+ #
334
+ # if Flor.potential_domain_name?(s)
335
+ #
336
+ # path = [ opts[:domain], s ].compact.join('.')
337
+ # elts = path.split('.')
338
+ # flow = @loader.library(path)
339
+ #
340
+ # fail ArgumentError.new(
341
+ # "flow not found at #{path.inspect}"
342
+ # ) unless flow
343
+ #
344
+ # [ elts[0..-2].join('.'), flow ]
345
+ #
346
+ # else
347
+ #
348
+ # [ opts[:domain], s ]
349
+ # end
350
+ # end
351
+
352
+ def reload
353
+
354
+ now = Time.now
355
+
356
+ return if @reloaded_at && (now - @reloaded_at < @reload_frequency)
357
+
358
+ @mutex.synchronize do
359
+
360
+ @reloaded_at = now
361
+ @timers = load_timers
362
+ @exids = load_exids
363
+ end
364
+ end
365
+
366
+ def load_timers
367
+
368
+ return [] if @thread_status != :running
369
+ @storage.load_timers.sort_by(&:ntime)
370
+ end
371
+
372
+ def load_exids
373
+
374
+ return [] if @thread_status != :running
375
+ @storage.load_exids
376
+ end
377
+
378
+ def trigger_timers
379
+
380
+ now = Time.now.utc
381
+ to_re_add = []
382
+
383
+ loop do
384
+
385
+ timer = @timers.first
386
+ break if timer == nil || timer.ntime_t > now
387
+
388
+ t = @mutex.synchronize { @timers.shift }
389
+ r = @storage.trigger_timer(t)
390
+ to_re_add << r if r
391
+ end
392
+
393
+ if to_re_add.any?
394
+
395
+ @mutex.synchronize do
396
+ @timers.concat(to_re_add)
397
+ @timers.sort_by! { |t| t.ntime }
398
+ end
399
+ end
400
+ end
401
+
402
+ def trigger_executions
403
+
404
+ return if @exids.empty?
405
+
406
+ while exid = @mutex.synchronize { @exids.shift }
407
+
408
+ @executors = @executors.select { |e| e.alive? }
409
+ # drop done executors
410
+
411
+ break if @executors.size > @max_executors
412
+ next if @executors.find { |e| e.exid == exid }
413
+
414
+ @executors << UnitExecutor.new(self, exid).run
415
+ end
416
+ end
417
+ end
418
+ end
419
+