flor 0.0.1 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (84) hide show
  1. data/CHANGELOG.md +13 -0
  2. data/LICENSE.txt +1 -1
  3. data/Makefile +66 -0
  4. data/README.md +57 -0
  5. data/fail.txt +7 -0
  6. data/flor.gemspec +12 -9
  7. data/intercepted.txt +123 -0
  8. data/lib/flor/colours.rb +140 -0
  9. data/lib/flor/conf.rb +88 -0
  10. data/lib/flor/core/executor.rb +473 -0
  11. data/lib/flor/core/node.rb +397 -0
  12. data/lib/flor/core/procedure.rb +600 -0
  13. data/lib/flor/core/texecutor.rb +209 -0
  14. data/lib/flor/core.rb +93 -0
  15. data/lib/flor/dollar.rb +248 -0
  16. data/lib/flor/errors.rb +36 -0
  17. data/lib/flor/flor.rb +556 -0
  18. data/lib/flor/log.rb +336 -0
  19. data/lib/flor/migrations/0001_tables.rb +122 -0
  20. data/lib/flor/parser.rb +414 -0
  21. data/lib/flor/pcore/_arr.rb +49 -0
  22. data/lib/flor/pcore/_atom.rb +43 -0
  23. data/lib/flor/pcore/_att.rb +160 -0
  24. data/lib/flor/pcore/_dump.rb +60 -0
  25. data/lib/flor/pcore/_err.rb +30 -0
  26. data/lib/flor/pcore/_happly.rb +73 -0
  27. data/lib/flor/pcore/_obj.rb +65 -0
  28. data/lib/flor/pcore/_skip.rb +63 -0
  29. data/lib/flor/pcore/apply.rb +60 -0
  30. data/lib/flor/pcore/arith.rb +46 -0
  31. data/lib/flor/pcore/break.rb +71 -0
  32. data/lib/flor/pcore/cmp.rb +72 -0
  33. data/lib/flor/pcore/cond.rb +57 -0
  34. data/lib/flor/pcore/cursor.rb +223 -0
  35. data/lib/flor/pcore/define.rb +96 -0
  36. data/lib/flor/pcore/fail.rb +45 -0
  37. data/lib/flor/pcore/ife.rb +56 -0
  38. data/lib/flor/pcore/loop.rb +53 -0
  39. data/lib/flor/pcore/map.rb +75 -0
  40. data/lib/flor/pcore/match.rb +70 -0
  41. data/lib/flor/pcore/move.rb +65 -0
  42. data/lib/flor/pcore/noeval.rb +46 -0
  43. data/lib/flor/pcore/noret.rb +47 -0
  44. data/lib/flor/pcore/push.rb +69 -0
  45. data/lib/flor/pcore/sequence.rb +39 -0
  46. data/lib/flor/pcore/set.rb +76 -0
  47. data/lib/flor/pcore/stall.rb +35 -0
  48. data/lib/flor/pcore/until.rb +122 -0
  49. data/lib/flor/pcore/val.rb +40 -0
  50. data/lib/flor/punit/cancel.rb +69 -0
  51. data/lib/flor/punit/cmap.rb +76 -0
  52. data/lib/flor/punit/concurrence.rb +149 -0
  53. data/lib/flor/punit/every.rb +46 -0
  54. data/lib/flor/punit/on.rb +81 -0
  55. data/lib/flor/punit/schedule.rb +68 -0
  56. data/lib/flor/punit/signal.rb +47 -0
  57. data/lib/flor/punit/sleep.rb +53 -0
  58. data/lib/flor/punit/task.rb +109 -0
  59. data/lib/flor/punit/trace.rb +51 -0
  60. data/lib/flor/punit/trap.rb +100 -0
  61. data/lib/flor/to_string.rb +81 -0
  62. data/lib/flor/tools/env.rb +103 -0
  63. data/lib/flor/tools/repl.rb +231 -0
  64. data/lib/flor/unit/executor.rb +260 -0
  65. data/lib/flor/unit/hooker.rb +186 -0
  66. data/lib/flor/unit/journal.rb +52 -0
  67. data/lib/flor/unit/loader.rb +181 -0
  68. data/lib/flor/unit/logger.rb +181 -0
  69. data/lib/flor/unit/models/execution.rb +105 -0
  70. data/lib/flor/unit/models/pointer.rb +31 -0
  71. data/lib/flor/unit/models/timer.rb +52 -0
  72. data/lib/flor/unit/models/trace.rb +31 -0
  73. data/lib/flor/unit/models/trap.rb +130 -0
  74. data/lib/flor/unit/models.rb +106 -0
  75. data/lib/flor/unit/scheduler.rb +419 -0
  76. data/lib/flor/unit/storage.rb +633 -0
  77. data/lib/flor/unit/tasker.rb +191 -0
  78. data/lib/flor/unit/waiter.rb +146 -0
  79. data/lib/flor/unit/wlist.rb +77 -0
  80. data/lib/flor/unit.rb +50 -0
  81. data/lib/flor.rb +40 -3
  82. metadata +152 -22
  83. checksums.yaml +0 -7
  84. data/Rakefile +0 -52
@@ -0,0 +1,633 @@
1
+ #--
2
+ # Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ # of this software and associated documentation files (the "Software"), to deal
6
+ # in the Software without restriction, including without limitation the rights
7
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
+ # copies of the Software, and to permit persons to whom the Software is
9
+ # furnished to do so, subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in
12
+ # all copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
+ # THE SOFTWARE.
21
+ #
22
+ # Made in Japan.
23
+ #++
24
+
25
+ require 'zlib'
26
+
27
+
28
+ module Flor
29
+
30
+ class Storage
31
+
32
+ attr_reader :unit, :db, :models
33
+
34
+ attr_reader :mutex
35
+ # might be useful for some implementations
36
+
37
+ def initialize(unit)
38
+
39
+ @unit = unit
40
+
41
+ @models = {}
42
+ @archive = @unit.conf['sto_archive']
43
+ @mutex = @unit.conf['sto_sync'] ? Mutex.new : nil
44
+
45
+ connect
46
+ end
47
+
48
+ def shutdown
49
+
50
+ @db.disconnect
51
+ #p [ :disconnected, @db.object_id ]
52
+ end
53
+
54
+ def db_version
55
+
56
+ (@db[:schema_info].first rescue {})[:version]
57
+ end
58
+
59
+ def migration_version
60
+
61
+ Dir[File.join(File.dirname(__FILE__), '../migrations/*.rb')]
62
+ .inject([]) { |a, fn|
63
+ m = File.basename(fn).match(/^(\d{4})_/)
64
+ a << m[1].to_i if m
65
+ a
66
+ }
67
+ .max
68
+ end
69
+
70
+ def ready?
71
+
72
+ db_version == migration_version
73
+ end
74
+
75
+ def synchronize(sync=true, &block)
76
+
77
+ Thread.current[:sto_errored_items] = nil
78
+
79
+ if @mutex && sync
80
+ @mutex.synchronize(&block)
81
+ else
82
+ block.call
83
+ end
84
+ end
85
+
86
+ def migrate(to=nil, from=nil)
87
+
88
+ dir =
89
+ @unit.conf['db_migrations'] ||
90
+ File.absolute_path(
91
+ File.join(
92
+ File.dirname(__FILE__), '..', 'migrations'))
93
+
94
+ synchronize do
95
+
96
+ Sequel::Migrator.run(
97
+ @db, dir,
98
+ :target => to, :current => from)
99
+
100
+ # defaults for the migration version table:
101
+ #:table => :schema_info
102
+ #:column => :version
103
+ end
104
+ end
105
+
106
+ def delete_tables
107
+
108
+ @db.tables.each { |t| @db[t].delete if t.to_s.match(/^flor_/) }
109
+ end
110
+
111
+ def load_exids
112
+
113
+ @db[:flor_messages]
114
+ .select(:exid)
115
+ .where(status: 'created')
116
+ .order_by(:ctime)
117
+ .distinct
118
+ .all
119
+ .collect { |r| r[:exid] }
120
+
121
+ rescue => err
122
+
123
+ @unit.logger.warn("#{self.class}#load_exids", err, '(returning [])')
124
+
125
+ []
126
+ end
127
+
128
+ def load_execution(exid)
129
+
130
+ e = @db[:flor_executions]
131
+ .select(:id, :content)
132
+ .where(exid: exid) # status active or terminated doesn't matter
133
+ .first
134
+
135
+ if e
136
+ ex = from_blob(e[:content])
137
+ fail("couldn't parse execution (db id #{e[:id]})") unless ex
138
+ ex['id'] = e[:id]
139
+ ex['size'] = e[:content].length
140
+ ex
141
+ else
142
+ put_execution({
143
+ 'exid' => exid, 'nodes' => {}, 'errors' => [], 'tasks' => {},
144
+ #'ashes' => {},
145
+ 'counters' => {}, 'start' => Flor.tstamp,
146
+ 'size' => -1
147
+ })
148
+ end
149
+ end
150
+
151
+ def put_execution(ex)
152
+
153
+ if i = ex['id']
154
+
155
+ status =
156
+ if ex['nodes']['0']['removed']
157
+ 'terminated'
158
+ else
159
+ 'active'
160
+ end
161
+
162
+ ex['end'] ||= Flor.tstamp \
163
+ if status == 'terminated'
164
+ ex['duration'] = Time.parse(ex['end']) - Time.parse(ex['start']) \
165
+ if ex['end']
166
+
167
+ data = to_blob(ex)
168
+ ex['size'] = data.length
169
+
170
+ synchronize do
171
+
172
+ @db.transaction do
173
+
174
+ now = Flor.tstamp
175
+
176
+ @db[:flor_executions]
177
+ .where(id: i)
178
+ .update(
179
+ content: data,
180
+ status: status,
181
+ mtime: now)
182
+
183
+ remove_nodes(ex, status, now)
184
+ update_pointers(ex, status, now)
185
+ end
186
+ end
187
+ else
188
+
189
+ data = to_blob(ex)
190
+ ex['size'] = data.length
191
+
192
+ synchronize do
193
+
194
+ @db.transaction do
195
+
196
+ now = Flor.tstamp
197
+
198
+ ex['id'] =
199
+ @db[:flor_executions]
200
+ .insert(
201
+ domain: Flor.domain(ex['exid']),
202
+ exid: ex['exid'],
203
+ content: data,
204
+ status: 'active',
205
+ ctime: now,
206
+ mtime: now)
207
+
208
+ remove_nodes(ex, status, now)
209
+ update_pointers(ex, status, now)
210
+ end
211
+ end
212
+ end
213
+
214
+ ex
215
+
216
+ rescue => err
217
+ Thread.current[:sto_errored_items] = [ ex ]
218
+ raise err
219
+ end
220
+
221
+ def fetch_messages(exid)
222
+
223
+ synchronize do
224
+ @db.transaction do
225
+
226
+ ms = @db[:flor_messages]
227
+ .select(:id, :content)
228
+ .where(status: 'created', exid: exid)
229
+ .order_by(:id)
230
+ .map { |m| r = from_blob(m[:content]) || {}; r['mid'] = m[:id]; r }
231
+
232
+ @db[:flor_messages]
233
+ .where(id: ms.collect { |m| m['mid'] })
234
+ .update(status: 'loaded')
235
+ #
236
+ # flag them as "loaded" so that other scheduler don't pick them
237
+
238
+ ms
239
+ end
240
+ end
241
+ end
242
+
243
+ def fetch_traps(exid)
244
+
245
+ traps
246
+ .where(status: 'active')
247
+ .where(domain: split_domain(exid))
248
+ .all
249
+
250
+ rescue => err
251
+
252
+ @unit.logger.warn("#{self.class}#fetch_traps()", err, '(returning [])')
253
+
254
+ []
255
+ end
256
+
257
+ def consume(messages)
258
+
259
+ synchronize do
260
+ if @archive
261
+ @db[:flor_messages]
262
+ .where(id: messages.collect { |m| m['mid'] }.compact)
263
+ .update(status: 'consumed', mtime: Flor.tstamp)
264
+ else
265
+ @db[:flor_messages]
266
+ .where(id: messages.collect { |m| m['mid'] }.compact)
267
+ .delete
268
+ end
269
+ end
270
+
271
+ rescue => err
272
+ Thread.current[:sto_errored_items] = messages
273
+ raise err
274
+ end
275
+
276
+ def load_timers
277
+
278
+ timers
279
+ .select(:id, :content)
280
+ .where(status: 'active')
281
+ .order_by(:id)
282
+ .all
283
+
284
+ rescue => err
285
+
286
+ @unit.logger.warn("#{self.class}#load_timers()", err, '(returning [])')
287
+
288
+ []
289
+ end
290
+
291
+ def put_messages(ms, syn=true)
292
+
293
+ return if ms.empty?
294
+
295
+ n = Flor.tstamp
296
+
297
+ synchronize(syn) do
298
+
299
+ @db[:flor_messages]
300
+ .import(
301
+ [ :domain, :exid, :point, :content,
302
+ :status, :ctime, :mtime ],
303
+ ms.map { |m|
304
+ [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
305
+ 'created', n, n ]
306
+ })
307
+ end
308
+
309
+ @unit.wake_up_executions(ms.collect { |m| m['exid'] }.uniq)
310
+
311
+ rescue => err
312
+ Thread.current[:sto_errored_items] = ms
313
+ raise err
314
+ end
315
+
316
+ def put_message(m)
317
+
318
+ put_messages([ m ])
319
+ end
320
+
321
+ def put_timer(message)
322
+
323
+ type, string = determine_type_and_schedule(message)
324
+
325
+ next_time = compute_next_time(type, string)
326
+
327
+ now = Flor.tstamp
328
+
329
+ id =
330
+ synchronize do
331
+ @db[:flor_timers].insert(
332
+ domain: Flor.domain(message['exid']),
333
+ exid: message['exid'],
334
+ nid: message['nid'],
335
+ type: type,
336
+ schedule: string,
337
+ ntime: next_time,
338
+ content: to_blob(message),
339
+ count: 0,
340
+ status: 'active',
341
+ ctime: now,
342
+ mtime: now)
343
+ end
344
+
345
+ @unit.timers[id]
346
+
347
+ rescue => err
348
+ Thread.current[:sto_errored_items] = [ message ]
349
+ raise err
350
+ end
351
+
352
+ # Returns the timer if it is rescheduling
353
+ #
354
+ def trigger_timer(timer)
355
+
356
+ r = nil
357
+
358
+ synchronize do
359
+ @db.transaction do
360
+
361
+ # TODO: cron/every stop conditions maybe?
362
+
363
+ if timer.type != 'at' && timer.type != 'in'
364
+
365
+ @db[:flor_timers]
366
+ .where(id: timer.id)
367
+ .update(
368
+ count: timer.count + 1,
369
+ ntime: compute_next_time(timer.type, timer.schedule),
370
+ mtime: Flor.tstamp)
371
+ r = timers[timer.id]
372
+
373
+ elsif @archive
374
+
375
+ @db[:flor_timers]
376
+ .where(id: timer.id)
377
+ .update(
378
+ count: timer.count + 1,
379
+ status: 'triggered',
380
+ mtime: Flor.tstamp)
381
+
382
+ else
383
+
384
+ @db[:flor_timers]
385
+ .where(id: timer.id)
386
+ .delete
387
+ end
388
+
389
+ put_messages([ timer.to_trigger_message ], false)
390
+ end
391
+ end
392
+
393
+ r
394
+
395
+ rescue => err
396
+ Thread.current[:sto_errored_items] = [ timer ]
397
+ raise err
398
+ end
399
+
400
+ def put_trap(node, tra)
401
+
402
+ exid = node['exid']
403
+ dom = Flor.domain(exid)
404
+ now = Flor.tstamp
405
+
406
+ id =
407
+ synchronize do
408
+ @db.transaction do
409
+
410
+ @db[:flor_traps].insert(
411
+ domain: dom,
412
+ exid: exid,
413
+ nid: tra['bnid'],
414
+ onid: node['nid'],
415
+ trange: tra['range'],
416
+ tpoints: tra['points'],
417
+ ttags: tra['tags'],
418
+ theats: tra['heats'],
419
+ theaps: tra['heaps'],
420
+ content: to_blob(tra),
421
+ status: 'active',
422
+ ctime: now,
423
+ mtime: now)
424
+ end
425
+ end
426
+
427
+ traps[id]
428
+
429
+ rescue => err
430
+ Thread.current[:sto_errored_items] = [ node, tra ]
431
+ raise err
432
+ end
433
+
434
+ def trace(exid, nid, tracer, text)
435
+
436
+ text = text.is_a?(String) ? text : JSON.dump(text)
437
+
438
+ synchronize do
439
+
440
+ @db[:flor_traces].insert(
441
+ domain: Flor.domain(exid),
442
+ exid: exid,
443
+ nid: nid,
444
+ tracer: tracer,
445
+ text: text,
446
+ ctime: Flor.tstamp)
447
+ end
448
+ end
449
+
450
+ def put_task_pointer(msg, tname, tconf)
451
+
452
+ exid = msg['exid']
453
+ dom = Flor.domain(exid)
454
+
455
+ synchronize do
456
+
457
+ @db[:flor_pointers]
458
+ .insert(
459
+ domain: dom,
460
+ exid: exid,
461
+ nid: msg['nid'],
462
+ type: 'tasker',
463
+ name: tname,
464
+ ctime: Flor.tstamp)
465
+ end
466
+ end
467
+
468
+ protected
469
+
470
+ def remove_nodes(exe, status, now)
471
+
472
+ exid = exe['exid']
473
+
474
+ x = status == 'terminated' ? {} : { nid: exe['nodes'].keys }
475
+ # if 'terminated' include all nodes
476
+
477
+ if @archive
478
+ @db[:flor_timers].where(exid: exid).exclude(x).update(status: 'removed')
479
+ @db[:flor_traps].where(exid: exid).exclude(x).update(status: 'removed')
480
+ else
481
+ @db[:flor_timers].where(exid: exid).exclude(x).delete
482
+ @db[:flor_traps].where(exid: exid).exclude(x).delete
483
+ end
484
+
485
+ #@db[:flor_pointers].where(exid: exid).exclude(x).delete
486
+ # done in update_pointers
487
+ end
488
+
489
+ def update_pointers(exe, status, now)
490
+
491
+ exid = exe['exid']
492
+
493
+ if status == 'terminated'
494
+ @db[:flor_pointers].where(exid: exid).delete
495
+ return
496
+ end
497
+
498
+ @db[:flor_pointers]
499
+ .where(exid: exid)
500
+ .where(Sequel.|({ type: %w[ var ] }, Sequel.~(nid: exe['nodes'].keys)))
501
+ .delete
502
+ #
503
+ # Delete all pointer to vars, their value might have changed,
504
+ # let's reinsert them.
505
+ # Delete pointers to gone nodes.
506
+
507
+ dom = Flor.domain(exid)
508
+
509
+ pointers =
510
+ exe['nodes'].inject([]) { |a, (nid, node)|
511
+ ts = node['tags']
512
+ ts.each { |t| a << [ dom, exid, nid, 'tag', t, nil, now ] } if ts
513
+ a
514
+ }
515
+
516
+ pointers +=
517
+ (exe['nodes']['0'] || { 'vars' => {} })['vars'].collect { |k, v|
518
+ case v; when Integer, String, TrueClass, FalseClass
519
+ [ dom, exid, '0', 'var', k, v.to_s, now ]
520
+ when NilClass
521
+ [ dom, exid, '0', 'var', k, nil, now ]
522
+ else
523
+ nil
524
+ end
525
+ }.compact
526
+
527
+ pointers +=
528
+ exe['tasks'].collect { |nid, v|
529
+ [ dom, exid, nid, 'tasker', v['tasker'], v['name'], now ]
530
+ }
531
+
532
+ cps = @db[:flor_pointers] # current pointers
533
+ .where(exid: exid)
534
+ .select(:nid, :type, :name)
535
+ .all
536
+ pointers.reject! { |_, _, ni, ty, na, _, _|
537
+ cps.find { |cp| cp[:nid] == ni && cp[:type] == ty && cp[:name] == na } }
538
+ #
539
+ # don't insert when already inserted
540
+
541
+ @db[:flor_pointers]
542
+ .import(
543
+ [ :domain, :exid, :nid, :type, :name, :value, :ctime ],
544
+ pointers)
545
+ end
546
+
547
+ def determine_type_and_schedule(message)
548
+
549
+ t, s = message['type'], message['string']
550
+ return [ t, s ] if t
551
+
552
+ t = Fugit.determine_type(s)
553
+ return [ t, s ] if t
554
+
555
+ s = "every #{s}"
556
+ return [ 'cron', s ] if Fugit.parse_nat(s)
557
+
558
+ nil
559
+ end
560
+
561
+ def compute_next_time(type, string)
562
+
563
+ f =
564
+ case type
565
+ when 'cron' then Fugit.parse_cron(string) || Fugit.parse_nat(string)
566
+ when 'at' then Fugit.parse_at(string)
567
+ when 'in' then Fugit.parse_duration(string)
568
+ #when 'every' then Fugit.parse_duration(string)
569
+ else Fugit.parse(string)
570
+ end
571
+
572
+ nt = f.is_a?(Time) ? f : f.next_time(Time.now) # local...
573
+
574
+ Flor.tstamp(nt.utc)
575
+ end
576
+
577
+ def split_domain(exid)
578
+
579
+ Flor.domain(exid)
580
+ .split('.')
581
+ .inject([]) { |a, elt| a << [ a.last, elt ].compact.join('.'); a }
582
+ end
583
+
584
+ class DbLogger
585
+
586
+ def initialize(unit); @unit = unit; end
587
+
588
+ def info(msg); @unit.logger.db_log(:info, msg); end
589
+ def error(msg); @unit.logger.db_log(:error, msg); end
590
+ end
591
+
592
+ def connect
593
+
594
+ uri = @unit.conf['sto_uri']
595
+
596
+ #uri = DB.uri if uri == 'DB' && defined?(DB)
597
+ uri = (Kernel.const_get(uri).uri rescue uri) if uri.match(/\A[A-Z]+\z/)
598
+ # for cases where `sto_uri: "DB"`
599
+
600
+ @db = Sequel.connect(uri)
601
+
602
+ class << @db; attr_accessor :flor_unit; end
603
+ @db.flor_unit = @unit
604
+
605
+ if cv = @unit.conf['sto_connection_validation']
606
+
607
+ to = cv.is_a?(Numeric) || cv.is_a?(String) ? cv.to_i : -1
608
+
609
+ @db.extension(:connection_validator)
610
+ @db.pool.connection_validation_timeout = to
611
+ # NB: -1 means "check all the time"
612
+ end
613
+
614
+ @db_logger = DbLogger.new(@unit)
615
+ @db.loggers << @db_logger
616
+ end
617
+
618
+ def self.to_blob(h)
619
+
620
+ Sequel.blob(Zlib::Deflate.deflate(JSON.dump(h)))
621
+ #rescue => e; pp h; raise e
622
+ end
623
+
624
+ def self.from_blob(content)
625
+
626
+ JSON.parse(Zlib::Inflate.inflate(content))
627
+ end
628
+
629
+ def to_blob(h); self.class.to_blob(h); end
630
+ def from_blob(content); self.class.from_blob(content); end
631
+ end
632
+ end
633
+