flor 0.18.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +29 -0
- data/CREDITS.md +1 -0
- data/LICENSE.txt +1 -1
- data/Makefile +1 -1
- data/README.md +16 -2
- data/flor.gemspec +1 -2
- data/lib/flor.rb +3 -2
- data/lib/flor/colours.rb +5 -3
- data/lib/flor/conf.rb +1 -0
- data/lib/flor/core.rb +9 -8
- data/lib/flor/core/executor.rb +20 -16
- data/lib/flor/core/node.rb +1 -13
- data/lib/flor/core/procedure.rb +10 -1
- data/lib/flor/core/texecutor.rb +35 -3
- data/lib/flor/djan.rb +1 -0
- data/lib/flor/errors.rb +1 -0
- data/lib/flor/flor.rb +172 -26
- data/lib/flor/id.rb +5 -1
- data/lib/flor/log.rb +14 -8
- data/lib/flor/migrations/0001_tables.rb +1 -0
- data/lib/flor/migrations/0002_cunit_and_munit.rb +1 -0
- data/lib/flor/migrations/0003_timer_onid_bnid.rb +1 -0
- data/lib/flor/migrations/0004_trap_bnid.rb +1 -0
- data/lib/flor/migrations/0005_pointer_content.rb +1 -0
- data/lib/flor/parser.rb +19 -11
- data/lib/flor/pcore/_apply.rb +1 -0
- data/lib/flor/pcore/_arr.rb +1 -0
- data/lib/flor/pcore/_atom.rb +1 -0
- data/lib/flor/pcore/_att.rb +1 -0
- data/lib/flor/pcore/_coll.rb +1 -0
- data/lib/flor/pcore/_dmute.rb +1 -0
- data/lib/flor/pcore/_dol.rb +1 -0
- data/lib/flor/pcore/_dqs.rb +1 -0
- data/lib/flor/pcore/_dump.rb +1 -0
- data/lib/flor/pcore/_err.rb +1 -0
- data/lib/flor/pcore/_head.rb +1 -0
- data/lib/flor/pcore/_obj.rb +1 -0
- data/lib/flor/pcore/_pat_.rb +1 -0
- data/lib/flor/pcore/_pat_arr.rb +1 -0
- data/lib/flor/pcore/_pat_guard.rb +1 -0
- data/lib/flor/pcore/_pat_obj.rb +1 -0
- data/lib/flor/pcore/_pat_or.rb +1 -0
- data/lib/flor/pcore/_pat_regex.rb +1 -0
- data/lib/flor/pcore/_ref.rb +1 -0
- data/lib/flor/pcore/_rxs.rb +1 -0
- data/lib/flor/pcore/_skip.rb +1 -0
- data/lib/flor/pcore/_val.rb +1 -0
- data/lib/flor/pcore/all.rb +1 -0
- data/lib/flor/pcore/andor.rb +1 -0
- data/lib/flor/pcore/any.rb +1 -0
- data/lib/flor/pcore/apply.rb +1 -0
- data/lib/flor/pcore/arith.rb +1 -0
- data/lib/flor/pcore/array_qmark.rb +1 -0
- data/lib/flor/pcore/break.rb +1 -0
- data/lib/flor/pcore/case.rb +1 -0
- data/lib/flor/pcore/cmp.rb +1 -0
- data/lib/flor/pcore/collect.rb +2 -1
- data/lib/flor/pcore/cond.rb +1 -0
- data/lib/flor/pcore/cursor.rb +45 -3
- data/lib/flor/pcore/define.rb +1 -0
- data/lib/flor/pcore/detect.rb +1 -0
- data/lib/flor/pcore/do_return.rb +1 -0
- data/lib/flor/pcore/each.rb +1 -0
- data/lib/flor/pcore/echo.rb +1 -0
- data/lib/flor/pcore/empty.rb +1 -0
- data/lib/flor/pcore/fail.rb +1 -0
- data/lib/flor/pcore/filter.rb +1 -0
- data/lib/flor/pcore/find.rb +1 -0
- data/lib/flor/pcore/flatten.rb +1 -0
- data/lib/flor/pcore/for_each.rb +1 -0
- data/lib/flor/pcore/if.rb +1 -0
- data/lib/flor/pcore/includes.rb +1 -0
- data/lib/flor/pcore/inject.rb +1 -0
- data/lib/flor/pcore/iterator.rb +1 -0
- data/lib/flor/pcore/keys.rb +1 -0
- data/lib/flor/pcore/length.rb +1 -0
- data/lib/flor/pcore/loop.rb +1 -0
- data/lib/flor/pcore/map.rb +1 -0
- data/lib/flor/pcore/match.rb +1 -0
- data/lib/flor/pcore/matchr.rb +1 -0
- data/lib/flor/pcore/max.rb +1 -0
- data/lib/flor/pcore/merge.rb +1 -0
- data/lib/flor/pcore/move.rb +1 -0
- data/lib/flor/pcore/noeval.rb +1 -0
- data/lib/flor/pcore/noret.rb +1 -0
- data/lib/flor/pcore/not.rb +1 -0
- data/lib/flor/pcore/on.rb +4 -3
- data/lib/flor/pcore/on_cancel.rb +1 -0
- data/lib/flor/pcore/on_error.rb +1 -0
- data/lib/flor/pcore/push.rb +1 -0
- data/lib/flor/pcore/rand.rb +1 -0
- data/lib/flor/pcore/range.rb +1 -0
- data/lib/flor/pcore/reduce.rb +1 -0
- data/lib/flor/pcore/return.rb +1 -0
- data/lib/flor/pcore/reverse.rb +1 -0
- data/lib/flor/pcore/select.rb +1 -0
- data/lib/flor/pcore/sequence.rb +1 -0
- data/lib/flor/pcore/set.rb +1 -0
- data/lib/flor/pcore/shuffle.rb +1 -0
- data/lib/flor/pcore/slice.rb +1 -0
- data/lib/flor/pcore/sort.rb +1 -0
- data/lib/flor/pcore/sort_by.rb +1 -0
- data/lib/flor/pcore/split.rb +1 -0
- data/lib/flor/pcore/stall.rb +1 -0
- data/lib/flor/pcore/strings.rb +1 -0
- data/lib/flor/pcore/timestamp.rb +1 -0
- data/lib/flor/pcore/to_array.rb +1 -0
- data/lib/flor/pcore/twig.rb +1 -0
- data/lib/flor/pcore/type_of.rb +1 -0
- data/lib/flor/pcore/until.rb +1 -0
- data/lib/flor/punit/abort.rb +50 -0
- data/lib/flor/punit/c_collect.rb +1 -0
- data/lib/flor/punit/c_each.rb +19 -0
- data/lib/flor/punit/c_for_each.rb +2 -1
- data/lib/flor/punit/c_iterator.rb +2 -1
- data/lib/flor/punit/c_map.rb +1 -0
- data/lib/flor/punit/cancel.rb +1 -0
- data/lib/flor/punit/concurrence.rb +7 -5
- data/lib/flor/punit/cron.rb +1 -0
- data/lib/flor/punit/do_trap.rb +1 -0
- data/lib/flor/punit/every.rb +1 -0
- data/lib/flor/punit/graft.rb +1 -0
- data/lib/flor/punit/{m_ram.rb → m_receive_and_merge.rb} +5 -4
- data/lib/flor/punit/on_timeout.rb +1 -0
- data/lib/flor/punit/part.rb +1 -0
- data/lib/flor/punit/schedule.rb +1 -0
- data/lib/flor/punit/signal.rb +1 -0
- data/lib/flor/punit/sleep.rb +1 -0
- data/lib/flor/punit/task.rb +1 -0
- data/lib/flor/punit/trace.rb +1 -0
- data/lib/flor/punit/trap.rb +10 -1
- data/lib/flor/to_string.rb +1 -0
- data/lib/flor/tools/env.rb +1 -0
- data/lib/flor/tools/firb.rb +33 -0
- data/lib/flor/tools/shell.rb +13 -3
- data/lib/flor/tools/shell_out.rb +1 -0
- data/lib/flor/tt.rb +98 -0
- data/lib/flor/unit.rb +3 -0
- data/lib/flor/unit/caller.rb +158 -23
- data/lib/flor/unit/caller_jruby.rb +133 -0
- data/lib/flor/unit/dump.rb +36 -0
- data/lib/flor/unit/executor.rb +19 -13
- data/lib/flor/unit/ganger.rb +9 -12
- data/lib/flor/unit/gangers.rb +125 -0
- data/lib/flor/unit/hloader.rb +34 -7
- data/lib/flor/unit/hook.rb +3 -0
- data/lib/flor/unit/hooker.rb +32 -15
- data/lib/flor/unit/journal.rb +23 -0
- data/lib/flor/unit/loader.rb +142 -15
- data/lib/flor/unit/logger.rb +35 -7
- data/lib/flor/unit/models.rb +8 -1
- data/lib/flor/unit/models/execution.rb +51 -0
- data/lib/flor/unit/models/message.rb +6 -0
- data/lib/flor/unit/models/pointer.rb +21 -1
- data/lib/flor/unit/models/timer.rb +1 -0
- data/lib/flor/unit/models/trace.rb +1 -0
- data/lib/flor/unit/models/trap.rb +3 -2
- data/lib/flor/unit/scheduler.rb +51 -36
- data/lib/flor/unit/spooler.rb +1 -0
- data/lib/flor/unit/storage.rb +113 -84
- data/lib/flor/unit/taskers.rb +70 -1
- data/lib/flor/unit/waiter.rb +22 -17
- data/lib/flor/unit/wlist.rb +19 -8
- metadata +16 -11
data/lib/flor/unit/models.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
3
|
module Flor
|
3
4
|
|
@@ -92,6 +93,12 @@ module Flor
|
|
92
93
|
end
|
93
94
|
end
|
94
95
|
|
96
|
+
alias to_dump_h to_h
|
97
|
+
#
|
98
|
+
# Downsteam, #to_h answers are more complete, more standalone
|
99
|
+
# whereas, #to_dump_h answers are more compact (see florist).
|
100
|
+
# For now, they're just the same, it specializes downstream.
|
101
|
+
|
95
102
|
class << self
|
96
103
|
|
97
104
|
def from_h(h)
|
@@ -135,7 +142,7 @@ module Flor
|
|
135
142
|
|
136
143
|
@models[key] ||=
|
137
144
|
parent_module.const_set(
|
138
|
-
"#{c}#{
|
145
|
+
"#{c}#{self.object_id.to_s.gsub('-', 'M')}",
|
139
146
|
Class.new(parent_module.const_get(c)) do
|
140
147
|
self.dataset = s.db["#{table_prefix}#{key}".to_sym]
|
141
148
|
self.unit = s.unit
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
3
|
module Flor
|
3
4
|
|
@@ -98,6 +99,56 @@ module Flor
|
|
98
99
|
h
|
99
100
|
end
|
100
101
|
|
102
|
+
def lookup_nodes(query, opts={})
|
103
|
+
|
104
|
+
@node_index ||= nodes
|
105
|
+
.sort_by { |k, _| k }
|
106
|
+
.inject({}) { |h, (k, v)|
|
107
|
+
#
|
108
|
+
# nid => [ node ]
|
109
|
+
#
|
110
|
+
h[k] = [ v ]
|
111
|
+
#
|
112
|
+
# code => [ node0, node1, ... ]
|
113
|
+
#
|
114
|
+
t =
|
115
|
+
(lookup_tree(v['parent'])[1][Flor.child_id(v['nid'])] rescue nil) ||
|
116
|
+
lookup_tree(v['nid'])
|
117
|
+
s = Flor.tree_to_flor(t, chop: true)
|
118
|
+
(h[s] ||= []) << v
|
119
|
+
#
|
120
|
+
# tag => [ node0, node1, ... ]
|
121
|
+
#
|
122
|
+
ts = v['tags']
|
123
|
+
ts.each { |t| (h[t] ||= []) << v } if ts
|
124
|
+
#
|
125
|
+
h }
|
126
|
+
|
127
|
+
@node_index
|
128
|
+
.select { |k, v|
|
129
|
+
case query
|
130
|
+
when Regexp then k.match(query)
|
131
|
+
else k == query
|
132
|
+
end }
|
133
|
+
.values
|
134
|
+
.flatten(1)
|
135
|
+
end
|
136
|
+
|
137
|
+
def lookup_node(query, opts={})
|
138
|
+
|
139
|
+
lookup_nodes(query, opts).first
|
140
|
+
end
|
141
|
+
|
142
|
+
def lookup_nids(query, opts={})
|
143
|
+
|
144
|
+
lookup_nodes(query, opts).collect { |n| n['nid'] }
|
145
|
+
end
|
146
|
+
|
147
|
+
def lookup_nid(query, opts={})
|
148
|
+
|
149
|
+
lookup_node(query, opts)['nid']
|
150
|
+
end
|
151
|
+
|
101
152
|
class << self
|
102
153
|
|
103
154
|
def by_status(s)
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
3
|
module Flor
|
3
4
|
|
@@ -29,7 +30,7 @@ module Flor
|
|
29
30
|
#end
|
30
31
|
|
31
32
|
# If the pointer is a "var" pointer, returns the full value
|
32
|
-
# for the variable, as
|
33
|
+
# for the variable, as found in the execution's node "0".
|
33
34
|
#
|
34
35
|
def full_value
|
35
36
|
|
@@ -37,6 +38,25 @@ module Flor
|
|
37
38
|
|
38
39
|
node['vars'][name]
|
39
40
|
end
|
41
|
+
|
42
|
+
def attd
|
43
|
+
|
44
|
+
data['atts'].inject({}) { |h, (k, v)| h[k] = v if k; h }
|
45
|
+
|
46
|
+
rescue; []
|
47
|
+
end
|
48
|
+
|
49
|
+
def attl
|
50
|
+
|
51
|
+
data['atts'].inject([]) { |a, (k, v)| a << v if k == nil; a }
|
52
|
+
|
53
|
+
rescue; []
|
54
|
+
end
|
55
|
+
|
56
|
+
def att_texts
|
57
|
+
|
58
|
+
attl.select { |e| e.is_a?(String) }
|
59
|
+
end
|
40
60
|
end
|
41
61
|
end
|
42
62
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
3
|
module Flor
|
3
4
|
|
@@ -124,7 +125,7 @@ module Flor
|
|
124
125
|
'exid' => self.exid,
|
125
126
|
'nid' => self.nid,
|
126
127
|
'type' => 'trap',
|
127
|
-
'trap' =>
|
128
|
+
'trap' => to_trap_h,
|
128
129
|
'trap_id' => self.id,
|
129
130
|
'message' => msg,
|
130
131
|
'sm' => message['m'] }
|
@@ -133,7 +134,7 @@ module Flor
|
|
133
134
|
#'dbg' => xx }
|
134
135
|
end
|
135
136
|
|
136
|
-
def
|
137
|
+
def to_trap_h
|
137
138
|
|
138
139
|
values
|
139
140
|
.inject({}) { |h, (k, v)| h[k.to_s] = v if k != :content; h }
|
data/lib/flor/unit/scheduler.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
3
|
module Flor
|
3
4
|
|
@@ -9,6 +10,7 @@ module Flor
|
|
9
10
|
attr_reader :logger
|
10
11
|
|
11
12
|
attr_reader :thread_status
|
13
|
+
attr_reader :last_queued_message_id
|
12
14
|
|
13
15
|
attr_reader :archive
|
14
16
|
|
@@ -136,11 +138,7 @@ module Flor
|
|
136
138
|
|
137
139
|
# TODO heartbeat, every x minutes, when idle, log something
|
138
140
|
|
139
|
-
|
140
|
-
"database not ready, " +
|
141
|
-
"db ver: #{@storage.db_version.inspect}, " +
|
142
|
-
"mig ver: #{@storage.migration_version}"
|
143
|
-
) if !! @conf['sto_migration_check'] && @storage.ready?
|
141
|
+
check_migration_version
|
144
142
|
|
145
143
|
@thread_status = :running
|
146
144
|
|
@@ -160,6 +158,15 @@ module Flor
|
|
160
158
|
self
|
161
159
|
end
|
162
160
|
|
161
|
+
def check_migration_version
|
162
|
+
|
163
|
+
fail(
|
164
|
+
"database not ready, " +
|
165
|
+
"db ver: #{@storage.db_version.inspect}, " +
|
166
|
+
"mig ver: #{@storage.migration_version}"
|
167
|
+
) if !! @conf['sto_migration_check'] && @storage.ready?
|
168
|
+
end
|
169
|
+
|
163
170
|
def stop
|
164
171
|
|
165
172
|
@thread_status = :stop
|
@@ -222,7 +229,11 @@ module Flor
|
|
222
229
|
|
223
230
|
def queue(message, opts={})
|
224
231
|
|
225
|
-
@
|
232
|
+
@last_queued_message_id =
|
233
|
+
@storage.put_message(message)
|
234
|
+
|
235
|
+
# Nota bene:
|
236
|
+
# the #wait method is added to the Scheduler by Flor::WaitList
|
226
237
|
|
227
238
|
if opts[:wait]
|
228
239
|
wait(message['exid'], opts)
|
@@ -356,9 +367,13 @@ module Flor
|
|
356
367
|
) unless nid
|
357
368
|
|
358
369
|
fail ArgumentError.new(
|
359
|
-
"cannot add iteration to
|
370
|
+
"cannot add iteration to node #{nid.inspect} not present in tree"
|
360
371
|
) unless exe.lookup_tree(nid)
|
361
372
|
|
373
|
+
fail ArgumentError.new(
|
374
|
+
"cannot add iteration to node #{nid.inspect} not present in execution"
|
375
|
+
) unless exe.nodes.has_key?(nid)
|
376
|
+
|
362
377
|
queue(msg, opts)
|
363
378
|
end
|
364
379
|
alias add_iteration add_iterations
|
@@ -434,16 +449,14 @@ module Flor
|
|
434
449
|
#
|
435
450
|
# unit.dump() { |h| ... } # modify the has right before it's turned to JSON
|
436
451
|
#
|
452
|
+
# unit.dump(hash: true) # returns the hash (instead of JSONing it)
|
453
|
+
#
|
437
454
|
def dump(io=nil, opts=nil, &block)
|
438
455
|
|
439
456
|
io, opts = nil, io if io.is_a?(Hash)
|
440
457
|
opts ||= {}
|
441
458
|
|
442
|
-
|
443
|
-
#
|
444
|
-
exis = o[:exid]
|
445
|
-
doms = o[:domain]
|
446
|
-
sdms = o[:strict_domain] || o[:sdomain]
|
459
|
+
exis, doms, sdms = extract_dump_and_load_filters(opts)
|
447
460
|
#
|
448
461
|
filter = lambda { |q|
|
449
462
|
q = q.where(
|
@@ -458,25 +471,25 @@ module Flor
|
|
458
471
|
domain: sdms) if sdms
|
459
472
|
q }
|
460
473
|
|
461
|
-
|
474
|
+
hash =
|
462
475
|
storage.db.transaction {
|
463
|
-
[ filter[executions].collect(&:to_h),
|
464
|
-
filter[timers].collect(&:to_h),
|
465
|
-
filter[traps].collect(&:to_h),
|
466
|
-
filter[pointers].collect(&:to_h) ] }
|
467
476
|
|
468
|
-
|
477
|
+
h = {
|
478
|
+
timestamp: Flor.tstamp,
|
479
|
+
executions: filter[executions].collect(&:to_h),
|
480
|
+
timers: filter[timers].collect(&:to_h),
|
481
|
+
traps: filter[traps].collect(&:to_h),
|
482
|
+
pointers: filter[pointers].collect(&:to_h) }
|
469
483
|
|
470
|
-
|
471
|
-
timestamp: Flor.tstamp,
|
472
|
-
executions: exs,
|
473
|
-
timers: tms,
|
474
|
-
traps: tps,
|
475
|
-
pointers: pts }
|
484
|
+
block.call(h) if block
|
476
485
|
|
477
|
-
|
486
|
+
h }
|
478
487
|
|
479
|
-
|
488
|
+
return hash if opts[:hash] || opts[:h]
|
489
|
+
|
490
|
+
o = io ? io : StringIO.new
|
491
|
+
|
492
|
+
JSON.dump(hash, o)
|
480
493
|
|
481
494
|
io ? io : o.string
|
482
495
|
end
|
@@ -507,12 +520,7 @@ module Flor
|
|
507
520
|
mks = DUMP_KEYS - h.keys
|
508
521
|
fail Flor::FlorError.new("missing keys #{mks.inspect}") if mks.any?
|
509
522
|
|
510
|
-
|
511
|
-
#
|
512
|
-
exis = o[:exid]
|
513
|
-
doms = o[:domain]
|
514
|
-
sdms = o[:strict_domain] || o[:sdomain]
|
515
|
-
#
|
523
|
+
exis, doms, sdms = extract_dump_and_load_filters(opts)
|
516
524
|
doms = doms.collect { |d| /\A#{d}(\.#{Flor::NAME_REX})*\z/ } if doms
|
517
525
|
|
518
526
|
counts = { executions: 0, timers: 0, traps: 0, pointers: 0, total: 0 }
|
@@ -541,7 +549,7 @@ module Flor
|
|
541
549
|
cla.import(cols, rows) if rows.any?
|
542
550
|
end
|
543
551
|
|
544
|
-
block.call(h) if block
|
552
|
+
block.call(h, counts) if block
|
545
553
|
end
|
546
554
|
|
547
555
|
counts
|
@@ -549,6 +557,13 @@ module Flor
|
|
549
557
|
|
550
558
|
protected
|
551
559
|
|
560
|
+
def extract_dump_and_load_filters(opts)
|
561
|
+
|
562
|
+
o = lambda { |k| v = opts[k] || opts["#{k}s".to_sym]; v ? Array(v) : nil }
|
563
|
+
|
564
|
+
[ o[:exid], o[:domain], o[:strict_domain] || o[:sdomain] ]
|
565
|
+
end
|
566
|
+
|
552
567
|
def tick
|
553
568
|
|
554
569
|
t0 = Time.now
|
@@ -580,7 +595,7 @@ module Flor
|
|
580
595
|
|
581
596
|
rescue Exception => ex
|
582
597
|
|
583
|
-
puts
|
598
|
+
puts(on_start_exc(ex))
|
584
599
|
end
|
585
600
|
|
586
601
|
def prepare_message(point, args)
|
@@ -668,9 +683,9 @@ module Flor
|
|
668
683
|
def should_wake_up?
|
669
684
|
|
670
685
|
return true if @wake_up
|
671
|
-
return true if Time.now - @reloaded_at >= reload_after
|
686
|
+
return true if (Time.now - @reloaded_at) >= reload_after
|
672
687
|
|
673
|
-
@next_time && @next_time <= Flor.
|
688
|
+
@next_time && (@next_time <= Flor.tstam)
|
674
689
|
end
|
675
690
|
|
676
691
|
def unreserve_messages
|
data/lib/flor/unit/spooler.rb
CHANGED
data/lib/flor/unit/storage.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
3
|
require 'zlib'
|
3
4
|
|
@@ -6,6 +7,14 @@ module Flor
|
|
6
7
|
|
7
8
|
class Storage
|
8
9
|
|
10
|
+
MESSAGE_COLUMNS = [
|
11
|
+
:domain, :exid, :point, :content,
|
12
|
+
:status, :ctime, :mtime, :cunit, :munit
|
13
|
+
].freeze
|
14
|
+
POINTER_COLUMNS = [
|
15
|
+
:domain, :exid, :nid, :type, :name, :value, :ctime, :cunit
|
16
|
+
].freeze
|
17
|
+
|
9
18
|
attr_reader :unit, :db, :models
|
10
19
|
|
11
20
|
attr_reader :mutex
|
@@ -30,9 +39,11 @@ module Flor
|
|
30
39
|
#p [ :disconnected, @db.object_id ]
|
31
40
|
end
|
32
41
|
|
33
|
-
def db_version
|
42
|
+
def db_version(opts={})
|
43
|
+
|
44
|
+
table, column = migration_table_and_column(opts)
|
34
45
|
|
35
|
-
(@db[
|
46
|
+
(@db[table].first rescue {})[column]
|
36
47
|
end
|
37
48
|
|
38
49
|
def migration_version
|
@@ -79,34 +90,27 @@ module Flor
|
|
79
90
|
opts[:target] ||= to if to.is_a?(Integer)
|
80
91
|
opts[:current] ||= from if from.is_a?(Integer)
|
81
92
|
|
82
|
-
opts[:table] = (
|
83
|
-
|
84
|
-
|
85
|
-
:schema_info
|
86
|
-
|
87
|
-
@unit.conf['db_migration_column'] ||
|
88
|
-
@unit.conf['sto_migration_column'] ||
|
89
|
-
:version).to_sym
|
90
|
-
#
|
91
|
-
# defaults for the migration version table:
|
92
|
-
# { table: :schema_info,
|
93
|
-
# column: :version }
|
93
|
+
opts[:table], opts[:column] = migration_table_and_column(opts)
|
94
|
+
#
|
95
|
+
# defaults for the migration version table:
|
96
|
+
# { table: :schema_info,
|
97
|
+
# column: :version }
|
94
98
|
|
95
99
|
skip =
|
100
|
+
opts[:sparse_migrations] ||
|
96
101
|
@unit.conf['db_sparse_migrations'] ||
|
97
|
-
@unit.conf['sto_sparse_migrations']
|
98
|
-
opts[:sparse_migrations]
|
102
|
+
@unit.conf['sto_sparse_migrations']
|
99
103
|
if skip && ! opts.has_key?(:allow_missing_migration_files)
|
100
104
|
opts[:allow_missing_migration_files] = true
|
101
105
|
end
|
102
106
|
|
103
107
|
dir =
|
108
|
+
opts[:migrations] ||
|
109
|
+
opts[:migration_dir] ||
|
104
110
|
@unit.conf['db_migrations'] ||
|
105
111
|
@unit.conf['db_migration_dir'] ||
|
106
112
|
@unit.conf['sto_migrations'] ||
|
107
113
|
@unit.conf['sto_migration_dir'] ||
|
108
|
-
opts[:migrations] ||
|
109
|
-
opts[:migration_dir] ||
|
110
114
|
Flor.migration_dir
|
111
115
|
|
112
116
|
synchronize do
|
@@ -336,35 +340,39 @@ module Flor
|
|
336
340
|
n = Flor.tstamp
|
337
341
|
u = @unit.identifier
|
338
342
|
|
339
|
-
|
343
|
+
id =
|
344
|
+
synchronize(syn) do
|
340
345
|
|
341
|
-
|
346
|
+
stored, unstored = ms.partition { |m| m['mid'] }
|
342
347
|
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
348
|
+
#
|
349
|
+
# de-reserve any previously stored message, might happen
|
350
|
+
# for "terminated" messages that got queued back to let
|
351
|
+
# other messages get processed
|
347
352
|
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
353
|
+
@db[:flor_messages]
|
354
|
+
.where(id: stored.collect { |m| m['mid'] })
|
355
|
+
.update(status: 'created', mtime: n, munit: u) \
|
356
|
+
if stored.any?
|
352
357
|
|
353
|
-
|
354
|
-
|
358
|
+
#
|
359
|
+
# store new messages
|
355
360
|
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
361
|
+
@db[:flor_messages]
|
362
|
+
.import(
|
363
|
+
MESSAGE_COLUMNS,
|
364
|
+
unstored.map { |m|
|
365
|
+
[ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
|
366
|
+
'created', n, n, u, u ] }) \
|
367
|
+
if unstored.any?
|
368
|
+
|
369
|
+
@db[:flor_messages].max(:id)
|
370
|
+
end
|
365
371
|
|
366
372
|
@unit.wake_up
|
367
373
|
|
374
|
+
id
|
375
|
+
|
368
376
|
rescue => err
|
369
377
|
|
370
378
|
Thread.current[:sto_errored_items] = ms
|
@@ -516,25 +524,6 @@ module Flor
|
|
516
524
|
end
|
517
525
|
end
|
518
526
|
|
519
|
-
def put_task_pointer(msg, tname, tconf)
|
520
|
-
|
521
|
-
exid = msg['exid']
|
522
|
-
dom = Flor.domain(exid)
|
523
|
-
|
524
|
-
synchronize do
|
525
|
-
|
526
|
-
@db[:flor_pointers]
|
527
|
-
.insert(
|
528
|
-
domain: dom,
|
529
|
-
exid: exid,
|
530
|
-
nid: msg['nid'],
|
531
|
-
type: 'tasker',
|
532
|
-
name: tname,
|
533
|
-
ctime: Flor.tstamp,
|
534
|
-
cunit: @unit.identifier)
|
535
|
-
end
|
536
|
-
end
|
537
|
-
|
538
527
|
def fetch_next_time
|
539
528
|
|
540
529
|
t =
|
@@ -557,6 +546,18 @@ module Flor
|
|
557
546
|
|
558
547
|
protected
|
559
548
|
|
549
|
+
def migration_table_and_column(opts={})
|
550
|
+
|
551
|
+
[ (opts[:migration_table] ||
|
552
|
+
@unit.conf['db_migration_table'] ||
|
553
|
+
@unit.conf['sto_migration_table'] ||
|
554
|
+
:schema_info).to_sym,
|
555
|
+
(opts[:migration_column] ||
|
556
|
+
@unit.conf['db_migration_column'] ||
|
557
|
+
@unit.conf['sto_migration_column'] ||
|
558
|
+
:version).to_sym ]
|
559
|
+
end
|
560
|
+
|
560
561
|
def _commaify(o)
|
561
562
|
|
562
563
|
if Flor.is_regex_tree?(o)
|
@@ -585,14 +586,13 @@ module Flor
|
|
585
586
|
|
586
587
|
@db[:flor_messages]
|
587
588
|
.where(
|
588
|
-
id: messages.collect { |m| m['mid'] }.compact)
|
589
|
+
id: messages.collect { |m| m['mid'] }.uniq.compact)
|
589
590
|
.update(
|
590
591
|
status: 'consumed', mtime: n, munit: u)
|
591
592
|
|
592
593
|
@db[:flor_messages]
|
593
594
|
.import(
|
594
|
-
|
595
|
-
:status, :ctime, :mtime, :cunit, :munit ],
|
595
|
+
MESSAGE_COLUMNS,
|
596
596
|
messages
|
597
597
|
.select { |m|
|
598
598
|
! m['mid'] && POINTS_TO_ARCHIVE.include?(m['point']) }
|
@@ -608,19 +608,16 @@ module Flor
|
|
608
608
|
|
609
609
|
@db[:flor_messages]
|
610
610
|
.where(
|
611
|
-
id: messages.collect { |m| m['mid'] }.compact)
|
611
|
+
id: messages.collect { |m| m['mid'] }.uniq.compact)
|
612
612
|
.delete
|
613
613
|
end
|
614
614
|
end
|
615
615
|
|
616
616
|
def load_timers
|
617
617
|
|
618
|
-
now = Flor.tstamp
|
619
|
-
no = now[0, now.rindex('.')]
|
620
|
-
|
621
618
|
timers
|
622
619
|
.where(status: 'active')
|
623
|
-
.where { ntime <=
|
620
|
+
.where { ntime <= Flor.tstam }
|
624
621
|
.order(:ntime)
|
625
622
|
.all
|
626
623
|
|
@@ -690,9 +687,9 @@ module Flor
|
|
690
687
|
|
691
688
|
def update_pointers(exe, status, now)
|
692
689
|
|
693
|
-
# Q
|
694
|
-
#
|
695
|
-
#
|
690
|
+
# Q Should we archive old pointers?
|
691
|
+
# A Well, it might be better to only archive the execution and leave
|
692
|
+
# in there enough information...
|
696
693
|
|
697
694
|
exid = exe['exid']
|
698
695
|
|
@@ -718,23 +715,25 @@ module Flor
|
|
718
715
|
|
719
716
|
ts = node['tags']
|
720
717
|
ts.each { |t|
|
721
|
-
a << [ dom, exid, nid, 'tag', t, nil, now, u ] } if ts
|
718
|
+
a << [ dom, exid, nid, 'tag', t, nil, now, u, nil ] } if ts
|
722
719
|
|
723
720
|
vs = nid == '0' ? node['vars'] : nil
|
724
721
|
vs.each { |k, v|
|
725
722
|
case v; when Numeric, String, TrueClass, FalseClass, NilClass
|
726
|
-
a << [ dom, exid, '0', 'var', k, v.to_s, now, u ]
|
723
|
+
a << [ dom, exid, '0', 'var', k, v.to_s, now, u, nil ]
|
727
724
|
when Array, Hash
|
728
725
|
s = '(array)'; s = '(object)' if v.is_a?(Hash)
|
729
|
-
a << [ dom, exid, '0', 'var', k, s, now, u ]
|
726
|
+
a << [ dom, exid, '0', 'var', k, s, now, u, nil ]
|
730
727
|
else
|
731
|
-
a << [ dom, exid, '0', 'var', k, nil, now, u ]
|
728
|
+
a << [ dom, exid, '0', 'var', k, nil, now, u, nil ]
|
732
729
|
end } if vs
|
733
730
|
|
734
|
-
|
735
|
-
|
736
|
-
|
737
|
-
|
731
|
+
if ta = node['task']
|
732
|
+
tasker = ta['tasker']
|
733
|
+
name = ta['name']
|
734
|
+
content = { message: node['message'], atts: node['atts'] }
|
735
|
+
a << [ dom, exid, nid, 'tasker', tasker, name, now, u, content ]
|
736
|
+
end
|
738
737
|
|
739
738
|
a }
|
740
739
|
|
@@ -742,17 +741,35 @@ module Flor
|
|
742
741
|
.where(exid: exid)
|
743
742
|
.select(:nid, :type, :name)
|
744
743
|
.all
|
745
|
-
pointers.reject! { |_, _, ni, ty, na, _, _, _|
|
744
|
+
pointers.reject! { |_, _, ni, ty, na, _, _, _, _|
|
746
745
|
cps.find { |cp| cp[:nid] == ni && cp[:type] == ty && cp[:name] == na } }
|
747
746
|
#
|
748
747
|
# don't insert when already inserted
|
749
748
|
|
749
|
+
if pointer_columns.include?(:content)
|
750
|
+
pointers.each { |ptr|
|
751
|
+
c = ptr[8]; ptr[8] = to_blob(c) if c }
|
752
|
+
else
|
753
|
+
pointers.each { |ptr|
|
754
|
+
ptr.pop }
|
755
|
+
end
|
756
|
+
|
750
757
|
@db[:flor_pointers]
|
751
758
|
.import(
|
752
|
-
|
759
|
+
pointer_columns,
|
753
760
|
pointers)
|
754
761
|
end
|
755
762
|
|
763
|
+
def pointer_columns
|
764
|
+
|
765
|
+
@pointer_columns ||=
|
766
|
+
if @db[:flor_pointers].columns.include?(:content)
|
767
|
+
POINTER_COLUMNS + [ :content ]
|
768
|
+
else
|
769
|
+
POINTER_COLUMNS
|
770
|
+
end
|
771
|
+
end
|
772
|
+
|
756
773
|
def determine_type_and_schedule(message)
|
757
774
|
|
758
775
|
t, s = message['type'], message['string']
|
@@ -798,15 +815,27 @@ module Flor
|
|
798
815
|
def error(msg); @unit.logger.db_log(:error, msg); end
|
799
816
|
end
|
800
817
|
|
801
|
-
def
|
818
|
+
def derive_db
|
819
|
+
|
820
|
+
db = @unit.conf['sto_db']
|
821
|
+
|
822
|
+
return db if db
|
802
823
|
|
803
824
|
uri = @unit.conf['sto_uri']
|
804
825
|
|
805
|
-
|
806
|
-
|
807
|
-
|
826
|
+
fail ArgumentError.new("no 'sto_uri' conf, cannot connect to db") \
|
827
|
+
unless uri
|
828
|
+
|
829
|
+
return Kernel.const_get(uri) \
|
830
|
+
if uri.is_a?(String) && uri.match(/\A[A-Z]+\z/)
|
831
|
+
# for cases where uri == 'DB'
|
832
|
+
|
833
|
+
Sequel.connect(uri)
|
834
|
+
end
|
835
|
+
|
836
|
+
def connect
|
808
837
|
|
809
|
-
@db =
|
838
|
+
@db = derive_db
|
810
839
|
|
811
840
|
class << @db; attr_accessor :flor_unit; end
|
812
841
|
@db.flor_unit = @unit
|