flor 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +12 -0
- data/README.md +5 -1
- data/fail.txt +3 -0
- data/lib/flor.rb +1 -1
- data/lib/flor/conf.rb +1 -1
- data/lib/flor/core/executor.rb +34 -6
- data/lib/flor/core/node.rb +15 -1
- data/lib/flor/core/texecutor.rb +19 -4
- data/lib/flor/djan.rb +1 -1
- data/lib/flor/flor.rb +37 -2
- data/lib/flor/parser.rb +55 -29
- data/lib/flor/pcore/apply.rb +23 -24
- data/lib/flor/pcore/case.rb +24 -26
- data/lib/flor/pcore/fail.rb +10 -0
- data/lib/flor/pcore/if.rb +36 -1
- data/lib/flor/pcore/twig.rb +32 -0
- data/lib/flor/pcore/val.rb +1 -1
- data/lib/flor/punit/graft.rb +39 -1
- data/lib/flor/unit.rb +3 -0
- data/lib/flor/unit/executor.rb +1 -0
- data/lib/flor/unit/ganger.rb +19 -68
- data/lib/flor/unit/hook.rb +32 -0
- data/lib/flor/unit/hooker.rb +5 -12
- data/lib/flor/unit/hooks.rb +37 -0
- data/lib/flor/unit/loader.rb +44 -17
- data/lib/flor/unit/logger.rb +1 -0
- data/lib/flor/unit/models/trap.rb +22 -26
- data/lib/flor/unit/runner.rb +84 -0
- data/lib/flor/unit/scheduler.rb +57 -46
- data/lib/flor/unit/spooler.rb +109 -0
- data/lib/flor/unit/storage.rb +1 -1
- data/lib/flor/unit/waiter.rb +2 -4
- data/lib/flor/unit/wlist.rb +13 -0
- metadata +8 -2
data/lib/flor/unit/hooker.rb
CHANGED
@@ -53,7 +53,9 @@ module Flor
|
|
53
53
|
|
54
54
|
def notify(executor, message)
|
55
55
|
|
56
|
-
(
|
56
|
+
(
|
57
|
+
@hooks + executor.traps_and_hooks
|
58
|
+
)
|
57
59
|
.inject([]) do |a, (_, opts, hook, block)|
|
58
60
|
# name of hook is piped into "_" oblivion
|
59
61
|
|
@@ -64,17 +66,8 @@ module Flor
|
|
64
66
|
executor.trigger_trap(hook, message)
|
65
67
|
elsif hook
|
66
68
|
executor.trigger_hook(hook, message)
|
67
|
-
else
|
68
|
-
|
69
|
-
if block.arity == 1
|
70
|
-
block.call(message)
|
71
|
-
elsif block.arity == 2
|
72
|
-
block.call(message, opts)
|
73
|
-
else
|
74
|
-
block.call(executor, message, opts)
|
75
|
-
end
|
76
|
-
r.is_a?(Array) && r.all? { |e| e.is_a?(Hash) } ? r : []
|
77
|
-
# be lenient with block hooks, help them return an array
|
69
|
+
else
|
70
|
+
executor.trigger_block(block, opts, message)
|
78
71
|
end)
|
79
72
|
end
|
80
73
|
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
|
2
|
+
module Flor
|
3
|
+
|
4
|
+
class BasicHook
|
5
|
+
|
6
|
+
def initialize(executor, opts, message)
|
7
|
+
|
8
|
+
@executor = executor
|
9
|
+
@opts = opts
|
10
|
+
@message = message
|
11
|
+
end
|
12
|
+
|
13
|
+
protected
|
14
|
+
|
15
|
+
def point
|
16
|
+
|
17
|
+
@message['point']
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class PointHook < BasicHook
|
22
|
+
|
23
|
+
def on
|
24
|
+
|
25
|
+
m = "on_#{point}"
|
26
|
+
|
27
|
+
respond_to?(m) ? send(m) : on_default
|
28
|
+
end
|
29
|
+
|
30
|
+
protected
|
31
|
+
|
32
|
+
def on_default
|
33
|
+
|
34
|
+
[]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/flor/unit/loader.rb
CHANGED
@@ -11,6 +11,8 @@ module Flor
|
|
11
11
|
|
12
12
|
@cache = {}
|
13
13
|
@mutex = Mutex.new
|
14
|
+
|
15
|
+
@root = File.absolute_path(@unit.conf['lod_path'] || @unit.conf['root'])
|
14
16
|
end
|
15
17
|
|
16
18
|
def shutdown
|
@@ -18,7 +20,7 @@ module Flor
|
|
18
20
|
|
19
21
|
def variables(domain)
|
20
22
|
|
21
|
-
Dir[File.join(root, '**/*.json')]
|
23
|
+
Dir[File.join(@root, '**/*.json')]
|
22
24
|
.select { |f| f.index('/etc/variables/') }
|
23
25
|
.collect { |pa| [ pa, expose_d(pa, {}) ] }
|
24
26
|
.select { |pa, d| is_subdomain?(domain, d) }
|
@@ -39,7 +41,11 @@ module Flor
|
|
39
41
|
domain, name, opts = [ domain, nil, name ] if name.is_a?(Hash)
|
40
42
|
domain, name = split_dn(domain, name)
|
41
43
|
|
42
|
-
|
44
|
+
if m = name.match(/(\.flor?)\z/)
|
45
|
+
name = name[0..m[1].length - 1]
|
46
|
+
end
|
47
|
+
|
48
|
+
path, d, n = (Dir[File.join(@root, '**/*.{flo,flor}')])
|
43
49
|
.select { |f| f.index('/lib/') }
|
44
50
|
.collect { |pa| [ pa, *expose_dn(pa, opts) ] }
|
45
51
|
.select { |pa, d, n| n == name && is_subdomain?(domain, d) }
|
@@ -51,9 +57,12 @@ module Flor
|
|
51
57
|
|
52
58
|
def tasker(domain, name=nil)
|
53
59
|
|
60
|
+
# NB: do not relativize path, because Ruby load path != cwd,
|
61
|
+
# stay absolute for `require` and `load`
|
62
|
+
|
54
63
|
domain, name = split_dn(domain, name)
|
55
64
|
|
56
|
-
path, d, n = Dir[File.join(root, '**/*.json')]
|
65
|
+
path, d, n = Dir[File.join(@root, '**/*.json')]
|
57
66
|
.select { |pa| pa.index('/lib/taskers/') }
|
58
67
|
.collect { |pa| [ pa, *expose_dn(pa, {}) ] }
|
59
68
|
.select { |pa, d, n|
|
@@ -72,7 +81,34 @@ module Flor
|
|
72
81
|
|
73
82
|
return nil unless con
|
74
83
|
|
75
|
-
|
84
|
+
pa = conf['_path']
|
85
|
+
|
86
|
+
if con.is_a?(Array)
|
87
|
+
con.each { |c| c['_path'] = pa }
|
88
|
+
else
|
89
|
+
con['_path'] = pa
|
90
|
+
end
|
91
|
+
|
92
|
+
con
|
93
|
+
end
|
94
|
+
|
95
|
+
def hooks(domain, name=nil)
|
96
|
+
|
97
|
+
Dir[File.join(@root, '**/*.json')]
|
98
|
+
.select { |f| f.index('/lib/hooks/') }
|
99
|
+
.collect { |pa| [ pa, expose_d(pa, {}) ] }
|
100
|
+
.select { |pa, d| is_subdomain?(domain, d) }
|
101
|
+
.sort_by { |pa, d| d.count('.') }
|
102
|
+
.collect { |pa, d|
|
103
|
+
interpret(pa).each_with_index { |h, i|
|
104
|
+
h['_path'] = Flor.relativize_path(pa) + ":#{i}" } }
|
105
|
+
.flatten(1)
|
106
|
+
end
|
107
|
+
|
108
|
+
def load_hooks(exid)
|
109
|
+
|
110
|
+
hooks(Flor.domain(exid))
|
111
|
+
.collect { |h| Flor::Hook.new(@unit, exid, h) }
|
76
112
|
end
|
77
113
|
|
78
114
|
protected
|
@@ -98,19 +134,19 @@ module Flor
|
|
98
134
|
|
99
135
|
def expose_d(path, opts)
|
100
136
|
|
101
|
-
pa = path[root.length..-1]
|
137
|
+
pa = path[@root.length..-1]
|
102
138
|
pa = pa[5..-1] if pa[0, 5] == '/usr/'
|
103
139
|
|
104
140
|
libregex =
|
105
141
|
opts[:subflows] ?
|
106
|
-
/\/lib\/(subflows|flows|taskers)\// :
|
107
|
-
/\/lib\/(flows|taskers)\//
|
142
|
+
/\/lib\/(subflows|flows|hooks|taskers)\// :
|
143
|
+
/\/lib\/(flows|hooks|taskers)\//
|
108
144
|
|
109
145
|
pa
|
110
146
|
.sub(/\/etc\/variables\//, '/')
|
111
147
|
.sub(libregex, '/')
|
112
148
|
.sub(/\/\z/, '')
|
113
|
-
.sub(/\/(flo|flor|dot)\.json\z/, '')
|
149
|
+
.sub(/\/(flo|flor|dot|hooks)\.json\z/, '')
|
114
150
|
.sub(/\.(flo|flor|json)\z/, '')
|
115
151
|
.sub(/\A\//, '')
|
116
152
|
.gsub(/\//, '.')
|
@@ -127,15 +163,6 @@ module Flor
|
|
127
163
|
end
|
128
164
|
end
|
129
165
|
|
130
|
-
def root
|
131
|
-
|
132
|
-
if lp = @unit.conf['lod_path']
|
133
|
-
File.absolute_path(lp)
|
134
|
-
else
|
135
|
-
File.dirname(File.absolute_path(@unit.conf['_path'] + '/..'))
|
136
|
-
end
|
137
|
-
end
|
138
|
-
|
139
166
|
def interpret(path)
|
140
167
|
|
141
168
|
@mutex.synchronize do
|
data/lib/flor/unit/logger.rb
CHANGED
@@ -5,33 +5,29 @@ module Flor
|
|
5
5
|
|
6
6
|
def to_hook
|
7
7
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
opts[:exid] = exid
|
30
|
-
opts[:subnid] = true
|
31
|
-
end
|
8
|
+
opts = {}
|
9
|
+
|
10
|
+
opts[:consumed] = tconsumed
|
11
|
+
|
12
|
+
opts[:point] = tpoints.split(',') if tpoints
|
13
|
+
opts[:heap] = theaps.split(',') if theaps
|
14
|
+
opts[:heat] = theats.split(',') if theats
|
15
|
+
|
16
|
+
opts[:name] = data['names']
|
17
|
+
|
18
|
+
case trange
|
19
|
+
when 'execution'
|
20
|
+
opts[:exid] = exid
|
21
|
+
when 'subdomain'
|
22
|
+
opts[:subdomain] = Flor.domain(exid)
|
23
|
+
when 'domain'
|
24
|
+
opts[:domain] = Flor.domain(exid)
|
25
|
+
else #'subnid' # default
|
26
|
+
opts[:exid] = exid
|
27
|
+
opts[:subnid] = true
|
28
|
+
end
|
32
29
|
|
33
|
-
|
34
|
-
end
|
30
|
+
[ "trap#{id}", opts, self, nil ]
|
35
31
|
end
|
36
32
|
|
37
33
|
def trigger(executor, message)
|
@@ -0,0 +1,84 @@
|
|
1
|
+
|
2
|
+
module Flor
|
3
|
+
|
4
|
+
class Runner
|
5
|
+
|
6
|
+
# NB: tasker configuration entries start with "rnr_"
|
7
|
+
|
8
|
+
def initialize(unit)
|
9
|
+
|
10
|
+
@unit = unit
|
11
|
+
end
|
12
|
+
|
13
|
+
def shutdown
|
14
|
+
end
|
15
|
+
|
16
|
+
def run(service, conf, message)
|
17
|
+
|
18
|
+
return ruby_run(service, conf, message) if conf['class'] || conf['module']
|
19
|
+
return cmd_run(service, conf, message) if conf['cmd']
|
20
|
+
|
21
|
+
fail ArgumentError.new("don't know how to run item at #{conf['_path']}")
|
22
|
+
end
|
23
|
+
|
24
|
+
protected
|
25
|
+
|
26
|
+
def fjoin(root, path)
|
27
|
+
|
28
|
+
root == '.' ? path : File.join(root, path)
|
29
|
+
end
|
30
|
+
|
31
|
+
def ruby_run(service, conf, message)
|
32
|
+
|
33
|
+
root = File.dirname(conf['_path'])
|
34
|
+
|
35
|
+
Flor.h_fetch_a(conf, 'require').each { |pa|
|
36
|
+
fail ArgumentError.new('".." not allowed in paths') if pa =~ /\.\./
|
37
|
+
require(fjoin(root, pa)) }
|
38
|
+
Flor.h_fetch_a(conf, 'load').each { |pa|
|
39
|
+
fail ArgumentError.new('".." not allowed in paths') if pa =~ /\.\./
|
40
|
+
load(fjoin(root, pa)) }
|
41
|
+
|
42
|
+
k = Flor.const_lookup(conf['class'] || conf['module'])
|
43
|
+
|
44
|
+
o =
|
45
|
+
if k.class == Module
|
46
|
+
k
|
47
|
+
else
|
48
|
+
case k.instance_method(:initialize).arity
|
49
|
+
when 1 then k.new(service)
|
50
|
+
when 2 then k.new(service, conf)
|
51
|
+
when 3 then k.new(service, conf, message)
|
52
|
+
when -1 then k.new({
|
53
|
+
service: service, configuration: conf, message: message })
|
54
|
+
else k.new
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
p = message['point']
|
59
|
+
m = :on
|
60
|
+
m = "on_#{p}" if ! o.respond_to?(m)
|
61
|
+
m = p if ! o.respond_to?(m)
|
62
|
+
m = :cancel if m == 'detask' && ! o.respond_to?(m)
|
63
|
+
|
64
|
+
fail(
|
65
|
+
"#{k.class.to_s.downcase} #{k} doesn't respond to on, on_#{p} or #{p}"
|
66
|
+
) if ! o.respond_to?(m)
|
67
|
+
|
68
|
+
case o.method(m).arity
|
69
|
+
when 1 then o.send(m, message)
|
70
|
+
when 2 then o.send(m, conf, message)
|
71
|
+
when 3 then o.send(m, executor, conf, message)
|
72
|
+
when -1 then o.send(m, {
|
73
|
+
service: service, configuration: conf, message: message })
|
74
|
+
else o.send(m)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def cmd_run(service, conf, message)
|
79
|
+
|
80
|
+
fail NotImplementedError
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
data/lib/flor/unit/scheduler.rb
CHANGED
@@ -5,7 +5,7 @@ module Flor
|
|
5
5
|
|
6
6
|
attr_reader :conf, :env
|
7
7
|
|
8
|
-
attr_reader :hooker, :storage, :loader, :ganger
|
8
|
+
attr_reader :hooker, :storage, :loader, :ganger, :runner
|
9
9
|
attr_reader :logger
|
10
10
|
|
11
11
|
attr_reader :thread_status
|
@@ -27,20 +27,24 @@ module Flor
|
|
27
27
|
@env = (Kernel.const_get(@env) rescue @env) if @env.match(/\A[A-Z]+\z/)
|
28
28
|
# when env is "RAILS_ENV" for example...
|
29
29
|
|
30
|
+
@loader =
|
31
|
+
(Flor::Conf.get_class(@conf, 'loader') || Flor::Loader).new(self)
|
32
|
+
@runner =
|
33
|
+
(Flor::Conf.get_class(@conf, 'runner') || Flor::Runner).new(self)
|
30
34
|
@hooker =
|
31
35
|
(Flor::Conf.get_class(@conf, 'hooker') || Flor::Hooker).new(self)
|
32
36
|
@storage =
|
33
37
|
(Flor::Conf.get_class(@conf, 'storage') || Flor::Storage).new(self)
|
34
|
-
@loader =
|
35
|
-
(Flor::Conf.get_class(@conf, 'loader') || Flor::Loader).new(self)
|
36
38
|
@ganger =
|
37
39
|
(Flor::Conf.get_class(@conf, 'ganger') || Flor::Ganger).new(self)
|
38
40
|
|
39
41
|
@logger =
|
40
42
|
(Flor::Conf.get_class(@conf, 'logger') || Flor::Logger).new(self)
|
43
|
+
@wlist =
|
44
|
+
(Flor::Conf.get_class(@conf, 'wlist') || Flor::WaitList).new(self)
|
41
45
|
|
42
|
-
@
|
43
|
-
|
46
|
+
@spooler =
|
47
|
+
(Flor::Conf.get_class(@conf, 'spooler') || Flor::Spooler).new(self)
|
44
48
|
|
45
49
|
@heart_rate = @conf[:sch_heart_rate] || 0.3
|
46
50
|
@reload_after = @conf[:sch_reload_after] || 60
|
@@ -82,7 +86,7 @@ module Flor
|
|
82
86
|
Socket.ip_address_list.find { |a| a.ipv4_private? } ||
|
83
87
|
Socket.ip_address_list.find { |a| a.ip_address != '::1' }
|
84
88
|
ip =
|
85
|
-
ai ? ai.ip_address : '::1'
|
89
|
+
(ai ? ai.ip_address : '::1').split('%').first
|
86
90
|
[
|
87
91
|
'sch', self.name,
|
88
92
|
'i' + ip,
|
@@ -149,46 +153,13 @@ module Flor
|
|
149
153
|
|
150
154
|
@thread =
|
151
155
|
if @thread
|
152
|
-
|
153
156
|
@thread.run
|
154
|
-
|
155
157
|
else
|
156
|
-
|
157
158
|
Thread.new do
|
158
|
-
|
159
159
|
loop do
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
Thread.stop if @thread_status == :stop
|
164
|
-
break if @thread_status == :shutdown
|
165
|
-
|
166
|
-
t0 = Time.now
|
167
|
-
|
168
|
-
if should_wake_up?
|
169
|
-
|
170
|
-
unreserve_messages
|
171
|
-
|
172
|
-
trigger_timers
|
173
|
-
trigger_executions
|
174
|
-
|
175
|
-
reload_next_time
|
176
|
-
reload_wake_up
|
177
|
-
@reloaded_at = Time.now
|
178
|
-
|
179
|
-
elsif @executors.empty?
|
180
|
-
|
181
|
-
@idle_count += 1
|
182
|
-
notify(nil, make_idle_message)
|
183
|
-
end
|
184
|
-
|
185
|
-
sleep [ @heart_rate - (Time.now - t0), 0 ].max #\
|
186
|
-
#unless should_wake_up?
|
187
|
-
|
188
|
-
rescue Exception => ex
|
189
|
-
|
190
|
-
puts on_start_exc(ex)
|
191
|
-
end
|
160
|
+
Thread.stop if @thread_status == :stop
|
161
|
+
break if @thread_status == :shutdown
|
162
|
+
tick
|
192
163
|
end
|
193
164
|
end
|
194
165
|
end
|
@@ -360,6 +331,37 @@ module Flor
|
|
360
331
|
|
361
332
|
protected
|
362
333
|
|
334
|
+
def tick
|
335
|
+
|
336
|
+
t0 = Time.now
|
337
|
+
|
338
|
+
spool
|
339
|
+
|
340
|
+
if should_wake_up?
|
341
|
+
|
342
|
+
unreserve_messages
|
343
|
+
|
344
|
+
trigger_timers
|
345
|
+
trigger_executions
|
346
|
+
|
347
|
+
reload_next_time
|
348
|
+
reload_wake_up
|
349
|
+
@reloaded_at = Time.now
|
350
|
+
|
351
|
+
elsif @executors.empty?
|
352
|
+
|
353
|
+
@idle_count += 1
|
354
|
+
notify(nil, make_idle_message)
|
355
|
+
end
|
356
|
+
|
357
|
+
sleep [ @heart_rate - (Time.now - t0), 0 ].max #\
|
358
|
+
#unless should_wake_up?
|
359
|
+
|
360
|
+
rescue Exception => ex
|
361
|
+
|
362
|
+
puts on_start_exc(ex)
|
363
|
+
end
|
364
|
+
|
363
365
|
def prepare_on_receive_last(h)
|
364
366
|
|
365
367
|
ei = h[:exid] || h['exid']
|
@@ -409,14 +411,17 @@ module Flor
|
|
409
411
|
m
|
410
412
|
end
|
411
413
|
|
412
|
-
def
|
414
|
+
def reload_after
|
415
|
+
|
416
|
+
@idle_count < 5 ? 1 : @reload_after
|
417
|
+
end
|
413
418
|
|
414
|
-
|
419
|
+
def should_wake_up?
|
415
420
|
|
416
421
|
return true if @wake_up
|
417
|
-
return
|
422
|
+
return true if Time.now - @reloaded_at >= reload_after
|
418
423
|
|
419
|
-
@next_time <= Flor.tstamp.split('.').first
|
424
|
+
@next_time && @next_time <= Flor.tstamp.split('.').first
|
420
425
|
end
|
421
426
|
|
422
427
|
def unreserve_messages
|
@@ -428,6 +433,12 @@ module Flor
|
|
428
433
|
) if c > 0
|
429
434
|
end
|
430
435
|
|
436
|
+
def spool
|
437
|
+
|
438
|
+
count = @spooler.spool
|
439
|
+
@wake_up = true if count > 0
|
440
|
+
end
|
441
|
+
|
431
442
|
def trigger_timers
|
432
443
|
|
433
444
|
@storage.trigger_timers
|