bud 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +33 -16
- data/bin/budplot +42 -65
- data/bin/budtimelines +235 -0
- data/bin/budvis +24 -122
- data/bin/rebl +1 -0
- data/docs/README.md +21 -10
- data/docs/bfs.md +4 -6
- data/docs/c.html +251 -0
- data/docs/cheat.md +45 -30
- data/docs/deploy.md +26 -26
- data/docs/getstarted.md +6 -4
- data/docs/visualizations.md +43 -31
- data/examples/chat/chat.rb +4 -9
- data/examples/chat/chat_server.rb +1 -8
- data/examples/deploy/deploy_ip_port +1 -0
- data/examples/deploy/keys.rb +5 -0
- data/examples/deploy/tokenring-ec2.rb +9 -9
- data/examples/deploy/{tokenring-local.rb → tokenring-fork.rb} +3 -5
- data/examples/deploy/tokenring-thread.rb +15 -0
- data/examples/deploy/tokenring.rb +25 -17
- data/lib/bud/aggs.rb +87 -25
- data/lib/bud/bud_meta.rb +48 -31
- data/lib/bud/bust/bust.rb +16 -15
- data/lib/bud/collections.rb +207 -232
- data/lib/bud/depanalysis.rb +1 -0
- data/lib/bud/deploy/countatomicdelivery.rb +8 -20
- data/lib/bud/deploy/deployer.rb +16 -16
- data/lib/bud/deploy/ec2deploy.rb +34 -35
- data/lib/bud/deploy/forkdeploy.rb +90 -0
- data/lib/bud/deploy/threaddeploy.rb +38 -0
- data/lib/bud/graphs.rb +103 -199
- data/lib/bud/joins.rb +190 -41
- data/lib/bud/monkeypatch.rb +84 -0
- data/lib/bud/rebl.rb +8 -1
- data/lib/bud/rewrite.rb +152 -49
- data/lib/bud/server.rb +1 -0
- data/lib/bud/state.rb +24 -10
- data/lib/bud/storage/dbm.rb +170 -0
- data/lib/bud/storage/tokyocabinet.rb +5 -1
- data/lib/bud/stratify.rb +6 -7
- data/lib/bud/viz.rb +31 -17
- data/lib/bud/viz_util.rb +204 -0
- data/lib/bud.rb +271 -244
- data/lib/bud.rb.orig +806 -0
- metadata +43 -22
- data/docs/bfs.raw +0 -251
- data/docs/diffs +0 -181
- data/examples/basics/out +0 -1103
- data/examples/basics/out.new +0 -856
- data/lib/bud/deploy/localdeploy.rb +0 -53
data/lib/bud.rb
CHANGED
@@ -5,105 +5,33 @@ require 'socket'
|
|
5
5
|
require 'superators'
|
6
6
|
require 'thread'
|
7
7
|
|
8
|
+
require 'bud/monkeypatch'
|
9
|
+
|
8
10
|
require 'bud/aggs'
|
9
11
|
require 'bud/bud_meta'
|
10
12
|
require 'bud/collections'
|
13
|
+
require 'bud/depanalysis'
|
14
|
+
require 'bud/deploy/forkdeploy'
|
15
|
+
require 'bud/deploy/threaddeploy'
|
11
16
|
require 'bud/errors'
|
12
17
|
require 'bud/joins'
|
13
18
|
require 'bud/rtrace'
|
14
19
|
require 'bud/server'
|
15
20
|
require 'bud/state'
|
21
|
+
require 'bud/storage/dbm'
|
16
22
|
require 'bud/storage/tokyocabinet'
|
17
23
|
require 'bud/storage/zookeeper'
|
24
|
+
require 'bud/stratify'
|
18
25
|
require 'bud/viz'
|
19
26
|
|
20
|
-
|
21
|
-
|
22
|
-
# We monkeypatch Module to add support for Bloom state and code declarations.
|
23
|
-
class Module
|
24
|
-
|
25
|
-
# import another module and assign to a qualifier symbol: <tt>import MyModule => :m</tt>
|
26
|
-
def import(spec)
|
27
|
-
raise Bud::CompileError unless (spec.class <= Hash and spec.length == 1)
|
28
|
-
mod, local_name = spec.first
|
29
|
-
raise Bud::CompileError unless (mod.class <= Module and local_name.class <= Symbol)
|
30
|
-
|
31
|
-
# To correctly expand qualified references to an imported module, we keep a
|
32
|
-
# table with the local bind names of all the modules imported by this
|
33
|
-
# module. To handle nested references (a.b.c.d etc.), the import table for
|
34
|
-
# module X points to X's own nested import table.
|
35
|
-
@bud_import_tbl ||= {}
|
36
|
-
child_tbl = mod.bud_import_table
|
37
|
-
raise Bud::CompileError if @bud_import_tbl.has_key? local_name
|
38
|
-
@bud_import_tbl[local_name] = child_tbl.clone # XXX: clone needed?
|
39
|
-
|
40
|
-
rewritten_mod_name = ModuleRewriter.do_import(self, mod, local_name)
|
41
|
-
self.module_eval "include #{rewritten_mod_name}"
|
42
|
-
end
|
43
|
-
|
44
|
-
# the block of Bloom collection declarations. one per module.
|
45
|
-
def state(&block)
|
46
|
-
meth_name = Module.make_state_meth_name(self)
|
47
|
-
define_method(meth_name, &block)
|
48
|
-
end
|
49
|
-
|
50
|
-
# a ruby block to be run before timestep 1. one per module.
|
51
|
-
def bootstrap(&block)
|
52
|
-
meth_name = "__bootstrap__#{Module.get_class_name(self)}".to_sym
|
53
|
-
define_method(meth_name, &block)
|
54
|
-
end
|
55
|
-
|
56
|
-
# bloom statements to be registered with Bud runtime. optional +block_name+
|
57
|
-
# allows for multiple bloom blocks per module, and overriding
|
58
|
-
def bloom(block_name=nil, &block)
|
59
|
-
# If no block name was specified, generate a unique name
|
60
|
-
if block_name.nil?
|
61
|
-
@block_id ||= 0
|
62
|
-
block_name = "#{Module.get_class_name(self)}__#{@block_id.to_s}"
|
63
|
-
@block_id += 1
|
64
|
-
else
|
65
|
-
unless block_name.class <= Symbol
|
66
|
-
raise Bud::CompileError, "Bloom block names must be a symbol: #{block_name}"
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
# Note that we don't encode the module name ("self") into the name of the
|
71
|
-
# method. This allows named blocks to be overridden (via inheritance or
|
72
|
-
# mixin) in the same way as normal Ruby methods.
|
73
|
-
meth_name = "__bloom__#{block_name}"
|
74
|
-
|
75
|
-
# Don't allow duplicate named bloom blocks to be defined within a single
|
76
|
-
# module; this indicates a likely programmer error.
|
77
|
-
if instance_methods(false).include? meth_name
|
78
|
-
raise Bud::CompileError, "Duplicate named bloom block: '#{block_name}' in #{self}"
|
79
|
-
end
|
80
|
-
define_method(meth_name.to_sym, &block)
|
81
|
-
end
|
82
|
-
|
83
|
-
def bud_import_table() #:nodoc: all
|
84
|
-
@bud_import_tbl ||= {}
|
85
|
-
@bud_import_tbl
|
86
|
-
end
|
27
|
+
ILLEGAL_INSTANCE_ID = -1
|
28
|
+
SIGNAL_CHECK_PERIOD = 0.2
|
87
29
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
def self.get_class_name(klass)
|
94
|
-
klass.name.split("::").last
|
95
|
-
end
|
96
|
-
|
97
|
-
# State method blocks are named using an auto-incrementing counter. This is to
|
98
|
-
# ensure that we can rediscover the possible dependencies between these blocks
|
99
|
-
# after module import (see Bud#call_state_methods).
|
100
|
-
def self.make_state_meth_name(klass)
|
101
|
-
@state_meth_id ||= 0
|
102
|
-
r = "__state#{@state_meth_id}__#{Module.get_class_name(klass)}".to_sym
|
103
|
-
@state_meth_id += 1
|
104
|
-
return r
|
105
|
-
end
|
106
|
-
end
|
30
|
+
$signal_lock = Mutex.new
|
31
|
+
$got_shutdown_signal = false
|
32
|
+
$signal_handler_setup = false
|
33
|
+
$instance_id = 0
|
34
|
+
$bud_instances = {} # Map from instance id => Bud instance
|
107
35
|
|
108
36
|
# The root Bud module. To cause an instance of Bud to begin executing, there are
|
109
37
|
# three main options:
|
@@ -128,51 +56,64 @@ end
|
|
128
56
|
# :main: Bud
|
129
57
|
module Bud
|
130
58
|
attr_reader :strata, :budtime, :inbound, :options, :meta_parser, :viz, :rtracer
|
131
|
-
attr_reader :dsock
|
132
|
-
attr_reader :tables, :
|
133
|
-
attr_reader :stratum_first_iter
|
59
|
+
attr_reader :dsock, :ip, :port
|
60
|
+
attr_reader :tables, :channels, :tc_tables, :zk_tables, :dbm_tables
|
61
|
+
attr_reader :stratum_first_iter, :joinstate
|
134
62
|
attr_accessor :lazy # This can be changed on-the-fly by REBL
|
63
|
+
attr_accessor :stratum_collection_map
|
135
64
|
|
136
|
-
# options to the
|
65
|
+
# options to the Bud runtime are passed in a hash, with the following keys
|
137
66
|
# * network configuration
|
138
67
|
# * <tt>:ip</tt> IP address string for this instance
|
139
68
|
# * <tt>:port</tt> port number for this instance
|
140
69
|
# * <tt>:ext_ip</tt> IP address at which external nodes can contact this instance
|
141
|
-
# * <tt>:ext_port</tt> port number to go with
|
142
|
-
# * <tt>:bust_port</tt> port number for the restful
|
70
|
+
# * <tt>:ext_port</tt> port number to go with <tt>:ext_ip</tt>
|
71
|
+
# * <tt>:bust_port</tt> port number for the restful HTTP messages
|
143
72
|
# * operating system interaction
|
144
|
-
# * <tt>:
|
145
|
-
# * <tt>:
|
73
|
+
# * <tt>:stdin</tt> if non-nil, reading from the +stdio+ collection results in reading from this +IO+ handle
|
74
|
+
# * <tt>:stdout</tt> writing to the +stdio+ collection results in writing to this +IO+ handle; defaults to <tt>$stdout</tt>
|
75
|
+
# * <tt>:no_signal_handlers</tt> if true, runtime ignores +SIGINT+ and +SIGTERM+
|
146
76
|
# * tracing and output
|
147
77
|
# * <tt>:quiet</tt> if true, suppress certain messages
|
148
|
-
# * <tt>:trace</tt> if true, generate budvis outputs
|
149
|
-
# * <tt>:rtrace</tt> if true, generate budplot outputs
|
78
|
+
# * <tt>:trace</tt> if true, generate +budvis+ outputs
|
79
|
+
# * <tt>:rtrace</tt> if true, generate +budplot+ outputs
|
150
80
|
# * <tt>:dump_rewrite</tt> if true, dump results of internal rewriting of Bloom code to a file
|
151
|
-
# * controlling execution
|
81
|
+
# * controlling execution
|
152
82
|
# * <tt>:lazy</tt> if true, prevents runtime from ticking except on external calls to +tick+
|
153
83
|
# * <tt>:tag</tt> a name for this instance, suitable for display during tracing and visualization
|
154
84
|
# * storage configuration
|
155
|
-
# * <tt>:
|
156
|
-
# * <tt>:
|
85
|
+
# * <tt>:dbm_dir</tt> filesystem directory to hold DBM-backed collections
|
86
|
+
# * <tt>:dbm_truncate</tt> if true, DBM-backed collections are opened with +OTRUNC+
|
87
|
+
# * <tt>:tc_dir</tt> filesystem directory to hold TokyoCabinet-backed collections
|
88
|
+
# * <tt>:tc_truncate</tt> if true, TokyoCabinet-backed collections are opened with +OTRUNC+
|
89
|
+
# * deployment
|
90
|
+
# * <tt>:deploy</tt> enable deployment
|
91
|
+
# * <tt>:deploy_child_opts</tt> option hash to pass to deployed instances
|
157
92
|
def initialize(options={})
|
158
93
|
@tables = {}
|
159
94
|
@table_meta = []
|
160
95
|
@rewritten_strata = []
|
161
96
|
@channels = {}
|
162
97
|
@tc_tables = {}
|
98
|
+
@dbm_tables = {}
|
163
99
|
@zk_tables = {}
|
164
100
|
@callbacks = {}
|
165
101
|
@callback_id = 0
|
102
|
+
@shutdown_callbacks = []
|
103
|
+
@post_shutdown_callbacks = []
|
166
104
|
@timers = []
|
105
|
+
@inside_tick = false
|
106
|
+
@tick_clock_time = nil
|
167
107
|
@budtime = 0
|
168
108
|
@inbound = []
|
169
109
|
@done_bootstrap = false
|
170
110
|
@joinstate = {} # joins are stateful, their state needs to be kept inside the Bud instance
|
111
|
+
@instance_id = ILLEGAL_INSTANCE_ID # Assigned when we start running
|
171
112
|
|
172
113
|
# Setup options (named arguments), along with default values
|
173
|
-
@options = options
|
114
|
+
@options = options.clone
|
174
115
|
@lazy = @options[:lazy] ||= false
|
175
|
-
@options[:ip] ||= "
|
116
|
+
@options[:ip] ||= "127.0.0.1"
|
176
117
|
@ip = @options[:ip]
|
177
118
|
@options[:port] ||= 0
|
178
119
|
@options[:port] = @options[:port].to_i
|
@@ -188,12 +129,6 @@ module Bud
|
|
188
129
|
|
189
130
|
init_state
|
190
131
|
|
191
|
-
# NB: Somewhat hacky. Dependency analysis and stratification are implemented
|
192
|
-
# by Bud programs, so in order for those programs to parse, we need the
|
193
|
-
# "Bud" class to have been defined first.
|
194
|
-
require 'bud/depanalysis'
|
195
|
-
require 'bud/stratify'
|
196
|
-
|
197
132
|
@viz = VizOnline.new(self) if @options[:trace]
|
198
133
|
@rtracer = RTrace.new(self) if @options[:rtrace]
|
199
134
|
|
@@ -208,6 +143,7 @@ module Bud
|
|
208
143
|
# array, so we need to convert it before loading the rewritten strata.
|
209
144
|
@strata = []
|
210
145
|
@rule_src = []
|
146
|
+
@rule_orig_src = []
|
211
147
|
declaration
|
212
148
|
@strata.each_with_index do |s,i|
|
213
149
|
raise BudError if s.class <= Array
|
@@ -219,9 +155,11 @@ module Bud
|
|
219
155
|
@rewritten_strata.each_with_index do |src_ary,i|
|
220
156
|
@strata[i] ||= []
|
221
157
|
@rule_src[i] ||= []
|
222
|
-
|
158
|
+
@rule_orig_src[i] ||= []
|
159
|
+
src_ary.each_with_index do |src, j|
|
223
160
|
@strata[i] << eval("lambda { #{src} }")
|
224
161
|
@rule_src[i] << src
|
162
|
+
@rule_orig_src[i] << @no_attr_rewrite_strata[i][j]
|
225
163
|
end
|
226
164
|
end
|
227
165
|
end
|
@@ -313,7 +251,7 @@ module Bud
|
|
313
251
|
|
314
252
|
def do_rewrite
|
315
253
|
@meta_parser = BudMeta.new(self, @declarations)
|
316
|
-
@rewritten_strata = @meta_parser.meta_rewrite
|
254
|
+
@rewritten_strata, @no_attr_rewrite_strata = @meta_parser.meta_rewrite
|
317
255
|
end
|
318
256
|
|
319
257
|
public
|
@@ -349,19 +287,25 @@ module Bud
|
|
349
287
|
# Run Bud in the "foreground" -- the caller's thread will be used to run the
|
350
288
|
# Bud interpreter. This means this method won't return unless an error
|
351
289
|
# occurs. It is often more useful to run Bud asynchronously -- see run_bg.
|
352
|
-
#
|
353
|
-
# Note that run_fg cannot be invoked if run_bg has already been called in the
|
354
|
-
# same Ruby process.
|
355
|
-
#
|
356
|
-
# Execution proceeds in time ticks, a la Dedalus.
|
357
|
-
# * Within each tick there may be multiple strata.
|
358
|
-
# * Within each stratum we do multiple semi-naive iterations.
|
359
290
|
def run_fg
|
360
|
-
|
291
|
+
# If we're called from the EventMachine thread (and EM is running), blocking
|
292
|
+
# the current thread would imply deadlocking ourselves.
|
293
|
+
if Thread.current == EventMachine::reactor_thread and EventMachine::reactor_running?
|
294
|
+
raise BudError, "Cannot invoke run_fg from inside EventMachine"
|
295
|
+
end
|
361
296
|
|
362
|
-
|
363
|
-
|
364
|
-
|
297
|
+
q = Queue.new
|
298
|
+
# Note that this must be a post-shutdown callback: if this is the only
|
299
|
+
# thread, then the program might exit after run_fg() returns. If run_fg()
|
300
|
+
# blocked on a normal shutdown callback, the program might exit before the
|
301
|
+
# other shutdown callbacks have a chance to run.
|
302
|
+
post_shutdown do
|
303
|
+
q.push(true)
|
304
|
+
end
|
305
|
+
|
306
|
+
run_bg
|
307
|
+
# Block caller's thread until Bud has shutdown
|
308
|
+
q.pop
|
365
309
|
end
|
366
310
|
|
367
311
|
# Shutdown a Bud instance that is running asynchronously. This method blocks
|
@@ -369,15 +313,34 @@ module Bud
|
|
369
313
|
# loop is also shutdown; this will interfere with the execution of any other
|
370
314
|
# Bud instances in the same process (as well as anything else that happens to
|
371
315
|
# use EventMachine).
|
372
|
-
def stop_bg(stop_em=false)
|
316
|
+
def stop_bg(stop_em=false, do_shutdown_cb=true)
|
317
|
+
schedule_and_wait do
|
318
|
+
do_shutdown(do_shutdown_cb)
|
319
|
+
end
|
320
|
+
|
373
321
|
if stop_em
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
322
|
+
Bud.stop_em_loop
|
323
|
+
EventMachine::reactor_thread.join
|
324
|
+
end
|
325
|
+
end
|
326
|
+
|
327
|
+
# Register a callback that will be invoked when this instance of Bud is
|
328
|
+
# shutting down.
|
329
|
+
def on_shutdown(&blk)
|
330
|
+
# Start EM if not yet started
|
331
|
+
start_reactor
|
332
|
+
schedule_and_wait do
|
333
|
+
@shutdown_callbacks << blk
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
# Register a callback that will be invoked when *after* this instance of Bud
|
338
|
+
# has been shutdown.
|
339
|
+
def post_shutdown(&blk)
|
340
|
+
# Start EM if not yet started
|
341
|
+
start_reactor
|
342
|
+
schedule_and_wait do
|
343
|
+
@post_shutdown_callbacks << blk
|
381
344
|
end
|
382
345
|
end
|
383
346
|
|
@@ -414,7 +377,7 @@ module Bud
|
|
414
377
|
end
|
415
378
|
|
416
379
|
# Shutdown any persistent tables used by the current Bud instance. If you are
|
417
|
-
# running Bud via tick() and using
|
380
|
+
# running Bud via tick() and using +tctable+ collections, you should call this
|
418
381
|
# after you're finished using Bud. Programs that use Bud via run_fg() or
|
419
382
|
# run_bg() don't need to call this manually.
|
420
383
|
def close_tables
|
@@ -485,7 +448,7 @@ module Bud
|
|
485
448
|
return result
|
486
449
|
end
|
487
450
|
|
488
|
-
#
|
451
|
+
# A common special case for sync_callback: block on a delta to a table.
|
489
452
|
def delta(out_tbl)
|
490
453
|
sync_callback(nil, nil, out_tbl)
|
491
454
|
end
|
@@ -508,6 +471,7 @@ module Bud
|
|
508
471
|
EventMachine::error_handler do |e|
|
509
472
|
puts "Unexpected Bud error: #{e.inspect}"
|
510
473
|
puts e.backtrace.join("\n")
|
474
|
+
Bud.shutdown_all_instances
|
511
475
|
raise e
|
512
476
|
end
|
513
477
|
|
@@ -517,10 +481,8 @@ module Bud
|
|
517
481
|
# EventMachine's event loop.
|
518
482
|
Thread.new do
|
519
483
|
EventMachine.run do
|
520
|
-
q
|
484
|
+
q.push(true)
|
521
485
|
end
|
522
|
-
# Executed only after EventMachine::stop_event_loop is done
|
523
|
-
$em_stopped << true
|
524
486
|
end
|
525
487
|
# Block waiting for EM's event loop to start up.
|
526
488
|
q.pop
|
@@ -529,9 +491,12 @@ module Bud
|
|
529
491
|
# Schedule a block to be evaluated by EventMachine in the future, and
|
530
492
|
# block until this has happened.
|
531
493
|
def schedule_and_wait
|
532
|
-
#
|
533
|
-
#
|
534
|
-
|
494
|
+
# If EM isn't running, just run the user's block immediately
|
495
|
+
# XXX: not clear that this is the right behavior
|
496
|
+
unless EventMachine::reactor_running?
|
497
|
+
yield
|
498
|
+
return
|
499
|
+
end
|
535
500
|
|
536
501
|
q = Queue.new
|
537
502
|
EventMachine::schedule do
|
@@ -548,43 +513,33 @@ module Bud
|
|
548
513
|
raise resp if resp
|
549
514
|
end
|
550
515
|
|
551
|
-
def do_shutdown(
|
552
|
-
|
553
|
-
|
516
|
+
def do_shutdown(do_shutdown_cb=true)
|
517
|
+
# Silently ignore duplicate shutdown requests or attempts to shutdown an
|
518
|
+
# instance that hasn't been started yet.
|
519
|
+
return if @instance_id == ILLEGAL_INSTANCE_ID
|
520
|
+
|
521
|
+
$signal_lock.synchronize {
|
522
|
+
raise unless $bud_instances.has_key? @instance_id
|
523
|
+
$bud_instances.delete @instance_id
|
524
|
+
@instance_id = ILLEGAL_INSTANCE_ID
|
525
|
+
}
|
526
|
+
|
527
|
+
if do_shutdown_cb
|
528
|
+
@shutdown_callbacks.each {|cb| cb.call}
|
554
529
|
end
|
530
|
+
@timers.each {|t| t.cancel}
|
555
531
|
close_tables
|
556
|
-
@dsock.close_connection
|
557
|
-
|
558
|
-
|
559
|
-
# has completely shutdown, we use the @em_stopped queue.
|
560
|
-
EventMachine::stop_event_loop if stop_em
|
561
|
-
end
|
562
|
-
|
563
|
-
# Schedule a "graceful" shutdown for a future EM tick. If EM is not currently
|
564
|
-
# running, shutdown immediately.
|
565
|
-
def schedule_shutdown(stop_em=false)
|
566
|
-
if EventMachine::reactor_running?
|
567
|
-
EventMachine::schedule do
|
568
|
-
do_shutdown(stop_em)
|
569
|
-
end
|
570
|
-
else
|
571
|
-
do_shutdown(stop_em)
|
532
|
+
@dsock.close_connection if EventMachine::reactor_running?
|
533
|
+
if do_shutdown_cb
|
534
|
+
@post_shutdown_callbacks.each {|cb| cb.call}
|
572
535
|
end
|
573
536
|
end
|
574
537
|
|
538
|
+
private
|
575
539
|
def start_bud
|
576
540
|
raise BudError unless EventMachine::reactor_thread?
|
577
541
|
|
578
|
-
|
579
|
-
unless @options[:no_signal_handlers]
|
580
|
-
Signal.trap("INT") do
|
581
|
-
schedule_shutdown(true)
|
582
|
-
end
|
583
|
-
Signal.trap("TRAP") do
|
584
|
-
schedule_shutdown(true)
|
585
|
-
end
|
586
|
-
end
|
587
|
-
|
542
|
+
@instance_id = Bud.init_signal_handlers(self)
|
588
543
|
do_start_server
|
589
544
|
|
590
545
|
# Initialize periodics
|
@@ -594,7 +549,7 @@ module Bud
|
|
594
549
|
|
595
550
|
# Arrange for Bud to read from stdin if enabled. Note that we can't do this
|
596
551
|
# earlier because we need to wait for EventMachine startup.
|
597
|
-
@stdio.start_stdin_reader if @options[:
|
552
|
+
@stdio.start_stdin_reader if @options[:stdin]
|
598
553
|
@zk_tables.each_value {|t| t.start_watchers}
|
599
554
|
|
600
555
|
# Compute a fixpoint; this will also invoke any bootstrap blocks.
|
@@ -611,12 +566,13 @@ module Bud
|
|
611
566
|
|
612
567
|
public
|
613
568
|
|
614
|
-
# Returns the
|
615
|
-
# and port, the user may define an external IP and/or port.
|
616
|
-
# version of each is returned if available. If not, the local
|
617
|
-
# returned. There are use cases for mixing and matching local and
|
618
|
-
# local_ip:external_port would be if you have local port
|
619
|
-
# external_ip:local_port would be if you're in a DMZ, for
|
569
|
+
# Returns the IP and port of the Bud instance as a string. In addition to the
|
570
|
+
# local IP and port, the user may define an external IP and/or port. The
|
571
|
+
# external version of each is returned if available. If not, the local
|
572
|
+
# version is returned. There are use cases for mixing and matching local and
|
573
|
+
# external. local_ip:external_port would be if you have local port
|
574
|
+
# forwarding, and external_ip:local_port would be if you're in a DMZ, for
|
575
|
+
# example.
|
620
576
|
def ip_port
|
621
577
|
raise BudError, "ip_port called before port defined" if @port.nil? and @options[:port] == 0 and not @options[:ext_port]
|
622
578
|
|
@@ -626,28 +582,43 @@ module Bud
|
|
626
582
|
ip + ":" + port
|
627
583
|
end
|
628
584
|
|
629
|
-
# Returns the internal IP and port. See ip_port
|
585
|
+
# Returns the internal IP and port. See ip_port.
|
630
586
|
def int_ip_port
|
631
587
|
raise BudError, "ip_port called before port defined" if @port.nil? and @options[:port] == 0
|
632
588
|
@port.nil? ? "#{@ip}:#{@options[:port]}" : "#{@ip}:#{@port}"
|
633
589
|
end
|
634
590
|
|
635
|
-
#
|
591
|
+
# Manually trigger one timestep of Bloom execution.
|
636
592
|
def tick
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
593
|
+
begin
|
594
|
+
@inside_tick = true
|
595
|
+
@tables.each_value do |t|
|
596
|
+
t.tick
|
597
|
+
end
|
598
|
+
|
599
|
+
@joinstate = {}
|
600
|
+
|
601
|
+
do_bootstrap unless @done_bootstrap
|
602
|
+
receive_inbound
|
642
603
|
|
643
|
-
|
644
|
-
|
604
|
+
@strata.each_with_index { |s,i| stratum_fixpoint(s, i) }
|
605
|
+
@viz.do_cards if @options[:trace]
|
606
|
+
do_flush
|
607
|
+
invoke_callbacks
|
608
|
+
@budtime += 1
|
609
|
+
ensure
|
610
|
+
@inside_tick = false
|
611
|
+
@tick_clock_time = nil
|
612
|
+
end
|
613
|
+
end
|
645
614
|
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
615
|
+
# Returns the wallclock time associated with the current Bud tick. That is,
|
616
|
+
# this value is guaranteed to remain the same for the duration of a single
|
617
|
+
# tick, but will likely change between ticks.
|
618
|
+
def bud_clock
|
619
|
+
raise BudError, "bud_clock undefined outside tick" unless @inside_tick
|
620
|
+
@tick_clock_time ||= Time.now
|
621
|
+
@tick_clock_time
|
651
622
|
end
|
652
623
|
|
653
624
|
private
|
@@ -656,18 +627,20 @@ module Bud
|
|
656
627
|
# standard "state" syntax, but we want to ensure that builtin state is
|
657
628
|
# initialized before user-defined state.
|
658
629
|
def builtin_state
|
659
|
-
|
630
|
+
loopback :localtick, [:col1]
|
660
631
|
@stdio = terminal :stdio
|
661
632
|
@periodics = table :periodics_tbl, [:pername] => [:ident, :period]
|
662
633
|
|
663
634
|
# for BUD reflection
|
664
|
-
table :t_rules, [:rule_id] => [:lhs, :op, :src]
|
635
|
+
table :t_rules, [:rule_id] => [:lhs, :op, :src, :orig_src]
|
665
636
|
table :t_depends, [:rule_id, :lhs, :op, :body] => [:nm]
|
666
637
|
table :t_depends_tc, [:head, :body, :via, :neg, :temporal]
|
667
638
|
table :t_provides, [:interface] => [:input]
|
668
639
|
table :t_underspecified, t_provides.schema
|
669
640
|
table :t_stratum, [:predicate] => [:stratum]
|
670
641
|
table :t_cycle, [:predicate, :via, :neg, :temporal]
|
642
|
+
table :t_table_info, [:tab_name, :tab_type]
|
643
|
+
table :t_table_schema, [:tab_name, :col_name, :ord, :loc]
|
671
644
|
end
|
672
645
|
|
673
646
|
# Handle any inbound tuples off the wire and then clear. Received messages are
|
@@ -684,14 +657,15 @@ module Bud
|
|
684
657
|
# 1. Emit outgoing tuples in channels and ZK tables.
|
685
658
|
# 2. Commit to disk any changes made to on-disk tables.
|
686
659
|
def do_flush
|
687
|
-
@channels.
|
660
|
+
@channels.each_value { |c| c.flush }
|
688
661
|
@zk_tables.each_value { |t| t.flush }
|
689
662
|
@tc_tables.each_value { |t| t.flush }
|
663
|
+
@dbm_tables.each_value { |t| t.flush }
|
690
664
|
end
|
691
665
|
|
692
666
|
def stratum_fixpoint(strat, strat_num)
|
693
|
-
# This routine uses semi-naive evaluation to compute
|
694
|
-
#
|
667
|
+
# This routine uses semi-naive evaluation to compute a fixpoint of the rules
|
668
|
+
# in strat.
|
695
669
|
#
|
696
670
|
# As described in lib/collections.rb, each collection has three
|
697
671
|
# sub-collections of note here:
|
@@ -699,13 +673,13 @@ module Bud
|
|
699
673
|
# @delta: tuples that should be used to drive derivation of new facts
|
700
674
|
# @new_delta: a place to store newly-derived facts
|
701
675
|
#
|
702
|
-
# The first time through this loop we mark @stratum_first_iter=true,
|
703
|
-
#
|
704
|
-
#
|
705
|
-
#
|
676
|
+
# The first time through this loop we mark @stratum_first_iter=true, which
|
677
|
+
# tells the Join::each code to join up all its @storage subcollections to
|
678
|
+
# start. In subsequent iterations the join code uses some table's @delta to
|
679
|
+
# ensure that only new tuples are derived.
|
706
680
|
#
|
707
|
-
# Note that calling "each" on a non-Join collection will iterate through
|
708
|
-
# storage and delta.
|
681
|
+
# Note that calling "each" on a non-Join collection will iterate through
|
682
|
+
# both storage and delta.
|
709
683
|
#
|
710
684
|
# At the end of each iteration of this loop we transition:
|
711
685
|
# - @delta tuples are merged into @storage
|
@@ -713,22 +687,22 @@ module Bud
|
|
713
687
|
# - @new_delta is set to empty
|
714
688
|
#
|
715
689
|
# XXX as a performance optimization, it would be nice to bypass the delta
|
716
|
-
# tables for any preds that don't participate in a rhs Join -- in that
|
717
|
-
#
|
718
|
-
#
|
719
|
-
|
720
|
-
# In semi-naive, the first iteration should join up tables
|
721
|
-
#
|
722
|
-
#
|
723
|
-
# these cases.
|
690
|
+
# tables for any preds that don't participate in a rhs Join -- in that case
|
691
|
+
# there's pointless extra tuple movement letting tuples "graduate" through
|
692
|
+
# @new_delta and @delta.
|
693
|
+
|
694
|
+
# In semi-naive, the first iteration should join up tables on their storage
|
695
|
+
# fields; subsequent iterations do the delta-joins only. The
|
696
|
+
# stratum_first_iter field here distinguishes these cases.
|
724
697
|
@stratum_first_iter = true
|
725
698
|
begin
|
726
699
|
strat.each_with_index do |r,i|
|
700
|
+
fixpoint = false
|
727
701
|
begin
|
728
702
|
r.call
|
729
703
|
rescue Exception => e
|
730
704
|
# Don't report source text for certain rules (old-style rule blocks)
|
731
|
-
rule_src = @
|
705
|
+
rule_src = @rule_orig_src[strat_num][i] unless @rule_orig_src[strat_num].nil?
|
732
706
|
src_msg = ""
|
733
707
|
unless rule_src == ""
|
734
708
|
src_msg = "\nRule: #{rule_src}"
|
@@ -742,57 +716,110 @@ module Bud
|
|
742
716
|
end
|
743
717
|
end
|
744
718
|
@stratum_first_iter = false
|
745
|
-
|
746
|
-
# we
|
747
|
-
|
748
|
-
@tables.
|
749
|
-
|
719
|
+
fixpoint = true
|
720
|
+
# tick collections in this stratum; if we don't have info on that, tick all collections
|
721
|
+
colls = @stratum_collection_map[strat_num] if @stratum_collection_map
|
722
|
+
colls ||= @tables.keys
|
723
|
+
colls.each do |name|
|
724
|
+
begin
|
725
|
+
coll = self.send(name)
|
726
|
+
unless coll.delta.empty? and coll.new_delta.empty?
|
727
|
+
coll.tick_deltas
|
728
|
+
fixpoint = false
|
729
|
+
end
|
730
|
+
rescue
|
731
|
+
# ignore missing tables; rebl for example deletes them mid-stream
|
732
|
+
end
|
733
|
+
end
|
734
|
+
end while not fixpoint
|
750
735
|
end
|
751
736
|
|
752
|
-
|
753
|
-
def wrap_map(j, &blk)
|
754
|
-
if blk.nil?
|
755
|
-
return j
|
756
|
-
else
|
757
|
-
return j.map(&blk)
|
758
|
-
end
|
759
|
-
end
|
737
|
+
private
|
760
738
|
|
761
|
-
|
762
|
-
def
|
763
|
-
|
739
|
+
######## ids and timers
|
740
|
+
def gen_id
|
741
|
+
Time.new.to_i.to_s << rand.to_s
|
764
742
|
end
|
765
743
|
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
744
|
+
def set_periodic_timer(name, id, period)
|
745
|
+
EventMachine::PeriodicTimer.new(period) do
|
746
|
+
@tables[name].add_periodic_tuple(id)
|
747
|
+
tick
|
748
|
+
end
|
771
749
|
end
|
772
750
|
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
751
|
+
# Fork a new process. This is identical to Kernel#fork, except that it also
|
752
|
+
# cleans up Bud and EventMachine-related state. As with Kernel#fork, the
|
753
|
+
# caller supplies a code block that is run in the child process; the PID of
|
754
|
+
# the child is returned by this method.
|
755
|
+
def self.do_fork
|
756
|
+
Kernel.fork do
|
757
|
+
srand
|
758
|
+
# This is somewhat grotty: we basically clone what EM::fork_reactor does,
|
759
|
+
# except that we don't want the user-supplied block to be invoked by the
|
760
|
+
# reactor thread.
|
761
|
+
if EventMachine::reactor_running?
|
762
|
+
EventMachine::stop_event_loop
|
763
|
+
EventMachine::release_machine
|
764
|
+
EventMachine::instance_variable_set('@reactor_running', false)
|
765
|
+
end
|
766
|
+
# Shutdown all the Bud instances inherited from the parent process, but
|
767
|
+
# don't invoke their shutdown callbacks
|
768
|
+
Bud.shutdown_all_instances(false)
|
769
|
+
|
770
|
+
$got_shutdown_signal = false
|
771
|
+
$setup_signal_handler = false
|
772
|
+
|
773
|
+
yield
|
774
|
+
end
|
777
775
|
end
|
778
776
|
|
779
|
-
#
|
780
|
-
#
|
781
|
-
|
782
|
-
|
777
|
+
# Note that this affects anyone else in the same process who happens to be
|
778
|
+
# using EventMachine! This is also a non-blocking call; to block until EM
|
779
|
+
# has completely shutdown, join on EM::reactor_thread.
|
780
|
+
def self.stop_em_loop
|
781
|
+
EventMachine::stop_event_loop
|
782
|
+
|
783
|
+
# If another instance of Bud is started later, we'll need to reinitialize
|
784
|
+
# the signal handlers (since they depend on EM).
|
785
|
+
$signal_handler_setup = false
|
783
786
|
end
|
784
787
|
|
785
|
-
|
788
|
+
# Signal handling. If multiple Bud instances are running inside a single
|
789
|
+
# process, we want a SIGINT or SIGTERM signal to cleanly shutdown all of them.
|
790
|
+
def self.init_signal_handlers(b)
|
791
|
+
$signal_lock.synchronize {
|
792
|
+
# If we setup signal handlers and then fork a new process, we want to
|
793
|
+
# reinitialize the signal handler in the child process.
|
794
|
+
unless b.options[:no_signal_handlers] or $signal_handler_setup
|
795
|
+
EventMachine::PeriodicTimer.new(SIGNAL_CHECK_PERIOD) do
|
796
|
+
if $got_shutdown_signal
|
797
|
+
Bud.shutdown_all_instances
|
798
|
+
Bud.stop_em_loop
|
799
|
+
$got_shutdown_signal = false
|
800
|
+
end
|
801
|
+
end
|
786
802
|
|
787
|
-
|
788
|
-
|
789
|
-
|
803
|
+
["INT", "TERM"].each do |signal|
|
804
|
+
Signal.trap(signal) {
|
805
|
+
$got_shutdown_signal = true
|
806
|
+
}
|
807
|
+
end
|
808
|
+
$setup_signal_handler_pid = true
|
809
|
+
end
|
810
|
+
|
811
|
+
$instance_id += 1
|
812
|
+
$bud_instances[$instance_id] = b
|
813
|
+
return $instance_id
|
814
|
+
}
|
790
815
|
end
|
791
816
|
|
792
|
-
def
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
817
|
+
def self.shutdown_all_instances(do_shutdown_cb=true)
|
818
|
+
instances = nil
|
819
|
+
$signal_lock.synchronize {
|
820
|
+
instances = $bud_instances.clone
|
821
|
+
}
|
822
|
+
|
823
|
+
instances.each_value {|b| b.stop_bg(false, do_shutdown_cb) }
|
797
824
|
end
|
798
825
|
end
|