bud 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/README +33 -16
  2. data/bin/budplot +42 -65
  3. data/bin/budtimelines +235 -0
  4. data/bin/budvis +24 -122
  5. data/bin/rebl +1 -0
  6. data/docs/README.md +21 -10
  7. data/docs/bfs.md +4 -6
  8. data/docs/c.html +251 -0
  9. data/docs/cheat.md +45 -30
  10. data/docs/deploy.md +26 -26
  11. data/docs/getstarted.md +6 -4
  12. data/docs/visualizations.md +43 -31
  13. data/examples/chat/chat.rb +4 -9
  14. data/examples/chat/chat_server.rb +1 -8
  15. data/examples/deploy/deploy_ip_port +1 -0
  16. data/examples/deploy/keys.rb +5 -0
  17. data/examples/deploy/tokenring-ec2.rb +9 -9
  18. data/examples/deploy/{tokenring-local.rb → tokenring-fork.rb} +3 -5
  19. data/examples/deploy/tokenring-thread.rb +15 -0
  20. data/examples/deploy/tokenring.rb +25 -17
  21. data/lib/bud/aggs.rb +87 -25
  22. data/lib/bud/bud_meta.rb +48 -31
  23. data/lib/bud/bust/bust.rb +16 -15
  24. data/lib/bud/collections.rb +207 -232
  25. data/lib/bud/depanalysis.rb +1 -0
  26. data/lib/bud/deploy/countatomicdelivery.rb +8 -20
  27. data/lib/bud/deploy/deployer.rb +16 -16
  28. data/lib/bud/deploy/ec2deploy.rb +34 -35
  29. data/lib/bud/deploy/forkdeploy.rb +90 -0
  30. data/lib/bud/deploy/threaddeploy.rb +38 -0
  31. data/lib/bud/graphs.rb +103 -199
  32. data/lib/bud/joins.rb +190 -41
  33. data/lib/bud/monkeypatch.rb +84 -0
  34. data/lib/bud/rebl.rb +8 -1
  35. data/lib/bud/rewrite.rb +152 -49
  36. data/lib/bud/server.rb +1 -0
  37. data/lib/bud/state.rb +24 -10
  38. data/lib/bud/storage/dbm.rb +170 -0
  39. data/lib/bud/storage/tokyocabinet.rb +5 -1
  40. data/lib/bud/stratify.rb +6 -7
  41. data/lib/bud/viz.rb +31 -17
  42. data/lib/bud/viz_util.rb +204 -0
  43. data/lib/bud.rb +271 -244
  44. data/lib/bud.rb.orig +806 -0
  45. metadata +43 -22
  46. data/docs/bfs.raw +0 -251
  47. data/docs/diffs +0 -181
  48. data/examples/basics/out +0 -1103
  49. data/examples/basics/out.new +0 -856
  50. data/lib/bud/deploy/localdeploy.rb +0 -53
data/lib/bud.rb CHANGED
@@ -5,105 +5,33 @@ require 'socket'
5
5
  require 'superators'
6
6
  require 'thread'
7
7
 
8
+ require 'bud/monkeypatch'
9
+
8
10
  require 'bud/aggs'
9
11
  require 'bud/bud_meta'
10
12
  require 'bud/collections'
13
+ require 'bud/depanalysis'
14
+ require 'bud/deploy/forkdeploy'
15
+ require 'bud/deploy/threaddeploy'
11
16
  require 'bud/errors'
12
17
  require 'bud/joins'
13
18
  require 'bud/rtrace'
14
19
  require 'bud/server'
15
20
  require 'bud/state'
21
+ require 'bud/storage/dbm'
16
22
  require 'bud/storage/tokyocabinet'
17
23
  require 'bud/storage/zookeeper'
24
+ require 'bud/stratify'
18
25
  require 'bud/viz'
19
26
 
20
- $em_stopped = Queue.new
21
-
22
- # We monkeypatch Module to add support for Bloom state and code declarations.
23
- class Module
24
-
25
- # import another module and assign to a qualifier symbol: <tt>import MyModule => :m</tt>
26
- def import(spec)
27
- raise Bud::CompileError unless (spec.class <= Hash and spec.length == 1)
28
- mod, local_name = spec.first
29
- raise Bud::CompileError unless (mod.class <= Module and local_name.class <= Symbol)
30
-
31
- # To correctly expand qualified references to an imported module, we keep a
32
- # table with the local bind names of all the modules imported by this
33
- # module. To handle nested references (a.b.c.d etc.), the import table for
34
- # module X points to X's own nested import table.
35
- @bud_import_tbl ||= {}
36
- child_tbl = mod.bud_import_table
37
- raise Bud::CompileError if @bud_import_tbl.has_key? local_name
38
- @bud_import_tbl[local_name] = child_tbl.clone # XXX: clone needed?
39
-
40
- rewritten_mod_name = ModuleRewriter.do_import(self, mod, local_name)
41
- self.module_eval "include #{rewritten_mod_name}"
42
- end
43
-
44
- # the block of Bloom collection declarations. one per module.
45
- def state(&block)
46
- meth_name = Module.make_state_meth_name(self)
47
- define_method(meth_name, &block)
48
- end
49
-
50
- # a ruby block to be run before timestep 1. one per module.
51
- def bootstrap(&block)
52
- meth_name = "__bootstrap__#{Module.get_class_name(self)}".to_sym
53
- define_method(meth_name, &block)
54
- end
55
-
56
- # bloom statements to be registered with Bud runtime. optional +block_name+
57
- # allows for multiple bloom blocks per module, and overriding
58
- def bloom(block_name=nil, &block)
59
- # If no block name was specified, generate a unique name
60
- if block_name.nil?
61
- @block_id ||= 0
62
- block_name = "#{Module.get_class_name(self)}__#{@block_id.to_s}"
63
- @block_id += 1
64
- else
65
- unless block_name.class <= Symbol
66
- raise Bud::CompileError, "Bloom block names must be a symbol: #{block_name}"
67
- end
68
- end
69
-
70
- # Note that we don't encode the module name ("self") into the name of the
71
- # method. This allows named blocks to be overridden (via inheritance or
72
- # mixin) in the same way as normal Ruby methods.
73
- meth_name = "__bloom__#{block_name}"
74
-
75
- # Don't allow duplicate named bloom blocks to be defined within a single
76
- # module; this indicates a likely programmer error.
77
- if instance_methods(false).include? meth_name
78
- raise Bud::CompileError, "Duplicate named bloom block: '#{block_name}' in #{self}"
79
- end
80
- define_method(meth_name.to_sym, &block)
81
- end
82
-
83
- def bud_import_table() #:nodoc: all
84
- @bud_import_tbl ||= {}
85
- @bud_import_tbl
86
- end
27
+ ILLEGAL_INSTANCE_ID = -1
28
+ SIGNAL_CHECK_PERIOD = 0.2
87
29
 
88
- private
89
- # Return a string with a version of the class name appropriate for embedding
90
- # into a method name. Annoyingly, if you define class X nested inside
91
- # class/module Y, X's class name is the string "Y::X". We don't want to define
92
- # method names with semicolons in them, so just return "X" instead.
93
- def self.get_class_name(klass)
94
- klass.name.split("::").last
95
- end
96
-
97
- # State method blocks are named using an auto-incrementing counter. This is to
98
- # ensure that we can rediscover the possible dependencies between these blocks
99
- # after module import (see Bud#call_state_methods).
100
- def self.make_state_meth_name(klass)
101
- @state_meth_id ||= 0
102
- r = "__state#{@state_meth_id}__#{Module.get_class_name(klass)}".to_sym
103
- @state_meth_id += 1
104
- return r
105
- end
106
- end
30
+ $signal_lock = Mutex.new
31
+ $got_shutdown_signal = false
32
+ $signal_handler_setup = false
33
+ $instance_id = 0
34
+ $bud_instances = {} # Map from instance id => Bud instance
107
35
 
108
36
  # The root Bud module. To cause an instance of Bud to begin executing, there are
109
37
  # three main options:
@@ -128,51 +56,64 @@ end
128
56
  # :main: Bud
129
57
  module Bud
130
58
  attr_reader :strata, :budtime, :inbound, :options, :meta_parser, :viz, :rtracer
131
- attr_reader :dsock
132
- attr_reader :tables, :ip, :port
133
- attr_reader :stratum_first_iter
59
+ attr_reader :dsock, :ip, :port
60
+ attr_reader :tables, :channels, :tc_tables, :zk_tables, :dbm_tables
61
+ attr_reader :stratum_first_iter, :joinstate
134
62
  attr_accessor :lazy # This can be changed on-the-fly by REBL
63
+ attr_accessor :stratum_collection_map
135
64
 
136
- # options to the bud runtime are passed in a hash, with the following keys
65
+ # options to the Bud runtime are passed in a hash, with the following keys
137
66
  # * network configuration
138
67
  # * <tt>:ip</tt> IP address string for this instance
139
68
  # * <tt>:port</tt> port number for this instance
140
69
  # * <tt>:ext_ip</tt> IP address at which external nodes can contact this instance
141
- # * <tt>:ext_port</tt> port number to go with :ext_ip
142
- # * <tt>:bust_port</tt> port number for the restful http messages
70
+ # * <tt>:ext_port</tt> port number to go with <tt>:ext_ip</tt>
71
+ # * <tt>:bust_port</tt> port number for the restful HTTP messages
143
72
  # * operating system interaction
144
- # * <tt>:read_stdin</tt> if true, captures stdin via the stdio collection
145
- # * <tt>:no_signal_handlers</tt> if true, runtime ignores SIGINT and SIGTERM
73
+ # * <tt>:stdin</tt> if non-nil, reading from the +stdio+ collection results in reading from this +IO+ handle
74
+ # * <tt>:stdout</tt> writing to the +stdio+ collection results in writing to this +IO+ handle; defaults to <tt>$stdout</tt>
75
+ # * <tt>:no_signal_handlers</tt> if true, runtime ignores +SIGINT+ and +SIGTERM+
146
76
  # * tracing and output
147
77
  # * <tt>:quiet</tt> if true, suppress certain messages
148
- # * <tt>:trace</tt> if true, generate budvis outputs
149
- # * <tt>:rtrace</tt> if true, generate budplot outputs
78
+ # * <tt>:trace</tt> if true, generate +budvis+ outputs
79
+ # * <tt>:rtrace</tt> if true, generate +budplot+ outputs
150
80
  # * <tt>:dump_rewrite</tt> if true, dump results of internal rewriting of Bloom code to a file
151
- # * controlling execution
81
+ # * controlling execution
152
82
  # * <tt>:lazy</tt> if true, prevents runtime from ticking except on external calls to +tick+
153
83
  # * <tt>:tag</tt> a name for this instance, suitable for display during tracing and visualization
154
84
  # * storage configuration
155
- # * <tt>:tc_dir</tt> filesystem directory to hold TokyoCabinet data stores
156
- # * <tt>:tc_truncate</tt> if true, TokyoCabinet collections are opened with OTRUNC
85
+ # * <tt>:dbm_dir</tt> filesystem directory to hold DBM-backed collections
86
+ # * <tt>:dbm_truncate</tt> if true, DBM-backed collections are opened with +OTRUNC+
87
+ # * <tt>:tc_dir</tt> filesystem directory to hold TokyoCabinet-backed collections
88
+ # * <tt>:tc_truncate</tt> if true, TokyoCabinet-backed collections are opened with +OTRUNC+
89
+ # * deployment
90
+ # * <tt>:deploy</tt> enable deployment
91
+ # * <tt>:deploy_child_opts</tt> option hash to pass to deployed instances
157
92
  def initialize(options={})
158
93
  @tables = {}
159
94
  @table_meta = []
160
95
  @rewritten_strata = []
161
96
  @channels = {}
162
97
  @tc_tables = {}
98
+ @dbm_tables = {}
163
99
  @zk_tables = {}
164
100
  @callbacks = {}
165
101
  @callback_id = 0
102
+ @shutdown_callbacks = []
103
+ @post_shutdown_callbacks = []
166
104
  @timers = []
105
+ @inside_tick = false
106
+ @tick_clock_time = nil
167
107
  @budtime = 0
168
108
  @inbound = []
169
109
  @done_bootstrap = false
170
110
  @joinstate = {} # joins are stateful, their state needs to be kept inside the Bud instance
111
+ @instance_id = ILLEGAL_INSTANCE_ID # Assigned when we start running
171
112
 
172
113
  # Setup options (named arguments), along with default values
173
- @options = options
114
+ @options = options.clone
174
115
  @lazy = @options[:lazy] ||= false
175
- @options[:ip] ||= "localhost"
116
+ @options[:ip] ||= "127.0.0.1"
176
117
  @ip = @options[:ip]
177
118
  @options[:port] ||= 0
178
119
  @options[:port] = @options[:port].to_i
@@ -188,12 +129,6 @@ module Bud
188
129
 
189
130
  init_state
190
131
 
191
- # NB: Somewhat hacky. Dependency analysis and stratification are implemented
192
- # by Bud programs, so in order for those programs to parse, we need the
193
- # "Bud" class to have been defined first.
194
- require 'bud/depanalysis'
195
- require 'bud/stratify'
196
-
197
132
  @viz = VizOnline.new(self) if @options[:trace]
198
133
  @rtracer = RTrace.new(self) if @options[:rtrace]
199
134
 
@@ -208,6 +143,7 @@ module Bud
208
143
  # array, so we need to convert it before loading the rewritten strata.
209
144
  @strata = []
210
145
  @rule_src = []
146
+ @rule_orig_src = []
211
147
  declaration
212
148
  @strata.each_with_index do |s,i|
213
149
  raise BudError if s.class <= Array
@@ -219,9 +155,11 @@ module Bud
219
155
  @rewritten_strata.each_with_index do |src_ary,i|
220
156
  @strata[i] ||= []
221
157
  @rule_src[i] ||= []
222
- src_ary.each do |src|
158
+ @rule_orig_src[i] ||= []
159
+ src_ary.each_with_index do |src, j|
223
160
  @strata[i] << eval("lambda { #{src} }")
224
161
  @rule_src[i] << src
162
+ @rule_orig_src[i] << @no_attr_rewrite_strata[i][j]
225
163
  end
226
164
  end
227
165
  end
@@ -313,7 +251,7 @@ module Bud
313
251
 
314
252
  def do_rewrite
315
253
  @meta_parser = BudMeta.new(self, @declarations)
316
- @rewritten_strata = @meta_parser.meta_rewrite
254
+ @rewritten_strata, @no_attr_rewrite_strata = @meta_parser.meta_rewrite
317
255
  end
318
256
 
319
257
  public
@@ -349,19 +287,25 @@ module Bud
349
287
  # Run Bud in the "foreground" -- the caller's thread will be used to run the
350
288
  # Bud interpreter. This means this method won't return unless an error
351
289
  # occurs. It is often more useful to run Bud asynchronously -- see run_bg.
352
- #
353
- # Note that run_fg cannot be invoked if run_bg has already been called in the
354
- # same Ruby process.
355
- #
356
- # Execution proceeds in time ticks, a la Dedalus.
357
- # * Within each tick there may be multiple strata.
358
- # * Within each stratum we do multiple semi-naive iterations.
359
290
  def run_fg
360
- raise BudError if EventMachine::reactor_running?
291
+ # If we're called from the EventMachine thread (and EM is running), blocking
292
+ # the current thread would imply deadlocking ourselves.
293
+ if Thread.current == EventMachine::reactor_thread and EventMachine::reactor_running?
294
+ raise BudError, "Cannot invoke run_fg from inside EventMachine"
295
+ end
361
296
 
362
- EventMachine::run {
363
- start_bud
364
- }
297
+ q = Queue.new
298
+ # Note that this must be a post-shutdown callback: if this is the only
299
+ # thread, then the program might exit after run_fg() returns. If run_fg()
300
+ # blocked on a normal shutdown callback, the program might exit before the
301
+ # other shutdown callbacks have a chance to run.
302
+ post_shutdown do
303
+ q.push(true)
304
+ end
305
+
306
+ run_bg
307
+ # Block caller's thread until Bud has shutdown
308
+ q.pop
365
309
  end
366
310
 
367
311
  # Shutdown a Bud instance that is running asynchronously. This method blocks
@@ -369,15 +313,34 @@ module Bud
369
313
  # loop is also shutdown; this will interfere with the execution of any other
370
314
  # Bud instances in the same process (as well as anything else that happens to
371
315
  # use EventMachine).
372
- def stop_bg(stop_em=false)
316
+ def stop_bg(stop_em=false, do_shutdown_cb=true)
317
+ schedule_and_wait do
318
+ do_shutdown(do_shutdown_cb)
319
+ end
320
+
373
321
  if stop_em
374
- schedule_shutdown(true)
375
- # Wait until EM has completely shutdown before we return.
376
- $em_stopped.pop
377
- else
378
- schedule_and_wait do
379
- do_shutdown(false)
380
- end
322
+ Bud.stop_em_loop
323
+ EventMachine::reactor_thread.join
324
+ end
325
+ end
326
+
327
+ # Register a callback that will be invoked when this instance of Bud is
328
+ # shutting down.
329
+ def on_shutdown(&blk)
330
+ # Start EM if not yet started
331
+ start_reactor
332
+ schedule_and_wait do
333
+ @shutdown_callbacks << blk
334
+ end
335
+ end
336
+
337
+ # Register a callback that will be invoked when *after* this instance of Bud
338
+ # has been shutdown.
339
+ def post_shutdown(&blk)
340
+ # Start EM if not yet started
341
+ start_reactor
342
+ schedule_and_wait do
343
+ @post_shutdown_callbacks << blk
381
344
  end
382
345
  end
383
346
 
@@ -414,7 +377,7 @@ module Bud
414
377
  end
415
378
 
416
379
  # Shutdown any persistent tables used by the current Bud instance. If you are
417
- # running Bud via tick() and using `tctable` collections, you should call this
380
+ # running Bud via tick() and using +tctable+ collections, you should call this
418
381
  # after you're finished using Bud. Programs that use Bud via run_fg() or
419
382
  # run_bg() don't need to call this manually.
420
383
  def close_tables
@@ -485,7 +448,7 @@ module Bud
485
448
  return result
486
449
  end
487
450
 
488
- # a common special case for sync_callback: block on a delta to a table.
451
+ # A common special case for sync_callback: block on a delta to a table.
489
452
  def delta(out_tbl)
490
453
  sync_callback(nil, nil, out_tbl)
491
454
  end
@@ -508,6 +471,7 @@ module Bud
508
471
  EventMachine::error_handler do |e|
509
472
  puts "Unexpected Bud error: #{e.inspect}"
510
473
  puts e.backtrace.join("\n")
474
+ Bud.shutdown_all_instances
511
475
  raise e
512
476
  end
513
477
 
@@ -517,10 +481,8 @@ module Bud
517
481
  # EventMachine's event loop.
518
482
  Thread.new do
519
483
  EventMachine.run do
520
- q << true
484
+ q.push(true)
521
485
  end
522
- # Executed only after EventMachine::stop_event_loop is done
523
- $em_stopped << true
524
486
  end
525
487
  # Block waiting for EM's event loop to start up.
526
488
  q.pop
@@ -529,9 +491,12 @@ module Bud
529
491
  # Schedule a block to be evaluated by EventMachine in the future, and
530
492
  # block until this has happened.
531
493
  def schedule_and_wait
532
- # Try to defend against error situations in which EM has stopped, but we've
533
- # been called nonetheless. This is racy, but better than nothing.
534
- raise BudError, "EM not running" unless EventMachine::reactor_running?
494
+ # If EM isn't running, just run the user's block immediately
495
+ # XXX: not clear that this is the right behavior
496
+ unless EventMachine::reactor_running?
497
+ yield
498
+ return
499
+ end
535
500
 
536
501
  q = Queue.new
537
502
  EventMachine::schedule do
@@ -548,43 +513,33 @@ module Bud
548
513
  raise resp if resp
549
514
  end
550
515
 
551
- def do_shutdown(stop_em=false)
552
- @timers.each do |t|
553
- t.cancel
516
+ def do_shutdown(do_shutdown_cb=true)
517
+ # Silently ignore duplicate shutdown requests or attempts to shutdown an
518
+ # instance that hasn't been started yet.
519
+ return if @instance_id == ILLEGAL_INSTANCE_ID
520
+
521
+ $signal_lock.synchronize {
522
+ raise unless $bud_instances.has_key? @instance_id
523
+ $bud_instances.delete @instance_id
524
+ @instance_id = ILLEGAL_INSTANCE_ID
525
+ }
526
+
527
+ if do_shutdown_cb
528
+ @shutdown_callbacks.each {|cb| cb.call}
554
529
  end
530
+ @timers.each {|t| t.cancel}
555
531
  close_tables
556
- @dsock.close_connection
557
- # Note that this affects anyone else in the same process who happens to be
558
- # using EventMachine! This is also a non-blocking call; to block until EM
559
- # has completely shutdown, we use the @em_stopped queue.
560
- EventMachine::stop_event_loop if stop_em
561
- end
562
-
563
- # Schedule a "graceful" shutdown for a future EM tick. If EM is not currently
564
- # running, shutdown immediately.
565
- def schedule_shutdown(stop_em=false)
566
- if EventMachine::reactor_running?
567
- EventMachine::schedule do
568
- do_shutdown(stop_em)
569
- end
570
- else
571
- do_shutdown(stop_em)
532
+ @dsock.close_connection if EventMachine::reactor_running?
533
+ if do_shutdown_cb
534
+ @post_shutdown_callbacks.each {|cb| cb.call}
572
535
  end
573
536
  end
574
537
 
538
+ private
575
539
  def start_bud
576
540
  raise BudError unless EventMachine::reactor_thread?
577
541
 
578
- # If we get SIGINT or SIGTERM, shutdown gracefully
579
- unless @options[:no_signal_handlers]
580
- Signal.trap("INT") do
581
- schedule_shutdown(true)
582
- end
583
- Signal.trap("TRAP") do
584
- schedule_shutdown(true)
585
- end
586
- end
587
-
542
+ @instance_id = Bud.init_signal_handlers(self)
588
543
  do_start_server
589
544
 
590
545
  # Initialize periodics
@@ -594,7 +549,7 @@ module Bud
594
549
 
595
550
  # Arrange for Bud to read from stdin if enabled. Note that we can't do this
596
551
  # earlier because we need to wait for EventMachine startup.
597
- @stdio.start_stdin_reader if @options[:read_stdin]
552
+ @stdio.start_stdin_reader if @options[:stdin]
598
553
  @zk_tables.each_value {|t| t.start_watchers}
599
554
 
600
555
  # Compute a fixpoint; this will also invoke any bootstrap blocks.
@@ -611,12 +566,13 @@ module Bud
611
566
 
612
567
  public
613
568
 
614
- # Returns the ip and port of the Bud instance. In addition to the local IP
615
- # and port, the user may define an external IP and/or port. the external
616
- # version of each is returned if available. If not, the local version is
617
- # returned. There are use cases for mixing and matching local and external.
618
- # local_ip:external_port would be if you have local port forwarding, and
619
- # external_ip:local_port would be if you're in a DMZ, for example
569
+ # Returns the IP and port of the Bud instance as a string. In addition to the
570
+ # local IP and port, the user may define an external IP and/or port. The
571
+ # external version of each is returned if available. If not, the local
572
+ # version is returned. There are use cases for mixing and matching local and
573
+ # external. local_ip:external_port would be if you have local port
574
+ # forwarding, and external_ip:local_port would be if you're in a DMZ, for
575
+ # example.
620
576
  def ip_port
621
577
  raise BudError, "ip_port called before port defined" if @port.nil? and @options[:port] == 0 and not @options[:ext_port]
622
578
 
@@ -626,28 +582,43 @@ module Bud
626
582
  ip + ":" + port
627
583
  end
628
584
 
629
- # Returns the internal IP and port. See ip_port
585
+ # Returns the internal IP and port. See ip_port.
630
586
  def int_ip_port
631
587
  raise BudError, "ip_port called before port defined" if @port.nil? and @options[:port] == 0
632
588
  @port.nil? ? "#{@ip}:#{@options[:port]}" : "#{@ip}:#{@port}"
633
589
  end
634
590
 
635
- # manually trigger one timestep of Bloom execution.
591
+ # Manually trigger one timestep of Bloom execution.
636
592
  def tick
637
- @tables.each_value do |t|
638
- t.tick
639
- end
640
-
641
- @joinstate = {}
593
+ begin
594
+ @inside_tick = true
595
+ @tables.each_value do |t|
596
+ t.tick
597
+ end
598
+
599
+ @joinstate = {}
600
+
601
+ do_bootstrap unless @done_bootstrap
602
+ receive_inbound
642
603
 
643
- do_bootstrap unless @done_bootstrap
644
- receive_inbound
604
+ @strata.each_with_index { |s,i| stratum_fixpoint(s, i) }
605
+ @viz.do_cards if @options[:trace]
606
+ do_flush
607
+ invoke_callbacks
608
+ @budtime += 1
609
+ ensure
610
+ @inside_tick = false
611
+ @tick_clock_time = nil
612
+ end
613
+ end
645
614
 
646
- @strata.each_with_index { |s,i| stratum_fixpoint(s, i) }
647
- @viz.do_cards if @options[:trace]
648
- do_flush
649
- invoke_callbacks
650
- @budtime += 1
615
+ # Returns the wallclock time associated with the current Bud tick. That is,
616
+ # this value is guaranteed to remain the same for the duration of a single
617
+ # tick, but will likely change between ticks.
618
+ def bud_clock
619
+ raise BudError, "bud_clock undefined outside tick" unless @inside_tick
620
+ @tick_clock_time ||= Time.now
621
+ @tick_clock_time
651
622
  end
652
623
 
653
624
  private
@@ -656,18 +627,20 @@ module Bud
656
627
  # standard "state" syntax, but we want to ensure that builtin state is
657
628
  # initialized before user-defined state.
658
629
  def builtin_state
659
- channel :localtick, [:col1]
630
+ loopback :localtick, [:col1]
660
631
  @stdio = terminal :stdio
661
632
  @periodics = table :periodics_tbl, [:pername] => [:ident, :period]
662
633
 
663
634
  # for BUD reflection
664
- table :t_rules, [:rule_id] => [:lhs, :op, :src]
635
+ table :t_rules, [:rule_id] => [:lhs, :op, :src, :orig_src]
665
636
  table :t_depends, [:rule_id, :lhs, :op, :body] => [:nm]
666
637
  table :t_depends_tc, [:head, :body, :via, :neg, :temporal]
667
638
  table :t_provides, [:interface] => [:input]
668
639
  table :t_underspecified, t_provides.schema
669
640
  table :t_stratum, [:predicate] => [:stratum]
670
641
  table :t_cycle, [:predicate, :via, :neg, :temporal]
642
+ table :t_table_info, [:tab_name, :tab_type]
643
+ table :t_table_schema, [:tab_name, :col_name, :ord, :loc]
671
644
  end
672
645
 
673
646
  # Handle any inbound tuples off the wire and then clear. Received messages are
@@ -684,14 +657,15 @@ module Bud
684
657
  # 1. Emit outgoing tuples in channels and ZK tables.
685
658
  # 2. Commit to disk any changes made to on-disk tables.
686
659
  def do_flush
687
- @channels.each { |c| @tables[c[0]].flush }
660
+ @channels.each_value { |c| c.flush }
688
661
  @zk_tables.each_value { |t| t.flush }
689
662
  @tc_tables.each_value { |t| t.flush }
663
+ @dbm_tables.each_value { |t| t.flush }
690
664
  end
691
665
 
692
666
  def stratum_fixpoint(strat, strat_num)
693
- # This routine uses semi-naive evaluation to compute
694
- # a fixpoint of the rules in strat.
667
+ # This routine uses semi-naive evaluation to compute a fixpoint of the rules
668
+ # in strat.
695
669
  #
696
670
  # As described in lib/collections.rb, each collection has three
697
671
  # sub-collections of note here:
@@ -699,13 +673,13 @@ module Bud
699
673
  # @delta: tuples that should be used to drive derivation of new facts
700
674
  # @new_delta: a place to store newly-derived facts
701
675
  #
702
- # The first time through this loop we mark @stratum_first_iter=true,
703
- # while tells the Join::each code to join up all its @storage subcollections
704
- # to start. In subsequent iterations the join code uses some table's @delta
705
- # to ensure that only new tuples are derived.
676
+ # The first time through this loop we mark @stratum_first_iter=true, which
677
+ # tells the Join::each code to join up all its @storage subcollections to
678
+ # start. In subsequent iterations the join code uses some table's @delta to
679
+ # ensure that only new tuples are derived.
706
680
  #
707
- # Note that calling "each" on a non-Join collection will iterate through both
708
- # storage and delta.
681
+ # Note that calling "each" on a non-Join collection will iterate through
682
+ # both storage and delta.
709
683
  #
710
684
  # At the end of each iteration of this loop we transition:
711
685
  # - @delta tuples are merged into @storage
@@ -713,22 +687,22 @@ module Bud
713
687
  # - @new_delta is set to empty
714
688
  #
715
689
  # XXX as a performance optimization, it would be nice to bypass the delta
716
- # tables for any preds that don't participate in a rhs Join -- in that
717
- # case there's pointless extra tuple movement letting tuples "graduate"
718
- # through @new_delta and @delta.
719
-
720
- # In semi-naive, the first iteration should join up tables
721
- # on their storage fields; subsequent iterations do the
722
- # delta-joins only. The stratum_first_iter field here distinguishes
723
- # these cases.
690
+ # tables for any preds that don't participate in a rhs Join -- in that case
691
+ # there's pointless extra tuple movement letting tuples "graduate" through
692
+ # @new_delta and @delta.
693
+
694
+ # In semi-naive, the first iteration should join up tables on their storage
695
+ # fields; subsequent iterations do the delta-joins only. The
696
+ # stratum_first_iter field here distinguishes these cases.
724
697
  @stratum_first_iter = true
725
698
  begin
726
699
  strat.each_with_index do |r,i|
700
+ fixpoint = false
727
701
  begin
728
702
  r.call
729
703
  rescue Exception => e
730
704
  # Don't report source text for certain rules (old-style rule blocks)
731
- rule_src = @rule_src[strat_num][i]
705
+ rule_src = @rule_orig_src[strat_num][i] unless @rule_orig_src[strat_num].nil?
732
706
  src_msg = ""
733
707
  unless rule_src == ""
734
708
  src_msg = "\nRule: #{rule_src}"
@@ -742,57 +716,110 @@ module Bud
742
716
  end
743
717
  end
744
718
  @stratum_first_iter = false
745
- # XXX this next line is inefficient.
746
- # we could call tick_deltas only on predicates in this stratum.
747
- # but it's not easy right now (??) to pull out tables in a given stratum
748
- @tables.each{|name,coll| coll.tick_deltas}
749
- end while not @tables.all?{|name,coll| coll.new_delta.empty? and coll.delta.empty?}
719
+ fixpoint = true
720
+ # tick collections in this stratum; if we don't have info on that, tick all collections
721
+ colls = @stratum_collection_map[strat_num] if @stratum_collection_map
722
+ colls ||= @tables.keys
723
+ colls.each do |name|
724
+ begin
725
+ coll = self.send(name)
726
+ unless coll.delta.empty? and coll.new_delta.empty?
727
+ coll.tick_deltas
728
+ fixpoint = false
729
+ end
730
+ rescue
731
+ # ignore missing tables; rebl for example deletes them mid-stream
732
+ end
733
+ end
734
+ end while not fixpoint
750
735
  end
751
736
 
752
- ####### Joins
753
- def wrap_map(j, &blk)
754
- if blk.nil?
755
- return j
756
- else
757
- return j.map(&blk)
758
- end
759
- end
737
+ private
760
738
 
761
- public
762
- def joinstate # :nodoc: all
763
- @joinstate
739
+ ######## ids and timers
740
+ def gen_id
741
+ Time.new.to_i.to_s << rand.to_s
764
742
  end
765
743
 
766
- public
767
- def join(collections, *preds, &blk) # :nodoc: all
768
- # since joins are stateful, we want to allocate them once and store in this Bud instance
769
- # we ID them on their tablenames, preds, and block
770
- return wrap_map(BudJoin.new(collections, self, preds), &blk)
744
+ def set_periodic_timer(name, id, period)
745
+ EventMachine::PeriodicTimer.new(period) do
746
+ @tables[name].add_periodic_tuple(id)
747
+ tick
748
+ end
771
749
  end
772
750
 
773
- def natjoin(collections, &blk) # :nodoc: all
774
- # for all pairs of relations, add predicates on matching column names
775
- preds = BudJoin::natural_preds(self, collections)
776
- join(collections, *preds, &blk)
751
+ # Fork a new process. This is identical to Kernel#fork, except that it also
752
+ # cleans up Bud and EventMachine-related state. As with Kernel#fork, the
753
+ # caller supplies a code block that is run in the child process; the PID of
754
+ # the child is returned by this method.
755
+ def self.do_fork
756
+ Kernel.fork do
757
+ srand
758
+ # This is somewhat grotty: we basically clone what EM::fork_reactor does,
759
+ # except that we don't want the user-supplied block to be invoked by the
760
+ # reactor thread.
761
+ if EventMachine::reactor_running?
762
+ EventMachine::stop_event_loop
763
+ EventMachine::release_machine
764
+ EventMachine::instance_variable_set('@reactor_running', false)
765
+ end
766
+ # Shutdown all the Bud instances inherited from the parent process, but
767
+ # don't invoke their shutdown callbacks
768
+ Bud.shutdown_all_instances(false)
769
+
770
+ $got_shutdown_signal = false
771
+ $setup_signal_handler = false
772
+
773
+ yield
774
+ end
777
775
  end
778
776
 
779
- # left-outer-join syntax to be used in rhs of Bloom statements.
780
- # first argument an array of 2 collections, second argument an array of predicates (as in Bud::BudCollection.pairs)
781
- def leftjoin(collections, *preds, &blk)
782
- return wrap_map(BudLeftJoin.new(collections, self, preds), &blk)
777
+ # Note that this affects anyone else in the same process who happens to be
778
+ # using EventMachine! This is also a non-blocking call; to block until EM
779
+ # has completely shutdown, join on EM::reactor_thread.
780
+ def self.stop_em_loop
781
+ EventMachine::stop_event_loop
782
+
783
+ # If another instance of Bud is started later, we'll need to reinitialize
784
+ # the signal handlers (since they depend on EM).
785
+ $signal_handler_setup = false
783
786
  end
784
787
 
785
- private
788
+ # Signal handling. If multiple Bud instances are running inside a single
789
+ # process, we want a SIGINT or SIGTERM signal to cleanly shutdown all of them.
790
+ def self.init_signal_handlers(b)
791
+ $signal_lock.synchronize {
792
+ # If we setup signal handlers and then fork a new process, we want to
793
+ # reinitialize the signal handler in the child process.
794
+ unless b.options[:no_signal_handlers] or $signal_handler_setup
795
+ EventMachine::PeriodicTimer.new(SIGNAL_CHECK_PERIOD) do
796
+ if $got_shutdown_signal
797
+ Bud.shutdown_all_instances
798
+ Bud.stop_em_loop
799
+ $got_shutdown_signal = false
800
+ end
801
+ end
786
802
 
787
- ######## ids and timers
788
- def gen_id
789
- Time.new.to_i.to_s << rand.to_s
803
+ ["INT", "TERM"].each do |signal|
804
+ Signal.trap(signal) {
805
+ $got_shutdown_signal = true
806
+ }
807
+ end
808
+ $setup_signal_handler_pid = true
809
+ end
810
+
811
+ $instance_id += 1
812
+ $bud_instances[$instance_id] = b
813
+ return $instance_id
814
+ }
790
815
  end
791
816
 
792
- def set_periodic_timer(name, id, period)
793
- EventMachine::PeriodicTimer.new(period) do
794
- @tables[name] <+ [[id, Time.new.to_s]]
795
- tick
796
- end
817
+ def self.shutdown_all_instances(do_shutdown_cb=true)
818
+ instances = nil
819
+ $signal_lock.synchronize {
820
+ instances = $bud_instances.clone
821
+ }
822
+
823
+ instances.each_value {|b| b.stop_bg(false, do_shutdown_cb) }
797
824
  end
798
825
  end