bud 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (62) hide show
  1. data/LICENSE +9 -0
  2. data/README +30 -0
  3. data/bin/budplot +134 -0
  4. data/bin/budvis +201 -0
  5. data/bin/rebl +4 -0
  6. data/docs/README.md +13 -0
  7. data/docs/bfs.md +379 -0
  8. data/docs/bfs.raw +251 -0
  9. data/docs/bfs_arch.png +0 -0
  10. data/docs/bloom-loop.png +0 -0
  11. data/docs/bust.md +83 -0
  12. data/docs/cheat.md +291 -0
  13. data/docs/deploy.md +96 -0
  14. data/docs/diffs +181 -0
  15. data/docs/getstarted.md +296 -0
  16. data/docs/intro.md +36 -0
  17. data/docs/modules.md +112 -0
  18. data/docs/operational.md +96 -0
  19. data/docs/rebl.md +99 -0
  20. data/docs/ruby_hooks.md +19 -0
  21. data/docs/visualizations.md +75 -0
  22. data/examples/README +1 -0
  23. data/examples/basics/hello.rb +12 -0
  24. data/examples/basics/out +1103 -0
  25. data/examples/basics/out.new +856 -0
  26. data/examples/basics/paths.rb +51 -0
  27. data/examples/bust/README.md +9 -0
  28. data/examples/bust/bustclient-example.rb +23 -0
  29. data/examples/bust/bustinspector.html +135 -0
  30. data/examples/bust/bustserver-example.rb +18 -0
  31. data/examples/chat/README.md +9 -0
  32. data/examples/chat/chat.rb +45 -0
  33. data/examples/chat/chat_protocol.rb +8 -0
  34. data/examples/chat/chat_server.rb +29 -0
  35. data/examples/deploy/tokenring-ec2.rb +26 -0
  36. data/examples/deploy/tokenring-local.rb +17 -0
  37. data/examples/deploy/tokenring.rb +39 -0
  38. data/lib/bud/aggs.rb +126 -0
  39. data/lib/bud/bud_meta.rb +185 -0
  40. data/lib/bud/bust/bust.rb +126 -0
  41. data/lib/bud/bust/client/idempotence.rb +10 -0
  42. data/lib/bud/bust/client/restclient.rb +49 -0
  43. data/lib/bud/collections.rb +937 -0
  44. data/lib/bud/depanalysis.rb +44 -0
  45. data/lib/bud/deploy/countatomicdelivery.rb +50 -0
  46. data/lib/bud/deploy/deployer.rb +67 -0
  47. data/lib/bud/deploy/ec2deploy.rb +200 -0
  48. data/lib/bud/deploy/localdeploy.rb +41 -0
  49. data/lib/bud/errors.rb +15 -0
  50. data/lib/bud/graphs.rb +405 -0
  51. data/lib/bud/joins.rb +300 -0
  52. data/lib/bud/rebl.rb +314 -0
  53. data/lib/bud/rewrite.rb +523 -0
  54. data/lib/bud/rtrace.rb +27 -0
  55. data/lib/bud/server.rb +43 -0
  56. data/lib/bud/state.rb +108 -0
  57. data/lib/bud/storage/tokyocabinet.rb +170 -0
  58. data/lib/bud/storage/zookeeper.rb +178 -0
  59. data/lib/bud/stratify.rb +83 -0
  60. data/lib/bud/viz.rb +65 -0
  61. data/lib/bud.rb +797 -0
  62. metadata +330 -0
data/lib/bud/rebl.rb ADDED
@@ -0,0 +1,314 @@
1
+ #!/usr/bin/env ruby
2
+ require 'readline'
3
+ require 'rubygems'
4
+ require 'bud'
5
+ require 'abbrev'
6
+
7
+ TABLE_TYPES = ["table", "scratch", "channel"]
8
+
9
+ # The class to which rebl adds user-specified rules and declarations.
10
+ class ReblClass
11
+ include Bud
12
+ attr_accessor:port, :ip
13
+
14
+ # Support for breakpoints
15
+ state { scratch :rebl_breakpoint }
16
+ end
17
+
18
+
19
+ # Static class that contains constants and functions for the rebl shell.
20
+ class ReblShell
21
+ @@histfile = File::expand_path("~/.rebl_history")
22
+ @@maxhistsize = 100
23
+ @@escape_char = '/'
24
+ @@commands =
25
+ {"tick" => [lambda {|lib,argv| lib.tick([Integer(argv[1]), 1].max)},
26
+ "tick [x]:\texecutes x (or 1) timesteps"],
27
+
28
+ "run" => [lambda {|lib,_| lib.run},
29
+ "run:\ttick until quiescence, a breakpoint, or #{@@escape_char}stop"],
30
+
31
+ "stop" => [lambda {|lib,_| lib.stop},
32
+ "stop:\tstop ticking"],
33
+
34
+ "lsrules" => [lambda {|lib,_| lib.rules.sort{|a,b| a[0] <=> b[0]}.each {|k,v| puts "#{k}: "+v}},
35
+ "lsrules:\tlist rules"],
36
+
37
+ "rmrule" => [lambda {|lib,argv| lib.del_rule(Integer(argv[1]))},
38
+ "rmrule x:\tremove rule number x"],
39
+
40
+ "lscollections" => [lambda {|lib,_| lib.state.sort{|a,b| a[0] <=> b[0]}.each {|k,v| puts "#{k}: "+v}},
41
+ "lscollections:\tlist collections"],
42
+
43
+ "dump" => [lambda {|lib,argv| lib.dump(argv[1])},
44
+ "dump c:\tdump contents of collection c"],
45
+
46
+ "exit" => [lambda {|_,_| do_exit}, "exit:\texit rebl"],
47
+
48
+ "quit" => [lambda {|_,_| do_exit}, "quit:\texit rebl"],
49
+
50
+ "help" => [lambda {|_,_| pretty_help},
51
+ "help:\tprint this help message"]}
52
+ @@abbrevs = @@commands.keys.abbrev
53
+ @@exit_message = "Rebellion quashed."
54
+
55
+ # Starts a rebl shell.
56
+ #--
57
+ # This function is not covered by testcases, but setup
58
+ # and rebl_loop are.
59
+ #++
60
+ def self.run
61
+ lib = setup
62
+ loop do
63
+ begin
64
+ rebl_loop(lib)
65
+ rescue Exception
66
+ puts "exception: #{$!}"
67
+ end
68
+ end
69
+ end
70
+
71
+ # Performs setup as part of starting a rebl shell, and returns the instance of
72
+ # LibRebl that is created; testcases call this directly.
73
+ def self.setup
74
+ Signal.trap("INT") {do_exit}
75
+ Signal.trap("TRAP") {do_exit}
76
+
77
+ ipport = ARGV[0] ? ARGV[0].split(":") : []
78
+ lib = LibRebl.new(*[(ipport[0] or "localhost"), (ipport[1] or 0)])
79
+ setup_history
80
+
81
+ comp = proc do |s|
82
+ @@commands.keys.map do |c|
83
+ @@escape_char+c
84
+ end.grep( /^#{Regexp.escape(s)}/ )
85
+ end
86
+ Readline.completion_append_character = ' '
87
+ Readline.completion_proc = comp
88
+
89
+ welcome
90
+ return lib
91
+ end
92
+
93
+ # One step of the rebl shell loop: processes one rebl shell line from stdin
94
+ # and returns. May raise an Exception.
95
+ def self.rebl_loop(lib,noreadline=false)
96
+ begin
97
+ line = Readline::readline('rebl> ') unless noreadline
98
+ line = gets if noreadline
99
+ do_exit if line.nil?
100
+ line = line.lstrip.rstrip
101
+ Readline::HISTORY.push(line) unless noreadline
102
+ split_line = line.split(" ")
103
+ if line[0..0] == @@escape_char then
104
+ # Command
105
+ split_line[0].slice! 0
106
+ if command split_line[0]
107
+ command(split_line[0]).call(lib, split_line)
108
+ else
109
+ puts "invalid command or ambiguous command prefix"
110
+ end
111
+ elsif TABLE_TYPES.include? split_line[0]
112
+ # Collection
113
+ lib.add_collection(line)
114
+ else
115
+ # Rule
116
+ lib.add_rule(line)
117
+ end
118
+ rescue Interrupt
119
+ abort(do_exit)
120
+ end
121
+ end
122
+
123
+ # Reads permanent history from @@histfile. This code is pretty much the same
124
+ # as irb's code.
125
+ def self.setup_history
126
+ begin
127
+ if File::exists?(@@histfile)
128
+ lines = IO::readlines(@@histfile).collect { |line| line.chomp }
129
+ Readline::HISTORY.push(*lines)
130
+ end
131
+ rescue Exception
132
+ puts "Error when configuring permanent history: #{$!}"
133
+ end
134
+ end
135
+
136
+ # lookup full command from abbreviation
137
+ def self.command(c)
138
+ return @@abbrevs[c].nil? ? nil : @@commands[@@abbrevs[c]][0]
139
+ end
140
+
141
+ private
142
+ # pretty-printed help
143
+ def self.pretty_help
144
+ puts "rebl commands are prefixed by '#{@@escape_char}'"
145
+ puts "other input is interpreted as Bloom code."
146
+ puts
147
+ puts "rebl commands:"
148
+ maxlen = @@commands.keys.sort{|a,b| b.size - a.size}.first.size
149
+ cmd_list = @@commands.keys.sort
150
+ cmd_list.each do |c|
151
+ v = @@commands[c]
152
+ puts @@escape_char +
153
+ v[1].gsub(/\t/, " "*(maxlen + 3 - v[1].split(':')[0].size))
154
+ end
155
+ puts "\nbreakpoints:"
156
+ puts "a breakpoint is a rule with the 'breakpoint' scratch on the left of "+
157
+ "a '<=' operator.\n'#{@@escape_char}run' will stop ticking at the end of a "+
158
+ "timestep where a 'breakpoint' tuple exists."
159
+ end
160
+
161
+ private
162
+ def self.welcome
163
+ puts "Welcome to rebl, the interactive Bloom terminal."
164
+ puts
165
+ puts "Type: " + @@escape_char + "h for help"
166
+ puts " " + @@escape_char + "q to quit"
167
+ puts
168
+ end
169
+
170
+ private
171
+ # Called on exit. Writes the session's history to @@histfile, and stops the
172
+ # bud instance from listening.
173
+ def self.do_exit
174
+ begin
175
+ lines = Readline::HISTORY.to_a.reverse.uniq.reverse
176
+ lines = lines[-@@maxhistsize, @@maxhistsize] if lines.nitems>@@maxhistsize
177
+ File::open(@@histfile, File::WRONLY|File::CREAT|File::TRUNC) do |io|
178
+ io.puts lines.join("\n")
179
+ end
180
+ rescue Exception
181
+ puts "Error when saving permanent history: #{$!}"
182
+ end
183
+ @rebl_class_inst.stop_bg if @rebl_class_inst
184
+ puts "\n" + @@exit_message
185
+ exit!
186
+ end
187
+ end
188
+
189
+
190
+ # Library of functions used by rebl. More generally, this can be viewed as a
191
+ # way to have a bud class that you can add and remove rules from, and that you
192
+ # can step through the execution of.
193
+ class LibRebl
194
+ attr_accessor :rules, :state
195
+ attr_reader :ip, :port, :rebl_class_inst
196
+ @@builtin_tables = [:stdio, :t_depends, :periodics_tbl, :t_cycle, :localtick,
197
+ :t_provides, :t_rules, :t_depends_tc, :t_stratum,
198
+ :rebl_breakpoint]
199
+
200
+ def initialize(ip, port)
201
+ @ip = ip
202
+ @port = port
203
+ @rules = {}
204
+ @ruleid = 0
205
+ @state = {}
206
+ @stateid = 0
207
+ @rebl_class = nil
208
+ @rebl_class_inst = nil
209
+ @old_inst = nil
210
+ reinstantiate
211
+ end
212
+
213
+ # Runs the bud instance (until a breakpoint, or stop() is called)
214
+ def run
215
+ @rebl_class_inst.sync_do {@rebl_class_inst.lazy = false}
216
+ end
217
+
218
+ # Stops the bud instance (and then performs another tick)
219
+ def stop
220
+ @rebl_class_inst.sync_do {@rebl_class_inst.lazy = true}
221
+ end
222
+
223
+ # Ticks the bud instance a specified integer number of times.
224
+ def tick(x)
225
+ x.times {@rebl_class_inst.sync_do}
226
+ end
227
+
228
+ # Dumps the contents of a table at the current time.
229
+ def dump(c)
230
+ tups = @rebl_class_inst.instance_eval("#{c}.inspected")
231
+ puts(tups.empty? ? "(empty)" : tups.sort.join("\n"))
232
+ end
233
+
234
+ # Declares a new collection.
235
+ def add_collection(c)
236
+ @state[@stateid += 1] = c
237
+ begin
238
+ reinstantiate
239
+ rescue Exception
240
+ @state.delete(@stateid)
241
+ raise
242
+ end
243
+ end
244
+
245
+ # Deactivates a rule at the current time; any tuples derived by the rule at
246
+ # a previous time are still available.
247
+ def del_rule(rid)
248
+ @rules.delete(rid)
249
+ reinstantiate
250
+ end
251
+
252
+ # Adds a new rule at the current time; only derives tuples based on data that
253
+ # exists at the current or a future time.
254
+ def add_rule(r)
255
+ @rules[@ruleid += 1] = r
256
+ begin
257
+ reinstantiate
258
+ rescue Exception
259
+ @rules.delete(@ruleid)
260
+ raise
261
+ end
262
+ end
263
+
264
+ private
265
+ def reinstantiate
266
+ # New anonymous subclass.
267
+ @rebl_class = Class.new(ReblClass)
268
+
269
+ begin
270
+ if not @rules.empty?
271
+ @rebl_class.class_eval("bloom :rebl_rules do\n" +
272
+ @rules.sort.map {|_,r| r}.join("\n") + "\nend")
273
+ end
274
+ if not @state.empty?
275
+ @rebl_class.class_eval("state do\n" + @state.values.join("\n") + "\nend")
276
+ end
277
+ rescue
278
+ raise
279
+ end
280
+
281
+ @old_inst = @rebl_class_inst
282
+ @rebl_class_inst = @rebl_class.new(:no_signal_handlers => true, :ip => @ip,
283
+ :port => @port, :lazy => true)
284
+
285
+ # Copy the tables over.
286
+ if @old_inst
287
+ @rebl_class_inst.tables.merge!(@old_inst.tables.reject do |k,v|
288
+ @@builtin_tables.include? k
289
+ end)
290
+ # Fix the bud instance pointers from copied tables.
291
+ @rebl_class_inst.tables.values.each do |v|
292
+ v.bud_instance = @rebl_class_inst
293
+ end
294
+ end
295
+
296
+ # Run lazily in background, shutting down old instance.
297
+ begin
298
+ @old_inst.stop_bg if @old_inst
299
+ # Lazify the instance upon a breakpoint (no effect if instance is
300
+ # already lazy)
301
+ @rebl_class_inst.register_callback(:rebl_breakpoint) do
302
+ @rebl_class_inst.lazy = true
303
+ end
304
+ @rebl_class_inst.run_bg
305
+ @ip = @rebl_class_inst.ip
306
+ @port = @rebl_class_inst.port
307
+ puts "Listening on #{@rebl_class_inst.ip_port}" if not @old_inst
308
+ rescue
309
+ # The above two need to be atomic, or we're in trouble.
310
+ puts "unrecoverable error, please file a bug: #{$!}"
311
+ abort
312
+ end
313
+ end
314
+ end