bud 0.0.8 → 0.1.0.pre1

metadata

@@ -1,27 +1,30 @@
 --- !ruby/object:Gem::Specification 
 name: bud
 version: !ruby/object:Gem::Version 
-  hash: 15
-  prerelease: 
+  hash: 1923831995
+  prerelease: 6
   segments: 
   - 0
+  - 1
   - 0
-  - 8
-  version: 0.0.8
+  - pre
+  - 1
+  version: 0.1.0.pre1
 platform: ruby
 authors: 
 - Peter Alvaro
 - Neil Conway
 - Joseph M. Hellerstein
 - William R. Marczak
+- Sriram Srinivasan
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2011-10-10 00:00:00 Z
+date: 2012-03-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: backports
+  name: eventmachine
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
@@ -35,7 +38,7 @@ dependencies:
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: eventmachine
+  name: fastercsv
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -49,7 +52,7 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: fastercsv
+  name: gchart
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -63,7 +66,7 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: gchart
+  name: getopt
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -77,7 +80,7 @@ dependencies:
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: getopt
+  name: i18n
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -91,7 +94,7 @@ dependencies:
   type: :runtime
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: i18n
+  name: json
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
@@ -105,7 +108,7 @@ dependencies:
   type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
-  name: json
+  name: minitest
   prerelease: false
   requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
@@ -147,7 +150,7 @@ dependencies:
   type: :runtime
   version_requirements: *id009
 - !ruby/object:Gem::Dependency 
-  name: ParseTree
+  name: ruby-graphviz
   prerelease: false
   requirement: &id010 !ruby/object:Gem::Requirement 
     none: false
@@ -160,24 +163,10 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id010
-- !ruby/object:Gem::Dependency 
-  name: ruby-graphviz
-  prerelease: false
-  requirement: &id011 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id011
 - !ruby/object:Gem::Dependency 
   name: ruby2ruby
   prerelease: false
-  requirement: &id012 !ruby/object:Gem::Requirement 
+  requirement: &id011 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - <
@@ -189,11 +178,11 @@ dependencies:
         - 1
         version: 1.3.1
   type: :runtime
-  version_requirements: *id012
+  version_requirements: *id011
 - !ruby/object:Gem::Dependency 
-  name: sexp_path
+  name: ruby_parser
   prerelease: false
-  requirement: &id013 !ruby/object:Gem::Requirement 
+  requirement: &id012 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -203,11 +192,11 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  version_requirements: *id013
+  version_requirements: *id012
 - !ruby/object:Gem::Dependency 
-  name: superators
+  name: superators19
   prerelease: false
-  requirement: &id014 !ruby/object:Gem::Requirement 
+  requirement: &id013 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -217,11 +206,11 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  version_requirements: *id014
+  version_requirements: *id013
 - !ruby/object:Gem::Dependency 
   name: syntax
   prerelease: false
-  requirement: &id015 !ruby/object:Gem::Requirement 
+  requirement: &id014 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -231,11 +220,11 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  version_requirements: *id015
+  version_requirements: *id014
 - !ruby/object:Gem::Dependency 
   name: uuid
   prerelease: false
-  requirement: &id016 !ruby/object:Gem::Requirement 
+  requirement: &id015 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -245,7 +234,7 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  version_requirements: *id016
+  version_requirements: *id015
 description: A prototype of the Bloom distributed programming language, as a Ruby DSL.
 email: 
 - bloomdevs@gmail.com
@@ -267,11 +256,10 @@ files:
 - lib/bud/collections.rb
 - lib/bud/depanalysis.rb
 - lib/bud/deploy/countatomicdelivery.rb
-- lib/bud/deploy/deployer.rb
-- lib/bud/deploy/ec2deploy.rb
-- lib/bud/deploy/forkdeploy.rb
-- lib/bud/deploy/threaddeploy.rb
 - lib/bud/errors.rb
+- lib/bud/executor/elements.rb
+- lib/bud/executor/group.rb
+- lib/bud/executor/join.rb
 - lib/bud/graphs.rb
 - lib/bud/joins.rb
 - lib/bud/meta_algebra.rb
@@ -281,11 +269,10 @@ files:
 - lib/bud/rewrite.rb
 - lib/bud/rtrace.rb
 - lib/bud/server.rb
+- lib/bud/source.rb
 - lib/bud/state.rb
 - lib/bud/storage/dbm.rb
-- lib/bud/storage/tokyocabinet.rb
 - lib/bud/storage/zookeeper.rb
-- lib/bud/stratify.rb
 - lib/bud/viz.rb
 - lib/bud/viz_util.rb
 - lib/bud.rb
@@ -317,10 +304,6 @@ files:
 - examples/chat/chat_protocol.rb
 - examples/chat/chat_server.rb
 - examples/chat/README.md
-- examples/deploy/tokenring-ec2.rb
-- examples/deploy/tokenring-fork.rb
-- examples/deploy/tokenring-thread.rb
-- examples/deploy/tokenring.rb
 - examples/README
 - README
 - LICENSE
@@ -335,7 +318,7 @@ require_paths:
 required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ~>
+  - - ">="
     - !ruby/object:Gem::Version 
       hash: 57
       segments: 
@@ -346,16 +329,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
-      hash: 3
+      hash: 25
       segments: 
-      - 0
-      version: "0"
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
 requirements: []
 
 rubyforge_project: bloom-lang
-rubygems_version: 1.8.6
+rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: A prototype Bloom DSL for distributed programming.

data/examples/deploy/tokenring-ec2.rb

@@ -1,26 +0,0 @@
-require 'rubygems'
-require 'bud'
-require 'bud/deploy/ec2deploy'
-require 'tokenring'
-
-class RingEC2
-  include Bud
-  include TokenRing
-  include EC2Deploy
-
-  deploystrap do
-    raise "keys.rb must exist in the current directory" unless File.exists? "keys.rb"
-    eval(IO.read('keys.rb'), binding)
-    node_count << [10]
-    ruby_command << ["ruby tokenring-ec2.rb"]
-    init_dir << ["."]
-  end
-end
-
-ip, port = ARGV[0].split(':')
-ext_ip, ext_port = ARGV[1].split(':')
-RingEC2.new(:ip => ip,
-            :port => port,
-            :ext_ip => ext_ip,
-            :ext_port => ext_port,
-            :deploy => ARGV[2]).run_fg

data/examples/deploy/tokenring-fork.rb

@@ -1,15 +0,0 @@
-require 'rubygems'
-require 'bud'
-require 'tokenring'
-
-class RingFork
-  include Bud
-  include TokenRing
-  include ForkDeploy
-
-  deploystrap do
-    node_count << [10]
-  end
-end
-
-RingFork.new(:deploy => true).run_fg

data/examples/deploy/tokenring-thread.rb

@@ -1,15 +0,0 @@
-require 'rubygems'
-require 'bud'
-require 'tokenring'
-
-class RingThread
-  include Bud
-  include TokenRing
-  include ThreadDeploy
-
-  deploystrap do
-    node_count << [10]
-  end
-end
-
-RingThread.new(:deploy => true).run_fg

data/examples/deploy/tokenring.rb

@@ -1,47 +0,0 @@
-require 'rubygems'
-require 'bud'
-
-module TokenRing
-  state do
-    table :next_node, [] => [:addr]
-    channel :next_node_chan, [:@loc] => [:next]
-    scratch :send_next_node, [:node, :next]
-    table :sent_next_node, [:addr]
-    channel :token, [:@loc]
-    table :token_persist, [:loc]
-  end
-
-  bloom :make_ring do
-    # Once a node and its successor have both been computed, send a message to
-    # the node with the address of its successor
-    send_next_node <= (node * node * node_ready).combos do |n1, n2, nr|
-      succ_id = (n1.uid + 1) % node_count[[]].num
-      if n2.uid == succ_id and not sent_next_node.has_key? [n1.uid]
-        [n1.addr, n2.addr]
-      end
-    end
-    next_node_chan <~ send_next_node
-    sent_next_node <+ send_next_node {|n| [n.node]}
-
-    next_node <= next_node_chan {|n| [n.next]}
-
-    # The deployer sends an initial message to the node with ID 0
-    token <~ (node_ready * node).rights(:uid => :uid) do |n|
-      [n.addr] if (@options[:deploy] and n.uid == 0)
-    end
-  end
-
-  bloom :pass_token do
-    # Persist the token for as long as necessary
-    token_persist <= token
-    token_persist <- (token_persist * next_node).lefts
-    # Pass on the token
-    token <~ (token_persist * next_node).rights do |nn|
-      [nn.addr]
-    end
-  end
-
-  bloom :print_token do
-    stdio <~ token {["#{@node_id}: Got token! (@ #{ip_port})"]}
-  end
-end

data/lib/bud/deploy/deployer.rb

@@ -1,67 +0,0 @@
-require 'rubygems'
-require 'bud'
-require 'bud/deploy/countatomicdelivery'
-
-class Module
-  def deploystrap(&block)
-    meth_name = "__deploystrap__#{Module.get_class_name(self)}".to_sym
-    define_method(meth_name, &block)
-  end
-end
-
-module Deployer # :nodoc: all
-  include CountAtomicDelivery
-
-  state do
-    table :node, [:uid] => [:addr]
-    table :node_count, [] => [:num]
-    # At the deployer node, this collection will contain a fact for the first
-    # tick in which the given node is ready
-    scratch :node_ready, [:uid]
-
-    table :initial_data, [:uid, :pred, :data]
-    channel :dont_care, [:@loc]
-    table :dead, [:dead]
-  end
-
-  def depl_idempotent(r) (dead.include? r) ? false : dead.insert(r) end
-
-  def do_deploystrap
-    self.class.ancestors.each do |anc|
-      anc.instance_methods(false).each do |m|
-        if /^__deploystrap__/.match m
-          self.method(m.to_sym).call
-        end
-      end
-    end
-  end
-
-  def initialize(opt={})
-    super
-    do_deploystrap if opt[:deploy]
-  end
-
-  # Distribute the EDB to each node.
-  #
-  # XXX: this may break coordination protocols that assume the EDB is present
-  # before any messages are received.  In order to fix this, we would probably
-  # need to globally synchronize to ensure that "timestamp 0" gets "fully
-  # evaluated" before any messages can be sent.
-  bloom :distribute_data do
-    atomic_data_in <= (node_ready * node * initial_data).combos(node_ready.uid => node.uid, node.uid => initial_data.uid) do |nr, n, i|
-      [n.addr, [i.pred, i.data]]
-    end
-
-    # Add all tuples at once.
-    dont_care <~ atomic_data_out do |a|
-      a.tuple[1].map do |d|
-        eval a.tuple[0].to_s + " <+ [" + d.inspect + "]"
-      end
-      [ip_port]
-    end
-  end
-
-  bloom :print_ready do
-    stdio <~ node_ready {|nr| ["Child node ready: #{nr.uid}"]}
-  end
-end

data/lib/bud/deploy/ec2deploy.rb

@@ -1,199 +0,0 @@
-require 'rubygems'
-require 'AWS'
-require 'pp'
-require 'net/ssh'
-require 'net/scp'
-require 'bud'
-require 'bud/deploy/deployer'
-
-# Logic to deploy Bloom programs on EC2
-# TODO:
-#  * add support for deploy_child_opts
-#  * avoid blocking I/O calls
-#  * emit "node_ready" event when each node is ready
-module EC2Deploy
-  include Deployer
-
-  state do
-    table :access_key_id, [] => [:key]
-    table :secret_access_key, [] => [:key]
-    table :image_id, [] => [:img]
-    table :key_name, [] => [:name]
-    table :ec2_key_location, [] => [:loc]
-    table :ec2_conn, [] => [:conn]
-    table :ec2_insts, [] => [:insts]
-    table :reservation_id, [] => [:rid]
-    periodic :spinup_timer, 6
-    scratch :the_reservation, [] => [:reservation]
-    scratch :the_reservation_next, [] => [:reservation]
-    scratch :node_up, [:node] => [:bool]
-    table :init_dir, [] => [:dir]
-    table :temp_node, [:uid, :node, :localip]
-    table :all_up, [:bool]
-    table :ruby_command, [] => [:cmd]
-    table :deploy_node, [:uid] => [:node]
-    channel :ready, [:@loc, :sender]
-    table :ready_tab, [:sender]
-    scratch :ready_count, [:num]
-  end
-
-  deploystrap do
-    # Write the IP & port to a file; we'll send this to each EC2 node.
-    File.open("deploy_ip_port", "w") do |f|
-      f.puts ip_port
-    end
-  end
-
-  bootstrap do
-    # The official BUD AMI.
-    image_id <= [["ami-f434c99d"]]
-    unless @options[:deploy]
-      # Send message to the deployer telling 'em we's up.
-      File.open("deploy_ip_port", "r") do |f|
-        ready <~ [[f.readline.rstrip, ip_port]]
-      end
-    end
-  end
-
-  bloom :spinup do
-    ec2_conn <= (access_key_id * secret_access_key).pairs do
-      if depl_idempotent [:ec2_comm]
-        [AWS::EC2::Base.new(:access_key_id => access_key_id[[]].key,
-                            :secret_access_key => secret_access_key[[]].key)]
-      end
-    end
-
-    ec2_insts <= (image_id * node_count * key_name * ec2_conn).combos do
-      if depl_idempotent [:ec2_insts]
-        print "Starting up EC2 instances"
-        $stdout.flush
-        # First, we create the security group.
-        begin
-          ec2_conn[[]].conn.create_security_group(:group_name => "bud", :group_description => "bud")
-        rescue AWS::InvalidGroupDuplicate
-          # Group already exists; ok, maybe we created it previously.
-        else
-          # Add SSH permission.
-          ec2_conn[[]].conn.authorize_security_group_ingress(:group_name => "bud",
-                                                             :ip_protocol => "tcp",
-                                                             :from_port => 22,
-                                                             :to_port => 22,
-                                                             :cidr_ip => "0.0.0.0/0")
-          # Add unlimited UDP permission from any node not in the security group.
-          # XXX: make this more restrictive?
-          ec2_conn[[]].conn.authorize_security_group_ingress(:group_name => "bud",
-                                                             :ip_protocol => "udp",
-                                                             :from_port => 0,
-                                                             :to_port => 65535,
-                                                             :cidr_ip => "0.0.0.0/0")
-        end
-
-        # Finally, start up the instances.
-        [ec2_conn[[]].conn.run_instances(:image_id => image_id[[]].img,
-                                         :min_count => node_count[[]].num,
-                                         :max_count => node_count[[]].num,
-                                         :key_name => key_name[[]].name,
-                                         :security_group => "bud")]
-      end
-    end
-
-    the_reservation <= (spinup_timer * ec2_conn * ec2_insts).combos do |t,c,i|
-      if depl_idempotent [[:the_reservation, t.val]] and not all_up.include? [true]
-        to_ret = nil
-        begin
-          to_ret = [ec2_conn[[]].conn.describe_instances()["reservationSet"]["item"].find do |j|
-                      j["reservationId"] == ec2_insts[[]].insts["reservationId"]
-                    end]
-        rescue SocketError
-          print "E"
-        else
-          print "."
-        end
-        $stdout.flush
-        to_ret
-      end
-    end
-
-    # XXX: No upsert operator, so we have to do this.
-    the_reservation_next <+ the_reservation
-
-    node_up <= (((ec2_insts * the_reservation).pairs do
-                   if not all_up.include? [true]
-                     the_reservation[[]].reservation["instancesSet"]["item"].map do |i|
-                       [i, i["instanceState"]["code"] == "16"]
-                     end
-                   end
-                 end)[0] or [])
-
-    all_up <+ node_up do
-      if node_up.find {|n1| n1.bool == false} == nil and node_up.find {|n2| n2.bool == true} != nil
-        if depl_idempotent [:nodes_all_up]
-          puts "done"
-          $stdout.flush
-          [true]
-        end
-      end
-    end
-
-    # XXX: Fixed port 54321
-    temp_node <= (all_up * the_reservation_next).pairs do
-      break(((0..(the_reservation_next[[]].reservation["instancesSet"]["item"].size-1)).to_a.zip(the_reservation_next[[]].reservation["instancesSet"]["item"].map {|i| [i["ipAddress"], i["privateIpAddress"]]})).map {|n,ips| [n, ips[0] + ":54321", ips[1] + ":54321"]})
-    end
-
-    deploy_node <= (temp_node * init_dir * ruby_command).combos do |t, i, r|
-      if depl_idempotent [[:node_startup, t.node]]
-        ip = t.node.split(":")[0]
-        port = t.node.split(":")[1]
-        print "Deploying to #{ip} (#{t.uid}/#{node_count[[]].num-1})."
-        $stdout.flush
-
-        # Upload files and run commands.
-        ctr = 0
-        while ctr < 10
-          begin
-            Net::SSH.start(ip, 'ec2-user', :keys => [ec2_key_location[[]].loc],
-                           :timeout => 5, :paranoid => false) do |session|
-              # Upload init_dir, and the IP and port of the deployer
-              session.scp.upload!("deploy_ip_port", "/home/ec2-user")
-              session.scp.upload!(init_dir[[]].dir, "/home/ec2-user",
-                                  :recursive  => true)
-              # Update the Bud gem
-              channel = session.open_channel do |ch|
-                channel.request_pty do |_, success|
-                  raise "couldn't open a PTY on #{t.node}" if !success
-                end
-                channel.exec("sudo gem update --no-ri --no-rdoc bud")
-              end
-              channel.wait
-              # Run the ruby_command
-              session.exec!('nohup ' + ruby_command[[]].cmd + ' ' + t.localip +
-                            ' ' + t.node + ' >metarecv.out 2>metarecv.err </dev/null &')
-            end
-            break true
-          rescue Exception
-            ctr += 1
-            print "."
-            $stdout.flush
-            sleep 10
-            next
-          end
-        end or raise "EC2 SSH failed after 10 retries"
-
-        puts "done"
-        [t.uid, t.node]
-      end
-    end
-  end
-
-  bloom :all_nodes do
-    stdio <~ ready {|_,s| ["Ready: #{s}"]}
-    # Persist ready messages
-    ready_tab <= ready {|_, s| [s]}
-    # Compute a count of ready messages
-    ready_count <= ready_tab.group(nil, count)
-    # Copy deploy_node into node when all nodes are up
-    node <= (ready_count * node_count).pairs(:num => :num) do
-      break deploy_node
-    end
-  end
-end