zillabyte-cli 0.1.18 → 0.1.19

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZDQ2N2E3MWI0OWM1MWYxNDkyNTZkMWIxNGVjZTkxZDIxOWYwNTBhMg==
+    OGE2NDJmZDQxOTAzNGE1YTYwYWE2MzZkNjFmODc5YmRhMDFlMTliZQ==
   data.tar.gz: !binary |-
-    N2U0ZDZjYmVlYWU5Mzg3MGI2NWYxYmU4MDhiMGJkY2MyZGQwMzQ0OQ==
+    ZDc5YzRhNzYwMjRmNDE4OTc3NzI3ZWQxNWJmNDA0OWY0ZWI0NjY4Yg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZGU4ZmI4MDljNmJmZjNjZDhjOTE4ZWI1ZTI5YzA1MzQxYTg1MjYzYjc0ZTUw
-    YWJiNjE5NTY3NjMxYjQ2ZWZlNTQ1Mzk4OGVjZmEyZDM5NzJjZDIzOTAwM2U0
-    MTg3MTJiYmQyNTY3NWZmOWViNGVjMzM4Njk1ZTU1MTAzNGE2ODE=
+    NGI5MWJkNTkzZDY2ZjYxNGRkNThkNTM3YWExNzRhYmFhZTY2OTRlZjlhY2Yy
+    NzZkYTFiMzdkZDFhMmI3NWE4NjYwMWY3MjBhYzVkM2YzNThhMDE4Y2I5NWU0
+    ODI0MTZkN2E5MWRhMDQ1MWZmYTRiNzRkNTk3Yzc3MmZlOGNhZTI=
   data.tar.gz: !binary |-
-    MTA1OTE2ZWM2ODNlZTY0NTQyNjNiYmM2ZDkzMzdlZGYyZWIzYTBkZGQyZGZh
-    ZGQ1MjM3MGUyMDFmNTVjNDYxMjkyNmY1YmNlZjBhZDQwNDU3ODQ3NWM1NzY3
-    MGZmMTc4OTgyOTY5NzBiMTkwYWRlMGI0YjZlYzMzYjU3NDQ5YzE=
+    MTFiNzgxNTgwOWMyNzAzNWI1Zjg5YTQ1NWNiYTVlZDVmNDYzZDY3NTFmYTgx
+    ZjMyMDRiZTc4N2NlNTk1MDFjZGEzNmFkZWRmNjJlZjE3NTNhMDRhYTZmNGRm
+    ZjZjZmM1OTg3YzU1ZGU2MDZiZjg4YjE0NmU4Yjc2MDdiM2Q0ZTY=

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.1.18"
+    VERSION = "0.1.19"
   end
 end

data/lib/zillabyte/runner/app_runner.rb

@@ -22,10 +22,13 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
     display "inferring your app details..."
     describe_app(meta)
 
-    # Setup streams
+    # Setup nodes and arcs
     @nodes = meta["nodes"]
+    @node_map = {}
+    @nodes.each do |n|
+      @node_map[n["name"]] = n
+    end
 
-    # Index stream consummers and emitters by stream name
     @arcs = meta["arcs"]
 
     # Organize component pipes
@@ -38,7 +41,6 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
       display "starting cycle #{cycle}" unless  cycles == 1
       
       begin
-
         # Setup component pipes
         @nodes.each do |n|
 
@@ -50,15 +52,29 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
           end
           
           # Create two new pipes in the parent.
-          rd_child, wr_parent = IO.pipe()
-          rd_parent, wr_child = IO.pipe()
+          rd_child_1, wr_parent_1 = IO.pipe()
+          rd_parent_1, wr_child_1 = IO.pipe()
 
           @operation_pipes[name] = {
-            :rd_child => rd_child,      
-            :wr_child => wr_child,
-            :rd_parent => rd_parent,        
-            :wr_parent => wr_parent
+            "rd_child_1" => rd_child_1,      
+            "wr_child_1" => wr_child_1,
+            "rd_parent_1" => rd_parent_1,        
+            "wr_parent_1" => wr_parent_1
           }
+
+
+          # Add a second(right hand side) set ofpipes for joins
+          if type == "join"
+            # Create two new pipes in the parent.
+            rd_child_2, wr_parent_2 = IO.pipe()
+            rd_parent_2, wr_child_2 = IO.pipe()
+            @operation_pipes[name]["rd_child_2"] = rd_child_2
+            @operation_pipes[name]["wr_child_2"] = wr_child_2
+            @operation_pipes[name]["rd_parent_2"] = rd_parent_2
+            @operation_pipes[name]["wr_parent_2"] = wr_parent_2
+          end
+
+
         end
 
         # Maps origin => {stream => [destinations]}
@@ -72,59 +88,124 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
           @arc_map[origin][name] << a["dest"]
         end
 
-        # Spawn component threads
+        # # Spawn component threads
         @nodes.each do |n|
 
           name = n["name"]
           type = n["type"]
           emits = n["emits"]
 
+
           pipes = @operation_pipes[name] 
-          rd_child = pipes[:rd_child]
-          wr_child = pipes[:wr_child]
-          rd_parent = pipes[:rd_parent]        
-          wr_parent = pipes[:wr_parent]
 
           # Fork.
           pid = fork()
           if pid # In parent
-            # Close the reading end of the child so we can write to the child.
-            rd_child.close()
-            # Close the writing end of the child so we can read from the child.
-            wr_child.close()
-
+            # Close the reading end of the first child so we can write to the child.
+            pipes["rd_child_1"].close()
+            # Close the writing end of the first child so we can read from the child.
+            pipes["wr_child_1"].close()
+
+            if type == "join"
+              # Close the reading end of the second child so we can write to the child.
+              pipes["rd_child_2"].close()
+              # Close the writing end of the second child so we can read from the child.
+              pipes["wr_child_2"].close()
+            end
           else # in child
-            # Close the writing end of the parent so we can read from the parent.
-            wr_parent.close()
-            # Close the reading end of the parent so we can write to the parent.
-            rd_parent.close()
-            begin
+            # Close the writing end of the first parent so we can read from the parent.
+             pipes["wr_parent_1"].close()
+            # Close the reading end of the first parent so we can write to the parent.
+             pipes["rd_parent_1"].close()
+
+            if type == "join"
+              # Close the reading end of the second child so we can write to the child.
+              pipes["rd_parent_2"].close()
+              # Close the writing end of the second child so we can read from the child.
+              pipes["wr_parent_2"].close()
+            end
 
 
+            begin
+
               # Setup reading and writing pipes for communicating with consumee component
-              in_pipe = {:rd_child => @operation_pipes[name][:rd_child], :wr_child => @operation_pipes[name][:wr_child]}
+              if type != "join"
+                in_pipes = {"rd_child_1" => @operation_pipes[name]["rd_child_1"], "wr_child_1" => @operation_pipes[name]["wr_child_1"]}
+              
+              # Add join specific options
+              else
+                options[:join_options] = {}
+                in_pipes = {}
+                @arcs.each do |a|
+
+                  if (a["dest"] == name)
+                    # Left Side
+                    if (a["left"] == 1)
+                      options[:join_options][:lhs] = a["origin"]
+                      in_pipes["rd_child_1"] = @operation_pipes[name]["rd_child_1"]
+                      in_pipes["wr_child_1"] = @operation_pipes[name]["wr_child_1"]
+                    # Right Side
+                    elsif (a["right"] == 1)
+                      options[:join_options][:rhs] = a["origin"]
+                      in_pipes["rd_child_2"] = @operation_pipes[name]["rd_child_2"]
+                      in_pipes["wr_child_2"] = @operation_pipes[name]["wr_child_2"]
+                    end
+                  end
+                end
+              end
 
               # Index consumer pipes by stream name, consumer_name
               out_pipes = {}
 
-
+              # Check if you are the consumee for a downstream join in order to select the correct pipe
               if type != "sink"
                 @arc_map[name].each_pair do |stream, destinations|
                   out_pipes[stream] ||= {}
+                  
+
                   destinations.each do |dest|
-                    out_pipes[stream][dest] = {:wr_parent => @operation_pipes[dest][:wr_parent], :rd_parent => @operation_pipes[dest][:rd_parent] }
+                    out_pipes[stream][dest] ||= {}
+
+                    # Check for a join at the destination
+                    if (@node_map[dest]["type"] == "join")                      
+                      @arcs.each do |a|
+                        if (a["dest"] == dest && a["origin"] == name)
+                          # Left Side
+                          if (a["left"] == 1)
+                            out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"] 
+                            out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
+                            break
+                          elsif (a["right"] == 1)
+                            out_pipes[stream][dest]["wr_parent_2"] = @operation_pipes[dest]["wr_parent_2"] 
+                            out_pipes[stream][dest]["rd_parent_2"] = @operation_pipes[dest]["rd_parent_2"]
+                            break
+                          end
+                        end
+                      end
+                    else
+                      out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"] 
+                      out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
+                    end
                   end
                 end
               end
 
               # Run the child process 
-              Zillabyte::Runner::MultilangOperation.run(n, dir, in_pipe, out_pipes, self, meta, options)
-    
+              Zillabyte::Runner::MultilangOperation.run(n, dir, in_pipes, out_pipes, self, meta, options)
+
+            rescue => e
+              display e.message
+              display e.backtrace
             ensure
               # Close the reading end of the child
-              rd_child.close()
+              pipes["rd_child_1"].close()
               # Close the writing end of the child
-              wr_child.close()
+              pipes["wr_child_1"].close()
+
+              # Close secondary join child
+              pipes["rd_child_2"].close() if pipes["rd_child_2"]
+              pipes["wr_child_2"].close() if pipes["wr_child_2"]
+
               exit!(-1)
             end
 
@@ -148,17 +229,24 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
               next
             end
             # Send tuple to source
-           @operation_pipes["source_1"][:wr_parent].puts msg
+           @operation_pipes["source_1"]["wr_parent_1"].puts msg
           end
         end 
-
+      rescue => e
+        display e.message
+        display e.backtrace
       ensure
         Process.waitall()
         @operation_pipes.each do |name, pipes|
           #Close the  writing end of the parent
-          pipes[:wr_parent].close()
+          pipes["wr_parent_1"].close()
           # Close the reading end of the parent
-          pipes[:rd_parent].close()
+          pipes["rd_parent_1"].close()
+
+          # Close secondary join parent
+          pipes["wr_parent_2"].close() if pipes["wr_parent_2"]
+          pipes["rd_parent_2"].close() if pipes["rd_parent_2"]
+
         end
       end
 

data/lib/zillabyte/runner/component_operation.rb

@@ -11,14 +11,13 @@ class Zillabyte::Runner::ComponentOperation
   ENDMARKER = "\nend\n"
 
   # Run the operation
-  def self.run(node, dir, consumee, consumer_pipes, tester, meta, options = {})
-
+  def self.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options = {})
 
     @__node = node
     @__name = node["name"]
     @__type = node["type"]
     @__dir = dir
-    @__consumee = consumee
+    @__consumee_pipes = consumee_pipes
     @__consumer_pipes = consumer_pipes
     @__tester = tester
 
@@ -43,7 +42,7 @@ class Zillabyte::Runner::ComponentOperation
 
       # Component outputs act in same manner as sinks
       else
-        Zillabyte::Runner::MultilangOperation.run(node, dir, consumee, consumer_pipes, tester, meta, options)
+        Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
       end
     rescue => e
       cdisplay e.message
@@ -76,14 +75,7 @@ class Zillabyte::Runner::ComponentOperation
         end
 
         # Index streams and consumers by their pipes for lookup
-        consumer_hash = {}
-        @__emit_queues.each_pair do |stream, consumers|
-          consumers.each_key do |consumer|
-            read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
-            consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-          end
-        end
-
+        consumer_hash = build_consumer_hash()
 
         # Send first tuple
         @__emit_queues.each_pair do |stream, consumers|
@@ -119,8 +111,8 @@ class Zillabyte::Runner::ComponentOperation
               if tuple_json.nil?
                 write_stream = get_write_stream(stream, consumer)
                 cdisplay "ending cycle for #{consumer}"
-                write_message(write_stream, END_CYCLE_MESSAGE)
-                write_message(write_stream, DONE_MESSAGE)
+                send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                send_command_tuple(stream, consumer, DONE_MESSAGE)
 
               else
                 # Emit tuple to consumer
@@ -138,16 +130,15 @@ class Zillabyte::Runner::ComponentOperation
       end
       
     else
-      stdin = @__consumee[:rd_child]
+      stdin = @__consumee_pipes["rd_child_1"]
       loop do 
 
         msg = stdin.gets
         if msg == "end\n"
           @__consumer_pipes.each_pair do |stream, consumers|
             consumers.each_pair do |consumer, pipe| 
-              write_stream = pipe[:wr_parent]
-              write_message(write_stream, END_CYCLE_MESSAGE)
-              write_message(write_stream, DONE_MESSAGE)
+              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+              send_command_tuple(stream, consumer, DONE_MESSAGE)
             end
           end
 
@@ -187,13 +178,7 @@ class Zillabyte::Runner::ComponentOperation
   def self.run_rpc_component()
 
     # Index streams and consumers by their pipes for lookup
-    consumer_hash = {}
-    @__emit_queues.each_pair do |stream, consumers|
-      consumers.each_key do |consumer|
-        read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
-        consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-      end
-    end
+    consumer_hash = build_consumer_hash()
     
     # Keep track of how many consumers to handle before exiting
     consumers_running = consumer_hash.keys.length
@@ -204,7 +189,7 @@ class Zillabyte::Runner::ComponentOperation
 
     # TODO multiple inputs
     # The input component(singular at the moment)
-    read_streams = consumer_hash.keys.concat [@__consumee[:rd_child]]
+    read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"]]
 
     # Start communication with API
     api = @__tester.session.api
@@ -335,7 +320,7 @@ class Zillabyte::Runner::ComponentOperation
           end
 
           # Ask for next tuple
-          write_message(@__consumee[:wr_child], NEXT_MESSAGE)
+          write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
 
 
         # End cycle
@@ -352,8 +337,7 @@ class Zillabyte::Runner::ComponentOperation
 
             # End cycle for consumer if it has processed all tuples
             if tuple_json.nil? && end_cycle_received
-              write_stream = get_write_stream(stream, consumer)
-              write_message(write_stream, END_CYCLE_MESSAGE)
+              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
               consumers_running -= 1
               if consumers_running == 0
                 break
@@ -489,11 +473,42 @@ class Zillabyte::Runner::ComponentOperation
     end
   end
 
+
+   # Build the hash of consumer streams for lookup when receiving responses
+  def self.build_consumer_hash()
+    consumer_hash = {}
+    @__emit_queues.each_pair do |stream, consumers|
+      consumers.each_key do |consumer|
+
+        pipes = @__consumer_pipes[stream][consumer]
+        if pipes.has_key? "rd_parent_1"
+          read_stream = pipes["rd_parent_1"]
+          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+
+        elsif pipes.has_key? "rd_parent_2"
+          read_stream = pipes["rd_parent_2"]
+          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+        end
+      end
+    end
+
+    return consumer_hash
+  end
+
   # Send object to every consumer of the operation, regardless of stream
   def self.send_to_consumers(json_obj)
     @__consumer_pipes.each_pair do |stream, consumers|
       consumers.each_pair do |consumer, pipe| 
-        write_message(pipe[:wr_parent], json_obj)
+
+        # Single or Left hand pipe
+        if (pipe.has_key? "wr_parent_1")
+          write_stream = get_write_stream(stream, consumer, 1)
+          write_message(write_stream, json_obj)
+        elsif (pipe.has_key? "wr_parent_2")
+          write_stream = get_write_stream(stream, consumer, 2)
+          write_message(write_stream, json_obj)
+        end
+
         cdisplay "emitted #{json_obj} to #{consumer}"
       end
     end
@@ -501,8 +516,9 @@ class Zillabyte::Runner::ComponentOperation
 
 
   # Get the write pipe of the stream consumer
-  def self.get_write_stream(stream, consumer)
-     @__consumer_pipes[stream][consumer][:wr_parent]
+  def self.get_write_stream(stream, consumer, number=1)
+     wr_pipe = "wr_parent_" + number.to_s
+     @__consumer_pipes[stream][consumer][wr_pipe]
   end
 
 
@@ -512,6 +528,22 @@ class Zillabyte::Runner::ComponentOperation
   end
 
 
+  def self.send_command_tuple(stream, consumer, json_obj)
+    pipe = @__consumer_pipes[stream][consumer]
+     # Single or Left hand pipe
+    if (pipe.has_key? "wr_parent_1")
+      write_stream = get_write_stream(stream, consumer, 1)
+      write_message(write_stream, json_obj)
+
+    # Right hand pipe
+    elsif (pipe.has_key? "wr_parent_2")
+      write_stream = get_write_stream(stream, consumer, 2)
+      write_message(write_stream, json_obj)
+    end
+    @__emit_queues[stream][consumer][:ready] = false
+  end
+
+
   # Emit tuple_json to the consumer of a stream
   def self.emit_consumer_tuple(stream, consumer, tuple_json)
     begin
@@ -519,8 +551,19 @@ class Zillabyte::Runner::ComponentOperation
     rescue JSON::ParserError
       cdisplay "Error: invalid JSON"
     end
-    write_stream = get_write_stream(stream, consumer)
-    write_message(write_stream, tuple_json)
+
+    pipe = @__consumer_pipes[stream][consumer]
+     # Single or Left hand pipe
+    if (pipe.has_key? "wr_parent_1")
+      write_stream = get_write_stream(stream, consumer, 1)
+      write_message(write_stream, tuple_json)
+
+    # Right hand pipe
+    elsif (pipe.has_key? "wr_parent_2")
+      write_stream = get_write_stream(stream, consumer, 2)
+      write_message(write_stream, tuple_json)
+    end
+
     @__emit_queues[stream][consumer][:ready] = false
     cdisplay "emitted tuple #{display_json} to #{consumer} "
   end