zillabyte-cli 0.1.18 → 0.1.19

data/lib/zillabyte/runner/component_runner.rb

@@ -25,6 +25,10 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
 
     # Setup streams
     @nodes = meta["nodes"]
+    @node_map = {}
+    @nodes.each do |n|
+      @node_map[n["name"]] = n
+    end
 
     # Index stream consummers and emitters by stream name
     @arcs = meta["arcs"]
@@ -45,18 +49,28 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
         if n["type"] ==  "source"
           fields = n["fields"]
         end
-        
-        # Create two new pipes in the parent.
-        rd_child, wr_parent = IO.pipe()
-        rd_parent, wr_child = IO.pipe()
 
+        # Create two new pipes in the parent.
+        rd_child_1, wr_parent_1 = IO.pipe()
+        rd_parent_1, wr_child_1 = IO.pipe()
+        
         @operation_pipes[name] = {
-          :node => n,
-          :rd_child => rd_child,      
-          :wr_child => wr_child,
-          :rd_parent => rd_parent,        
-          :wr_parent => wr_parent
+          "rd_child_1" => rd_child_1,      
+          "wr_child_1" => wr_child_1,
+          "rd_parent_1" => rd_parent_1,        
+          "wr_parent_1" => wr_parent_1
         }
+
+        # Add a second(right hand side) set ofpipes for joins
+        if type == "join"
+          # Create two new pipes in the parent.
+          rd_child_2, wr_parent_2 = IO.pipe()
+          rd_parent_2, wr_child_2 = IO.pipe()
+          @operation_pipes[name]["rd_child_2"] = rd_child_2
+          @operation_pipes[name]["wr_child_2"] = wr_child_2
+          @operation_pipes[name]["rd_parent_2"] = rd_parent_2
+          @operation_pipes[name]["wr_parent_2"] = wr_parent_2
+        end
       end
 
       # Maps origin => {stream => [destinations]}
@@ -79,56 +93,115 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
         emits = n["emits"]
 
         pipes = @operation_pipes[name] 
-        rd_child = pipes[:rd_child]
-        wr_child = pipes[:wr_child]
-        rd_parent = pipes[:rd_parent]        
-        wr_parent = pipes[:wr_parent]
 
         # Fork.
         pid = fork()
         if pid # In parent
-          # Close the reading end of the child so we can write to the child.
-          rd_child.close()
-          # Close the writing end of the child so we can read from the child.
-          wr_child.close()
-
+          # Close the reading end of the first child so we can write to the child.
+          pipes["rd_child_1"].close()
+          # Close the writing end of the first child so we can read from the child.
+          pipes["wr_child_1"].close()
+
+          if type == "join"
+            # Close the reading end of the second child so we can write to the child.
+            pipes["rd_child_2"].close()
+            # Close the writing end of the second child so we can read from the child.
+            pipes["wr_child_2"].close()
+          end
         else # in child
-          # Close the writing end of the parent so we can read from the parent.
-          wr_parent.close()
-          # Close the reading end of the parent so we can write to the parent.
-          rd_parent.close()
-          begin
+          # Close the writing end of the first parent so we can read from the parent.
+           pipes["wr_parent_1"].close()
+          # Close the reading end of the first parent so we can write to the parent.
+           pipes["rd_parent_1"].close()
+
+          if type == "join"
+            # Close the reading end of the second child so we can write to the child.
+            pipes["rd_parent_2"].close()
+            # Close the writing end of the second child so we can read from the child.
+            pipes["wr_parent_2"].close()
+          end
 
+          begin
             # Setup reading and writing pipes for communicating with consumee component
-            in_pipe = {:rd_child => @operation_pipes[name][:rd_child], :wr_child => @operation_pipes[name][:wr_child]}
+            if type != "join"
+              in_pipes = {"rd_child_1" => @operation_pipes[name]["rd_child_1"], "wr_child_1" => @operation_pipes[name]["wr_child_1"]}
+            
+            # Add join specific options
+            else
+              options[:join_options] = {}
+              in_pipes = {}
+              @arcs.each do |a|
+
+                if (a["dest"] == name)
+                  # Left Side
+                  if (a["left"] == 1)
+                    options[:join_options][:lhs] = a["origin"]
+                    in_pipes["rd_child_1"] = @operation_pipes[name]["rd_child_1"]
+                    in_pipes["wr_child_1"] = @operation_pipes[name]["wr_child_1"]
+                  # Right Side
+                  elsif (a["right"] == 1)
+                    options[:join_options][:rhs] = a["origin"]
+                    in_pipes["rd_child_2"] = @operation_pipes[name]["rd_child_2"]
+                    in_pipes["wr_child_2"] = @operation_pipes[name]["wr_child_2"]
+                  end
+                end
+              end
+            end
 
-            # Index consumer pipes by consumer_name
+            # Index consumer pipes by stream name, consumer_name
             out_pipes = {}
 
+            # Check if you are the consumee for a downstream join in order to select the correct pipe
             if type != "sink"
               @arc_map[name].each_pair do |stream, destinations|
                 out_pipes[stream] ||= {}
+                
                 destinations.each do |dest|
-                  out_pipes[stream][dest] = {:wr_parent => @operation_pipes[dest][:wr_parent], :rd_parent => @operation_pipes[dest][:rd_parent] }
+                  out_pipes[stream][dest] ||= {}
+
+                  # Check for a join at the destination
+                  if (@node_map[dest]["type"] == "join")                      
+                    @arcs.each do |a|
+                      if (a["dest"] == dest && a["origin"] == name)
+                        # Left Side
+                        if (a["left"] == 1)
+                          out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"] 
+                          out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
+                          break
+                        elsif (a["right"] == 1)
+                          out_pipes[stream][dest]["wr_parent_2"] = @operation_pipes[dest]["wr_parent_2"] 
+                          out_pipes[stream][dest]["rd_parent_2"] = @operation_pipes[dest]["rd_parent_2"]
+                          break
+                        end
+                      end
+                    end
+                  else
+                    out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"] 
+                    out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
+                  end
                 end
               end
             end
 
             # Run the child process 
-            Zillabyte::Runner::ComponentOperation.run(n, dir, in_pipe, out_pipes, self, meta, options)
-
-         rescue => e
-          display e
+            Zillabyte::Runner::ComponentOperation.run(n, dir, in_pipes, out_pipes, self, meta, options)
 
+          rescue => e
+            display e.message
+            display e.backtrace
           ensure
             # Close the reading end of the child
-            rd_child.close()
+            pipes["rd_child_1"].close()
             # Close the writing end of the child
-            wr_child.close()
-            exit!(-1)
-          end
+            pipes["wr_child_1"].close()
 
-        end #end child
+            # Close secondary join child
+            pipes["rd_child_2"].close() if pipes["rd_child_2"]
+            pipes["wr_child_2"].close() if pipes["wr_child_2"]
+
+            exit!(-1)
+          end #end child
+        end
       end 
 
 
@@ -164,20 +237,26 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
           display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
           msg = ask
           # Send tuple to source
-          @operation_pipes[source][:wr_parent].puts msg
+          @operation_pipes[source]["wr_parent_1"].puts msg
         end
       end
+    rescue => e
+      display e.message
+      display e.backtrace
    
     ensure
       Process.waitall()
       @operation_pipes.each do |name, pipes|
         #Close the  writing end of the parent
-        pipes[:wr_parent].close()
+        pipes["wr_parent_1"].close()
         # Close the reading end of the parent
-        pipes[:rd_parent].close()
+        pipes["rd_parent_1"].close()
+
+        # Close secondary join parent
+        pipes["wr_parent_2"].close() if pipes["wr_parent_2"]
+        pipes["rd_parent_2"].close() if pipes["rd_parent_2"]
       end
     end
-
   end
 
   def session
@@ -185,7 +264,7 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
   end
 
 
- def cdisplay(name, message, useName=true)
+  def cdisplay(name, message, useName=true)
     color = @colors[name] || :default
     if message == ""
       display ""
@@ -205,7 +284,6 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
     end
   end
 
-
   def display(message, newline = true)
     @session.display(message, newline)
   end
@@ -241,8 +319,4 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
     end
   end
 
-
-
-
-
 end

data/lib/zillabyte/runner/multilang_operation.rb

@@ -11,24 +11,25 @@ module Zillabyte; module Runner; class MultilangOperation
   PONG_SUFFIX = "\"}\n"  
   ENDMARKER = "\nend\n"
 
-  def self.run(node, dir, consumee, consumer_pipes, tester, meta, options = {})
+  def self.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options = {})
     require("mkfifo")
     require("zillabyte/runner/component_operation")
 
     require("pty")
     require("open3")
+
     @__node = node
     @__name = node["name"]
+    @__tester = tester
+
     @__type = node["type"]
     @__dir = dir
-    @__consumee = consumee
-    @__consumer_pipes = consumer_pipes
-    @__tester = tester
 
+    @__consumee_pipes = consumee_pipes
+    @__consumer_pipes = consumer_pipes
     @__meta = meta
     @__options = options
     @__output_type = options[:output_type]
-
     # Each consumer of a stream gets its own queue and message passing
     @__emit_queues = {}
     @__consumer_pipes.each_pair do |stream, consumers|
@@ -44,12 +45,14 @@ module Zillabyte; module Runner; class MultilangOperation
         self.run_source()
       when "group_by"
         self.run_group_by()
+      when "join"
+        self.run_join()
       when "each"
         self.run_each()
       when "filter"
         self.run_filter()
       when "component"
-        Zillabyte::Runner::ComponentOperation.run(node, dir, consumee, consumer_pipes, tester, meta, options = {})
+        Zillabyte::Runner::ComponentOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options = {})
       when "sink"
         self.run_sink()
       else
@@ -70,7 +73,7 @@ module Zillabyte; module Runner; class MultilangOperation
     if @__options[:interactive]
       loop do 
 
-        msg = @__consumee[:rd_child].gets
+        msg = @__consumee_pipes["rd_child_1"].gets
 
           # Build tuple
           begin
@@ -97,13 +100,10 @@ module Zillabyte; module Runner; class MultilangOperation
         cdisplay("Could not find data that matches your 'matches' clause")
         exit(-1)
       end
-      cdisplay("Received #{rows.length} rows!")
-
       # Enqueue rows for sending to consumers
-      column_aliases = res["column_aliases"]
+      column_aliases = res['column_aliases']
       rows.each do |tuple|
         tuple_json = build_tuple_json(tuple, nil, column_aliases)
-
         @__emit_queues.each_pair do |stream, consumers|
           consumers.each_pair do |consumer, emitter|
             emitter[:write_queue] << tuple_json
@@ -112,13 +112,7 @@ module Zillabyte; module Runner; class MultilangOperation
       end
 
       # Index streams and consumers by their pipes for lookup
-      consumer_hash = {}
-      @__emit_queues.each_pair do |stream, consumers|
-        consumers.each_key do |consumer|
-          read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
-          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-        end
-      end
+      consumer_hash = build_consumer_hash()
 
 
       # Send first tuple
@@ -155,9 +149,8 @@ module Zillabyte; module Runner; class MultilangOperation
             if tuple_json.nil?
               write_stream = get_write_stream(stream, consumer)
               cdisplay "ending cycle for #{consumer}"
-              write_message(write_stream, END_CYCLE_MESSAGE)
-              write_message(write_stream, DONE_MESSAGE)
-
+              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+              send_command_tuple(stream, consumer, DONE_MESSAGE)
             else
               # Emit tuple to consumer
               emit_consumer_tuple(stream, consumer, tuple_json)
@@ -175,15 +168,9 @@ module Zillabyte; module Runner; class MultilangOperation
     # Custom source
     else
 
-      # Index streams and consumers by their pipes for lookup
 
-      consumer_hash ={}
-      @__emit_queues.each_pair do |stream, consumers|
-        consumers.each_key do |consumer|
-          read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
-          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-        end
-      end
+      # Index streams and consumers by their pipes for lookup
+      consumer_hash = build_consumer_hash()
 
       # Keep track of how many consumers to handle before exiting
       consumers_running = consumer_hash.keys.length
@@ -209,6 +196,7 @@ module Zillabyte; module Runner; class MultilangOperation
             # Setup streams from consumers and multilang
             read_streams = consumer_hash.keys.concat [stdout, ml_output]
 
+
             # Handshake
             handshake(ml_input, ml_output)
 
@@ -242,11 +230,11 @@ module Zillabyte; module Runner; class MultilangOperation
                 end
 
                 if obj["command"] 
+
                   case obj["command"]
 
                   # Multilang emitted a tuple
                   when "emit"
-
                     stream = obj['stream']
                     # Check for null emit
                     if end_cycle_policy != "explicit"
@@ -272,10 +260,10 @@ module Zillabyte; module Runner; class MultilangOperation
                     # Send or enqueue tuple for each consumer
                     tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
 
- 
                     @__emit_queues[stream].each_pair do |consumer, emitter|
                       if emitter[:ready]
                         emit_consumer_tuple(stream, consumer, tuple_json)
+
                       else
                         @__emit_queues[stream][consumer][:write_queue] << tuple_json
                       end
@@ -291,8 +279,7 @@ module Zillabyte; module Runner; class MultilangOperation
                     # End cycle for consumer if it has processed all tuples
                     if tuple_json.nil? && end_cycle_received
 
-                      write_stream = get_write_stream(stream, consumer)
-                      write_message(write_stream, END_CYCLE_MESSAGE)
+                      send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
                       consumers_running -= 1
                       if consumers_running == 0
                         break
@@ -329,8 +316,7 @@ module Zillabyte; module Runner; class MultilangOperation
                       @__emit_queues.each_pair do |stream, consumers|
                         consumers.each_pair do |consumer, emitter|
                           if emitter[:ready]
-                            write_stream = get_write_stream(stream, consumer)
-                            write_message(write_stream, END_CYCLE_MESSAGE)
+                            send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
                             consumers_running -= 1
                             if consumers_running == 0
                               break
@@ -393,15 +379,9 @@ module Zillabyte; module Runner; class MultilangOperation
 
   def self.run_each()
 
-    # Index streams and consumers by their pipes for lookup
-    consumer_hash = {}
-    @__emit_queues.each_pair do |stream, consumers|
-      consumers.each_key do |consumer|
-        read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
-        consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-      end
-    end
 
+    # Index streams and consumers by their pipes for lookup
+    consumer_hash = build_consumer_hash()
 
     # Keep track of how many consumers to handle before exiting
     consumers_running = consumer_hash.keys.length
@@ -423,7 +403,7 @@ module Zillabyte; module Runner; class MultilangOperation
           ml_output = File.open("#{ml_pipe}.in", "r+")
 
           # Setup streams from consumers, multilang, and the consumee
-          read_streams = consumer_hash.keys.concat [@__consumee[:rd_child], ml_output, stdout]
+          read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"], ml_output, stdout]
 
           # Handshake
           handshake(ml_input, ml_output)
@@ -432,6 +412,7 @@ module Zillabyte; module Runner; class MultilangOperation
           multilang_queue = []
           mutlilang_count = 0
           end_cycle_received = false
+          column_aliases = nil
 
 
           # Receive and handle messages
@@ -461,7 +442,7 @@ module Zillabyte; module Runner; class MultilangOperation
                   stream = obj["stream"]
                  
                   # Send or enqueue tuple for each consumer
-                  tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
+                  tuple_json = build_tuple_json(obj['tuple'], obj['meta'], column_aliases)
 
                   @__emit_queues[stream].each_pair do |consumer, emitter|
                     if emitter[:ready]
@@ -473,17 +454,16 @@ module Zillabyte; module Runner; class MultilangOperation
 
                 # Consumer is ready for a message
                 when "next"
-
                   stream = consumer_hash[r][:stream]
                   consumer = consumer_hash[r][:consumer]
 
+
                   @__emit_queues[stream][consumer][:ready] = true
                   tuple_json = get_consumer_tuple(stream, consumer)
 
                   # End cycle for consumer if it has processed all tuples
                   if tuple_json.nil? && end_cycle_received
-                    write_stream = get_write_stream(stream, consumer)
-                    write_message(write_stream, END_CYCLE_MESSAGE)
+                    send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
                     consumers_running -= 1
                     if consumers_running == 0
                       break
@@ -507,8 +487,7 @@ module Zillabyte; module Runner; class MultilangOperation
 
                   # Request next tuple from consumee
                   elsif !end_cycle_received
-                    write_message(@__consumee[:wr_child], NEXT_MESSAGE)
-                    
+                    write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
 
                   # If there are no more messages to send, we are done
                   elsif end_cycle_received && mutlilang_count == 0
@@ -518,8 +497,7 @@ module Zillabyte; module Runner; class MultilangOperation
                     @__emit_queues.each_pair do |stream, consumers|
                       consumers.each_pair do |consumer, emitter|
                         if emitter[:ready]
-                          write_stream = get_write_stream(stream, consumer)
-                          write_message(write_stream, END_CYCLE_MESSAGE)
+                          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
                           consumers_running -= 1
                           if consumers_running == 0
                             break
@@ -540,13 +518,11 @@ module Zillabyte; module Runner; class MultilangOperation
                 # Consumee operation sent signal to end_cycle
                 when "end_cycle"
                   end_cycle_received = true
-
                   if mutlilang_count == 0
                     @__emit_queues.each_pair do |stream, consumers|
                       consumers.each_pair do |consumer, emitter|                      
                         if emitter[:ready]
-                          write_stream = get_write_stream(stream, consumer)
-                          write_message(write_stream, END_CYCLE_MESSAGE)
+                          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
                           consumers_running -= 1
                           if consumers_running == 0
                             break
@@ -561,7 +537,7 @@ module Zillabyte; module Runner; class MultilangOperation
 
               # Received a tuple from consumee
               elsif obj['tuple']
-              
+                column_aliases = obj['column_aliases']
                 # Send or enqueue to multilang
                 mutlilang_count += 1
                 if multilang_queue.empty?
@@ -604,13 +580,7 @@ module Zillabyte; module Runner; class MultilangOperation
   def self.run_group_by()
 
     # Index streams and consumers by their pipes for lookup
-    consumer_hash = {}
-    @__emit_queues.each_pair do |stream, consumers|
-      consumers.each_key do |consumer|
-        read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
-        consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-      end
-    end
+    consumer_hash = build_consumer_hash
 
     # Keep track of how many consumers to handle before exiting
     consumers_running = consumer_hash.keys.length
@@ -638,7 +608,7 @@ module Zillabyte; module Runner; class MultilangOperation
           # Multilang output tuples
           ml_output = File.open("#{ml_pipe}.in", "r+")
           # Setup streams from consumers, multilang, and the consumee
-          read_streams = consumer_hash.keys.concat [stdout, ml_output, @__consumee[:rd_child]]
+          read_streams = consumer_hash.keys.concat [stdout, ml_output, @__consumee_pipes["rd_child_1"]]
 
           # Handshake
           handshake(ml_input, ml_output)
@@ -750,7 +720,7 @@ module Zillabyte; module Runner; class MultilangOperation
                 end
 
                 # Ask operation for next tuple
-                write_message(@__consumee[:wr_child], NEXT_MESSAGE)
+                write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
 
               # Multilang sent a ping
               elsif obj['ping']
@@ -839,6 +809,330 @@ module Zillabyte; module Runner; class MultilangOperation
   end
 
 
+  def self.run_join()
+
+    lhs_fields = @__node["lhs_fields"]
+    rhs_fields = @__node["rhs_fields"]
+    join_type = @__node["join_type"]
+
+
+    # Sanity check
+    if lhs_fields.nil? || rhs_fields.nil? || join_type.nil?
+      return
+    end
+
+
+    # Index the consumee streams for left and right sides
+    consumer_hash = build_consumer_hash()
+
+    # read all tuples from lefthand and right hand streams
+    read_streams = [@__consumee_pipes["rd_child_1"], @__consumee_pipes["rd_child_2"]]
+
+     # Index left and right hand consumees by their streams for lookup
+    consumee_hash = {}
+    # Left side
+    lhs = @__options[:join_options][:lhs]
+    consumee_hash[@__consumee_pipes["rd_child_1"]] = lhs
+    # Right side
+    rhs = @__options[:join_options][:rhs]
+    consumee_hash[@__consumee_pipes["rd_child_2"]] = rhs
+  
+    # Keep track of how many consumers to handle before exiting
+    consumers_running = 1
+
+    # Index the incoming tuples by their join key
+    lhs_tuples = {}
+    rhs_tuples = {}
+
+    tuple_queue = []
+
+    # Begin cycle
+    left_end_cycle_received = false
+    right_end_cycle_received = false
+    
+    # Receive and handle messages
+    loop do
+
+      # Read from a stream
+      rs = select_read_streams(read_streams)
+      rs.each do |r|
+
+        # Receive an object
+        obj = read_message(r)
+      
+        if obj["command"] 
+          case obj["command"]
+
+          # A consumee is done emitting
+          when "end_cycle"
+
+            if consumee_hash[r] == lhs
+              left_end_cycle_received = true
+            elsif consumee_hash[r] == rhs
+              right_end_cycle_received = true
+            end
+
+            # We are done receiving from streams
+            if left_end_cycle_received && right_end_cycle_received
+              break
+            end
+          end
+
+         # Received a tuple from consumee
+        elsif obj['tuple']
+          tuple = obj['tuple']
+          if consumee_hash[r] == lhs
+            lhs_tuples[tuple[lhs_fields]] ||= []
+            lhs_tuples[tuple[lhs_fields]] << tuple
+          elsif consumee_hash[r] == rhs
+            rhs_tuples[tuple[rhs_fields]] ||= []
+            rhs_tuples[tuple[rhs_fields]] << tuple
+          end
+        end
+
+        #Ask operation for next tuple
+        if consumee_hash[r] == lhs
+          write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
+        elsif consumee_hash[r] == rhs
+          write_message(@__consumee_pipes["wr_child_2"], NEXT_MESSAGE)
+        end  
+      end
+
+      # Break out if emits ended
+      if left_end_cycle_received && right_end_cycle_received
+        break
+      end
+    end
+    # Build the joined tuples
+    joined_tuples = []
+
+    # If no joined tuples, end the cycle
+    if lhs_tuples.empty? && rhs_fields.empty?
+        @__emit_queues.each_pair do |stream, consumers|
+        consumers.each_pair do |consumer, emitter|                      
+          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+        end
+      end
+      return
+    end
+    joined_fields = (lhs_tuples.values[0][0].keys.concat rhs_tuples.values[0][0].keys).uniq
+    case join_type
+    when "inner"
+      lhs_tuples.each_pair do |key, lhs_tuples|
+        lhs_tuples.each do |lhs_tuple|
+
+          tuple = {}
+          if !rhs_tuples[key].nil? 
+            rhs_tuples[key].each do |rhs_tuple|
+              # Check for a valid join
+              if !lhs_tuple[lhs_fields].nil? && !rhs_tuple[rhs_fields].nil?
+                # Add the fields
+                joined_fields.each do |field|
+                  if lhs_tuple.has_key? field
+                    tuple[field] = lhs_tuple[field]
+                  else
+                    tuple[field] = rhs_tuple[field]
+                  end
+                end
+
+                # Add the tuple
+                joined_tuples << tuple
+              end
+            end
+          end
+        end
+      end
+    when "left"
+      lhs_tuples.each_pair do |key, lhs_tuples|
+        lhs_tuples.each do |lhs_tuple|
+
+          joined = false
+          if rhs_tuples.has_key? key
+            rhs_tuples[key].each do |rhs_tuple|
+              # Check for a valid join
+              if !lhs_tuple[lhs_fields].nil? && !rhs_tuple[rhs_fields].nil?
+                tuple = {}
+
+                # Add the fields
+                joined_fields.each do |field|
+                  if lhs_tuple.has_key? field
+                    tuple[field] = lhs_tuple[field]
+                  else
+                    tuple[field] = rhs_tuple[field]
+                  end
+                end
+                joined_tuples << tuple
+                joined = true
+              end
+            end
+          end
+          if !joined
+            tuple = {}
+            lhs_tuples.each.each do |hash|
+              hash.each do |key, value|
+                tuple[key] = value
+              end
+            end
+
+            joined_fields.each do |field|
+              if !tuple.has_key? field
+                tuple[field] = nil
+              end
+            end
+            # Add the tuple
+            joined_tuples << tuple
+          end
+        end
+      end
+    when "right"
+      rhs_tuples.each_pair do |key, rhs_tuples|
+        rhs_tuples.each do |rhs_tuple|
+
+          joined = false
+          if lhs_tuples.has_key? key
+            lhs_tuples[key].each do |lhs_tuple|
+              # Check for a valid join
+              if !rhs_tuple[lhs_fields].nil? && !lhs_tuple[rhs_fields].nil?
+                tuple = {}
+
+                # Add the fields
+                joined_fields.each do |field|
+                  if rhs_tuple.has_key? field
+                    tuple[field] = rhs_tuple[field]
+                  else
+                    tuple[field] = lhs_tuple[field]
+                  end
+                end
+                joined_tuples << tuple
+                joined = true
+              end
+            end
+          end
+          if !joined
+            tuple = {}
+            rhs_tuples.each.each do |hash|
+              hash.each do |key, value|
+                tuple[key] = value
+              end
+            end
+
+            joined_fields.each do |field|
+              if !tuple.has_key? field
+                tuple[field] = nil
+              end
+            end
+            # Add the tuple
+            joined_tuples << tuple
+          end
+        end
+      end
+    when "outer"
+      lhs_tuples.each_pair do |key, lhs_tuples|
+        lhs_tuples.each do |lhs_tuple|
+
+          joined = false
+          if rhs_tuples.has_key? key
+            rhs_tuples[key].each do |rhs_tuple|
+              # Check for a valid join
+              if !lhs_tuple[lhs_fields].nil? && !rhs_tuple[rhs_fields].nil?
+                tuple = {}
+
+                # Add the fields
+                joined_fields.each do |field|
+                  if lhs_tuple.has_key? field
+                    tuple[field] = lhs_tuple[field]
+                  else
+                    tuple[field] = rhs_tuple[field]
+                  end
+                end
+                joined_tuples << tuple
+                joined = true
+              end
+            end
+          end
+          if !joined
+            tuple = {}
+            lhs_tuples.each.each do |hash|
+              hash.each do |key, value|
+                tuple[key] = value
+              end
+            end
+
+            joined_fields.each do |field|
+              if !tuple.has_key? field
+                tuple[field] = nil
+              end
+            end
+            # Add the tuple
+            joined_tuples << tuple
+          end
+        end
+      end
+    end
+
+    # Setup output queues
+    joined_tuples.each do |tuple|
+      tuple_json = build_tuple_json(tuple)
+      @__emit_queues.each_pair do |stream, consumers|
+        consumers.each_key do |consumer|
+          @__emit_queues[stream][consumer][:write_queue] << tuple_json  
+        end
+      end
+    end
+
+    # Send first tuple
+    @__emit_queues.each_pair do |stream, consumers|
+      consumers.each_key do |consumer|
+
+        tuple_json = get_consumer_tuple(stream, consumer)
+        if tuple_json.nil?
+          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+          consumers_running -= 1
+          if consumers_running == 0
+            return
+          end
+        else
+          emit_consumer_tuple(stream, consumer, tuple_json)
+        end
+      end
+    end
+
+    # Sent tuples to consumers as appropriate
+    loop do 
+
+      # Retrieve messages from consumers
+      rs, ws, es = IO.select(consumer_hash.keys, [], [])
+
+      # Emit tuples to consumers
+      rs.each do |r|
+
+        # Read from consumer
+        msg = read_message(r)
+        consumer = consumer_hash[r][:consumer]
+        stream = consumer_hash[r][:stream]
+
+        # Consumer is ready for next message
+        if msg["command"] && msg["command"] == "next"
+
+          @__emit_queues[stream][consumer][:ready] = true
+          tuple_json = get_consumer_tuple(stream, consumer)
+
+          # If all messages have been sent to a consumer, end its cycle
+          if tuple_json.nil?
+            send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+            consumers_running -= 1
+            if consumers_running == 0
+              return
+            end
+          else
+            # Emit tuple to consumer
+            emit_consumer_tuple(stream, consumer, tuple_json)
+          end
+        end
+      end
+    end
+  end
+
   def self.run_filter()
     self.run_each()
   end
@@ -861,14 +1155,12 @@ module Zillabyte; module Runner; class MultilangOperation
       col_map[key] = type_map[type]
     end
 
-
-
     messages = []
 
     output = @__options["output"]
     loop do
       # Read messages
-      msg = read_message(@__consumee[:rd_child])
+      msg = read_message(@__consumee_pipes["rd_child_1"])
       obj = msg
 
       # Add row
@@ -882,14 +1174,20 @@ module Zillabyte; module Runner; class MultilangOperation
           cdisplay("\n \nPress Ctrl-C to exit", false)
           return
         end
-
         # Check tuple columns for valid entry
         columns_to_check = col_map.length
-        tuple.each_pair do |col,value|
+        tuple.keys.each do |col|
+          value = tuple[col]
           types = type_map.each_value.map {|t| value.is_a? t}
-          if col_map.include? col
-            if value.is_a? col_map[col]
+          matched_column = check_tuple_for_alias(col, obj['column_aliases'], col_map.keys)
+          if !matched_column.nil?
+            if value.nil? || (value.is_a? col_map[matched_column])
               columns_to_check -= 1
+              # Set the proper column name
+              if col != matched_column
+                tuple[matched_column] = value
+                tuple.delete(col)
+              end
             else
               break
             end
@@ -901,12 +1199,11 @@ module Zillabyte; module Runner; class MultilangOperation
           cdisplay("\n \nPress Ctrl-C to exit", false)
           return
         end
-
         messages << msg
         if @__options[:interactive]
           cdisplay "received #{display_json}"
         end
-        write_message(@__consumee[:wr_child], NEXT_MESSAGE)
+        write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
 
       # End cycle
       elsif obj['command'] && obj['command'] == "end_cycle"
@@ -943,7 +1240,7 @@ module Zillabyte; module Runner; class MultilangOperation
 
             vals = [t.values, m.values].flatten
             csv << vals
-            table << vals.flat_map{|v| "#{v}"[0..100]}
+            table << vals.flat_map{|v| "#{v.to_s}"[0..100]}
           end
         rescue JSON::ParserError
           cdisplay("invalid JSON")
@@ -990,7 +1287,6 @@ module Zillabyte; module Runner; class MultilangOperation
       end
     end
     return rs unless rs.empty?
-
     rs, ws, es = IO.select(read_streams, [], [])
     return rs
   end
@@ -999,18 +1295,17 @@ module Zillabyte; module Runner; class MultilangOperation
   # Read a JSON message
   def self.read_message(read_stream)
 
+
     @__read_buffers[read_stream] ||= ""
     @__read_buffered_messages[read_stream] ||= []
     if !@__read_buffered_messages[read_stream].empty?
         obj = @__read_buffered_messages[read_stream].shift
       return obj
     end
-
     # read message from stream
     loop do
 
       while !@__read_buffers[read_stream].include? ENDMARKER
-
         segment = read_stream.sysread(BUFSIZE)
         @__read_buffers[read_stream] << segment
       end
@@ -1096,12 +1391,42 @@ module Zillabyte; module Runner; class MultilangOperation
     end
   end
 
+  # Build the hash of consumer streams for lookup when receiving responses
+  def self.build_consumer_hash()
+    consumer_hash = {}
+    @__emit_queues.each_pair do |stream, consumers|
+      consumers.each_key do |consumer|
+
+        pipes = @__consumer_pipes[stream][consumer]
+        if pipes.has_key? "rd_parent_1"
+          read_stream = pipes["rd_parent_1"]
+          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+
+        elsif pipes.has_key? "rd_parent_2"
+          read_stream = pipes["rd_parent_2"]
+          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+        end
+      end
+    end
+
+    return consumer_hash
+  end
+
 
   # Send object to every consumer of the operation, regardless of stream
   def self.send_to_consumers(json_obj)
     @__consumer_pipes.each_pair do |stream, consumers|
       consumers.each_pair do |consumer, pipe| 
-        write_message(pipe[:wr_parent], json_obj)
+
+        # Single or Left hand pipe
+        if (pipe.has_key? "wr_parent_1")
+          write_stream = get_write_stream(stream, consumer, 1)
+          write_message(write_stream, json_obj)
+        elsif (pipe.has_key? "wr_parent_2")
+          write_stream = get_write_stream(stream, consumer, 2)
+          write_message(write_stream, json_obj)
+        end
+
         cdisplay "emitted #{json_obj} to #{consumer}"
       end
     end
@@ -1109,8 +1434,9 @@ module Zillabyte; module Runner; class MultilangOperation
 
 
   # Get the write pipe of the stream consumer
-  def self.get_write_stream(stream, consumer)
-     @__consumer_pipes[stream][consumer][:wr_parent]
+  def self.get_write_stream(stream, consumer, number=1)
+     wr_pipe = "wr_parent_" + number.to_s
+     @__consumer_pipes[stream][consumer][wr_pipe]
   end
 
 
@@ -1120,6 +1446,23 @@ module Zillabyte; module Runner; class MultilangOperation
   end
 
 
+  # Send a command message to a consumer
+  def self.send_command_tuple(stream, consumer, json_obj)
+    pipe = @__consumer_pipes[stream][consumer]
+     # Single or Left hand pipe
+    if (pipe.has_key? "wr_parent_1")
+      write_stream = get_write_stream(stream, consumer, 1)
+      write_message(write_stream, json_obj)
+
+    # Right hand pipe
+    elsif (pipe.has_key? "wr_parent_2")
+      write_stream = get_write_stream(stream, consumer, 2)
+      write_message(write_stream, json_obj)
+    end
+    @__emit_queues[stream][consumer][:ready] = false
+  end
+
+
   # Emit tuple_json to the consumer of a stream
   def self.emit_consumer_tuple(stream, consumer, tuple_json)
     begin
@@ -1127,13 +1470,43 @@ module Zillabyte; module Runner; class MultilangOperation
     rescue JSON::ParserError
       cdisplay "Error: invalid JSON"
     end
-    write_stream = get_write_stream(stream, consumer)
-    write_message(write_stream, tuple_json)
+
+    pipe = @__consumer_pipes[stream][consumer]
+     # Single or Left hand pipe
+    if (pipe.has_key? "wr_parent_1")
+      write_stream = get_write_stream(stream, consumer, 1)
+      write_message(write_stream, tuple_json)
+
+    # Right hand pipe
+    elsif (pipe.has_key? "wr_parent_2")
+      write_stream = get_write_stream(stream, consumer, 2)
+      write_message(write_stream, tuple_json)
+    end
+
     @__emit_queues[stream][consumer][:ready] = false
     cdisplay "emitted tuple #{display_json} to #{consumer} "
   end
 
 
+  # Check if a tuple's column or its aliases matches an expected column and returns the valid column
+  def self.check_tuple_for_alias(column, column_aliases, valid_columns)
+
+    valid_columns.each do |expected|
+      if column == expected
+        return expected
+      end
+
+      column_aliases.each do |hash|
+        concrete_name = hash["concrete_name"]
+        if (column == concrete_name || column == hash['alias']) && hash["alias"] == expected
+          return expected
+        end
+      end
+    end
+
+    return nil
+  end
+
   # Build a tuple and format into JSON
   def self.build_tuple_json(tuple, meta = nil, column_aliases = nil)
     meta ||= {}
@@ -1144,11 +1517,12 @@ module Zillabyte; module Runner; class MultilangOperation
        next
      elsif(k == "confidence" or k == "since" or k == "source")
        meta[k] = v
-     else  
+     else
       values[k] = v
      end
     end
-    tuple_json = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
+    tuple_json = {"op" => @__name, "tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
+
     return tuple_json
   end