zillabyte-cli 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eea29081785924f4123050342b0106d98355f0c8
-  data.tar.gz: 9e5e777aa135171b8230ced0011622519e450539
+  metadata.gz: 90127131598b38dedc88b0903dc0b984537442a1
+  data.tar.gz: acf6b8bb47b8476621ea36834310ed48d479463a
 SHA512:
-  metadata.gz: 9b6441517bb14bd08ce83ddc6502e99287434f80403e6a1ee634c9880546ac772c32ea49bffe320737d72b7c70f1135e65862026df7ae89df3542f61447d5788
-  data.tar.gz: ad27686e26e9ebe20d2a4e38879449108dc2f8df4be5c70fecd97617cd409ec2f457dc337a09f3b4772144d5cf586d0e71533150a3551d7d5b5e4537e73e7924
+  metadata.gz: d99663ba95dfbc8e76ef06656bf55621812f71c400c2ed170f00c5528cf04225b188b4b47e003fab82d8ac6f8f48f262261a47bd4f3750ef6d7715431ce85eee
+  data.tar.gz: 32ab54910d7b01b6171123ec1b7312a8fb8bafcb612c5465b3af5dfeaf77a0eda9e6c957bc179487bba29ea18550d63828b20e091a3640b03283448151d54d68

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.9.0"
+    VERSION = "0.9.1"
   end
 end

data/lib/zillabyte/runner/component_operation.rb

@@ -37,11 +37,13 @@ class Zillabyte::Runner::ComponentOperation
     end
     begin
       case @__type
-      when "source"
+      when "input"
         self.run_input()
       when "component"
         self.run_rpc_component()
-
+      when "output"
+        node["type"] = "sink"
+        Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
       # Component outputs act in same manner as sinks
       else
         Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
@@ -50,6 +52,7 @@ class Zillabyte::Runner::ComponentOperation
       cdisplay e.message
       cdisplay e.backtrace
     end
+    cdisplay "EXIT"
   end
 
 
@@ -142,9 +145,8 @@ class Zillabyte::Runner::ComponentOperation
       loop do 
 
         msg = stdin.gets
-        if msg == END_CYCLE_MESSAGE
-          send_to_consumers(END_CYCLE_MESSAGE)
-          send_to_consumers(DONE_MESSAGE)
+        if msg == KILL_CYCLE_MESSAGE
+          send_to_consumers(KILL_CYCLE_MESSAGE)
           return
         else
           # Build tuples
@@ -271,13 +273,6 @@ class Zillabyte::Runner::ComponentOperation
                   # Handle Tuple in here
                   data_streams = hash['data']
 
-                  # if stream_hash.nil?
-                  #   cdisplay("error : no results!")
-                  #   next
-                  # else
-                  #   stream_hash.each_pair
-                  # end
-
                   # Handle each resulting tuple
                   data_streams.each_pair do |stream, data_tuples|
                     data_tuples.each do |data_tuple|
@@ -334,7 +329,7 @@ class Zillabyte::Runner::ComponentOperation
                 send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
                 consumers_running -= 1
                 if consumers_running == 0
-                  break
+                  return
                 end
 
                 # TODO break if last consumer
@@ -356,7 +351,7 @@ class Zillabyte::Runner::ComponentOperation
 
         # Exit after ending consumer cycles
         if consumers_running == 0
-          break
+          return
         end
       end
 

data/lib/zillabyte/runner/component_runner.rb

@@ -5,7 +5,7 @@ require "zillabyte/runner/component_operation"
 class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base 
   include Zillabyte::Helpers
 
-  END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
+  KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
 
   def run (meta, dir = Dir.pwd, session = nil, options = {})
 
@@ -154,7 +154,7 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
             out_pipes = {}
 
             # Check if you are the consumee for a downstream join in order to select the correct pipe
-            if type != "sink"
+            if type != "output"
               @arc_map[name].each_pair do |stream, destinations|
                 out_pipes[stream] ||= {}
                 
@@ -216,7 +216,7 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
           name = n["name"]
           type = n["type"]
 
-          if type == "source"
+          if type == "input"
             if source == ""
               source = name
             else
@@ -235,8 +235,13 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
           fields = @node_map[source]['fields'].map {|h| h.keys[0].upcase }
           display "Enter an input tuple in the form : #{fields.join(' ')}"
           msg = ask
+
+          # Kill the cycle
           if msg == 'end'
-            msg = END_CYCLE_MESSAGE
+            @operation_pipes[source]["wr_parent_1"].puts KILL_CYCLE_MESSAGE
+            break
+            
+          # Check arguments
           else
             args = msg.scan(/(?:\w|"[^"]*")+/)
             if (args.length % fields.length != 0)
@@ -244,11 +249,12 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
               next
             end
           end
-
           # Send tuple to source
           @operation_pipes[source]["wr_parent_1"].puts msg
+         
         end
       end
+
     rescue => e
       display e.message
       display e.backtrace
@@ -311,12 +317,12 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
       display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
 
       # Convert metadata typing to that of components
-      if node['type'] == "source"
+      if node['type'] == "input"
         type = "input"
         display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
         display "#{"fields".rjust(rjust)}: #{node['fields'].to_s.colorize(color)}"
         display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
-      elsif node['type'] == "sink"
+      elsif node['type'] == "output"
         type = "output"
         display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
         display "#{"columns".rjust(rjust)}: #{node['columns'].to_s.colorize(color)}"

data/lib/zillabyte/runner/multilang_operation.rb

@@ -3,6 +3,7 @@
 module Zillabyte; module Runner; class MultilangOperation
 
   HANDSHAKE_MESSAGE = "{\"pidDir\": \"/tmp\"}\n"
+  PREPARE_MESSAGE = " {\"command\": \"prepare\"}\n"
   DONE_MESSAGE = "{\"command\": \"done\"}\n"
   NEXT_MESSAGE = "{\"command\": \"next\"}\n"
   BEGIN_CYCLE_MESSAGE = "{\"command\": \"begin_cycle\"}\n"
@@ -77,9 +78,8 @@ module Zillabyte; module Runner; class MultilangOperation
 
         msg = @__consumee_pipes["rd_child_1"].gets
 
-        if msg == END_CYCLE_MESSAGE            
-          send_to_consumers(END_CYCLE_MESSAGE)
-          send_to_consumers(DONE_MESSAGE)
+        if msg == KILL_CYCLE_MESSAGE            
+          send_to_consumers(KILL_CYCLE_MESSAGE)
           return
         else
           # Build tuple
@@ -193,217 +193,211 @@ module Zillabyte; module Runner; class MultilangOperation
       # Kill the cycle on error
       cycle_killed = false
 
-      # Setup multilang pipe
-      ml_pipe = "#{@__name}_pipe"
-      if File.exists?("#{ml_pipe}.in")
-        File.delete("#{ml_pipe}.in")
-      end
-      File.mkfifo("#{ml_pipe}.in")
-
-
+      # Setup multilang socket
+      require 'socket'
+      host = "0.0.0.0"
+      server = TCPServer.new(0)
+      port = server.addr[1]
+    
       # Spawn multilang process
-      cmd = command("--execute_live --name #{@__name} --pipe #{ml_pipe}")
+      cmd = command("--execute_live --name #{@__name}  --host \"#{host}\" --port #{port}")
+
       begin
 
-        Open3.popen3(cmd) do |ml_input, stdout, stderr, wait_thread|
+        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
           begin
+            server_thread = Thread.new do 
+              ml_socket = server.accept()
 
-            # Multilang output tuples
-            ml_output = File.open("#{ml_pipe}.in", "r")
-
-            # Setup streams from consumers and multilang
-            read_streams = consumer_hash.keys.concat [stdout, ml_output]
-
-
-            # Handshake
-            handshake(ml_input, ml_output)
-
-            # Begin cycle
-            begin_cycle(ml_input, ml_output)
-            emitted = false
-            write_message(ml_input, NEXT_MESSAGE)
-            multilang_queue = []
-            end_cycle_policy = @__options[:end_cycle_policy]
-            end_cycle_received = false
-
-            # Receive and handle messages
-            loop do
-             
-              # Read from a stream
-              rs = select_read_streams(read_streams)
-              rs.each do |r|
-
-                # Read stdout straight to user
-                if r == stdout && consumers_running > 0
-                  msg = r.gets
-                  msg = msg.sub(/\n/, "")
-                  cdisplay("log: #{msg}")
-                  next
-                end
-
-                obj = read_message(r)
-
-                if obj.nil?
-                  next
-                end
+              # RUN SOURCE
+              begin
+                # Setup streams from consumers and multilang(stdout and socket communication)
+                read_streams = consumer_hash.keys.concat [stdout, ml_socket]
 
-                if obj["command"] 
+                # Handshake
+                handshake(ml_socket, ml_socket)
+                prepare(ml_socket, ml_socket)
 
-                  case obj["command"]
+                # Begin cycle
+                begin_cycle(ml_socket, ml_socket)
+                emitted = false
+                write_message(ml_socket, NEXT_MESSAGE)
+                multilang_queue = []
+                end_cycle_policy = @__options[:end_cycle_policy]
+                end_cycle_received = false
+
+                # Receive and handle messages
+                loop do
+                  # Read from a stream
+                  rs = select_read_streams(read_streams)
+                  rs.each do |r|
+                    # Read stdout straight to user
+                    if r == stdout && consumers_running > 0
+                      msg = r.gets
+                      msg = msg.sub(/\n/, "")
+                      cdisplay("log: #{msg}")
+                      next
+                    end
 
-                  # Multilang emitted a tuple
-                  when "emit"
-                    stream = obj['stream']
-                    # Check for null emit
-                    if end_cycle_policy != "explicit"
+                    obj = read_message(r)
 
-                      if obj['tuple'].nil?
-                        end_cycle_received = true
-                      else
-                        nil_values = false
-                        obj['tuple'].each_value do |v|
-                          if v.nil?
-                            nil_values = true
-                            break       
-                          end                     
-                        end
-                        end_cycle_received = nil_values
-                        next unless !end_cycle_received
-                      end
+                    if obj.nil?
+                      next
                     end
 
-                    # Valid emit
-                    emitted = true
-
-                    # Send or enqueue tuple for each consumer
-                    tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
+                    if obj["command"] 
+                      case obj["command"]
+
+                      # Multilang emitted a tuple
+                      when "emit"
+                        stream = obj['stream']
+                        # Check for null emit
+                        if end_cycle_policy != "explicit"
+
+                          if obj['tuple'].nil?
+                            end_cycle_received = true
+                          else
+                            nil_values = false
+                            obj['tuple'].each_value do |v|
+                              if v.nil?
+                                nil_values = true
+                                break       
+                              end                     
+                            end
+                            end_cycle_received = nil_values
+                            next unless !end_cycle_received
+                          end
+                        end
 
-                    @__emit_queues[stream].each_pair do |consumer, emitter|
-                      if emitter[:ready]
-                        emit_consumer_tuple(stream, consumer, tuple_json)
+                        # Valid emit
+                        emitted = true
 
-                      else
-                        @__emit_queues[stream][consumer][:write_queue] << tuple_json
-                      end
-                    end
+                        # Send or enqueue tuple for each consumer
+                        tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
 
-                  # Consumer is ready for a message
-                  when "next"
-                    stream = consumer_hash[r][:stream]
-                    consumer = consumer_hash[r][:consumer]
-                    @__emit_queues[stream][consumer][:ready] = true
-                    tuple_json = get_consumer_tuple(stream, consumer)
+                        @__emit_queues[stream].each_pair do |consumer, emitter|
+                          if emitter[:ready]
+                            emit_consumer_tuple(stream, consumer, tuple_json)
 
-                    # End cycle for consumer if it has processed all tuples
-                    if tuple_json.nil? && end_cycle_received
+                          else
+                            @__emit_queues[stream][consumer][:write_queue] << tuple_json
+                          end
+                        end
 
-                      send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                      consumers_running -= 1
-                      if consumers_running == 0
-                        break
-                      end
+                      # Consumer is ready for a message
+                      when "next"
+                        stream = consumer_hash[r][:stream]
+                        consumer = consumer_hash[r][:consumer]
+                        @__emit_queues[stream][consumer][:ready] = true
+                        tuple_json = get_consumer_tuple(stream, consumer)
 
-                      # TODO break if last consumer
-                    elsif !tuple_json.nil?
-                      # Emit tuple to consumer
-                      emit_consumer_tuple(stream, consumer, tuple_json)
-                      emitted = true
-                    end
+                        # End cycle for consumer if it has processed all tuples
+                        if tuple_json.nil? && end_cycle_received
 
-                  # Multilang is done emitting a group of tuples
-                  when "done"
-                    # End cycle if no tuples were emitted
-                    if !emitted && end_cycle_policy == "null_emit"
-                      end_cycle_received = true
-                    else
-                      emitted = false
-                    end
-                  
-                    # Send the next tuple to multilang
-                    if !multilang_queue.empty?
-                      write_message(ml_input, multilang_queue.shift)
+                          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                          consumers_running -= 1
+                          if consumers_running == 0
+                            exit(0)
+                          end
 
-                    # Request next tuple from mutilang
-                    elsif !end_cycle_received
-                      write_message(ml_input, NEXT_MESSAGE)
+                          # TODO break if last consumer
+                        elsif !tuple_json.nil?
+                          # Emit tuple to consumer
+                          emit_consumer_tuple(stream, consumer, tuple_json)
+                          emitted = true
+                        end
 
-                    # If there are no more messages to send, we are done
-                    else end_cycle_received
-                      finished = true
-                       # End cycle for ready consumers
-                      @__emit_queues.each_pair do |stream, consumers|
-                        consumers.each_pair do |consumer, emitter|
-                          if emitter[:ready]
-                            send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                            consumers_running -= 1
-                            if consumers_running == 0
-                              break
+                      # Multilang is done emitting a group of tuples
+                      when "done"
+                        # End cycle if no tuples were emitted
+                        if !emitted && end_cycle_policy == "null_emit"
+                          end_cycle_received = true
+                        else
+                          emitted = false
+                        end
+                      
+                        # Send the next tuple to multilang
+                        if !multilang_queue.empty?
+                          write_message(ml_socket, multilang_queue.shift)
+
+                        # Request next tuple from mutilang
+                        elsif !end_cycle_received
+                          write_message(ml_socket, NEXT_MESSAGE)
+
+                        # If there are no more messages to send, we are done
+                        else end_cycle_received
+                          finished = true
+                           # End cycle for ready consumers
+                          @__emit_queues.each_pair do |stream, consumers|
+                            consumers.each_pair do |consumer, emitter|
+                              if emitter[:ready]
+                                send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                                consumers_running -= 1
+                                if consumers_running == 0
+                                  exit(0)
+                                end
+                              end
                             end
+
                           end
                         end
 
+                      # Multilang sent an error message
+                      when "fail"
+                        cdisplay("ERROR : #{obj['msg']}")
+                        cycle_killed = true
+                        exit(0)
+                      
+                      # Multilang sent a log message
+                      when "log"
+                        cdisplay "LOG: #{obj['msg']}"
+
+                      # Multilang sent signal to end the cycle
+                      when "end_cycle"
+                        if end_cycle_policy != "explicit"
+                          cdisplay "received end_cycle command for non explicit policy"
+                          next
+                        end
+                        end_cycle_received = true
+                      when "kill_cycle"
+                        cycle_killed = true
+                        exit(0)
                       end
-                    end
-
-                  # Multilang sent an error message
-                  when "fail"
-                    cdisplay("ERROR : #{obj['msg']}")
                   
-                  # Multilang sent a log message
-                  when "log"
-                    cdisplay "LOG: #{obj['msg']}"
-
-                  # Multilang sent signal to end the cycle
-                  when "end_cycle"
-                    if end_cycle_policy != "explicit"
-                      cdisplay "received end_cycle command for non explicit policy"
-                      next
+                    # Multilang sent a ping
+                    elsif obj['ping']
+                      write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
                     end
-                    end_cycle_received = true
-                  when "kill_cycle"
-                    cycle_killed = true
-                    return
                   end
-                
-                # Multilang sent a ping
-                elsif obj['ping']
-                  write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
                 end
-              end
 
-              # Exit after ending consumer cycles
-              if consumers_running == 0
-                return
+                # Exit after ending consumer cycles
+                if consumers_running == 0
+                  exit(0)
+                end
+              rescue => e
+                cycle_killed = true
+                cdisplay e.message
+                cdisplay e.backtrace
+              ensure
+                ml_socket.close()
               end
-              
             end
+            server_thread.join()
           rescue => e
-            cycle_killed = true
-            cdisplay e.message
-            cdisplay e.backtrace
           ensure
-
             # cleanup
             if cycle_killed
               send_to_consumers(KILL_CYCLE_MESSAGE, false)
             end
             pid = wait_thread[:pid] 
-            ml_input.close
-            if File.exists?("#{ml_pipe}.in")
-              ml_output.close
-              File.delete("#{ml_pipe}.in")
-            end
             stdout.close
-            stderr.close
             Process.kill('INT', pid)
             Process.exit!(true)
           end
         end
 
       rescue PTY::ChildExited
-        if File.exists?("#{ml_pipe}.in")
-          File.delete("#{ml_pipe}.in")
-        end
       end
     end
   end
@@ -411,7 +405,6 @@ module Zillabyte; module Runner; class MultilangOperation
 
   def self.run_each()
 
-
     # Index streams and consumers by their pipes for lookup
     consumer_hash = build_consumer_hash()
 
@@ -421,182 +414,187 @@ module Zillabyte; module Runner; class MultilangOperation
     # Kill the cycle on error
     cycle_killed = false
 
-    # Setup multilang pipe
-    ml_pipe = "#{@__name}_pipe"
-    if File.exists?("#{ml_pipe}.in")
-      File.delete("#{ml_pipe}.in")
-    end
-    File.mkfifo("#{ml_pipe}.in")
-
-    cmd = command("--execute_live --name #{@__name} --pipe #{ml_pipe}")
+    # Setup multilang socket
+    require 'socket'
+    host = "0.0.0.0"
+    server = TCPServer.new(0)
+    port = server.addr[1]
+  
+    # Spawn multilang process
+    cmd = command("--execute_live --name #{@__name}  --host \"#{host}\" --port #{port}")
+    
     begin
       # Start the operation... 
-      Open3.popen3(cmd) do |ml_input, stdout, stderr, wait_thread|
+      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
         begin
+          server_thread = Thread.new do 
+            ml_socket = server.accept()
+            begin
+              # Setup streams from consumers, multilang, and the consumee
+              read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"], ml_socket, stdout]
 
-          # Multilang output tuples
-          ml_output = File.open("#{ml_pipe}.in", "r+")
-
-          # Setup streams from consumers, multilang, and the consumee
-          read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"], ml_output, stdout]
-
-          # Handshake
-          handshake(ml_input, ml_output)
+              # Handshake
+              handshake(ml_socket, ml_socket)
+              prepare(ml_socket, ml_socket)
 
-          # Begin cycle
-          multilang_queue = []
-          mutlilang_count = 0
-          end_cycle_received = false
-          column_aliases = nil
+              # Begin cycle
+              multilang_queue = []
+              mutlilang_count = 0
+              end_cycle_received = false
+              column_aliases = nil
 
 
-          # Receive and handle messages
-          loop do
+              # Receive and handle messages
+              loop do
 
-            # Read from a stream
-            rs = select_read_streams(read_streams)
-            rs.each do |r|
+                # Read from a stream
+                rs = select_read_streams(read_streams)
+                rs.each do |r|
 
-              # Read STDOUT from program straight to user
-              if r == stdout
-                msg = r.gets
-                if !msg.nil?
-                  msg = msg.sub(/\n/, "")
-                  cdisplay("LOG: #{msg}")
-                end
-                next
-              end
+                  # Read STDOUT from program straight to user
+                  if r == stdout
+                    msg = r.gets
+                    if !msg.nil?
+                      msg = msg.sub(/\n/, "")
+                      cdisplay("LOG: #{msg}")
+                    end
+                    next
+                  end
 
-              # Receive an object
-              obj = read_message(r)
+                  # Receive an object
+                  obj = read_message(r)
 
-              if obj["command"] 
-                case obj["command"]
+                  if obj["command"] 
+                    case obj["command"]
 
-                # Multilang emitted a tuple
-                when "emit"
+                    # Multilang emitted a tuple
+                    when "emit"
 
-                  stream = obj["stream"]
-                 
-                  # Send or enqueue tuple for each consumer
-                  tuple_json = build_tuple_json(obj['tuple'], obj['meta'], column_aliases)
+                      stream = obj["stream"]
+                     
+                      # Send or enqueue tuple for each consumer
+                      tuple_json = build_tuple_json(obj['tuple'], obj['meta'], column_aliases)
 
-                  @__emit_queues[stream].each_pair do |consumer, emitter|
-                    if emitter[:ready]
-                      emit_consumer_tuple(stream, consumer, tuple_json)
-                    else
-                      @__emit_queues[stream][consumer][:write_queue] << tuple_json
-                    end
-                  end
-
-                # Consumer is ready for a message
-                when "next"
-                  stream = consumer_hash[r][:stream]
-                  consumer = consumer_hash[r][:consumer]
+                      @__emit_queues[stream].each_pair do |consumer, emitter|
+                        if emitter[:ready]
+                          emit_consumer_tuple(stream, consumer, tuple_json)
+                        else
+                          @__emit_queues[stream][consumer][:write_queue] << tuple_json
+                        end
+                      end
 
+                    # Consumer is ready for a message
+                    when "next"
+                      stream = consumer_hash[r][:stream]
+                      consumer = consumer_hash[r][:consumer]
 
-                  @__emit_queues[stream][consumer][:ready] = true
-                  tuple_json = get_consumer_tuple(stream, consumer)
 
-                  # End cycle for consumer if it has processed all tuples
-                  if tuple_json.nil? && end_cycle_received
-                    send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                    consumers_running -= 1
-                    if consumers_running == 0
-                      break
-                    end
+                      @__emit_queues[stream][consumer][:ready] = true
+                      tuple_json = get_consumer_tuple(stream, consumer)
 
+                      # End cycle for consumer if it has processed all tuples
+                      if tuple_json.nil? && end_cycle_received
+                        send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                        consumers_running -= 1
+                        if consumers_running == 0
+                          break
+                        end
 
-                    # TODO break if last consumer
-                  elsif !tuple_json.nil?
-                    # Emit tuple to consumer
-                    emit_consumer_tuple(stream, consumer, tuple_json)
-                    emitted = true
-                  end
 
-                # Multilang is done emitting a group of tuples
-                when "done" 
-                  mutlilang_count -= 1                  
+                        # TODO break if last consumer
+                      elsif !tuple_json.nil?
+                        # Emit tuple to consumer
+                        emit_consumer_tuple(stream, consumer, tuple_json)
+                        emitted = true
+                      end
 
-                  # Send the next tuple to multilang
-                  if !multilang_queue.empty?
-                    write_message(ml_input, multilang_queue.shift)
+                    # Multilang is done emitting a group of tuples
+                    when "done" 
+                      mutlilang_count -= 1                  
 
-                  # Request next tuple from consumee
-                  elsif !end_cycle_received
-                    write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
+                      # Send the next tuple to multilang
+                      if !multilang_queue.empty?
+                        write_message(ml_socket, multilang_queue.shift)
 
-                  # If there are no more messages to send, we are done
-                  elsif end_cycle_received && mutlilang_count == 0
-                    finished = true
+                      # If there are no more messages to send, we are done
+                      elsif end_cycle_received && mutlilang_count == 0
+                        finished = true
 
-                   # End cycle for ready consumers
-                    @__emit_queues.each_pair do |stream, consumers|
-                      consumers.each_pair do |consumer, emitter|
-                        if emitter[:ready]
-                          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                          consumers_running -= 1
-                          if consumers_running == 0
-                            break
+                       # End cycle for ready consumers
+                        @__emit_queues.each_pair do |stream, consumers|
+                          consumers.each_pair do |consumer, emitter|
+                            if emitter[:ready]
+                              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                              consumers_running -= 1
+                              if consumers_running == 0
+                                break
+                              end
+                            end
                           end
                         end
                       end
-                    end
-                  end
 
-                # Multilang sent an error message
-                when "fail"
-                  cdisplay("ERROR : #{obj['msg']}")
-                
-                # Multilang sent a log message
-                when "log"
-                  cdisplay "LOG: #{obj['msg']}"
-
-                # Consumee operation sent signal to end_cycle
-                when "end_cycle"
-                  end_cycle_received = true
-                  if mutlilang_count == 0
-                    @__emit_queues.each_pair do |stream, consumers|
-                      consumers.each_pair do |consumer, emitter|                      
-                        if emitter[:ready]
-                          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                          consumers_running -= 1
-                          if consumers_running == 0
-                            return
+                    # Multilang sent an error message
+                    when "fail"
+                      cdisplay("ERROR : #{obj['msg']}")
+                      cycle_killed = true
+                      exit(0)
+                    
+                    # Multilang sent a log message
+                    when "log"
+                      cdisplay "LOG: #{obj['msg']}"
+
+                    # Consumee operation sent signal to end_cycle
+                    when "end_cycle"
+                      end_cycle_received = true
+                      if mutlilang_count == 0
+                        @__emit_queues.each_pair do |stream, consumers|
+                          consumers.each_pair do |consumer, emitter|                      
+                            if emitter[:ready]
+                              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                              consumers_running -= 1
+                              if consumers_running == 0
+                                exit(0)
+                              end
+                            end
                           end
                         end
                       end
+           
+                    when "kill_cycle"
+                      cycle_killed = true
+                      exit(0)
                     end
+
+                  # Received a tuple from consumee
+                  elsif obj['tuple']
+                    column_aliases = obj['column_aliases']
+                    # Send or enqueue to multilang
+                    mutlilang_count += 1
+                    if multilang_queue.empty?
+                      write_message(ml_socket, obj.to_json)
+                    else
+                      multilang_queue << obj.to_json
+                    end
+                    write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
+                  
+                  # Multilang sent a ping
+                  elsif obj['ping']
+                    write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
                   end
-       
-                when "kill_cycle"
-                  cycle_killed = true
-                  return
                 end
 
-              # Received a tuple from consumee
-              elsif obj['tuple']
-                column_aliases = obj['column_aliases']
-                # Send or enqueue to multilang
-                mutlilang_count += 1
-                if multilang_queue.empty?
-                  write_message(ml_input, obj.to_json)
-                else
-                  multilang_queue << obj.to_json
+                # Exit after ending consumer cycles
+                if consumers_running == 0
+                  exit(0)
                 end
-              
-              # Multilang sent a ping
-              elsif obj['ping']
-                write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
-              end
-            end
 
-            # Exit after ending consumer cycles
-            if consumers_running == 0
-              return
+              end
+            ensure
+              ml_socket.close()
             end
-
           end
+          server_thread.join()
         rescue => e
            cdisplay e.message
            cdisplay e.backtrace
@@ -604,28 +602,18 @@ module Zillabyte; module Runner; class MultilangOperation
           # cleanup
           if cycle_killed
             send_to_consumers(KILL_CYCLE_MESSAGE, false)
-            send_to_consumees(KILL_CYCLE_MESSAGE, false)
+            send_to_consumees(KILL_CYCLE_MESSAGE)
           end
           pid = wait_thread[:pid] 
-          ml_input.close
-          if File.exists?("#{ml_pipe}.in")
-            ml_output.close
-            File.delete("#{ml_pipe}.in")
-          end
+          stdin.close
           stdout.close
           stderr.close
-          Process.kill('INT', pid)
-          Process.exit!(true)
         end
       end
     rescue PTY::ChildExited
-     
       cdisplay("The child process exited!")
-    ensure
-      if File.exists?("#{ml_pipe}.in")
-        File.delete("#{ml_pipe}.in")
-      end
     end
+
   end
 
 
@@ -648,219 +636,223 @@ module Zillabyte; module Runner; class MultilangOperation
     tuple_queue = []
 
     # Setup multilang pipe
-    ml_pipe = "#{@__name}_pipe"
-    if File.exists?("#{ml_pipe}.in")
-      File.delete("#{ml_pipe}.in")
-    end
-    File.mkfifo("#{ml_pipe}.in")
-    
-    cmd = command("--execute_live --name #{@__name} --pipe #{ml_pipe}")
+    require 'socket'
+    host = "0.0.0.0"
+    server = TCPServer.new(0)
+    port = server.addr[1]
+
+    # Spawn multilang process
+    cmd = command("--execute_live --name #{@__name}  --host \"#{host}\" --port #{port}")
+
     begin
       # Start the operation... 
-      Open3.popen3(cmd) do |ml_input, stdout, stderr, wait_thread|
+      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
         begin
+          server_thread = Thread.new do 
+            ml_socket = server.accept()
+            begin
+              # Setup streams from consumers, multilang, and the consumee
+              read_streams = consumer_hash.keys.concat [stdout, ml_socket, @__consumee_pipes["rd_child_1"]]
 
-          # Multilang output tuples
-          ml_output = File.open("#{ml_pipe}.in", "r+")
-          # Setup streams from consumers, multilang, and the consumee
-          read_streams = consumer_hash.keys.concat [stdout, ml_output, @__consumee_pipes["rd_child_1"]]
-
-          # Handshake
-          handshake(ml_input, ml_output)
+              # Handshake
+              handshake(ml_socket, ml_socket)
+              prepare(ml_socket, ml_socket)
 
-          # Begin cycle
-          end_cycle_received = false
-          finished_emitting = false
+              # Begin cycle
+              end_cycle_received = false
+              finished_emitting = false
 
-          # select a stream
-          loop do
+               # Handle streams
+              loop do
+                # Read from a stream
+                rs = select_read_streams(read_streams)
+                rs.each do |r|
 
-            # Read from a stream
-            rs = select_read_streams(read_streams)
-            rs.each do |r|
+                  # Read STDOUT from program straight to user
+                  if r == stdout
+                    msg = r.gets
+                    msg = msg.sub(/\n/, "")
+                    cdisplay("log: #{msg}")
+                    next
+                  end
 
-              # Read STDOUT from program straight to user
-              if r == stdout
-                msg = r.gets
-                msg = msg.sub(/\n/, "")
-                cdisplay("log: #{msg}")
-                next
-              end
 
+                  # Receive an object
+                  obj = read_message(r)
+    
+                  if obj["command"] 
+                    case obj["command"]
 
-              # Receive an object
-              obj = read_message(r)
-            
-              if obj["command"] 
-                case obj["command"]
+                    when "done"
 
-                when "done"
+                      if end_cycle_received
+                        tuple_json = tuple_queue.shift
+                        if !tuple_json.nil?
+                          write_message(ml_socket, tuple_json)
+                        end
+                      end
 
-                  if end_cycle_received
-                    tuple_json = tuple_queue.shift
-                    if !tuple_json.nil?
-                      write_message(ml_input, tuple_json)
-                    end
-                  end
+                      next
 
-                  next
+                    # Begin aggregation
+                    when "end_cycle"
+                      end_cycle_received = true
+                      read_streams = [ml_socket]
 
-                # Begin aggregation
-                when "end_cycle"
-                  end_cycle_received = true
-                  read_streams = [ml_output]
+                      group_tuples.each do |group_tuple, tuples|
+                        tuple_queue << "{\"command\": \"begin_group\", \"tuple\": #{group_tuple.to_json}, \"meta\":{}}\n"
+                        tuples.each do |t|
+                          tuple_queue << "{\"command\": \"aggregate\", #{t}}\n"
+                        end
+                        tuple_queue << "{\"command\": \"end_group\"}\n"
 
-                  group_tuples.each do |group_tuple, tuples|
-                    tuple_queue << "{\"command\": \"begin_group\", \"tuple\": #{group_tuple.to_json}, \"meta\":{}}\n"
-                    tuples.each do |t|
-                      tuple_queue << "{\"command\": \"aggregate\", #{t}}\n"
-                    end
-                   tuple_queue << "{\"command\": \"end_group\"}\n"
+                        # keep track of how many emits are expected
+                        emit_count += 1
+                      end
 
-                    # keep track of how many emits are expected
-                    emit_count += 1
-                  end
+                      tuple_json = tuple_queue.shift
+                      if !tuple_json.nil?
+                        write_message(ml_socket, tuple_json)
+                      end
 
-                  tuple_json = tuple_queue.shift
-                  if !tuple_json.nil?
-                    write_message(ml_input, tuple_json)
-                  end
+                    # Multilang has emitted a grouped tuple
+                    when "emit"
+                      stream = obj['stream']
+                      emit_count -= 1
+                      # Enqueue for consumers
+                      tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
+                      @__emit_queues.each_pair do |stream, consumers|
+                        consumers.each_key do |consumer|
+                          @__emit_queues[stream][consumer][:write_queue] << tuple_json  
+                        end
+                      end
 
-                # Multilang has emitted a grouped tuple
-                when "emit"
-                  stream = obj['stream']
-                  emit_count -= 1
-                  # Enqueue for consumers
-                  tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
-                  @__emit_queues.each_pair do |stream, consumers|
-                    consumers.each_key do |consumer|
-                      @__emit_queues[stream][consumer][:write_queue] << tuple_json  
+                      # End cycle when done emitting
+                      if end_cycle_received && emit_count == 0
+                        finished_emitting = true
+                        break      
+                      elsif end_cycle_received
+                        tuple_json = tuple_queue.shift
+                        if !tuple_json.nil?
+                          write_message(ml_socket, tuple_json)
+                        end
+                      end
+                  
+                    # An error has occured
+                    when "kill_cycle"
+                      cycle_killed = true
+                      exit(0)
                     end
-                  end
 
-                  # End cycle when done emitting
-                  if end_cycle_received && emit_count == 0
-                    finished_emitting = true
-                    break      
-                  elsif end_cycle_received
-                    tuple_json = tuple_queue.shift
-                    if !tuple_json.nil?
-                      write_message(ml_input, tuple_json)
+                  # Received a tuple from operation
+                  elsif obj["tuple"]
+                    tuple = obj["tuple"].to_json
+                    meta = obj["meta"].to_json
+                    column_aliases = obj["column_aliases"] || {}
+                    aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
+                    gt = {}
+
+                    # Get the column names to group on
+                    group_by.each do |field|
+                      field_name = aliases[field] || field
+                      gt[field] = obj["tuple"][field_name]
                     end
-                  end
-              
-                # An error has occured
-                when "kill_cycle"
-                  cycle_killed = true
-                  return
-                end
 
-              # Received a tuple from operation
-              elsif obj["tuple"]
-                tuple = obj["tuple"].to_json
-                meta = obj["meta"].to_json
-                column_aliases = obj["column_aliases"] || {}
-                aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
-                gt = {}
-
-                # Get the column names to group on
-                group_by.each do |field|
-                  field_name = aliases[field] || field
-                  gt[field] = obj["tuple"][field_name]
-                end
+                    msg_no_brackets = "\"tuple\": #{tuple}, \"meta\": #{meta}, \"column_aliases\": #{column_aliases.to_json}"
 
-                msg_no_brackets = "\"tuple\": #{tuple}, \"meta\": #{meta}, \"column_aliases\": #{column_aliases.to_json}"
-
-                # Group tuple into existing group or create new group
-                if group_tuples[gt]
-                  group_tuples[gt] << msg_no_brackets
-                else
-                  group_tuples[gt] = [msg_no_brackets]
-                end
+                    # Group tuple into existing group or create new group
+                    if group_tuples[gt]
+                      group_tuples[gt] << msg_no_brackets
+                    else
+                      group_tuples[gt] = [msg_no_brackets]
+                    end
 
-                # Ask operation for next tuple
-                write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
+                    # Ask operation for next tuple
+                    write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
 
-              # Multilang sent a ping
-              elsif obj['ping']
-                write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
-              end
-            end
+                  # Multilang sent a ping
+                  elsif obj['ping']
+                    write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
+                  end
+                end
 
-            # Send tuples to consumers
-            if finished_emitting && consumers_running > 0
+                # Send tuples to consumers
+                if finished_emitting && consumers_running > 0
 
-              # Send first tuple
-              @__emit_queues.each_pair do |stream, consumers|
-                consumers.each_key do |consumer|
-                  tuple_json = get_consumer_tuple(stream, consumer)
-                  emit_consumer_tuple(stream, consumer, tuple_json)
-                end
-              end
+                  # Send first tuple
+                  @__emit_queues.each_pair do |stream, consumers|
+                    consumers.each_key do |consumer|
+                      tuple_json = get_consumer_tuple(stream, consumer)
+                      emit_consumer_tuple(stream, consumer, tuple_json)
+                    end
+                  end
 
-              # Sent tuples to consumers as appropriate
-              loop do 
+                  # Sent tuples to consumers as appropriate
+                  loop do 
 
-                # Retrieve messages from consumers
-                rs, ws, es = IO.select(consumer_hash.keys, [], [])
+                    # Retrieve messages from consumers
+                    rs, ws, es = IO.select(consumer_hash.keys, [], [])
 
-                # Emit tuples to consumers
-                emitted = false
-                rs.each do |r|
+                    # Emit tuples to consumers
+                    emitted = false
+                    rs.each do |r|
 
-                  # Read from consumer
-                  msg = read_message(r)
-                  consumer = consumer_hash[r][:consumer]
-                  stream = consumer_hash[r][:stream]
+                      # Read from consumer
+                      msg = read_message(r)
+                      consumer = consumer_hash[r][:consumer]
+                      stream = consumer_hash[r][:stream]
 
-                  # Consumer is ready for next message
-                  if msg["command"] && msg["command"] == "next"
+                      # Consumer is ready for next message
+                      if msg["command"] && msg["command"] == "next"
 
-                    @__emit_queues[stream][consumer][:ready] = true
-                    tuple_json = get_consumer_tuple(stream, consumer)
+                        @__emit_queues[stream][consumer][:ready] = true
+                        tuple_json = get_consumer_tuple(stream, consumer)
 
-                    # If all messages have been sent to a consumer, end its cycle
-                    if tuple_json.nil?
-                      write_stream = get_write_stream(stream, consumer)
-                      write_message(write_stream, END_CYCLE_MESSAGE)
-                      consumers_running -= 1
-                      if consumers_running == 0
-                        break
+                        # If all messages have been sent to a consumer, end its cycle
+                        if tuple_json.nil?
+                          write_stream = get_write_stream(stream, consumer)
+                          write_message(write_stream, END_CYCLE_MESSAGE)
+                          consumers_running -= 1
+                          if consumers_running == 0
+                            break
+                          end
+                        else
+                          # Emit tuple to consumer
+                          emit_consumer_tuple(stream, consumer, tuple_json)
+                          emitted = true
+                        end
                       end
-                    else
-                      # Emit tuple to consumer
-                      emit_consumer_tuple(stream, consumer, tuple_json)
-                      emitted = true
+
+                    end
+                     # Exit when done emitting
+                    if !emitted
+                      exit(0)
                     end
                   end
+                  break
 
-                end
-                 # Exit when done emitting
-                if !emitted
-                  return
-                end
+                # Exit after ending all consumer cycles
+                elsif consumers_running == 0
+                  break
+                end                
               end
-              break
-
-            # Exit after ending all consumer cycles
-            elsif consumers_running == 0
-              break
+            ensure
+              ml_socket.close()
             end
-          
           end
+          server_thread.join()
+
         rescue Errno::EIO
            cdisplay("Errno:EIO error")
         ensure
           # cleanup
           if cycle_killed
             send_to_consumers(KILL_CYCLE_MESSAGE, false)
-            send_to_consumers(KILL_CYCLE_MESSAGE, false)
+            send_to_consumees(KILL_CYCLE_MESSAGE)
           end
 
           pid = wait_thread[:pid]  
-          ml_input.close 
-          ml_output.close
-          File.delete("#{ml_pipe}.in")
+          stdin.close
           stdout.close
           stderr.close
           Process.kill('INT', pid)
@@ -1226,7 +1218,9 @@ module Zillabyte; module Runner; class MultilangOperation
       "string" => String,
       "double" => Float,
       "integer" => Integer,
-      "float" => Float
+      "float" => Float,
+      "array" => Array,
+      "map" => Hash
     }
 
     col_map = {}
@@ -1449,6 +1443,18 @@ module Zillabyte; module Runner; class MultilangOperation
   end
 
 
+  # Instruct multilang to run prepare step
+  def self.prepare(write_stream, read_stream)
+    begin
+      write_message write_stream, PREPARE_MESSAGE
+      msg = read_message(read_stream)
+    rescue => e
+      cdisplay("Error running prepare")
+      raise e
+    end
+  end 
+
+
   # Instruct multilang to begin cycle
   def self.begin_cycle(write_stream, read_stream)
     write_message(write_stream, BEGIN_CYCLE_MESSAGE)
@@ -1495,14 +1501,14 @@ module Zillabyte; module Runner; class MultilangOperation
   def self.send_to_consumees(json_obj)
     pipes = @__consumee_pipes
     # Left hand(or singular) input
-    if (pipes.has_key? "wr_parent_1")
-      write_stream = get_write_stream(stream, consumer, 1)
+    if (pipes.has_key? "wr_child_1")
+      write_stream = pipes["wr_child_1"]
       write_message(write_stream, json_obj)
     end
 
     # Right hand input 
-    if (pipes.has_key? "wr_parent_2")
-      write_stream = get_write_stream(stream, consumer, 2)
+    if (pipes.has_key? "wr_child_2")
+      write_stream = pipes["wr_child_2"]
       write_message(write_stream, json_obj)
     end
   end
@@ -1511,7 +1517,6 @@ module Zillabyte; module Runner; class MultilangOperation
   def self.send_to_consumers(json_obj, display = false)
     @__consumer_pipes.each_pair do |stream, consumers|
       consumers.each_pair do |consumer, pipe| 
-        @__consumee_pipes["wr_child_1"]
         # Single or Left hand pipe
         if (pipe.has_key? "wr_parent_1")
           write_stream = get_write_stream(stream, consumer, 1)
@@ -1626,7 +1631,6 @@ module Zillabyte; module Runner; class MultilangOperation
   # Construct a multilang command
   def self.command(arg, ignore_stderr=false)
     cdisplay("could not extract meta information. missing zillabyte.conf.yml?") if @__meta.nil?
-
     full_script = File.join(@__dir, @__meta["script"])
     stderr_opt = "2> /dev/null" if ignore_stderr