zillabyte-cli 0.9.20 → 0.9.21

data/lib/zillabyte/runner/component_runner.rb

@@ -1,337 +0,0 @@
-require "zillabyte/runner/multilang_operation"
-require "zillabyte/runner/component_operation"
-
-# HIDDEN:
-class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base 
-  include Zillabyte::Helpers
-
-  KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
-
-  def run (meta, dir = Dir.pwd, session = nil, options = {})
-
-    if meta.nil? or session.nil?
-      return
-    end
-
-    @session = session
-    @colors = {}
-
-    # Get options
-    input = options[:input]
-    output = options[:output]
-    otype = options[:output_type] 
-
-    # Show the user what we know about their app... 
-    display "inferring your app details..."
-    describe_component(meta)
-
-    # Setup streams
-    @nodes = meta["nodes"]
-    @node_map = {}
-    @nodes.each do |n|
-      @node_map[n["name"]] = n
-    end
-
-    # Index stream consummers and emitters by stream name
-    @arcs = meta["arcs"]
-
-    # Organize component pipes
-    @operations = {}
-    @operation_pipes = {}
-
-
-    # Start component
-    begin
-
-      # Setup operation pipes
-      @nodes.each do |n|
-
-        name = n["name"]
-        type = n["type"]
-        if n["type"] ==  "source"
-          fields = n["fields"]
-        end
-
-        # Create two new pipes in the parent.
-        rd_child_1, wr_parent_1 = IO.pipe()
-        rd_parent_1, wr_child_1 = IO.pipe()
-        
-        @operation_pipes[name] = {
-          "rd_child_1" => rd_child_1,      
-          "wr_child_1" => wr_child_1,
-          "rd_parent_1" => rd_parent_1,        
-          "wr_parent_1" => wr_parent_1
-        }
-
-        # Add a second(right hand side) set ofpipes for joins
-        if type == "join"
-          # Create two new pipes in the parent.
-          rd_child_2, wr_parent_2 = IO.pipe()
-          rd_parent_2, wr_child_2 = IO.pipe()
-          @operation_pipes[name]["rd_child_2"] = rd_child_2
-          @operation_pipes[name]["wr_child_2"] = wr_child_2
-          @operation_pipes[name]["rd_parent_2"] = rd_parent_2
-          @operation_pipes[name]["wr_parent_2"] = wr_parent_2
-        end
-      end
-
-      # Maps origin => {stream => [destinations]}
-      @arc_map = {}
-      @arcs.each do |a|
-        origin = a["origin"]
-        name = a["name"]
-        dest = a["dest"]
-        @arc_map[origin] ||= {}
-        @arc_map[origin][name] ||= []
-        @arc_map[origin][name] << a["dest"]
-      end
-
-
-      # Spawn component threads
-      @nodes.each do |n|
-
-        name = n["name"]
-        type = n["type"]
-        emits = n["emits"]
-
-        pipes = @operation_pipes[name] 
-
-        # Fork.
-        pid = fork()
-        if pid # In parent
-          # Close the reading end of the first child so we can write to the child.
-          pipes["rd_child_1"].close()
-          # Close the writing end of the first child so we can read from the child.
-          pipes["wr_child_1"].close()
-
-          if type == "join"
-            # Close the reading end of the second child so we can write to the child.
-            pipes["rd_child_2"].close()
-            # Close the writing end of the second child so we can read from the child.
-            pipes["wr_child_2"].close()
-          end
-        else # in child
-          # Close the writing end of the first parent so we can read from the parent.
-           pipes["wr_parent_1"].close()
-          # Close the reading end of the first parent so we can write to the parent.
-           pipes["rd_parent_1"].close()
-
-          if type == "join"
-            # Close the reading end of the second child so we can write to the child.
-            pipes["rd_parent_2"].close()
-            # Close the writing end of the second child so we can read from the child.
-            pipes["wr_parent_2"].close()
-          end
-
-          begin
-            # Setup reading and writing pipes for communicating with consumee component
-            if type != "join"
-              in_pipes = {"rd_child_1" => @operation_pipes[name]["rd_child_1"], "wr_child_1" => @operation_pipes[name]["wr_child_1"]}
-            
-            # Add join specific options
-            else
-              options[:join_options] = {}
-              in_pipes = {}
-              @arcs.each do |a|
-
-                if (a["dest"] == name)
-                  # Left Side
-                  if (a["left"] == 1)
-                    options[:join_options][:lhs] = a["origin"]
-                    in_pipes["rd_child_1"] = @operation_pipes[name]["rd_child_1"]
-                    in_pipes["wr_child_1"] = @operation_pipes[name]["wr_child_1"]
-                  # Right Side
-                  elsif (a["right"] == 1)
-                    options[:join_options][:rhs] = a["origin"]
-                    in_pipes["rd_child_2"] = @operation_pipes[name]["rd_child_2"]
-                    in_pipes["wr_child_2"] = @operation_pipes[name]["wr_child_2"]
-                  end
-                end
-              end
-            end
-
-            # Index consumer pipes by stream name, consumer_name
-            out_pipes = {}
-
-            # Check if you are the consumee for a downstream join in order to select the correct pipe
-            if type != "output"
-              @arc_map[name].each_pair do |stream, destinations|
-                out_pipes[stream] ||= {}
-                
-                destinations.each do |dest|
-                  out_pipes[stream][dest] ||= {}
-
-                  # Check for a join at the destination
-                  if (@node_map[dest]["type"] == "join")                      
-                    @arcs.each do |a|
-                      if (a["dest"] == dest && a["origin"] == name)
-                        # Left Side
-                        if (a["left"] == 1)
-                          out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"] 
-                          out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
-                          break
-                        elsif (a["right"] == 1)
-                          out_pipes[stream][dest]["wr_parent_2"] = @operation_pipes[dest]["wr_parent_2"] 
-                          out_pipes[stream][dest]["rd_parent_2"] = @operation_pipes[dest]["rd_parent_2"]
-                          break
-                        end
-                      end
-                    end
-                  else
-                    out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"] 
-                    out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
-                  end
-                end
-              end
-            end
-
-            # Run the child process 
-            Zillabyte::Runner::ComponentOperation.run(n, dir, in_pipes, out_pipes, self, meta, options)
-
-          rescue => e
-            display e.message
-            display e.backtrace
-          ensure
-            # Close the reading end of the child
-            pipes["rd_child_1"].close()
-            # Close the writing end of the child
-            pipes["wr_child_1"].close()
-
-            # Close secondary join child
-            pipes["rd_child_2"].close() if pipes["rd_child_2"]
-            pipes["wr_child_2"].close() if pipes["wr_child_2"]
-
-            exit!(-1)
-          end #end child
-        end
-      end 
-
-
-      # If no input file, read from STDIN
-      # TODO handle inputs
-      if input.nil?
-
-        source = ""
-        @nodes.each do |n|
-          name = n["name"]
-          type = n["type"]
-
-          if type == "input"
-            if source == ""
-              source = name
-            else
-              display "Error: Cannot run component with multiple input sources without input files"
-              return
-            end
-          end
-        end
-          
-        display ""
-        display "To view results:  Enter 'end'    "
-        display ""
-
-        while true
-
-          fields = @node_map[source]['fields'].map {|h| h.keys[0].upcase }
-          display "Enter an input tuple in the form : #{fields.join(' ')}"
-          msg = ask
-
-          # Kill the cycle
-          if msg == 'end'
-            @operation_pipes[source]["wr_parent_1"].puts KILL_CYCLE_MESSAGE
-            break
-            
-          # Check arguments
-          else
-            args = msg.scan(/(?:\w|"[^"]*")+/)
-            if (args.length % fields.length != 0)
-              display "Error: Argument length must be a multiple of the schema length"
-              next
-            end
-          end
-          # Send tuple to source
-          @operation_pipes[source]["wr_parent_1"].puts msg
-         
-        end
-      end
-
-    rescue => e
-      display e.message
-      display e.backtrace
-   
-    ensure
-      Process.waitall()
-      @operation_pipes.each do |name, pipes|
-        #Close the  writing end of the parent
-        pipes["wr_parent_1"].close()
-        # Close the reading end of the parent
-        pipes["rd_parent_1"].close()
-
-        # Close secondary join parent
-        pipes["wr_parent_2"].close() if pipes["wr_parent_2"]
-        pipes["rd_parent_2"].close() if pipes["rd_parent_2"]
-      end
-    end
-  end
-
-  def session
-    @session
-  end
-
-
-  def cdisplay(name, message, useName=true)
-    color = @colors[name] || :default
-    if message.nil? || message == ""
-      return
-    else
-      
-      if message.is_a?(Array)
-        lines = message
-      else
-        lines = message.split("\n")
-      end
-
-      prefix = useName ? "#{name} - " : ""
-      display "#{prefix}#{lines.first}".colorize(color)
-      lines[1..-1].each do |line|
-        display "#{' '*prefix.size}#{line}".colorize(color)
-      end
-    end
-  end
-
-  def display(message, newline = true)
-    @session.display(message, newline)
-  end
-
-  def describe_component(meta)
-    require("colorize")
-    require("indentation")
-    colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
-    rjust = 20
-
-    display "#{'component name'.rjust(rjust)}: #{meta['name']}"
-    display "#{'component language'.rjust(rjust)}: #{meta['language']}"
-    meta['nodes'].each_with_index do |node, index|
-      color = @colors[node['name']] ||= colors[index % colors.length]
-      display (("="*rjust + "  operation ##{index}").colorize(color))
-      display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
-
-      # Convert metadata typing to that of components
-      if node['type'] == "input"
-        type = "input"
-        display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
-        display "#{"fields".rjust(rjust)}: #{node['fields'].to_s.colorize(color)}"
-        display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
-      elsif node['type'] == "output"
-        type = "output"
-        display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
-        display "#{"columns".rjust(rjust)}: #{node['columns'].to_s.colorize(color)}"
-
-      else
-        type = node['type']
-        display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
-      end
-    end
-  end
-
-end

data/lib/zillabyte/runner/multilang_operation.rb

@@ -1,1662 +0,0 @@
-
-# Emulate a multilang operation
-module Zillabyte; module Runner; class MultilangOperation
-
-  HANDSHAKE_MESSAGE = "{\"pidDir\": \"/tmp\"}\n"
-  PREPARE_MESSAGE = " {\"command\": \"prepare\"}\n"
-  DONE_MESSAGE = "{\"command\": \"done\"}\n"
-  NEXT_MESSAGE = "{\"command\": \"next\"}\n"
-  BEGIN_CYCLE_MESSAGE = "{\"command\": \"begin_cycle\"}\n"
-  END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
-  KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
-  PONG_PREFIX = "{\"pong\": \""
-  PONG_SUFFIX = "\"}\n"  
-  ENDMARKER = "\nend\n"
-
-  def self.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options = {})
-    require("mkfifo")
-    require("zillabyte/runner/component_operation")
-    require("zillabyte/runner/operation")
-
-    require("pty")
-    require("open3")
-
-    @__node = node
-    @__name = node["name"]
-    @__tester = tester
-
-    @__type = node["type"]
-    @__dir = dir
-
-    @__consumee_pipes = consumee_pipes
-    @__consumer_pipes = consumer_pipes
-    @__meta = meta
-    @__options = options
-    @__output_type = options[:output_type]
-    # Each consumer of a stream gets its own queue and message passing
-    @__emit_queues = {}
-    @__consumer_pipes.each_pair do |stream, consumers|
-      consumers.each_key do |consumer|
-        @__emit_queues[stream] ||= {}
-        @__emit_queues[stream][consumer] = {:write_queue => [], :ready => true}
-      end
-    end
-
-    begin
-      case @__type
-      when "source"
-        self.run_source()
-      when "group_by"
-        self.run_group_by()
-      when "join"
-        self.run_join()
-      when "each"
-        self.run_each()
-      when "filter"
-        self.run_filter()
-      when "component"
-        Zillabyte::Runner::ComponentOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options = {})
-      when "sink"
-        self.run_sink()
-      else
-        cdisplay("invalid operation type #{@__type}")
-      end
-    rescue => e
-      cdisplay e.message
-    end
-
-  end
-
-
-  def self.run_source()
-
-    end_cycle_policy = @__node["end_cycle_policy"]
-
-    # Interactive source
-    if @__options[:interactive]
-      loop do 
-
-        msg = @__consumee_pipes["rd_child_1"].gets
-
-        if msg == KILL_CYCLE_MESSAGE            
-          send_to_consumers(KILL_CYCLE_MESSAGE)
-          return
-        else
-          # Build tuple
-          begin
-            obj = JSON.parse(msg)
-          rescue JSON::ParserError 
-            cdisplay "Error: invalid JSON"           
-            next
-          end
-                  
-          tuple_json = build_tuple_json(obj)
-          display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
-          send_to_consumers(tuple_json)   
-          
-        end
-      end
-
-    # Source from relation
-    elsif @__node['matches'] or @__node["relation"]
-
-      # Query API for rows  
-      matches = @__node['matches'] || (@__node["relation"]["query"])
-      cdisplay("Fetching remote data...")
-      res = @__tester.query_agnostic(matches)
-      rows = res["rows"]
-      if(rows.nil? or rows.length == 0)
-        cdisplay("Could not find data that matches your 'matches' clause")
-        exit(-1)
-      end
-      # Enqueue rows for sending to consumers
-      column_aliases = res['column_aliases']
-      rows.each do |tuple|
-        tuple_json = build_tuple_json(tuple, nil, column_aliases)
-        @__emit_queues.each_pair do |stream, consumers|
-          consumers.each_pair do |consumer, emitter|
-            emitter[:write_queue] << tuple_json
-          end
-        end
-      end
-
-      # Index streams and consumers by their pipes for lookup
-      consumer_hash = build_consumer_hash()
-
-
-      # Send first tuple
-      @__emit_queues.each_pair do |stream, consumers|
-        consumers.each_key do |consumer|
-          tuple_json = get_consumer_tuple(stream, consumer)
-          emit_consumer_tuple(stream, consumer, tuple_json)
-        end
-      end
-
-      # Sent tuples to consumers as appropriate
-      loop do 
-
-        # Retrieve messages from consumers
-        rs, ws, es = IO.select(consumer_hash.keys, [], [])
-
-        # Emit tuples to consumers
-        emitted = false
-        rs.each do |r|
-
-          # Read from consumer
-          msg = read_message(r)
-
-          stream = consumer_hash[r][:stream]
-          consumer = consumer_hash[r][:consumer]
-
-          # Consumer is ready for next message
-          if msg["command"] 
-
-            case msg["command"]
-            when "next"
-
-              @__emit_queues[stream][consumer][:ready] = true
-              tuple_json = get_consumer_tuple(stream, consumer)
-
-              # If all messages have been sent to consumer, end their cycle
-              if tuple_json.nil?
-                write_stream = get_write_stream(stream, consumer)
-                cdisplay "ending cycle for #{consumer}"
-                send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                send_command_tuple(stream, consumer, DONE_MESSAGE)
-              else
-                # Emit tuple to consumer
-                emit_consumer_tuple(stream, consumer, tuple_json)
-                emitted = true
-              end
-            when "kill_cycle"
-              send_to_consumers(KILL_CYCLE_MESSAGE, false)
-              return
-            end
-          end
-        end
-
-        # Exit when done emitting
-        if !emitted
-          return
-        end
-      end
-      
-    # Custom source
-    else
-
-      # Index streams and consumers by their pipes for lookup
-      consumer_hash = build_consumer_hash()
-
-      # Keep track of how many consumers to handle before exiting
-      consumers_running = consumer_hash.keys.length
-
-      # Kill the cycle on error
-      cycle_killed = false
-
-      # Setup multilang socket
-      require 'socket'
-      host = "0.0.0.0"
-      server = TCPServer.new(0)
-      port = server.addr[1]
-    
-      # Spawn multilang process
-      cmd = command("--execute_live --name #{@__name}  --host \"#{host}\" --port #{port}")
-
-      begin
-
-        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
-          begin
-            server_thread = Thread.new do 
-              ml_socket = server.accept()
-
-              # RUN SOURCE
-              begin
-                # Setup streams from consumers and multilang(stdout and socket communication)
-                read_streams = consumer_hash.keys.concat [stdout, ml_socket]
-
-                # Handshake
-                handshake(ml_socket, ml_socket)
-                prepare(ml_socket, ml_socket)
-
-                # Begin cycle
-                begin_cycle(ml_socket, ml_socket)
-                emitted = false
-                write_message(ml_socket, NEXT_MESSAGE)
-                multilang_queue = []
-                end_cycle_policy = @__options[:end_cycle_policy]
-                end_cycle_received = false
-
-                # Receive and handle messages
-                loop do
-                  # Read from a stream
-                  rs = select_read_streams(read_streams)
-                  rs.each do |r|
-                    # Read stdout straight to user
-                    if r == stdout && consumers_running > 0
-                      msg = r.gets
-                      msg = msg.sub(/\n/, "")
-                      cdisplay("log: #{msg}")
-                      next
-                    end
-
-                    obj = read_message(r)
-
-                    if obj.nil?
-                      next
-                    end
-
-                    if obj["command"] 
-                      case obj["command"]
-
-                      # Multilang emitted a tuple
-                      when "emit"
-                        stream = obj['stream']
-                        # Check for null emit
-                        if end_cycle_policy != "explicit"
-
-                          if obj['tuple'].nil?
-                            end_cycle_received = true
-                          else
-                            nil_values = false
-                            obj['tuple'].each_value do |v|
-                              if v.nil?
-                                nil_values = true
-                                break       
-                              end                     
-                            end
-                            end_cycle_received = nil_values
-                            next unless !end_cycle_received
-                          end
-                        end
-
-                        # Valid emit
-                        emitted = true
-
-                        # Send or enqueue tuple for each consumer
-                        tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
-
-                        @__emit_queues[stream].each_pair do |consumer, emitter|
-                          if emitter[:ready]
-                            emit_consumer_tuple(stream, consumer, tuple_json)
-
-                          else
-                            @__emit_queues[stream][consumer][:write_queue] << tuple_json
-                          end
-                        end
-
-                      # Consumer is ready for a message
-                      when "next"
-                        stream = consumer_hash[r][:stream]
-                        consumer = consumer_hash[r][:consumer]
-                        @__emit_queues[stream][consumer][:ready] = true
-                        tuple_json = get_consumer_tuple(stream, consumer)
-
-                        # End cycle for consumer if it has processed all tuples
-                        if tuple_json.nil? && end_cycle_received
-
-                          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                          consumers_running -= 1
-                          if consumers_running == 0
-                            exit(0)
-                          end
-
-                          # TODO break if last consumer
-                        elsif !tuple_json.nil?
-                          # Emit tuple to consumer
-                          emit_consumer_tuple(stream, consumer, tuple_json)
-                          emitted = true
-                        end
-
-                      # Multilang is done emitting a group of tuples
-                      when "done"
-                        # End cycle if no tuples were emitted
-                        if !emitted && end_cycle_policy == "null_emit"
-                          end_cycle_received = true
-                        else
-                          emitted = false
-                        end
-                      
-                        # Send the next tuple to multilang
-                        if !multilang_queue.empty?
-                          write_message(ml_socket, multilang_queue.shift)
-
-                        # Request next tuple from mutilang
-                        elsif !end_cycle_received
-                          write_message(ml_socket, NEXT_MESSAGE)
-
-                        # If there are no more messages to send, we are done
-                        else end_cycle_received
-                          finished = true
-                           # End cycle for ready consumers
-                          @__emit_queues.each_pair do |stream, consumers|
-                            consumers.each_pair do |consumer, emitter|
-                              if emitter[:ready]
-                                send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                                consumers_running -= 1
-                                if consumers_running == 0
-                                  exit(0)
-                                end
-                              end
-                            end
-
-                          end
-                        end
-
-                      # Multilang sent an error message
-                      when "fail"
-                        cdisplay("ERROR : #{obj['msg']}")
-                        cycle_killed = true
-                        exit(0)
-                      
-                      # Multilang sent a log message
-                      when "log"
-                        cdisplay "LOG: #{obj['msg']}"
-
-                      # Multilang sent signal to end the cycle
-                      when "end_cycle"
-                        if end_cycle_policy != "explicit"
-                          cdisplay "received end_cycle command for non explicit policy"
-                          next
-                        end
-                        end_cycle_received = true
-                      when "kill_cycle"
-                        cycle_killed = true
-                        exit(0)
-                      end
-                  
-                    # Multilang sent a ping
-                    elsif obj['ping']
-                      write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
-                    end
-                  end
-                end
-
-                # Exit after ending consumer cycles
-                if consumers_running == 0
-                  exit(0)
-                end
-              rescue => e
-                cycle_killed = true
-                cdisplay e.message
-                cdisplay e.backtrace
-              ensure
-                ml_socket.close()
-              end
-            end
-            server_thread.join()
-          rescue => e
-          ensure
-            # cleanup
-            if cycle_killed
-              send_to_consumers(KILL_CYCLE_MESSAGE, false)
-            end
-            pid = wait_thread[:pid] 
-            stdout.close
-            Process.kill('INT', pid)
-            Process.exit!(true)
-          end
-        end
-
-      rescue PTY::ChildExited
-      end
-    end
-  end
-
-
-  def self.run_each()
-
-    # Index streams and consumers by their pipes for lookup
-    consumer_hash = build_consumer_hash()
-
-    # Keep track of how many consumers to handle before exiting
-    consumers_running = consumer_hash.keys.length
-
-    # Kill the cycle on error
-    cycle_killed = false
-
-    # Setup multilang socket
-    require 'socket'
-    host = "0.0.0.0"
-    server = TCPServer.new(0)
-    port = server.addr[1]
-  
-    # Spawn multilang process
-    cmd = command("--execute_live --name #{@__name}  --host \"#{host}\" --port #{port}")
-    
-    begin
-      # Start the operation... 
-      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
-        begin
-          server_thread = Thread.new do 
-            ml_socket = server.accept()
-            begin
-              # Setup streams from consumers, multilang, and the consumee
-              read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"], ml_socket, stdout]
-
-              # Handshake
-              handshake(ml_socket, ml_socket)
-              prepare(ml_socket, ml_socket)
-
-              # Begin cycle
-              multilang_queue = []
-              mutlilang_count = 0
-              end_cycle_received = false
-              column_aliases = nil
-
-
-              # Receive and handle messages
-              loop do
-
-                # Read from a stream
-                rs = select_read_streams(read_streams)
-                rs.each do |r|
-
-                  # Read STDOUT from program straight to user
-                  if r == stdout
-                    msg = r.gets
-                    if !msg.nil?
-                      msg = msg.sub(/\n/, "")
-                      cdisplay("LOG: #{msg}")
-                    end
-                    next
-                  end
-
-                  # Receive an object
-                  obj = read_message(r)
-
-                  if obj["command"] 
-                    case obj["command"]
-
-                    # Multilang emitted a tuple
-                    when "emit"
-
-                      stream = obj["stream"]
-                     
-                      # Send or enqueue tuple for each consumer
-                      tuple_json = build_tuple_json(obj['tuple'], obj['meta'], column_aliases)
-
-                      @__emit_queues[stream].each_pair do |consumer, emitter|
-                        if emitter[:ready]
-                          emit_consumer_tuple(stream, consumer, tuple_json)
-                        else
-                          @__emit_queues[stream][consumer][:write_queue] << tuple_json
-                        end
-                      end
-
-                    # Consumer is ready for a message
-                    when "next"
-                      stream = consumer_hash[r][:stream]
-                      consumer = consumer_hash[r][:consumer]
-
-
-                      @__emit_queues[stream][consumer][:ready] = true
-                      tuple_json = get_consumer_tuple(stream, consumer)
-
-                      # End cycle for consumer if it has processed all tuples
-                      if tuple_json.nil? && end_cycle_received
-                        send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                        consumers_running -= 1
-                        if consumers_running == 0
-                          break
-                        end
-
-
-                        # TODO break if last consumer
-                      elsif !tuple_json.nil?
-                        # Emit tuple to consumer
-                        emit_consumer_tuple(stream, consumer, tuple_json)
-                        emitted = true
-                      end
-
-                    # Multilang is done emitting a group of tuples
-                    when "done" 
-                      mutlilang_count -= 1                  
-
-                      # Send the next tuple to multilang
-                      if !multilang_queue.empty?
-                        write_message(ml_socket, multilang_queue.shift)
-
-                      # If there are no more messages to send, we are done
-                      elsif end_cycle_received && mutlilang_count == 0
-                        finished = true
-
-                       # End cycle for ready consumers
-                        @__emit_queues.each_pair do |stream, consumers|
-                          consumers.each_pair do |consumer, emitter|
-                            if emitter[:ready]
-                              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                              consumers_running -= 1
-                              if consumers_running == 0
-                                break
-                              end
-                            end
-                          end
-                        end
-                      end
-
-                    # Multilang sent an error message
-                    when "fail"
-                      cdisplay("ERROR : #{obj['msg']}")
-                      cycle_killed = true
-                      exit(0)
-                    
-                    # Multilang sent a log message
-                    when "log"
-                      cdisplay "LOG: #{obj['msg']}"
-
-                    # Consumee operation sent signal to end_cycle
-                    when "end_cycle"
-                      end_cycle_received = true
-                      if mutlilang_count == 0
-                        @__emit_queues.each_pair do |stream, consumers|
-                          consumers.each_pair do |consumer, emitter|                      
-                            if emitter[:ready]
-                              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                              consumers_running -= 1
-                              if consumers_running == 0
-                                exit(0)
-                              end
-                            end
-                          end
-                        end
-                      end
-           
-                    when "kill_cycle"
-                      cycle_killed = true
-                      exit(0)
-                    end
-
-                  # Received a tuple from consumee
-                  elsif obj['tuple']
-                    column_aliases = obj['column_aliases']
-                    # Send or enqueue to multilang
-                    mutlilang_count += 1
-                    if multilang_queue.empty?
-                      write_message(ml_socket, obj.to_json)
-                    else
-                      multilang_queue << obj.to_json
-                    end
-                    write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
-                  
-                  # Multilang sent a ping
-                  elsif obj['ping']
-                    write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
-                  end
-                end
-
-                # Exit after ending consumer cycles
-                if consumers_running == 0
-                  exit(0)
-                end
-
-              end
-            ensure
-              ml_socket.close()
-            end
-          end
-          server_thread.join()
-        rescue => e
-           cdisplay e.message
-           cdisplay e.backtrace
-        ensure
-          # cleanup
-          if cycle_killed
-            send_to_consumers(KILL_CYCLE_MESSAGE, false)
-            send_to_consumees(KILL_CYCLE_MESSAGE)
-          end
-          pid = wait_thread[:pid] 
-          stdin.close
-          stdout.close
-          stderr.close
-        end
-      end
-    rescue PTY::ChildExited
-      cdisplay("The child process exited!")
-    end
-
-  end
-
-
-  def self.run_group_by()
-
-    # Index streams and consumers by their pipes for lookup
-    consumer_hash = build_consumer_hash
-
-    # Keep track of how many consumers to handle before exiting
-    consumers_running = consumer_hash.keys.length
-
-    # Kill the cycle on error
-    cycle_killed = false
-
-    # Setup groups
-    group_by = @__node['group_by']
-    group_tuples = {}
-    emit_count = 0    # used to track how many emits are expected once groups are sent to multilang
-    emitted_tuples = [] # used to send to consumers after once groupings are emitted
-    tuple_queue = []
-
-    # Setup multilang pipe
-    require 'socket'
-    host = "0.0.0.0"
-    server = TCPServer.new(0)
-    port = server.addr[1]
-
-    # Spawn multilang process
-    cmd = command("--execute_live --name #{@__name}  --host \"#{host}\" --port #{port}")
-
-    begin
-      # Start the operation... 
-      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
-        begin
-          server_thread = Thread.new do 
-            ml_socket = server.accept()
-            begin
-              # Setup streams from consumers, multilang, and the consumee
-              read_streams = consumer_hash.keys.concat [stdout, ml_socket, @__consumee_pipes["rd_child_1"]]
-
-              # Handshake
-              handshake(ml_socket, ml_socket)
-              prepare(ml_socket, ml_socket)
-
-              # Begin cycle
-              end_cycle_received = false
-              finished_emitting = false
-
-               # Handle streams
-              loop do
-                # Read from a stream
-                rs = select_read_streams(read_streams)
-                rs.each do |r|
-
-                  # Read STDOUT from program straight to user
-                  if r == stdout
-                    msg = r.gets
-                    msg = msg.sub(/\n/, "")
-                    cdisplay("log: #{msg}")
-                    next
-                  end
-
-
-                  # Receive an object
-                  obj = read_message(r)
-    
-                  if obj["command"] 
-                    case obj["command"]
-
-                    when "done"
-
-                      if end_cycle_received
-                        tuple_json = tuple_queue.shift
-                        if !tuple_json.nil?
-                          write_message(ml_socket, tuple_json)
-                        end
-                      end
-
-                      next
-
-                    # Begin aggregation
-                    when "end_cycle"
-                      end_cycle_received = true
-                      read_streams = [ml_socket]
-
-                      group_tuples.each do |group_tuple, tuples|
-                        tuple_queue << "{\"command\": \"begin_group\", \"tuple\": #{group_tuple.to_json}, \"meta\":{}}\n"
-                        tuples.each do |t|
-                          tuple_queue << "{\"command\": \"aggregate\", #{t}}\n"
-                        end
-                        tuple_queue << "{\"command\": \"end_group\"}\n"
-
-                        # keep track of how many emits are expected
-                        emit_count += 1
-                      end
-
-                      tuple_json = tuple_queue.shift
-                      if !tuple_json.nil?
-                        write_message(ml_socket, tuple_json)
-                      end
-
-                    # Multilang has emitted a grouped tuple
-                    when "emit"
-                      stream = obj['stream']
-                      emit_count -= 1
-                      # Enqueue for consumers
-                      tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
-                      @__emit_queues.each_pair do |stream, consumers|
-                        consumers.each_key do |consumer|
-                          @__emit_queues[stream][consumer][:write_queue] << tuple_json  
-                        end
-                      end
-
-                      # End cycle when done emitting
-                      if end_cycle_received && emit_count == 0
-                        finished_emitting = true
-                        break      
-                      elsif end_cycle_received
-                        tuple_json = tuple_queue.shift
-                        if !tuple_json.nil?
-                          write_message(ml_socket, tuple_json)
-                        end
-                      end
-                  
-                    # An error has occured
-                    when "kill_cycle"
-                      cycle_killed = true
-                      exit(0)
-                    end
-
-                  # Received a tuple from operation
-                  elsif obj["tuple"]
-                    tuple = obj["tuple"].to_json
-                    meta = obj["meta"].to_json
-                    column_aliases = obj["column_aliases"] || {}
-                    aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
-                    gt = {}
-
-                    # Get the column names to group on
-                    group_by.each do |field|
-                      field_name = aliases[field] || field
-                      gt[field] = obj["tuple"][field_name]
-                    end
-
-                    msg_no_brackets = "\"tuple\": #{tuple}, \"meta\": #{meta}, \"column_aliases\": #{column_aliases.to_json}"
-
-                    # Group tuple into existing group or create new group
-                    if group_tuples[gt]
-                      group_tuples[gt] << msg_no_brackets
-                    else
-                      group_tuples[gt] = [msg_no_brackets]
-                    end
-
-                    # Ask operation for next tuple
-                    write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
-
-                  # Multilang sent a ping
-                  elsif obj['ping']
-                    write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
-                  end
-                end
-
-                # Send tuples to consumers
-                if finished_emitting && consumers_running > 0
-
-                  # Send first tuple
-                  @__emit_queues.each_pair do |stream, consumers|
-                    consumers.each_key do |consumer|
-                      tuple_json = get_consumer_tuple(stream, consumer)
-                      emit_consumer_tuple(stream, consumer, tuple_json)
-                    end
-                  end
-
-                  # Sent tuples to consumers as appropriate
-                  loop do 
-
-                    # Retrieve messages from consumers
-                    rs, ws, es = IO.select(consumer_hash.keys, [], [])
-
-                    # Emit tuples to consumers
-                    emitted = false
-                    rs.each do |r|
-
-                      # Read from consumer
-                      msg = read_message(r)
-                      consumer = consumer_hash[r][:consumer]
-                      stream = consumer_hash[r][:stream]
-
-                      # Consumer is ready for next message
-                      if msg["command"] && msg["command"] == "next"
-
-                        @__emit_queues[stream][consumer][:ready] = true
-                        tuple_json = get_consumer_tuple(stream, consumer)
-
-                        # If all messages have been sent to a consumer, end its cycle
-                        if tuple_json.nil?
-                          write_stream = get_write_stream(stream, consumer)
-                          write_message(write_stream, END_CYCLE_MESSAGE)
-                          consumers_running -= 1
-                          if consumers_running == 0
-                            break
-                          end
-                        else
-                          # Emit tuple to consumer
-                          emit_consumer_tuple(stream, consumer, tuple_json)
-                          emitted = true
-                        end
-                      end
-
-                    end
-                     # Exit when done emitting
-                    if !emitted
-                      exit(0)
-                    end
-                  end
-                  break
-
-                # Exit after ending all consumer cycles
-                elsif consumers_running == 0
-                  break
-                end                
-              end
-            ensure
-              ml_socket.close()
-            end
-          end
-          server_thread.join()
-
-        rescue Errno::EIO
-           cdisplay("Errno:EIO error")
-        ensure
-          # cleanup
-          if cycle_killed
-            send_to_consumers(KILL_CYCLE_MESSAGE, false)
-            send_to_consumees(KILL_CYCLE_MESSAGE)
-          end
-
-          pid = wait_thread[:pid]  
-          stdin.close
-          stdout.close
-          stderr.close
-          Process.kill('INT', pid)
-          Process.exit!(true)
-        end  
-      end
-    rescue PTY::ChildExited
-      if File.exists?("#{ml_pipe}.in")
-        File.delete("#{ml_pipe}.in")
-      end
-      cdisplay("The child process exited!")
-    end
-  end
-
-
-  def self.run_join()
-
-    lhs_fields = @__node["lhs_fields"]
-    rhs_fields = @__node["rhs_fields"]
-    join_type = @__node["join_type"]
-
-
-    # Sanity check
-    if lhs_fields.nil? || rhs_fields.nil? || join_type.nil?
-      return
-    end
-
-
-    # Index the consumee streams for left and right sides
-    consumer_hash = build_consumer_hash()
-
-    # Kill the cycle on error
-    cycle_killed = false
-
-    # read all tuples from lefthand and right hand streams
-    read_streams = [@__consumee_pipes["rd_child_1"], @__consumee_pipes["rd_child_2"]]
-
-     # Index left and right hand consumees by their streams for lookup
-    consumee_hash = {}
-    # Left side
-    lhs = @__options[:join_options][:lhs]
-    consumee_hash[@__consumee_pipes["rd_child_1"]] = lhs
-    # Right side
-    rhs = @__options[:join_options][:rhs]
-    consumee_hash[@__consumee_pipes["rd_child_2"]] = rhs
-  
-    # Keep track of how many consumers to handle before exiting
-    consumers_running = 1
-
-    # Index the incoming tuples by their join key
-    lhs_tuples = {}
-    rhs_tuples = {}
-
-    tuple_queue = []
-
-    # Begin cycle
-    left_end_cycle_received = false
-    right_end_cycle_received = false
-    
-    # Receive and handle messages
-    loop do
-
-      # Read from a stream
-      rs = select_read_streams(read_streams)
-      rs.each do |r|
-
-        # Receive an object
-        obj = read_message(r)
-      
-        if obj["command"] 
-          case obj["command"]
-
-          # A consumee is done emitting
-          when "end_cycle"
-
-            if consumee_hash[r] == lhs
-              left_end_cycle_received = true
-            elsif consumee_hash[r] == rhs
-              right_end_cycle_received = true
-            end
-
-            # We are done receiving from streams
-            if left_end_cycle_received && right_end_cycle_received
-              break
-            end
-         
-          when "kill_cycle"
-            cycle_killed = true
-            return
-          end
-
-         # Received a tuple from consumee
-        elsif obj['tuple']
-          tuple = obj['tuple']
-          if consumee_hash[r] == lhs
-            lhs_tuples[tuple[lhs_fields]] ||= []
-            lhs_tuples[tuple[lhs_fields]] << tuple
-          elsif consumee_hash[r] == rhs
-            rhs_tuples[tuple[rhs_fields]] ||= []
-            rhs_tuples[tuple[rhs_fields]] << tuple
-          end
-        end
-
-        #Ask operation for next tuple
-        if consumee_hash[r] == lhs
-          write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
-        elsif consumee_hash[r] == rhs
-          write_message(@__consumee_pipes["wr_child_2"], NEXT_MESSAGE)
-        end  
-      end
-
-      # Break out if emits ended
-      if left_end_cycle_received && right_end_cycle_received
-        break
-      end
-    end
-    # Build the joined tuples
-    joined_tuples = []
-
-    # If no joined tuples, end the cycle
-    if lhs_tuples.empty? && rhs_fields.empty?
-        @__emit_queues.each_pair do |stream, consumers|
-        consumers.each_pair do |consumer, emitter|                      
-          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-        end
-      end
-      return
-    end
-    joined_fields = (lhs_tuples.values[0][0].keys.concat rhs_tuples.values[0][0].keys).uniq
-    case join_type
-    when "inner"
-      lhs_tuples.each_pair do |key, lhs_tuples|
-        lhs_tuples.each do |lhs_tuple|
-
-          tuple = {}
-          if !rhs_tuples[key].nil? 
-            rhs_tuples[key].each do |rhs_tuple|
-              # Check for a valid join
-              if !lhs_tuple[lhs_fields].nil? && !rhs_tuple[rhs_fields].nil?
-                # Add the fields
-                joined_fields.each do |field|
-                  if lhs_tuple.has_key? field
-                    tuple[field] = lhs_tuple[field]
-                  else
-                    tuple[field] = rhs_tuple[field]
-                  end
-                end
-
-                # Add the tuple
-                joined_tuples << tuple
-              end
-            end
-          end
-        end
-      end
-    when "left"
-      lhs_tuples.each_pair do |key, lhs_tuples|
-        lhs_tuples.each do |lhs_tuple|
-
-          joined = false
-          if rhs_tuples.has_key? key
-            rhs_tuples[key].each do |rhs_tuple|
-              # Check for a valid join
-              if !lhs_tuple[lhs_fields].nil? && !rhs_tuple[rhs_fields].nil?
-                tuple = {}
-
-                # Add the fields
-                joined_fields.each do |field|
-                  if lhs_tuple.has_key? field
-                    tuple[field] = lhs_tuple[field]
-                  else
-                    tuple[field] = rhs_tuple[field]
-                  end
-                end
-                joined_tuples << tuple
-                joined = true
-              end
-            end
-          end
-          if !joined
-            tuple = {}
-            lhs_tuples.each.each do |hash|
-              hash.each do |key, value|
-                tuple[key] = value
-              end
-            end
-
-            joined_fields.each do |field|
-              if !tuple.has_key? field
-                tuple[field] = nil
-              end
-            end
-            # Add the tuple
-            joined_tuples << tuple
-          end
-        end
-      end
-    when "right"
-      rhs_tuples.each_pair do |key, rhs_tuples|
-        rhs_tuples.each do |rhs_tuple|
-
-          joined = false
-          if lhs_tuples.has_key? key
-            lhs_tuples[key].each do |lhs_tuple|
-              # Check for a valid join
-              if !rhs_tuple[lhs_fields].nil? && !lhs_tuple[rhs_fields].nil?
-                tuple = {}
-
-                # Add the fields
-                joined_fields.each do |field|
-                  if rhs_tuple.has_key? field
-                    tuple[field] = rhs_tuple[field]
-                  else
-                    tuple[field] = lhs_tuple[field]
-                  end
-                end
-                joined_tuples << tuple
-                joined = true
-              end
-            end
-          end
-          if !joined
-            tuple = {}
-            rhs_tuples.each.each do |hash|
-              hash.each do |key, value|
-                tuple[key] = value
-              end
-            end
-
-            joined_fields.each do |field|
-              if !tuple.has_key? field
-                tuple[field] = nil
-              end
-            end
-            # Add the tuple
-            joined_tuples << tuple
-          end
-        end
-      end
-    when "outer"
-      lhs_tuples.each_pair do |key, lhs_tuples|
-        lhs_tuples.each do |lhs_tuple|
-
-          joined = false
-          if rhs_tuples.has_key? key
-            rhs_tuples[key].each do |rhs_tuple|
-              # Check for a valid join
-              if !lhs_tuple[lhs_fields].nil? && !rhs_tuple[rhs_fields].nil?
-                tuple = {}
-
-                # Add the fields
-                joined_fields.each do |field|
-                  if lhs_tuple.has_key? field
-                    tuple[field] = lhs_tuple[field]
-                  else
-                    tuple[field] = rhs_tuple[field]
-                  end
-                end
-                joined_tuples << tuple
-                joined = true
-              end
-            end
-          end
-          if !joined
-            tuple = {}
-            lhs_tuples.each.each do |hash|
-              hash.each do |key, value|
-                tuple[key] = value
-              end
-            end
-
-            joined_fields.each do |field|
-              if !tuple.has_key? field
-                tuple[field] = nil
-              end
-            end
-            # Add the tuple
-            joined_tuples << tuple
-          end
-        end
-      end
-    end
-
-    # Setup output queues
-    joined_tuples.each do |tuple|
-      tuple_json = build_tuple_json(tuple)
-      @__emit_queues.each_pair do |stream, consumers|
-        consumers.each_key do |consumer|
-          @__emit_queues[stream][consumer][:write_queue] << tuple_json  
-        end
-      end
-    end
-
-    # Send first tuple
-    @__emit_queues.each_pair do |stream, consumers|
-      consumers.each_key do |consumer|
-
-        tuple_json = get_consumer_tuple(stream, consumer)
-        if tuple_json.nil?
-          send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-          consumers_running -= 1
-          if consumers_running == 0
-            return
-          end
-        else
-          emit_consumer_tuple(stream, consumer, tuple_json)
-        end
-      end
-    end
-
-    # Sent tuples to consumers as appropriate
-    loop do 
-
-      # Retrieve messages from consumers
-      rs, ws, es = IO.select(consumer_hash.keys, [], [])
-
-      # Emit tuples to consumers
-      rs.each do |r|
-
-        # Read from consumer
-        msg = read_message(r)
-        consumer = consumer_hash[r][:consumer]
-        stream = consumer_hash[r][:stream]
-
-        # Consumer is ready for next message
-        if msg["command"] 
-          case msg["command"] 
-          when"next"
-
-            @__emit_queues[stream][consumer][:ready] = true
-            tuple_json = get_consumer_tuple(stream, consumer)
-
-            # If all messages have been sent to a consumer, end its cycle
-            if tuple_json.nil?
-              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-              consumers_running -= 1
-              if consumers_running == 0
-                return
-              end
-            else
-              # Emit tuple to consumer
-              emit_consumer_tuple(stream, consumer, tuple_json)
-            end
-          when "kill_cycle"
-            cycle_killed = true
-            return
-          end
-        end
-      end
-    end
-  end
-
-  def self.run_filter()
-    self.run_each()
-  end
-
-
-  # Send a message to all consumers of the operation
-  def self.run_sink()
-    columns = @__node["columns"]
-
-    type_map = {
-      "string" => String,
-      "double" => Float,
-      "integer" => Integer,
-      "float" => Float,
-      "array" => Array,
-      "map" => Hash
-    }
-
-    col_map = {}
-    columns.each do |hash|
-      key = hash.keys[0]
-      type = hash[key]
-      col_map[key] = type_map[type]
-    end
-
-    tuples = []
-
-    output = @__options["output"]
-    loop do
-      # Read messages
-      obj = read_message(@__consumee_pipes["rd_child_1"])
-
-      # Add row
-      if obj['tuple']
-
-        tuple = obj['tuple']
-        display_json = Hash[obj['tuple'].map{|k, v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
-
-        if col_map.keys.length > tuple.keys.length
-          cdisplay "Error: invalid keys for sink tuple : Expected #{col_map.keys} , got: #{tuple.keys}"
-          cdisplay("\n \nPress Ctrl-C to exit", false)
-          return
-        end
-        # Check tuple columns for valid entry
-        columns_to_check = col_map.length
-        tuple.keys.each do |col|
-          value = tuple[col]
-          types = type_map.each_value.map {|t| value.is_a? t}
-          matched_column = check_tuple_for_alias(col, obj['column_aliases'], col_map.keys)
-          if !matched_column.nil?
-            if value.nil? || (value.is_a? col_map[matched_column])
-              columns_to_check -= 1
-              # Set the proper column name
-              if col != matched_column
-                tuple[matched_column] = value
-                tuple.delete(col)
-              end
-            else
-              break
-            end
-          end
-        end
-
-        if columns_to_check != 0
-          cdisplay "Error: invalid schema for sink tuple #{display_json}"
-          return
-        end
-        tuples << obj
-        if @__options[:interactive]
-          cdisplay "received #{display_json}"
-        end
-        write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
-
-      # End cycle
-      elsif obj['command']
-        case obj['command']
-        when "end_cycle" 
-          break
-        when "kill_cycle"
-          break
-        end
-      end
-    end
-
-    if tuples.empty?
-      cdisplay "No tuples received"
-      return
-    end
-
-    # Build table
-    require("terminal-table")
-    table = Terminal::Table.new :title => @__name
-    require("csv")
-    csv_str = CSV.generate do |csv|
-      header_written = false;
-      tuples.each do |obj|
-        begin
-
-          t = obj['tuple']
-          m = obj['meta'] || {}
-
-           if t
-            if header_written == false
-              keys = [t.keys, m.keys].flatten
-              csv << keys
-              table << keys
-              table << :separator
-              header_written = true
-            end
-
-            vals = [t.values, m.values].flatten
-            csv << vals
-            table << vals.flat_map{|v| "#{v.to_s}"[0..100]}
-          end
-        rescue JSON::ParserError
-          cdisplay("invalid JSON")
-          next
-        rescue => e
-          cdisplay e.message
-          cdisplay e.backtrace
-        end
-      end
-    end
-
-    # Output table
-    cdisplay("\n#{table.to_s}")
-    cdisplay "#{tuples.length} rows"
-      
-    # Write file
-    if output
-      filename = "#{output}.csv"
-      f = File.open(filename, "w")
-      f.write(csv_str)
-      f.close()
-      cdisplay("output written to #{filename}")
-    end
-  end
-
-
-  private
-
-  BUFSIZE = 8192
-
-  # Each reading pipe has a read buffer and message queue
-  @__read_buffers = {}
-  @__read_buffered_messages = {}
-
-
-  # Return availible reading streams
-  def self.select_read_streams(read_streams)
-
-    rs = []
-    read_streams.each do |read_stream|
-      @__read_buffered_messages[read_stream] ||= []
-      if !@__read_buffered_messages[read_stream].empty?
-        rs << read_stream
-      end
-    end
-    return rs unless rs.empty?
-    rs, ws, es = IO.select(read_streams, [], [])
-    return rs
-  end
-
-
-  # Read a JSON message
-  def self.read_message(read_stream)
-
-    @__read_buffers[read_stream] ||= ""
-    @__read_buffered_messages[read_stream] ||= []
-    if !@__read_buffered_messages[read_stream].empty?
-        obj = @__read_buffered_messages[read_stream].shift
-      return obj
-    end
-    # read message from stream
-    loop do
-
-      while !@__read_buffers[read_stream].include? ENDMARKER
-        segment = read_stream.sysread(BUFSIZE)
-        @__read_buffers[read_stream] << segment
-      end
-
-      # cdisplay "READMESSAGE: read #{segment.length} bytes, read buffer length : #{@__read_buffer.length}"
-      # TODO this include is redundant
-      read_buffer = @__read_buffers[read_stream]
-      if read_buffer.include? ENDMARKER
-        objs = read_buffer.split(ENDMARKER)
-        ends = read_buffer.scan(ENDMARKER)
-        if objs.count == ends.count # We have a full number of messages
-          objs.each do |obj|
-            begin
-              @__read_buffered_messages[read_stream] << JSON.parse(obj)
-              # cdisplay "READMESSAGE: got hash #{hash}"
-            rescue JSON::ParserError 
-              cdisplay "READMESSAGE: invalid JSON #{obj}"
-            end
-          end
-          @__read_buffers[read_stream] = ""
-          return @__read_buffered_messages[read_stream].shift
-        else
-
-          (0..ends.count-1).each do |i|
-            obj = objs[i] 
-            begin
-              @__read_buffered_messages[read_stream] << JSON.parse(obj)
-            rescue JSON::ParserError 
-              cdisplay "READMESSAGE: invalid JSON #{obj}" 
-            end
-          end
-
-         # cdisplay "adding leftovers : \n  #{objs[ends.count..-1]}"
-          @__read_buffers[read_stream] = objs[ends.count..-1].join(ENDMARKER)
-          return @__read_buffered_messages[read_stream].shift
-        end
-      end
-    end
-  end
-
-
-  # Write JSON message
-  def self.write_message(write_stream, msg)
-    write_msg = msg.strip + ENDMARKER
-    write_stream.write write_msg
-    write_stream.flush
-  end
-
-
-  # Handshake connection to multilang
-  def self.handshake(write_stream, read_stream)
-    begin
-      write_message write_stream, HANDSHAKE_MESSAGE
-      msg = read_message(read_stream)
-    rescue => e
-      cdisplay("Error handshaking node")
-      raise e
-    end
-  end
-
-
-  # Instruct multilang to run prepare step
-  def self.prepare(write_stream, read_stream)
-    begin
-      write_message write_stream, PREPARE_MESSAGE
-      msg = read_message(read_stream)
-    rescue => e
-      cdisplay("Error running prepare")
-      raise e
-    end
-  end 
-
-
-  # Instruct multilang to begin cycle
-  def self.begin_cycle(write_stream, read_stream)
-    write_message(write_stream, BEGIN_CYCLE_MESSAGE)
-   
-    while 1 do
-
-      msg = read_message(read_stream)
-      obj = Hash[msg]
-      case obj["command"]
-        when "log"
-          cdisplay "LOG: #{obj['msg']}"
-        when "done"
-          break
-        else
-          cdisplay("Error beginning cycle")
-          raise obj["msg"]
-        end
-     end
-  
-  end
-
-  # Build the hash of consumer streams for lookup when receiving responses
-  def self.build_consumer_hash()
-    consumer_hash = {}
-    @__emit_queues.each_pair do |stream, consumers|
-      consumers.each_key do |consumer|
-
-        pipes = @__consumer_pipes[stream][consumer]
-        if pipes.has_key? "rd_parent_1"
-          read_stream = pipes["rd_parent_1"]
-          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-
-        elsif pipes.has_key? "rd_parent_2"
-          read_stream = pipes["rd_parent_2"]
-          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-        end
-      end
-    end
-
-    return consumer_hash
-  end
-
-  # Send object to every consumer of the operation, regardless of stream
-  def self.send_to_consumees(json_obj)
-    pipes = @__consumee_pipes
-    # Left hand(or singular) input
-    if (pipes.has_key? "wr_child_1")
-      write_stream = pipes["wr_child_1"]
-      write_message(write_stream, json_obj)
-    end
-
-    # Right hand input 
-    if (pipes.has_key? "wr_child_2")
-      write_stream = pipes["wr_child_2"]
-      write_message(write_stream, json_obj)
-    end
-  end
-
-  # Send object to every consumer of the operation, regardless of stream
-  def self.send_to_consumers(json_obj, display = false)
-    @__consumer_pipes.each_pair do |stream, consumers|
-      consumers.each_pair do |consumer, pipe| 
-        # Single or Left hand pipe
-        if (pipe.has_key? "wr_parent_1")
-          write_stream = get_write_stream(stream, consumer, 1)
-          write_message(write_stream, json_obj)
-        elsif (pipe.has_key? "wr_parent_2")
-          write_stream = get_write_stream(stream, consumer, 2)
-          write_message(write_stream, json_obj)
-        end
-        if display
-          cdisplay "emitted #{json_obj.chomp} to #{consumer}"
-        end
-      end
-    end
-  end
-
-
-  # Get the write pipe of the stream consumer
-  def self.get_write_stream(stream, consumer, number=1)
-     wr_pipe = "wr_parent_" + number.to_s
-     @__consumer_pipes[stream][consumer][wr_pipe]
-  end
-
-
-  # Get tuple for sending to consumer of stream
-  def self.get_consumer_tuple(stream, consumer)
-    @__emit_queues[stream][consumer][:write_queue].shift
-  end
-
-
-
-  # Send a command message to a consumer
-  def self.send_command_tuple(stream, consumer, json_obj)
-    pipe = @__consumer_pipes[stream][consumer]
-     # Single or Left hand pipe
-    if (pipe.has_key? "wr_parent_1")
-      write_stream = get_write_stream(stream, consumer, 1)
-      write_message(write_stream, json_obj)
-
-    # Right hand pipe
-    elsif (pipe.has_key? "wr_parent_2")
-      write_stream = get_write_stream(stream, consumer, 2)
-      write_message(write_stream, json_obj)
-    end
-    @__emit_queues[stream][consumer][:ready] = false
-  end
-
-
-  # Emit tuple_json to the consumer of a stream
-  def self.emit_consumer_tuple(stream, consumer, tuple_json)
-    begin
-      display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
-    rescue JSON::ParserError
-      cdisplay "Error: invalid JSON"
-    end
-
-    pipe = @__consumer_pipes[stream][consumer]
-     # Single or Left hand pipe
-    if (pipe.has_key? "wr_parent_1")
-      write_stream = get_write_stream(stream, consumer, 1)
-      write_message(write_stream, tuple_json)
-
-    # Right hand pipe
-    elsif (pipe.has_key? "wr_parent_2")
-      write_stream = get_write_stream(stream, consumer, 2)
-      write_message(write_stream, tuple_json)
-    end
-
-    @__emit_queues[stream][consumer][:ready] = false
-    cdisplay "emitted tuple #{display_json} to #{consumer} "
-  end
-
-
-  # Check if a tuple's column or its aliases matches an expected column and returns the valid column
-  def self.check_tuple_for_alias(column, column_aliases, valid_columns)
-
-    valid_columns.each do |expected|
-      if column == expected
-        return expected
-      end
-
-      column_aliases.each do |hash|
-        concrete_name = hash["concrete_name"]
-        if (column == concrete_name || column == hash['alias']) && hash["alias"] == expected
-          return expected
-        end
-      end
-    end
-
-    return nil
-  end
-
-  # Build a tuple and format into JSON
-  def self.build_tuple_json(tuple, meta = nil, column_aliases = nil)
-    meta ||= {}
-    column_aliases ||= {}
-    values = {}
-    tuple.each do |k, v|
-     if(k == "id")
-       next
-     elsif(k == "confidence" or k == "since" or k == "source")
-       meta[k] = v
-     else
-      values[k] = v
-     end
-    end
-    tuple_json = {"op" => @__name, "tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
-
-    return tuple_json
-  end
-
-
-  # Construct a multilang command
-  def self.command(arg, ignore_stderr=false)
-    cdisplay("could not extract meta information. missing zillabyte.conf.yml?") if @__meta.nil?
-    full_script = File.join(@__dir, @__meta["script"])
-    stderr_opt = "2> /dev/null" if ignore_stderr
-
-    case @__meta["language"]
-    when "ruby"
-      # Execute in the bundler context
-      cmd = "cd \"#{@__dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}  #{stderr_opt}"
-    when "python"#{
-      if(File.directory?("#{@__dir}/vEnv"))
-        cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{@__dir}/vEnv/bin/python \"#{full_script}\" #{arg}  #{stderr_opt}"
-      else
-        cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}  #{stderr_opt}"
-      end
-    when "js"
-      cmd = "cd \"#{@__dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN}  \"#{full_script}\" #{arg} #{stderr_opt}"
-    else
-      cdisplay("no language specified")
-    end
-    return cmd
-  end
-
-
-  # Display a colored, formatted message
-  def self.cdisplay(msg, useName=true)
-    @__tester.cdisplay(@__name, msg, useName)
-  end
-
-end; end; end
-