zillabyte-cli 0.0.24 → 0.1.0

data/lib/zillabyte/runner/multilang_operation.rb

@@ -0,0 +1,1133 @@
+require 'json'
+require 'mkfifo'
+require "zillabyte/runner/component_operation"
+
+
+# Emulate a multilang operation
+class Zillabyte::Runner::MultilangOperation
+
+  HANDSHAKE_MESSAGE = "{\"pidDir\": \"/tmp\"}\n"
+  DONE_MESSAGE = "{\"command\": \"done\"}\n"
+  NEXT_MESSAGE = "{\"command\": \"next\"}\n"
+  BEGIN_CYCLE_MESSAGE = "{\"command\": \"begin_cycle\"}\n"
+  END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
+  PONG_PREFIX = "{\"pong\": \""
+  PONG_SUFFIX = "\"}\n"  
+  ENDMARKER = "\nend\n"
+
+  def self.run(node, dir, consumee, consumer_pipes, tester, meta, options = {})
+
+    @__node = node
+    @__name = node["name"]
+    @__type = node["type"]
+    @__dir = dir
+    @__consumee = consumee
+    @__consumer_pipes = consumer_pipes
+    @__tester = tester
+
+    @__meta = meta
+    @__options = options
+    @__output_type = options[:output_type]
+
+    # Each consumer of a stream gets its own queue and message passing
+    @__emit_queues = {}
+    @__consumer_pipes.each_pair do |stream, consumers|
+      consumers.each_key do |consumer|
+        @__emit_queues[stream] ||= {}
+        @__emit_queues[stream][consumer] = {:write_queue => [], :ready => true}
+      end
+    end
+
+    begin
+      case @__type
+      when "source"
+        self.run_source()
+      when "group_by"
+        self.run_group_by()
+      when "each"
+        self.run_each()
+      when "filter"
+        self.run_filter()
+      when "component"
+        Zillabyte::Runner::ComponentOperation.run(node, dir, consumee, consumer_pipes, tester, meta, options = {})
+      when "sink"
+        self.run_sink()
+      else
+        cdisplay("invalid operation type #{@__type}")
+      end
+    rescue => e
+      cdisplay e.message
+      cdisplay e.backtrace
+    end
+  end
+
+
+  def self.run_source()
+
+    end_cycle_policy = @__node["end_cycle_policy"]
+
+    # Interactive source
+    if @__options[:interactive]
+      loop do 
+
+        msg = @__consumee[:rd_child].gets
+
+          # Build tuple
+          begin
+            tuple = JSON.parse(msg)
+          rescue JSON::ParserError 
+            cdisplay "Error: invalid JSON"           
+            next
+          end
+
+          tuple_json = build_tuple_json(tuple)
+          display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [truncate_message(k), truncate_message(v)]}].to_json
+          send_to_consumers(tuple_json)   
+      end
+
+    # Source from relation
+    elsif @__node['matches'] or @__node["relation"]
+
+      # Query API for rows  
+      matches = @__node['matches'] || (@__node["relation"]["query"])
+      cdisplay("Fetching remote data...")
+      res = @__tester.query_agnostic(matches)
+      rows = res["rows"]
+      if(rows.nil? or rows.length == 0)
+        cdisplay("Could not find data that matches your 'matches' clause")
+        exit(-1)
+      end
+      cdisplay("Received #{rows.length} rows!")
+
+      # Enqueue rows for sending to consumers
+      column_aliases = res["column_aliases"]
+      rows.each do |tuple|
+        tuple_json = build_tuple_json(tuple, nil, column_aliases)
+
+        @__emit_queues.each_pair do |stream, consumers|
+          consumers.each_pair do |consumer, emitter|
+            emitter[:write_queue] << tuple_json
+          end
+        end
+      end
+
+      # Index streams and consumers by their pipes for lookup
+      consumer_hash = {}
+      @__emit_queues.each_pair do |stream, consumers|
+        consumers.each_key do |consumer|
+          read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
+          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+        end
+      end
+
+
+      # Send first tuple
+      @__emit_queues.each_pair do |stream, consumers|
+        consumers.each_key do |consumer|
+          tuple_json = get_consumer_tuple(stream, consumer)
+          emit_consumer_tuple(stream, consumer, tuple_json)
+        end
+      end
+
+      # Sent tuples to consumers as appropriate
+      loop do 
+
+        # Retrieve messages from consumers
+        rs, ws, es = IO.select(consumer_hash.keys, [], [])
+
+        # Emit tuples to consumers
+        emitted = false
+        rs.each do |r|
+
+          # Read from consumer
+          msg = read_message(r)
+
+          stream = consumer_hash[r][:stream]
+          consumer = consumer_hash[r][:consumer]
+
+          # Consumer is ready for next message
+          if msg["command"] && msg["command"] == "next"
+
+            @__emit_queues[stream][consumer][:ready] = true
+            tuple_json = get_consumer_tuple(stream, consumer)
+
+            # If all messages have been sent to consumer, end their cycle
+            if tuple_json.nil?
+              write_stream = get_write_stream(stream, consumer)
+              cdisplay "ending cycle for #{consumer}"
+              write_message(write_stream, END_CYCLE_MESSAGE)
+              write_message(write_stream, DONE_MESSAGE)
+
+            else
+              # Emit tuple to consumer
+              emit_consumer_tuple(stream, consumer, tuple_json)
+              emitted = true
+            end
+          end
+        end
+
+        # Exit when done emitting
+        if !emitted
+          return
+        end
+      end
+      
+    # Custom source
+    else
+
+      # Index streams and consumers by their pipes for lookup
+
+      consumer_hash ={}
+      @__emit_queues.each_pair do |stream, consumers|
+        consumers.each_key do |consumer|
+          read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
+          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+        end
+      end
+
+      # Keep track of how many consumers to handle before exiting
+      consumers_running = consumer_hash.keys.length
+
+      # Setup multilang pipe
+      ml_pipe = "#{@__name}_pipe"
+      if File.exists?("#{ml_pipe}.in")
+        File.delete("#{ml_pipe}.in")
+      end
+      File.mkfifo("#{ml_pipe}.in")
+
+
+      # Spawn multilang process
+      cmd = command("--execute_live --name #{@__name} --pipe #{ml_pipe}")
+      begin
+
+        Open3.popen3(cmd) do |ml_input, stdout, stderr, wait_thread|
+          begin
+
+            # Multilang output tuples
+            ml_output = File.open("#{ml_pipe}.in", "r")
+
+            # Setup streams from consumers and multilang
+            read_streams = consumer_hash.keys.concat [stdout, ml_output]
+
+            # Handshake
+            handshake(ml_input, ml_output)
+
+            # Begin cycle
+            begin_cycle(ml_input, ml_output)
+            emitted = false
+            write_message(ml_input, NEXT_MESSAGE)
+            multilang_queue = []
+            end_cycle_policy = @__options[:end_cycle_policy]
+            end_cycle_received = false
+
+            # Receive and handle messages
+            loop do
+             
+              # Read from a stream
+              rs = select_read_streams(read_streams)
+              rs.each do |r|
+
+                # Read stdout straight to user
+                if r == stdout && consumers_running > 0
+                  msg = r.gets
+                  msg = msg.sub(/\n/, "")
+                  cdisplay("log: #{msg}")
+                  next
+                end
+
+                obj = read_message(r)
+
+                if obj.nil?
+                  next
+                end
+
+                if obj["command"] 
+                  case obj["command"]
+
+                  # Multilang emitted a tuple
+                  when "emit"
+
+                    stream = obj['stream']
+                    # Check for null emit
+                    if end_cycle_policy != "explicit"
+
+                      if obj['tuple'].nil?
+                        end_cycle_received = true
+                      else
+                        nil_values = false
+                        obj['tuple'].each_value do |v|
+                          if v.nil?
+                            nil_values = true
+                            break       
+                          end                     
+                        end
+                        end_cycle_received = nil_values
+                        next unless !end_cycle_received
+                      end
+                    end
+
+                    # Valid emit
+                    emitted = true
+
+                    # Send or enqueue tuple for each consumer
+                    tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
+
+ 
+                    @__emit_queues[stream].each_pair do |consumer, emitter|
+                      if emitter[:ready]
+                        emit_consumer_tuple(stream, consumer, tuple_json)
+                      else
+                        @__emit_queues[stream][consumer][:write_queue] << tuple_json
+                      end
+                    end
+
+                  # Consumer is ready for a message
+                  when "next"
+                    stream = consumer_hash[r][:stream]
+                    consumer = consumer_hash[r][:consumer]
+                    @__emit_queues[stream][consumer][:ready] = true
+                    tuple_json = get_consumer_tuple(stream, consumer)
+
+                    # End cycle for consumer if it has processed all tuples
+                    if tuple_json.nil? && end_cycle_received
+
+                      write_stream = get_write_stream(stream, consumer)
+                      write_message(write_stream, END_CYCLE_MESSAGE)
+                      consumers_running -= 1
+                      if consumers_running == 0
+                        break
+                      end
+
+                      # TODO break if last consumer
+                    elsif !tuple_json.nil?
+                      # Emit tuple to consumer
+                      emit_consumer_tuple(stream, consumer, tuple_json)
+                      emitted = true
+                    end
+
+                  # Multilang is done emitting a group of tuples
+                  when "done"
+                    # End cycle if no tuples were emitted
+                    if !emitted && end_cycle_policy == "null_emit"
+                      end_cycle_received = true
+                    else
+                      emitted = false
+                    end
+                  
+                    # Send the next tuple to multilang
+                    if !multilang_queue.empty?
+                      write_message(ml_input, multilang_queue.shift)
+
+                    # Request next tuple from mutilang
+                    elsif !end_cycle_received
+                      write_message(ml_input, NEXT_MESSAGE)
+
+                    # If there are no more messages to send, we are done
+                    else end_cycle_received
+                      finished = true
+                       # End cycle for ready consumers
+                      @__emit_queues.each_pair do |stream, consumers|
+                        consumers.each_pair do |consumer, emitter|
+                          if emitter[:ready]
+                            write_stream = get_write_stream(stream, consumer)
+                            write_message(write_stream, END_CYCLE_MESSAGE)
+                            consumers_running -= 1
+                            if consumers_running == 0
+                              break
+                            end
+                          end
+                        end
+
+                      end
+                    end
+
+                  # Multilang sent an error message
+                  when "fail"
+                    cdisplay("ERROR : #{obj['msg']}")
+                  
+                  # Multilang sent a log message
+                  when "log"
+                    cdisplay "LOG: #{obj['msg']}"
+
+                  # Multilang sent signal to end the cycle
+                  when "end_cycle"
+                    if end_cycle_policy != "explicit"
+                      cdisplay "received end_cycle command for non explicit policy"
+                      next
+                    end
+                    end_cycle_received = true
+
+                  end
+                
+                # Multilang sent a ping
+                elsif obj['ping']
+                  write_message(to_ml, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
+                end
+              end
+
+              # Exit after ending consumer cycles
+              if consumers_running == 0
+                return
+              end
+              
+            end
+          rescue Errno::EIO
+             cdisplay("Errno:EIO error")
+          ensure
+            # cleanup
+            pid = wait_thread[:pid] 
+            ml_input.close
+            ml_output.close
+            File.delete("#{ml_pipe}.in")
+            stdout.close
+            stderr.close
+            Process.kill('INT', pid)
+          end
+        end
+      rescue PTY::ChildExited
+        cdisplay("The child process exited!")
+      end
+    end
+  end
+
+
+  def self.run_each()
+
+    # Index streams and consumers by their pipes for lookup
+    consumer_hash = {}
+    @__emit_queues.each_pair do |stream, consumers|
+      consumers.each_key do |consumer|
+        read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
+        consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+      end
+    end
+
+
+    # Keep track of how many consumers to handle before exiting
+    consumers_running = consumer_hash.keys.length
+
+    # Setup multilang pipe
+    ml_pipe = "#{@__name}_pipe"
+    if File.exists?("#{ml_pipe}.in")
+      File.delete("#{ml_pipe}.in")
+    end
+    File.mkfifo("#{ml_pipe}.in")
+
+    cmd = command("--execute_live --name #{@__name} --pipe #{ml_pipe}")
+    begin
+      # Start the operation... 
+      Open3.popen3(cmd) do |ml_input, stdout, stderr, wait_thread|
+        begin
+
+          # Multilang output tuples
+          ml_output = File.open("#{ml_pipe}.in", "r+")
+
+          # Setup streams from consumers, multilang, and the consumee
+          read_streams = consumer_hash.keys.concat [@__consumee[:rd_child], ml_output, stdout]
+
+          # Handshake
+          handshake(ml_input, ml_output)
+
+          # Begin cycle
+          multilang_queue = []
+          mutlilang_count = 0
+          end_cycle_received = false
+
+          # Receive and handle messages
+          loop do
+
+            # Read from a stream
+            rs = select_read_streams(read_streams)
+            rs.each do |r|
+
+              # Read STDOUT from program straight to user
+              if r == stdout
+                msg = r.gets
+                msg = msg.sub(/\n/, "")
+                cdisplay("log: #{msg}")
+                next
+              end
+
+              # Receive an object
+              obj = read_message(r)
+              if obj["command"] 
+                case obj["command"]
+
+                # Multilang emitted a tuple
+                when "emit"
+
+                  stream = obj["stream"]
+                 
+                  # Send or enqueue tuple for each consumer
+                  tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
+
+                  @__emit_queues[stream].each_pair do |consumer, emitter|
+                    if emitter[:ready]
+                      emit_consumer_tuple(stream, consumer, tuple_json)
+                    else
+                      @__emit_queues[stream][consumer][:write_queue] << tuple_json
+                    end
+                  end
+
+                # Consumer is ready for a message
+                when "next"
+
+                  stream = consumer_hash[r][:stream]
+                  consumer = consumer_hash[r][:consumer]
+
+                  @__emit_queues[stream][consumer][:ready] = true
+                  tuple_json = get_consumer_tuple(stream, consumer)
+
+                  # End cycle for consumer if it has processed all tuples
+                  if tuple_json.nil? && end_cycle_received
+                    write_stream = get_write_stream(stream, consumer)
+                    write_message(write_stream, END_CYCLE_MESSAGE)
+                    consumers_running -= 1
+                    if consumers_running == 0
+                      break
+                    end
+
+
+                    # TODO break if last consumer
+                  elsif !tuple_json.nil?
+                    # Emit tuple to consumer
+                    emit_consumer_tuple(stream, consumer, tuple_json)
+                    emitted = true
+                  end
+
+                # Multilang is done emitting a group of tuples
+                when "done" 
+                  mutlilang_count -= 1                  
+
+                  # Send the next tuple to multilang
+                  if !multilang_queue.empty?
+                    write_message(ml_input, multilang_queue.shift)
+
+                  # Request next tuple from consumee
+                  elsif !end_cycle_received
+                    write_message(@__consumee[:wr_child], NEXT_MESSAGE)
+                    
+
+                  # If there are no more messages to send, we are done
+                  elsif end_cycle_received && mutlilang_count == 0
+                    finished = true
+
+                   # End cycle for ready consumers
+                    @__emit_queues.each_pair do |stream, consumers|
+                      consumers.each_pair do |consumer, emitter|
+                        if emitter[:ready]
+                          write_stream = get_write_stream(stream, consumer)
+                          write_message(write_stream, END_CYCLE_MESSAGE)
+                          consumers_running -= 1
+                          if consumers_running == 0
+                            break
+                          end
+                        end
+                      end
+                    end
+                  end
+
+                # Multilang sent an error message
+                when "fail"
+                  cdisplay("ERROR : #{obj['msg']}")
+                
+                # Multilang sent a log message
+                when "log"
+                  cdisplay "LOG: #{obj['msg']}"
+
+                # Consumee operation sent signal to end_cycle
+                when "end_cycle"
+                  end_cycle_received = true
+
+                  if mutlilang_count == 0
+                    
+                    @__emit_queues.each_pair do |stream, consumers|
+                      consumers.each_pair do |consumer, emitter|                      
+                        if emitter[:ready]
+                          write_stream = get_write_stream(stream, consumer)
+                          write_message(write_stream, END_CYCLE_MESSAGE)
+                          consumers_running -= 1
+                          if consumers_running == 0
+                            break
+                          end
+                        end
+                      end
+                    end
+
+                  end
+
+                end
+
+              # Received a tuple from consumee
+              elsif obj['tuple']
+              
+                # Send or enqueue to multilang
+                mutlilang_count += 1
+                if multilang_queue.empty?
+                  write_message(ml_input, obj.to_json)
+                else
+                  multilang_queue << obj.to_json
+                end
+              
+              # Multilang sent a ping
+              elsif obj['ping']
+                write_message(to_ml, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
+              end
+            end
+
+            # Exit after ending consumer cycles
+            if consumers_running == 0
+              break
+            end
+
+          end
+        rescue Errno::EIO
+           cdisplay("Errno:EIO error")
+        ensure
+          # cleanup
+          pid = wait_thread[:pid] 
+          ml_input.close  
+          ml_output.close
+          File.delete("#{ml_pipe}.in")
+          stdout.close
+          stderr.close
+          Process.kill('INT', pid)
+        end
+      end
+    rescue PTY::ChildExited
+      cdisplay("The child process exited!")
+    end
+  end
+
+
+  def self.run_group_by()
+
+    # Index streams and consumers by their pipes for lookup
+    consumer_hash = {}
+    @__emit_queues.each_pair do |stream, consumers|
+      consumers.each_key do |consumer|
+        read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
+        consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+      end
+    end
+
+    # Keep track of how many consumers to handle before exiting
+    consumers_running = consumer_hash.keys.length
+
+    # Setup groups
+    group_by = @__node['group_by']
+    group_tuples = {}
+    emit_count = 0    # used to track how many emits are expected once groups are sent to multilang
+    emitted_tuples = [] # used to send to consumers after once groupings are emitted
+    tuple_queue = []
+
+    # Setup multilang pipe
+    ml_pipe = "#{@__name}_pipe"
+    if File.exists?("#{ml_pipe}.in")
+      File.delete("#{ml_pipe}.in")
+    end
+    File.mkfifo("#{ml_pipe}.in")
+    
+    cmd = command("--execute_live --name #{@__name} --pipe #{ml_pipe}")
+    begin
+      # Start the operation... 
+      Open3.popen3(cmd) do |ml_input, stdout, stderr, wait_thread|
+        begin
+
+          # Multilang output tuples
+          ml_output = File.open("#{ml_pipe}.in", "r+")
+          # Setup streams from consumers, multilang, and the consumee
+          read_streams = consumer_hash.keys.concat [stdout, ml_output, @__consumee[:rd_child]]
+
+          # Handshake
+          handshake(ml_input, ml_output)
+
+          # Begin cycle
+          end_cycle_received = false
+          finished_emitting = false
+
+          # select a stream
+          loop do
+
+            # Read from a stream
+            rs = select_read_streams(read_streams)
+            rs.each do |r|
+
+              # Read STDOUT from program straight to user
+              if r == stdout
+                msg = r.gets
+                msg = msg.sub(/\n/, "")
+                cdisplay("log: #{msg}")
+                next
+              end
+
+
+              # Receive an object
+              obj = read_message(r)
+            
+              if obj["command"] 
+                case obj["command"]
+
+                when "done"
+
+                  if end_cycle_received
+                    tuple_json = tuple_queue.shift
+                    if !tuple_json.nil?
+                      write_message(ml_input, tuple_json)
+                    end
+                  end
+
+                  next
+
+                # Begin aggregation
+                when "end_cycle"
+                  end_cycle_received = true
+                  read_streams = [ml_output]
+
+                  group_tuples.each do |group_tuple, tuples|
+                    tuple_queue << "{\"command\": \"begin_group\", \"tuple\": #{group_tuple.to_json}, \"meta\":{}}\n"
+                    tuples.each do |t|
+                      tuple_queue << "{\"command\": \"aggregate\", #{t}}\n"
+                    end
+                   tuple_queue << "{\"command\": \"end_group\"}\n"
+
+                    # keep track of how many emits are expected
+                    emit_count += 1
+                  end
+
+                  tuple_json = tuple_queue.shift
+                  if !tuple_json.nil?
+                    write_message(ml_input, tuple_json)
+                  end
+
+                # Multilang has emitted a grouped tuple
+                when "emit"
+                  stream = obj['stream']
+                  emit_count -= 1
+                  # Enqueue for consumers
+                  tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
+                  @__emit_queues.each_pair do |stream, consumers|
+                    consumers.each_key do |consumer|
+                      @__emit_queues[stream][consumer][:write_queue] << tuple_json  
+                    end
+                  end
+
+                  # End cycle when done emitting
+                  if end_cycle_received && emit_count == 0
+                    finished_emitting = true
+                    break      
+                  elsif end_cycle_received
+                    tuple_json = tuple_queue.shift
+                    if !tuple_json.nil?
+                      write_message(ml_input, tuple_json)
+                    end
+                  end
+
+                end
+
+              # Received a tuple from operation
+              elsif obj["tuple"]
+                tuple = obj["tuple"].to_json
+                meta = obj["meta"].to_json
+                column_aliases = obj["column_aliases"] || {}
+                aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
+                gt = {}
+
+                # Get the column names to group on
+                group_by.each do |field|
+                  field_name = aliases[field] || field
+                  gt[field] = obj["tuple"][field_name]
+                end
+
+                msg_no_brackets = "\"tuple\": #{tuple}, \"meta\": #{meta}, \"column_aliases\": #{column_aliases.to_json}"
+
+                # Group tuple into existing group or create new group
+                if group_tuples[gt]
+                  group_tuples[gt] << msg_no_brackets
+                else
+                  group_tuples[gt] = [msg_no_brackets]
+                end
+
+                # Ask operation for next tuple
+                write_message(@__consumee[:wr_child], NEXT_MESSAGE)
+
+              end
+            end
+
+            # Send tuples to consumers
+            if finished_emitting && consumers_running > 0
+
+              # Send first tuple
+              @__emit_queues.each_pair do |stream, consumers|
+                consumers.each_key do |consumer|
+                  tuple_json = get_consumer_tuple(stream, consumer)
+                  emit_consumer_tuple(stream, consumer, tuple_json)
+                end
+              end
+
+              # Sent tuples to consumers as appropriate
+              loop do 
+
+                # Retrieve messages from consumers
+                rs, ws, es = IO.select(consumer_hash.keys, [], [])
+
+                # Emit tuples to consumers
+                emitted = false
+                rs.each do |r|
+
+                  # Read from consumer
+                  msg = read_message(r)
+                  consumer = consumer_hash[r][:consumer]
+                  stream = consumer_hash[r][:stream]
+
+                  # Consumer is ready for next message
+                  if msg["command"] && msg["command"] == "next"
+
+                    @__emit_queues[stream][consumer][:ready] = true
+                    tuple_json = get_consumer_tuple(stream, consumer)
+
+                    # If all messages have been sent to a consumer, end its cycle
+                    if tuple_json.nil?
+                      write_stream = get_write_stream(stream, consumer)
+                      write_message(write_stream, END_CYCLE_MESSAGE)
+                      consumers_running -= 1
+                      if consumers_running == 0
+                        break
+                      end
+                    else
+                      # Emit tuple to consumer
+                      emit_consumer_tuple(stream, consumer, tuple_json)
+                      emitted = true
+                    end
+                  end
+
+                end
+                 # Exit when done emitting
+                if !emitted
+                  return
+                end
+              end
+              break
+
+            # Exit after ending all consumer cycles
+            elsif consumers_running == 0
+              break
+            end
+
+            
+          end
+        rescue Errno::EIO
+           cdisplay("Errno:EIO error")
+        ensure
+          # cleanup
+          pid = wait_thread[:pid]  
+          ml_input.close 
+          ml_output.close
+          File.delete("#{ml_pipe}.in")
+          stdout.close
+          stderr.close
+          Process.kill('INT', pid)
+        end  
+      end
+    rescue PTY::ChildExited
+      cdisplay("The child process exited!")
+    end
+  end
+
+
+  def self.run_filter()
+    self.run_each()
+  end
+
+
+  # Send a message to all consumers of the operation
+  def self.run_sink()
+    output = @__options[:output]
+    messages = []
+    loop do
+      # Read messages
+      msg = read_message(@__consumee[:rd_child])
+      obj = msg
+
+      # Add row
+      if obj['tuple']
+        messages << msg
+        display_json = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}].to_json
+        if @__options[:interactive]
+          cdisplay "received #{display_json}"
+        end
+        write_message(@__consumee[:wr_child], NEXT_MESSAGE)
+
+      # End cycle
+      elsif obj['command'] && obj['command'] == "end_cycle"
+        break
+      end
+    end
+
+    if messages.empty?
+      cdisplay "empty relation"
+      return
+    end
+
+    # Build table
+    table = Terminal::Table.new :title => @__name
+    csv_str = CSV.generate do |csv|
+      header_written = false;
+      messages.each do |obj|
+        begin
+
+          t = obj['tuple']
+          m = obj['meta'] || {}
+
+           if t
+            if header_written == false
+              keys = [t.keys, m.keys].flatten
+              csv << keys
+              table << keys
+              table << :separator
+              header_written = true
+            end
+
+            vals = [t.values, m.values].flatten
+            csv << vals
+            table << vals.flat_map{|v| "#{v}"[0..100]}
+          end
+        rescue JSON::ParserError
+          cdisplay("invalid JSON")
+          next
+        rescue Exception => e
+          cdisplay e
+        end
+      end
+    end
+
+    # Output table
+    cdisplay("\n#{table.to_s}")
+    cdisplay ""
+      
+    # Write file
+    if output
+      filename = "#{output}.csv"
+      f = File.open(filename, "w")
+      f.write(csv_str)
+      f.close()
+      cdisplay("output written to #{filename}")
+    end
+  end
+
+
+  private
+
+  BUFSIZE = 8192
+
+  # Each reading pipe has a read buffer and message queue
+  @__read_buffers = {}
+  @__read_buffered_messages = {}
+
+
+  # Return availible reading streams
+  def self.select_read_streams(read_streams)
+
+    rs = []
+    read_streams.each do |read_stream|
+      @__read_buffered_messages[read_stream] ||= []
+      if !@__read_buffered_messages[read_stream].empty?
+        rs << read_stream
+      end
+    end
+    return rs unless rs.empty?
+
+    rs, ws, es = IO.select(read_streams, [], [])
+    return rs
+  end
+
+
+  # Read a JSON message
+  def self.read_message(read_stream)
+
+    @__read_buffers[read_stream] ||= ""
+    @__read_buffered_messages[read_stream] ||= []
+    if !@__read_buffered_messages[read_stream].empty?
+        obj = @__read_buffered_messages[read_stream].shift
+      return obj
+    end
+
+    # read message from stream
+    loop do
+
+      while !@__read_buffers[read_stream].include? ENDMARKER
+        segment = read_stream.sysread(BUFSIZE)
+        @__read_buffers[read_stream] << segment
+      end
+
+      # cdisplay "READMESSAGE: read #{segment.length} bytes, read buffer length : #{@__read_buffer.length}"
+      # TODO this include is redundant
+      read_buffer = @__read_buffers[read_stream]
+      if read_buffer.include? ENDMARKER
+        objs = read_buffer.split(ENDMARKER)
+        ends = read_buffer.scan(ENDMARKER)
+        if objs.count == ends.count # We have a full number of messages
+          objs.each do |obj|
+            begin
+              @__read_buffered_messages[read_stream] << JSON.parse(obj)
+              # cdisplay "READMESSAGE: got hash #{hash}"
+            rescue JSON::ParserError 
+              cdisplay "READMESSAGE: invalid JSON #{obj}"
+            end
+          end
+          @__read_buffers[read_stream] = ""
+          return @__read_buffered_messages[read_stream].shift
+        else
+
+          (0..ends.count-1).each do |i|
+            obj = objs[i] 
+            begin
+              @__read_buffered_messages[read_stream] << JSON.parse(obj)
+            rescue JSON::ParserError 
+              cdisplay "READMESSAGE: invalid JSON #{obj}" 
+            end
+          end
+
+         # cdisplay "adding leftovers : \n  #{objs[ends.count..-1]}"
+          @__read_buffers[read_stream] = objs[ends.count..-1].join(ENDMARKER)
+          return @__read_buffered_messages[read_stream].shift
+        end
+      end
+    end
+  end
+
+
+  # Write JSON message
+  def self.write_message(write_stream, msg)
+    write_msg = msg.strip + ENDMARKER
+    write_stream.write write_msg
+    write_stream.flush
+  end
+
+  # Format a message for display
+  def self.truncate_message(msg)
+    return msg if(!msg.instance_of?(String))
+    t_length = 50 # truncates entries to this length
+    m_length = msg.length
+    msg_out = m_length > t_length ? msg[0..t_length-3]+"..." : msg
+    msg_out
+  end
+
+
+  # Handshake connection to multilang
+  def self.handshake(write_stream, read_stream)
+    begin
+      write_message write_stream, HANDSHAKE_MESSAGE
+      msg = read_message(read_stream)
+    rescue Exception => e
+      cdisplay(e)
+      cdisplay("Error handshaking node")
+      raise e
+    end
+  end
+
+
+  # Instruct multilang to begin cycle
+  def self.begin_cycle(write_stream, read_stream)
+    begin
+      write_message(write_stream, BEGIN_CYCLE_MESSAGE)
+      msg = read_message(read_stream)
+      obj = Hash[msg]
+      if obj["command"] != "done"
+        raise "Invalid response from multilang #{msg}"
+      end
+    rescue Exception => e
+      cdisplay(e)
+    end
+  end
+
+
+  # Send object to every consumer of the operation, regardless of stream
+  def self.send_to_consumers(json_obj)
+    @__consumer_pipes.each_pair do |stream, consumers|
+      consumers.each_pair do |consumer, pipe| 
+        write_message(pipe[:wr_parent], json_obj)
+        cdisplay "emitted #{json_obj} to #{consumer}"
+      end
+    end
+  end
+
+
+  # Get the write pipe of the stream consumer
+  def self.get_write_stream(stream, consumer)
+     @__consumer_pipes[stream][consumer][:wr_parent]
+  end
+
+
+  # Get tuple for sending to consumer of stream
+  def self.get_consumer_tuple(stream, consumer)
+    @__emit_queues[stream][consumer][:write_queue].shift
+  end
+
+
+  # Emit tuple_json to the consumer of a stream
+  def self.emit_consumer_tuple(stream, consumer, tuple_json)
+    begin
+      display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [truncate_message(k), truncate_message(v)]}].to_json
+    rescue JSON::ParserError
+      cdisplay "Error: invalid JSON"
+    end
+    write_stream = get_write_stream(stream, consumer)
+    write_message(write_stream, tuple_json)
+    @__emit_queues[stream][consumer][:ready] = false
+    cdisplay "emitted tuple #{display_json} to #{consumer} "
+  end
+
+
+  # Build a tuple and format into JSON
+  def self.build_tuple_json(tuple, meta = nil, column_aliases = nil)
+    meta ||= {}
+    column_aliases ||= {}
+    values = {}
+    tuple.each do |k, v|
+     if(k == "id")
+       next
+     elsif(k == "confidence" or k == "since" or k == "source")
+       meta[k] = v
+     else  
+      values[k] = v
+     end
+    end
+    tuple_json = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
+    return tuple_json
+  end
+
+
+  # Construct a multilang command
+  def self.command(arg, ignore_stderr=false)
+    cdisplay("could not extract meta information. missing zillabyte.conf.yml?") if @__meta.nil?
+
+    full_script = File.join(@__dir, @__meta["script"])
+    stderr_opt = "2> /dev/null" if ignore_stderr
+
+    case @__meta["language"]
+    when "ruby"
+      # Execute in the bundler context
+      cmd = "cd \"#{@__dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}  #{stderr_opt}"
+    when "python"#{
+      if(File.directory?("#{@__dir}/vEnv"))
+        cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{@__dir}/vEnv/bin/python \"#{full_script}\" #{arg}  #{stderr_opt}"
+      else
+        cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}  #{stderr_opt}"
+      end
+    when "js"
+      cmd = "cd \"#{@__dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN}  \"#{full_script}\" #{arg} #{stderr_opt}"
+    else
+      cdisplay("no language specified")
+    end
+    return cmd
+  end
+
+
+  # Display a colored, formatted message
+  def self.cdisplay(msg)
+    @__tester.cdisplay(@__name, msg)
+  end
+
+
+end