zillabyte-cli 0.9.20 → 0.9.21

data/lib/#zillabyte-cli.rb#

@@ -1,5 +0,0 @@
-require "zillabyte-cli/version"
-require "zillabyte/api"
-require "zillabyte/cli"
-require "zillabyte/common"
-require "zillabyte/queries"

data/lib/zillabyte/cli/#logs.rb#

@@ -1,12 +0,0 @@
-require "zillabyte/cli/base"
-
-# manage custom logs
-#
-class Zillabyte::Command::Logs < Zillabyte::Command::Base
-
-
-
-  
-  
-
-end

data/lib/zillabyte/cli/#repl.rb#

@@ -1,43 +0,0 @@
-require "zillabyte/cli/base"
-require 'readline'
-# REPL console for zillabyte commands
-#
-class Zillabyte::Command::Repl < Zillabyte::Command::Base
-
-  # repl
-  #
-  # start a console session for zillabyte
-  # --quiet # HIDDEN
-  # --history HISTORY# HIDDEN hack to allow history for readline
-  def index
-    if !options[:quiet]
-      v = `zillabyte version`
-      display "\n#{v}Type q,exit or Ctrl+D to quit\n\n"
-    end
-    server = `echo $ZILLABYTE_API_HOST` || ""
-    prompt = ""
-    if server && server.chomp.length > 0
-      prompt = "#{server.chomp} " 
-    end
-    prompt += "zillabyte $ "
-    if options[:history]
-      #p options[:history]
-      history = JSON.parse(options[:history])
-      history.last(50).each do |his|
-        # TODO: Handle single quotes ??
-        Readline::HISTORY << his
-      end
-    end
-    # TODO: Add tab completion for basic commands, app/relation names etc.
-    while cmd = Readline.readline(prompt, true)
-      if cmd && cmd.length > 0
-        if cmd.downcase == "exit" || cmd.downcase == "q"
-          display "" # TODO Make Ctrl+D print a newline too
-          return
-        else
-          exec "zillabyte #{cmd}; zillabyte repl --quiet --history '#{Readline::HISTORY.to_a.to_json}'"
-        end
-      end
-    end
-  end
-end

data/lib/zillabyte/cli/templates/python/#simple_function.py#

@@ -1,27 +0,0 @@
-import zillabyte
-
-def prep(controller):
-  return
-
-# This is the heart of your algorithm.  It's processed on every
-# web page.  This algorithm is run in parallel on possibly hundreds
-# of machines.
-def execute(controller, tup):
-  if("hello world" in tup.values["html"]):
-    controller.emit("has_hello_world",{"url":tup.values["url"]})
-  return
-
-zillabyte.simple_function(\
-  # This directive instructs zillabyte to give your function every
-  # web page in our known universe.  Your function will have access
-  # to two fields: URL and HTML
-  matches = "select * from web_pa", \
-
-  # This directive tells Zillabyte what kind of data your function
-  # produces.  In this case, we're saying we will emit a tuple that
-  # is one-column wide and contains the field 'URL' 
-  emits = [["has_hello_world", [{"url":"string"}]]], \
-
-  prepare = prep, \
-  execute = execute\
-)

data/lib/zillabyte/runner.rb

@@ -1,6 +0,0 @@
-require "zillabyte/helpers"
-
-
-module Zillabyte::Runner
-
-end

data/lib/zillabyte/runner/app_runner.rb

@@ -1,320 +0,0 @@
-require "zillabyte/runner/multilang_operation"
-
-# HIDDEN:
-class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base 
-  include Zillabyte::Helpers
-
-  END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
-
-  def run (meta, dir = Dir.pwd, session = nil, options = {})
-
-    if meta.nil? or session.nil?
-      return
-    end
-
-    @session = session
-
-    @colors = {}
-    output = options[:output]
-    otype = options[:output_type] 
-    interactive = options[:interactive]
-    cycles = (options[:cycles] || "1").to_i
-
-    # Show the user what we know about their app... 
-    display "inferring your app details..."
-    describe_app(meta)
-
-    # Setup nodes and arcs
-    @nodes = meta["nodes"]
-    @node_map = {}
-    @nodes.each do |n|
-      @node_map[n["name"]] = n
-    end
-
-    @arcs = meta["arcs"]
-
-    # Organize component pipes
-    @operations = {}
-    @operation_pipes = {}
-
-    # On each cycle, setup and tear down a test harness
-    (1..cycles).each do |cycle|
-
-      display "starting cycle #{cycle}" unless  cycles == 1
-      
-      begin
-        # Setup component pipes
-        @nodes.each do |n|
-
-          name = n["name"]
-          type = n["type"]
-          emits = n["emits"]
-          if n["type"] ==  "source"
-            options[:end_cycle_policy] = n["end_cycle_policy"]
-          end
-          
-          # Create two new pipes in the parent.
-          rd_child_1, wr_parent_1 = IO.pipe()
-          rd_parent_1, wr_child_1 = IO.pipe()
-
-          @operation_pipes[name] = {
-            "rd_child_1" => rd_child_1,      
-            "wr_child_1" => wr_child_1,
-            "rd_parent_1" => rd_parent_1,        
-            "wr_parent_1" => wr_parent_1
-          }
-
-
-          # Add a second(right hand side) set ofpipes for joins
-          if type == "join"
-            # Create two new pipes in the parent.
-            rd_child_2, wr_parent_2 = IO.pipe()
-            rd_parent_2, wr_child_2 = IO.pipe()
-            @operation_pipes[name]["rd_child_2"] = rd_child_2
-            @operation_pipes[name]["wr_child_2"] = wr_child_2
-            @operation_pipes[name]["rd_parent_2"] = rd_parent_2
-            @operation_pipes[name]["wr_parent_2"] = wr_parent_2
-          end
-
-
-        end
-
-        # Maps origin => {stream => [destinations]}
-        @arc_map = {}
-        @arcs.each do |a|
-          origin = a["origin"]
-          name = a["name"]
-          dest = a["dest"]
-          @arc_map[origin] ||= {}
-          @arc_map[origin][name] ||= []
-          @arc_map[origin][name] << a["dest"]
-        end
-
-        # # Spawn component threads
-        @nodes.each do |n|
-
-          name = n["name"]
-          type = n["type"]
-          emits = n["emits"]
-
-
-          pipes = @operation_pipes[name] 
-
-          # Fork.
-          pid = fork()
-          if pid # In parent
-            # Close the reading end of the first child so we can write to the child.
-            pipes["rd_child_1"].close()
-            # Close the writing end of the first child so we can read from the child.
-            pipes["wr_child_1"].close()
-
-            if type == "join"
-              # Close the reading end of the second child so we can write to the child.
-              pipes["rd_child_2"].close()
-              # Close the writing end of the second child so we can read from the child.
-              pipes["wr_child_2"].close()
-            end
-          else # in child
-            # Close the writing end of the first parent so we can read from the parent.
-             pipes["wr_parent_1"].close()
-            # Close the reading end of the first parent so we can write to the parent.
-             pipes["rd_parent_1"].close()
-
-            if type == "join"
-              # Close the reading end of the second child so we can write to the child.
-              pipes["rd_parent_2"].close()
-              # Close the writing end of the second child so we can read from the child.
-              pipes["wr_parent_2"].close()
-            end
-
-
-            begin
-
-              # Setup reading and writing pipes for communicating with consumee component
-              if type != "join"
-                in_pipes = {"rd_child_1" => @operation_pipes[name]["rd_child_1"], "wr_child_1" => @operation_pipes[name]["wr_child_1"]}
-              
-              # Add join specific options
-              else
-                options[:join_options] = {}
-                in_pipes = {}
-                @arcs.each do |a|
-
-                  if (a["dest"] == name)
-                    # Left Side
-                    if (a["left"] == 1)
-                      options[:join_options][:lhs] = a["origin"]
-                      in_pipes["rd_child_1"] = @operation_pipes[name]["rd_child_1"]
-                      in_pipes["wr_child_1"] = @operation_pipes[name]["wr_child_1"]
-                    # Right Side
-                    elsif (a["right"] == 1)
-                      options[:join_options][:rhs] = a["origin"]
-                      in_pipes["rd_child_2"] = @operation_pipes[name]["rd_child_2"]
-                      in_pipes["wr_child_2"] = @operation_pipes[name]["wr_child_2"]
-                    end
-                  end
-                end
-              end
-
-              # Index consumer pipes by stream name, consumer_name
-              out_pipes = {}
-
-              # Check if you are the consumee for a downstream join in order to select the correct pipe
-              if type != "sink"
-                @arc_map[name].each_pair do |stream, destinations|
-                  out_pipes[stream] ||= {}
-                  
-
-                  destinations.each do |dest|
-                    out_pipes[stream][dest] ||= {}
-
-                    # Check for a join at the destination
-                    if (@node_map[dest]["type"] == "join")                      
-                      @arcs.each do |a|
-                        if (a["dest"] == dest && a["origin"] == name)
-                          # Left Side
-                          if (a["left"] == 1)
-                            out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"] 
-                            out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
-                            break
-                          elsif (a["right"] == 1)
-                            out_pipes[stream][dest]["wr_parent_2"] = @operation_pipes[dest]["wr_parent_2"] 
-                            out_pipes[stream][dest]["rd_parent_2"] = @operation_pipes[dest]["rd_parent_2"]
-                            break
-                          end
-                        end
-                      end
-                    else
-                      out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"] 
-                      out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
-                    end
-                  end
-                end
-              end
-
-              # Run the child process 
-              Zillabyte::Runner::MultilangOperation.run(n, dir, in_pipes, out_pipes, self, meta, options)
-
-            rescue => e
-              display e.message
-              display e.backtrace
-            ensure
-
-              # Close the reading end of the child
-              pipes["rd_child_1"].close()
-              # Close the writing end of the child
-              pipes["wr_child_1"].close()
-
-              # Close secondary join child
-              pipes["rd_child_2"].close() if pipes["rd_child_2"]
-              pipes["wr_child_2"].close() if pipes["wr_child_2"]
-
-              exit!(-1)
-            end
-
-          end #end child
-        end 
-
-        if interactive
-          display "To view results:  Enter 'end'    "
-          display ""
-
-  
-          while true
-            display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
-            msg = ask
-
-            
-            if msg == "end"
-              msg = END_CYCLE_MESSAGE
-            else
-              begin
-                JSON.parse(msg)
-              rescue JSON::ParserError
-                display "Received invalid JSON object"
-                next
-              end
-            end
-            # Send tuple to source
-           @operation_pipes["source_1"]["wr_parent_1"].puts msg
-          end
-        end 
-      rescue => e
-        display e.message
-        display e.backtrace
-      ensure
-        Process.waitall()
-        @operation_pipes.each do |name, pipes|
-          #Close the  writing end of the parent
-          pipes["wr_parent_1"].close()
-          # Close the reading end of the parent
-          pipes["rd_parent_1"].close()
-
-          # Close secondary join parent
-          pipes["wr_parent_2"].close() if pipes["wr_parent_2"]
-          pipes["rd_parent_2"].close() if pipes["rd_parent_2"]
-
-        end
-      end
-
-    end 
-  end
-
-  def session
-    @session
-  end
-
-
-  def cdisplay(name, message, useName=true)
-
-    color = @colors[name] || :default
-    if message.nil? || message == ""
-      return
-    else
-      
-      if message.is_a?(Array)
-        lines = message
-      else
-        lines = message.split("\n")
-      end
-
-      prefix = useName ? "#{name} - " : ""      
-      display "#{prefix}#{lines.first}".colorize(color)
-      lines[1..-1].each do |line|
-        display "#{' '*prefix.size}#{line}".colorize(color)
-      end
-    end
-  end
-
-
-  def query_agnostic(query)
-    require("zillabyte/api")
-    @session.api.query.agnostic(query)
-  end
-
-  def display(message, newline = true)
-    @session.display(message, newline)
-  end
-
-
-  def describe_app(meta)
-    require("colorize")
-    require("indentation")
-    colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
-    rjust = 20
-    display "#{'app name'.rjust(rjust)}: #{meta['name']}"
-    display "#{'app language'.rjust(rjust)}: #{meta['language']}"
-    meta['nodes'].each_with_index do |node, index|
-      @colors[node['name']] ||= colors[index % colors.length]
-      color = @colors[node['name']]
-      display (("="*rjust + "  operation ##{index}").colorize(color))
-      display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
-      display "#{"type".rjust(rjust)}: #{node['type'].to_s.colorize(color)}"
-      display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
-      display "#{"consumes".rjust(rjust)}: #{node['consumes'].to_s.colorize(color)}" if node['consumes']
-      display "#{"emits".rjust(rjust)}: #{JSON.pretty_generate(node['emits']).indent(rjust+2).lstrip.colorize(color)}" if node['emits']
-    end
-  end
-
-
-end

data/lib/zillabyte/runner/component_operation.rb

@@ -1,636 +0,0 @@
-
-
-
-# Emulate Component Operations
-class Zillabyte::Runner::ComponentOperation
-
-  NEXT_MESSAGE = "{\"command\": \"next\"}\n"
-  DONE_MESSAGE = "{\"command\": \"done\"}\n"
-  KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
-  END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
-  ENDMARKER = "\nend\n"
-
-  # Run the operation
-  def self.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options = {})
-    require "zillabyte/runner/multilang_operation"
-    require("zillabyte/runner/operation")
-
-    @__node = node
-    @__name = node["name"]
-    @__type = node["type"]
-    @__dir = dir
-    @__consumee_pipes = consumee_pipes
-    @__consumer_pipes = consumer_pipes
-    @__tester = tester
-
-    @__meta = meta
-    @__options = options
-    @__output_type = options[:output_type]
-
-    # Each consumer of a stream gets its own queue and message passing
-    @__emit_queues = {}
-    @__consumer_pipes.each_pair do |stream, consumers|
-      consumers.each_key do |consumer|
-        @__emit_queues[stream] ||= {}
-        @__emit_queues[stream][consumer] = {:write_queue => [], :ready => true}
-      end
-    end
-    begin
-      case @__type
-      when "input"
-        self.run_input()
-      when "component"
-        self.run_rpc_component()
-      when "output"
-        node["type"] = "sink"
-        Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
-      # Component outputs act in same manner as sinks
-      else
-        Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
-      end
-    rescue => e
-      cdisplay e.message
-      cdisplay e.backtrace
-    end
-    cdisplay "EXIT"
-  end
-
-
-  # Run a component input
-  def self.run_input()
-    input = @__options[:input]
-
-    fields = @__node["fields"].map {|f| f.keys[0]}
-
-    # Read input from file
-    if input
-      messages = []
-      cdisplay "reading from file...."
-      csv_rows = CSV.read("#{input}")
-      csv_rows.each do |row|
-        tuple = {}
-        fields.each {|f| tuple[f] = row.shift}
-
-        tuple_json = build_tuple_json(tuple)
-
-        @__emit_queues.each_pair do |stream, consumers|
-          consumers.each_pair do |consumer, emitter|
-            emitter[:write_queue] << tuple_json
-          end
-        end
-
-        # Index streams and consumers by their pipes for lookup
-        consumer_hash = build_consumer_hash()
-
-        # Send first tuple
-        @__emit_queues.each_pair do |stream, consumers|
-          consumers.each_key do |consumer|
-            tuple_json = get_consumer_tuple(stream, consumer)
-            emit_consumer_tuple(stream, consumer, tuple_json)
-          end
-        end
-
-        # Sent tuples to consumers as appropriate
-        loop do 
-
-          # Retrieve messages from consumers
-          rs, ws, es = IO.select(consumer_hash.keys, [], [])
-
-          # Emit tuples to consumers
-          emitted = false
-          rs.each do |r|
-
-            # Read from consumer
-            msg = read_message(r)
-
-            stream = consumer_hash[r][:stream]
-            consumer = consumer_hash[r][:consumer]
-
-            # Consumer is ready for next message
-            if msg["command"]
-              case msg["command"]
-              when "next"
-
-                @__emit_queues[stream][consumer][:ready] = true
-                tuple_json = get_consumer_tuple(stream, consumer)
-
-                # If all messages have been sent to consumer, end their cycle
-                if tuple_json.nil?
-                  write_stream = get_write_stream(stream, consumer)
-                  cdisplay "ending cycle for #{consumer}"
-                  send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                  send_command_tuple(stream, consumer, DONE_MESSAGE)
-
-                else
-                  # Emit tuple to consumer
-                  emit_consumer_tuple(stream, consumer, tuple_json)
-                  emitted = true
-                end
-
-              when "kill_cycle"
-                send_to_consumers(KILL_CYCLE_MESSAGE)                
-              end
-            end
-          end
-
-          # Exit when done emitting
-          if !emitted
-            return
-          end
-        end
-      end
-      
-    else
-      stdin = @__consumee_pipes["rd_child_1"]
-      loop do 
-
-        msg = stdin.gets
-        if msg == KILL_CYCLE_MESSAGE
-          send_to_consumers(KILL_CYCLE_MESSAGE)
-          return
-        else
-          # Build tuples
-          args = msg.scan(/(?:\w|"[^"]*")+/).map {|s| s.gsub(/[\'\"]/, '')}
-          component_args = []
-
-          while(true) do
-            break if args.empty?
-            tuple = {}
-            fields.each {|f| tuple[f] = args.shift}
-            tuple_json = build_tuple_json(tuple)
-            display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
-            send_to_consumers(tuple_json)
-          end
-        end
-      end
-    end
-  end
-
-
-  # Send to and manage an RPC component 
-  def self.run_rpc_component()
-
-    # Index streams and consumers by their pipes for lookup
-    consumer_hash = build_consumer_hash()
-    
-    # Keep track of how many consumers to handle before exiting
-    consumers_running = consumer_hash.keys.length
-
-    # Kill the cycle on error
-    cycle_killed = false
-
-    # Begin cycle
-    end_cycle_received = false
-
-    # The input component(singular at the moment)
-    read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"]]
-
-    # Start communication with API
-    api = @__tester.session.api
-    require("zillabyte/api/components")
-    caller = Zillabyte::API::Components.new(api)
-    component_id = @__node["id"]
-    output_format = @__node["output_format"]
-    component_info = api.request(
-      :expects  => 200,
-      :method   => :get,
-      :path     => "/flows/#{component_id}"
-    )
-    component_schema = component_info.body["schema"]
-    cdisplay("error: The requested component is not properly registered.") if component_info.nil?
-
-    component_nodes = component_schema["nodes"]
-    source_nodes = []
-    component_nodes.each do |node|
-      source_nodes << node if node["type"] == "source"
-    end
-    cdisplay("error: This component has multiple input streams. Currently we only support single input streams, sorry!") if source_nodes.size > 1
-
-    fields = source_nodes[0]["fields"]
-
-    begin
-
-      # Receive and handle messages
-      loop do
-
-        # Read from a stream
-        rs = select_read_streams(read_streams)
-        rs.each do |r|
-          # Read a message
-          obj = read_message(r)
-
-          # Handle tuple through RPC
-          if obj['tuple']
-            cdisplay("error: The number of inputs to the component does not match the declared number for stream #{source_nodes[0]["name"]}.") if obj['tuple'].size != fields.size
-            rpc_inputs = []
-            fields.each do |field|
-              rpc_inputs << obj['tuple'][field.keys[0]]
-            end
-
-            display_json = Hash[obj['tuple'].map{|k, v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
-            
-
-            # Send RPC call
-            cdisplay "sending RPC call..."
-            call_options = {
-              :rpc_inputs => [rpc_inputs],
-              :output_format => output_format
-            }
-
-            res = caller.rpc(component_id, call_options)
-            cdisplay "received API response : #{res}"
-
-            #TODO handle errors
-            if res['error']
-              cdisplay("error: #{res['error']}")
-              next
-            end
-            run_ids = []
-            res['execute_ids'].each do |q, qid|
-              run_ids << qid
-            end
-            status = res['status']
-
-
-            # Send result call
-            loop do
-
-              # If a call for results is not ready, save the execution ID for the next cycle
-              ids_in_progress = []
-              res = caller.get_rpc_results(component_id, {:execute_ids => run_ids})
-              if res['error']
-                cdisplay("error: #{res['error']}")
-                next
-              end
-
-          
-              # check results
-              res['results'].each do |run_id, hash|
-                
-                # We have results
-                if hash['status'] == "complete"
-
-                  # Handle Tuple in here
-                  data_streams = hash['data']
-
-                  # Handle each resulting tuple
-                  data_streams.each_pair do |stream, data_tuples|
-                    data_tuples.each do |data_tuple|
-                      tuple_json = build_tuple_json(data_tuple)
-
-                      #send or enqueue the tuple to all consumers of the stream
-                      @__emit_queues.each_pair do |stream, consumers|
-                        consumers.each_pair do |consumer, emitter|
-                          if emitter[:ready]
-                            emit_consumer_tuple(stream, consumer, tuple_json)
-                          else
-                            @__emit_queues[stream][consumer][:write_queue] << tuple_json
-                          end
-                        end
-                      end
-
-                    end
-                  end
-
-                else
-                  cdisplay "RPC #{run_id} has not completed... "
-                  ids_in_progress << run_id
-                end
-              end
-              run_ids = ids_in_progress
-
-              # If no more IDs to run, we are done
-              if run_ids.empty?
-                break
-              end
-
-              # Dont spam the API
-              sleep(2)
-            end
-
-            # Ask for next tuple
-            write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
-
-
-          # End cycle
-          elsif obj['command'] 
-            case obj["command"]
-
-            # Consumer is ready for a message
-            when "next"
-              stream = consumer_hash[r][:stream]
-              consumer = consumer_hash[r][:consumer]
-
-              @__emit_queues[stream][consumer][:ready] = true
-              tuple_json = get_consumer_tuple(stream, consumer)
-
-              # End cycle for consumer if it has processed all tuples
-              if tuple_json.nil? && end_cycle_received
-                send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                consumers_running -= 1
-                if consumers_running == 0
-                  return
-                end
-
-                # TODO break if last consumer
-              elsif !tuple_json.nil?
-                # Emit tuple to consumer
-                emit_consumer_tuple(stream, consumer, tuple_json)
-                emitted = true
-              end
-
-            when "end_cycle"
-              end_cycle_received = true
-            when "kill_cycle"
-              cycle_killed = true
-              return
-            end
-          end
-        end
-      
-
-        # Exit after ending consumer cycles
-        if consumers_running == 0
-          return
-        end
-      end
-
-    rescue => e
-      cdisplay e.message
-      cdisplay e.backtrace
-    ensure
-      # cleanup
-      if cycle_killed
-        send_to_consumers(KILL_CYCLE_MESSAGE, false)
-        send_to_consumees(KILL_CYCLE_MESSAGE, false)
-      end
-    end
-
-
-  end
-
-
-
-  private
-
-  BUFSIZE = 8192
-
-  # Each reading pipe has a read buffer and message queue
-  @__read_buffers = {}
-  @__read_buffered_messages = {}
-
-
-  # Return availible reading streams
-  def self.select_read_streams(read_streams)
-
-    rs = []
-    read_streams.each do |read_stream|
-      @__read_buffered_messages[read_stream] ||= []
-      if !@__read_buffered_messages[read_stream].empty?
-        rs << read_stream
-      end
-    end
-    return rs unless rs.empty?
-
-    rs, ws, es = IO.select(read_streams, [], [])
-    return rs
-  end
-
-
-  # Read a JSON message
-  def self.read_message(read_stream)
-
-    @__read_buffers[read_stream] ||= ""
-    @__read_buffered_messages[read_stream] ||= []
-    if !@__read_buffered_messages[read_stream].empty?
-        obj = @__read_buffered_messages[read_stream].shift
-      return obj
-    end
-
-    # read message from stream
-    loop do
-
-      while !@__read_buffers[read_stream].include? ENDMARKER
-        segment = read_stream.sysread(BUFSIZE)
-        @__read_buffers[read_stream] << segment
-      end
-
-      read_buffer = @__read_buffers[read_stream]
-      if read_buffer.include? ENDMARKER
-        objs = read_buffer.split(ENDMARKER)
-        ends = read_buffer.scan(ENDMARKER)
-        if objs.count == ends.count # We have a full number of messages
-          objs.each do |obj|
-            begin
-              @__read_buffered_messages[read_stream] << JSON.parse(obj)
-            rescue JSON::ParserError 
-              cdisplay "READMESSAGE: invalid JSON #{obj}"
-            end
-          end
-          @__read_buffers[read_stream] = ""
-          return @__read_buffered_messages[read_stream].shift
-        else
-
-          (0..ends.count-1).each do |i|
-            obj = objs[i] 
-            begin
-              @__read_buffered_messages[read_stream] << JSON.parse(obj)
-            rescue JSON::ParserError 
-              cdisplay "READMESSAGE: invalid JSON #{obj}" 
-            end
-          end
-
-          @__read_buffers[read_stream] = objs[ends.count..-1].join(ENDMARKER)
-          return @__read_buffered_messages[read_stream].shift
-        end
-      end
-    end
-  end
-
-
-  # Write JSON message
-  def self.write_message(write_stream, msg)
-    write_msg = msg.strip + ENDMARKER
-    write_stream.write write_msg
-    write_stream.flush
-  end
-
-  # Handshake connection to multilang
-  def self.handshake(write_stream, read_stream)
-    begin
-      write_message write_stream, HANDSHAKE_MESSAGE
-      msg = read_message(read_stream)
-    rescue Exception => e
-      cdisplay(e)
-      cdisplay("Error handshaking node")
-      raise e
-    end
-  end
-
-
-   # Build the hash of consumer streams for lookup when receiving responses
-  def self.build_consumer_hash()
-    consumer_hash = {}
-    @__emit_queues.each_pair do |stream, consumers|
-      consumers.each_key do |consumer|
-
-        pipes = @__consumer_pipes[stream][consumer]
-        if pipes.has_key? "rd_parent_1"
-          read_stream = pipes["rd_parent_1"]
-          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-
-        elsif pipes.has_key? "rd_parent_2"
-          read_stream = pipes["rd_parent_2"]
-          consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
-        end
-      end
-    end
-
-    return consumer_hash
-  end
-
-
-  # Send object to every consumer of the operation, regardless of stream
-  def self.send_to_consumers(json_obj)
-    @__consumer_pipes.each_pair do |stream, consumers|
-      consumers.each_pair do |consumer, pipe| 
-
-        # Single or Left hand pipe
-        if (pipe.has_key? "wr_parent_1")
-          write_stream = get_write_stream(stream, consumer, 1)
-          write_message(write_stream, json_obj)
-        elsif (pipe.has_key? "wr_parent_2")
-          write_stream = get_write_stream(stream, consumer, 2)
-          write_message(write_stream, json_obj)
-        end
-
-        cdisplay "emitted #{json_obj.chomp} to #{consumer}"
-      end
-    end
-  end
-
-
-  # Send object to every consumer of the operation, regardless of stream
-  def self.send_to_consumees(json_obj)
-    pipes = @__consumee_pipes
-    # Left hand(or singular) input
-    if (pipes.has_key? "wr_parent_1")
-      write_stream = get_write_stream(stream, consumer, 1)
-      write_message(write_stream, json_obj)
-    end
-
-    # Right hand input 
-    if (pipes.has_key? "wr_parent_2")
-      write_stream = get_write_stream(stream, consumer, 2)
-      write_message(write_stream, json_obj)
-    end
-  end
-
-
-  # Get the write pipe of the stream consumer
-  def self.get_write_stream(stream, consumer, number=1)
-     wr_pipe = "wr_parent_" + number.to_s
-     @__consumer_pipes[stream][consumer][wr_pipe]
-  end
-
-
-  # Get tuple for sending to consumer of stream
-  def self.get_consumer_tuple(stream, consumer)
-    @__emit_queues[stream][consumer][:write_queue].shift
-  end
-
-
-  def self.send_command_tuple(stream, consumer, json_obj)
-    pipe = @__consumer_pipes[stream][consumer]
-     # Single or Left hand pipe
-    if (pipe.has_key? "wr_parent_1")
-      write_stream = get_write_stream(stream, consumer, 1)
-      write_message(write_stream, json_obj)
-
-    # Right hand pipe
-    elsif (pipe.has_key? "wr_parent_2")
-      write_stream = get_write_stream(stream, consumer, 2)
-      write_message(write_stream, json_obj)
-    end
-    @__emit_queues[stream][consumer][:ready] = false
-  end
-
-
-  # Emit tuple_json to the consumer of a stream
-  def self.emit_consumer_tuple(stream, consumer, tuple_json)
-    begin
-      display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
-    rescue JSON::ParserError
-      cdisplay "Error: invalid JSON"
-    end
-
-    pipe = @__consumer_pipes[stream][consumer]
-     # Single or Left hand pipe
-    if (pipe.has_key? "wr_parent_1")
-      write_stream = get_write_stream(stream, consumer, 1)
-      write_message(write_stream, tuple_json)
-
-    # Right hand pipe
-    elsif (pipe.has_key? "wr_parent_2")
-      write_stream = get_write_stream(stream, consumer, 2)
-      write_message(write_stream, tuple_json)
-    end
-
-    @__emit_queues[stream][consumer][:ready] = false
-    cdisplay "emitted tuple #{display_json} to #{consumer} "
-  end
-
-
-  # Build a tuple and format into JSON
-  def self.build_tuple_json(tuple, meta = nil, column_aliases = nil)
-    meta ||= {}
-    column_aliases ||= {}
-    values = {}
-    tuple.each do |k, v|
-     if(k == "id")
-       nextx
-     elsif(k == "confidence" or k == "since" or k == "source")
-       meta[k] = v
-     else  
-      values[k] = v
-     end
-    end
-    tuple_json = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
-    return tuple_json
-  end
-
-
-  # Construct a multilang command
-  def self.command(arg, ignore_stderr=false)
-    cdisplay("could not extract meta information. missing zillabyte.conf.yml?") if @__meta.nil?
-
-    full_script = File.join(@__dir, @__meta["script"])
-    stderr_opt = "2> /dev/null" if ignore_stderr
-
-    case @__meta["language"]
-    when "ruby"
-      # Execute in the bundler context
-      cmd = "cd \"#{@__dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}  #{stderr_opt}"
-    when "python"#{
-      if(File.directory?("#{@__dir}/vEnv"))
-        cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{@__dir}/vEnv/bin/python \"#{full_script}\" #{arg}  #{stderr_opt}"
-      else
-        cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}  #{stderr_opt}"
-      end
-    when "js"
-      require("zillabyte/api/settings")
-      cmd = "cd \"#{@__dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN}  \"#{full_script}\" #{arg} #{stderr_opt}"
-    else
-      cdisplay("no language specified")
-    end
-    return cmd
-  end
-
-
-  # Display a colored, formatted message
-  def self.cdisplay(msg, useName=true)
-    @__tester.cdisplay(@__name, msg, useName)
-  end
-
-
-end