zillabyte-cli 0.0.21 → 0.0.22

data/lib/zillabyte/cli/apps.rb.orig

@@ -1,1158 +0,0 @@
-require "zillabyte/cli/base"
-require "zillabyte/cli/config"
-require "zillabyte/common"
-require "pty"
-require 'indentation'
-require 'open3'
-require 'securerandom'
-require 'colorize'
-require 'time_difference'
-
-# manage custom apps
-#
-class Zillabyte::Command::Apps < Zillabyte::Command::Base
-
-  MAX_POLL_SECONDS = 60 * 5
-  POLL_SLEEP = 0.5
-
-  # apps
-  #
-  # list custom apps
-  # --type TYPE     # specify an output type i.e. json
-  #
-  def index
-    self.list
-  end
-
-  # apps
-  #
-  # list custom apps
-  # --type TYPE # specify an output type i.e. json
-  def list
-    type = options[:type]
-    
-    headings = ["id", "name", "state", "cycles"]
-    rows = api.app.list.map do |row|
-      if headings.size == 0
-        headings = row.keys 
-        headings.delete("rel_dir")
-      end
-      v = row.values_at *headings
-      v
-    end
-
-    display "apps:\n" if type.nil?
-    display TableOutputBuilder.build_table(headings, rows, type)
-    display "Total number of apps: "+rows.length.to_s if type.nil?
-  end
-
-
-  # apps:push [DIR]
-  #
-  # uploads an app
-  # 
-  # --config CONFIG_FILE # use the given config file
-  # --type TYPE # specify an output type i.e. json
-  # --directory DIR # app directory
-  #
-  #Examples:
-  #
-  # $ zillabyte apps:push . 
-  #
-  def push
-    
-    since = Time.now.utc.to_s
-    dir = options[:directory] || shift_argument
-    if dir.nil?
-      dir = Dir.pwd
-    else
-      dir = File.expand_path(dir)
-    end
-    options[:directory] = dir
-    type = options[:type]
-
-    res = api.apps.push_directory dir, session, options
-    
-    if res['error']
-      error("error: #{res['error_message']}", type)
-    else
-      display "app ##{res['id']} #{res['action']}" if type.nil?
-    end
-
-    display "Starting up your app...please wait..." if type.nil?
-    sleep(2) # wait for kill command 
-    
-    lf = LogFormatter::Startup.new
-    api.logs.get_startup(res['id'], "_ALL_", {:push => true}) do |hash|
-      
-      # Error?
-      error(hash['error_message'], type) if hash['error']
-      
-      # Print it
-      lf.print_log_line(hash) if type.nil?
-      
-      # Exit when we get the 'done' message
-      # exit(0)  if (hash['line'] || '').downcase.include?("app deployed")
-      
-    end
-    display "{}" if type == "json"
-      
-  end
-  alias_command "push", "apps:push"
-  
-  
-  
-  # apps:pull ID DIR
-  #
-  # pulls an app source to a directory.
-  #
-  # --force # pulls even if the directory exists
-  # --type TYPE # specify an output type i.e. json
-  # --directory DIR # Directory of the app
-  #
-  #Examples:
-  #
-  # $ zillabyte apps:pull . 
-  #
-  def pull
-
-    app_id = options[:id] || shift_argument
-
-    if !(app_id =~ /^\d*$/)
-      options[:is_name] = true
-    end
-
-    dir = options[:directory] || shift_argument
-    error("no directory given", type) if dir.nil?
-    dir = File.expand_path(dir)
-
-    type = options[:type]
-    
-    error("no id given", type) if app_id.nil?
-    
-    # Create if not exists..
-    if File.exists?(dir)
-      if Dir.entries(dir).size != 2 and options[:force].nil?
-        error("target directory not empty. use --force to override", type)
-      end
-    else
-      FileUtils.mkdir_p(dir)
-    end
-    
-    res = api.apps.pull_to_directory app_id, dir, session, options
-    
-    if res['error']
-      error("error: #{res['error_message']}", type)
-    else
-      if type == "json"
-        display "{}"
-      else
-        display "app ##{res['id']} pulled to #{dir}"
-      end
-    end
-    
-  end
-  alias_command "pull", "apps:pull"
-
-
-  # apps:delete ID
-  #
-  # deletes an app. if the app is running, this command will kill it. 
-  #
-  # -f, --force # don't ask for confirmation
-  # --type TYPE # specify an output type i.e. json
-  #
-  def delete
-    app_id = options[:id] || shift_argument
-
-    if app_id.nil?
-      app_id = read_name_from_conf(options)
-      options[:is_name] = true
-    elsif !(app_id =~ /^\d*$/)
-      options[:is_name] = true
-    end
-    forced = options[:force]
-    type = options[:type]
-
-    if not forced
-
-      if !type.nil?
-        error("specify -f, --force to confirm deletion", type)
-      end
-
-      while true
-        
-        display "This operation cannot be undone. Are you sure you want to delete this app? (yes/no):", false
-        confirm = ask
-        break if confirm == "yes" || confirm == "no"
-        display "Please enter 'yes' to delete the app or 'no' to exit"
-      end
-    end
-
-    confirmed = forced || confirm == "yes"
-
-    if confirmed
-      response = api.apps.delete(app_id, options)
-      if type == "json"
-        display "{}"
-      else
-        display response["body"]
-      end
-    end
-
-  end
-
-  
-  # apps:prep [DIR]
-  #
-  # --directory DIR       # app directory
-  # --type TYPE           # specify an output type i.e. json
-  #
-  def prep
-
-    type = options[:type]
-    dir = options[:directory] || shift_argument
-    if dir.nil?
-      dir = Dir.pwd
-    else
-      dir = File.expand_path(dir)
-    end
-    options[:directory] = dir
-    meta = Zillabyte::CLI::Config.get_config_info(dir)
-
-    if meta.nil?
-      error("The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file.", type)
-    end
-
-    case meta["language"]
-    when "ruby"
-
-      # Execute in the bundler context
-      full_script = File.join(dir, meta["script"])
-      cmd = "cd \"#{meta['home_dir']}\"; unset BUNDLE_GEMFILE; unset RUBYOPT; bundle install"
-      exec(cmd)
-      
-    when "python"
-      vDir = "#{meta['home_dir']}/vEnv"
-      lock_file = meta['home_dir']+"/zillabyte_thread_lock_file"
-      if File.exists?(lock_file)
-        sleep(1) while File.exists?(lock_file)
-      else
-        begin
-          cmd = "touch #{lock_file}; virtualenv --clear --system-site-packages #{vDir}; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{vDir}/bin/pip install -r #{meta['home_dir']}/requirements.txt"
-          system cmd, :out => :out
-        ensure
-          File.delete(lock_file)
-        end
-      end
-      
-    when "js"
-    end
-    
-  end
-  alias_command "prep", "apps:prep"
-
-
-
-  
-  # apps:init [LANG] [DIR]
-  #
-  # initializes a new executable in DIR
-  # [LANG] defaults to ruby, and [DIR] to the current directory
-  #
-  # --type TYPE # specify an output type i.e. json
-  # --directory DIR # Directory of the app
-  #
-  #Examples:
-  #
-  # $ zillabyte apps:init python contact_extractor
-  #
-  def init
-    
-    lang = options[:lang] || shift_argument || "ruby"
-    dir = options[:directory] || shift_argument
-    if dir.nil?
-      dir = Dir.pwd
-    else
-      dir = File.expand_path(dir)
-    end
-    type = options[:type]
-
-    languages = ["ruby","python", "js"]
-
-    error("Unsupported language #{lang}. We only support #{languages.join(', ')}.", type) if not languages.include? lang
-
-    display "initializing empty #{lang} app in #{dir}" if type.nil?
-    FileUtils.cp_r( File.expand_path("../templates/#{lang}", __FILE__) + "/." , dir )
-    
-    
-  end
-
-
-
-  # apps:logs ID [OPERATION_NAME]
-  #
-  # streams logs from the distributed workers
-  # 
-  # --type TYPE # specify an output type i.e. json
-  # -v, --verbose LEVEL  # sets the verbosity (error, info, debug) (default: info)
-  #
-  def logs
-
-    app_id = options[:id] || shift_argument
-    if app_id.nil?
-      app_id = read_name_from_conf(options)
-      options[:is_name] = true
-    elsif !(app_id =~ /^\d*$/)
-      options[:is_name] = true
-    end
-
-    operation_id = options[:operation] || shift_argument || '_ALL_'
-    category = options[:verbose] || '_ALL_'
-    type = options[:type]
-
-    carry_settings = {
-      :category => category
-    }
-
-    display "Retrieving logs for app ##{app_id}...please wait..." if type.nil?
-    lf = LogFormatter::Operation.new
-    self.api.logs.get(app_id, operation_id, options) do |line|
-      
-      error(line['error_message'], type) if line['error']
-      lf.print_log_line(line) if type.nil?
-      
-    end
-
-  end  
-  alias_command "logs", "apps:logs"
-
-
-
-  # apps:errors ID
-  #
-  # Show recent errors generated by the app
-  # --type TYPE           # specify an output type i.e. json
-  #
-  def errors
-    
-    # Init
-    app_id = options[:id] || shift_argument
-    
-    # No name?
-    if app_id.nil?
-      app_id = read_name_from_conf(options)
-      options[:is_name] = true
-    elsif !(app_id =~ /^\d*$/)
-      options[:is_name] = true
-    end
-
-    type = options[:type]
-
-    # Make the request
-    res = api.request(
-      :expects => 200, 
-      :method => :get,
-      :body    => options.to_json,
-      :path => "/flows/#{CGI.escape(app_id)}/errors"
-    )
-    
-    # Render
-    display "Recent errors:" if type.nil?
-    headings = ["operation", "date", "error"]
-    rows = (res.body["recent_errors"] || []).map do |row|
-      if row['date'] 
-        d = Time.at(row['date']/1000)
-      else
-        d = nil
-      end
-      [row['name'], d, row['message']]
-    end
-    rows.sort! do |a,b|
-      a[1] <=> b[1]
-    end
-    color_map = {}
-    colors = LogFormatter::COLORS.clone
-    rows.each do |row|
-      name = row[0]
-      time = row[1]
-      message = row[2].strip
-      color_map[name] ||= colors.shift
-      if time 
-        display "#{"* #{name} - #{time_ago_in_words(time)} ago".colorize(color_map[name])}:" if type.nil?
-      else
-        display "#{"* #{name}".colorize(color_map[name])}:" if type.nil?
-      end
-      message.split('\n').each do |sub_line|
-        display "    #{sub_line}" if type.nil?
-      end
-    end
-    
-  end
-
-  # apps:cycles ID [OPTIONS]
-  #  
-  # operations on the app's cycles (batches).
-  # with no options, the command lists the apps cycles
-  # -n, --next            # request the app to move to the next cycle
-  # -f, --forever         # don't wait on cycles any more
-  # --type TYPE           # specify an output type i.e. json
-  #
-  def cycles
-    app_id = options[:id] || shift_argument
-    type = options[:type]
-
-    trigger_next = options[:next] || false
-    trigger_forever = options[:forever] || false
-
-    if app_id.nil?
-      app_id = read_name_from_conf(options)
-      options[:is_name] = true
-    elsif !(app_id =~ /^\d*$/)
-      options[:is_name] = true
-    end
-
-    if trigger_next
-      # Trigger the next app
-      response = api.apps.create_cycle(app_id, options)
-    elsif trigger_forever
-      response = api.apps.run_forever(app_id, options)
-    else
-      # List the apps
-      response = api.apps.list_cycles(app_id, options)
-      # TODO List the sequence number for this app.
-      display "Most recent cyles of the app:" if type.nil?
-      headings = ["Cycle_id", "State", "Start", "End"]
-      rows = response["cycles"] 
-      rows = rows.map do |row|
-        start_time = DateTime.parse(row["start"]).strftime("%m/%d/%Y %I:%M%p")
-        end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
-        [row["cycle_id"], row["state"], start_time, end_time ] #TODO Pretty print time
-      end
-
-      display TableOutputBuilder.build_table(headings, rows, type)
-      display "Total number of cycles executed: #{response['total']}" if type.nil?
-      return
-    end
-
-    if response["job_id"]
-      options[:job_id] = response["job_id"]
-      app_id = response["flow_id"]
-      options.delete :is_name
-
-      start = Time.now.utc
-      display "Next cycle request sent. If your app was RETIRED this may take slightly longer." if type.nil?
-
-      while(Time.now.utc < start + MAX_POLL_SECONDS) do
-
-        # Poll
-        res = self.api.apps.cycles_poll(app_id, options)
-
-        case res['status']
-        when 'completed'
-          if res['return']
-            if type == "json"
-              return "{}"
-            else
-              display res['return']
-            end
-          else
-            throw "something is wrong: #{res}"
-          end
-          # success! continue below
-          break
-        when 'running'
-          sleep(POLL_SLEEP)
-       #   display ".", false
-        else
-          throw "unknown status: #{res}"
-        end
-
-      end
-    elsif response["error"]
-      error(response["error"], type)
-    else
-      error("remote server error (a380)", type) 
-    end
-  end
-  
-
-  # apps:test [DIR]
-  #
-  # tests a local app with sample data
-  # 
-  # --config CONFIG_FILE  # use the given config file
-  # --output OUTPUT_FILE  # writes sink output to a file
-  # --wait MAX            # max time to spend on each operation (default 10 seconds)
-  # --batches BATCHES     # number of batches to emit (default 1)
-  # --directory DIR       # app directory
-  #
-  def test
-    
-    output = options[:output]
-    otype = options[:type] #type is used below for something else
-
-    max_seconds = (options[:wait] || "30").to_i
-    batches = (options[:batches] || "1").to_i
-
-    def read_message(read_stream, color)
-      msg = nil
-      read_stream.each do |line|
-        line.strip!
-        if(line == "end")
-          return msg
-        end
-        begin
-          hash = JSON.parse(line)
-          if(hash["command"] == "done")
-            msg = "done"
-          else
-            msg = hash.to_json
-          end
-        rescue
-          next
-        end
-      end
-      msg
-    end
-
-    def write_message(write_stream, msg)
-      write_stream.write msg.strip + "\n"
-      write_stream.write "end\n"
-      write_stream.flush
-    end
-
-    def truncate_message(msg)
-      return msg if(!msg.instance_of?(String))
-      t_length = 50 # truncates entries to this length
-      m_length = msg.length
-      msg_out = m_length > t_length ? msg[0..t_length-3]+"..." : msg
-      msg_out
-    end
-
-    def handshake(write_stream, read_stream, node, color)
-      begin
-        write_message write_stream, "{\"pidDir\": \"/tmp\"}\n"
-        read_message read_stream, color # Read to "end\n"
-      rescue Exception => e
-        puts "Error handshaking node: #{node}"
-        raise e
-      end
-    end
-
-
-
-    # INIT
-     dir = options[:directory] || shift_argument
-    if dir.nil?
-      dir = Dir.pwd
-    else
-      dir = File.expand_path(dir)
-    end
-    options[:directory] = dir
-
-    meta = Zillabyte::API::Apps.get_rich_meta_info_from_script(dir, self, {:test => true})
-    if meta.nil?
-      error "this is not a valid zillabyte app directory"
-      exit
-    end    
-
-    # Show the user what we know about their app... 
-    display "inferring your app details..."
-    colors = {}
-    describe_app(meta, colors)
-
-    
-    # Extract the app's information.. 
-    nodes = meta["nodes"]
-    write_to_next_each = []
-    write_queue = []
-    stream_messages = {}
-    default_stream = "_default"
-    
-    # Iterate all nodes sequentially and invoke them in separate processes... 
-    nodes.each do |node|
-
-      # Init 
-      type = node["type"]
-      name = node["name"]
-      consumes = node["consumes"]
-      emits = node["emits"]
-
-      color = colors[name] || :default
-      
-      op_display = lambda do |msg, override_color = nil|
-        display "#{name} - #{msg}".colorize(override_color || color)
-      end
-
-      # A Source? 
-      if type == "source"
-
-        # A source from relation? 
-        if node['matches'] or node["relation"]
-          matches = node['matches'] || (node["relation"]["query"])
-          emits = emits.first #For spouting from a relation, there should only be one emits
-          op_display.call "Grabbing remote data"
-          
-          res = api.query.agnostic(matches)
-          rows = res["rows"]
-          column_aliases = res["column_aliases"]
-          
-          
-          if(rows.nil? or rows.length == 0)
-            raise NameError, "Could not find data that matches your 'matches' clause"
-          end
-          rows.each do |tuple|
-            values = {}
-            meta = {}
-            tuple.each do |k, v|
-              if(k == "id")
-                next
-              elsif(k == "confidence" or k == "since" or k == "source")
-                meta[k] = v
-              else
-                values[k] = v
-              end
-            end
-            read_msg = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
-            values = Hash[values.map{|k, v| [truncate_message(k), truncate_message(v)]}]
-            op_display.call "emitted: #{values} #{meta} to #{emits}"
-            stream_messages[emits] ||= []
-            stream_messages[emits] << read_msg
-          end
-          
-          # Done processing... 
-          next 
-          
-        else
-          
-          # A regular source.. 
-          stream_messages[default_stream] ||= []
-          stream_messages[default_stream] << "{\"command\": \"begin_cycle\"}\n"
-          emits.each {|ss| stream_messages[ss] = []} #initialize streams
-          stream_size_at_last_call_to_next_tuple = Hash[emits.map {|ss| [ss, 0]}] #initialize initial size of streams (all 0)
-            # the above initializations are used to deal with the case where end_cycle_policy == "null_emit"
-          n_batches_emitted = 1
-          end_cycle_received = false
-          last_call_next_tuple = false
- 
-        end
-
-      # An Aggregate?  
-      elsif type == "aggregate"
-        if node['consumes']
-          input_stream = node['consumes']
-        else
-          input_stream = stream_messages.keys.first
-        end
-        messages = stream_messages[input_stream] || []
-        stream_messages[input_stream] = []
-        
-        group_by = node['group_by']
-        group_tuples = {}
-        messages.each do |msg|
-          msg = JSON.parse(msg)
-          tuple = msg["tuple"].to_json
-          meta = msg["meta"].to_json
-          column_aliases = msg["column_aliases"] || {}
-          aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
-          gt = {}
-          group_by.each do |field|
-            field_name = aliases[field] || field
-            gt[field] = msg["tuple"][field_name]
-          end
-          
-          msg_no_brackets = "\"tuple\": #{tuple}, \"meta\": #{meta}, \"column_aliases\": #{column_aliases.to_json}"
-          if group_tuples[gt]
-            group_tuples[gt] << msg_no_brackets
-          else
-            group_tuples[gt] = [msg_no_brackets]
-          end
-        end
-
-        group_tuples.each do |group_tuple, tuples|
-          stream_messages[input_stream] << "{\"command\": \"begin_group\", \"tuple\": #{group_tuple.to_json}, \"meta\":{}}\n"
-          tuples.each do |t|
-            stream_messages[input_stream] << "{\"command\": \"aggregate\", #{t}}\n"
-          end
-          stream_messages[input_stream] << "{\"command\": \"end_group\"}\n"
-        end
-     
-      # A Sink? 
-      elsif type == "sink"
-       
-        if consumes.nil?
-          error "The node #{name} must declare which stream it 'consumes'"
-        end
-        messages = stream_messages[consumes] || []
-        
-        table = Terminal::Table.new :title => name
-        csv_str = CSV.generate do |csv|
-          header_written = false;
-          messages.each do |msg|
-            obj = JSON.parse(msg)
-            if obj['tuple']
-              if header_written == false
-                keys = [obj['tuple'].keys, obj['meta'].keys].flatten
-                csv << keys
-                table << keys
-                table << :separator
-                header_written = true
-              end
-              vals = [obj['tuple'].values, obj['meta'].values].flatten
-              csv << vals
-              table << vals
-            end
-          end
-        end
-        
-        display table.to_s.colorize(color)
-        
-        if output
-          filename = "#{output}.csv"
-          f = File.open(filename, "w")
-          f.write(csv_str)
-          f.close()
-          op_display.call "output written to #{filename}"
-        end
-        
-        next
-      end
-      
-
-      cmd = command("--execute_live --name #{name}", otype, dir)
-      begin
-        
-        # Start the operation... 
-        op_display.call "beginning #{type} #{name}"
-        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
-          begin
-            
-            # Init 
-            handshake stdin, stdout, node, color
-            write_queue = []
-            read_queue = []
-          
-            if consumes.nil?
-              # Assume default stream (this should only happen for the source)
-              stream_name = default_stream
-            else
-              stream_name = consumes
-            end
-            write_queue = stream_messages[stream_name].clone
-            stream_messages.delete(stream_name)
-          
-            # Start writing the messages... 
-            stuff_to_read = false
-            writing_thread = Thread.start do 
-
-              while(true)
-
-                case type
-                when 'source'
-                  break if n_batches_emitted > batches
-                  if write_queue.empty?
-                    sleep 0.5
-                    next
-                  end
-                else
-                  break if write_queue.empty?
-                end
-
-                # Make sure we're not reading anything... 
-                while(stuff_to_read)   # TODO: semaphores
-                  sleep 0.5 # spin wait
-                end
-                
-                # Get next mesage
-                write_msg = write_queue.shift
-                
-                # Make it human-understable
-                write_json = JSON.parse(write_msg)
-                if write_json['tuple']
-                  display_hash = Hash[write_json['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
-                  op_display.call "receiving: #{display_hash}"
-                elsif write_json['command'] == 'next'
-                  last_call_next_tuple = true
-                  op_display.call "getting next set of tuples in the batch"
-                else
-                  puts write_json
-                end
-                
-                # Actually send it to the process 
-                begin
-                  write_message stdin, write_msg
-                  stuff_to_read = true
-                  sleep 0.1
-                rescue Exception => e
-                  puts "Error running #{cmd}: #{e}"
-                  raise e
-                end
-              end
-            end
-            
-            # Start reading messages... 
-            reading_thread = Thread.start do 
-              while(true)
-
-                # If the end cycle command is received, we either trigger the next cycle if the number of emitted
-                # cycles is less than what the user requested, or we break
-                if type == "source" and end_cycle_received
-                  write_queue << "{\"command\": \"begin_cycle\"}\n"
-                  n_batches_emitted += 1
-                  end_cycle_received = false
-                  last_call_next_tuple = false
-                  stuff_to_read = false
-                  break if n_batches_emitted > batches
-                  sleep 0.5
-                  next
-                end
-
-                # Get next message
-                read_msg = read_message(stdout, color)
-                if read_msg == "done" || read_msg.nil?
-                  stuff_to_read = false
-
-                  # For sources, if we receive a "done", check to see if any of the streams emitted by the source has
-                  # increased in size since the last call to next_tuple. If so, the cycle isn't over, otherwise, the
-                  # current call to next_tuple emitted nothing and if the end_cycle_policy is set to null_emit, this
-                  # should end the current cycle.
-                  if type == "source"
-                    if last_call_next_tuple and node["end_cycle_policy"] == "null_emit"
-                      end_cycle_received = true
-                      emits.each do |ss|
-                        end_cycle_received = false if stream_messages[ss].size > stream_size_at_last_call_to_next_tuple[ss]
-                        break
-                      end
-                      next if end_cycle_received
-                    end 
-
-                    # If the policy isn't "null_emit", then just request next_tuple again
-                    write_queue << "{\"command\": \"next\"}\n"
-                  end
-  
-                  # For other operations, if the queue is empty then we're done
-                  if write_queue.empty?
-                    break  # exit while loop
-                  else
-                    sleep 0.5 # spin wait
-                    next
-                  end
-                end
-                stuff_to_read = true
-                
-                # Process message
-                obj = JSON.parse(read_msg)
-   
-                # process the received tuple or other commands
-                if obj['tuple']
-                 
-                  # if 
-                  tt = obj['tuple']
-                  tt.each do |kk, vv|
-                    if tt[kk].nil? and type == "source" and node["end_cycle_policy"] == "null_emit"
-                      end_cycle_received = true
-                      # read rest of stuff in stdout buffer until "done"
-                      mm = nil
-                      while(mm != "done")
-                        mm = read_message(stdout, color)
-                      end
-                      break
-                    end
-                  end
-                  next if end_cycle_received
-                    
-                  # Convert to a incoming tuple for the next operation
-                  next_msg = {
-                    :tuple => obj['tuple'],
-                    :meta => obj['meta']
-                  }
-                  emit_stream = obj['stream']
-                  stream_messages[emit_stream] ||= []
-                  stream_messages[emit_stream] << next_msg.to_json
-
-                  display_hash = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
-                  op_display.call "emitted: #{display_hash} to #{emit_stream}"
-
-                  # track stream message size to end cycles when necessary
-                  stream_size_at_last_call_to_next_tuple[emit_stream] = stream_messages[emit_stream].size if type == "source"
-                elsif obj['command'] == 'end_cycle'
-                  end_cycle_received = true
-                  # command:end_cycle should always be followed by done, read it (below) so 
-                  # that it doesn't interfere with next
-                  read_message(stdout, color)
-                elsif obj['command'] == 'log'
-                  op_display.call "log: #{obj['msg']}"
-                elsif obj['command'] == 'fail'
-                  op_display.call "error: #{obj['msg']}", :red
-                  exit(1)
-                else
-                  error "unknown message: #{read_msg}"
-                end
-                
-              end
-            end
-            
-            # stderr thread 
-            stderr_thread = Thread.start do 
-              stderr.each do |line|
-                op_display.call("stderr: #{line}", :red)
-              end
-            end
-            
-            begin
-              killed = Timeout.timeout(max_seconds) do
-                reading_thread.join()
-                writing_thread.join()
-                stderr_thread.kill()
-                op_display.call "completed #{type} #{name}"
-              end
-            rescue Timeout::Error
-              op_display.call "max time reached. preempting #{type} #{name}. set --wait to increase", :red
-              reading_thread.kill() if reading_thread.alive?
-              writing_thread.kill() if writing_thread.alive?
-              stderr_thread.kill() if stderr_thread.alive?
-            end
-
-          rescue Errno::EIO
-            puts "Errno:EIO error, but this probably just means " +
-                 "that the process has finished giving output"
-          end
-        end
-      rescue PTY::ChildExited
-        puts "The child process exited!"
-      end
-    end
-    
-  end  
-  alias_command "test", "apps:test"
-  
-  
-  # apps:kill ID
-  #
-  # kills the given app
-  # 
-  # --config CONFIG_FILE # use the given config file
-  # --type TYPE # specify an output type i.e. json
-  #
-  def kill
-
-    id = options[:id] || shift_argument
-    type = options[:type]
-
-    if id.nil?
-      id = read_name_from_conf(options)
-      options[:is_name] = true
-    elsif !(id =~ /^\d*$/)
-      options[:is_name] = true
-    end
-
-    display "Killing app ##{id}...please wait..." if type.nil?
-    api.apps.kill(id, options)
-
-    if type == "json"
-      display "{}"
-    else
-      display "App ##{id} killed"
-    end
-
-  end
-  
-
-  # apps:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]
-  #
-  # runs a local app with live data
-  # 
-  # --config CONFIG_FILE # use the given config file
-  # --type TYPE # specify an output type i.e. json
-  # --directory DIR # Directory of the app
-  #
-  # HIDDEN:
-  def live_run
-
-    name = options[:name] || shift_argument
-    type = options[:type]
-
-    thread_id = options[:thread] || shift_argument || ""
-    dir = options[:directory] || shift_argument
-    if dir.nil?
-      dir = Dir.pwd
-    else
-      dir = File.expand_path(dir)
-    end
-    options[:directory] = dir
-
-    meta = Zillabyte::CLI::Config.get_config_info(dir, options)
-  
-    if meta.nil?
-      error("could not find meta information for: #{dir}", type)
-    end
-   
-    if(thread_id == "")
-      exec(command("--execute_live --name #{name.to_s}",type, dir))
-    else 
-      exec(command("--execute_live --name #{name.to_s} --pipe #{thread_id}",type, dir))
-    end
-  end  
-  alias_command "live_run", "apps:live_run"
-  
-  
-  # apps:info [DIR]
-  # 
-  # outputs the info for the app in the dir.
-  # 
-  # --pretty # Pretty prints the info output
-  # --type TYPE # specify an output type i.e. json
-  # --directory DIR # Directory of the app
-  #
-  def info
-
-    dir = options[:directory] || shift_argument
-    if dir.nil?
-      dir = Dir.pwd
-    else
-      dir = File.expand_path(dir)
-    end
-    options[:directory] = dir
-
-    info_file = "#{dir}/#{SecureRandom.uuid}"
-    type = options[:type]
-
-    cmd = command("--info --file #{info_file}", type, dir)
-<<<<<<< HEAD
-    app_info = Zillabyte::Command::Apps.get_info(cmd, info_file, dir)
-=======
-    app_info = Zillabyte::Command::Apps.get_info(cmd, info_file, options)
->>>>>>> b7501a641428a0952068d1837b20293619aba0cd
-
-    if type == "json"
-      puts app_info
-    else
-      if options[:pretty]
-        puts JSON.pretty_generate(JSON.parse(app_info))
-      else
-        puts app_info
-      end
-    end
-
-    exit
-  end
-  alias_command "info", "apps:info"
-
-
-  #
-  # --type TYPE # specify an output type i.e. json
-  #
-<<<<<<< HEAD
-  def self.get_info(cmd, info_file, dir, options = {})
-=======
-  def self.get_info(cmd, info_file, options = {})
-
->>>>>>> b7501a641428a0952068d1837b20293619aba0cd
-    type = options[:type]
-
-    response = `#{cmd}`
-    if($?.exitstatus == 1)
-
-      File.delete("#{info_file}") if File.exists?(info_file)
-
-      if options[:type].nil?
-        exit(1)
-      else
-        Zillabyte::Helpers.error("error: #{response}", type)
-      end
-    end
-
-    app_info = {}
-    File.open("#{info_file}", "r+").each do |line|
-      line = JSON.parse(line)
-      if(line["type"])
-        app_info["nodes"] << line
-      else
-        app_info = line
-        app_info["nodes"] = []
-      end
-    end
-    File.delete("#{info_file}")
-
-    app_info = app_info.to_json
-    if(File.exists?("#{dir}/info_to_java.in"))
-      java_pipe = open("#{dir}/info_to_java.in","w+")
-      java_pipe.puts(app_info+"\n")
-      java_pipe.flush
-      java_pipe.close()
-    end
-
-    app_info
-  end
-
-
-  private 
-  
-  #
-  # --type TYPE # specify an output type i.e. json
-  #
-  def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false)
-    meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
-
-    #meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
-    error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
-    error(meta["error_message"], type) if meta['status'] == "error"
-    full_script = File.join(dir, meta["script"])
-    stderr_opt = "2> /dev/null" if ignore_stderr
-
-    case meta["language"]
-    when "ruby"
-      # Execute in the bundler context
-      cmd = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}  #{stderr_opt}"
-      
-    when "python"#{
-      if(File.directory?("#{dir}/vEnv"))
-        cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}  #{stderr_opt}"
-      else
-        cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}  #{stderr_opt}"
-      end
- 
-    when "js"
-#      cmd = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS}  \"#{full_script}\" #{arg}"
-      cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN}  \"#{full_script}\" #{arg} #{stderr_opt}"
-
-    else
-      error("no language specified", type)
-    end
-    
-    return cmd
-
-  end
-  
-  #
-  #
-  #
-  def describe_app(meta, colors = {})
-    @colors ||= [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
-    rjust = 20
-    display "#{'app name'.rjust(rjust)}: #{meta['name']}"
-    display "#{'app language'.rjust(rjust)}: #{meta['language']}"
-    
-    meta['nodes'].each_with_index do |node, index|
-      colors[node['name']] ||= @colors.shift
-      color = colors[node['name']]
-      display (("="*rjust + "  operation ##{index}").colorize(color))
-      display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
-      display "#{"type".rjust(rjust)}: #{node['type'].to_s.colorize(color)}"
-      display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
-      display "#{"emits".rjust(rjust)}: #{JSON.pretty_generate(node['emits']).indent(rjust+2).lstrip.colorize(color)}" if node['emits']
-    end
-    
-  end
-
-  #
-  #
-  def read_name_from_conf(options = {})
-    type = options[:type]
-    hash = Zillabyte::API::Apps.get_rich_meta_info_from_script Dir.pwd, options
-    error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify an app id or run command from the directory containing the app.",type) if hash["error"]
-    hash["name"]
-  end
-
-end