zillabyte-cli 0.0.20 → 0.0.21

data/lib/zillabyte/cli/apps.rb.orig

@@ -0,0 +1,1158 @@
+require "zillabyte/cli/base"
+require "zillabyte/cli/config"
+require "zillabyte/common"
+require "pty"
+require 'indentation'
+require 'open3'
+require 'securerandom'
+require 'colorize'
+require 'time_difference'
+
+# manage custom apps
+#
+class Zillabyte::Command::Apps < Zillabyte::Command::Base
+
+  MAX_POLL_SECONDS = 60 * 5
+  POLL_SLEEP = 0.5
+
+  # apps
+  #
+  # list custom apps
+  # --type TYPE     # specify an output type i.e. json
+  #
+  def index
+    self.list
+  end
+
+  # apps
+  #
+  # list custom apps
+  # --type TYPE # specify an output type i.e. json
+  def list
+    type = options[:type]
+    
+    headings = ["id", "name", "state", "cycles"]
+    rows = api.app.list.map do |row|
+      if headings.size == 0
+        headings = row.keys 
+        headings.delete("rel_dir")
+      end
+      v = row.values_at *headings
+      v
+    end
+
+    display "apps:\n" if type.nil?
+    display TableOutputBuilder.build_table(headings, rows, type)
+    display "Total number of apps: "+rows.length.to_s if type.nil?
+  end
+
+
+  # apps:push [DIR]
+  #
+  # uploads an app
+  # 
+  # --config CONFIG_FILE # use the given config file
+  # --type TYPE # specify an output type i.e. json
+  # --directory DIR # app directory
+  #
+  #Examples:
+  #
+  # $ zillabyte apps:push . 
+  #
+  def push
+    
+    since = Time.now.utc.to_s
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+    type = options[:type]
+
+    res = api.apps.push_directory dir, session, options
+    
+    if res['error']
+      error("error: #{res['error_message']}", type)
+    else
+      display "app ##{res['id']} #{res['action']}" if type.nil?
+    end
+
+    display "Starting up your app...please wait..." if type.nil?
+    sleep(2) # wait for kill command 
+    
+    lf = LogFormatter::Startup.new
+    api.logs.get_startup(res['id'], "_ALL_", {:push => true}) do |hash|
+      
+      # Error?
+      error(hash['error_message'], type) if hash['error']
+      
+      # Print it
+      lf.print_log_line(hash) if type.nil?
+      
+      # Exit when we get the 'done' message
+      # exit(0)  if (hash['line'] || '').downcase.include?("app deployed")
+      
+    end
+    display "{}" if type == "json"
+      
+  end
+  alias_command "push", "apps:push"
+  
+  
+  
+  # apps:pull ID DIR
+  #
+  # pulls an app source to a directory.
+  #
+  # --force # pulls even if the directory exists
+  # --type TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the app
+  #
+  #Examples:
+  #
+  # $ zillabyte apps:pull . 
+  #
+  def pull
+
+    app_id = options[:id] || shift_argument
+
+    if !(app_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    dir = options[:directory] || shift_argument
+    error("no directory given", type) if dir.nil?
+    dir = File.expand_path(dir)
+
+    type = options[:type]
+    
+    error("no id given", type) if app_id.nil?
+    
+    # Create if not exists..
+    if File.exists?(dir)
+      if Dir.entries(dir).size != 2 and options[:force].nil?
+        error("target directory not empty. use --force to override", type)
+      end
+    else
+      FileUtils.mkdir_p(dir)
+    end
+    
+    res = api.apps.pull_to_directory app_id, dir, session, options
+    
+    if res['error']
+      error("error: #{res['error_message']}", type)
+    else
+      if type == "json"
+        display "{}"
+      else
+        display "app ##{res['id']} pulled to #{dir}"
+      end
+    end
+    
+  end
+  alias_command "pull", "apps:pull"
+
+
+  # apps:delete ID
+  #
+  # deletes an app. if the app is running, this command will kill it. 
+  #
+  # -f, --force # don't ask for confirmation
+  # --type TYPE # specify an output type i.e. json
+  #
+  def delete
+    app_id = options[:id] || shift_argument
+
+    if app_id.nil?
+      app_id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(app_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+    forced = options[:force]
+    type = options[:type]
+
+    if not forced
+
+      if !type.nil?
+        error("specify -f, --force to confirm deletion", type)
+      end
+
+      while true
+        
+        display "This operation cannot be undone. Are you sure you want to delete this app? (yes/no):", false
+        confirm = ask
+        break if confirm == "yes" || confirm == "no"
+        display "Please enter 'yes' to delete the app or 'no' to exit"
+      end
+    end
+
+    confirmed = forced || confirm == "yes"
+
+    if confirmed
+      response = api.apps.delete(app_id, options)
+      if type == "json"
+        display "{}"
+      else
+        display response["body"]
+      end
+    end
+
+  end
+
+  
+  # apps:prep [DIR]
+  #
+  # --directory DIR       # app directory
+  # --type TYPE           # specify an output type i.e. json
+  #
+  def prep
+
+    type = options[:type]
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+    meta = Zillabyte::CLI::Config.get_config_info(dir)
+
+    if meta.nil?
+      error("The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file.", type)
+    end
+
+    case meta["language"]
+    when "ruby"
+
+      # Execute in the bundler context
+      full_script = File.join(dir, meta["script"])
+      cmd = "cd \"#{meta['home_dir']}\"; unset BUNDLE_GEMFILE; unset RUBYOPT; bundle install"
+      exec(cmd)
+      
+    when "python"
+      vDir = "#{meta['home_dir']}/vEnv"
+      lock_file = meta['home_dir']+"/zillabyte_thread_lock_file"
+      if File.exists?(lock_file)
+        sleep(1) while File.exists?(lock_file)
+      else
+        begin
+          cmd = "touch #{lock_file}; virtualenv --clear --system-site-packages #{vDir}; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{vDir}/bin/pip install -r #{meta['home_dir']}/requirements.txt"
+          system cmd, :out => :out
+        ensure
+          File.delete(lock_file)
+        end
+      end
+      
+    when "js"
+    end
+    
+  end
+  alias_command "prep", "apps:prep"
+
+
+
+  
+  # apps:init [LANG] [DIR]
+  #
+  # initializes a new executable in DIR
+  # [LANG] defaults to ruby, and [DIR] to the current directory
+  #
+  # --type TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the app
+  #
+  #Examples:
+  #
+  # $ zillabyte apps:init python contact_extractor
+  #
+  def init
+    
+    lang = options[:lang] || shift_argument || "ruby"
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    type = options[:type]
+
+    languages = ["ruby","python", "js"]
+
+    error("Unsupported language #{lang}. We only support #{languages.join(', ')}.", type) if not languages.include? lang
+
+    display "initializing empty #{lang} app in #{dir}" if type.nil?
+    FileUtils.cp_r( File.expand_path("../templates/#{lang}", __FILE__) + "/." , dir )
+    
+    
+  end
+
+
+
+  # apps:logs ID [OPERATION_NAME]
+  #
+  # streams logs from the distributed workers
+  # 
+  # --type TYPE # specify an output type i.e. json
+  # -v, --verbose LEVEL  # sets the verbosity (error, info, debug) (default: info)
+  #
+  def logs
+
+    app_id = options[:id] || shift_argument
+    if app_id.nil?
+      app_id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(app_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    operation_id = options[:operation] || shift_argument || '_ALL_'
+    category = options[:verbose] || '_ALL_'
+    type = options[:type]
+
+    carry_settings = {
+      :category => category
+    }
+
+    display "Retrieving logs for app ##{app_id}...please wait..." if type.nil?
+    lf = LogFormatter::Operation.new
+    self.api.logs.get(app_id, operation_id, options) do |line|
+      
+      error(line['error_message'], type) if line['error']
+      lf.print_log_line(line) if type.nil?
+      
+    end
+
+  end  
+  alias_command "logs", "apps:logs"
+
+
+
+  # apps:errors ID
+  #
+  # Show recent errors generated by the app
+  # --type TYPE           # specify an output type i.e. json
+  #
+  def errors
+    
+    # Init
+    app_id = options[:id] || shift_argument
+    
+    # No name?
+    if app_id.nil?
+      app_id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(app_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    type = options[:type]
+
+    # Make the request
+    res = api.request(
+      :expects => 200, 
+      :method => :get,
+      :body    => options.to_json,
+      :path => "/flows/#{CGI.escape(app_id)}/errors"
+    )
+    
+    # Render
+    display "Recent errors:" if type.nil?
+    headings = ["operation", "date", "error"]
+    rows = (res.body["recent_errors"] || []).map do |row|
+      if row['date'] 
+        d = Time.at(row['date']/1000)
+      else
+        d = nil
+      end
+      [row['name'], d, row['message']]
+    end
+    rows.sort! do |a,b|
+      a[1] <=> b[1]
+    end
+    color_map = {}
+    colors = LogFormatter::COLORS.clone
+    rows.each do |row|
+      name = row[0]
+      time = row[1]
+      message = row[2].strip
+      color_map[name] ||= colors.shift
+      if time 
+        display "#{"* #{name} - #{time_ago_in_words(time)} ago".colorize(color_map[name])}:" if type.nil?
+      else
+        display "#{"* #{name}".colorize(color_map[name])}:" if type.nil?
+      end
+      message.split('\n').each do |sub_line|
+        display "    #{sub_line}" if type.nil?
+      end
+    end
+    
+  end
+
+  # apps:cycles ID [OPTIONS]
+  #  
+  # operations on the app's cycles (batches).
+  # with no options, the command lists the apps cycles
+  # -n, --next            # request the app to move to the next cycle
+  # -f, --forever         # don't wait on cycles any more
+  # --type TYPE           # specify an output type i.e. json
+  #
+  def cycles
+    app_id = options[:id] || shift_argument
+    type = options[:type]
+
+    trigger_next = options[:next] || false
+    trigger_forever = options[:forever] || false
+
+    if app_id.nil?
+      app_id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(app_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    if trigger_next
+      # Trigger the next app
+      response = api.apps.create_cycle(app_id, options)
+    elsif trigger_forever
+      response = api.apps.run_forever(app_id, options)
+    else
+      # List the apps
+      response = api.apps.list_cycles(app_id, options)
+      # TODO List the sequence number for this app.
+      display "Most recent cyles of the app:" if type.nil?
+      headings = ["Cycle_id", "State", "Start", "End"]
+      rows = response["cycles"] 
+      rows = rows.map do |row|
+        start_time = DateTime.parse(row["start"]).strftime("%m/%d/%Y %I:%M%p")
+        end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
+        [row["cycle_id"], row["state"], start_time, end_time ] #TODO Pretty print time
+      end
+
+      display TableOutputBuilder.build_table(headings, rows, type)
+      display "Total number of cycles executed: #{response['total']}" if type.nil?
+      return
+    end
+
+    if response["job_id"]
+      options[:job_id] = response["job_id"]
+      app_id = response["flow_id"]
+      options.delete :is_name
+
+      start = Time.now.utc
+      display "Next cycle request sent. If your app was RETIRED this may take slightly longer." if type.nil?
+
+      while(Time.now.utc < start + MAX_POLL_SECONDS) do
+
+        # Poll
+        res = self.api.apps.cycles_poll(app_id, options)
+
+        case res['status']
+        when 'completed'
+          if res['return']
+            if type == "json"
+              return "{}"
+            else
+              display res['return']
+            end
+          else
+            throw "something is wrong: #{res}"
+          end
+          # success! continue below
+          break
+        when 'running'
+          sleep(POLL_SLEEP)
+       #   display ".", false
+        else
+          throw "unknown status: #{res}"
+        end
+
+      end
+    elsif response["error"]
+      error(response["error"], type)
+    else
+      error("remote server error (a380)", type) 
+    end
+  end
+  
+
+  # apps:test [DIR]
+  #
+  # tests a local app with sample data
+  # 
+  # --config CONFIG_FILE  # use the given config file
+  # --output OUTPUT_FILE  # writes sink output to a file
+  # --wait MAX            # max time to spend on each operation (default 10 seconds)
+  # --batches BATCHES     # number of batches to emit (default 1)
+  # --directory DIR       # app directory
+  #
+  def test
+    
+    output = options[:output]
+    otype = options[:type] #type is used below for something else
+
+    max_seconds = (options[:wait] || "30").to_i
+    batches = (options[:batches] || "1").to_i
+
+    def read_message(read_stream, color)
+      msg = nil
+      read_stream.each do |line|
+        line.strip!
+        if(line == "end")
+          return msg
+        end
+        begin
+          hash = JSON.parse(line)
+          if(hash["command"] == "done")
+            msg = "done"
+          else
+            msg = hash.to_json
+          end
+        rescue
+          next
+        end
+      end
+      msg
+    end
+
+    def write_message(write_stream, msg)
+      write_stream.write msg.strip + "\n"
+      write_stream.write "end\n"
+      write_stream.flush
+    end
+
+    def truncate_message(msg)
+      return msg if(!msg.instance_of?(String))
+      t_length = 50 # truncates entries to this length
+      m_length = msg.length
+      msg_out = m_length > t_length ? msg[0..t_length-3]+"..." : msg
+      msg_out
+    end
+
+    def handshake(write_stream, read_stream, node, color)
+      begin
+        write_message write_stream, "{\"pidDir\": \"/tmp\"}\n"
+        read_message read_stream, color # Read to "end\n"
+      rescue Exception => e
+        puts "Error handshaking node: #{node}"
+        raise e
+      end
+    end
+
+
+
+    # INIT
+     dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+
+    meta = Zillabyte::API::Apps.get_rich_meta_info_from_script(dir, self, {:test => true})
+    if meta.nil?
+      error "this is not a valid zillabyte app directory"
+      exit
+    end    
+
+    # Show the user what we know about their app... 
+    display "inferring your app details..."
+    colors = {}
+    describe_app(meta, colors)
+
+    
+    # Extract the app's information.. 
+    nodes = meta["nodes"]
+    write_to_next_each = []
+    write_queue = []
+    stream_messages = {}
+    default_stream = "_default"
+    
+    # Iterate all nodes sequentially and invoke them in separate processes... 
+    nodes.each do |node|
+
+      # Init 
+      type = node["type"]
+      name = node["name"]
+      consumes = node["consumes"]
+      emits = node["emits"]
+
+      color = colors[name] || :default
+      
+      op_display = lambda do |msg, override_color = nil|
+        display "#{name} - #{msg}".colorize(override_color || color)
+      end
+
+      # A Source? 
+      if type == "source"
+
+        # A source from relation? 
+        if node['matches'] or node["relation"]
+          matches = node['matches'] || (node["relation"]["query"])
+          emits = emits.first #For spouting from a relation, there should only be one emits
+          op_display.call "Grabbing remote data"
+          
+          res = api.query.agnostic(matches)
+          rows = res["rows"]
+          column_aliases = res["column_aliases"]
+          
+          
+          if(rows.nil? or rows.length == 0)
+            raise NameError, "Could not find data that matches your 'matches' clause"
+          end
+          rows.each do |tuple|
+            values = {}
+            meta = {}
+            tuple.each do |k, v|
+              if(k == "id")
+                next
+              elsif(k == "confidence" or k == "since" or k == "source")
+                meta[k] = v
+              else
+                values[k] = v
+              end
+            end
+            read_msg = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
+            values = Hash[values.map{|k, v| [truncate_message(k), truncate_message(v)]}]
+            op_display.call "emitted: #{values} #{meta} to #{emits}"
+            stream_messages[emits] ||= []
+            stream_messages[emits] << read_msg
+          end
+          
+          # Done processing... 
+          next 
+          
+        else
+          
+          # A regular source.. 
+          stream_messages[default_stream] ||= []
+          stream_messages[default_stream] << "{\"command\": \"begin_cycle\"}\n"
+          emits.each {|ss| stream_messages[ss] = []} #initialize streams
+          stream_size_at_last_call_to_next_tuple = Hash[emits.map {|ss| [ss, 0]}] #initialize initial size of streams (all 0)
+            # the above initializations are used to deal with the case where end_cycle_policy == "null_emit"
+          n_batches_emitted = 1
+          end_cycle_received = false
+          last_call_next_tuple = false
+ 
+        end
+
+      # An Aggregate?  
+      elsif type == "aggregate"
+        if node['consumes']
+          input_stream = node['consumes']
+        else
+          input_stream = stream_messages.keys.first
+        end
+        messages = stream_messages[input_stream] || []
+        stream_messages[input_stream] = []
+        
+        group_by = node['group_by']
+        group_tuples = {}
+        messages.each do |msg|
+          msg = JSON.parse(msg)
+          tuple = msg["tuple"].to_json
+          meta = msg["meta"].to_json
+          column_aliases = msg["column_aliases"] || {}
+          aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
+          gt = {}
+          group_by.each do |field|
+            field_name = aliases[field] || field
+            gt[field] = msg["tuple"][field_name]
+          end
+          
+          msg_no_brackets = "\"tuple\": #{tuple}, \"meta\": #{meta}, \"column_aliases\": #{column_aliases.to_json}"
+          if group_tuples[gt]
+            group_tuples[gt] << msg_no_brackets
+          else
+            group_tuples[gt] = [msg_no_brackets]
+          end
+        end
+
+        group_tuples.each do |group_tuple, tuples|
+          stream_messages[input_stream] << "{\"command\": \"begin_group\", \"tuple\": #{group_tuple.to_json}, \"meta\":{}}\n"
+          tuples.each do |t|
+            stream_messages[input_stream] << "{\"command\": \"aggregate\", #{t}}\n"
+          end
+          stream_messages[input_stream] << "{\"command\": \"end_group\"}\n"
+        end
+     
+      # A Sink? 
+      elsif type == "sink"
+       
+        if consumes.nil?
+          error "The node #{name} must declare which stream it 'consumes'"
+        end
+        messages = stream_messages[consumes] || []
+        
+        table = Terminal::Table.new :title => name
+        csv_str = CSV.generate do |csv|
+          header_written = false;
+          messages.each do |msg|
+            obj = JSON.parse(msg)
+            if obj['tuple']
+              if header_written == false
+                keys = [obj['tuple'].keys, obj['meta'].keys].flatten
+                csv << keys
+                table << keys
+                table << :separator
+                header_written = true
+              end
+              vals = [obj['tuple'].values, obj['meta'].values].flatten
+              csv << vals
+              table << vals
+            end
+          end
+        end
+        
+        display table.to_s.colorize(color)
+        
+        if output
+          filename = "#{output}.csv"
+          f = File.open(filename, "w")
+          f.write(csv_str)
+          f.close()
+          op_display.call "output written to #{filename}"
+        end
+        
+        next
+      end
+      
+
+      cmd = command("--execute_live --name #{name}", otype, dir)
+      begin
+        
+        # Start the operation... 
+        op_display.call "beginning #{type} #{name}"
+        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
+          begin
+            
+            # Init 
+            handshake stdin, stdout, node, color
+            write_queue = []
+            read_queue = []
+          
+            if consumes.nil?
+              # Assume default stream (this should only happen for the source)
+              stream_name = default_stream
+            else
+              stream_name = consumes
+            end
+            write_queue = stream_messages[stream_name].clone
+            stream_messages.delete(stream_name)
+          
+            # Start writing the messages... 
+            stuff_to_read = false
+            writing_thread = Thread.start do 
+
+              while(true)
+
+                case type
+                when 'source'
+                  break if n_batches_emitted > batches
+                  if write_queue.empty?
+                    sleep 0.5
+                    next
+                  end
+                else
+                  break if write_queue.empty?
+                end
+
+                # Make sure we're not reading anything... 
+                while(stuff_to_read)   # TODO: semaphores
+                  sleep 0.5 # spin wait
+                end
+                
+                # Get next mesage
+                write_msg = write_queue.shift
+                
+                # Make it human-understable
+                write_json = JSON.parse(write_msg)
+                if write_json['tuple']
+                  display_hash = Hash[write_json['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
+                  op_display.call "receiving: #{display_hash}"
+                elsif write_json['command'] == 'next'
+                  last_call_next_tuple = true
+                  op_display.call "getting next set of tuples in the batch"
+                else
+                  puts write_json
+                end
+                
+                # Actually send it to the process 
+                begin
+                  write_message stdin, write_msg
+                  stuff_to_read = true
+                  sleep 0.1
+                rescue Exception => e
+                  puts "Error running #{cmd}: #{e}"
+                  raise e
+                end
+              end
+            end
+            
+            # Start reading messages... 
+            reading_thread = Thread.start do 
+              while(true)
+
+                # If the end cycle command is received, we either trigger the next cycle if the number of emitted
+                # cycles is less than what the user requested, or we break
+                if type == "source" and end_cycle_received
+                  write_queue << "{\"command\": \"begin_cycle\"}\n"
+                  n_batches_emitted += 1
+                  end_cycle_received = false
+                  last_call_next_tuple = false
+                  stuff_to_read = false
+                  break if n_batches_emitted > batches
+                  sleep 0.5
+                  next
+                end
+
+                # Get next message
+                read_msg = read_message(stdout, color)
+                if read_msg == "done" || read_msg.nil?
+                  stuff_to_read = false
+
+                  # For sources, if we receive a "done", check to see if any of the streams emitted by the source has
+                  # increased in size since the last call to next_tuple. If so, the cycle isn't over, otherwise, the
+                  # current call to next_tuple emitted nothing and if the end_cycle_policy is set to null_emit, this
+                  # should end the current cycle.
+                  if type == "source"
+                    if last_call_next_tuple and node["end_cycle_policy"] == "null_emit"
+                      end_cycle_received = true
+                      emits.each do |ss|
+                        end_cycle_received = false if stream_messages[ss].size > stream_size_at_last_call_to_next_tuple[ss]
+                        break
+                      end
+                      next if end_cycle_received
+                    end 
+
+                    # If the policy isn't "null_emit", then just request next_tuple again
+                    write_queue << "{\"command\": \"next\"}\n"
+                  end
+  
+                  # For other operations, if the queue is empty then we're done
+                  if write_queue.empty?
+                    break  # exit while loop
+                  else
+                    sleep 0.5 # spin wait
+                    next
+                  end
+                end
+                stuff_to_read = true
+                
+                # Process message
+                obj = JSON.parse(read_msg)
+   
+                # process the received tuple or other commands
+                if obj['tuple']
+                 
+                  # if 
+                  tt = obj['tuple']
+                  tt.each do |kk, vv|
+                    if tt[kk].nil? and type == "source" and node["end_cycle_policy"] == "null_emit"
+                      end_cycle_received = true
+                      # read rest of stuff in stdout buffer until "done"
+                      mm = nil
+                      while(mm != "done")
+                        mm = read_message(stdout, color)
+                      end
+                      break
+                    end
+                  end
+                  next if end_cycle_received
+                    
+                  # Convert to a incoming tuple for the next operation
+                  next_msg = {
+                    :tuple => obj['tuple'],
+                    :meta => obj['meta']
+                  }
+                  emit_stream = obj['stream']
+                  stream_messages[emit_stream] ||= []
+                  stream_messages[emit_stream] << next_msg.to_json
+
+                  display_hash = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
+                  op_display.call "emitted: #{display_hash} to #{emit_stream}"
+
+                  # track stream message size to end cycles when necessary
+                  stream_size_at_last_call_to_next_tuple[emit_stream] = stream_messages[emit_stream].size if type == "source"
+                elsif obj['command'] == 'end_cycle'
+                  end_cycle_received = true
+                  # command:end_cycle should always be followed by done, read it (below) so 
+                  # that it doesn't interfere with next
+                  read_message(stdout, color)
+                elsif obj['command'] == 'log'
+                  op_display.call "log: #{obj['msg']}"
+                elsif obj['command'] == 'fail'
+                  op_display.call "error: #{obj['msg']}", :red
+                  exit(1)
+                else
+                  error "unknown message: #{read_msg}"
+                end
+                
+              end
+            end
+            
+            # stderr thread 
+            stderr_thread = Thread.start do 
+              stderr.each do |line|
+                op_display.call("stderr: #{line}", :red)
+              end
+            end
+            
+            begin
+              killed = Timeout.timeout(max_seconds) do
+                reading_thread.join()
+                writing_thread.join()
+                stderr_thread.kill()
+                op_display.call "completed #{type} #{name}"
+              end
+            rescue Timeout::Error
+              op_display.call "max time reached. preempting #{type} #{name}. set --wait to increase", :red
+              reading_thread.kill() if reading_thread.alive?
+              writing_thread.kill() if writing_thread.alive?
+              stderr_thread.kill() if stderr_thread.alive?
+            end
+
+          rescue Errno::EIO
+            puts "Errno:EIO error, but this probably just means " +
+                 "that the process has finished giving output"
+          end
+        end
+      rescue PTY::ChildExited
+        puts "The child process exited!"
+      end
+    end
+    
+  end  
+  alias_command "test", "apps:test"
+  
+  
+  # apps:kill ID
+  #
+  # kills the given app
+  # 
+  # --config CONFIG_FILE # use the given config file
+  # --type TYPE # specify an output type i.e. json
+  #
+  def kill
+
+    id = options[:id] || shift_argument
+    type = options[:type]
+
+    if id.nil?
+      id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    display "Killing app ##{id}...please wait..." if type.nil?
+    api.apps.kill(id, options)
+
+    if type == "json"
+      display "{}"
+    else
+      display "App ##{id} killed"
+    end
+
+  end
+  
+
+  # apps:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]
+  #
+  # runs a local app with live data
+  # 
+  # --config CONFIG_FILE # use the given config file
+  # --type TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the app
+  #
+  # HIDDEN:
+  def live_run
+
+    name = options[:name] || shift_argument
+    type = options[:type]
+
+    thread_id = options[:thread] || shift_argument || ""
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+
+    meta = Zillabyte::CLI::Config.get_config_info(dir, options)
+  
+    if meta.nil?
+      error("could not find meta information for: #{dir}", type)
+    end
+   
+    if(thread_id == "")
+      exec(command("--execute_live --name #{name.to_s}",type, dir))
+    else 
+      exec(command("--execute_live --name #{name.to_s} --pipe #{thread_id}",type, dir))
+    end
+  end  
+  alias_command "live_run", "apps:live_run"
+  
+  
+  # apps:info [DIR]
+  # 
+  # outputs the info for the app in the dir.
+  # 
+  # --pretty # Pretty prints the info output
+  # --type TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the app
+  #
+  def info
+
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+
+    info_file = "#{dir}/#{SecureRandom.uuid}"
+    type = options[:type]
+
+    cmd = command("--info --file #{info_file}", type, dir)
+<<<<<<< HEAD
+    app_info = Zillabyte::Command::Apps.get_info(cmd, info_file, dir)
+=======
+    app_info = Zillabyte::Command::Apps.get_info(cmd, info_file, options)
+>>>>>>> b7501a641428a0952068d1837b20293619aba0cd
+
+    if type == "json"
+      puts app_info
+    else
+      if options[:pretty]
+        puts JSON.pretty_generate(JSON.parse(app_info))
+      else
+        puts app_info
+      end
+    end
+
+    exit
+  end
+  alias_command "info", "apps:info"
+
+
+  #
+  # --type TYPE # specify an output type i.e. json
+  #
+<<<<<<< HEAD
+  def self.get_info(cmd, info_file, dir, options = {})
+=======
+  def self.get_info(cmd, info_file, options = {})
+
+>>>>>>> b7501a641428a0952068d1837b20293619aba0cd
+    type = options[:type]
+
+    response = `#{cmd}`
+    if($?.exitstatus == 1)
+
+      File.delete("#{info_file}") if File.exists?(info_file)
+
+      if options[:type].nil?
+        exit(1)
+      else
+        Zillabyte::Helpers.error("error: #{response}", type)
+      end
+    end
+
+    app_info = {}
+    File.open("#{info_file}", "r+").each do |line|
+      line = JSON.parse(line)
+      if(line["type"])
+        app_info["nodes"] << line
+      else
+        app_info = line
+        app_info["nodes"] = []
+      end
+    end
+    File.delete("#{info_file}")
+
+    app_info = app_info.to_json
+    if(File.exists?("#{dir}/info_to_java.in"))
+      java_pipe = open("#{dir}/info_to_java.in","w+")
+      java_pipe.puts(app_info+"\n")
+      java_pipe.flush
+      java_pipe.close()
+    end
+
+    app_info
+  end
+
+
+  private 
+  
+  #
+  # --type TYPE # specify an output type i.e. json
+  #
+  def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false)
+    meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
+
+    #meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
+    error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
+    error(meta["error_message"], type) if meta['status'] == "error"
+    full_script = File.join(dir, meta["script"])
+    stderr_opt = "2> /dev/null" if ignore_stderr
+
+    case meta["language"]
+    when "ruby"
+      # Execute in the bundler context
+      cmd = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}  #{stderr_opt}"
+      
+    when "python"#{
+      if(File.directory?("#{dir}/vEnv"))
+        cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}  #{stderr_opt}"
+      else
+        cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}  #{stderr_opt}"
+      end
+ 
+    when "js"
+#      cmd = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS}  \"#{full_script}\" #{arg}"
+      cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN}  \"#{full_script}\" #{arg} #{stderr_opt}"
+
+    else
+      error("no language specified", type)
+    end
+    
+    return cmd
+
+  end
+  
+  #
+  #
+  #
+  def describe_app(meta, colors = {})
+    @colors ||= [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
+    rjust = 20
+    display "#{'app name'.rjust(rjust)}: #{meta['name']}"
+    display "#{'app language'.rjust(rjust)}: #{meta['language']}"
+    
+    meta['nodes'].each_with_index do |node, index|
+      colors[node['name']] ||= @colors.shift
+      color = colors[node['name']]
+      display (("="*rjust + "  operation ##{index}").colorize(color))
+      display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
+      display "#{"type".rjust(rjust)}: #{node['type'].to_s.colorize(color)}"
+      display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
+      display "#{"emits".rjust(rjust)}: #{JSON.pretty_generate(node['emits']).indent(rjust+2).lstrip.colorize(color)}" if node['emits']
+    end
+    
+  end
+
+  #
+  #
+  def read_name_from_conf(options = {})
+    type = options[:type]
+    hash = Zillabyte::API::Apps.get_rich_meta_info_from_script Dir.pwd, options
+    error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify an app id or run command from the directory containing the app.",type) if hash["error"]
+    hash["name"]
+  end
+
+end