zillabyte-cli 0.0.15 → 0.0.16

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YTcwMDBjODg0OTEyY2YxMzYzMjU4OTgwNmVhYTllOGRiZjEzZTZjYw==
+    MDRiNDYwNjlhZDQwYjc4NGNiNDZlNDkzYjE2NGNjZDhjYzMyNTdlMA==
   data.tar.gz: !binary |-
-    YzYwNWU4ZThlMDM0OWMxOTA2OTFkYjEyZGEyOTE4NTcwODg4ZTA0Ng==
-SHA512:
+    YTUzMjI0MGQxYTkyZjE4ODQ0MDU4ODg3MTBkNWM0ZDliNGEwNWQ2NA==
+!binary "U0hBNTEy":
   metadata.gz: !binary |-
-    OGQzMDYyZjg4ZDkwODVlOTNlM2ZkNTc4MmZkMjJlMGY0MGIxMGM3ZDRkZTM4
-    NzQyODU3MjkwN2NjMDM4YTRjZTAxYjliMmJkMjZmZTk0MWI3NGM1ZGM3MTZk
-    YTQ4OWE2OTM4MmZjOWYzMDQ4ZTEzMTRmNDQ4NGU4ZjM3MmFhMjE=
+    MmE4Y2RhZWY0ODE2ZTQ2MWIzMTgxMjk1Yzg4NzU0NzhhZWE2YzE2MWJhYzU5
+    ZTJmZjE0NGJmNTBlMjBhOTQ2YTVjNTZlNGQ1ODMzOGMwYTU4N2M4MjhiZTIy
+    MzlkYjc2ZmU2OTM2MjJlMTU1NTQxMDRiMzRmMTYwYTY2YTVhY2U=
   data.tar.gz: !binary |-
-    ZWI1YWZhYzk3MzMzNjQxOTVhYWEyY2ZlNjQ4MTZiMTM3MjIxN2Y3NGViNmEy
-    NWI0MDU0NDhhYzQ2NjE3YzNiOTgzYTNkYTg0YWUxNjEwOGFhM2M4ZDk1ZmRj
-    YjYzMTgzMTAyMGE2YTQ5NzRmNmQ5NzYxMTJkODYyMTVmNGUxZTM=
+    ZjAxMzlmOTEyYjI4ZTk2MmU2MWE1NThjMTQ5ODg4MmUxMjc1NDMzMmYxMjYz
+    ZmFlYWEyZjFlNTFmOGIwYTBhZDVkZWI5ZjZkNjI5NDBiMWExOGIzNDY3MTk0
+    ZGY5Yzc4Y2FjMTZhNDY0NTg2M2Q3ZTIwZjg1ZjA3MzA3Y2Y1YmQ=

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.0.15"
+    VERSION = "0.0.16"
   end
 end

data/lib/zillabyte/api.rb

@@ -62,13 +62,70 @@ class Zillabyte::API
 
   def request(params, &block)
     begin
-# The @connection object handles connections to api.zillabyte.com via Excon, this sends a command to the Rails api code
-# with parameters in params (one of which contains the path to the method to be run in the form of a url, e.g. if the
-# :path parameter is set to /functions/#{id}?tar=1, then the show method in functions_controller.rb will be automatically
-# called...it will have it's own set of parameters which correspond to the items in the query string).
-      response = @connection.request(params, &block)
+      
+      # If a block is given, then we assume we're streaming (i.e. for logs)
+      if block_given?
+        
+        # We tell excon to emit a chunk ever 50 bytes.  This is an excon thing, doesn't 
+        # affect the network IO chunk size
+        params[:chunk_size] = 50;
+        last_string = ""
+        
+        # Executed on each chunk excon bubbles up
+        params[:response_block] = lambda do |chunk, remaining_bytes, total_bytes|
+
+          # assume single-line json, split by '\n'
+          last_string += chunk
+          ary = last_string.split("\n", -1)
+          
+          # Are we at the end of a JSON record? 
+          if ary.size > 1
+            last_string = ary.last + (if last_string.end_with?("\n") then "\n" else "" end)
+            json_records = ary[0..-2]
+            json_records.each do |json_record|
+              
+              # Blank?
+              next if json_record.strip == ""
+              
+              if json_record.start_with?("HTTP/1.1 50")
+                puts "internal server error (code 500)"
+                exit(1)
+              end
+              
+              # Offer all the json records to the listening block
+              begin
+                block.call(JSON.parse(json_record))
+              rescue JSON::ParserError => e
+                error "unable to parse logs"
+                exit(1)
+              end
+              
+            end
+          end
+        end
+      end
+      
+      # Execute the request
+      response = @connection.request(params)
+      
     rescue Excon::Errors::HTTPStatusError => error
-      @progress.error "internal server error" if @progress
+      if @progress
+        
+        # Show helpful error message
+        if error.response.body
+          begin
+            hash = JSON.parse(error.response.body)
+            if hash['error_message']
+              @progress.error hash['error_message']
+            end
+          rescue JSON::ParserError => e
+            # Trickle to generic message below
+          end
+        end
+        
+        @progress.error "internal server error"
+        
+      end
       raise error
     end
     

data/lib/zillabyte/api/logs.rb

@@ -5,17 +5,17 @@ class Zillabyte::API::Logs < Zillabyte::API::Base
 
 
   # GET /flows/ID/logs
-  def get(flow_id, operation_id, options = {})
+  def get(flow_id, operation_id, options = {}, &block)
     
     options = {
     }.merge(options)
    
-    res = @api.request(
+    res = @api.request({
       :expects  => 200,
       :method   => :get,
-      :path     => "/flows/#{flow_id}/logs/#{operation_id}",
+      :path     => "/flows/#{flow_id}/logs/#{operation_id}/stream",
       :body     => options.to_json
-    )
+    }, &block)
 
     res.body
 
@@ -23,6 +23,23 @@ class Zillabyte::API::Logs < Zillabyte::API::Base
   
   
   
+  def get_startup(flow_id, operation_id, options = {}, &block)
+    
+    options = {
+    }.merge(options)
+   
+    res = @api.request({
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{flow_id}/logs/#{operation_id}/startup",
+      :body     => options.to_json
+    }, &block)
+
+    res.body
+
+  end
+  
+  
   
 
 end

data/lib/zillabyte/cli/flows.rb

@@ -63,23 +63,22 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     end
 
     display "Starting up your flow...please wait..."
-    previous_logs = {}
-    while true
-      all_logs = api.logs.get(res['id'], "_ALL_", {:push => true})
-      all_logs.each_pair do |op, logs|
-        next if logs.length == 0
-        difference = previous_logs[op] ? logs - previous_logs[op] : logs
-        difference.each do |log|
-          if ["start_up", "error"].include?(log["category"])
-            display log["line"]
-          end
-          exit(0) if log["line"].downcase.include?("flow deployed")
-        end
-        previous_logs[op] = logs
-      end
-      sleep(0.5)
-    end
+    sleep(2) # wait for kill command 
     
+    lf = LogFormatter::Startup.new
+    api.logs.get_startup(res['id'], "_ALL_", {:push => true}) do |hash|
+      
+      # Error?
+      error hash['error_message'] if hash['error']
+      
+      # Print it
+      lf.print_log_line(hash)
+      
+      # Exit when we get the 'done' message
+      # exit(0)  if (hash['line'] || '').downcase.include?("flow deployed")
+      
+    end
+      
   end
   alias_command "push", "flows:push"
   
@@ -87,7 +86,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   
   # flows:pull ID DIR
   #
-  # pulls a flow source to a directory. The target directory must be empty
+  # pulls a flow source to a directory.
   #
   # --force # pulls even if the directory exists
   #
@@ -227,14 +226,12 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   #
   # streams logs from the distributed workers
   # 
-  # -t, --tail  # continuously watches for new results
   # -v, --verbose LEVEL  # sets the verbosity (error, info, debug) (default: info)
   #
   def logs
 
     flow_id = options[:flow] || shift_argument
     operation_id = options[:operation] || shift_argument || '_ALL_'
-    tail = options[:tail]
     category = options[:verbose] || '_ALL_'
     carry_settings = {
       :category => category
@@ -247,17 +244,13 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     end
    
     display "Retrieving logs for flow ##{flow_id}...please wait..." 
-    begin
+    lf = LogFormatter::Operation.new
+    self.api.logs.get(flow_id, operation_id, api_options) do |line|
       
-      res = self.api.logs.get(flow_id, operation_id, api_options)
-
-      carry_settings = show_log_output(res, carry_settings)
-      api_options[:since] = carry_settings[:since]
+      error line['error_message'] if line['error']
+      lf.print_log_line(line)
       
-      if (tail)
-        sleep(5) # HACK
-      end
-    end while(tail)
+    end
 
   end  
   alias_command "logs", "flows:logs"
@@ -307,7 +300,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   end
   
 
-  # flows:test [TEST_DATASET_ID]
+  # flows:test
   #
   # tests a local flow with sample data
   # 
@@ -349,6 +342,14 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
       write_stream.flush
     end
 
+    def truncate_message(msg)
+      return msg if(!msg.instance_of?(String))
+      t_length = 50 # truncates entries to this length
+      m_length = msg.length
+      msg_out = m_length > t_length ? msg[0..t_length-3]+"..." : msg
+      msg_out
+    end
+
     def handshake(write_stream, read_stream, node, color)
       begin
         write_message write_stream, "{\"pidDir\": \"/tmp\"}\n"
@@ -361,7 +362,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
 
 
     # INIT
-    test_data = options[:test_data] || shift_argument
     dir = options[:dir] || Dir.pwd
 
     meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, self, {:test => true})
@@ -397,10 +397,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
         display "#{name} - #{msg}".colorize(override_color || color)
       end
 
-      # A Spout? 
-      if type == "spout"
+      # A Source? 
+      if type == "source"
 
-        # A spout from relation? 
+        # A source from relation? 
         if node['matches'] or node["relation"]
           matches = node['matches'] || (node["relation"]["query"])
           op_display.call "Grabbing remote data"
@@ -426,6 +426,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
               end
             end
             read_msg = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
+            values = Hash[values.map{|k, v| [truncate_message(k), truncate_message(v)]}]
             op_display.call "emit tuple: #{values} #{meta}"
             stream_messages[default_stream] ||= []
             stream_messages[default_stream] << read_msg
@@ -436,7 +437,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
           
         else
           
-          # A regular spout.. 
+          # A regular source.. 
           stream_messages[default_stream] ||= []
           batches.times do |i|
             stream_messages[default_stream] << "{\"command\": \"next\"}\n"
@@ -580,9 +581,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
                 # Make it human-understable
                 write_json = JSON.parse(write_msg)
                 if write_json['tuple']
-                  op_display.call "receiving: #{write_json['tuple']}"
+                  display_hash = Hash[write_json['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
+                  op_display.call "receiving: #{display_hash}"
                 elsif write_json['command'] == 'next'
-                  op_display.call "starting next spout batch"
+                  op_display.call "starting next source batch"
                 else
                   puts write_json
                 end
@@ -628,7 +630,8 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
                   emit_stream = obj['stream'] || default_stream
                   stream_messages[emit_stream] ||= []
                   stream_messages[emit_stream] << next_msg.to_json
-                  op_display.call "emitted: #{obj['tuple']} to #{emit_stream}"
+                  display_hash = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
+                  op_display.call "emitted: #{display_hash} to #{emit_stream}"
                 elsif obj['command'] == 'log'
                   op_display.call "log: #{obj['msg']}"
                 else
@@ -671,7 +674,9 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
         # We need to check here to see if we need to turn on multistream mode.
         # Note that we still need the redundant checks above, in case the user
         # didn't honor the emits contract.
-        split_branches = true if node["emits"].size() > 1
+        if node["emits"] && node["emits"].size() > 1
+          split_branches = true
+        end
       end
     end
     
@@ -709,7 +714,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   # runs a local flow with live data
   # 
   # --config CONFIG_FILE # use the given config file
-  #
+  # HIDDEN:
   def live_run
 
     name = options[:name] || shift_argument
@@ -735,7 +740,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   # outputs the info for the flow in the dir.
   # 
   # --pretty # Pretty prints the info output
-  #
+  # 
   def info
     info_file = SecureRandom.uuid
     cmd = command("--info --file #{info_file}")
@@ -785,71 +790,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
 
   private 
   
-  def show_log_output(lines, settings = {})
-    
-    max_category_size = 0
-    settings["start_up"] = :green
-    settings["kill"] = :yellow
-    settings["ipc"] = :cyan
-    settings["error"] = :red
-    settings["system"] = :white
-    settings["run"] = :magenta
-    all_lines = []
-    
-    lines.each_pair do |operation, lines|
-      lines.each do |line_h|
-        
-        line = line_h['line']
-        category = line_h['category']
-        date = line_h['date']
-        
-        all_lines << {
-          :line => line,
-          :category => category,
-          :color => settings[category],
-          :date => date
-        }
-        max_category_size = [max_category_size, category.size].max
-        
-        if settings[:since]
-          settings[:since] = [date, settings[:since]].max
-        else
-          settings[:since] = date
-        end
-        
-      end
-      
-    end
-    
-    all_lines.sort! do |a,b|
-      a[:date] <=> b[:date]
-    end
-    
-    requested_category = settings[:category]
-
-    all_lines.each do |h|
-      color = h[:color]
-      
-      case h[:category]
-      when nil
-        next
-      when 'start_up'
-        next if ['kill', 'run', 'ipc'].member?(requested_category)
-      when 'run'
-        next if ['kill', 'start_up', 'ipc'].member?(requested_category)
-      when 'ipc'
-        next if ['kill', 'start_up', 'run'].member?(requested_category)
-      when 'kill'
-        next if ['start_up', 'run', 'ipc'].member?(requested_category)
-      end
-      
-      display "#{h[:category].rjust(max_category_size)} - #{h[:date]} UTC - #{h[:line]}".colorize(color)
-    end
-    
-    settings
-  end
-  
-
 
   
   def command(arg="--execute_live", ignore_stderr = false)

data/lib/zillabyte/cli/help.rb

@@ -8,8 +8,16 @@ class Zillabyte::Command::Help < Zillabyte::Command::Base
   
 
   def index(*direct_args)
+    
     if command = args.shift || direct_args.shift
-      help_for_command(command)
+      if command == "aliases"
+        puts
+        puts "Useful aliases: "
+        puts 
+        summary_for_aliases(Zillabyte::Command.command_aliases)
+      else
+        help_for_command(command)
+      end
     else
       help_for_root
     end
@@ -18,6 +26,7 @@ class Zillabyte::Command::Help < Zillabyte::Command::Base
   alias_command "-h", "help"
   alias_command "--help", "help"
 
+
 private
 
   def commands_for_namespace(name)
@@ -74,13 +83,14 @@ private
   end
 
   def summary_for_aliases(command_aliases)
-    a_size = longest(command_aliases.map { |al,al_to| al })
-    a_to_size = longest(command_aliases.map { |al,al_to| al_to })
-    command_aliases.sort_by{|k,v| v}.each do |command_alias, alias_to|
+    used_aliases = command_aliases.select{|c,a| !c.include?("-") and !c.include?(":") and !skip_command?(commands[a])}
+    a_size = longest(used_aliases.map { |al,al_to| al }) + 2
+    a_to_size = longest(used_aliases.map { |al,al_to| al_to }) + 2
+    used_aliases.sort_by{|k,v| v}.each do |command_alias, alias_to|
       if command_alias.include?("-") or command_alias.include?(":")
         next #skip -h, -help and live-run etc.
       end
-      puts command_alias.ljust(a_size) + " --> " + alias_to.ljust(a_to_size + 2) + " # " + commands[alias_to][:summary]
+      puts command_alias.ljust(a_size) + " --> " + alias_to.ljust(a_to_size) + " # " + commands[alias_to][:summary]
     end
   end
 
@@ -95,11 +105,9 @@ private
     puts
     summary_for_namespaces(additional_namespaces)
     puts
-    puts "Useful aliases: "
-    puts 
-    summary_for_aliases(Zillabyte::Command.command_aliases)
+    puts "Type \"zillabyte help aliases\" for shortcuts to common commands."
     puts
-    puts "For help getting started, visit http://docs.zillabyte.com."
+    puts "Visit our documentation for more help: http://docs.zillabyte.com. "
     puts
   end
 
@@ -107,7 +115,7 @@ private
     namespace_commands = commands_for_namespace(name)
 
     unless namespace_commands.empty?
-      size = longest(namespace_commands.map { |c| c[:banner] })
+      size = longest(namespace_commands.select{|c| !skip_command?(c)}.map { |c| c[:banner] })
       namespace_commands.sort_by { |c| c[:banner].to_s }.each do |command|
         next if skip_command?(command)
         command[:summary] ||= legacy_help_for_command(command[:command])
@@ -141,10 +149,10 @@ private
     end
 
     if !namespace_commands.empty?
-      puts "Additional commands, type \"zillabyte help COMMAND\" for more details:"
       puts
       help_for_namespace(name)
       puts
+      puts "To see additional help for each of the commands, type \"zillabyte help COMMAND\"."
     elsif command.nil?
       error "#{name} is not a zillabyte command. See `zillabyte help`."
     end

data/lib/zillabyte/cli/log_formatter.rb

@@ -0,0 +1,103 @@
+class LogFormatter
+  
+  ########################################################
+  ## Base / Generic ######################################
+  ########################################################
+  
+  class Base
+    
+    def initialize
+      @all_colors ||= [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
+      @color_map = {
+        'flow' => :white
+      }
+    end
+    
+    def print_log_line(hash)
+    
+      # Dno't show heartbeats
+      return if hash['heartbeat']
+    
+      # Init 
+      line = hash['line']
+      category = hash['category']
+      date = hash['date']
+      name = hash['name']
+      index = hash['index']
+      
+      # Don't display 'hidden' lines
+      if category == 'hidden'
+        
+        handle_hidden(name, index, category, date, line);
+        
+      else
+        
+        # Print it out
+        display(name, index, category, date, line)
+        
+      end
+      
+    
+    end
+    
+  
+    protected
+  
+    def handle_hidden(operation_name, operation_index, category, date, line)
+    end
+  
+    def display(operation_name, operation_index, category, date, line)
+      puts "#{get_name(operation_name, operation_index).rjust(9).colorize(color_for(operation_name))} - #{line}"
+    end
+    
+    
+    def get_name(operation_name, operation_index)
+      if operation_index and operation_index != ""
+        return "#{operation_name}.#{operation_index}"
+      else
+        return operation_name
+      end
+    end
+    
+    def color_for(operation_name)
+      @color_map[operation_name] ||= @all_colors[ @color_map.size % @all_colors.size ]
+      @color_map[operation_name]
+    end
+    
+  end
+  
+  
+  ########################################################
+  ## Startup #############################################
+  ########################################################
+  
+  class Startup < Base
+    
+    def display(operation_name, operation_index, category, date, line)
+      puts "#{get_name(operation_name, operation_index).rjust(15).colorize(color_for(operation_name))} : #{line}"
+    end
+    
+    def handle_hidden(operation_name, operation_index, category, date, line)
+      case line
+      when 'REGISTER_FLOW_DEPLOYED', 'REGISTER_FLOW_TIMEOUT'
+        # done reading the logs
+        exit(0)
+      end
+    end
+    
+  end
+  
+  
+  ########################################################
+  ## Operation Colorized #################################
+  ########################################################
+  
+  class Operation < Base
+    # TODO: color'ify each operation
+  end
+  
+  
+  
+end
+
+

data/lib/zillabyte/cli/query.rb

@@ -163,9 +163,9 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
   alias_command "sql", "query:sql"
 
 
-  # query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET S3_FILE_PATH
+  # query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
   #
-  # pulls query data to S3_BUCKET/FILE_PATH/part***.gz
+  # pulls query data to S3_BUCKET/FILE_KEY/part***.gz
   #
   def pull_to_s3
 
@@ -175,14 +175,14 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
     user_s3_access_key = options[:s3_access_key] || shift_argument
     user_s3_secret = options[:s3_secret] || shift_argument
     user_s3_bucket = options[:s3_bucket] || shift_argument
-    user_s3_file_path = options[:s3_file_path] || shift_argument
+    user_s3_file_key = options[:s3_file_key] || shift_argument
     error "no s3 access key provided" if user_s3_access_key.nil?
     error "no s3 secret provided" if user_s3_secret.nil?
     error "no s3 bucket provided" if user_s3_bucket.nil?
-    error "no s3 file path specified" if user_s3_file_path.nil?
+    error "no s3 file path specified" if user_s3_file_key.nil?
 
     s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret, 
-                 :s3_bucket => user_s3_bucket, :s3_object_key => user_s3_file_path}
+                 :s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
 
     res = self.api.queries.pull_to_s3(query, s3_params)
     display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"

data/lib/zillabyte/cli/relations.rb

@@ -82,7 +82,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   # --schema SCHEMA              # Column names and types in the format "field_1:type_1,field_2:type_2,..."
   # --public SCOPE               # Make the relation public
   # --file FILE                  # A data file 
-  # --type FILE_TYPE             # type of data file [csv (default), xlsx]
+  # --type TYPE                  # File format type, defaults to csv
   # --description DESCRIPTION    # Description of relation contents
   # --aliases ALIASES            # Relation name aliases in the format "alias_1,alias_2,..."
   #
@@ -122,7 +122,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   #
   # adds data to an existing relation
   #
-  # --type TYPE # interpret file as type [csv, xlsx]
+  # --type TYPE # File format type, defaults to csv
   #
   def append
 
@@ -149,7 +149,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   #
   # pulls relation data into OUTPUT.gz
   #
-  #  --cycle_id [cycle_id]   # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
+  #  --cycle_id [cycle_id]   # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
   #
   def pull
 
@@ -157,7 +157,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     file = options[:file] || shift_argument
     error "no id given" if id.nil?
     error "no output file given" if file.nil?
-    file = file+".gz"
+    file = "#{file}.gz" unless File.extname(file) == ".gz"
 
     res = self.api.data.pull(id, options)
     if(res["uri"])
@@ -194,11 +194,11 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   end
   alias_command "rl:pull", "relations:pull"
 
-  # relations:pull:s3 ID S3_KEY S3_SECRET S3_BUCKET S3_FILE_PATH
+  # relations:pull:s3 ID S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
   #
-  # pulls relation data to S3_BUCKET/FILE_PATH/part***.gz
+  # pulls relation data to S3_BUCKET/FILE_KEY/part***.gz
   #
-  #  --cycle_id [cycle_id]   # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
+  #  --cycle_id [cycle_id]   # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
   #
   def pull_to_s3
 
@@ -208,14 +208,14 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     user_s3_access_key = options[:s3_access_key] || shift_argument
     user_s3_secret = options[:s3_secret] || shift_argument
     user_s3_bucket = options[:s3_bucket] || shift_argument
-    user_s3_file_path = options[:s3_file_path] || shift_argument
+    user_s3_file_key = options[:s3_file_key] || shift_argument
     error "no s3 access key provided" if user_s3_access_key.nil?
     error "no s3 secret provided" if user_s3_secret.nil?
     error "no s3 bucket provided" if user_s3_bucket.nil?
-    error "no s3 file path specified" if user_s3_file_path.nil?
+    error "no s3 file path specified" if user_s3_file_key.nil?
 
     s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret, 
-                 :s3_bucket => user_s3_bucket, :s3_object_key => user_s3_file_path}
+                 :s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
     s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
 
     res = self.api.data.pull_to_s3(id, s3_params)
@@ -229,9 +229,9 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
 
   # relations:show ID
   #
-  # shows raw data in a relation. run 'queries' for more elaborate functionality
+  # shows a sample of the data in a relation. see 'zillabyte queries' for more elaborate functionality
   #
-  #  --cycle_id [cycle_id]   # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
+  #  --cycle_id [cycle_id]   # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
   #  --no_truncation         # don't truncate long strings
   #
   def show

data/lib/zillabyte/cli/templates/ruby/simple_function.rb

@@ -1,22 +1,29 @@
 require 'zillabyte'
 
-Zillabyte.simple_function do |fn|
+Zillabyte.simple_function do
+
+  # This specifies the function's name and is mandatory.
+  name "simple_function"
   
   # This directive instructs zillabyte to give your function every 
   # web page in our known universe.  Your function will have access
   # to two fields: URL and HTML
-  fn.matches "select * from web_pages"
+  matches "select * from web_pages"
   
   # This directive tells Zillabyte what kind of data your function
   # produces.  In this case, we're saying we will emit a tuple that 
   # is one-column wide and contains the field 'URL'
-  fn.emits   [["has_hello_world", [{"URL"=>:string}]]]
+  emits [ 
+          [ 
+            "has_hello_world", [ {"URL"=>:string} ]
+          ]
+        ]
 
   
   # This is the heart of your algorithm.  It's processed on every
   # web page.  This algorithm is run in parallel on possibly hundreds
   # of machines. 
-  fn.execute do |controller, tuple|
+  execute do |tuple|
     
     # get the fields
     url = tuple['url']
@@ -27,13 +34,9 @@ Zillabyte.simple_function do |fn|
     # instead of
     # if html.include?('hello world')
     if html.include?('hello world')
-      controller.emit("has_hello_world", "URL" => url)
+      emit("has_hello_world", "URL" => url)
     end
     
   end
 
 end
-  
-  
-
-

data/zillabyte-cli.gemspec

@@ -28,6 +28,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "excon", "~> 0.31"
   spec.add_dependency "terminal-table", "~> 1.4"
   spec.add_dependency "activesupport", "~> 3.2.11"
+  spec.add_dependency "multi_json", "~> 1.0"
+  spec.add_dependency "mini_portile", "~> 0.5.0"
   
   spec.add_dependency "bundler", "~> 1.3"
   spec.add_dependency "colorize", "~> 0.6"
@@ -36,4 +38,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "indentation", "~> 0.1"
   spec.add_dependency "aws-sdk", "~> 1.33.0"
   spec.add_dependency "time_difference"
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.16
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-25 00:00:00.000000000 Z
+date: 2014-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -94,6 +94,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 3.2.11
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mini_portile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -201,7 +229,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/zillabyte
-- lib/zillabyte/#api.rb#
 - lib/zillabyte/api/base.rb
 - lib/zillabyte/api/data.rb
 - lib/zillabyte/api/flows.rb
@@ -215,7 +242,6 @@ files:
 - lib/zillabyte/api/zillalogs.rb
 - lib/zillabyte/api.rb
 - lib/zillabyte/auth.rb
-- lib/zillabyte/cli/#logs.rb#
 - lib/zillabyte/cli/auth.rb
 - lib/zillabyte/cli/base.rb
 - lib/zillabyte/cli/config.rb
@@ -224,12 +250,12 @@ files:
 - lib/zillabyte/cli/help.rb
 - lib/zillabyte/cli/helpers/data_schema_builder.rb
 - lib/zillabyte/cli/host.rb
+- lib/zillabyte/cli/log_formatter.rb
 - lib/zillabyte/cli/query.rb
 - lib/zillabyte/cli/relations.rb
 - lib/zillabyte/cli/sources.rb
 - lib/zillabyte/cli/templates/js/simple_function.js
 - lib/zillabyte/cli/templates/js/zillabyte.conf.yaml
-- lib/zillabyte/cli/templates/python/#simple_function.py#
 - lib/zillabyte/cli/templates/python/requirements.txt
 - lib/zillabyte/cli/templates/python/simple_function.py
 - lib/zillabyte/cli/templates/python/zillabyte.conf.yaml
@@ -273,7 +299,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.10
+rubygems_version: 2.0.7
 signing_key: 
 specification_version: 4
 summary: The Official Zillabyte CLI Gem

data/lib/zillabyte/#api.rb#

@@ -1,170 +0,0 @@
-# load('zillabyte/helpers.rb') # reload helpers after possible inject_loadpath
-# load('zillabyte/updater.rb') # reload updater after possible inject_loadpath
-
-require "base64"
-require "excon"
-# require "securerandom"
-require "uri"
-require "zlib"
-require "active_support/core_ext/object/to_query.rb"
-require 'json'
-
-
-# workaround for rescue/reraise to define errors in command.rb failing in 1.8.6
-if RUBY_VERSION =~ /^1.8.6/
-  # require('zillabyte-api')
-  require('rest_client')
-end
-
-class Zillabyte::API
-
-  HEADERS = {
-    'Accept'                => 'application/json',
-    'Content-Type'          => 'application/json',
-    # 'Accept-Encoding'       => 'gzip',
-    'User-Agent'            => "zillabyte/#{Zillabyte::CLI::VERSION}",
-    'X-Ruby-Version'        => RUBY_VERSION,
-    'X-Ruby-Platform'       => RUBY_PLATFORM
-  }
-
-  OPTIONS = {
-    :headers  => {},
-    :host     => ENV['ZILLABYTE_API_HOST'] || 'api.zillabyte.com',
-    :port     => (ENV['ZILLABYTE_API_PORT'] || '80').to_i,
-    :nonblock => false,
-    :scheme   => 'http'
-  }
-
-  
-
-  def initialize(options={})
-    options = OPTIONS.merge(options)
-
-    @progress = options.delete(:progress)
-    @api_key = options.delete(:api_key) || ENV['ZILLABYTE_API_KEY']
-    
-    options[:headers] = HEADERS.merge({ 
-      'Authorization' => "auth_token #{@api_key}",
-    }).merge(options[:headers])
-
-    @options = options.clone
-    @options[:bubble_exceptions] = ENV['BUBBLE_EXCEPTIONS'] || false
-
-    @connection = Excon.new("#{options[:scheme]}://#{options[:host]}", options)
-
-  end
-  
-  
-  def host
-    @options[:host]
-  end
-  
-
-  def request(params, &block)
-    begin
-# The @connection object handles connections to api.zillabyte.com via Excon, this sends a command to the Rails api code
-# with parameters in params (one of which contains the path to the method to be run in the form of a url, e.g. if the
-# :path parameter is set to /functions/#{id}?tar=1, then the show method in functions_controller.rb will be automatically
-# called...it will have it's own set of parameters which correspond to the items in the query string).
-      response = @connection.request(params, &block)
-    rescue Excon::Errors::HTTPStatusError => error
-      @progress.error "internal server error" if @progress
-      raise error
-    end
-    
-    if response.body && !response.body.empty?
-      # decompress_response!(response)
-      begin
-        response.body = JSON.parse(response.body)
-      rescue Exception => e
-        raise e if @options[:bubble_exceptions]
-        @progress.error "unknown server response: \n #{response.body}" if @progress
-      end
-    end
-    
-    # Errors?
-    if response.body.is_a?(Hash)
-      if response.body["status"] == "error"
-        if @progress
-          if response.body["error"] == "authentication_error"
-            @progress.error "authentication error. run 'zillabyte login'"
-          else
-            @progress.error response.body["error_message"] 
-          end
-        end
-      end
-    end
-    
-
-    # reset (non-persistent) connection
-    @connection.reset
-
-    response
-  end
-  
-  
-  def data
-    @_data ||= ::Zillabyte::API::Data.new(self)
-  end
-  
-  def logs
-    @_logs ||= ::Zillabyte::API::Logs.new(self)
-  end
-  
-  
-  def queries
-    @_queries ||= ::Zillabyte::API::Queries.new(self)
-  end
-  alias_method :query, :queries
-  
-  def flows
-    @_flows ||= ::Zillabyte::API::Flows.new(self)
-  end
-  alias_method :flow, :flows
-  
-  def metrics
-    @_metrics ||= ::Zillabyte::API::Metrics.new(self)
-  end
-  alias_method :metric, :metrics
-  
-  def zillalogs
-    @_rules ||= ::Zillabyte::API::Zillalogs.new(self)
-  end
-  alias_method :zillalog, :zillalogs
-  
-  def sources
-    @_sources ||= ::Zillabyte::API::Sources.new(self)
-  end
-  alias_method :source, :sources
-  
-  
-  def collectors
-    @_collectors ||= ::Zillabyte::API::Collectors.new(self)
-  end
-  alias_method :collector, :collectors
-  
-  def semantic_tags
-    @_rules ||= ::Zillabyte::API::SemanticTags.new(self)
-  end
-  alias_method :zillalog, :zillalogs
-  
-  
-  
-  
-  
-  def self.load
-    Dir[File.join(File.dirname(__FILE__), "api", "*.rb")].sort.each do |file|
-      require file
-    end
-    
-    Dir[File.join(File.dirname(__FILE__), "api", "helpers", "*.rb")].sort.each do |file|
-      require file
-    end
-  end
-
-
-end
-
-
-# Load all helpers...
-Zillabyte::API.load()

data/lib/zillabyte/cli/#logs.rb#

@@ -1,12 +0,0 @@
-require "zillabyte/cli/base"
-
-# manage custom logs
-#
-class Zillabyte::Command::Logs < Zillabyte::Command::Base
-
-
-
-  
-  
-
-end

data/lib/zillabyte/cli/templates/python/#simple_function.py#

@@ -1,27 +0,0 @@
-import zillabyte
-
-def prep(controller):
-  return
-
-# This is the heart of your algorithm.  It's processed on every
-# web page.  This algorithm is run in parallel on possibly hundreds
-# of machines.
-def execute(controller, tup):
-  if("hello world" in tup.values["html"]):
-    controller.emit("has_hello_world",{"url":tup.values["url"]})
-  return
-
-zillabyte.simple_function(\
-  # This directive instructs zillabyte to give your function every
-  # web page in our known universe.  Your function will have access
-  # to two fields: URL and HTML
-  matches = "select * from web_pa", \
-
-  # This directive tells Zillabyte what kind of data your function
-  # produces.  In this case, we're saying we will emit a tuple that
-  # is one-column wide and contains the field 'URL' 
-  emits = [["has_hello_world", [{"url":"string"}]]], \
-
-  prepare = prep, \
-  execute = execute\
-)