zillabyte-cli 0.0.16 → 0.0.17

data/lib/zillabyte/cli/relations.rb

@@ -1,34 +1,42 @@
 require "zillabyte/cli/base"
+require "zillabyte/cli/helpers/table_output_builder"
 require "csv"
 require "open-uri"
 require "aws-sdk"
+require "json"
+
 
 # manage custom relations
 #
 class Zillabyte::Command::Relations < Zillabyte::Command::Base
 
+  MAX_POLL_SECONDS = 60 * 5
+  POLL_SLEEP = 0.5
+
   # relations
   #
   # lists your custom relations 
+  # --type TYPE                   # specify an output type i.e. json
   # 
   def index
     self.list
   end
   
 
-
   # relations
-  #
+  #  v 
   # lists your custom relations 
   # 
+  #  --type TYPE # The output format type
   def list
-    
+    type = options[:type] || nil
+
     response = api.request(
       :expects  => 200,
       :method   => :get,
       :path     => "/relations"
     )
-      
+
     headings = []
     rows = response.body.map do |row|
       headings = row.keys if headings.size == 0
@@ -37,10 +45,11 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
       vals = row.values_at *headings
       vals 
     end
-    
-    display "relations:\n" + Terminal::Table.new(:headings => headings, :rows => rows).to_s
-    display "Total number of relations: "+rows.length.to_s
-    
+
+    display "relations\n" if type.nil? && rows.size > 0
+    display TableOutputBuilder.build_table(headings, rows, type)
+    display "Total number of relations: "+rows.length.to_s if type.nil?
+
   end
   alias_command "rl:list", "relations:list"
 
@@ -49,13 +58,20 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   # 
   # deletes a relation 
   # 
-  # -f, --force # Delete without asking for confirmation
+  # -f, --force         # Delete without asking for confirmation
+  # --type TYPE     # The output format type
   def delete 
     id = options[:id] || shift_argument
     forced = options[:force]
-    if not forced
+    type = options[:type] || nil
+
+    if !forced
+
+      if !type.nil?
+        error("specify -f, --force to confirm deletion", type)
+      end
+
       while true
-        
         display "This operation cannot be undone. Are you sure you want to delete this relation? (yes/no):", false
         confirm = ask
         break if confirm == "yes" || confirm == "no"
@@ -66,11 +82,17 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     confirmed = forced || confirm == "yes"
 
     if confirmed
-      response = api.data.delete(id)
-      display response["body"]
+      res = api.data.delete(id)
+      if res['error']
+        error(res['error'], type)
+      else
+        if type == "json"
+          display {}.to_json
+        else
+          display res["body"]
+        end
+      end
     end
-    
-    
   end
   alias_command "rl:delete", "relations:delete"
 
@@ -82,36 +104,47 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   # --schema SCHEMA              # Column names and types in the format "field_1:type_1,field_2:type_2,..."
   # --public SCOPE               # Make the relation public
   # --file FILE                  # A data file 
-  # --type TYPE                  # File format type, defaults to csv
+  # --filetype FILETYPE          # File format type, defaults to csv
   # --description DESCRIPTION    # Description of relation contents
   # --aliases ALIASES            # Relation name aliases in the format "alias_1,alias_2,..."
+  # --type TYPE                  # The output format type
   #
   def create
     
     name = options[:name] || shift_argument
     file = options[:file] || nil
-    type = options[:type] || nil
+    filetype = options[:filetype] || nil
+    type = options[:type]
 
-    error "no name given" if name.nil?
+    error("no name given", type) if name.nil?
 
     schema = options[:schema] if options[:schema]
     is_public = options[:public] || nil
     description = options[:description] || nil
     aliases = options[:aliases] || nil
-   
-    hash = set_relation_properties(schema,is_public,description,aliases)
+
+    if type.nil?
+      hash = get_relation_properties(schema,is_public,description,aliases)
+    else
+      hash = hash_relation_properties(schema,is_public,description,aliases, type)
+    end
+
     if file
-      type ||= File.extname(file).gsub(".", "")
-      rows = sanity_check_file(file,type, {"columns" => hash[:schema]})
+      filetype ||= File.extname(file).gsub(".", "")
+      rows = sanity_check_file(file,filetype, {"columns" => hash[:schema]}, type)
       hash[:rows] = rows
     end
 
     res = api.data.create name, hash
     
     if res['error']
-      display "error: #{res['error_message']}"
+      error("#{res['error_message']}", type)
     else
-      display "relation ##{res['id']} #{res['action']}. size: #{res['size'] || 0} rows."
+      if type == "json"
+        display {}.to_json
+      else
+        display "relation ##{res['id']} #{res['action']}. size: #{res['size'] || 0} rows."
+      end
     end
     
   end
@@ -121,8 +154,8 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   # relations:append ID FILE
   #
   # adds data to an existing relation
-  #
-  # --type TYPE # File format type, defaults to csv
+  # --filetype FILETYPE                 # Input File format type, defaults to csv
+  # --type TYPE                     # Output formatting type i.e. json
   #
   def append
 
@@ -130,16 +163,22 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     file = options[:file] || shift_argument
     type = options[:type]
 
-    error "no id given" if id.nil?
-    error "no file given" if file.nil?
-    type ||= File.extname(file).gsub(".", "")
+    filetype = options[:filetype]
+    filetype ||= File.extname(file).gsub(".", "")
+    error("no id given", type) if id.nil?
+    error("no file given", type) if file.nil?
+
 
     relation = self.api.data.get(id)
     columns = relation["columns"].map{|col| {col["index"] => col["type"]}}
-    rows = sanity_check_file(file,type,{"columns" => columns})
+    rows = sanity_check_file(file,filetype,{"columns" => columns}, type)
    
     res = self.api.data.append(id, {:rows => rows})
-    display "relation ##{id} appended #{res["size"]} rows"
+    if type == "json"
+      display {}.to_json
+    else
+      display "relation ##{id} appended #{res["size"]} rows"
+    end
 
   end
   alias_command "append", "relations:append"
@@ -149,31 +188,36 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   #
   # pulls relation data into OUTPUT.gz
   #
-  #  --cycle_id [cycle_id]   # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
+  #  --cycle_id [cycle_id]              # retrieve data generated for that cycle, only if this relation is associated with an app. (defaults to the last cycle
+  #  --type TYPE                        # the output format type
   #
   def pull
 
     id = options[:id] || shift_argument
     file = options[:file] || shift_argument
-    error "no id given" if id.nil?
-    error "no output file given" if file.nil?
+    type = options[:type]
+    error("no id given", type) if id.nil?
+    error("no file given", type) if file.nil?
+
     file = "#{file}.gz" unless File.extname(file) == ".gz"
 
     res = self.api.data.pull(id, options)
+
     if(res["uri"])
-      display "Waiting for download."
+      display "Waiting for download." if type.nil?
       File.open(file, "w") do |f|
         f.write open(res["uri"]).read
       end
     elsif(res["s3_credentials"])
-      display "Request sent. Depending on the size of your file, this may take a while."
+      display "Request sent. Depending on the size of your file, this may take a while." if type.nil?
       s3 = AWS::S3.new(res["s3_credentials"])
       bucket = s3.buckets[res["s3_bucket"]]
       obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
+
       while(true)
         keys = obj.collect(&:key)
         if keys.length > 0 and keys.include?("#{res["s3_file_key"]}/manifest")
-          display "Starting to write to file..."
+          display "Starting to write to file..." if type.nil?
           File.open(file, "w") do |f|
             obj.each do |o|
               if o.key == "#{res["s3_file_key"]}/manifest"
@@ -189,7 +233,12 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
         end
       end
     end
-    display "pulled data from relation ##{id} to file #{file}"
+
+    if type == "json"
+      display {}.to_json
+    else
+      display "pulled data from relation ##{id} to file #{file}"
+    end
 
   end
   alias_command "rl:pull", "relations:pull"
@@ -198,30 +247,37 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   #
   # pulls relation data to S3_BUCKET/FILE_KEY/part***.gz
   #
-  #  --cycle_id [cycle_id]   # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
+  #  --cycle_id [cycle_id]                                # retrieve data generated for that cycle, only if this relation is associated with an app. (defaults to the last cycle)
+  #  -t, --type TYPE                                      # the output format types
   #
+
   def pull_to_s3
 
     id = options[:id] || shift_argument
-    error "no id given" if id.nil?
+    type = options[:type]
+    error("no id given", type) if id.nil?
 
     user_s3_access_key = options[:s3_access_key] || shift_argument
     user_s3_secret = options[:s3_secret] || shift_argument
     user_s3_bucket = options[:s3_bucket] || shift_argument
     user_s3_file_key = options[:s3_file_key] || shift_argument
-    error "no s3 access key provided" if user_s3_access_key.nil?
-    error "no s3 secret provided" if user_s3_secret.nil?
-    error "no s3 bucket provided" if user_s3_bucket.nil?
-    error "no s3 file path specified" if user_s3_file_key.nil?
+    error("no s3 access key provided", type) if user_s3_access_key.nil?
+    error("no s3 access secret provided", type) if user_s3_secret.nil?
+    error("no s3 access bucket provided", type) if user_s3_bucket.nil?
+    error("no s3 file path provided", type) if user_s3_file_key.nil?
 
     s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret, 
                  :s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
     s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
 
     res = self.api.data.pull_to_s3(id, s3_params)
-    display "downloading relation data to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
-    display "if the relation is large, this may take a while, please check your s3 account after a few minutes"
 
+    if type == "json"
+      display {}.to_json
+    else
+      display "downloading relation data to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
+      display "if the relation is large, this may take a while, please check your s3 account after a few minutes"
+    end
   end
   alias_command "relations:pull:s3", "relations:pull_to_s3"
   alias_command "rl:pull:s3", "relations:pull_to_s3"
@@ -231,23 +287,65 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   #
   # shows a sample of the data in a relation. see 'zillabyte queries' for more elaborate functionality
   #
-  #  --cycle_id [cycle_id]   # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
+  #  --cycle_id [cycle_id]   # retrieve data generated for that cycle, only if this relation is associated with an app. (defaults to the last cycle)
   #  --no_truncation         # don't truncate long strings
-  #
+  #  --type TYPE             # the type of the output
   def show
-    
     name = options[:name] || shift_argument
+    type = options[:type]
     error "no id given" if name.nil?
     
-    res = self.api.data.get(name, options)
-    
+    # Initial request..
+    res = self.api.data.show(name, :post, options)
+
+    if res['job_id']
+      job_id = res['job_id']
+      options[:job_id] = job_id
+
+      # Poll until the results are ready... 
+      start = Time.now.utc
+
+      display "Fetching your data, please wait..." if type.nil?
+
+      while(Time.now.utc < start + MAX_POLL_SECONDS) do
+
+        # Poll
+        res = self.api.data.show(name, :get, options)
+
+        # Status?
+        case res['status']
+        when 'completed'
+          if res['return']
+            res = res['return']
+          else
+            throw "something is wrong: #{res}"
+          end
+          # success! continue below
+          break
+        when 'running'
+          sleep(POLL_SLEEP)
+       #   display ".", false
+        else
+          throw "unknown status: #{res}"
+        end
+        
+      end
+    else
+      if res['error']
+        error(res['error_message'] || res['error'], type)
+      else
+        error("remote server error (r256)", type)
+      end
+    end 
+
+    # We only reach here after polling is complete...
     if res["rows"].size > 0
       headings = []
       concrete_headings = res["rows"].first.keys
       concrete_headings.delete("id")
       concrete_headings.each do |ch|
         has_alias = false
-        res['column_aliases'].each do |al|
+        (res['column_aliases'] || []).each do |al|
           if(al["concrete_name"] == ch)
             headings << al["alias"]
             has_alias = true
@@ -256,9 +354,8 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
         headings << ch if !has_alias
       end
 
-      table = Terminal::Table.new :headings => headings    
+      rows = []
       res["rows"].each do |obj|
-        
         new_row = concrete_headings.map do |heading|
           if options[:no_truncation]
             obj[heading]
@@ -270,16 +367,16 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
             end
           end
         end
-        
-        table.add_row new_row
-        
+        rows << new_row
       end
-      display table.to_s
-      
+
+      display TableOutputBuilder.build_table(headings, rows, type)
     else
-      
-      display "empty relation"
-      
+      if type == "json"
+        display {}.to_json
+      else
+        display "empty relation"
+      end
     end
     
   end
@@ -289,8 +386,53 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   private
 
 
+  def hash_relation_properties(schema, is_public, description, aliases, type)
+    if !aliases.nil?
+      aliases = aliases.strip.split(",").map{|x| x.strip}.uniq
+      aliases.each do |a|
+        if(!(a =~ /^[a-zA-Z0-9\_]+$/i))
+          error_message = "(\"#{a}\" contains illegal characters. Only letters, numbers and underscore are allowed in alias names."
+          error_message += "If you would like to give alias names to your relation, please enter them below. (comma separated, only letters, number and underscore allowed)"
+          error(error_message, type)
+        end
+      end
+    else
+      aliases = []
+    end
 
-  def set_relation_properties(schema,is_public,description,aliases)
+    valid_types=["string", "integer", "float"]
+    meta_names=["id", "confidence", "since", "source"]
+
+    if description.nil?
+      description = ""
+    end
+
+    if schema.nil?
+      error("specify a schema", type)
+    end
+    schema = schema.split(",").map { |x| Hash[*x.split(":")] }
+    schema.each do |s|
+      cname = s.keys[0]
+      if(cname =~ /^v[0-9]+$/i or meta_names.member?(cname.downcase))
+        error("\"v[number]\", \"id\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your column name something else.", type)
+      end
+      ctype = s[cname]
+      unless valid_types.member?(ctype)
+        error("invalid type: #{ctype}", type)
+        next
+      end
+    end
+
+    if is_public.nil?
+      error("specify public or private using --public")
+    end
+
+    {:schema => schema, :public => is_public, :description => description, :aliases => aliases}
+  end
+
+
+
+  def get_relation_properties(schema, is_public, description, aliases)
     if aliases.nil?
       display "If you would like to give alias names to your relation, please enter them below. (comma separated, only letters, number and underscore allowed)"
       aliases = ask.strip
@@ -371,7 +513,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     {:schema => schema, :public => is_public, :description => description, :aliases => aliases}
   end
 
-  def sanity_check_file(file,type,dataset)
+  def sanity_check_file(file, filetype, dataset, type = nil)
     rows = []
   
     n_columns = dataset["columns"].size
@@ -387,27 +529,33 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
       end
     end
 
-    case type
+    case filetype
     when "csv"
-      CSV.foreach(file) do |row|
-        if row.size != n_columns
-          error "relation expect #{n_columns} column(s). Found a row with #{row.size}::\n #{row}"
-        end
-        float_cols.each do |i|
-          if(!(row[i] =~ /^[+-]?(\d+(.\d+)?)$/))
-            error "column #{i+1} should be a FLOAT in row::\n #{row}"
+      begin
+        CSV.foreach(file) do |row|
+          if row.size != n_columns
+            error("relation expect #{n_columns} column(s). Found a row with #{row.size}::\n #{row}", type)
           end
-        end
-        int_cols.each do |i|
-          if(!(row[i] =~ /^[+-]?(\d+)$/))
-            error "column #{i+1} should be an INTEGER in row::\n #{row}"
+
+          float_cols.each do |i|
+            if (!(row[i] =~ /^[+-]?(\d+(.\d+)?)$/))
+            error("column #{i+1} should be an INTEGER in row ::\n #{row}", type)
+            end
+          end
+
+          int_cols.each do |i|
+            if (!(row[i] =~ /^[+-]?(\d+)$/))
+             error("column #{i+1} should be an INTEGER in row #{row}::\n #{row}", type)
+            end
           end
+          rows << row
         end
-        rows << row
+      rescue Exception => e
+        error("unable to parse csv", type)
       end
       
     else
-      error "unsupported type: #{type}"
+      error "unsupported type: #{filetype}"
       
     end
     rows

data/lib/zillabyte/cli/sources.rb

@@ -34,4 +34,4 @@
 # 
 #   
 # 
-# end
+# end

data/lib/zillabyte/cli/templates/js/simple_function.js

@@ -15,6 +15,11 @@ function exec(controller, tuple) {
 }
 
 zillabyte.simple_function({
+/**
+ * This specifies the function's name and is mandatory.
+ */
+  name: "simple_function",
+
 /** 
  * This directive instructs zillabyte to give your function every
  * web page in our known universe.  Your function will have access

data/lib/zillabyte/cli/templates/python/#simple_function.py#

@@ -0,0 +1,27 @@
+import zillabyte
+
+def prep(controller):
+  return
+
+# This is the heart of your algorithm.  It's processed on every
+# web page.  This algorithm is run in parallel on possibly hundreds
+# of machines.
+def execute(controller, tup):
+  if("hello world" in tup.values["html"]):
+    controller.emit("has_hello_world",{"url":tup.values["url"]})
+  return
+
+zillabyte.simple_function(\
+  # This directive instructs zillabyte to give your function every
+  # web page in our known universe.  Your function will have access
+  # to two fields: URL and HTML
+  matches = "select * from web_pa", \
+
+  # This directive tells Zillabyte what kind of data your function
+  # produces.  In this case, we're saying we will emit a tuple that
+  # is one-column wide and contains the field 'URL' 
+  emits = [["has_hello_world", [{"url":"string"}]]], \
+
+  prepare = prep, \
+  execute = execute\
+)

data/lib/zillabyte/cli/templates/python/simple_function.py

@@ -12,6 +12,9 @@ def execute(controller, tup):
   return
 
 zillabyte.simple_function(\
+  # This specifies the function's name and is mandatory.
+  name = "simple_function", \
+
   # This directive instructs zillabyte to give your function every
   # web page in our known universe.  Your function will have access
   # to two fields: URL and HTML

data/lib/zillabyte/cli/templates/ruby/{simple_function.rb → simple_app.rb}

@@ -1,16 +1,16 @@
 require 'zillabyte'
 
-Zillabyte.simple_function do
+Zillabyte.simple_app do
 
-  # This specifies the function's name and is mandatory.
-  name "simple_function"
+  # This specifies the app's name and is mandatory.
+  name "simple_app"
   
-  # This directive instructs zillabyte to give your function every 
-  # web page in our known universe.  Your function will have access
+  # This directive instructs zillabyte to give your app every 
+  # web page in our known universe.  Your app will have access
   # to two fields: URL and HTML
   matches "select * from web_pages"
   
-  # This directive tells Zillabyte what kind of data your function
+  # This directive tells Zillabyte what kind of data your app
   # produces.  In this case, we're saying we will emit a tuple that 
   # is one-column wide and contains the field 'URL'
   emits [ 

data/lib/zillabyte/cli/templates/ruby/zillabyte.conf.yaml

@@ -1,2 +1,2 @@
 language: ruby
-script: simple_function.rb
+script: simple_app.rb

data/lib/zillabyte/cli/version.rb

@@ -8,4 +8,4 @@ class Zillabyte::Command::Version < Zillabyte::Command::Base
     display "Zillabyte CLI Version #{Zillabyte::CLI::VERSION}"
   end
 
-end
+end

data/lib/zillabyte/cli/zillalogs.rb

@@ -83,4 +83,4 @@
 #   
 # 
 # 
-# end
+# end

data/lib/zillabyte/command.rb

@@ -208,9 +208,17 @@ module Zillabyte
         object, method = prepare_run(cmd, arguments.dup)
         object.send(method)
       rescue Excon::Errors::SocketError => e
-        error "remote server error: #{e.message}"
+        if ENV['BUBBLE_EXCEPTIONS']
+          raise e
+        else
+          error "remote server error: #{e.message}"
+        end
       rescue Errno::ECONNREFUSED => e
-        error "remote server error: #{e.message}"
+        if ENV['BUBBLE_EXCEPTIONS']
+          raise e
+        else
+          error "remote server error: #{e.message}"
+        end
       end
     end
 

data/lib/zillabyte/common/{progress.rb → session.rb}

@@ -1,5 +1,5 @@
 module Zillabyte::Common
-  class Progress
+  class Session
     
     def update(*args)
       display(*args)