zillabyte-cli 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54c7334eca6bc7a0ee7d5b20093411539b51b891
-  data.tar.gz: 6239db222616513ebc77fbfadf8c3484d215800a
+  metadata.gz: 284b1b673fa8e9159a4f9f5ac108ffdf2f158fe0
+  data.tar.gz: a8d5398aa5b99320f6d050c797c952b977b10dc1
 SHA512:
-  metadata.gz: 1a3382cdff1fa803334611716c351f8bd27ce899b43180713331ef6f8e37279b38accbf100126ad9cd0b48b3df3217c9dc8ffbc420725950d61d716e35773da0
-  data.tar.gz: e0a1dceff8dd68bf110c403979b42224d06ebb07415e58f7eb5446c5a1a4bf3ba5936d0d059ace0bacd105ad4cc9602218f882975f8327eb92528df29f66e49f
+  metadata.gz: 5786156d1bb899860e7a8c9468b1e0a4ac5dcb4f7d4afe7b59bf4174b455eaefe2d3764db9bfc7c056f972cb7e1bec8c570bf48fc378ab5d5d2b049a7dbbc2f2
+  data.tar.gz: f56ec56c826a691b1b911264e38358ec66efe9ee77e0560e09690f95bf93605a99d5f58262d8254e33d8ecc981c8e4c7c8ab30edc248966786a9780f88bc6c71

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

data/lib/zillabyte/api/apps.rb

@@ -42,24 +42,7 @@ class Zillabyte::API::Apps < Zillabyte::API::Flows
 
   end
 
-  # POST /apps/id/kill
-  def kill(id, options = {})
 
-    options = {
-      # TODO
-    }.merge(options)
-
-    res = @api.request(
-      :expects  => 200,
-      :method   => :post,
-      :path     => "/apps/#{id}/kill",
-      :body     => options.to_json
-    )
-
-    res.body
-
-  end
-  
   def list_cycles(id, options={})
     res = @api.request(
       :expects => 200,
@@ -110,6 +93,10 @@ class Zillabyte::API::Apps < Zillabyte::API::Flows
     # Get the meta info
     options[:local_flag] = 1
     hash = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, session, options)
+    if hash["flow_type"] == "component"
+      session.error("To push up a component please use git. The \"zillabyte push\" command is only supported for apps.") if session
+    end
+
     if hash.nil?
       session.error("unable to extract app meta information", type) if session
     end

data/lib/zillabyte/api/components.rb

@@ -94,22 +94,4 @@ class Zillabyte::API::Components < Zillabyte::API::Flows
     res.body
   end
 
-  # POST /components/id/kill
-  def kill(id, options = {})
-
-    options = {
-      # TODO
-    }.merge(options)
-
-    res = @api.request(
-      :expects  => 200,
-      :method   => :post,
-      :path     => "/components/#{id}/kill",
-      :body     => options.to_json
-    )
-
-    res.body
-
-  end
-
 end

data/lib/zillabyte/api/flows.rb

@@ -14,6 +14,78 @@ class Zillabyte::API::Flows < Zillabyte::API::Base
     res.body
   end
 
+  # POST /flows/id/kill
+  def create_kill(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/flows/#{id}/kill",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+  # GET /flows/id/kill
+  def poll_kill(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{id}/kill",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+  # POST /flows/id/delete
+  def create_delete(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/flows/#{id}/delete",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+  # GET /flows/id/delete
+  def poll_delete(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{id}/delete",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
   # GET /flows/id/download
   def pull_to_directory(id, dir, session = nil, options = {})
     

data/lib/zillabyte/auth.rb

@@ -12,6 +12,7 @@ class Zillabyte::Auth
     def login(email, auth_token)
       if valid_credentials?(email, auth_token)
         write_credentials(email, auth_token)
+        display "Authentication complete."
       end
     end
 
@@ -113,7 +114,6 @@ class Zillabyte::Auth
         display error.message
         exit 1
       end
-      display "Authentication complete"
     end
 
     def delete_credentials
@@ -198,8 +198,44 @@ class Zillabyte::Auth
           if not valid_credentials?(email, auth_token)
             raise Zillabyte::Auth::InvalidCredentialsException
           else
+            
             # Success... 
             write_credentials(email, auth_token)
+            
+            # Maybe add ssh keys while we're at it... 
+            possible_keys = [
+              "~/.ssh/id_rsa.pub", 
+              "~/.ssh/id_dsa.pub"
+            ]
+            
+            key_added = false
+            possible_keys.each do |keypath|
+              if File.exists?(File.expand_path(keypath))
+              
+                display "Register SSH key #{keypath}? (y/N)", false
+                break if (ask() || "").strip.downcase[0] != "y"
+                  
+                api = Zillabyte::API.new(:api_key => auth_token, :session => self)
+                begin
+                  key = File.binread(File.expand_path(keypath))
+                rescue => e
+                  error(e.message, type)
+                  break
+                end
+                message = api.keys.add("default", key)
+                display("SSH keys added.  Use 'zillabyte keys' to manage.")
+                key_added = true
+                break
+
+              end
+            end
+            
+            unless key_added
+              display "Please run zillabyte keys:add to add your ssh keys."
+            end
+            
+            display "Authentication complete."
+            
           end
         end
       rescue Zillabyte::Auth::InvalidCredentialsException => e

data/lib/zillabyte/cli/apps.rb

@@ -15,7 +15,7 @@ require 'net/http'
 class Zillabyte::Command::Apps < Zillabyte::Command::Flows
 
   MAX_POLL_SECONDS = 60 * 15
-  POLL_SLEEP = 0.5
+  POLL_SLEEP = 2.0
 
   # apps
   #
@@ -279,11 +279,15 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Flows
   def init
     
     name = options[:name] || shift_argument
+    error("name required") if name.nil? 
+    
     lang = options[:lang] || options[:language] || "ruby"
-    dir = options[:dir] || options[:directory] || File.join(Dir.pwd, name)
+    dir = options[:dir] || options[:directory] || Dir.pwd
+    remote = options[:remote] || "zillabyte"
+    
     dir = File.expand_path(dir) unless dir.nil?
     type = options[:output_type]
-
+    
     languages = ["ruby","python", "js"]
 
     error("Unsupported language #{lang}. Zillabyte currently supports #{languages.join(', ')}.", type) if not languages.include? lang
@@ -303,6 +307,8 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Flows
       
     end
     
+    # Add the git remote... 
+    add_git_remote(name, remote)
     
   end
 
@@ -380,7 +386,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Flows
       options.delete :is_name
 
       start = Time.now.utc
-      display "Next cycle request sent. If your app was RETIRED this may take a few minutes. Check 'zillabyte logs' for progress." if type.nil?
+      display "Next cycle request sent. If your app was RETIRED this may take a few minutes. Please wait or check 'zillabyte logs' for progress." if type.nil?
 
       while(Time.now.utc < start + MAX_POLL_SECONDS) do
 
@@ -439,24 +445,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Flows
   # --output_type OUTPUT_TYPE # specify an output type i.e. json
   #
   def kill
-    id = options[:id] || shift_argument
-    type = options[:output_type]
-
-    if id.nil?
-      id = read_name_from_conf(options)
-      options[:is_name] = true
-    elsif !(id =~ /^\d*$/)
-      options[:is_name] = true
-    end
-
-    display "Killing app ##{id}...please wait..." if type.nil?
-    api.apps.kill(id, options)
-
-    if type == "json"
-      display "{}"
-    else
-      display "App ##{id} killed"
-    end
+    super
   end
   
 

data/lib/zillabyte/cli/components.rb

@@ -15,7 +15,58 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
   def index
     self.list
   end
+
+
+
+  
   
+  # components:init [NAME]
+  #
+  # initializes a new component
+  #
+  # --lang LANG       # which language to use [ruby, python]. default 'ruby'.
+  # --dir DIR         # target directory of the app. 
+  # --remote REMOTE   # the git remote name. default 'zillabyte'
+  #
+  #Examples:
+  #
+  # $ zillabyte components:init domain_extractor --lang ruby
+  #
+  def init
+    
+    name = options[:name] || shift_argument
+    error("name required") if name.nil?
+      
+    lang = options[:lang] || options[:language] || "ruby"
+    dir = options[:dir] || options[:directory] || Dir.pwd
+    remote = options[:remote] || "zillabyte"
+    
+    dir = File.expand_path(dir) unless dir.nil?
+    type = options[:output_type]
+
+    languages = ["ruby","python", "js"]
+
+    error("Unsupported language #{lang}. Zillabyte currently supports #{languages.join(', ')}.", type) if not languages.include? lang
+
+    display "initializing empty #{lang} component in #{dir}" if type.nil?
+    erb_binding = binding
+    FileUtils.mkdir_p dir
+    
+    Dir[File.join(File.expand_path("../templates/components/#{lang}", __FILE__), "*")].each do |source_file|
+      
+      next if File.directory?(source_file)
+      erb = ERB.new(File.read(source_file))
+      erb.filename = source_file
+      
+      dest_file = File.join(dir, File.basename(source_file).gsub(/\.erb$/, ""))
+      File.open(dest_file, 'w') {|file| file.write(erb.result(erb_binding))}
+      
+    end
+    
+    # Add the git remote... 
+    add_git_remote(name, remote)
+    
+  end
 
 
   # components
@@ -25,7 +76,7 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
   def list
     type = options[:output_type]
     
-    headings = ["id", "name", "state", "inputs", "outputs"]
+    headings = ["id", "name", "state", "inputs", "outputs", "scope"]
     rows = api.component.list.map do |row|
       if headings.size == 0
         headings = row.keys 
@@ -90,24 +141,7 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
   # --output_type OUTPUT_TYPE # specify an output type i.e. json
   #
   def kill
-    id = options[:id] || shift_argument
-    type = options[:output_type]
-
-    if id.nil?
-      id = read_name_from_conf(options)
-      options[:is_name] = true
-    elsif !(id =~ /^\d*$/)
-      options[:is_name] = true
-    end
-
-    display "Killing component ##{id}...please wait..." if type.nil?
-    api.components.kill(id, options)
-
-    if type == "json"
-      display "{}"
-    else
-      display "Component ##{id} killed"
-    end
+    super
   end
 
   # components:info [DIR]
@@ -130,7 +164,7 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
   # --config CONFIG_FILE            # use the given config file
   # --input INPUT_FILE              # uses a CSV for component input
   # --output OUTPUT_FILE            # write output to a CSV
-  # --directory DIR                 # app directory
+  # --directory DIR                 # component directory
   #
   def test
     super
@@ -150,6 +184,7 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
   # components:errors ID
   #
   # show recent errors generated by the componeny rpc
+  #
   # --output_type OUTPUT_TYPE           # specify an output type i.e. json
   #
   def errors
@@ -169,13 +204,32 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
     super
   end
  
-  # components:rpc [ID] [QUERY_1_INPUT_1,QUERY_1_INPUT_2,...] [QUERY_2_INPUT_1,QUERY_2_INPUT_2,...], ...
+  
+  # components:rpc [ID] [INPUT_1] [INPUT_2] ...
   #
-  # runs a component as an rpc
+  # Submits a single query to the rpc. The input parameters MUST be
+  # listed in the same order as that given in the component inputs.
+  # TO SUBMIT MULTIPLE queries, use the --file switch to specify a
+  # csv file containing the queries without listing any inputs on
+  # the command line. Each line of the file should correspond to a
+  # unique query.
+  # 
+  # --async                    # run the command asynchronously
+  # --input_file  INTPUT_FILE  # input csv file containing parameters for multiple queries
+  # --output_file OUTPUT_FILE  # output csv file containing query parameters and run ids
+  #
+  # Examples: 
+  #
+  # Single query:
+  # $ zillabyte components:rpc 'web_screenshot' 'google.com'
+  #
+  # Multiple queries:
+  # $ zillabyte components:rpc 'web_screenshot' --file 'url_list.csv'
   #
-  # --output_type OUTPUT_TYPE # specify an output type i.e. json
   def rpc
+    
     component_id = options[:id] || shift_argument
+    async = options[:async] || false
 
     if component_id.nil?
       component_id = read_name_from_conf(options)
@@ -185,31 +239,91 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
     end
 
     type = options[:output_type]
+    file = options[:input_file]
+    out_file = options[:output_file]
 
     component_args = []
-    while(true) do
-      next_arg = shift_argument
-      break if next_arg.nil?
-      component_args << next_arg.split(",")
+    if file
+      CSV.foreach(file) {|line| component_args << line}
+    else
+      args = []
+      while(true) do
+        next_arg = shift_argument
+        break if next_arg.nil?
+        args << next_arg
+      end
+      component_args << args if !args.empty?
     end
 
-    res = api.components.rpc(component_id, {:rpc_inputs => component_args})
+    opts = {:rpc_inputs => component_args} if component_args.size != 0
+    res = api.components.rpc(component_id, opts)
+    
+    input_par_for_id = {}
     if res['error']
       error("error: #{res['error_message']}", type)
     else
       if type.nil?
-        display "Request submitted to component ##{res['id']}. The run ids are (query: run_id):"
-        res["execute_ids"].each do |pars, id|
-          display "\t #{pars}: #{id}"
+        display "Request submitted to component ##{res['id']}."
+      end
+
+      if res["execute_ids"]
+        input_par_for_id = res["execute_ids"].invert
+        if out_file
+          CSV.open(out_file, "w") {|csv| res["execute_ids"].to_a.each {|elem| csv << elem}}
+        end
+      end
+    end
+
+    if async
+      # Async -- return right away.. 
+      display "The run ids are (query: run_id):"
+      res["execute_ids"].each do |pars, id|
+        display "\t #{pars}: #{id}"
+      end
+      display "Please use \"zillabyte components:results [run id]\" to check on the status of your query and to retrieve your results.\nYou may also wish to check \"zillabyte logs\" for errors."
+    else
+      # Sync -- poll until it's done...
+      if res["execute_ids"] 
+        run_ids = res["execute_ids"].values
+        while true
+          res = api.components.get_rpc_results(component_id, {:execute_ids => run_ids})
+          if res['error']
+            error("error: #{res['error_message']}", type)
+          else
+            if res["results"].detect{|id, res| ["running", "waiting"].member?(res["status"].downcase)}
+              # poll again
+              sleep(2)
+            else
+              if type.nil?
+                # All results are done (or rather, not 'running). Display them
+                res["results"].each do |id, hash|
+                  display "Query #{input_par_for_id[id]}:"
+                  if hash["data"].size == 0
+                    display "\t NO OUTPUT"
+                  else
+                    display "\t #{hash["data"].to_json()}"
+                  end
+                end
+              end
+              break
+            end
+          end
         end
-        display "Please use \"zillabyte components:results [run id]\" to check on the status of your query and to retrieve your results.\nYou may also wish to check \"zillabyte logs\" for errors."
       end
     end
   end
+  alias_command "rpc", "components:rpc"
 
-  # components:results [ID] [RUN_ID]
+
+
+  # components:results [ID] [RUN_ID_1] [RUN_ID_2] ...
   #
-  # gets the result of an rpc request if it is done running, otherwise gets current run status
+  # Gets the result of rpc requests corresponding to RUN_ID if it is 
+  # done running, otherwise gets current run status. A list of RUN_IDs
+  # may also be specified in an input csv file like that generated by
+  # components:rpc with the --output_file flag.
+  # 
+  # --input_file  INTPUT_FILE  # input csv file containing query parameters and run ids (this can be generated automatically by components:rpc using the --output_file flag)
   #
   def results
     component_id = options[:id] || shift_argument
@@ -222,12 +336,19 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
     end
 
     type = options[:output_type]
+    file = options[:input_file]
 
     run_ids = []
-    while(true) do
-      next_arg = shift_argument
-      break if next_arg.nil?
-      run_ids << next_arg
+    if file
+      CSV.foreach(file) do |line|
+        run_ids << line[1]
+      end
+    else
+      while(true) do
+        next_arg = shift_argument
+        break if next_arg.nil?
+        run_ids << next_arg
+      end
     end
 
     res = api.components.get_rpc_results(component_id, {:execute_ids => run_ids})
@@ -237,9 +358,16 @@ class Zillabyte::Command::Components < Zillabyte::Command::Flows
       if type.nil?
         res["results"].each do |id, hash|
           display "#{id}: #{hash["status"]}"
-          display "\t #{hash["data"]}" if hash["status"] == "complete"
+          if hash["status"] == "complete"
+            if hash["data"].size == 0
+              display "\t NO OUTPUT"
+            else
+              display "\t #{hash["data"]}"
+            end
+          end
         end
       end
     end
   end
+  
 end