zillabyte-cli 0.9.36 → 0.9.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 408d5fdeafe9e0a644166eb8256b1b53acc31443
-  data.tar.gz: 3e77adae220ec1e1db02255818438a48050c2c57
+  metadata.gz: bd191d13ac7c5d6d01086ece5b6d708ce181ca46
+  data.tar.gz: 7df48d7dd5cf14ff444460fae31dbdfdc8c7dfa0
 SHA512:
-  metadata.gz: 41b6dbacd035e30232f0d7b00264ef60ae34fbc23753619bc4c5867430b520549cde60c27a963d41cb754cc0aaa016e294529332854640e03ef3acb1e6015c9c
-  data.tar.gz: 33d9b3b2f732a984d006c68590b001d6fb41c1b42d38ff7e20110adbfa8e5bd8f1c6b61dbe700744ca8677b17e284ea1623ee1d56431d55c86913118705cbe4e
+  metadata.gz: 7c9a0577619d019f5f5135e0e2f55cd36cc940234d64f755d1b81c9d9acd393bcc09d3c4b9d86037f96848212b09e9aec9f7f1961d243b53c4b887a3f9d73812
+  data.tar.gz: 639ef72ed7257daa641b6f38aea8f8b820cfda6ea366570e69bc17333efba865983f6261e54206bb651475a88c700fe74dfdd7d3689d7d0259f342ef5ee53c6f

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.9.36"
+    VERSION = "0.9.37"
   end
 end

data/lib/zillabyte/api/flows.rb

@@ -49,6 +49,115 @@ class Zillabyte::API::Flows < Zillabyte::API::Base
 
   end
 
+ # POST /flows/id/scale
+  def create_scale(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/flows/#{id}/scale",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+  # GET /flows/id/scale
+  def poll_scale(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{id}/scale",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+
+   # POST /flows/id/pause
+  def create_pause(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/flows/#{id}/pause",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+  # GET /flows/id/pause
+  def poll_pause(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{id}/pause",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+     # POST /flows/id/kill
+  def create_resume(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/flows/#{id}/resume",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+  # GET /flows/id/kill
+  def poll_resume(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{id}/resume",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
+
   # POST /flows/id/delete
   def create_delete(id, options = {})
 
@@ -85,6 +194,21 @@ class Zillabyte::API::Flows < Zillabyte::API::Base
 
   end
 
+
+  # GET /flows/id/config
+  def get_meta(id, options = {})
+
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{id}/config",
+      :body     => options.to_json
+    )
+
+    res.body
+
+  end
+
   # GET /flows/id/download
   def pull_to_directory(id, dir, session = nil, options = {})
     require("zillabyte/common/tar")

data/lib/zillabyte/cli.rb

@@ -7,7 +7,9 @@ module Zillabyte; module CLI
       command = args.shift().strip() rescue "help"
       require("zillabyte/command")
       Zillabyte::Command.load(command)
+      thr = Thread.new{Zillabyte::Command.tracker(command, args)}
       Zillabyte::Command.run(command, args)
+      thr.join
     rescue Interrupt
       `stty icanon echo`
       Zillabyte::Helpers.error("Command cancelled.")

data/lib/zillabyte/cli/apps.rb

@@ -289,6 +289,34 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Flows
   def test
     super
   end  
+
+
+  # apps:pause [ID]
+  #
+  # Pause an app
+  # 
+  # 
+  def pause
+    super
+  end  
+
+  # apps:resume [ID]
+  #
+  # Resume a paused app
+  # 
+  # 
+  def resume
+    super
+  end  
+
+
+  # apps:scale ID OPERATION_NAME PARALLELISM
+  #
+  # Change the parallelism of of app or operation
+  def scale
+    super
+  end
+  
   
   
   

data/lib/zillabyte/cli/data.rb

@@ -280,7 +280,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
   #
   # Pulls dataset into OUTPUT.gz.
   #
-  # --cycle_id [cycle_id]                       # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle]
+  # --version_id [version_id]                   # Retrieve data generated with a specific version of the app if the dataset is associated with an app [default: last version]
   # --output_type OUTPUT_TYPE                   # Specify an output type i.e. json #HIDDEN
   #
   def pull
@@ -324,7 +324,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
   #   2) s3://s3_bucket/s3_key: also supply --s3_access and --s3_secret OR set the environment variables S3_ACCESS and S3_SECRET
   #   3) s3_key: also supply --s3_access, --s3_secret and --s3_bucket OR set the environment variables S3_ACCESS and S3_SECRET and supply --s3_bucket
   #
-  # --cycle_id [cycle_id]                       # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle]
+  # --version_id [version_id]                   # Retrieve data generated with a specific version of the app if the dataset is associated with an app [default: last version]
   # --s3_access [s3_access_key]                 # S3 access key
   # --s3_secret [s3_secret_key]                 # S3 secret key
   # --s3_bucket [s3_bucket]                     # S3 bucket to store data at
@@ -362,7 +362,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
 
     s3_params = {:s3_access_key => s3_access, :s3_secret => s3_secret, 
                  :s3_bucket => s3_bucket, :s3_file_key => s3_key}
-    s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
+    s3_params[:version_id] = options[:version_id] if options[:version_id]
 
     res = self.api.data.pull_to_s3(id, s3_params)
 
@@ -382,10 +382,9 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
 
   # data:show ID
   #
-  # Shows a sample of the dataset. See 'zillabyte queries' for 
-  # more elaborate functionality.
+  # Shows a sample of the dataset. 
   #
-  # --cycle_id [cycle_id]                       # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle]
+  # --version_id [version_id]                   # Retrieve data generated with a specific version of the app if the dataset is associated with an app [default: last version]
   # --no_truncation                             # Don't truncate long strings
   # --meta                                      # Show metadata columns (since, confidence, source)
   # --output_type OUTPUT_TYPE                   # Specify an output type i.e. json #HIDDEN

data/lib/zillabyte/cli/download.rb

@@ -5,7 +5,7 @@ require("zillabyte/cli/base")
 class Zillabyte::Command::Download < Zillabyte::Command::Base
 
   MIN_LOCAL_JAR_VERSION = "0.0.3"
-  LATEST_JAR_VERSION = "0.9.36"
+  LATEST_JAR_VERSION = "0.9.37"
   MIN_JAVA_VERSION = "1.7.0"
 
   # --local                                   # HIDDEN

data/lib/zillabyte/cli/flows.rb

@@ -398,7 +398,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     heading1 = ["type", "name"]
 
     # a level deeper
-    heading2 = ["consumed", "emitted", "errors"]
+    heading2 = ["consumed", "emitted", "errors", "parallelism"]
 
     # for the table output
     headings = heading1 + heading2
@@ -415,10 +415,13 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     # get the values for the headings
     stats = instances.map do |instance|
 
+
       heading2.map do |heading|
         instance["stats"][heading]
       end
 
+
+
     end
 
     # combine the arrays
@@ -456,7 +459,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     end
 
     # rows to be output to the CLI
-    rows = rows.group_by{ |type, name, consumed, emitted, errors| name }.map{ |hashkey, array| [array[0][0], hashkey, array.inject(0){ |sum, i | sum + i[2].to_i }, array.inject(0){ |sum, i | sum + i[3].to_i },array.inject(0){ |sum, i | sum + i[4].to_i }]    }
+    rows = rows.group_by { |type, name, consumed, emitted, errors, parallelism| name }.map{ |hashkey, array| [array[0][0], hashkey, array.inject(0){ |sum, i | sum + i[2].to_i }, array.inject(0){ |sum, i | sum + i[3].to_i },array.inject(0){ |sum, i | sum + i[4].to_i }, array.inject(0){|value, i| i[5].to_i}]}
     # example sums: [["sink", "has_facebook_likebox_and_vwo", 73, 0, 0], ["source", "sharder.1", 16, 10367872, 1], ["each", "each_3", 10367872, 73, 0]]
 
     # calculate percentage for the user to see
@@ -829,6 +832,213 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   alias_command "local", "flows:local"
 
 
+
+  # flows:pause ID
+  #
+  # Pause a flow
+  # 
+  def pause
+    id = options[:id] || shift_argument
+    type = options[:output_type]
+
+    if id.nil?
+      id = read_name_from_conf(options)
+    end
+
+    display "Pausing flow ##{id}...please wait..." if type.nil?
+    response = api.flows.create_pause(id, options)
+
+    if response["job_id"]
+      options[:job_id] = response["job_id"]
+      flow_id = response["flow_id"]
+
+      start = Time.now.utc
+      display "Pause request sent. Please wait..." if type.nil?
+
+      while(Time.now.utc < start + MAX_POLL_SECONDS) do
+
+        # Poll
+        res = self.api.flows.poll_pause(flow_id, options)
+
+        case res['status']
+        when 'completed'
+          if res['return']
+            if type == "json"
+              display "{}"
+            else
+              display "Flow ##{id} paused"
+            end
+          else
+            throw "something is wrong: #{res}"
+          end
+          # success! continue below
+          break
+        when 'running'
+          sleep(POLL_SLEEP)
+       #   display ".", false
+        else
+          throw "unknown status: #{res}"
+        end
+
+      end
+    elsif response["error"]
+      error(response["error"], type)
+    else
+      error("remote server error (f569)", type)
+    end
+
+  end
+  alias_command "pause", "flows:pause"
+
+  # flows:resume
+  #
+  # Resume a paused flow ID
+  def resume
+    id = options[:id] || shift_argument
+    type = options[:output_type]
+
+    if id.nil?
+      id = read_name_from_conf(options)
+    end
+
+    display "Resume flow ##{id}...please wait..." if type.nil?
+    response = api.flows.create_resume(id, options)
+
+    if response["job_id"]
+      options[:job_id] = response["job_id"]
+      flow_id = response["flow_id"]
+
+      start = Time.now.utc
+      display "Resume request sent. Please wait..." if type.nil?
+
+      while(Time.now.utc < start + MAX_POLL_SECONDS) do
+
+        # Poll
+        res = self.api.flows.poll_resume(flow_id, options)
+
+        case res['status']
+        when 'completed'
+          if res['return']
+            if type == "json"
+              display "{}"
+            else
+              display "Flow ##{id} resumed"
+            end
+          else
+            throw "something is wrong: #{res}"
+          end
+          # success! continue below
+          break
+        when 'running'
+          sleep(POLL_SLEEP)
+       #   display ".", false
+        else
+          throw "unknown status: #{res}"
+        end
+
+      end
+    elsif response["error"]
+      error(response["error"], type)
+    else
+      error("remote server error (f569)", type)
+    end
+  end
+  alias_command "resume", "flows:resume"
+
+
+
+
+
+  # apps:scale ID OPERATION_NAME PARALLELISM
+  #
+  # Change the parallelism of of app or operation
+  def scale
+    id = options[:id] || shift_argument
+
+    if id.nil?
+      id = read_name_from_conf(options)
+    end
+
+    name = options[:name] || shift_argument
+    parallelism = options[:parallelism] || shift_argument
+    type = options[:output_type]
+
+    # Read current metadata from API
+    display "Fetching configuration for flow ##{id}...please wait..." if type.nil?
+    options[:flow_id] = id
+
+    response = api.flows.get_meta(id, options)
+    if response["meta"]
+      meta = response["meta"]
+      nodes = meta["nodes"]
+
+      # Set new parallelism if necessary
+      scaling = false
+      nodes.each_with_index do |node, i|
+
+        if node["name"] == name && node["parallelism"] != parallelism
+          meta["nodes"][i]["parallelism"] = parallelism
+          scaling = true
+          break
+        end
+      end
+
+      if scaling
+        display "Scaling flow ##{id}...please wait..." if type.nil?
+        options[:flow_config] = meta
+        response = api.flows.create_scale(id, options)
+
+        if response["job_id"]
+          options[:job_id] = response["job_id"]
+          flow_id = response["flow_id"]
+
+          start = Time.now.utc
+          display "Scaling request sent. Please wait..." if type.nil?
+
+          while(Time.now.utc < start + MAX_POLL_SECONDS) do
+
+            # Poll
+            options.delete(:flow_config)
+            res = self.api.flows.poll_scale(flow_id, options)
+
+            case res['status']
+            when 'completed'
+              if res['return']
+                if type == "json"
+                  display "{}"
+                else
+                  display "Flow ##{id} scaled"
+                end
+              else
+                throw "something is wrong: #{res}"
+              end
+              # success! continue below
+              break
+            when 'running'
+              sleep(POLL_SLEEP)
+           #   display ".", false
+            else
+              throw "unknown status: #{res}"
+            end
+
+          end
+        elsif response["error"]
+          error(response["error"], type)
+        else
+          error("remote server error (f569)", type)
+        end
+      else
+        error("Unable to find operation #{name}", type)
+      end
+    elsif response["error"]
+      error(response["error"], type)
+    end
+
+  end
+  alias_command "scale", "flows:scale"
+  
+
+
   # flows:kill ID
   #
   # Kills the given flow.

data/lib/zillabyte/command.rb

@@ -222,8 +222,66 @@ module Zillabyte
         ].compact.join("\n"))
       end
     end
+    
+    def self.tracker(cmd,arguments=[])
+      # tell us what the user is doing so we can get better.
+      options = {}
+
+      # get user id AND/OR install id
+      ## check for user id.
+
+      require("netrc")
+
+      default_path = Netrc.default_path
+      if File.exists?(default_path)
+        encrypted = default_path + ".gpg"
+        if File.exists?(encrypted)
+          default_path = encrypted
+        end
+
+        if Netrc.read(Netrc.default_path)["api.zillabyte.com"]
+          options[:user_id] = Netrc.read(Netrc.default_path)["api.zillabyte.com"][1]
+          options[:email] = Netrc.read(Netrc.default_path)["api.zillabyte.com"][0]
+        end
+      end
+
+      ## check for install id.
+      location = "#{Dir.home}/.zillabyte/.install_id"
+
+      unless File.file?(location)
+        # it doesn't exist, so create one.
+        require "securerandom"
+        uuid_home = "#{Dir.home}/.zillabyte/"
+        FileUtils.mkdir_p(uuid_home)
+        File.open(location,'w') { |f| f.write(SecureRandom.uuid)}
+      end
+
+      options[:install_id] = File.read(location)
+      options[:cli_command] = cmd
+      options[:date] = Time.now
+      options[:args] = arguments.to_s
+      # hit our server
+      begin
+        require 'excon'
+        Excon.new(
+          "http://tracking.zillabyte.com",
+          :headers => {
+            'Content-Type'          => "application/json",
+          }
+        ).request(
+          :expects  => 200,
+          :method   => :post,
+          :path     => "/track",
+          :body     => options.to_json
+        )
+      rescue Exception => e
+        # do nothing
+      end
+    end
+      
 
     def self.run(cmd, arguments=[])
+      
       require("excon/errors")
       require("zillabyte/helpers")
       extend(Zillabyte::Helpers)
@@ -243,6 +301,7 @@ module Zillabyte
           error "remote server error: #{e.message}"
         end
       end
+            
     end
 
     def self.parse(cmd)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte-cli
 version: !ruby/object:Gem::Version
-  version: 0.9.36
+  version: 0.9.37
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-22 00:00:00.000000000 Z
+date: 2014-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake