zillabyte-cli 0.0.12 → 0.0.13

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NDQxYTUwOWJlM2ZjNDc2NjEyZGYxYzIxMjJiOGNhOGIxMzFiM2YzMg==
+  data.tar.gz: !binary |-
+    ZTFlNWQ5MDYwMjFlYjdlMmY0MGMwNmQ5MTNkNTg0ZDExMjdmNDY5Mw==
+SHA512:
+  metadata.gz: !binary |-
+    MzhlMTRkYmUxYjRmNDljNDEyMzYwMGQxZWMzMDNhZGJkZmY4OWM3ZmViM2Mx
+    ZTIwOGU3OGVmNjBhN2Q5NTVjYTY2NDMyYTlmYWRhYjYzNTQ1Y2EzYWJmYzRm
+    NjYwNzAyNjlhNzVkMDQyM2JhNGEzNDM5ODJmNjg3NTE4YWJmMzE=
+  data.tar.gz: !binary |-
+    MGRhYWEyYzU1NWE5OWFjZjMxMzkwZDY4NWJiMzQ0YzU3YTkxOWMzZWRiZmY5
+    YTcyODcyOTliZjk5MjRhMWJlMWZlNWZiNmJhZWI0MTI1ODZjNWU0NzU4ZmM5
+    NjI4OGNlMzkyNmZjNzFlNTJmNTE4NTViZjk5NTI0YzliNWQ5YWI=

data/lib/zillabyte/cli/#logs.rb#

@@ -0,0 +1,12 @@
+require "zillabyte/cli/base"
+
+# manage custom logs
+#
+class Zillabyte::Command::Logs < Zillabyte::Command::Base
+
+
+
+  
+  
+
+end

data/lib/zillabyte/cli/flows.rb

@@ -6,6 +6,7 @@ require 'indentation'
 require 'open3'
 require 'securerandom'
 require 'colorize'
+require 'time_difference'
 
 # manage custom flows
 #
@@ -51,6 +52,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   #
   def push
     
+    since = Time.now.utc.to_s
     dir = options[:directory] || shift_argument || Dir.pwd
     res = api.flows.push_directory dir, progress, options
     
@@ -59,6 +61,24 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     else
       display "flow ##{res['id']} #{res['action']}"
     end
+
+    display "Starting up your flow...please wait..."
+    previous_logs = {}
+    while true
+      all_logs = api.logs.get(res['id'], "_ALL_", {:since => since})
+      all_logs.each_pair do |op, logs|
+        next if logs.length == 0
+        difference = previous_logs[op] ? logs - previous_logs[op] : logs
+        difference.each do |log|
+          if ["start_up", "error"].include?(log["category"])
+            display log["line"]
+          end
+          exit(0) if log["line"].downcase.include?("flow deployed")
+        end
+        previous_logs[op] = logs
+      end
+      sleep(0.5)
+    end
     
   end
   alias_command "push", "flows:push"
@@ -199,7 +219,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
 
 
 
-  # flow:show FLOW_ID [OPERATION]
+  # flow:show FLOW_ID [OPERATION_NAME]
   #
   # streams logs from the distributed workers
   # 
@@ -211,9 +231,9 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     flow_id = options[:flow] || shift_argument
     operation_id = options[:operation] || shift_argument || '_ALL_'
     tail = options[:tail]
-    priority = options[:verbose] || 'debug'
+    category = options[:verbose] || '_ALL_'
     carry_settings = {
-      :priority => priority
+      :category => category
     }
     api_options = {}
     
@@ -255,16 +275,27 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     else
       # List the flows
       response = api.flows.list_cycles(flow_id)
-
+      # TODO List the sequence number for this flow.
+      display "Most recent cyles of the flow:"
+      headings = ["State", "Start", "End", "Duration"]
+      rows = response["cycles"] 
+      rows = rows.map do |row|
+        start_time = DateTime.parse(row["start"]).strftime("%m/%d/%Y %I:%M%p")
+        end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
+        duration = end_time == "---" ? "---" : TimeDifference.between(DateTime.parse(row["start"]), DateTime.parse(row["end"])).in_minutes.to_s + " minutes"
+        [row["state"], start_time, end_time , duration ] #TODO Pretty print time
+      end
+      display Terminal::Table.new(:headings => headings, :rows => rows).to_s
+      display "Total number of cycles executed: #{response['total']}"
+    end
+    if response["body"]
+      display  response["body"]
+    elsif  response["error"]
+      error response["error"]
     end
-
-    
-    display response
   end
   
 
-
-
   # flows:test [TEST_DATASET_ID]
   #
   # tests a local flow with sample data
@@ -739,28 +770,29 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   
   def show_log_output(lines, settings = {})
     
-    @colors ||= [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
-
     max_category_size = 0
+    settings["start_up"] = :green
+    settings["kill"] = :yellow
+    settings["ipc"] = :cyan
+    settings["error"] = :red
+    settings["system"] = :white
+    settings["run"] = :magenta
     all_lines = []
     
     lines.each_pair do |operation, lines|
-      settings[operation] ||= {}      
-      settings[operation][:color] ||= @colors.shift
       lines.each do |line_h|
         
         line = line_h['line']
-        priority = line_h['priority']
+        category = line_h['category']
         date = line_h['date']
         
         all_lines << {
           :line => line,
-          :category => operation,
-          :priority => priority,
-          :color => settings[operation][:color],
+          :category => category,
+          :color => settings[category],
           :date => date
         }
-        max_category_size = [max_category_size, operation.size].max
+        max_category_size = [max_category_size, category.size].max
         
         if settings[:since]
           settings[:since] = [date, settings[:since]].max
@@ -776,22 +808,25 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
       a[:date] <=> b[:date]
     end
     
-    requested_priority = settings[:priority]
+    requested_category = settings[:category]
 
     all_lines.each do |h|
       color = h[:color]
       
-      case h[:priority]
-      when 'error'
-        color = :red if h[:priority] == 'error'
-      when 'info'
-        next if ['error'].member?(requested_priority)
-      when 'system'
-      when 'debug'
-        next if ['info', 'error'].member?(requested_priority)
+      case h[:category]
+      when nil
+        next
+      when 'start_up'
+        next if ['kill', 'run', 'ipc'].member?(requested_category)
+      when 'run'
+        next if ['kill', 'start_up', 'ipc'].member?(requested_category)
+      when 'ipc'
+        next if ['kill', 'start_up', 'run'].member?(requested_category)
+      when 'kill'
+        next if ['start_up', 'run', 'ipc'].member?(requested_category)
       end
       
-      display "#{h[:category].rjust(max_category_size)} #{h[:priority].rjust(6)} - #{h[:line]}".colorize(color)
+      display "#{h[:category].rjust(max_category_size)} - #{h[:date]} UTC - #{h[:line]}".colorize(color)
     end
     
     settings

data/lib/zillabyte/cli/relations.rb

@@ -33,6 +33,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     rows = response.body.map do |row|
       headings = row.keys if headings.size == 0
       row["columns"] = row["columns"].map{|c| c['type']}.join(',')
+      row["aliases"] = row["aliases"].map{|a| a['name']}.join(',')
       vals = row.values_at *headings
       vals 
     end
@@ -124,11 +125,12 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
   #
   # creates a new relation
   #
-  # --schema SCHEMA              # a JSON object of column types
+  # --schema SCHEMA              # Column names and types in the format "field_1:type_1,field_2:type_2,..."
   # --public SCOPE               # Make the relation public
   # --file FILE                  # A data file 
   # --type FILE_TYPE             # type of data file [csv (default), xlsx]
   # --description DESCRIPTION    # Description of relation contents
+  # --aliases ALIASES            # Relation name aliases in the format "alias_1,alias_2,..."
   #
   def create
     
@@ -141,8 +143,9 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     schema = options[:schema] if options[:schema]
     is_public = options[:public] || nil
     description = options[:description] || nil
+    aliases = options[:aliases] || nil
    
-    hash = set_relation_properties(schema,is_public,description)
+    hash = set_relation_properties(schema,is_public,description,aliases)
     if file
       type ||= File.extname(file).gsub(".", "")
       rows = sanity_check_file(file,type, {"columns" => hash[:schema]})
@@ -190,8 +193,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
 
   # relations:pull ID OUTPUT
   #
-  # pulls relation data into OUTPUT
-  # if s3 parameters specified, pulls relation data to S3_BUCKET/OUTPUT/
+  # pulls relation data into OUTPUT.gz
   #
   # --type TYPE # interpret file as type [csv, xlsx]
   #
@@ -201,8 +203,8 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     file = options[:file] || shift_argument
     error "no id given" if id.nil?
     error "no output file given" if file.nil?
+    file = file+".gz"
     type = options[:type]
-    type ||= File.extname(file).gsub(".", "")
 
     res = self.api.data.pull(id)
     if(res["uri"])
@@ -239,7 +241,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
 
   # relations:pull:s3 ID S3_KEY S3_SECRET S3_BUCKET S3_FILE_PATH
   #
-  # pulls relation data to S3_BUCKET/FILE_PATH/
+  # pulls relation data to S3_BUCKET/FILE_PATH/part***.gz
   #
   # --type TYPE # interpret file as type [csv, xlsx]
   #
@@ -331,7 +333,27 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
 
 
 
-  def set_relation_properties(schema,is_public,description)
+  def set_relation_properties(schema,is_public,description,aliases)
+    if aliases.nil?
+      display "If you would like to give alias names to your relation, please enter them below. (comma separated, only letters, number and underscore allowed)"
+      aliases = ask.strip
+    end
+    while true
+      aliases = aliases.strip.split(",").map{|x| x.strip}.uniq
+      all_valid = true
+      aliases.each do |a|
+        if(!(a =~ /^[a-zA-Z0-9\_]+$/i))
+          display "\"#{a}\" contains illegal characters. Only letters, numbers and underscore are allowed in alias names."
+          display "If you would like to give alias names to your relation, please enter them below. (comma separated, only letters, number and underscore allowed)"
+          aliases = ask.strip
+          all_valid = false
+          break
+        end
+      end
+      break if all_valid
+    end
+    display "Will use the following aliases for this relation: #{aliases}" if !aliases.empty?
+
     if description.nil?
       display "Enter a description for your relation (defaults to empty string)."
       description = ask.strip
@@ -389,7 +411,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
       end
     end
 
-    {:schema => schema, :public => is_public, :description => description}
+    {:schema => schema, :public => is_public, :description => description, :aliases => aliases}
   end
 
   def sanity_check_file(file,type,dataset)

data/lib/zillabyte/cli/templates/python/#simple_function.py#

@@ -0,0 +1,27 @@
+import zillabyte
+
+def prep(controller):
+  return
+
+# This is the heart of your algorithm.  It's processed on every
+# web page.  This algorithm is run in parallel on possibly hundreds
+# of machines.
+def execute(controller, tup):
+  if("hello world" in tup.values["html"]):
+    controller.emit("has_hello_world",{"url":tup.values["url"]})
+  return
+
+zillabyte.simple_function(\
+  # This directive instructs zillabyte to give your function every
+  # web page in our known universe.  Your function will have access
+  # to two fields: URL and HTML
+  matches = "select * from web_pa", \
+
+  # This directive tells Zillabyte what kind of data your function
+  # produces.  In this case, we're saying we will emit a tuple that
+  # is one-column wide and contains the field 'URL' 
+  emits = [["has_hello_world", [{"url":"string"}]]], \
+
+  prepare = prep, \
+  execute = execute\
+)

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.0.12"
+    VERSION = "0.0.13"
   end
-end
+end

data/zillabyte-cli.gemspec

@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ascii_charts", "~> 0.9.1"
   spec.add_dependency "indentation", "~> 0.1"
   spec.add_dependency "aws-sdk", "~> 1.33.0"
+  spec.add_dependency "time_difference"
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.12
-  prerelease: 
+  version: 0.0.13
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-24 00:00:00.000000000 Z
+date: 2014-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -110,7 +97,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -118,7 +104,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -126,7 +111,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -134,7 +118,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -142,7 +125,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: chronic
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -150,7 +132,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -158,7 +139,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: ascii_charts
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -166,7 +146,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -174,7 +153,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: indentation
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -182,7 +160,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -190,7 +167,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -198,11 +174,24 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.33.0
+- !ruby/object:Gem::Dependency
+  name: time_difference
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: The Official Zillabyte CLI Gem
 email:
 - gem@zillabyte.com
@@ -225,6 +214,7 @@ files:
 - lib/zillabyte/api/zillalogs.rb
 - lib/zillabyte/api.rb
 - lib/zillabyte/auth.rb
+- lib/zillabyte/cli/#logs.rb#
 - lib/zillabyte/cli/auth.rb
 - lib/zillabyte/cli/base.rb
 - lib/zillabyte/cli/config.rb
@@ -238,6 +228,7 @@ files:
 - lib/zillabyte/cli/sources.rb
 - lib/zillabyte/cli/templates/js/simple_function.js
 - lib/zillabyte/cli/templates/js/zillabyte.conf.yaml
+- lib/zillabyte/cli/templates/python/#simple_function.py#
 - lib/zillabyte/cli/templates/python/requirements.txt
 - lib/zillabyte/cli/templates/python/simple_function.py
 - lib/zillabyte/cli/templates/python/zillabyte.conf.yaml
@@ -255,7 +246,6 @@ files:
 - lib/zillabyte/helpers.rb
 - lib/zillabyte/queries.rb
 - lib/zillabyte-cli/version.rb
-- lib/zillabyte-cli/version.rb~
 - lib/zillabyte-cli.rb
 - LICENSE
 - README.md
@@ -266,26 +256,25 @@ files:
 homepage: http://www.zillabyte.com
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.1.10
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: The Official Zillabyte CLI Gem
 test_files: []

data/lib/zillabyte-cli/version.rb~

@@ -1,5 +0,0 @@
-module Zillabyte
-  module CLI
-    VERSION = "0.0.11"
-  end
-end