zillabyte-cli 0.1.6 → 0.1.7

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Nzg4MjUyOWI3MDVkZGMwZjUyNWI2YmUzOTIyMjMyYjRhZWFmMTA1MA==
+    MjFiZWM2YzdkZGQwZTI0OWQzOWVlMjViNTM5YmY0NmVlMDM4MjFjMg==
   data.tar.gz: !binary |-
-    MGY3ODY4NTU3MzIyZmIxZDcxMmIzODY1MjYyZDgxNmE5N2IxYjk3Yg==
+    OGQ1NjRiYTQ2M2E0MWMyMzFhOTU3Yzg0MTMxZjJhODdiN2YyNzJkNA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NzQzZGRhNmYwOWFhMzVjM2RkODFhZjQ3NjNmYzNkNmIyY2JlZDg3Mjg5YTNh
-    YTRjZTgwODY0MjZiNDgxNzhjMjUzMDg5OWRiNmExYmZiNzU0MTM3MGFiOWMw
-    Nzc4YTU3NDJjZGY5ZDFkN2Q5MjBiNjM2ODY3YjE5OWUwNjc0NmU=
+    ZWRlMDM0NDc2NDhjODc2MDhiMDllZGQwOTc0YmE5YWRiYjYxMzg3YzZiNmEz
+    ZjMyYWFmNTM0NTE4NmU4ZmM2ODIwMWNlMWYyNjQ4YWE0OTkxMzMxYTczZTMw
+    MzJkMzE3MDc5MDAzMjc4YTU5MDcyYmJkNjY4NzExNjQ0NjIzOGY=
   data.tar.gz: !binary |-
-    MzQyMWNjYTViOTYwMjgzNjc4ZmY2M2U3ZTM1ZjM0NDgyYTI5M2EyMjcxMzQ1
-    MWVkNGZlMDM4ZmY2ZTVkZTJkOGVlNDM5ZDlmNzdhMDBjY2IzYjU2N2FjYmE0
-    ZjgzNzcxYzY2MTQzMTY2NjIwYzVkOTU0YTVlOThhZTNkY2ZiOTA=
+    OTY1NjVlYzYxZTQxMmY1MjQ5MjQxNTcyM2ZlNmI3MDFiYTM2YWNmYTA2NjYy
+    ZmYxYTNmOWYwMDZlNTgwMjI1NzBiM2FhNWVlNzk5ZmEyYTk4NTNiZTFjN2Vj
+    N2Q2ZjhlNjUyYzkxNmUxMDQ0ZjNkYWQzNTVhOGY4NTEwODA3Nzk=

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.1.6"
+    VERSION = "0.1.7"
   end
 end

data/lib/zillabyte/cli/apps.rb

@@ -364,7 +364,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Flows
       # List the apps
       response = api.apps.list_cycles(app_id, options)
       # TODO List the sequence number for this app.
-      display "Most recent cyles of the app:" if type.nil?
+      display "Most recent cycles of the app:" if type.nil?
       headings = ["Cycle_id", "State", "Start", "End"]
       rows = response["cycles"] 
       rows = rows.map do |row|

data/lib/zillabyte/cli/data.rb

@@ -252,11 +252,19 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
   alias_command "dataset:download", "data:pull"
   
 
-  # data:pull:s3 ID S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
+  # data:pull:s3 ID S3_PATH
   #
-  # Pulls dataset to S3_BUCKET/FILE_KEY/part***.gz.
+  # Pulls dataset to s3_bucket/s3_key/part***.gz using the given s3_access and s3_secret credentials.
+  # S3_PATH may be given in the following forms:
+  #   1) s3://s3_access:s3_secret@s3_bucket/s3_key
+  #   2) s3://s3_bucket/s3_key: also supply --s3_access and --s3_secret OR set the environment variables S3_ACCESS and S3_SECRET
+  #   3) s3_key: also supply --s3_access, --s3_secret and --s3_bucket OR set the environment variables S3_ACCESS and S3_SECRET and supply --s3_bucket
   #
   # --cycle_id [cycle_id]                       # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle]
+  # --s3_access [s3_access_key]                 # S3 access key
+  # --s3_secret [s3_secret_key]                 # S3 secret key
+  # --s3_bucket [s3_bucket]                     # S3 bucket to store data at
+  # --s3_key [s3_file_key]                      # S3 key to store data at
   # --output_type OUTPUT_TYPE                   # Specify an output type i.e. json #HIDDEN
   #
   def pull_to_s3
@@ -265,17 +273,29 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
     type = options[:output_type]
     error("no id given", type) if id.nil?
 
-    user_s3_access_key = options[:s3_access_key] || shift_argument
-    user_s3_secret = options[:s3_secret] || shift_argument
-    user_s3_bucket = options[:s3_bucket] || shift_argument
-    user_s3_file_key = options[:s3_file_key] || shift_argument
-    error("no s3 access key provided", type) if user_s3_access_key.nil?
-    error("no s3 access secret provided", type) if user_s3_secret.nil?
-    error("no s3 access bucket provided", type) if user_s3_bucket.nil?
-    error("no s3 file path provided", type) if user_s3_file_key.nil?
-
-    s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret, 
-                 :s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
+    s3_path = options[:s3_path] || shift_argument
+    matches = s3_path.match(/^s3:\/\/([A-Z0-9]{20}):([$-\/:-?{-~!"^_`\[\]\w]{40})@([\w\.]*)\/([-\w\/]*)$/)
+    if !matches.nil?
+      s3_access, s3_secret, s3_bucket, s3_key = matches.captures
+    else
+      matches = s3_path.match(/^s3:\/\/([\w\.]*)\/([-\w\/]*)$/)
+      if !matches.nil?
+        s3_bucket, s3_key = matches.captures
+      else
+        s3_key = s3_path
+      end
+    end
+    s3_access = options[:s3_access] || ENV["S3_ACCESS"] if s3_access == "" or s3_access.nil?
+    s3_secret = options[:s3_secret] || ENV["S3_SECRET"] if s3_secret == "" or s3_secret.nil?
+    s3_bucket = options[:s3_bucket] if s3_bucket == "" or s3_bucket.nil?
+
+    error("No s3 access key or invalid access key provided. Please check that you have entered the access key correctly.", type) if s3_access.nil?
+    error("No s3 access secret key or invalid secret key provided. Please check that you have entered the secret key correctly.", type) if s3_secret.nil?
+    error("No s3 access bucket or invalid bucket provided. Please check that you have entered the bucket correctly.", type) if s3_bucket.nil?
+    error("No s3 file key provided. Please check that you have entered the file key correctly.", type) if s3_key.nil?
+
+    s3_params = {:s3_access_key => s3_access, :s3_secret => s3_secret, 
+                 :s3_bucket => s3_bucket, :s3_file_key => s3_key}
     s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
 
     res = self.api.data.pull_to_s3(id, s3_params)

data/lib/zillabyte/runner/app_runner.rb

@@ -170,8 +170,8 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
   end
 
 
-  def cdisplay(name, message)
-    require("colorize")
+  def cdisplay(name, message, useName=true)
+
     color = @colors[name] || :default
     if message == ""
       display ""
@@ -182,8 +182,8 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
       else
         lines = message.split("\n")
       end
-      
-      prefix = "#{name} - "
+
+      prefix = useName ? "#{name} - " : ""      
       display "#{prefix}#{lines.first}".colorize(color)
       lines[1..-1].each do |line|
         display "#{' '*prefix.size}#{line}".colorize(color)

data/lib/zillabyte/runner/component_operation.rb

@@ -573,9 +573,8 @@ class Zillabyte::Runner::ComponentOperation
 
 
   # Display a colored, formatted message
-  def self.cdisplay(msg)
-
-    @__tester.cdisplay(@__name, msg)
+  def self.cdisplay(msg, useName=true)
+    @__tester.cdisplay(@__name, msg, useName)
   end
 
 

data/lib/zillabyte/runner/component_runner.rb

@@ -185,14 +185,24 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
   end
 
 
-  def cdisplay(name, message)
-     require("colorize")
-     color = @colors[name] || :default
-      if message == ""
-        display ""
+ def cdisplay(name, message, useName=true)
+    color = @colors[name] || :default
+    if message == ""
+      display ""
+    else
+      
+      if message.is_a?(Array)
+        lines = message
       else
-        display "#{name} - #{message}".colorize(color)
+        lines = message.split("\n")
+      end
+
+      prefix = useName ? "#{name} - " : ""
+      display "#{prefix}#{lines.first}".colorize(color)
+      lines[1..-1].each do |line|
+        display "#{' '*prefix.size}#{line}".colorize(color)
       end
+    end
   end
 
 

data/lib/zillabyte/runner/multilang_operation.rb

@@ -846,8 +846,26 @@ module Zillabyte; module Runner; class MultilangOperation
 
   # Send a message to all consumers of the operation
   def self.run_sink()
-    output = @__options[:output]
+    columns = @__node["columns"]
+
+    type_map = {
+      "string" => String,
+      "double" => Float,
+      "integer" => Integer
+    }
+
+    col_map = {}
+    columns.each do |hash|
+      key = hash.keys[0]
+      type = hash[key]
+      col_map[key] = type_map[type]
+    end
+
+
+
     messages = []
+
+    output = @__options["output"]
     loop do
       # Read messages
       msg = read_message(@__consumee[:rd_child])
@@ -855,8 +873,36 @@ module Zillabyte; module Runner; class MultilangOperation
 
       # Add row
       if obj['tuple']
-        messages << msg
+
+        tuple = obj['tuple']
         display_json = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}].to_json
+
+        if col_map.keys.length != tuple.keys.length
+          cdisplay "Error: invalid keys for sink tuple : Expected #{col_map.keys} , got: #{tuple.keys}"
+          cdisplay("\n \nPress Ctrl-C to exit", false)
+          return
+        end
+
+        # Check tuple columns for valid entry
+        columns_to_check = col_map.length
+        tuple.each_pair do |col,value|
+          types = type_map.each_value.map {|t| value.is_a? t}
+          if col_map.include? col
+            if value.is_a? col_map[col]
+              columns_to_check -= 1
+            else
+              break
+            end
+          end
+        end
+
+        if columns_to_check != 0
+          cdisplay "Error: invalid schema for sink tuple #{display_json}"
+          cdisplay("\n \nPress Ctrl-C to exit", false)
+          return
+        end
+
+        messages << msg
         if @__options[:interactive]
           cdisplay "received #{display_json}"
         end
@@ -910,7 +956,7 @@ module Zillabyte; module Runner; class MultilangOperation
 
     # Output table
     cdisplay("\n#{table.to_s}")
-    cdisplay ""
+    cdisplay "#{messages.length} rows"
       
     # Write file
     if output
@@ -920,7 +966,7 @@ module Zillabyte; module Runner; class MultilangOperation
       f.close()
       cdisplay("output written to #{filename}")
     end
-    cdisplay "\n \nPress Ctrl-C to exit"
+    cdisplay("\n \nPress Ctrl-C to exit", false)
   end
 
 
@@ -1134,9 +1180,9 @@ module Zillabyte; module Runner; class MultilangOperation
 
 
   # Display a colored, formatted message
-  def self.cdisplay(msg)
-    @__tester.cdisplay(@__name, msg)
+  def self.cdisplay(msg, useName=true)
+    @__tester.cdisplay(@__name, msg, useName)
   end
 
-
 end; end; end
+

data/zillabyte-cli.gemspec

@@ -41,7 +41,7 @@ Getting Started with Zillabyte
 
 (1) Register for an auth token at http://zillabyte.com
 (2) Log in by running 'zillabyte login' in the command line
-(3) Build an empty app by running 'zillabyte app:init'
+(3) Build an empty app by running 'zillabyte apps:init'
     Or... check out our Quick Starts by visiting http://docs.zillabyte.com/
 
 Questions, comments? Please visit us at http://docs.zillabyte.com

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-18 00:00:00.000000000 Z
+date: 2014-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -257,7 +257,7 @@ licenses:
 metadata: {}
 post_install_message: ! "\nGetting Started with Zillabyte\n==============================\n\n(1)
   Register for an auth token at http://zillabyte.com\n(2) Log in by running 'zillabyte
-  login' in the command line\n(3) Build an empty app by running 'zillabyte app:init'\n
+  login' in the command line\n(3) Build an empty app by running 'zillabyte apps:init'\n
   \   Or... check out our Quick Starts by visiting http://docs.zillabyte.com/\n\nQuestions,
   comments? Please visit us at http://docs.zillabyte.com\n"
 rdoc_options: []