zillabyte-cli 0.9.33 → 0.9.34

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    OGZjZTEyZTYyMmFmZTY5YzIwMDU0OTIyYjkyNDM3ZmViNWFlYzQ5ZA==
-  data.tar.gz: !binary |-
-    YzMzMDAzYjRjYTQzNjE4YzA0NTI0MTJhYWFjYzc2ZjE2MDcwMWIwMA==
+SHA1:
+  metadata.gz: 873b66c2ff0aefd25f8e83dd77b47ab67bbeaf11
+  data.tar.gz: ad39b03a444c5b52b5834de35ab2fa7641157516
 SHA512:
-  metadata.gz: !binary |-
-    ZDBhZGVmZDU5NzFlOGZmNGM4MWQ4ZTEyNmYyNDgwMzI5YTVkMjFiMjFlYjc1
-    Zjc4OWY2YmU2MmEwN2JmY2QxZjhiMzk0NTVlYmJiODhhMjg2ZGNiNjk5OTMx
-    OTgwYjA0M2YzNWI0OGFlZjE4NDE4OGU4YjI1ZDAyMTMwNjdlNGY=
-  data.tar.gz: !binary |-
-    NGQyNWUzNjVlZTFiMmRhYWNlMmU3NjJmYmMxMmRmNWRlNGM4M2YwOTE1MjFh
-    NjAwOGFmMDJlNzc2YTA1ZmQyM2U3MTZlZGM3NzgxNTNjMTcyMWUyNTQwZmQz
-    MGMzZDA5NjIyMDYxOGVkOGQ4ODhiNmU2Y2RkY2FmZjA1MWU2NmU=
+  metadata.gz: 14fdbd895bbb43c5c800a681365595a64b72875d782c6935bb062ab70592992a76a649224584283e6ad6e09dd2ff5ca8d96c0c8b1fe7e7aed94baac6f5448b8e
+  data.tar.gz: 45329f8e5348ab8313a688250929cf556f97ac6ca40b52c108a32ddb6f384ba4ffd09fc330660eb2bb5e2d04b4dcf6cbdc5a57726539fc551a02833e9665a725

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.9.33"
+    VERSION = "0.9.34"
   end
 end

data/lib/zillabyte/cli.rb

@@ -1,14 +1,15 @@
 module Zillabyte; module CLI
 
   def self.start(*args)
+    require("zillabyte/helpers")
     begin
+      Zillabyte::Helpers.error("We're sorry, but Zillabyte requires Ruby 1.9.3 or above.") if !Zillabyte::Helpers.version_okay?(RUBY_VERSION, "1.9.3")
       command = args.shift().strip() rescue "help"
       require("zillabyte/command")
       Zillabyte::Command.load(command)
       Zillabyte::Command.run(command, args)
     rescue Interrupt
       `stty icanon echo`
-      require("zillabyte/helpers")
       Zillabyte::Helpers.error("Command cancelled.")
     rescue => error
       raise

data/lib/zillabyte/cli/base.rb

@@ -171,22 +171,6 @@ class Zillabyte::Command::Base
     self
   end
 
-  def version_okay?(current_version, min_version)
-    current_version = current_version.split(".").map {|v| v.to_i}
-    minimum_version = min_version.split(".").map {|v| v.to_i}
-    if current_version[0] > minimum_version[0]
-      return true
-    elsif current_version[0] == minimum_version[0]
-      if current_version[1] > minimum_version[1]
-        return true
-      elsif current_version[1] == minimum_version[1]
-        return true if current_version[2] >= minimum_version[2]
-      end
-    end
-    return false
-  end
-  
-  
 end
 
 

data/lib/zillabyte/cli/data.rb

@@ -7,6 +7,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
   MAX_POLL_SECONDS = 60 * 5
   POLL_SLEEP = 1
   APPENDS_ROWS_SLICE = 5_000;
+  ONE_HUNDRED_MB = 1e8;
 
   # data
   #
@@ -138,39 +139,34 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
     description = options[:description] || nil
     aliases = options[:aliases] || nil
 
-    if type.nil?
-      hash = get_dataset_properties(schema,is_public,description,aliases)
-    else
-      hash = hash_dataset_properties(schema,is_public,description,aliases, type)
-    end
+    hash = get_dataset_properties(schema,is_public,description,aliases)
 
     if file
       filetype ||= File.extname(file).gsub(".", "")
-      rows = sanity_check_file(file,filetype, {"columns" => hash[:schema]}, type)
-      hash[:rows] = rows
+      n_shards = calculate_number_of_shards(file)
+      hash[:shards] = n_shards
+      display "Sharding into ~#{n_shards} shard(s) for upload. (Actual number may be smaller.)" if type.nil?
     end
 
     res = api.data.create name, hash
+
     if res['error']
       error("#{res['error_message']}", type)
     else
+      if file
+        row_size = save_rows(hash[:schema], file, filetype, res['uris'], type)
+      end
       if type == "json"
         display "{}"
       else
-        display "dataset ##{res['id']} #{res['action']}. size: #{res['size'] || 0} rows."
+        display "dataset ##{res['id']} #{res['action']}. size: #{row_size || 0} rows."
       end
     end
-
   end
   alias_command "relations:create", "data:create"
   alias_command "relation:create", "data:create"
   alias_command "datasets:create", "data:create"
   alias_command "dataset:create", "data:create"
-  
-  
-  
-  
-  
 
   # data:authorize [ID] [SCOPE]
   #
@@ -255,28 +251,21 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
     error("no file given", type) if file.nil?
     
     dataset = self.api.data.get(id, options)
-    columns = dataset["columns"].map{|col| {col["index"] => col["type"]}}
-    raw_rows = sanity_check_file(file,filetype,{"columns" => columns}, type)
+    schema = dataset["schema"]
+    n_shards = calculate_number_of_shards(file)
    
-    total_rows = 0 
     display("uploading content.", false)
-    raw_rows.each_slice(APPENDS_ROWS_SLICE) do |rows|
-      
-      
-      # TODO: post to direct signed s3 (http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/S3/PresignedPost.html)
-      display(".", false)
-      require("base64")
-      res = self.api.data.append(id, {:gzip_rows => Base64.encode64(gzip(rows.to_json()))})
-      # res = self.api.data.append(id, {:rows => rows})
-      
-      break unless res["size"]
-      total_rows += res["size"]
-    end
+    res = self.api.data.append(id, {:shards => n_shards})
     
-    if type == "json"
-      display({:rows => total_rows}.to_json)
+    if res['error']
+      error("#{res['error_message']}", type)
     else
-      display "dataset ##{id} appended #{total_rows} rows"
+      row_size = save_rows(schema, file, filetype, res['uris'], type)
+      if type == "json"
+        display({:rows => row_size}.to_json)
+      else
+        display "dataset ##{id} appended #{row_size} rows"
+      end
     end
 
   end
@@ -547,7 +536,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
     end
 
     if is_public.nil?
-      error("specify public or private using --public")
+      error("specify public or private using --public", type)
     end
 
     {:schema => schema, :public => is_public, :description => description, :aliases => aliases, :output_type => type}
@@ -613,85 +602,6 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
     {:schema => schema, :public => is_public, :description => description, :aliases => aliases}
   end
 
-  def sanity_check_file(file, filetype, dataset, type = nil)
-    rows = []
-  
-    n_columns = dataset["columns"].size
-    float_cols = []
-    int_cols = []
-    array_cols = []
-    map_cols = []
-    (0...n_columns).each do |i|
-      value = dataset["columns"][i].values[0].downcase
-      # use integer instead of numeric to be consistent with multilang type declarations
-      if(value == 'integer')
-        int_cols << i
-      elsif(value == "float" or value == "double")
-        float_cols << i
-      elsif(value == "array")
-        array_cols << i
-      elsif(value == "map")
-        map_cols << i
-      end
-    end
-
-    case filetype
-    when "csv"
-      begin
-        require("csv")
-        CSV.foreach(file) do |row|
-          if row.size != n_columns
-            error("Dataset expects #{n_columns} column(s). Found a row with #{row.size}::\n #{row}", type)
-          end
-
-          float_cols.each do |i|
-            if (!(row[i] =~ /^[+-]?(\d+(.\d+)?)$/))
-              error("Column #{i+1} should be a FLOAT or DOUBLE in row ::\n #{row}", type)
-            end
-          end
-
-          int_cols.each do |i|
-            if (!(row[i] =~ /^[+-]?(\d+)$/))
-              error("Column #{i+1} should be an INTEGER in row ::\n #{row}", type)
-            end
-          end
-
-          array_cols.each do |i|
-            err = "Column #{i+1} should be an ARRAY (i.e. [v0,v1,...]) in row ::\n #{row}\n Please remember to escape strings in the array!"
-            begin
-              array = JSON.parse(row[i])
-            rescue
-              error(err, type)
-            end
-            if !array.instance_of?(Array)
-              error(err, type)
-            end
-          end
-
-          map_cols.each do |i|
-            err = "Column #{i+1} should be a MAP (i.e. {\"k1\":v1,\"k2\":v2,...}) in row ::\n #{row}\n Please remember to escape strings in the map!"
-            begin
-              map = JSON.parse(row[i])
-            rescue
-              error(err, type)
-            end
-            if !map.instance_of?(Hash)
-              error(err, type)
-            end
-          end
-          rows << row
-        end
-      rescue Exception => e
-        error("error when parsing csv : #{e}", type)
-      end
-      
-    else
-      error "unsupported type: #{filetype}"
-      
-    end
-    rows
-  end
-
 #  # relations:status ID
 #  #
 #  # shows status of a relation
@@ -750,6 +660,130 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
       compressed
   end
 
+  def sanity_check_row(row, schema, n_columns, output_type)
+    if row.size != n_columns
+      error("Dataset expects #{n_columns} column(s). Found a row with #{row.size}::\n #{row}", output_type)
+    end
+
+    row_hash = {}
+    schema.each_with_index do |col, i|
+      case col.values.first
+      when "float", "double"
+        if (!(row[i] =~ /^[+-]?(\d+(.\d+)?)$/))
+          error("Column #{i+1} should be a FLOAT or DOUBLE in row ::\n #{row}", output_type)
+        end
+        row_hash[col.keys.first] = row[i].to_f
+      when "integer"
+        if (!(row[i] =~ /^[+-]?(\d+)$/))
+          error("Column #{i+1} should be an INTEGER in row ::\n #{row}", output_type)
+        end
+        row_hash[col.keys.first] = row[i].to_i
+      when "array"
+        err = "Column #{i+1} should be an ARRAY (i.e. [v0,v1,...]) in row ::\n #{row}\n Please remember to escape strings in the array!"
+        begin
+          row_hash[col.keys.first] = JSON.parse(row[i])
+        rescue
+          error(err, output_type)
+        end
+        if !row[i].instance_of?(Array)
+          error(err, output_type)
+        end
+      when "map"
+        err = "Column #{i+1} should be a MAP (i.e. {\"k1\":v1,\"k2\":v2,...}) in row ::\n #{row}\n Please remember to escape strings in the map!"
+        begin
+          row_hash[col.keys.first] = JSON.parse(row[i])
+        rescue
+          error(err, output_type)
+        end
+        if !row[i].instance_of?(Hash)
+          error(err, output_type)
+        end
+      else
+        row_hash[col.keys.first] = row[i]
+      end
+    end
+
+    row_hash.to_json
+  end
+
+  def flush_to_s3(rows, public_uris, output_type)
+    require "zlib"
+    while(true)
+      begin
+        str = StringIO.new("")
+        gz = Zlib::GzipWriter.new(str)
+        gz.write(rows.join("\n"))
+        gz.finish
+        str.rewind
+      rescue Exception => e
+        # just retry
+      end
+      break
+    end
+
+    require 'net/http'
+    require "uri"
+    display("Uploading.") if output_type.nil?
+    uri = URI.parse(public_uris.shift)
+    try_again = 1
+    while(try_again)
+      begin
+        Net::HTTP.start(uri.host, uri.port, :use_ssl => true) do |http|
+          request = Net::HTTP::Put.new(uri.request_uri)
+          request.body_stream = str
+          request['Content-Length'] = request.body_stream.size
+          request['Content-Type'] = ''
+          response = http.request(request)
+          if response.code.to_i >= 300
+            display("Received code #{response.code}...re-trying.") if output_type.nil?
+            try_again = 1
+            sleep(3)
+          else
+            try_again = nil
+          end
+        end
+      rescue Exception => e
+        try_again = 1
+      end
+    end
+  end 
+
+  def save_rows(schema, file, filetype, public_uris, output_type)
+    n_columns = schema.size
+    n_flushes = 1
+    total_rows = 0
+    display "Checking your file and constructing shard ##{n_flushes}." if output_type.nil?
+    case filetype
+    when "csv"
+      require("csv")
+      total_size = 0
+      rows = []
+      CSV.foreach(file) do |row|
+        total_rows += 1
+        print "." if(total_rows % 10000 == 0)
+        total_size += row.join.bytesize
+        row_json = sanity_check_row(row, schema, n_columns, output_type)
+        rows << row_json
+
+        if total_size > ONE_HUNDRED_MB
+          flush_to_s3(rows, public_uris, output_type)
+          total_size = 0
+          n_flushes += 1
+          display "Checking your file and constructing shard ##{n_flushes}." if output_type.nil?
+          rows = []
+        end
+      end
+      flush_to_s3(rows, public_uris, output_type) if total_size > 0
+    else
+      error "unsupported type: #{filetype}", output_type
+    end
 
+    total_rows
+  end
+
+  def calculate_number_of_shards(file)
+    filesize = File.size(file)
+    (filesize / ONE_HUNDRED_MB + 1).floor
+  end
 
 end

data/lib/zillabyte/cli/download.rb

@@ -5,7 +5,7 @@ require("zillabyte/cli/base")
 class Zillabyte::Command::Download < Zillabyte::Command::Base
 
   MIN_LOCAL_JAR_VERSION = "0.0.3"
-  LATEST_JAR_VERSION = "0.9.33"
+  LATEST_JAR_VERSION = "0.9.34"
   MIN_JAVA_VERSION = "1.7.0"
 
   # --local                                   # HIDDEN

data/lib/zillabyte/cli/templates/apps/ruby/README.md.erb

@@ -0,0 +1,2 @@
+# <%= name %>
+

data/lib/zillabyte/helpers.rb

@@ -324,6 +324,21 @@ module Zillabyte
       end
     end
 
+    def version_okay?(current_version, min_version)
+      current_version = current_version.split(".").map {|v| v.to_i}
+      minimum_version = min_version.split(".").map {|v| v.to_i}
+      if current_version[0] > minimum_version[0]
+        return true
+      elsif current_version[0] == minimum_version[0]
+        if current_version[1] > minimum_version[1]
+          return true
+        elsif current_version[1] == minimum_version[1]
+          return true if current_version[2] >= minimum_version[2]
+        end
+      end
+      return false
+    end
+
   end
 end
 

metadata

@@ -1,139 +1,139 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte-cli
 version: !ruby/object:Gem::Version
-  version: 0.9.33
+  version: 0.9.34
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-19 00:00:00.000000000 Z
+date: 2014-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.7
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.1
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.31'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.31'
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.2.11
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.2.11
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
   name: indentation
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
 description: The Official Zillabyte CLI Gem
@@ -194,6 +194,7 @@ files:
 - lib/zillabyte/cli/templates/apps/python/zillabyte.conf.yaml
 - lib/zillabyte/cli/templates/apps/ruby/Gemfile
 - lib/zillabyte/cli/templates/apps/ruby/README.md
+- lib/zillabyte/cli/templates/apps/ruby/README.md.erb
 - lib/zillabyte/cli/templates/apps/ruby/app.rb.erb
 - lib/zillabyte/cli/templates/apps/ruby/zillabyte.conf.yaml
 - lib/zillabyte/cli/templates/components/js/simple_function.js
@@ -220,22 +221,28 @@ homepage: http://www.zillabyte.com
 licenses:
 - MIT
 metadata: {}
-post_install_message: ! "\nGetting Started with Zillabyte\n==============================\n\n(1)
-  Register for an auth token at http://zillabyte.com\n(2) Log in by running 'zillabyte
-  login' in the command line\n(3) Build an empty app by running 'zillabyte apps:init'\n
-  \   Or... check out our Quick Starts by visiting http://docs.zillabyte.com/\n\nQuestions,
-  comments? Please visit us at http://docs.zillabyte.com\n"
+post_install_message: |2
+
+  Getting Started with Zillabyte
+  ==============================
+
+  (1) Register for an auth token at http://zillabyte.com
+  (2) Log in by running 'zillabyte login' in the command line
+  (3) Build an empty app by running 'zillabyte apps:init'
+      Or... check out our Quick Starts by visiting http://docs.zillabyte.com/
+
+  Questions, comments? Please visit us at http://docs.zillabyte.com
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []