zillabyte-cli 0.0.21 → 0.0.22

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2YyYjcyMWJlMGExNmUzOGI3NjE1N2UzMThiZjU4ZjZkYzc4ZDgyYg==
+    OTcxYjQ2YjVjYTYxNmZkN2ExNzAzZDA5ZTg1ZGNlMzQ2OTA0YTUxNg==
   data.tar.gz: !binary |-
-    MWIyNjZmMjVlYWJhYzM2OTdhOGU2NzlhY2U2NTg4YTRkMjJjMjJkMQ==
+    N2FhNTYxODc2NjMxYjVkOGIzZDVjODEyODEzY2Y4YmU1YTczNThjOA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    MWFmY2FiMzUzNjM0MmY3OGE2NzA3YjEwYTY4MzQ2NWMwZTdmNGE4NDA5ZDgy
-    NjdmNDI4YjllMjQyMGFiNDY1ZGQzYzY1OTliY2Y1YTRkMGJlMzQyM2VkZDA4
-    NzY5ZWFmMThkNjFjNDhmYzZmODM3Yzk3ZjQ0OGRkMDZkNjE1Njg=
+    OTQ1OTZjMzU4MjM2ODBlYWM0ZGZlMWM5NDdhYzFhMzlkNzg0ZDlmOTA4NTNj
+    ZWNhMWMzMDljZjNiYzQ3MzQ4NWUxYWQyMzVhZmM4NDUzZTdkNTkwOWE4MGQx
+    YjQwOTBkMzViNGViNDYzZjc2MTI4NTFiODZiYTY2YzY0MzEzNTg=
   data.tar.gz: !binary |-
-    YjA3ZGIxNDU4NjVkNDViZGNmN2M5MTg5YmNkYTQ0YzQzODU3NDJkMTNmMzZi
-    MDNjZTI3YmViMDhiM2JlNzA4NGFiNDVhZjk3MTg2ZmY0NmI1NDYzMmNlNDNi
-    YmIzYWY2ZjcyNGZjYTI0NmVkNzdhM2ZhMDEyNTY1OGYyNWI2YjA=
+    NTI5NThiMzMyMTVhYTA3Y2FkZGFhMmI0ZGE2OWExNzg2ZTcxYTI2NmFlMDE3
+    NzA2YjE4NTg1YTVlODJhNGZkZjJkMWM4YzYzZjZjODQ5ODg1MTdjZmIwNTE0
+    YjQ5OTYxMzcxYmY4NzliNjJlOTg2YzM1ZTUxNmZhYWVlNGJlZGY=

data/lib/zillabyte/api/data.rb

@@ -86,6 +86,7 @@ class Zillabyte::API::Data < Zillabyte::API::Base
       :expects  => 200,
       :method   => :post,
       :path     => "/relations/#{CGI.escape(id)}/append",
+      # :headers  => {"Content-Type" => "application/octet-stream"},
       :body     => options.to_json
     )
     hash = res.body
@@ -93,4 +94,8 @@ class Zillabyte::API::Data < Zillabyte::API::Base
  
   end
 
+
+  private
+
+
 end

data/lib/zillabyte/api.rb

@@ -53,6 +53,16 @@ class Zillabyte::API
   end
   
   
+  def connection
+    @connection
+  end
+  
+  
+  def headers
+    @options[:headers]
+  end
+  
+  
   def host
     @options[:host]
   end
@@ -65,15 +75,6 @@ class Zillabyte::API
     end
 
     type = nil
-    if params[:body]
-      begin
-          param_hash = JSON.parse(params[:body])
-          type = param_hash["type"]
-      rescue JSON::ParserError => e
-          @session.error("invalid parameters", type)
-      end
-    end
-
     begin
 
       # If a block is given, then we assume we're streaming (i.e. for logs)

data/lib/zillabyte/cli/apps.rb

@@ -823,15 +823,17 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
           header_written = false;
           messages.each do |msg|
             obj = JSON.parse(msg)
-            if obj['tuple']
+            t = obj['tuple']
+            m = obj['meta'] || {}
+            if t
               if header_written == false
-                keys = [obj['tuple'].keys, obj['meta'].keys].flatten
+                keys = [t.keys, m.keys].flatten
                 csv << keys
                 table << keys
                 table << :separator
                 header_written = true
               end
-              vals = [obj['tuple'].values, obj['meta'].values].flatten
+              vals = [t.values, m.values].flatten
               csv << vals
               table << vals
             end

data/lib/zillabyte/cli/query.rb

@@ -238,6 +238,77 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
   alias_command "sql", "query:sql"
 
 
+
+
+  # query:pull QUERY FILE
+  #
+  # executes a queries and downloads the results to FILE
+  #
+  # pulls query data to OUTPUT.gz
+  def pull
+
+    query = options[:query] || shift_argument
+    file = options[:file] || shift_argument
+    type = options[:type]
+    error("no query given", type) if query.nil?
+    error("no file given", type) if file.nil?
+    file = "#{file}.gz" unless File.extname(file) == ".gz"
+
+    options[:query] = query
+
+    res = api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/query_pull",
+      :body     => options.to_json 
+    )
+    res = res.body
+    
+    p res
+    if(res["uri"])
+      display "Waiting for download." if type.nil?
+      File.open(file, "w") do |f|
+        f.write open(res["uri"]).read
+      end
+    elsif(res["s3_credentials"])
+      display "Pulling query results." if type.nil?
+      s3 = AWS::S3.new(res["s3_credentials"])
+      bucket = s3.buckets[res["s3_bucket"]]
+      obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
+
+      while(true)
+        keys = obj.collect(&:key)
+        if keys.length > 0 and keys.include?("#{res["s3_file_key"]}/manifest")
+          display "Starting to write to file..." if type.nil?
+          File.open(file, "w") do |f|
+            obj.each do |o|
+              if o.key == "#{res["s3_file_key"]}/manifest"
+                next
+              end
+              f.write(o.read)
+            end
+          end
+          break
+        else
+          sleep(3)
+          obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
+        end
+      end
+    else
+      throw "bad response: #{res}"
+    end
+
+    if type == "json"
+      display "{}"
+    else
+      display "results downloaded to file #{file}"
+    end
+    
+  end
+  alias_command "sql:pull", "query:pull"
+
+
+
   # query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
   #
   # pulls query data to S3_BUCKET/FILE_KEY/part***.gz

data/lib/zillabyte/cli/relations.rb

@@ -4,6 +4,7 @@ require "csv"
 require "open-uri"
 require "aws-sdk"
 require "json"
+require "base64"
 
 
 # manage custom relations
@@ -12,6 +13,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
 
   MAX_POLL_SECONDS = 60 * 5
   POLL_SLEEP = 1
+  APPENDS_ROWS_SLICE = 5_000;
 
   # relations
   #
@@ -163,20 +165,30 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     type = options[:type]
 
     filetype = options[:filetype]
-    filetype ||= File.extname(file).gsub(".", "")
+    filetype ||= File.extname(file || "").gsub(".", "")
     error("no id given", type) if id.nil?
     error("no file given", type) if file.nil?
-
-
+    
     relation = self.api.data.get(id, options)
     columns = relation["columns"].map{|col| {col["index"] => col["type"]}}
-    rows = sanity_check_file(file,filetype,{"columns" => columns}, type)
+    raw_rows = sanity_check_file(file,filetype,{"columns" => columns}, type)
    
-    res = self.api.data.append(id, options.merge({:rows => rows}))
+    total_rows = 0 
+    display("uploading content.", false)
+    raw_rows.each_slice(APPENDS_ROWS_SLICE) do |rows|
+      
+      display(".", false)
+      res = self.api.data.append(id, {:gzip_rows => Base64.encode64(gzip(rows.to_json()))})
+      # res = self.api.data.append(id, {:rows => rows})
+      
+      break unless res["size"]
+      total_rows += res["size"]
+    end
+    
     if type == "json"
-      display "{}"
+      display({:rows => total_rows}.to_json)
     else
-      display "relation ##{id} appended #{res["size"]} rows"
+      display "relation ##{id} appended #{total_rows} rows"
     end
 
   end
@@ -431,32 +443,9 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
 
 
 
-  def get_relation_properties(schema, is_public, description, aliases)
-    if aliases.nil?
-      display "If you would like to give alias names to your relation, please enter them below. (comma separated, only letters, number and underscore allowed)"
-      aliases = ask.strip
-    end
-    while true
-      aliases = aliases.strip.split(",").map{|x| x.strip}.uniq
-      all_valid = true
-      aliases.each do |a|
-        if(!(a =~ /^[a-zA-Z0-9\_]+$/i))
-          display "\"#{a}\" contains illegal characters. Only letters, numbers and underscore are allowed in alias names."
-          display "If you would like to give alias names to your relation, please enter them below. (comma separated, only letters, number and underscore allowed)"
-          aliases = ask.strip
-          all_valid = false
-          break
-        end
-      end
-      break if all_valid
-    end
-    display "Will use the following aliases for this relation: #{aliases}" if !aliases.empty?
-
-    if description.nil?
-      display "Enter a description for your relation (defaults to empty string)."
-      description = ask.strip
-    end
-
+  def get_relation_properties(schema, is_public, description = "", aliases = nil)
+    
+    description ||= ""
     valid_types=["string", "integer", "float"]
     meta_names=["id", "confidence", "since", "source"]
     if schema.nil?
@@ -606,4 +595,16 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
 #  end
 #  alias_command "rl:status", "relations:status"
 
+
+  def gzip(string)
+      wio = StringIO.new("w")
+      w_gz = Zlib::GzipWriter.new(wio)
+      w_gz.write(string)
+      w_gz.close
+      compressed = wio.string
+      compressed
+  end
+
+
+
 end

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.0.21"
+    VERSION = "0.0.22"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.21
+  version: 0.0.22
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-05 00:00:00.000000000 Z
+date: 2014-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -257,7 +257,6 @@ files:
 - lib/zillabyte/api.rb
 - lib/zillabyte/auth.rb
 - lib/zillabyte/cli/apps.rb
-- lib/zillabyte/cli/apps.rb.orig
 - lib/zillabyte/cli/auth.rb
 - lib/zillabyte/cli/base.rb
 - lib/zillabyte/cli/config.rb
@@ -279,7 +278,6 @@ files:
 - lib/zillabyte/cli/templates/python/zillabyte.conf.yaml
 - lib/zillabyte/cli/templates/ruby/app.rb
 - lib/zillabyte/cli/templates/ruby/Gemfile
-- lib/zillabyte/cli/templates/ruby/Gemfile.lock
 - lib/zillabyte/cli/templates/ruby/zillabyte.conf.yaml
 - lib/zillabyte/cli/version.rb
 - lib/zillabyte/cli/zillalogs.rb
@@ -318,7 +316,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.0.7
 signing_key: 
 specification_version: 4
 summary: The Official Zillabyte CLI Gem