td-client 0.8.15 → 0.8.16

data/ChangeLog

@@ -1,4 +1,9 @@
 
+== 2012-05-10 version 0.8.16
+
+* Added bulk import feature
+
+
 == 2012-04-26 version 0.8.15
 
 * Result model replaces ResultSet model

data/lib/td/client.rb

@@ -156,6 +156,63 @@ class Client
     Job.new(self, job_id, :export, nil)
   end
 
+  # => nil
+  def create_bulk_import(name, database, table)
+    @api.create_bulk_import(name, database, table)
+  end
+
+  # => nil
+  def delete_bulk_import(name)
+    @api.delete_bulk_import(name)
+  end
+
+  # => nil
+  def freeze_bulk_import(name)
+    @api.freeze_bulk_import(name)
+  end
+
+  # => nil
+  def unfreeze_bulk_import(name)
+    @api.unfreeze_bulk_import(name)
+  end
+
+  # => Job
+  def perform_bulk_import(name)
+    job_id = @api.perform_bulk_import(name)
+    Job.new(self, job_id, :bulk_import, nil)
+  end
+
+  # => nil
+  def commit_bulk_import(name)
+    @api.commit_bulk_import(name)
+  end
+
+  # => records:[row:Hash]
+  def bulk_import_error_records(name, &block)
+    @api.bulk_import_error_records(name, &block)
+  end
+
+  # => [BulkImport]
+  def bulk_imports
+    @api.list_bulk_imports.map {|data|
+      BulkImport.new(self, data)
+    }
+  end
+
+  # => nil
+  def bulk_import_upload_part(name, part_name, stream, size)
+    @api.bulk_import_upload_part(name, part_name, stream, size)
+  end
+
+  # => nil
+  def bulk_import_delete_part(name, part_name)
+    @api.bulk_import_delete_part(name, part_name)
+  end
+
+  def list_bulk_import_parts(name)
+    @api.list_bulk_import_parts(name)
+  end
+
   # => first_time:Time
   def create_schedule(name, opts)
     raise ArgumentError, "'cron' option is required" unless opts[:cron] || opts['cron']

data/lib/td/client/api.rb

@@ -401,6 +401,147 @@ class API
     return js['job_id'].to_s
   end
 
+  ####
+  ## Bulk import API
+  ##
+
+  # => nil
+  def create_bulk_import(name, db, table, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/create/#{e name}/#{e db}/#{e table}", params)
+    if code != "200"
+      raise_error("Create bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => nil
+  def delete_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/delete/#{e name}", params)
+    if code != "200"
+      raise_error("Delete bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => result:[data:Hash]
+  def list_bulk_imports(opts={})
+    params = opts.dup
+    code, body, res = get("/v3/bulk_import/list", params)
+    if code != "200"
+      raise_error("List bulk imports failed", res)
+    end
+    js = checked_json(body, %w[bulk_imports])
+    return js['bulk_imports']
+  end
+
+  def list_bulk_import_parts(name, opts={})
+    params = opts.dup
+    code, body, res = get("/v3/bulk_import/list_parts/#{e name}", params)
+    if code != "200"
+      raise_error("List bulk import parts failed", res)
+    end
+    js = checked_json(body, %w[parts])
+    return js['parts']
+  end
+
+  # => nil
+  def bulk_import_upload_part(name, part_name, stream, size, opts={})
+    code, body, res = put("/v3/bulk_import/upload_part/#{e name}/#{e part_name}", stream, size)
+    if code[0] != ?2
+      raise_error("Upload a part failed", res)
+    end
+    return nil
+  end
+
+  # => nil
+  def bulk_import_delete_part(name, part_name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/delete_part/#{e name}/#{e part_name}", params)
+    if code[0] != ?2
+      raise_error("Delete a part failed", res)
+    end
+    return nil
+  end
+
+  # => nil
+  def freeze_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/freeze/#{e name}", params)
+    if code != "200"
+      raise_error("Freeze bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => nil
+  def unfreeze_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/unfreeze/#{e name}", params)
+    if code != "200"
+      raise_error("Unfreeze bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => jobId:String
+  def perform_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/perform/#{e name}", params)
+    if code != "200"
+      raise_error("Perform bulk import failed", res)
+    end
+    js = checked_json(body, %w[job_id])
+    return js['job_id'].to_s
+  end
+
+  # => nil
+  def commit_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/commit/#{e name}", params)
+    if code != "200"
+      raise_error("Commit bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => data...
+  def bulk_import_error_records(name, opts={}, &block)
+    params = opts.dup
+    code, body, res = get("/v3/bulk_import/error_records/#{e name}", params)
+    if code != "200"
+      raise_error("Commit bulk import failed", res)
+    end
+    if body.empty?
+      if block
+        return nil
+      else
+        return []
+      end
+    end
+    require 'zlib'
+    require 'stringio'
+    require 'msgpack'
+    u = MessagePack::Unpacker.new(Zlib::GzipReader.new(StringIO.new(body)))
+    if block
+      begin
+        u.each(&block)
+      rescue EOFError
+      end
+      nil
+    else
+      result = []
+      begin
+        u.each {|row|
+          result << row
+        }
+      rescue EOFError
+      end
+      return result
+    end
+  end
+
   ####
   ## Schedule API
   ##

data/lib/td/client/model.rb

@@ -343,6 +343,37 @@ class Result < Model
 end
 
 
+class BulkImport < Model
+  def initialize(client, data={})
+    super(client)
+    @name = data['name']
+    @database = data['database']
+    @table = data['table']
+    @status = data['status']
+    @upload_frozen = data['upload_frozen']
+    @job_id = data['job_id']
+    @valid_records = data['valid_records']
+    @error_records = data['error_records']
+    @valid_parts = data['valid_parts']
+    @error_parts = data['error_parts']
+  end
+
+  attr_reader :name
+  attr_reader :database
+  attr_reader :table
+  attr_reader :status
+  attr_reader :job_id
+  attr_reader :valid_records
+  attr_reader :error_records
+  attr_reader :valid_parts
+  attr_reader :error_parts
+
+  def upload_frozen?
+    @upload_frozen
+  end
+end
+
+
 class AggregationSchema < Model
   def initialize(client, name, relation_key, logs=nil, attributes=nil, timezone=nil)
     super(client)

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.8.15'
+VERSION = '0.8.16'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td-client
 version: !ruby/object:Gem::Version
-  version: 0.8.15
+  version: 0.8.16
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-27 00:00:00.000000000Z
+date: 2012-05-10 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70230453942020 !ruby/object:Gem::Requirement
+  requirement: &70147243679900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70230453942020
+  version_requirements: *70147243679900
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70230453937480 !ruby/object:Gem::Requirement
+  requirement: &70147243665900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70230453937480
+  version_requirements: *70147243665900
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70230453936700 !ruby/object:Gem::Requirement
+  requirement: &70147243665220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 2.8.0
   type: :development
   prerelease: false
-  version_requirements: *70230453936700
+  version_requirements: *70147243665220
 description: 
 email: 
 executables: []