td-client 1.0.0-java

data/lib/td/client/api_error.rb

@@ -0,0 +1,37 @@
+module TreasureData
+
+class ParameterValidationError < StandardError
+end
+
+# Generic API error
+class APIError < StandardError
+  attr_reader :api_backtrace
+
+  def initialize(error_message = nil, api_backtrace = nil)
+    super(error_message)
+    @api_backtrace = api_backtrace == '' ? nil : api_backtrace
+  end
+end
+
+# 401 API errors
+class AuthError < APIError
+end
+
+# 403 API errors, used for database permissions
+class ForbiddenError < APIError
+end
+
+# 409 API errors
+class AlreadyExistsError < APIError
+  attr_reader :conflicts_with
+  def initialize(error_message = nil, api_backtrace = nil, conflicts_with=nil)
+    super(error_message, api_backtrace)
+    @conflicts_with = conflicts_with
+  end
+end
+
+# 404 API errors
+class NotFoundError < APIError
+end
+
+end

data/lib/td/client/compat_gzip_reader.rb

@@ -0,0 +1,22 @@
+
+methods = Zlib::GzipReader.public_instance_methods
+if !methods.include?(:readpartial) && !methods.include?('readpartial')
+  class Zlib::GzipReader
+    # @param [Fixnum] size
+    # @param [IO] out
+    # @return [String]
+    def readpartial(size, out=nil)
+      o = read(size)
+      if o
+        if out
+          out.replace(o)
+          return out
+        else
+          return o
+        end
+      end
+      raise EOFError, "end of file reached"
+    end
+  end
+end
+

data/lib/td/client/model.rb

@@ -0,0 +1,816 @@
+require 'timeout'
+
+module TreasureData
+
+class Model
+  # @param [TreasureData::Client] client
+  def initialize(client)
+    @client = client
+  end
+
+  # @!attribute [r] client
+  # @return [TreasureData::Client] client
+  attr_reader :client
+end
+
+class Account < Model
+  # @param [TreasureData::Client] client
+  # @param [String] account_id
+  # @param [Fixnum] plan
+  # @param [Fixnum] storage_size
+  # @param [Fixnum] guaranteed_cores
+  # @param [Fixnum] maximum_cores
+  # @param [String] created_at
+  def initialize(client, account_id, plan, storage_size=nil, guaranteed_cores=nil, maximum_cores=nil, created_at=nil)
+    super(client)
+    @account_id = account_id
+    @plan = plan
+    @storage_size = storage_size
+    @guaranteed_cores = guaranteed_cores
+    @maximum_cores = maximum_cores
+    @created_at = created_at
+  end
+
+  # @!attribute [r] account_id
+  # @!attribute [r] plan
+  # @!attribute [r] storage_size
+  # @!attribute [r] guaranteed_cores
+  # @!attribute [r] maximum_cores
+  attr_reader :account_id, :plan, :storage_size, :guaranteed_cores, :maximum_cores
+
+  # @return <Time, nil>
+  def created_at
+    @created_at && !@created_at.empty? ? Time.parse(@created_at) : nil
+  end
+
+  # @return <String>
+  def storage_size_string
+    if @storage_size <= 1024 * 1024
+      return "0.0 GB"
+    elsif @storage_size <= 60 * 1024 * 1024
+      return "0.01 GB"
+    elsif @storage_size <= 60 * 1024 * 1024 * 1024
+      "%.1f GB" % (@storage_size.to_f / (1024 * 1024 * 1024))
+    else
+      "%d GB" % (@storage_size.to_f / (1024 * 1024 * 1024)).to_i
+    end
+  end
+end
+
+class Database < Model
+  PERMISSIONS = [:administrator, :full_access, :import_only, :query_only]
+  PERMISSION_LIST_TABLES = [:administrator, :full_access]
+
+  # @param [TreasureData::Client] client
+  # @param [String] db_name
+  # @param [Array<Table>] tables
+  # @param [Fixnum] count
+  # @param [String] created_at
+  # @param [String] updated_at
+  # @param [String] org_name
+  # @param [String] permission
+  def initialize(client, db_name, tables=nil, count=nil, created_at=nil, updated_at=nil, org_name=nil, permission=nil)
+    super(client)
+    @db_name = db_name
+    @tables = tables
+    @count = count
+    @created_at = created_at
+    @updated_at = updated_at
+    @permission = permission.to_sym
+  end
+
+  # @!attribute [r] org_name
+  # @!attribute [r] permission
+  # @!attribute [r] count
+  attr_reader :org_name, :permission, :count
+
+  # @return [String] db_name
+  def name
+    @db_name
+  end
+
+  # @return [Array<Table>]
+  def tables
+    update_tables! unless @tables
+    @tables
+  end
+
+  # @param [String] name
+  # @return [true]
+  def create_log_table(name)
+    @client.create_log_table(@db_name, name)
+  end
+
+  # @param [String] table_name
+  # @return [Table]
+  def table(table_name)
+    @client.table(@db_name, table_name)
+  end
+
+  # @return [Symbol]
+  def delete
+    @client.delete_database(@db_name)
+  end
+
+  # @param [String] q
+  # @return [Job]
+  def query(q)
+    @client.query(@db_name, q)
+  end
+
+  # @return [Time, nil]
+  def created_at
+    @created_at && !@created_at.empty? ? Time.parse(@created_at) : nil
+  end
+
+  # @return [Time, nil]
+  def updated_at
+    @updated_at && !@updated_at.empty? ? Time.parse(@updated_at) : nil
+  end
+
+  # @return [nil]
+  def update_tables!
+    @tables = @client.tables(@db_name)
+    # provide Table objects with a reference to the parent Database to avoid
+    # requesting the Database information (such as permission) every time
+    @tables.each {|table|
+      table.database = self
+    }
+  end
+
+end
+
+class Table < Model
+  # @param [TreasureData::Client] client
+  # @param [String] db_name
+  # @param [String] table_name
+  # @param [String] type
+  # @param [String] schema
+  # @param [Fixnum] count
+  # @param [String] created_at
+  # @param [String] updated_at
+  # @param [Fixnum] estimated_storage_size
+  # @param [String] last_import
+  # @param [String] last_log_timestamp
+  # @param [Fixnum, String] expire_days
+  def initialize(client, db_name, table_name, type, schema, count, created_at=nil, updated_at=nil, estimated_storage_size=nil, last_import=nil, last_log_timestamp=nil, expire_days=nil)
+    super(client)
+    @database = nil
+    @db_name = db_name
+    @table_name = table_name
+    @type = type
+    @schema = schema
+    @count = count
+    @created_at = created_at
+    @updated_at = updated_at
+    @estimated_storage_size = estimated_storage_size
+    @last_import = last_import
+    @last_log_timestamp = last_log_timestamp
+    @expire_days = expire_days
+  end
+
+  # @!attribute [r] type
+  # @!attribute [r] db_name
+  # @!attribute [r] table_name
+  # @!attribute [r] schema
+  # @!attribute [r] count
+  # @!attribute [r] estimated_storage_size
+  attr_reader :type, :db_name, :table_name, :schema, :count, :estimated_storage_size
+
+  alias database_name db_name
+  alias name table_name
+
+  # @param [String] database
+  def database=(database)
+    @database = database if database.instance_of?(Database)
+  end
+
+  # @return [Time, nil]
+  def created_at
+    @created_at && !@created_at.empty? ? Time.parse(@created_at) : nil
+  end
+
+  # @return [Time, nil]
+  def updated_at
+    @updated_at && !@updated_at.empty? ? Time.parse(@updated_at) : nil
+  end
+
+  # @return [Time, nil]
+  def last_import
+    @last_import && !@last_import.empty? ? Time.parse(@last_import) : nil
+  end
+
+  # @return [Time, nil]
+  def last_log_timestamp
+    @last_log_timestamp && !@last_log_timestamp.empty? ? Time.parse(@last_log_timestamp) : nil
+  end
+
+  # @return [Fixnum, nil]
+  def expire_days
+    @expire_days ? @expire_days.to_i : nil
+  end
+
+  # @return [Database]
+  def database
+    update_database! unless @database
+    @database
+  end
+
+  # get the database's permission as if they were the table's
+  # @return [String]
+  def permission
+    database.permission
+  end
+
+  # @return [String]
+  def identifier
+    "#{@db_name}.#{@table_name}"
+  end
+
+  # @return [Symbol]
+  def delete
+    @client.delete_table(@db_name, @table_name)
+  end
+
+  # @param [Fixnum] count
+  # @return [Array, nil]
+  def tail(count)
+    @client.tail(@db_name, @table_name, count)
+  end
+
+  # @param [String] format
+  # @param [String, StringIO] stream
+  # @param [Fixnum] size
+  # @return [Float]
+  def import(format, stream, size)
+    @client.import(@db_name, @table_name, format, stream, size)
+  end
+
+  # @param [String] storage_type
+  # @param [Hash] opts
+  # @return [Job]
+  def export(storage_type, opts={})
+    @client.export(@db_name, @table_name, storage_type, opts)
+  end
+
+  # @return [String]
+  def estimated_storage_size_string
+    if @estimated_storage_size <= 1024*1024
+      return "0.0 GB"
+    elsif @estimated_storage_size <= 60*1024*1024
+      return "0.01 GB"
+    elsif @estimated_storage_size <= 60*1024*1024*1024
+      "%.1f GB" % (@estimated_storage_size.to_f / (1024*1024*1024))
+    else
+      "%d GB" % (@estimated_storage_size.to_f / (1024*1024*1024)).to_i
+    end
+  end
+
+  def update_database!
+    @database = @client.database(@db_name)
+  end
+
+  # @return [String]
+  def inspect
+    %[#<%s:%#0#{1.size*2}x @db_name="%s" @table_name="%s">] %
+    [self.class.name, self.__id__*2, @db_name, @table_name]
+  end
+end
+
+class Schema
+  class Field
+    # @param [String] name
+    # @param [String] type
+    # @param [String] sql_alias
+    def initialize(name, type, sql_alias=nil)
+      if name == 'v' || name == 'time'
+        raise ParameterValidationError, "Column name '#{name}' is reserved."
+      end
+      API.validate_column_name(name)
+      API.validate_sql_alias_name(sql_alias) if sql_alias
+      @name = name
+      @type = type
+      @sql_alias = sql_alias
+    end
+
+    # @!attribute [r] name
+    # @!attribute [r] type
+    attr_reader :name
+    attr_reader :type
+    attr_reader :sql_alias
+  end
+
+  # @param [String] columns
+  # @return [Schema]
+  def self.parse(columns)
+    schema = Schema.new
+    columns.each {|column|
+      unless /\A(?<name>.*)(?::(?<type>[^:@]+))(?:@(?<sql_alias>[^:@]+))?\z/ =~ column
+        raise ParameterValidationError, "type must be specified"
+      end
+      schema.add_field(name, type, sql_alias)
+    }
+    schema
+  end
+
+  # @param [Array] fields
+  def initialize(fields=[])
+    @fields = fields
+    @names = {}
+    @fields.each do |f|
+      raise ArgumentError, "Column name '#{f.name}' is duplicated." if @names.key?(f.name)
+      @names[f.name] = true
+      if f.sql_alias
+        raise ArgumentError, "SQL Column alias '#{f.sql_alias}' is duplicated." if @names.key?(f.sql_alias)
+        @names[f.sql_alias] = true
+      end
+    end
+  end
+
+  # @!attribute [r] fields
+  attr_reader :fields
+
+  # @param [String] name
+  # @param [String] type
+  # @return [Array]
+  def add_field(name, type, sql_alias=nil)
+    if @names.key?(name)
+      raise ParameterValidationError, "Column name '#{name}' is duplicated."
+    end
+    @names[name] = true
+    if sql_alias && @names.key?(sql_alias)
+      raise ParameterValidationError, "SQL Column alias '#{sql_alias}' is duplicated."
+    end
+    @names[sql_alias] = true
+    @fields << Field.new(name, type, sql_alias)
+  end
+
+  # @param [Schema] schema
+  # @return [Schema]
+  def merge(schema)
+    nf = @fields.dup
+    schema.fields.each {|f|
+      if i = nf.find_index {|sf| sf.name == f.name }
+        nf[i] = f
+      else
+        nf << f
+      end
+    }
+    Schema.new(nf)
+  end
+
+  # @return [Array<Field>]
+  def to_json(*args)
+    @fields.map {|f| f.sql_alias ? [f.name, f.type, f.sql_alias] : [f.name, f.type] }.to_json(*args)
+  end
+
+  # @param [Object] obj
+  # @return [self]
+  def from_json(obj)
+    @fields = obj.map {|f|
+      Field.new(*f)
+    }
+    self
+  end
+end
+
+class Job < Model
+  STATUS_QUEUED = "queued"
+  STATUS_BOOTING = "booting"
+  STATUS_RUNNING = "running"
+  STATUS_SUCCESS = "success"
+  STATUS_ERROR = "error"
+  STATUS_KILLED = "killed"
+  FINISHED_STATUS = [STATUS_SUCCESS, STATUS_ERROR, STATUS_KILLED]
+
+  # @param [TreasureData::Client] client
+  # @param [String] job_id
+  # @param [String] type
+  # @param [String] query
+  # @param [Fixnum] status
+  # @param [String] url
+  # @param [Boolean] debug
+  # @param [String] start_at
+  # @param [String] end_at
+  # @param [String] cpu_time
+  # @param [String] result_size
+  # @param [Array] result
+  # @param [String] result_url
+  # @param [Array] hive_result_schema
+  # @param [Fixnum] priority
+  # @param [Fixnum] retry_limit
+  # @param [String] org_name
+  # @param [String] db_name
+  # @param [Fixnum] duration
+  # @param [Fixnum] num_records
+  def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, cpu_time=nil,
+                 result_size=nil, result=nil, result_url=nil, hive_result_schema=nil, priority=nil, retry_limit=nil,
+                 org_name=nil, db_name=nil, duration=nil, num_records=nil)
+    super(client)
+    @job_id = job_id
+    @type = type
+    @url = url
+    @query = query
+    @status = status
+    @debug = debug
+    @start_at = start_at
+    @end_at = end_at
+    @cpu_time = cpu_time
+    @result_size = result_size
+    @result = result
+    @result_url = result_url
+    @hive_result_schema = hive_result_schema
+    @priority = priority
+    @retry_limit = retry_limit
+    @db_name = db_name
+    @duration = duration
+    @num_records = num_records
+  end
+
+  # @!attribute [r] job_id
+  # @!attribute [r] type
+  # @!attribute [r] result_url
+  # @!attribute [r] priority
+  # @!attribute [r] retry_limit
+  # @!attribute [r] org_name
+  # @!attribute [r] db_name
+  # @!attribute [r] duration
+  # @!attribute [r] num_records
+  attr_reader :job_id, :type, :result_url
+  attr_reader :priority, :retry_limit, :org_name, :db_name
+  attr_reader :duration, :num_records
+
+  # @option timeout [Integer,nil] timeout in second
+  # @option wait_interval [Integer,nil] interval in second of polling the job status
+  # @param detail [Boolean] update job detail or not
+  # @param verbose [Boolean] out retry log to stderr or not
+  def wait(*args)
+    opthash = Hash.try_convert(args.last)
+    if opthash
+      args.pop
+      detail = opthash.fetch(:detail, false)
+      verbose = opthash.fetch(:verbose, ENV['TD_CLIENT_DEBUG'])
+    end
+    timeout = args[0]
+    wait_interval = args[1] || 2
+    deadline = monotonic_clock + timeout if timeout
+    timeout_klass = Class.new(Exception)
+    begin
+      if timeout
+        if deadline <= monotonic_clock
+          raise timeout_klass, "timeout (#{timeout}) exceeded wait_interval=#{wait_interval}"
+        end
+      end
+      sleep wait_interval
+      detail ? update_status! : update_progress!
+      yield self if block_given?
+    rescue timeout_klass
+      raise Timeout::Error, $!.message
+    rescue Timeout::Error, SystemCallError, EOFError, SocketError, HTTPClient::ConnectTimeoutError
+      $stderr.puts "ignore network error (#{$!}); retry..." if verbose
+    end until finished?
+  end
+
+  def kill!
+    # TODO
+  end
+
+  # @return [String]
+  def query
+    update_status! unless @query || finished?
+    @query
+  end
+
+  # @return [String]
+  def status
+    update_status! unless @status || finished?
+    @status
+  end
+
+  # @return [String]
+  def url
+    update_status! unless @url || finished?
+    @url
+  end
+
+  # @return [Boolean]
+  def debug
+    update_status! unless @debug || finished?
+    @debug
+  end
+
+  # @return [Time, nil]
+  def start_at
+    update_status! unless @start_at || finished?
+    @start_at && !@start_at.empty? ? Time.parse(@start_at) : nil
+  end
+
+  # @return [Time, nil]
+  def end_at
+    update_status! unless @end_at || finished?
+    @end_at && !@end_at.empty? ? Time.parse(@end_at) : nil
+  end
+
+  # @return [String]
+  def cpu_time
+    update_status! unless @cpu_time || finished?
+    @cpu_time
+  end
+
+  # @return [Array]
+  def hive_result_schema
+    update_status! unless @hive_result_schema.instance_of? Array || finished?
+    @hive_result_schema
+  end
+
+  # @return [String]
+  def result_size
+    update_status! unless @result_size || finished?
+    @result_size
+  end
+
+  # @return [Array]
+  def result
+    unless @result
+      return nil unless finished?
+      @result = @client.job_result(@job_id)
+    end
+    @result
+  end
+
+  # @param [String] format
+  # @param [IO] io
+  # @param [Proc] block
+  # @return [nil, String]
+  def result_format(format, io=nil, &block)
+    return nil unless finished?
+    @client.job_result_format(@job_id, format, io, &block)
+  end
+
+  def result_raw(format, io=nil, &block)
+    return nil unless finished?
+    @client.job_result_raw(@job_id, format, io, &block)
+  end
+
+  # @yield [result]
+  # @return [nil]
+  def result_each_with_compr_size(&block)
+    if @result
+      @result.each(&block)
+    else
+      @client.job_result_each_with_compr_size(@job_id, &block)
+    end
+    nil
+  end
+
+  # @yield [result]
+  # @return [nil]
+  def result_each(&block)
+    if @result
+      @result.each(&block)
+    else
+      @client.job_result_each(@job_id, &block)
+    end
+    nil
+  end
+
+  # @return [Boolean]
+  def finished?
+    update_progress! unless @status
+    FINISHED_STATUS.include?(@status)
+  end
+
+  # @return [Boolean]
+  def success?
+    update_progress! unless @status
+    @status == STATUS_SUCCESS
+  end
+
+  # @return [Boolean]
+  def error?
+    update_progress! unless @status
+    @status == STATUS_ERROR
+  end
+
+  # @return [Boolean]
+  def killed?
+    update_progress! unless @status
+    @status == STATUS_KILLED
+  end
+
+  # @return [Boolean]
+  def queued?
+    update_progress! unless @status
+    @status == STATUS_QUEUED
+  end
+
+  # @return [Boolean]
+  def running?
+    update_progress! unless @status
+    @status == STATUS_RUNNING
+  end
+
+  def update_progress!
+    @status = @client.job_status(@job_id)
+  end
+
+  def update_status!
+    type, query, status, url, debug, start_at, end_at, cpu_time,
+      result_size, result_url, hive_result_schema, priority, retry_limit,
+      org_name, db_name , duration, num_records = @client.api.show_job(@job_id)
+    @query = query
+    @status = status
+    @url = url
+    @debug = debug
+    @start_at = start_at
+    @end_at = end_at
+    @cpu_time = cpu_time
+    @result_size = result_size
+    @result_url = result_url
+    @hive_result_schema = hive_result_schema
+    @priority = priority
+    @retry_limit = retry_limit
+    @db_name = db_name
+    @duration = duration
+    @num_records = num_records
+    self
+  end
+
+  private
+  def monotonic_clock
+    if defined?(Process.clock_gettime)
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    else
+      Time.now.to_i
+    end
+  end
+end
+
+
+class ScheduledJob < Job
+  attr_reader :scheduled_at
+
+  # @param [TreasureData::Client] client
+  # @param [String] scheduled_at
+  # @param [...] super_args for Job#initialize
+  def initialize(client, scheduled_at, *super_args)
+    super(client, *super_args)
+    if scheduled_at.to_s.empty?
+      @scheduled_at = nil
+    else
+      @scheduled_at = Time.parse(scheduled_at) rescue nil
+    end
+  end
+end
+
+
+class Schedule < Model
+  # @param [TreasureData::Client] client
+  # @param [String] name
+  # @param [String] cron
+  # @param [String] query
+  # @param [Fixnum] database
+  # @param [String] result_url
+  # @param [String] timezone
+  # @param [String] delay
+  # @param [String] next_time
+  # @param [String] priority
+  # @param [String] retry_limit
+  # @param [String] org_name
+  def initialize(client, name, cron, query, database=nil, result_url=nil, timezone=nil, delay=nil, next_time=nil,
+                 priority=nil, retry_limit=nil, org_name=nil)
+    super(client)
+    @name = name
+    @cron = cron
+    @query = query
+    @database = database
+    @result_url = result_url
+    @timezone = timezone
+    @delay = delay
+    @next_time = next_time
+    @priority = priority
+    @retry_limit = retry_limit
+  end
+
+  # @!attribute [r] name
+  # @!attribute [r] cron
+  # @!attribute [r] query
+  # @!attribute [r] database
+  # @!attribute [r] result_url
+  # @!attribute [r] delay
+  # @!attribute [r] priority
+  # @!attribute [r] retry_limit
+  # @!attribute [r] org_name
+  attr_reader :name, :cron, :query, :database, :result_url, :timezone, :delay, :priority, :retry_limit, :org_name
+
+  # @return [Time, nil]
+  def next_time
+    @next_time ? Time.parse(@next_time) : nil
+  end
+
+  # @param [String] time
+  # @param [Fixnum] num
+  # @return [Array]
+  def run(time, num)
+    @client.run_schedule(@name, time, num)
+  end
+end
+
+
+class Result < Model
+  # @param [TreasureData::Client] client
+  # @param [String] name
+  # @param [String] url
+  # @param [String] org_name
+  def initialize(client, name, url, org_name)
+    super(client)
+    @name = name
+    @url = url
+  end
+
+  # @!attribute [r] name
+  # @!attribute [r] url
+  # @!attribute [r] org_name
+  attr_reader :name, :url, :org_name
+end
+
+
+class BulkImport < Model
+  # @param [TreasureData::Client] client
+  # @param [Hash] data
+  def initialize(client, data={})
+    super(client)
+    @name = data['name']
+    @database = data['database']
+    @table = data['table']
+    @status = data['status']
+    @upload_frozen = data['upload_frozen']
+    @job_id = data['job_id']
+    @valid_records = data['valid_records']
+    @error_records = data['error_records']
+    @valid_parts = data['valid_parts']
+    @error_parts = data['error_parts']
+  end
+
+  # @!attribute [r] name
+  # @!attribute [r] database
+  # @!attribute [r] table
+  # @!attribute [r] status
+  # @!attribute [r] job_id
+  # @!attribute [r] valid_records
+  # @!attribute [r] error_records
+  # @!attribute [r] valid_parts
+  # @!attribute [r] error_parts
+  # @!attribute [r] org_name
+  attr_reader :name
+  attr_reader :database
+  attr_reader :table
+  attr_reader :status
+  attr_reader :job_id
+  attr_reader :valid_records
+  attr_reader :error_records
+  attr_reader :valid_parts
+  attr_reader :error_parts
+  attr_reader :org_name
+
+  # @return [Boolean]
+  def upload_frozen?
+    @upload_frozen
+  end
+end
+
+
+class User < Model
+  def initialize(client, name, org_name, role_names, email)
+    super(client)
+    @name = name
+    @email = email
+  end
+
+  attr_reader :client, :name, :org_name, :role_names, :email
+end
+
+
+class AccessControl < Model
+  # @param [TreasureData::Client] client
+  # @param [String] subject
+  # @param [String] action
+  # @param [String] scope
+  # @param [Array] grant_option
+  def initialize(client, subject, action, scope, grant_option)
+    super(client)
+    @subject = subject
+    @action = action
+    @scope = scope
+    @grant_option = grant_option
+  end
+
+  # @!attribute [r] subject
+  # @!attribute [r] action
+  # @!attribute [r] scope
+  # @!attribute [r] grant_option
+  attr_reader :subject, :action, :scope, :grant_option
+end
+
+
+end