td 0.7.3 → 0.7.4

data/ChangeLog

@@ -1,4 +1,10 @@
 
+== 2011-08-18 version 0.7.4
+
+* Added set-schema subcommand
+* Added describe-table subcommand
+
+
 == 2011-08-15 version 0.7.3
 
 * show-jobs: removed --from and --around options

data/lib/td/api.rb

@@ -36,12 +36,43 @@ class API
     unless name =~ /^([a-z0-9_]+)$/
       raise "Name must consist only of alphabets, numbers, '_'."
     end
+    name
   end
 
   def self.validate_table_name(name)
     validate_database_name(name)
   end
 
+  def self.validate_column_name(name)
+    name = name.to_s
+    if name.empty?
+      raise "Empty column name is not allowed"
+    end
+    if 32 < name.length
+      raise "Column name must be to 32 characters, got #{name.length} characters."
+    end
+    unless name =~ /^([a-z0-9_]+)$/
+      raise "Column name must consist only of alphabets, numbers, '_'."
+    end
+  end
+
+  def self.normalize_type_name(name)
+    case name
+    when /int/i, /integer/i
+      "int"
+    when /long/i, /bigint/i
+      "long"
+    when /string/i
+      "string"
+    when /float/i
+      "float"
+    when /double/i
+      "double"
+    else
+      raise "Type name must eather of int, long, string float or double"
+    end
+  end
+
   ####
   ## Database API
   ##
@@ -94,19 +125,20 @@ class API
       name = m['name']
       type = (m['type'] || '?').to_sym
       count = (m['count'] || 0).to_i  # TODO?
-      result[name] = [type, count]
+      schema = JSON.parse(m['schema']) # FIXME?
+      result[name] = [type, schema, count]
     }
     return result
   end
 
-  # => true
-  def create_table(db, table, type)
+  def create_log_or_item_table(db, table, type)
     code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
     if code != "200"
       raise_error("Create #{type} table failed", res)
     end
     return true
   end
+  private :create_log_or_item_table
 
   # => true
   def create_log_table(db, table)
@@ -118,6 +150,25 @@ class API
     create_table(db, table, :item)
   end
 
+  def create_table(db, table, type)
+    schema = schema.to_s
+    code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
+    if code != "200"
+      raise_error("Create #{type} table failed", res)
+    end
+    return true
+  end
+  private :create_table
+
+  # => true
+  def update_schema(db, table, schema_json)
+    code, body, res = post("/v3/table/update-schema/#{e db}/#{e table}", {'schema'=>schema_json})
+    if code != "200"
+      raise_error("Create schema table failed", res)
+    end
+    return true
+  end
+
   # => type:Symbol
   def delete_table(db, table)
     code, body, res = post("/v3/table/delete/#{e db}/#{e table}")

data/lib/td/client.rb

@@ -59,18 +59,18 @@ class Client
   end
 
   # => true
-  def create_table(db_name, table_name, type)
-    @api.create_table(db_name, table_name, type)
+  def create_log_table(db_name, table_name)
+    @api.create_log_table(db_name, table_name)
   end
 
   # => true
-  def create_log_table(db_name, table_name)
-    create_table(db_name, table_name, :log)
+  def create_item_table(db_name, table_name)
+    @api.create_item_table(db_name, table_name)
   end
 
   # => true
-  def create_item_table(db_name, table_name)
-    create_table(db_name, table_name, :item)
+  def update_schema(db_name, table_name, schema)
+    @api.update_schema(db_name, table_name, schema.to_json)
   end
 
   # => type:Symbol
@@ -81,17 +81,17 @@ class Client
   # => [Table]
   def tables(db_name)
     m = @api.list_tables(db_name)
-    m.map {|table_name,(type,count)|
-      Table.new(self, db_name, table_name, type, count)
+    m.map {|table_name,(type,schema,count)|
+      schema = Schema.new.from_json(schema)
+      Table.new(self, db_name, table_name, type, schema, count)
     }
   end
 
   # => Table
   def table(db_name, table_name)
-    m = @api.list_tables(db_name)
-    m.each_pair {|name,(type,count)|
-      if name == table_name
-        return Table.new(self, db_name, name, type, count)
+    tables(db_name).each {|t|
+      if t.name == table_name
+        return t
       end
     }
     raise NotFoundError, "Table '#{db_name}.#{table_name}' does not exist"
@@ -168,16 +168,12 @@ class Database < Model
     @tables
   end
 
-  def create_table(name, type)
-    @client.create_table(@db_name, name, type)
-  end
-
   def create_log_table(name)
-    create_table(name, :log)
+    @client.create_log_table(@db_name, name)
   end
 
   def create_item_table(name)
-    create_table(name, :item)
+    @client.create_item_table(@db_name, name)
   end
 
   def table(table_name)
@@ -194,28 +190,24 @@ class Database < Model
 end
 
 class Table < Model
-  def initialize(client, db_name, table_name, type, count)
+  def initialize(client, db_name, table_name, type, schema, count)
     super(client)
     @db_name = db_name
     @table_name = table_name
     @type = type
+    @schema = schema
     @count = count
   end
 
-  attr_reader :type, :count
+  attr_reader :type, :db_name, :table_name, :schema, :count
 
-  def database_name
-    @db_name
-  end
+  alias database_name db_name
+  alias name table_name
 
   def database
     @client.database(@db_name)
   end
 
-  def name
-    @table_name
-  end
-
   def identifier
     "#{@db_name}.#{@table_name}"
   end
@@ -225,6 +217,50 @@ class Table < Model
   end
 end
 
+class Schema
+  class Field
+    def initialize(name, type)
+      @name = name
+      @type = type
+    end
+    attr_reader :name
+    attr_reader :type
+  end
+
+  def self.parse(cols)
+    fields = cols.split(',').map {|col|
+      name, type, *_ = col.split(':')
+      Field.new(name, type)
+    }
+    Schema.new(fields)
+  end
+
+  def initialize(fields=[])
+    @fields = fields
+  end
+
+  attr_reader :fields
+
+  def add_field(name, type)
+    @fields << Field.new(name, type)
+  end
+
+  def to_s
+    @fields.map {|f| "#{f.name}:#{f.type}" }.join(',')
+  end
+
+  def to_json(*args)
+    @fields.map {|f| [f.name, f.type] }.to_json(*args)
+  end
+
+  def from_json(obj)
+    @fields = obj.map {|f|
+      Field.new(f[0], f[1])
+    }
+    self
+  end
+end
+
 class Job < Model
   def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, result=nil)
     super(client)
@@ -332,6 +368,5 @@ class Job < Model
   end
 end
 
-
 end
 

data/lib/td/command/common.rb

@@ -1,11 +1,22 @@
 
 module TreasureData
+
+autoload :API, 'td/api'
+autoload :Client, 'td/client'
+autoload :Database, 'td/client'
+autoload :Table, 'td/client'
+autoload :Schema, 'td/client'
+autoload :Job, 'td/client'
+
 module Command
   private
   def cmd_opt(name, *args)
     if args.last.to_s =~ /_$/
       multi = true
       args.push args.pop.to_s[0..-2]+'...'
+    elsif args.last.to_s =~ /_\?$/
+      multi = true
+      args.push args.pop.to_s[0..-3]+'...?'
     end
 
     req_args, opt_args = args.partition {|a| a.to_s !~ /\?$/ }
@@ -61,7 +72,6 @@ EOF
     unless apikey
       raise ConfigError, "Account is not configured."
     end
-    require 'td/client'
     Client.new(apikey)
   end
 

data/lib/td/command/list.rb

@@ -33,25 +33,31 @@ module List
   add_list 'server', 'server-status', 'Show status of the Treasure Data server'
   add_list 'database', 'show-databases', 'Show list of databases'
   add_list 'table', 'show-tables', 'Show list of tables'
-  add_list 'query', 'show-jobs', 'Show list of jobs'
   add_list 'database', 'create-database', 'Create a database'
   add_list 'table', 'create-log-table', 'Create a log table'
   #add_list 'table', 'create-item-table', 'Create a item table'
+  add_list 'table', 'set-schema', 'Set a schema on a table'
+  add_list 'table', 'describe-table', 'Describe information of a table'
   add_list 'database', 'drop-database', 'Delete a database'
   add_list 'table', 'drop-table', 'Delete a table'
   add_list 'query', 'query', 'Start a query'
   add_list 'query', 'job', 'Show status and result of a job'
+  add_list 'query', 'show-jobs', 'Show list of jobs'
   add_list 'import', 'import', 'Import files to a table'
   add_list 'list', 'version', 'Show version'
 
   add_alias 'show-dbs', 'show-databases'
-  add_alias 'show-database', 'show-databases'
+  add_alias 'databases', 'show-databases'
+  add_alias 'dbs', 'show-databases'
   add_alias 'create-db', 'create-databases'
   add_alias 'drop-db', 'create-databases'
-  add_alias 'show-table', 'show-tables'
+  add_alias 'tables', 'show-tables'
+  add_alias 'table', 'describe-table'
+  add_alias 'show-table', 'describe-table'
   add_alias 'delete-database', 'drop-database'
   add_alias 'delete-table', 'drop-table'
   add_alias 'jobs', 'show-jobs'
+  add_alias 'update-schema', 'set-schema'
 
   add_guess 'create-table', 'create-log-table'
   add_guess 'drop-log-table', 'drop-table'

data/lib/td/command/table.rb

@@ -2,13 +2,17 @@
 module TreasureData
 module Command
 
-  def create_table_type(type, db_name, table_name)
+  def create_log_or_item_table(mode_log, db_name, table_name)
     client = get_client
 
     API.validate_table_name(table_name)
 
     begin
-      client.create_table(db_name, table_name, type)
+      if mode_log
+        client.create_log_table(db_name, table_name)
+      else
+        client.create_item_table(db_name, table_name)
+      end
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Database '#{db_name}' does not exist."
@@ -22,20 +26,20 @@ module Command
 
     $stderr.puts "Table '#{db_name}.#{table_name}' is created."
   end
-  private :create_table_type
+  private :create_log_or_item_table
 
   def create_log_table
     op = cmd_opt 'create-log-table', :db_name, :table_name
     db_name, table_name = op.cmd_parse
 
-    create_table_type(:log, db_name, table_name)
+    create_log_or_item_table(true, db_name, table_name)
   end
 
   def create_item_table
     op = cmd_opt 'create-item-table', :db_name, :table_name
     db_name, table_name = op.cmd_parse
 
-    create_table_type(:item, db_name, table_name)
+    create_log_or_item_table(false, db_name, table_name)
   end
 
   def drop_table
@@ -56,6 +60,42 @@ module Command
     $stderr.puts "Table '#{db_name}.#{table_name}' is deleted."
   end
 
+  def set_schema
+    op = cmd_opt 'set-schema', :db_name, :table_name, :columns_?
+
+    db_name, table_name, *columns = op.cmd_parse
+
+    schema = Schema.new
+    columns.each {|column|
+      name, type = column.split(':',2)
+      name = name.to_s
+      type = type.to_s
+
+      API.validate_column_name(name)
+      type = API.normalize_type_name(type)
+
+      if schema.fields.find {|f| f.name == name }
+        $stderr.puts "Column name '#{name}' is duplicated."
+        exit 1
+      end
+      schema.add_field(name, type)
+
+      if name == 'v' || name == 'time'
+        $stderr.puts "Column name '#{name}' is reserved."
+        exit 1
+      end
+    }
+
+    client = get_client
+
+    find_table(client, db_name, table_name)
+
+    client.update_schema(db_name, table_name, schema)
+
+    $stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
+    $stderr.puts "Use '#{$prog} describe-table #{db_name} #{table_name}' to confirm the schema."
+  end
+
   def show_tables
     op = cmd_opt 'show-tables', :db_name?
     db_name = op.cmd_parse
@@ -72,14 +112,17 @@ module Command
     rows = []
     dbs.each {|db|
       db.tables.each {|table|
-        rows << {:Database => db.name, :Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s}
+        pschema = table.schema.fields.map {|f|
+          "#{f.name}:#{f.type}"
+        }.join(', ')
+        rows << {:Database => db.name, :Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s, :Schema=>pschema.to_s}
       }
     }
     rows = rows.sort_by {|map|
       [map[:Database], map[:Type].size, map[:Table]]
     }
 
-    puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count])
+    puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count, :Schema])
 
     if rows.empty?
       if db_name
@@ -95,6 +138,24 @@ module Command
     end
   end
 
+  def describe_table
+    op = cmd_opt 'describe-table', :db_name, :table_name
+
+    db_name, table_name = op.cmd_parse
+
+    client = get_client
+
+    table = find_table(client, db_name, table_name)
+
+    puts "Name      : #{table.db_name}.#{table.name}"
+    puts "Type      : #{table.type}"
+    puts "Count     : #{table.count}"
+    puts "Schema    : ("
+    table.schema.fields.each {|f|
+      puts "    #{f.name}:#{f.type}"
+    }
+    puts ")"
+  end
 end
 end
 

data/lib/td/command/td.rb

@@ -1,9 +1,11 @@
 
 require 'optparse'
+require 'td/version'
 
 $prog = File.basename($0)
 
 op = OptionParser.new
+op.version = TreasureData::VERSION
 op.banner = <<EOF
 usage: #{$prog} [options] COMMAND [args]
 

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.7.2'
+VERSION = '0.7.4'
 
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: td
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 11
   prerelease: false
   segments: 
   - 0
   - 7
-  - 3
-  version: 0.7.3
+  - 4
+  version: 0.7.4
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-15 00:00:00 +09:00
+date: 2011-08-18 00:00:00 +09:00
 default_executable: td
 dependencies: 
 - !ruby/object:Gem::Dependency