td 0.10.84 → 0.10.85

data/ChangeLog

@@ -1,3 +1,11 @@
+== 2013-08-23 version 0.10.85
+
+* Fix 'undefined method job_priority_id_of' error
+* Add table:expire_days subcommand
+* Update bulk-import-java to 0.2.1
+* Add import command to support new bulk import
+
+
 == 2013-07-25 version 0.10.84
 
 * bulk_import now checks first argument is correct session name

data/build/update-td-bulk-import-java.sh

@@ -19,10 +19,13 @@ if [ -n "$chrev" ];then
 fi
 
 revname="$(git show --pretty=format:'%H %ad' | head -n 1)"
-vername="0.2.0-SNAPSHOT"
+vername="0.2.1"
 
 mvn package -Dmaven.test.skip=true || exit 1
+echo "copy td-bulk-import-${vername}.jar"
 cp target/td-bulk-import-${vername}.jar ../../java/td-bulk-import-${vername}.jar
+echo "copy logging.properties"
+cp src/test/resources/java/logging.properties ../../java/logging.properties
 
 if [ -n "$chrev" ];then
     git checkout master

data/java/logging.properties

@@ -0,0 +1,53 @@
+############################################################
+#  	Treasure Data BulkImport Logging Configuration File
+#
+# You can use a different file by specifying a filename
+# with the java.util.logging.config.file system property.  
+# For example java -Djava.util.logging.config.file=myfile
+############################################################
+
+############################################################
+#  	Global properties
+############################################################
+
+# "handlers" specifies a comma separated list of log Handler 
+# classes.  These handlers will be installed during VM startup.
+# Note that these classes must be on the system classpath.
+# By default we only configure a ConsoleHandler, which will only
+# show messages at the INFO and above levels.
+handlers= java.util.logging.FileHandler
+
+# To also add the FileHandler, use the following line instead.
+#handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler
+
+# Default global logging level.
+# This specifies which kinds of events are logged across
+# all loggers.  For any given facility this global level
+# can be overriden by a facility specific level
+# Note that the ConsoleHandler also has a separate level
+# setting to limit messages printed to the console.
+.level= INFO
+
+############################################################
+# Handler specific properties.
+# Describes specific configuration info for Handlers.
+############################################################
+
+java.util.logging.FileHandler.level = INFO
+java.util.logging.FileHandler.pattern=td-bulk-import.log
+java.util.logging.FileHandler.limit = 50000
+java.util.logging.FileHandler.count = 1
+java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter
+
+# Limit the message that are printed on the console to INFO and above.
+java.util.logging.ConsoleHandler.level = INFO
+java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
+
+############################################################
+# Facility specific properties.
+# Provides extra control for each logger.
+############################################################
+
+# For example, set the com.xyz.foo logger to only log SEVERE
+# messages:
+com.xyz.foo.level = SEVERE

data/java/td-bulk-import-java.version

@@ -1 +1 @@
-13ef573e6221c827f793ed68fea7f7e4978e7957 Fri Jun 28 16:14:15 2013 +0900
+9fab1d2cc1d026d0a355945c52ea103bb8b05e1d Wed Aug 21 15:28:09 2013 +0900

data/lib/td/command/import.rb

@@ -2,321 +2,213 @@
 module TreasureData
 module Command
 
-  IMPORT_TEMPLATES = {
-    'apache' => [
-                  /^([^ ]*) [^ ]* ([^ ]*) \[([^\]]*)\] "(\S+)(?: +([^ ]*) +\S*)?" ([^ ]*) ([^ ]*)(?: "([^\"]*)" "([^\"]*)")?$/,
-                  ['host', 'user', 'time', 'method', 'path', 'code', 'size', 'referer', 'agent'],
-                  "%d/%b/%Y:%H:%M:%S %z"],
-    'syslog' => [
-                  /^([^ ]* [^ ]* [^ ]*) ([^ ]*) ([a-zA-Z0-9_\/\.\-]*)(?:\[([0-9]+)\])?[^\:]*\: *(.*)$/,
-                  ['time', 'host', 'ident', 'pid', 'message'],
-                  "%b %d %H:%M:%S"],
-  }
-
-  # TODO import-item
-  # TODO tail
-
-  def table_import(op)
-    op.banner << "\nsupported formats:\n"
-    op.banner << "  apache\n"
-    op.banner << "  syslog\n"
-    op.banner << "  msgpack\n"
-    op.banner << "  json\n"
-
-    format = 'apache'
-    time_key = 'time'
-    auto_create = false
-
-    op.on('--format FORMAT', "file format (default: #{format})") {|s|
-      format = s
-    }
-
-    op.on('--apache', "same as --format apache; apache common log format") {
-      format = 'apache'
-    }
-
-    op.on('--syslog', "same as --format syslog; syslog") {
-      format = 'syslog'
-    }
-
-    op.on('--msgpack', "same as --format msgpack; msgpack stream format") {
-      format = 'msgpack'
-    }
-
-    op.on('--json', "same as --format json; LF-separated json format") {
-      format = 'json'
-    }
-
-    op.on('-t', '--time-key COL_NAME', "time key name for json and msgpack format (e.g. 'created_at')") {|s|
-      time_key = s
-    }
-
-    op.on('--auto-create-table', "Create table and database if doesn't exist", TrueClass) { |b|
-      auto_create = b
-    }
-
-    db_name, table_name, *paths = op.cmd_parse
-
-    client = get_client
-
-    if auto_create
-      # Merge with db_create and table_create after refactoring
-      API.validate_database_name(db_name)
-      begin
-        client.create_database(db_name)
-        $stderr.puts "Database '#{db_name}' is created."
-      rescue AlreadyExistsError
-      end
+  BASE_PATH = File.expand_path('../../..', File.dirname(__FILE__))
 
-      API.validate_table_name(table_name)
-      begin
-        client.create_log_table(db_name, table_name)
-        $stderr.puts "Table '#{db_name}.#{table_name}' is created."
-      rescue AlreadyExistsError
-      end
-    end
+  JAVA_COMMAND = "java"
+  JAVA_COMMAND_CHECK = "#{JAVA_COMMAND} -version"
+  JAVA_MAIN_CLASS = "com.treasure_data.bulk_import.BulkImportMain"
+  JAVA_HEAP_MAX_SIZE = "-Xmx1024m" # TODO
 
-    case format
-    when 'json', 'msgpack'
-      #unless time_key
-      #  $stderr.puts "-t, --time-key COL_NAME (e.g. '-t created_at') parameter is required for #{format} format"
-      #  exit 1
-      #end
-      if format == 'json'
-        require 'json'
-        require 'time'
-        parser = JsonParser.new(time_key)
-      else
-        parser = MessagePackParser.new(time_key)
-      end
+  APP_OPTION_PREPARE = "prepare"
+  APP_OPTION_UPLOAD = "upload"
 
-    else
-      regexp, names, time_format = IMPORT_TEMPLATES[format]
-      if !regexp || !names || !time_format
-        $stderr.puts "Unknown format '#{format}'"
-        exit 1
-      end
-      parser = TextParser.new(names, regexp, time_format)
-    end
+  def import_list(op)
+    require 'td/command/bulk_import'
+    bulk_import_list(op)
+  end
 
-    get_table(client, db_name, table_name)
+  def import_show(op)
+    require 'td/command/bulk_import'
+    bulk_import_show(op)
+  end
 
-    require 'zlib'
+  def import_create(op)
+    require 'td/command/bulk_import'
+    bulk_import_create(op)
+  end
 
-    files = paths.map {|path|
-      if path == '-'
-        $stdin
-      elsif path =~ /\.gz$/
-        require 'td/compat_gzip_reader'
-        Zlib::GzipReader.open(path)
-      else
-        File.open(path)
-      end
-    }
+  def import_prepare(op)
+    import_generic(APP_OPTION_PREPARE)
+  end
 
-    require 'msgpack'
-    require 'tempfile'
-    #require 'thread'
+  def import_upload(op)
+    import_generic(APP_OPTION_UPLOAD)
+  end
 
-    files.zip(paths).each {|file,path|
-      import_log_file(file, path, client, db_name, table_name, parser)
-    }
+  def import_perform(op)
+    require 'td/command/bulk_import'
+    bulk_import_perform(op)
+  end
+
+  def import_error_records(op)
+    require 'td/command/bulk_import'
+    bulk_import_error_records(op)
+  end
+
+  def import_commit(op)
+    require 'td/command/bulk_import'
+    bulk_import_commit(op)
+  end
+
+  def import_delete(op)
+    require 'td/command/bulk_import'
+    bulk_import_delete(op)
+  end
+
+  def import_freeze(op)
+    require 'td/command/bulk_import'
+    bulk_import_freeze(op)
+  end
 
-    puts "done."
+  def import_unfreeze(op)
+    require 'td/command/bulk_import'
+    bulk_importunfreeze(op)
   end
 
   private
-  def import_log_file(file, path, client, db_name, table_name, parser)
-    puts "importing #{path}..."
-
-    out = Tempfile.new('td-import')
-    out.binmode if out.respond_to?(:binmode)
-
-    writer = Zlib::GzipWriter.new(out)
-
-    n = 0
-    x = 0
-    has_bignum = false
-    parser.call(file, path) {|record|
-      entry = begin
-                record.to_msgpack
-              rescue RangeError
-                has_bignum = true
-                TreasureData::API.normalized_msgpack(record)
-              end
-      writer.write entry
-
-      n += 1
-      x += 1
-      if n % 10000 == 0
-        puts "  imported #{n} entries from #{path}..."
-
-      elsif out.pos > 1024*1024  # TODO size
-        puts "  imported #{n} entries from #{path}..."
-        begin
-          writer.finish
-          size = out.pos
-          out.pos = 0
-
-          puts "  uploading #{size} bytes..."
-          client.import(db_name, table_name, "msgpack.gz", out, size)
-
-          out.truncate(0)
-          out.pos = 0
-          x = 0
-          writer = Zlib::GzipWriter.new(out)
-        rescue
-          $stderr.puts "  #{$!}"
-          return 1 # TODO error
-        end
-      end
-    }
+  def import_generic(subcmd)
+    # has java runtime
+    check_java
+
+    # show help
+    show_help = ARGV.size == 0 || (ARGV.size == 1 || ARGV[0] =~ /^import:/)
 
-    if x != 0
-      writer.finish
-      size = out.pos
-      out.pos = 0
+    # configure jvm options
+    jvm_opts = [ JAVA_HEAP_MAX_SIZE ]
 
-      puts "  uploading #{size} bytes..."
-      # TODO upload on background thread
-      client.import(db_name, table_name, "msgpack.gz", out, size)
+    # configure java options
+    java_opts = [ "-cp \"#{find_td_bulk_import_jar()}\"" ]
+
+    # configure system properties
+    sysprops = set_sysprops()
+
+    # configure java command-line arguments
+    java_args = []
+    java_args << JAVA_MAIN_CLASS
+    java_args << subcmd
+    if show_help
+      java_args << "--help"
+    else
+      java_args << ARGV
     end
 
-    puts "  imported #{n} entries from #{path}."
-    $stderr.puts normalized_message if has_bignum
-  ensure
-    out.close rescue nil
-    writer.close rescue nil
+    # TODO consider parameters including spaces; don't use join(' ')
+    cmd = "#{JAVA_COMMAND} #{jvm_opts.join(' ')} #{java_opts.join(' ')} #{sysprops.join(' ')} #{java_args.join(' ')}"
+    exec cmd
   end
 
-  require 'date'  # DateTime#strptime
-  require 'time'  # Time#strptime, Time#parse
+  private
+  def check_java
+    pid = do_fork(JAVA_COMMAND_CHECK)
+    pid, stat = Process.waitpid2(pid)
 
-  class TextParser
-    def initialize(names, regexp, time_format)
-      @names = names
-      @regexp = regexp
-      @time_format = time_format
+    if stat.exitstatus != 0
+      $stderr.puts "Java is not installed. 'td import' command requires Java (version 1.6 or later). If Java is not installed yet, please use 'bulk_import' commands instead of this command."
+      exit 1
     end
+  end
 
-    def call(file, path, &block)
-      i = 0
-      file.each_line {|line|
-        i += 1
-        begin
-          line.rstrip!
-          m = @regexp.match(line)
-          unless m
-            raise "invalid log format at #{path}:#{i}"
-          end
-
-          record = {}
-
-          cap = m.captures
-          @names.each_with_index {|name,cap_i|
-            if value = cap[cap_i]
-              if name == "time"
-                value = parse_time(value).to_i
-              end
-              record[name] = value
-            end
-          }
-
-          block.call(record)
-
-        rescue
-          $stderr.puts "  skipped: #{$!}: #{line.dump}"
-        end
-      }
+  def do_fork(cmd)
+    Process.fork do
+      begin
+        Process.exec(*cmd)
+      ensure
+        exit! 127
+      end
     end
+  end
 
-    if Time.respond_to?(:strptime)
-      def parse_time(value)
-        Time.strptime(value, @time_format)
-      end
-    else
-      def parse_time(value)
-        Time.parse(DateTime.strptime(value, @time_format).to_s)
-      end
+  private
+  def find_td_bulk_import_jar
+    libjars = Dir.glob("#{BASE_PATH}/java/**/*.jar")
+    found = libjars.find { |path| File.basename(path) =~ /^td-bulk-import/ }
+    if found.nil?
+      $stderr.puts "td-bulk-import.jar is not found."
+      exit
     end
+    td_bulk_import_jar = libjars.delete(found)
+    td_bulk_import_jar
   end
 
-  class JsonParser
-    def initialize(time_key)
-      require 'json'
-      @time_key = time_key
+  private
+  def set_sysprops
+    sysprops = []
+
+    # set apiserver
+    set_sysprops_endpoint(sysprops)
+
+    # set http_proxy
+    set_sysprops_http_proxy(sysprops)
+
+    # set configuration file for logging
+    conf_file = find_logging_conf_file
+    if conf_file
+      sysprops << "-Djava.util.logging.config.file=#{conf_file}"
     end
 
-    def call(file, path, &block)
-      i = 0
-      file.each_line {|line|
-        i += 1
-        begin
-          record = JSON.parse(line)
-
-          unless record.is_a?(Hash)
-            raise "record must be a Hash"
-          end
-
-          time = record[@time_key]
-          unless time
-            raise "record doesn't have '#{@time_key}' column"
-          end
-
-          case time
-          when Integer
-            # do nothing
-          else
-            time = Time.parse(time.to_s).to_i
-          end
-          record['time'] = time
-
-          block.call(record)
-
-        rescue
-          $stderr.puts "  skipped: #{$!}: #{line.dump}"
+    # set API key
+    sysprops << "-Dtd.api.key=#{TreasureData::Config.apikey}"
+
+    sysprops
+  end
+
+  private
+  def set_sysprops_endpoint(sysprops)
+    endpoint = ENV['TD_API_SERVER']
+    if endpoint
+      require 'uri'
+
+      uri = URI.parse(endpoint)
+
+      case uri.scheme
+      when 'http', 'https'
+        host = uri.host
+        port = uri.port
+        ssl = uri.scheme == 'https'
+
+        port = 80 if port == 443 and ssl
+      else
+        if uri.port
+          # invalid URI
+          raise "Invalid endpoint: #{endpoint}"
         end
-      }
+
+        # generic URI
+        host, port = endpoint.split(':', 2)
+        port = port.to_i
+        # TODO support ssl
+        port = 80 if port == 0
+        ssl = false
+      end
+
+      sysprops << "-Dtd.api.server.host=#{host}"
+      sysprops << "-Dtd.api.server.port=#{port}"
     end
   end
 
-  class MessagePackParser
-    def initialize(time_key)
-      require 'msgpack'
-      @time_key = time_key
-    end
+  private
+  def set_sysprops_http_proxy(sysprops)
+    http_proxy = ENV['HTTP_PROXY']
+    if http_proxy
+      if http_proxy =~ /\Ahttp:\/\/(.*)\z/
+        http_proxy = $~[1]
+      end
+      proxy_host, proxy_port = http_proxy.split(':', 2)
+      proxy_port = (proxy_port ? proxy_port.to_i : 80)
 
-    def call(file, path, &block)
-      i = 0
-      MessagePack::Unpacker.new(file).each {|record|
-        i += 1
-        begin
-          unless record.is_a?(Hash)
-            raise "record must be a Hash"
-          end
-
-          time = record[@time_key]
-          unless time
-            raise "record doesn't have '#{@time_key}' column"
-          end
-
-          case time
-          when Integer
-            # do nothing
-          else
-            time = Time.parse(time.to_s).to_i
-          end
-          record['time'] = time
-
-          block.call(record)
-
-        rescue
-          $stderr.puts "  skipped: #{$!}: #{record.to_json}"
-        end
-      }
-    rescue EOFError
+      sysprops << "-Dhttp.proxyHost=#{proxy_host}"
+      sysprops << "-Dhttp.proxyPort=#{proxy_port}"
     end
   end
+
+  private
+  def find_logging_conf_file
+    libjars = Dir.glob("#{BASE_PATH}/java/**/*.properties")
+    found = libjars.find { |path| File.basename(path) =~ /^logging.properties/ }
+    return nil if found.nil?
+    logging_conf_file = libjars.delete(found)
+    logging_conf_file
+  end
+
 end
 end
-

data/lib/td/command/list.rb

@@ -228,6 +228,7 @@ module List
   add_list 'table:swap', %w[db table1 table2], 'Swap names of two tables', 'table:swap example_db table1 table2'
   add_list 'table:tail', %w[db table], 'Get recently imported logs', 'table:tail example_db table1', 'table:tail example_db table1 -t "2011-01-02 03:04:05" -n 30'
   add_list 'table:partial_delete', %w[db table], 'Delete logs from the table within the specified time range', 'table:partial_delete example_db table1 --from 1341000000 --to 1341003600'
+  add_list 'table:expire', %w[db table expire_days], 'Expire data in table after specified number of days', 'table:expire example_db table1 30'
 
   add_list 'bulk_import:list', %w[], 'List bulk import sessions', 'bulk_import:list'
   add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', 'bulk_import:show'
@@ -246,6 +247,18 @@ module List
   add_list 'bulk_import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', 'bulk_import:freeze logs_201201'
   add_list 'bulk_import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', 'bulk_import:unfreeze logs_201201'
 
+  add_list 'import:list', %w[], 'List bulk import sessions', 'import:list'
+  add_list 'import:show', %w[name], 'Show list of uploaded parts', 'import:show'
+  add_list 'import:create', %w[name db table], 'Create a new bulk import session to the the table', 'import:create logs_201201 example_db event_logs'
+  add_list 'import:prepare', %w[files_], 'Convert files into part file format', 'import:prepare logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
+  add_list 'import:upload', %w[name files_], 'Upload or re-upload files into a bulk import session', 'import:upload parts/* --parallel 4'
+  add_list 'import:perform', %w[name], 'Start to validate and convert uploaded files', 'import:perform logs_201201'
+  add_list 'import:error_records', %w[name], 'Show records which did not pass validations', 'import:error_records logs_201201'
+  add_list 'import:commit', %w[name], 'Start to commit a performed bulk import session', 'import:commit logs_201201'
+  add_list 'import:delete', %w[name], 'Delete a bulk import session', 'import:delete logs_201201'
+  add_list 'import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', 'import:freeze logs_201201'
+  add_list 'import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', 'import:unfreeze logs_201201'
+
   add_list 'result:list', %w[], 'Show list of result URLs', 'result:list', 'results'
   add_list 'result:show', %w[name], 'Describe information of a result URL', 'result mydb'
   add_list 'result:create', %w[name URL], 'Create a result URL', 'result:create mydb mysql://my-server/mydb'
@@ -413,7 +426,7 @@ module List
   add_guess 'show-job',         'job:show'
   add_guess 'show-jobs',        'job:list'
   add_guess 'server-status',    'server:status'
-  add_alias 'import',           'table:import'
+  add_alias 'import',           'import:upload'
 
   finishup
 end

data/lib/td/command/sched.rb

@@ -25,6 +25,8 @@ module Command
   end
 
   def sched_create(op)
+    require 'td/command/job'  # job_priority_id_of
+
     org = nil
     db_name = nil
     timezone = nil
@@ -131,6 +133,8 @@ module Command
   end
 
   def sched_update(op)
+    require 'td/command/job'  # job_priority_id_of
+
     cron = nil
     sql = nil
     db_name = nil

data/lib/td/command/table.rb

@@ -357,7 +357,338 @@ module Command
     end
   end
 
-  require 'td/command/import'  # table:import
+  def table_expire(op)
+    db_name, table_name, expire_days = op.cmd_parse
+
+    expire_days = expire_days.to_i
+    if expire_days <= 0
+      $stderr.puts "Table expiration days must be greater than 0."
+      return
+    end
+
+    client = get_client
+    client.update_expire(db_name, table_name, expire_days)
+
+    $stderr.puts "Table set to expire data older than #{expire_days} days."
+  end
+  
+
+  IMPORT_TEMPLATES = {
+    'apache' => [
+                  /^([^ ]*) [^ ]* ([^ ]*) \[([^\]]*)\] "(\S+)(?: +([^ ]*) +\S*)?" ([^ ]*) ([^ ]*)(?: "([^\"]*)" "([^\"]*)")?$/,
+                  ['host', 'user', 'time', 'method', 'path', 'code', 'size', 'referer', 'agent'],
+                  "%d/%b/%Y:%H:%M:%S %z"],
+    'syslog' => [
+                  /^([^ ]* [^ ]* [^ ]*) ([^ ]*) ([a-zA-Z0-9_\/\.\-]*)(?:\[([0-9]+)\])?[^\:]*\: *(.*)$/,
+                  ['time', 'host', 'ident', 'pid', 'message'],
+                  "%b %d %H:%M:%S"],
+  }
+
+  # TODO import-item
+  # TODO tail
+
+  def table_import(op)
+    op.banner << "\nsupported formats:\n"
+    op.banner << "  apache\n"
+    op.banner << "  syslog\n"
+    op.banner << "  msgpack\n"
+    op.banner << "  json\n"
+
+    format = 'apache'
+    time_key = 'time'
+    auto_create = false
+
+    op.on('--format FORMAT', "file format (default: #{format})") {|s|
+      format = s
+    }
+
+    op.on('--apache', "same as --format apache; apache common log format") {
+      format = 'apache'
+    }
+
+    op.on('--syslog', "same as --format syslog; syslog") {
+      format = 'syslog'
+    }
+
+    op.on('--msgpack', "same as --format msgpack; msgpack stream format") {
+      format = 'msgpack'
+    }
+
+    op.on('--json', "same as --format json; LF-separated json format") {
+      format = 'json'
+    }
+
+    op.on('-t', '--time-key COL_NAME', "time key name for json and msgpack format (e.g. 'created_at')") {|s|
+      time_key = s
+    }
+
+    op.on('--auto-create-table', "Create table and database if doesn't exist", TrueClass) { |b|
+      auto_create = b
+    }
+
+    db_name, table_name, *paths = op.cmd_parse
+
+    client = get_client
+
+    if auto_create
+      # Merge with db_create and table_create after refactoring
+      API.validate_database_name(db_name)
+      begin
+        client.create_database(db_name)
+        $stderr.puts "Database '#{db_name}' is created."
+      rescue AlreadyExistsError
+      end
+
+      API.validate_table_name(table_name)
+      begin
+        client.create_log_table(db_name, table_name)
+        $stderr.puts "Table '#{db_name}.#{table_name}' is created."
+      rescue AlreadyExistsError
+      end
+    end
+
+    case format
+    when 'json', 'msgpack'
+      #unless time_key
+      #  $stderr.puts "-t, --time-key COL_NAME (e.g. '-t created_at') parameter is required for #{format} format"
+      #  exit 1
+      #end
+      if format == 'json'
+        require 'json'
+        require 'time'
+        parser = JsonParser.new(time_key)
+      else
+        parser = MessagePackParser.new(time_key)
+      end
+
+    else
+      regexp, names, time_format = IMPORT_TEMPLATES[format]
+      if !regexp || !names || !time_format
+        $stderr.puts "Unknown format '#{format}'"
+        exit 1
+      end
+      parser = TextParser.new(names, regexp, time_format)
+    end
+
+    get_table(client, db_name, table_name)
+
+    require 'zlib'
+
+    files = paths.map {|path|
+      if path == '-'
+        $stdin
+      elsif path =~ /\.gz$/
+        require 'td/compat_gzip_reader'
+        Zlib::GzipReader.open(path)
+      else
+        File.open(path)
+      end
+    }
+
+    require 'msgpack'
+    require 'tempfile'
+    #require 'thread'
+
+    files.zip(paths).each {|file,path|
+      import_log_file(file, path, client, db_name, table_name, parser)
+    }
+
+    puts "done."
+  end
+
+  private
+  def import_log_file(file, path, client, db_name, table_name, parser)
+    puts "importing #{path}..."
+
+    out = Tempfile.new('td-import')
+    out.binmode if out.respond_to?(:binmode)
+
+    writer = Zlib::GzipWriter.new(out)
+
+    n = 0
+    x = 0
+    has_bignum = false
+    parser.call(file, path) {|record|
+      entry = begin
+                record.to_msgpack
+              rescue RangeError
+                has_bignum = true
+                TreasureData::API.normalized_msgpack(record)
+              end
+      writer.write entry
+
+      n += 1
+      x += 1
+      if n % 10000 == 0
+        puts "  imported #{n} entries from #{path}..."
+
+      elsif out.pos > 1024*1024  # TODO size
+        puts "  imported #{n} entries from #{path}..."
+        begin
+          writer.finish
+          size = out.pos
+          out.pos = 0
+
+          puts "  uploading #{size} bytes..."
+          client.import(db_name, table_name, "msgpack.gz", out, size)
+
+          out.truncate(0)
+          out.pos = 0
+          x = 0
+          writer = Zlib::GzipWriter.new(out)
+        rescue
+          $stderr.puts "  #{$!}"
+          return 1 # TODO error
+        end
+      end
+    }
+
+    if x != 0
+      writer.finish
+      size = out.pos
+      out.pos = 0
+
+      puts "  uploading #{size} bytes..."
+      # TODO upload on background thread
+      client.import(db_name, table_name, "msgpack.gz", out, size)
+    end
+
+    puts "  imported #{n} entries from #{path}."
+    $stderr.puts normalized_message if has_bignum
+  ensure
+    out.close rescue nil
+    writer.close rescue nil
+  end
+
+  require 'date'  # DateTime#strptime
+  require 'time'  # Time#strptime, Time#parse
+
+  class TextParser
+    def initialize(names, regexp, time_format)
+      @names = names
+      @regexp = regexp
+      @time_format = time_format
+    end
+
+    def call(file, path, &block)
+      i = 0
+      file.each_line {|line|
+        i += 1
+        begin
+          line.rstrip!
+          m = @regexp.match(line)
+          unless m
+            raise "invalid log format at #{path}:#{i}"
+          end
+
+          record = {}
+
+          cap = m.captures
+          @names.each_with_index {|name,cap_i|
+            if value = cap[cap_i]
+              if name == "time"
+                value = parse_time(value).to_i
+              end
+              record[name] = value
+            end
+          }
+
+          block.call(record)
+
+        rescue
+          $stderr.puts "  skipped: #{$!}: #{line.dump}"
+        end
+      }
+    end
+
+    if Time.respond_to?(:strptime)
+      def parse_time(value)
+        Time.strptime(value, @time_format)
+      end
+    else
+      def parse_time(value)
+        Time.parse(DateTime.strptime(value, @time_format).to_s)
+      end
+    end
+  end
+
+  class JsonParser
+    def initialize(time_key)
+      require 'json'
+      @time_key = time_key
+    end
+
+    def call(file, path, &block)
+      i = 0
+      file.each_line {|line|
+        i += 1
+        begin
+          record = JSON.parse(line)
+
+          unless record.is_a?(Hash)
+            raise "record must be a Hash"
+          end
+
+          time = record[@time_key]
+          unless time
+            raise "record doesn't have '#{@time_key}' column"
+          end
+
+          case time
+          when Integer
+            # do nothing
+          else
+            time = Time.parse(time.to_s).to_i
+          end
+          record['time'] = time
+
+          block.call(record)
+
+        rescue
+          $stderr.puts "  skipped: #{$!}: #{line.dump}"
+        end
+      }
+    end
+  end
+
+  class MessagePackParser
+    def initialize(time_key)
+      require 'msgpack'
+      @time_key = time_key
+    end
+
+    def call(file, path, &block)
+      i = 0
+      MessagePack::Unpacker.new(file).each {|record|
+        i += 1
+        begin
+          unless record.is_a?(Hash)
+            raise "record must be a Hash"
+          end
+
+          time = record[@time_key]
+          unless time
+            raise "record doesn't have '#{@time_key}' column"
+          end
+
+          case time
+          when Integer
+            # do nothing
+          else
+            time = Time.parse(time.to_s).to_i
+          end
+          record['time'] = time
+
+          block.call(record)
+
+        rescue
+          $stderr.puts "  skipped: #{$!}: #{record.to_json}"
+        end
+      }
+    rescue EOFError
+    end
+  end
+
   require 'td/command/export'  # table:export
   require 'td/command/job'  # wait_job
 end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.84'
+VERSION = '0.10.85'
 
 end

metadata

@@ -1,32 +1,34 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.84
+  version: 0.10.85
+  prerelease: 
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-25 00:00:00.000000000 Z
+date: 2013-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -35,20 +37,21 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - '!='
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -57,6 +60,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -64,6 +68,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -71,20 +76,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -92,6 +100,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -99,6 +108,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -106,6 +116,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -113,6 +124,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -120,6 +132,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -127,6 +140,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -134,6 +148,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -141,6 +156,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -148,6 +164,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -155,6 +172,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -162,6 +180,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -169,6 +188,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -176,6 +196,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -209,7 +230,8 @@ files:
 - dist/resources/pkg/postinstall
 - dist/resources/pkg/ruby-2.0.0-p0.pkg
 - dist/resources/pkg/td
-- java/td-bulk-import-0.2.0-SNAPSHOT.jar
+- java/logging.properties
+- java/td-bulk-import-0.2.1.jar
 - java/td-bulk-import-java.version
 - lib/td.rb
 - lib/td/command/account.rb
@@ -259,26 +281,33 @@ files:
 - td.gemspec
 homepage: http://treasure-data.com/
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -2194512597141251482
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -2194512597141251482
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:
 - spec/file_reader/filter_spec.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: c7c60a874b7228266a8436132aae4d0e2438b3e5
-  data.tar.gz: 9caacd3a0d3065d8fd4dca7bc7bae1cd29b9bbd5
-SHA512:
-  metadata.gz: baa729bcbfb73760f2c93cc70e6fefccc69b1cae9a41c62e7c869372bf4124975649e0abffb5db15b79156bfb59e59ab52875634bf179da86fab129979c7cdab
-  data.tar.gz: 10328c283f1b8a9a3e4d2b9ad23187da08b0b0391ae237fdd5fbe8ad4c610784f0d8a2f2ca8d00c758f5afac757426965545e5811b406df943769a826b9a27d7