td 0.10.66 → 0.10.67

data/.gitignore

@@ -0,0 +1,5 @@
+.bundle
+build/td-java
+Gemfile.lock
+vendor/*
+*~

data/ChangeLog

@@ -1,4 +1,9 @@
 
+== 2013-01-18 version 0.10.67
+
+* Added bulk_import:prepare_parts2 subcommand which is faster and more reliable
+
+
 == 2013-01-16 version 0.10.66
 
 * td-client v0.8.42

data/Rakefile

@@ -13,6 +13,10 @@ def version
   TreasureData::VERSION
 end
 
+task "jar" do
+  system('./build/update-td.sh')
+end
+
 def project_root_path(path)
   "#{PROJECT_ROOT}/#{path}"
 end

data/build/update-td.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+cd "$(dirname $0)"
+chrev="$1"
+
+if [ -d td-java/.git ];then
+    rm -rf td-java/*
+    cd td-java
+    git checkout . || exit 1
+    git pull || exit 1
+else
+    rm -rf td-java/
+    git clone git@github.com:treasure-data/td-java.git td-java || exit 1
+    cd td-java
+fi
+git checkout master
+
+if [ -n "$chrev" ];then
+    git checkout $chrev
+fi
+
+revname="$(git show --pretty=format:'%H %ad' | head -n 1)"
+
+mvn package -Dmaven.test.skip=true || exit 1
+cp target/td-0.1.1-SNAPSHOT.jar ../../java/td-0.1.1-SNAPSHOT.jar
+
+if [ -n "$chrev" ];then
+    git checkout master
+fi
+
+cd ../../
+
+echo "$revname" > java/td_java.version
+
+echo ""
+echo "git commit ./java -m \"updated td-java $revname\""
+git commit ./java/td-0.1.1-SNAPSHOT.jar ./java/td_java.version -m "updated td-java $revname" || exit 1
+

data/java/td_java.version

@@ -0,0 +1 @@
+a0bb26a737329cc3df9987c0f1a6ea7188d3b334 Fri Jan 18 15:03:20 2013 +0900

data/lib/td/command/bulk_import.rb

@@ -392,6 +392,102 @@ module Command
     }
   end
 
+  def bulk_import_prepare_parts2(op)
+    format = 'csv'
+    columns = nil
+    column_types = nil
+    has_header = nil
+    time_column = 'time'
+    time_value = nil
+    split_size_kb = PART_SPLIT_SIZE / 1024  # kb
+    outdir = nil
+
+    op.on('-f', '--format NAME', 'source file format [csv]') {|s|
+      format = s
+    }
+    op.on('-h', '--columns NAME,NAME,...', 'column names (use --column-header instead if the first line has column names)') {|s|
+      columns = s
+    }
+    op.on('--column-types TYPE,TYPE,...', 'column types [string, int, long]') {|s|
+      column_types = s
+    }
+    op.on('-H', '--column-header', 'first line includes column names', TrueClass) {|b|
+      has_header = b
+    }
+    op.on('-t', '--time-column NAME', 'name of the time column') {|s|
+      time_column = s
+    }
+    op.on('--time-value TIME', 'long value of the time column') {|s|
+      if s.to_i.to_s == s
+        time_value = s.to_i
+      else
+        require 'time'
+        time_value = Time.parse(s).to_i
+      end
+    }
+    op.on('-s', '--split-size SIZE_IN_KB', "size of each parts (default: #{split_size_kb})", Integer) {|i|
+      split_size_kb = i
+    }
+    op.on('-o', '--output DIR', 'output directory') {|s|
+      outdir = s
+    }
+
+    files = op.cmd_parse
+    files = [files] unless files.is_a?(Array) # TODO ruby 1.9
+
+    # options validation
+    unless column_types
+      $stderr.puts "--column-types TYPE,TYPE,... option is required."
+      exit 1
+    end
+    unless outdir
+      $stderr.puts "-o, --output DIR option is required."
+      exit 1
+    end
+
+    # java command
+    javacmd = 'java'
+
+    # make jvm options
+    jvm_opts = []
+    jvm_opts << "-Xmx1024m" # TODO
+
+    # find java/*.jar and td.jar
+    base_path = File.expand_path('../../..', File.dirname(__FILE__)) # TODO
+    libjars = Dir.glob("#{base_path}/java/**/*.jar")
+    found = libjars.find { |path| File.basename(path) =~ /^td-/ }
+    td_command_jar = libjars.delete(found)
+
+    # make application options
+    app_opts = []
+    app_opts << "-cp .:#{td_command_jar}"
+
+    # make system properties
+    sysprops = []
+    sysprops << "-Dtd.bulk_import.prepare_parts.format=#{format}"
+    sysprops << "-Dtd.bulk_import.prepare_parts.columns=#{columns}" if columns
+    sysprops << "-Dtd.bulk_import.prepare_parts.column-types=#{column_types}" if column_types
+    sysprops << "-Dtd.bulk_import.prepare_parts.column-header=#{has_header}" if has_header
+    sysprops << "-Dtd.bulk_import.prepare_parts.time-column=#{time_column}"
+    sysprops << "-Dtd.bulk_import.prepare_parts.time-value=#{time_value.to_s}" if time_value
+    sysprops << "-Dtd.bulk_import.prepare_parts.split-size=#{split_size_kb}"
+    sysprops << "-Dtd.bulk_import.prepare_parts.output-dir=#{outdir}"
+
+    # make application arguments
+    app_args = []
+    app_args << 'com.treasure_data.tools.BulkImportTool'
+    app_args << 'prepare_parts'
+    app_args << files
+
+    command = "#{javacmd} #{jvm_opts.join(' ')} #{app_opts.join(' ')} #{sysprops.join(' ')} #{app_args.join(' ')}"
+
+    begin
+      exec(command)
+    rescue
+      exit 1
+    end
+  end
+
   private
   def bulk_import_upload_impl(name, part_name, io, size, retry_limit, retry_wait)
     begin

data/lib/td/command/list.rb

@@ -233,6 +233,7 @@ module List
   add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', 'bulk_import:show'
   add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', 'bulk_import:create logs_201201 example_db event_logs'
   add_list 'bulk_import:prepare_parts', %w[files_], 'Convert files into part file format', 'bulk_import:prepare_parts logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
+  add_list 'bulk_import:prepare_parts2', %w[files_], 'Convert files into part file format', 'bulk_import:prepare_parts2 logs/*.csv --format csv --columns time,uid,price,count --column-types long,string,long,int --time-column "time" -o parts/'
   add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', 'bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz'
   add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', 'bulk_import:upload_parts parts/* --parallel 4'
   add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', 'bulk_import:delete_part logs_201201 01h'

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.66'
+VERSION = '0.10.67'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.66
+  version: 0.10.67
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-16 00:00:00.000000000 Z
+date: 2013-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -162,11 +162,13 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
 - ChangeLog
 - Gemfile
 - README.rdoc
 - Rakefile
 - bin/td
+- build/update-td.sh
 - data/sample_apache.json
 - data/sample_apache_gen.rb
 - dist/exe.rake
@@ -179,6 +181,8 @@ files:
 - dist/resources/pkg/PackageInfo.erb
 - dist/resources/pkg/postinstall
 - dist/resources/pkg/td
+- java/td-0.1.1-SNAPSHOT.jar
+- java/td_java.version
 - lib/td.rb
 - lib/td/command/account.rb
 - lib/td/command/acl.rb