td 0.10.71 → 0.10.72

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 756988829cba729420d255670f1b7db27c7f3bc3
+  data.tar.gz: ccf55b0be4c0ee2fd934ad822034336ad33a80da
+SHA512:
+  metadata.gz: 87c00a05581a54d35b4db41969a74156e5c522dbb3285dc9c615a4619e33dc6efd715f8c58b72ed82d19a841d80967be641933b93b0ab12c7af0292ca51b5faa
+  data.tar.gz: 735d45002439d770e565521033c9f470e846b93a4a3d904048c50fdb12f7b93c0c792787ea535aae923c1bc28adc2571b7672a0d3c36003dd23e332b4314e525

data/ChangeLog

@@ -1,3 +1,9 @@
+== 2013-02-27 version 0.10.72
+
+* Add -q(--query) option to query subcommand
+* Hide password input on Windows
+* 'td help sample' is now available
+
 == 2013-02-05 version 0.10.71
 
 * bulk_import:upload_parts: improved retry algorithm to use exponential wait time

data/Rakefile

@@ -14,7 +14,7 @@ def version
 end
 
 task "jar" do
-  system('./build/update-td.sh')
+  system('./build/update-td-bulk-import-java.sh')
 end
 
 def project_root_path(path)

data/build/update-td-bulk-import-java.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+cd "$(dirname $0)"
+chrev="$1"
+
+if [ -d td-bulk-import-java/.git ];then
+    rm -rf td-bulk-import-java/*
+    cd td-bulk-import-java
+    git checkout . || exit 1
+    git pull || exit 1
+else
+    rm -rf td-bulk-import-java/
+    git clone git@github.com:treasure-data/td-bulk-import-java.git td-bulk-import-java || exit 1
+    cd td-bulk-import-java
+fi
+git checkout master
+
+if [ -n "$chrev" ];then
+    git checkout $chrev
+fi
+
+revname="$(git show --pretty=format:'%H %ad' | head -n 1)"
+vername="0.1.2-SNAPSHOT"
+
+mvn package -Dmaven.test.skip=true || exit 1
+cp target/td-bulk-import-${vername}.jar ../../java/td-bulk-import-${vername}.jar
+
+if [ -n "$chrev" ];then
+    git checkout master
+fi
+
+cd ../../
+
+echo "$revname" > java/td-bulk-import-java.version
+
+echo ""
+echo "git commit ./java -m \"updated td-bulk-import-java $revname\""
+git commit ./java/td-bulk-import-${vername}.jar ./java/td-bulk-import-java.version -m "updated td-bulk-import-java $revname" || exit 1
+

data/java/td-bulk-import-java.version

@@ -0,0 +1 @@
+95b0662ffcf4f2407c6044478abd288a98be448d Sat Feb 23 22:43:05 2013 +0900

data/lib/td/command/account.rb

@@ -1,3 +1,4 @@
+require 'td/helpers'
 
 module TreasureData
 module Command
@@ -50,10 +51,8 @@ module Command
 
     3.times do
       begin
-        system "stty -echo"  # TODO termios
         print "Password (typing will be hidden): "
-        password = STDIN.gets || ""
-        password = password[0..-2]  # strip \n
+        password = get_password
       rescue Interrupt
         $stderr.print "\ncanceled."
         exit 1
@@ -98,6 +97,38 @@ module Command
     $stderr.puts "Storage:  #{a.storage_size_string}"
   end
 
+  private
+
+  if Helpers.on_windows?
+    require 'Win32API'
+
+    def get_char
+      Win32API.new('msvcrt', '_getch', [], 'L').Call
+    rescue Exception
+      Win32API.new('crtdll', '_getch', [], 'L').Call
+    end
+
+    def get_password
+      password = ''
+
+      while c = get_char
+        break if c == 13 || c == 10 # CR or NL
+        if c == 127 || c == 8  # 128: backspace, 8: delete
+          password.slice!(-1, 1)
+        else
+          password << c.chr
+        end
+      end
+
+      password
+    end
+  else
+    def get_password
+      system "stty -echo"  # TODO termios
+      password = STDIN.gets || ""
+      password[0..-2]  # strip \n
+    end
+  end
 end
 end
 

data/lib/td/command/bulk_import.rb

@@ -408,7 +408,105 @@ module Command
     }
   end
 
+  def bulk_import_upload_parts2(op)
+    upload_opts = upload_parts2_config(op)
+    prepare_opts = prepare_parts2_config(op)
+
+    # java command
+    javacmd = 'java'
+
+    # make jvm options
+    jvm_opts = []
+    jvm_opts << "-Xmx1024m" # TODO
+
+    # find java/*.jar and td.jar
+    base_path = File.expand_path('../../..', File.dirname(__FILE__)) # TODO
+    libjars = Dir.glob("#{base_path}/java/**/*.jar")
+    found = libjars.find { |path| File.basename(path) =~ /^td-bulk-import/ }
+    td_command_jar = libjars.delete(found)
+
+    # make application options
+    app_opts = []
+    app_opts << "-cp \"#{td_command_jar}\""
+
+    # make system properties
+    sysprops = []
+    sysprops.concat(upload_parts2_sysprops(opts))
+    sysprops.concat(prepare_parts2_sysprops(opts))
+
+    # make application arguments
+    app_args << 'com.treasure_data.tools.BulkImportTool'
+    app_args << 'upload_parts'
+    app_args << files
+
+    command = "#{javacmd} #{jvm_opts.join(' ')} #{app_opts.join(' ')} #{sysprops.join(' ')} #{app_args.join(' ')}"
+
+    exec command
+  end
+
   def bulk_import_prepare_parts2(op)
+    opts = prepare_parts2_config(op)
+
+    # java command
+    javacmd = 'java'
+
+    # make jvm options
+    jvm_opts = []
+    jvm_opts << "-Xmx1024m" # TODO
+
+    # find java/*.jar and td.jar
+    base_path = File.expand_path('../../..', File.dirname(__FILE__)) # TODO
+    libjars = Dir.glob("#{base_path}/java/**/*.jar")
+    found = libjars.find { |path| File.basename(path) =~ /^td-bulk-import/ }
+    td_command_jar = libjars.delete(found)
+
+    # make application options
+    app_opts = []
+    app_opts << "-cp \"#{td_command_jar}\""
+
+    # make system properties
+    sysprops = []
+    sysprops.concat(prepare_parts2_sysprops(opts))
+
+    # make application arguments
+    app_args = []
+    app_args << 'com.treasure_data.tools.BulkImportTool'
+    app_args << 'prepare_parts'
+    app_args << files
+
+    # TODO consider parameters including spaces; don't use join(' ')
+    command = "#{javacmd} #{jvm_opts.join(' ')} #{app_opts.join(' ')} #{sysprops.join(' ')} #{app_args.join(' ')}"
+
+    exec command
+  end
+
+  private
+  def prepare_parts2_sysprops(opts)
+    sysprops = []
+    sysprops << "-Dtd.bulk_import.prepare_parts.format=#{opts[0]}"
+    sysprops << "-Dtd.bulk_import.prepare_parts.columns=#{opts[1]}" if opts[1]
+    sysprops << "-Dtd.bulk_import.prepare_parts.column-types=#{opts[2]}" if opts[2]
+    sysprops << "-Dtd.bulk_import.prepare_parts.column-header=#{opts[3]}" if opts[3]
+    sysprops << "-Dtd.bulk_import.prepare_parts.time-column=#{opts[4]}"
+    sysprops << "-Dtd.bulk_import.prepare_parts.time-value=#{opts[5].to_s}" if opts[5]
+    sysprops << "-Dtd.bulk_import.prepare_parts.split-size=#{opts[6]}"
+    sysprops << "-Dtd.bulk_import.prepare_parts.output-dir=#{opts[7]}"
+    sysprops
+  end
+
+  private
+  def upload_parts2_sysprops(opts)
+    sysprops = []
+    sysprops << "td.bulk_import.upload_parts.parallel=#{opts[0]}"
+    sysprops << "td.bulk_import.upload_parts.auto-perform=#{opts[1]}"
+    sysprops << "td.bulk_import.upload_parts.auto-commit=#{opts[2]}"
+    sysprops << "td.bulk_import.upload_parts.retrycount=10"
+    sysprops << "td.bulk_import.upload_parts.waitsec=1"
+    sysprops
+  end
+
+  private
+  def prepare_parts2_config(op)
     format = 'csv'
     columns = nil
     column_types = nil
@@ -461,44 +559,26 @@ module Command
       exit 1
     end
 
-    # java command
-    javacmd = 'java'
-
-    # make jvm options
-    jvm_opts = []
-    jvm_opts << "-Xmx1024m" # TODO
-
-    # find java/*.jar and td.jar
-    base_path = File.expand_path('../../..', File.dirname(__FILE__)) # TODO
-    libjars = Dir.glob("#{base_path}/java/**/*.jar")
-    found = libjars.find { |path| File.basename(path) =~ /^td-/ }
-    td_command_jar = libjars.delete(found)
-
-    # make application options
-    app_opts = []
-    app_opts << "-cp \"#{td_command_jar}\""
-
-    # make system properties
-    sysprops = []
-    sysprops << "-Dtd.bulk_import.prepare_parts.format=#{format}"
-    sysprops << "-Dtd.bulk_import.prepare_parts.columns=#{columns}" if columns
-    sysprops << "-Dtd.bulk_import.prepare_parts.column-types=#{column_types}" if column_types
-    sysprops << "-Dtd.bulk_import.prepare_parts.column-header=#{has_header}" if has_header
-    sysprops << "-Dtd.bulk_import.prepare_parts.time-column=#{time_column}"
-    sysprops << "-Dtd.bulk_import.prepare_parts.time-value=#{time_value.to_s}" if time_value
-    sysprops << "-Dtd.bulk_import.prepare_parts.split-size=#{split_size_kb}"
-    sysprops << "-Dtd.bulk_import.prepare_parts.output-dir=#{outdir}"
+    return [ format, columns, column_types, has_header, time_column, time_value, split_size_kb, outdir ]
+  end
 
-    # make application arguments
-    app_args = []
-    app_args << 'com.treasure_data.tools.BulkImportTool'
-    app_args << 'prepare_parts'
-    app_args << files
+  private
+  def upload_parts2_config(op)
+    auto_perform = true
+    auto_commit = false
+    paraallel = 2
 
-    # TODO consider parameters including spaces; don't use join(' ')
-    command = "#{javacmd} #{jvm_opts.join(' ')} #{app_opts.join(' ')} #{sysprops.join(' ')} #{app_args.join(' ')}"
+    op.on('--auto-perform', 'perform bulk import job automatically', TrueClass) {|b|
+      auto_perform = b
+    }
+    op.on('--auto-commit', 'perform bulk import job automatically', FalseClass) {|b|
+      auto_perform = b
+    }
+    op.on('--parallel NUM', 'perform uploading in parallel (default: 2; max 8)', Integer) {|i|
+      parallel = i
+    }
 
-    exec command
+    return [ auto_perform, auto_commit, parallel ]
   end
 
   private

data/lib/td/command/list.rb

@@ -236,6 +236,7 @@ module List
   add_list 'bulk_import:prepare_parts2', %w[files_], 'Convert files into part file format', 'bulk_import:prepare_parts2 logs/*.csv --format csv --columns time,uid,price,count --column-types long,string,long,int --time-column "time" -o parts/'
   add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', 'bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz'
   add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', 'bulk_import:upload_parts parts/* --parallel 4'
+  add_list 'bulk_import:upload_parts2', %w[name files_], 'Upload or re-upload files into a bulk import session', 'bulk_import:upload_parts parts/* --parallel 4'
   add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', 'bulk_import:delete_part logs_201201 01h'
   add_list 'bulk_import:delete_parts', %w[name ids_], 'Delete uploaded files from a bulk import session', 'bulk_import:delete_parts logs_201201 01h 02h 03h'
   add_list 'bulk_import:perform', %w[name], 'Start to validate and convert uploaded files', 'bulk_import:perform logs_201201'
@@ -264,7 +265,8 @@ module List
   add_list 'sched:history', %w[name max?], 'Show history of scheduled queries', 'sched sched1 --page 1'
   add_list 'sched:run', %w[name time], 'Run scheduled queries for the specified time', 'sched:run sched1 "2013-01-01 00:00:00" -n 6'
 
-  add_list 'query', %w[sql], 'Issue a query', 'query -d example_db -w -r rset1 "select count(*) from table1"'
+  add_list 'query', %w[sql?], 'Issue a query', 'query -d example_db -w -r rset1 "select count(*) from table1"',
+                                               'query -d example_db -w -r rset1 -q query.txt'
 
   add_list 'job:show', %w[job_id], 'Show status and result of a job', 'job 1461'
   add_list 'job:status', %w[job_id], 'Show status progress of a job', 'job:status 1461'
@@ -375,6 +377,7 @@ module List
 
   add_alias 'apikey', 'apikey:show'
   add_alias 'server', 'server:status'
+  add_alias 'sample', 'sample:apache'
 
   add_alias 's', 'status'
 

data/lib/td/command/query.rb

@@ -14,6 +14,7 @@ module Command
     result_ask_password = false
     priority = nil
     retry_limit = nil
+    query = nil
 
     op.on('-g', '--org ORGANIZATION', "issue the query under this organization") {|s|
       org = s
@@ -54,6 +55,9 @@ module Command
     op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
       retry_limit = i
     }
+    op.on('-q', '--query PATH', 'use file instead of inline query') {|s|
+      query = File.open(s) { |f| f.read.strip }
+    }
 
     sql = op.cmd_parse
 
@@ -64,6 +68,13 @@ module Command
 
     if sql == '-'
       sql = STDIN.read
+    elsif sql.nil?
+      sql = query
+    end
+
+    unless sql
+      $stderr.puts "<sql> argument or -q,--query PATH option is required."
+      exit 1
     end
 
     if result_url

data/lib/td/command/runner.rb

@@ -51,6 +51,7 @@ Additional commands:
   status         # show scheds, jobs, tables and results
   apikey         # show/set API key
   server         # show status of the Treasure Data server
+  sample         # create a sample log file
   help           # show help messages
 
 Type 'td help COMMAND' for more information on a specific command.

data/lib/td/helpers.rb

@@ -0,0 +1,9 @@
+module TreasureData
+  module Helpers
+    module_function
+
+    def on_windows?
+      RUBY_PLATFORM =~ /mswin32|mingw32/
+    end
+  end
+end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.71'
+VERSION = '0.10.72'
 
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.71
-  prerelease: 
+  version: 0.10.72
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-05 00:00:00.000000000 Z
+date: 2013-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,23 +41,20 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -110,7 +97,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -118,7 +104,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -126,7 +111,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -134,7 +118,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -142,7 +125,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -150,7 +132,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -168,7 +149,7 @@ files:
 - README.rdoc
 - Rakefile
 - bin/td
-- build/update-td.sh
+- build/update-td-bulk-import-java.sh
 - data/sample_apache.json
 - data/sample_apache_gen.rb
 - dist/exe.rake
@@ -181,8 +162,8 @@ files:
 - dist/resources/pkg/PackageInfo.erb
 - dist/resources/pkg/postinstall
 - dist/resources/pkg/td
-- java/td-0.1.1.jar
-- java/td_java.version
+- java/td-bulk-import-0.1.2-SNAPSHOT.jar
+- java/td-bulk-import-java.version
 - lib/td.rb
 - lib/td/command/account.rb
 - lib/td/command/acl.rb
@@ -214,6 +195,7 @@ files:
 - lib/td/config.rb
 - lib/td/distribution.rb
 - lib/td/file_reader.rb
+- lib/td/helpers.rb
 - lib/td/version.rb
 - spec/file_reader/filter_spec.rb
 - spec/file_reader/io_filter_spec.rb
@@ -226,27 +208,26 @@ files:
 - td.gemspec
 homepage: http://treasure-data.com/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:
 - spec/file_reader/filter_spec.rb

data/build/update-td.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-cd "$(dirname $0)"
-chrev="$1"
-
-if [ -d td-java/.git ];then
-    rm -rf td-java/*
-    cd td-java
-    git checkout . || exit 1
-    git pull || exit 1
-else
-    rm -rf td-java/
-    git clone git@github.com:treasure-data/td-java.git td-java || exit 1
-    cd td-java
-fi
-git checkout master
-
-if [ -n "$chrev" ];then
-    git checkout $chrev
-fi
-
-revname="$(git show --pretty=format:'%H %ad' | head -n 1)"
-
-mvn package -Dmaven.test.skip=true || exit 1
-cp target/td-0.1.1.jar ../../java/td-0.1.1.jar
-
-if [ -n "$chrev" ];then
-    git checkout master
-fi
-
-cd ../../
-
-echo "$revname" > java/td_java.version
-
-echo ""
-echo "git commit ./java -m \"updated td-java $revname\""
-git commit ./java/td-0.1.1.jar ./java/td_java.version -m "updated td-java $revname" || exit 1
-

data/java/td_java.version

@@ -1 +0,0 @@
-19238aa12ef2b06cc07f4dc5cd008d35da1aa358 Tue Jan 22 21:46:12 2013 +0900