td 0.11.6 → 0.11.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fbc07ea58b09419ef7a3ba5dd850af222c8af299
-  data.tar.gz: db4fd8cfdece04ac1c94b3b68d8a642488c07be5
+  metadata.gz: e4b7f5341cbcebd3487c7df38951a4275204507e
+  data.tar.gz: 9aad2c3285c7beb623a97410b62844ff41f74574
 SHA512:
-  metadata.gz: cebcfe70d4474b64ae1b440981f9aee6c200f3b6b3e4b09c2dc1137faa7a2495e6f9e86ef8527181613c9d41af569d77ebf9091327d3e088e3e79bcfb95e7107
-  data.tar.gz: a261af1042634927563564043865d7e41e232c1bf048af3759025c5a46039fd64391b3d54900beb427371b93c3e6f19349194aa4246a14cc93ba3b17df1da2e0
+  metadata.gz: 24846c57a7df595675ca6999412b0c542ac7beb1850adda09de6f9832152954d33ff2be5335b293213f92e492e558ab711f132598587433b0249a4f803eedebc
+  data.tar.gz: 65eed33f8222fe6c3fb57067087d1cc477fb3a9bc64d9d427b0375964281c32726b11f8b65a0258ab77ce914a7530a0aadfaa92adb0d40ae243b5a63369294d9

data/ChangeLog

@@ -1,3 +1,23 @@
+== 2015-02-16 version 0.11.7
+
+* Fix server:endpoint command not working
+* Experimental new feature: timeout for import subcommands.  To prevent Java
+  process stall this option allows user to specify maximum execution seconds
+  for import subcommands by --bulk-import-timeout option. When execution
+  exceeds specified seconds the command fails after sending SIGQUIT to
+  internal java process if it's not running on Windows.
+  NOTE1: the execution time is measured against all subcommand execution. If
+  the command parameter designates perform and commit execution the whole
+  time could be longer than just uploading part files.
+  NOTE2: this option is not shown in help at this moment.
+* Show full query text for CSV, TSV, and JSON output format (It's now only
+  truncated in table format)
+* msgpack.gz to CSV/TSV perf optimization. It should be faster around 2 times
+  than before.
+* Allow to disable table data expiration by setting the period to 0.
+* Fix sched:history crash when scheduled_at == ""
+* Several spec fixes. Thanks @yakitorii!
+
 == 2014-12-03 version 0.11.6
 
 * Support HTTP proxy for bulk import

data/lib/td/command/import.rb

@@ -101,6 +101,7 @@ module Command
     show_help = ARGV.size == 0 || (ARGV.size == 1 || ARGV[0] =~ /^import:/)
 
     # configure java command-line arguments
+    timeout = nil
     java_args = []
     java_args.concat build_sysprops
     java_args.concat ["-cp", find_td_import_jar]
@@ -109,18 +110,70 @@ module Command
     if show_help
       java_args << "--help"
     else
+      0.upto(ARGV.length - 1) do |idx|
+        if ARGV[idx] == '--bulk-import-timeout'
+          timeout = ARGV[idx + 1]
+          if timeout.nil?
+            raise ArgumentError, 'timeout not given'
+          end
+          timeout = Integer(timeout)
+          ARGV.slice!(idx, 2)
+        end
+      end
       java_args.concat ARGV
     end
-
     cmd = [JAVA_COMMAND] + JVM_OPTS + java_args
-    system(*cmd)
-    if $?.exitstatus != 0
-      raise BulkImportExecutionError,
-            "Bulk Import returned error #{$?.exitstatus}. Please check the 'td-bulk-import.log' logfile for details."
-    end
+
+    CommandExecutor.new(cmd, timeout).execute
   end
 
+  class CommandExecutor
+    def initialize(cmd, timeout)
+      @cmd, @timeout = cmd, timeout
+    end
+
+    def execute
+      status = execute_command
+      if status.exitstatus != 0
+        raise BulkImportExecutionError,
+              "Bulk Import returned error #{status.exitstatus}. Please check the 'td-bulk-import.log' logfile for details."
+      end
+      status
+    end
+
   private
+
+    def execute_command
+      if @timeout
+        require 'timeout'
+        pid = nil
+        begin
+          Timeout.timeout(@timeout) do
+            pid = Process.spawn(*@cmd)
+            Process.waitpid(pid)
+            return $?
+          end
+        rescue Timeout::Error
+          if pid
+            require 'rbconfig'
+            # win32 ruby does not support QUIT and TERM
+            if RbConfig::CONFIG['host_os'] !~ /mswin|mingw|cygwin/
+              Process.kill('QUIT', pid)
+              Process.kill('TERM', pid)
+            else
+              # just kill without thread dump on win32 platforms
+              Process.kill('KILL', pid)
+            end
+          end
+          raise BulkImportExecutionError, "Bulk Import execution timed out: #{@timeout} [sec]"
+        end
+      else
+        system(*@cmd)
+        return $?
+      end
+    end
+  end
+
   def check_java
     if RbConfig::CONFIG["target_os"].downcase =~ /mswin(?!ce)|mingw|cygwin|bccwin/ # windows
       cmd = "#{JAVA_COMMAND} -version > NUL 2>&1"

data/lib/td/command/job.rb

@@ -75,12 +75,13 @@ module Command
       elapsed = Command.humanize_elapsed_time(start, job.end_at)
       cpu_time = Command.humanize_time(job.cpu_time, true)
       priority = job_priority_name_of(job.priority)
+      query = (op.render_format == 'table' || op.render_format.nil? ? job.query.to_s[0,50] + " ..." : job.query)
       rows << {
         :JobID => job.job_id,
         :Database => job.db_name,
         :Status => job.status,
         :Type => job.type,
-        :Query => job.query.to_s[0,50] + " ...",
+        :Query => query,
         :Start => (start ? start.localtime : ''),
         :Elapsed => elapsed.rjust(11),
         :CPUTime => cpu_time.rjust(17),
@@ -384,13 +385,15 @@ module Command
             job.result_size, 0.1, 1)
         end
         job.result_each_with_compr_size {|row, compr_size|
-          indicator.update(compr_size) unless output.nil?
           # TODO limit the # of columns
           writer << row.map {|col|
             dump_column(col)
           }
           n_rows += 1
-          writer.flush if n_rows % 100 == 0 # flush every 100 recods
+          if n_rows % 100 == 0 # flush every 100 recods
+            writer.flush
+            indicator.update(compr_size) unless output.nil?
+          end
           break if output.nil? and !limit.nil? and n_rows == limit
         }
         indicator.finish unless output.nil?
@@ -415,17 +418,12 @@ module Command
             job.result_size, 0.1, 1)
         end
         job.result_each_with_compr_size {|row, compr_size|
-          indicator.update(compr_size) unless output.nil?
-          n_cols = 0
-          row.each {|col|
-            f.write "\t" if n_cols > 0
-            # TODO limit the # of columns
-            f.write dump_column(col)
-            n_cols += 1
-          }
-          f.write "\n"
+          f.write row.map {|col| dump_column(col)}.join("\t") + "\n"
           n_rows += 1
-          f.flush if n_rows % 100 == 0 # flush every 100 recods
+          if n_rows % 100 == 0
+            f.flush # flush every 100 recods
+            indicator.update(compr_size) unless output.nil?
+          end
           break if output.nil? and !limit.nil? and n_rows == limit
         }
         indicator.finish unless output.nil?
@@ -495,7 +493,7 @@ module Command
       job.result_each_with_compr_size {|row, compr_size|
         indicator.update(compr_size)
         rows << row.map {|v|
-          dump_column(v)
+          dump_column_safe_utf8(v)
         }
         n_rows += 1
         break if !limit.nil? and n_rows == limit
@@ -517,9 +515,11 @@ module Command
   end
 
   def dump_column(v)
-    require 'yajl'
+    v.is_a?(String) ? v.to_s : Yajl.dump(v)
+  end
 
-    s = v.is_a?(String) ? v.to_s : Yajl.dump(v)
+  def dump_column_safe_utf8(v)
+    s = dump_column(v)
     # Here does UTF-8 -> UTF-16LE -> UTF8 conversion:
     #   a) to make sure the string doesn't include invalid byte sequence
     #   b) to display multi-byte characters as it is

data/lib/td/command/list.rb

@@ -230,7 +230,7 @@ module List
   add_list 'table:swap', %w[db table1 table2], 'Swap names of two tables', ['table:swap example_db table1 table2']
   add_list 'table:tail', %w[db table], 'Get recently imported logs', ['table:tail example_db table1', 'table:tail example_db table1 -t "2011-01-02 03:04:05" -n 30']
   add_list 'table:partial_delete', %w[db table], 'Delete logs from the table within the specified time range', ['table:partial_delete example_db table1 --from 1341000000 --to 1341003600']
-  add_list 'table:expire', %w[db table expire_days], 'Expire data in table after specified number of days', ['table:expire example_db table1 30']
+  add_list 'table:expire', %w[db table expire_days], 'Expire data in table after specified number of days. Set to 0 to disable the expiration.', ['table:expire example_db table1 30']
 
   add_list 'bulk_import:list', %w[], 'List bulk import sessions', ['bulk_import:list']
   add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', ['bulk_import:show']

data/lib/td/command/sched.rb

@@ -273,7 +273,8 @@ module Command
 
     rows = []
     history.each {|j|
-      rows << {:Time => j.scheduled_at.localtime, :JobID => j.job_id, :Status => j.status, :Priority => job_priority_name_of(j.priority), :Result=>j.result_url}
+      scheduled_at = j.scheduled_at ? j.scheduled_at.localtime : nil
+      rows << {:Time => scheduled_at, :JobID => j.job_id, :Status => j.status, :Priority => job_priority_name_of(j.priority), :Result=>j.result_url}
     }
 
     puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Priority, :Result], :render_format => op.render_format)

data/lib/td/command/server.rb

@@ -20,7 +20,7 @@ module Command
     end
 
     Command.validate_api_endpoint(endpoint)
-    Coomand.test_api_endpoint(endpoint)
+    Command.test_api_endpoint(endpoint)
 
     conf = nil
     begin

data/lib/td/command/table.rb

@@ -412,15 +412,19 @@ module Command
     db_name, table_name, expire_days = op.cmd_parse
 
     expire_days = expire_days.to_i
-    if expire_days <= 0
-      $stderr.puts "Table expiration days must be greater than 0."
+    if expire_days < 0
+      $stderr.puts "Table expiration days must be greater or equal to 0."
       return
     end
 
     client = get_client
     client.update_expire(db_name, table_name, expire_days)
 
-    $stderr.puts "Table set to expire data older than #{expire_days} days."
+    if expire_days == 0
+      puts "Data expiration disabled for this table."
+    else
+      puts "Table set to expire data older than #{expire_days} days."
+    end
   end
 
 

data/lib/td/file_reader.rb

@@ -43,7 +43,9 @@ module TreasureData
       end
 
       def forward
-        @u.each {|r| break r }
+        content = @u.each {|r| break r }
+        raise EOFError unless content
+        content
       end
     end
 

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.11.6'
+  TOOLBELT_VERSION = '0.11.7'
 end

data/spec/file_reader/line_reader_spec.rb

@@ -33,12 +33,21 @@ describe FileReader::LineReader do
 
   it 'initialize with specifid encoding' do
     if io.respond_to?(:external_encoding)
-      ee = io.external_encoding
+      original_encoding = io.external_encoding
+    end
+
+    # when RUBY_VERSION >= 2.0, default encoding is utf-8.
+    # ensure that external_encoding is differ from the original external_encoding(original_encoding).
+    if original_encoding == Encoding.find('utf-8')
+      specified_encoding = 'sjis'
+    else
+      specified_encoding = 'utf-8'
     end
-    FileReader::LineReader.new(io, error, {:encoding => 'utf-8'})
+
+    FileReader::LineReader.new(io, error, {:encoding => specified_encoding})
     if io.respond_to?(:external_encoding)
-      io.external_encoding.should_not == ee
-      io.external_encoding.should == Encoding.find('utf-8')
+      io.external_encoding.should_not == original_encoding
+      io.external_encoding.should == Encoding.find(specified_encoding)
     end
   end
 

data/spec/td/command/import_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+require 'td/command/common'
+require 'td/command/import'
+
+module TreasureData::Command
+  describe 'import commands' do
+    describe CommandExecutor do
+      it 'executes command' do
+        CommandExecutor.new(['echo'], nil).execute.exitstatus.should == 0
+      end
+
+      it 'executes command' do
+        expect {
+          CommandExecutor.new(['exit', '1'], nil).execute
+        }.to raise_error BulkImportExecutionError, /td-bulk-import\.log/
+      end
+
+      it 'terminates process on timeout' do
+        CommandExecutor.new(['sleep', '1'], 2).execute
+        expect {
+          CommandExecutor.new(['sleep', '2'], 1).execute
+        }.to raise_error BulkImportExecutionError, /timed out/
+      end
+    end
+  end
+end

data/spec/td/command/job_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+require 'td/command/common'
+require 'td/command/job'
+require 'td/command/list'
+require 'tempfile'
+
+module TreasureData::Command
+  describe 'job commands' do
+    let :command do
+      Class.new { include TreasureData::Command }.new
+    end
+
+    let :job do
+      job = TreasureData::Job.new(nil, 12345, 'hive', 'select * from employee')
+      job.instance_eval do
+        @result = [[["1", 2.0, {key:3}], 1], [["4", 5.0, {key:6}], 2], [["7", 8.0, {key:9}], 3]]
+        @result_size = 3
+        @status = 'success'
+      end
+      job
+    end
+
+    describe 'write_result' do
+      it 'supports json output' do
+        file = Tempfile.new("job_spec")
+        command.send(:show_result, job, file, nil, 'json')
+        File.read(file.path).should == %Q([["1",2.0,{"key":3}],\n["4",5.0,{"key":6}],\n["7",8.0,{"key":9}]])
+      end
+
+      it 'supports csv output' do
+        file = Tempfile.new("job_spec")
+        command.send(:show_result, job, file, nil, 'csv')
+        File.read(file.path).should == %Q(1,2.0,"{""key"":3}"\n4,5.0,"{""key"":6}"\n7,8.0,"{""key"":9}"\n)
+      end
+
+      it 'supports tsv output' do
+        file = Tempfile.new("job_spec")
+        command.send(:show_result, job, file, nil, 'tsv')
+        File.read(file.path).should == %Q(1\t2.0\t{"key":3}\n4\t5.0\t{"key":6}\n7\t8.0\t{"key":9}\n)
+      end
+    end
+  end
+end

data/spec/td/command_sched_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+require 'td/command/common'
+require 'td/config'
+require 'td/command/list'
+require 'td/command/sched'
+require 'td/client/model'
+require 'time'
+
+module TreasureData::Command
+
+  describe 'sched_history' do
+    it 'runs' do
+      client = Object.new
+      time = Time.now.xmlschema
+      job_params = ['job_id', :type, 'query', 'status', nil, nil, time, time, 123, 456]
+      job1 = TreasureData::ScheduledJob.new(client, '2015-02-17 13:22:52 +0900', *job_params)
+      job2 = TreasureData::ScheduledJob.new(client, nil, *job_params)
+      client.stub(:schedules).and_return([])
+      client.stub(:history).and_return([job1, job2])
+      command = Class.new { include TreasureData::Command }.new
+      command.stub(:get_client).and_return(client)
+      op = List::CommandParser.new('sched:history', %w[], %w[], false, [], [])
+      expect {
+        command.sched_history(op)
+      }.to_not raise_exception
+    end
+  end
+end

data/spec/td/updater_spec.rb

@@ -162,6 +162,10 @@ module TreasureData::Updater
         '/maven2/com/treasuredata/td-import/maven-metadata.xml',
         WEBrick::HTTPServlet::ProcHandler.new(method(:metadata).to_proc)
       )
+      @server.mount(
+        '/maven2/com/treasuredata/td-import/version/td-import-version-jar-with-dependencies.jar',
+        WEBrick::HTTPServlet::ProcHandler.new(method(:jar).to_proc)
+      )
       @server_thread = start_server_thread(@server)
       @server
     end
@@ -181,6 +185,11 @@ module TreasureData::Updater
       res.body = '<metadata><versioning><lastUpdated>20141204123456</lastUpdated><release>version</release></versioning></metadata>'
     end
 
+    def jar
+      res['content-type'] = 'application/octet-stream'
+      res.body = File.read(fixture_file('tmp.zip'))
+    end
+
     def start_server_thread(server)
       t = Thread.new {
         Thread.current.abort_on_exception = true

data/td.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.6.1"
-  gem.add_dependency "td-client", "~> 0.8.67"
+  gem.add_dependency "td-client", "~> 0.8.68"
   gem.add_dependency "td-logger", "~> 0.3.21"
   gem.add_dependency "rubyzip", "~> 0.9.9"
   gem.add_development_dependency "rake", "~> 0.9"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.11.6
+  version: 0.11.7
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-05 00:00:00.000000000 Z
+date: 2015-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -102,14 +102,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.67
+        version: 0.8.68
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.67
+        version: 0.8.68
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
@@ -248,6 +248,9 @@ files:
 - spec/file_reader/shared_context.rb
 - spec/file_reader_spec.rb
 - spec/spec_helper.rb
+- spec/td/command/import_spec.rb
+- spec/td/command/job_spec.rb
+- spec/td/command_sched_spec.rb
 - spec/td/common_spec.rb
 - spec/td/fixture/ca.cert
 - spec/td/fixture/server.cert
@@ -288,6 +291,9 @@ test_files:
 - spec/file_reader/shared_context.rb
 - spec/file_reader_spec.rb
 - spec/spec_helper.rb
+- spec/td/command/import_spec.rb
+- spec/td/command/job_spec.rb
+- spec/td/command_sched_spec.rb
 - spec/td/common_spec.rb
 - spec/td/fixture/ca.cert
 - spec/td/fixture/server.cert