td 0.13.2 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e20abc2bf25fadf9daeac46efc09aa0278f9d8cb
-  data.tar.gz: 83dfdb2445a9279810a3c651f979fc8b1af12ae0
+  metadata.gz: be3247f0bc28da7b0e3f1e5caa10aaf2f0b0ff52
+  data.tar.gz: bdf8a4ac5a21214084d2f66f791ae3f85b4777d9
 SHA512:
-  metadata.gz: 8a78076ea4a0b05fdd923980e63e0a7836e9dac065baa76a16358e5b6b5cd3043f6f8a792f67ef5ed8ad01782ff237fa042b356d9685502cee1cb678412dba08
-  data.tar.gz: 4c25dd9af339f532470e8d32322c50bbcdce1b987cd2891269f3dae1bc38688a0a43963d97d210473217ca26787b2ba9d05990283b6904a6dd6044e94714ab36
+  metadata.gz: a5dfba44b1698e8f4f89bf0ee72c791e1b1ac54ab11398d5450b8e51d4fb214b0051cd91acf325604d6696b6dad503a69528fd71ba7aba7eec616b9c4d94688a
+  data.tar.gz: deeecc28efaa52310e34a732e77788162e6c5429bb119c3e3f7fe9efc1d3130ffbc1d728ab24fd749b53c8465007c19186746c2f369456af58356a86a7100a02

data/.travis.yml

@@ -11,7 +11,7 @@ rvm:
 gemfile:
   - Gemfile
 
-script: bundle exec rake spec
+script: bundle exec rake spec SPEC_OPTS="-fd"
 
 matrix:
   allow_failures:

data/ChangeLog

@@ -1,3 +1,12 @@
+== 2016-04-25 version 0.14.0
+
+* Use parallel v1.8.0 #161
+* Support `td query --domain-key=KEY` #157
+* Add encryption of export worker #156 #159
+* Support dummy schedule #152
+* output bulkload config with `td job:show` #151
+* fix: make jar_update more solid #150
+
 == 2016-01-25 version 0.13.2
 
 * Replace Infinity and NaN inside Array or Hash

data/lib/td/command/export.rb

@@ -2,6 +2,7 @@
 module TreasureData
 module Command
   SUPPORTED_FORMATS = %W[json.gz line-json.gz tsv.gz]
+  SUPPORTED_ENCRYPT_METHOD = %W[s3]
 
   def table_export(op)
     from = nil
@@ -13,6 +14,7 @@ module Command
     file_prefix = nil
     file_format = "json.gz" # default
     pool_name = nil
+    encryption = nil
 
     op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b|
       wait = b
@@ -42,6 +44,10 @@ module Command
     op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s|
       pool_name = s
     }
+    op.on('-e', '--encryption ENCRYPT_METHOD', 'export with server side encryption with the ENCRYPT_METHOD') {|s|
+      raise ArgumentError, "#{s} is not a supported encryption method" unless SUPPORTED_ENCRYPT_METHOD.include?(s)
+      encryption = s
+    }
 
     db_name, table_name = op.cmd_parse
 
@@ -75,6 +81,7 @@ module Command
     s3_opts['access_key_id'] = aws_access_key_id
     s3_opts['secret_access_key'] = aws_secret_access_key
     s3_opts['pool_name'] = pool_name if pool_name
+    s3_opts['encryption'] = encryption if encryption
 
     job = client.export(db_name, table_name, "s3", s3_opts)
 

data/lib/td/command/job.rb

@@ -180,6 +180,9 @@ private
       $stdout.puts "Query       : #{job.query}"
     elsif job.type == :bulk_import_perform
       $stdout.puts "Destination : #{job.query}"
+    elsif job.type == :bulkload
+      require 'yaml'
+      $stdout.puts "Config      :\n#{YAML.dump(job.query)}"
     end
     # if the job is done and is of type hive, show the Map-Reduce cumulated CPU time
     if job.finished?

data/lib/td/command/query.rb

@@ -19,6 +19,7 @@ module Command
     limit = nil
     exclude = false
     pool_name = nil
+    domain_key = nil
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
@@ -85,6 +86,9 @@ module Command
     op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s|
       pool_name = s
     }
+    op.on('--domain-key DOMAIN_KEY', 'optional user-provided unique ID. You can include this ID with your `create` request to ensure idempotence') {|s|
+      domain_key = s
+    }
 
     sql = op.cmd_parse
 
@@ -134,6 +138,7 @@ module Command
     opts = {}
     opts['type'] = type if type
     opts['pool_name'] = pool_name if pool_name
+    opts['domain_key'] = domain_key if domain_key
     job = client.query(db_name, sql, result_url, priority, retry_limit, opts)
 
     $stdout.puts "Job #{job.job_id} is queued."

data/lib/td/command/runner.rb

@@ -185,7 +185,7 @@ EOF
       #   => NotFoundError
       #   => AuthError
       if ![ParameterConfigurationError, BulkImportExecutionError, UpdateError, ImportError,
-            APIError, ForbiddenError, NotFoundError, AuthError].include?(e.class) ||
+            APIError, ForbiddenError, NotFoundError, AuthError, AlreadyExistsError].include?(e.class) ||
          !ENV['TD_TOOLBELT_DEBUG'].nil? || $verbose
         show_backtrace "Error #{$!.class}: backtrace:", $!.backtrace
       end

data/lib/td/command/sched.rb

@@ -116,7 +116,11 @@ module Command
       exit 1
     end
 
-    $stderr.puts "Schedule '#{name}' is created. It starts at #{first_time.localtime}."
+    if first_time
+      $stderr.puts "Schedule '#{name}' is created. It starts at #{first_time.localtime}."
+    else
+      $stderr.puts "Schedule '#{name}' is created, which never runs."
+    end
   end
 
   def sched_delete(op)

data/lib/td/updater.rb

@@ -331,17 +331,6 @@ end # module ModuleDefinition
     if updated > last_updated
       FileUtils.mkdir_p(Updater.jarfile_dest_path) unless File.exists?(Updater.jarfile_dest_path)
       Dir.chdir(Updater.jarfile_dest_path) do
-        File.open('VERSION', 'w') {|f|
-          if hourly
-            f.print "#{version} via hourly jar auto-update"
-          else
-            f.print "#{version} via import:jar_update command"
-          end
-        }
-        File.open('td-import-java.version', 'w') {|f|
-          f.print "#{version} #{updated}"
-        }
-
         status = nil
         indicator = Command::TimeBasedDownloadProgressIndicator.new(
           "Updating td-import.jar", Time.new.to_i, 2)
@@ -355,8 +344,20 @@ end # module ModuleDefinition
         if status
           $stdout.puts "Installed td-import.jar v#{version} in '#{Updater.jarfile_dest_path}'.\n"
           File.rename 'td-import.jar.new', 'td-import.jar'
+
+          File.open('VERSION', 'w') {|f|
+            if hourly
+              f.print "#{version} via hourly jar auto-update"
+            else
+              f.print "#{version} via import:jar_update command"
+            end
+          }
+          File.open('td-import-java.version', 'w') {|f|
+            f.print "#{version} #{updated}"
+          }
         else
-          $stdout.puts "Update of td-import.jar failed." unless ENV['TD_TOOLBELT_DEBUG'].nil?
+          $stdout.puts "Update of td-import.jar failed."
+          $stdout.puts "Please execute 'td import:jar_update' later to update td-import.jar correctly."
           File.delete 'td-import.jar.new' if File.exists? 'td-import.jar.new'
         end
       end
@@ -366,7 +367,11 @@ end # module ModuleDefinition
   end
 
   def check_n_update_jar(hourly = false)
-    if hourly
+    if !File.exist?(File.join(Updater.jarfile_dest_path, 'td-import.jar')) ||
+      !File.exist?(File.join(Updater.jarfile_dest_path, 'VERSION'))
+      # remove all file to install cleanly if a file is missing
+      FileUtils.rm_rf(Updater.jarfile_dest_path)
+    elsif hourly
       if !ENV['TD_TOOLBELT_JAR_UPDATE'].nil?
         # also validates the TD_TOOLBELT_JAR_UPDATE environment variable value
         if ENV['TD_TOOLBELT_JAR_UPDATE'] == "0"

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.13.2'
+  TOOLBELT_VERSION = '0.14.0'
 end

data/spec/file_reader/filter_spec.rb

@@ -42,7 +42,7 @@ describe 'FileReader filters' do
 
     it 'initialize' do
       filter = FileReader::AutoTypeConvertParserFilter.new(parser, error, options)
-      filter.should_not be_nil
+      expect(filter).not_to be_nil
     end
 
     context 'after initialization' do
@@ -51,14 +51,14 @@ describe 'FileReader filters' do
       end
 
       it 'forward returns one converted line' do
-        filter.forward.should == dataset[0]
+        expect(filter.forward).to eq(dataset[0])
       end
 
       it 'feeds all lines' do
         begin
           i = 0
           while line = filter.forward
-            line.should == dataset[i]
+            expect(line).to eq(dataset[i])
             i += 1
           end
         rescue
@@ -79,7 +79,7 @@ describe 'FileReader filters' do
 
     it 'initialize' do
       builder = FileReader::HashBuilder.new(parser, error, columns)
-      builder.should_not be_nil
+      expect(builder).not_to be_nil
     end
 
     context 'after initialization' do
@@ -88,14 +88,14 @@ describe 'FileReader filters' do
       end
 
       it 'forward returns one converted line' do
-        builder.forward.should == built_dataset[0]
+        expect(builder.forward).to eq(built_dataset[0])
       end
 
       it 'feeds all lines' do
         begin
           i = 0
           while line = builder.forward
-            line.should == built_dataset[i]
+            expect(line).to eq(built_dataset[i])
             i += 1
           end
         rescue
@@ -111,7 +111,7 @@ describe 'FileReader filters' do
 
         it 'initialize' do
           filter = FileReader::TimeParserFilter.new(builder, error, :time_column => 'log_at')
-          filter.should_not be_nil
+          expect(filter).not_to be_nil
         end
 
         context 'after initialization' do
@@ -125,14 +125,14 @@ describe 'FileReader filters' do
           end
 
           it 'forward returns one parse line with parsed log_at' do
-            filter.forward.should == timed_dataset[0]
+            expect(filter.forward).to eq(timed_dataset[0])
           end
 
           it 'feeds all lines' do
             begin
               i = 0
               while line = filter.forward
-                line.should == timed_dataset[i]
+                expect(line).to eq(timed_dataset[i])
                 i += 1
               end
             rescue
@@ -157,7 +157,7 @@ describe 'FileReader filters' do
               rescue RSpec::Expectations::ExpectationNotMetError => e
                 fail
               rescue
-                i.should == 0
+                expect(i).to eq(0)
               end
             end
           end
@@ -182,7 +182,7 @@ describe 'FileReader filters' do
                 rescue RSpec::Expectations::ExpectationNotMetError => e
                   fail
                 rescue
-                  i.should == 0
+                  expect(i).to eq(0)
                 end
               end
             }
@@ -199,7 +199,7 @@ describe 'FileReader filters' do
 
         it 'initialize' do
           filter = FileReader::SetTimeParserFilter.new(builder, error, :time_value => Time.now.to_i)
-          filter.should_not be_nil
+          expect(filter).not_to be_nil
         end
 
         context 'after initialization' do
@@ -216,14 +216,14 @@ describe 'FileReader filters' do
           end
 
           it 'forward returns one converted line with time' do
-            filter.forward.should == timed_dataset[0]
+            expect(filter.forward).to eq(timed_dataset[0])
           end
 
           it 'feeds all lines' do
             begin
               i = 0
               while line = filter.forward
-                line.should == timed_dataset[i]
+                expect(line).to eq(timed_dataset[i])
                 i += 1
               end
             rescue

data/spec/file_reader/io_filter_spec.rb

@@ -47,12 +47,12 @@ describe 'FileReader io filters' do
 
       it 'returns Zlib::GzipReader with :gzip' do
         wrapped = FileReader::DecompressIOFilter.filter(gzipped_io, error, :compress => 'gzip')
-        wrapped.should be_an_instance_of(Zlib::GzipReader)
+        expect(wrapped).to be_an_instance_of(Zlib::GzipReader)
       end
 
       it 'returns Zlib::GzipReader with auto detection' do
         wrapped = FileReader::DecompressIOFilter.filter(gzipped_io, error, {})
-        wrapped.should be_an_instance_of(Zlib::GzipReader)
+        expect(wrapped).to be_an_instance_of(Zlib::GzipReader)
       end
 
       context 'after initialization' do
@@ -63,18 +63,18 @@ describe 'FileReader io filters' do
           end
 
           it 'forward_row returns one line' do
-            reader.forward_row.should == lines[0]
+            expect(reader.forward_row).to eq(lines[0])
           end
 
           it 'feeds all lines' do
             begin
               i = 0
               while line = reader.forward_row
-                line.should == lines[i]
+                expect(line).to eq(lines[i])
                 i += 1
               end
             rescue
-              gzipped_io.eof?.should be_true
+              expect(gzipped_io.eof?).to be_truthy
             end
           end
         }
@@ -84,12 +84,12 @@ describe 'FileReader io filters' do
     describe 'plain' do
       it 'returns passed io with :plain' do
         wrapped = FileReader::DecompressIOFilter.filter(io, error, :compress => 'plain')
-        wrapped.should be_an_instance_of(StringIO)
+        expect(wrapped).to be_an_instance_of(StringIO)
       end
 
       it 'returns passed io with auto detection' do
         wrapped = FileReader::DecompressIOFilter.filter(io, error, {})
-        wrapped.should be_an_instance_of(StringIO)
+        expect(wrapped).to be_an_instance_of(StringIO)
       end
     end
   end

data/spec/file_reader/line_reader_spec.rb

@@ -27,7 +27,7 @@ describe FileReader::LineReader do
     end
     FileReader::LineReader.new(io, error, {})
     if io.respond_to?(:external_encoding)
-      io.external_encoding.should == ee
+      expect(io.external_encoding).to eq(ee)
     end
   end
 
@@ -46,8 +46,8 @@ describe FileReader::LineReader do
 
     FileReader::LineReader.new(io, error, {:encoding => specified_encoding})
     if io.respond_to?(:external_encoding)
-      io.external_encoding.should_not == original_encoding
-      io.external_encoding.should == Encoding.find(specified_encoding)
+      expect(io.external_encoding).not_to eq(original_encoding)
+      expect(io.external_encoding).to eq(Encoding.find(specified_encoding))
     end
   end
 
@@ -57,7 +57,7 @@ describe FileReader::LineReader do
     end
 
     it 'forward_row returns one line' do
-      reader.forward_row.should == lines[0]
+      expect(reader.forward_row).to eq(lines[0])
     end
 
     # TODO: integrate with following shared_examples_for
@@ -65,13 +65,13 @@ describe FileReader::LineReader do
       begin
         i = 0
         while line = reader.forward_row
-          line.should == lines[i]
+          expect(line).to eq(lines[i])
           i += 1
         end
       rescue RSpec::Expectations::ExpectationNotMetError => e
         fail
       rescue
-        io.eof?.should be_true
+        expect(io.eof?).to be_truthy
       end
     end
 
@@ -82,13 +82,13 @@ describe FileReader::LineReader do
         begin
           i = 0
           while line = parser.forward
-            line.should == get_expected.call(i)
+            expect(line).to eq(get_expected.call(i))
             i += step
           end
         rescue RSpec::Expectations::ExpectationNotMetError => e
           fail
         rescue
-          io.eof?.should be_true
+          expect(io.eof?).to be_truthy
         end
       end
     end
@@ -96,7 +96,7 @@ describe FileReader::LineReader do
     describe FileReader::JSONParser do
       it 'initialize with LineReader' do
         parser = FileReader::JSONParser.new(reader, error, {})
-        parser.should_not be_nil
+        expect(parser).not_to be_nil
       end
 
       context 'after initialization' do
@@ -105,7 +105,7 @@ describe FileReader::LineReader do
         end
 
         it 'forward returns one line' do
-          parser.forward.should == JSON.parse(lines[0])
+          expect(parser.forward).to eq(JSON.parse(lines[0]))
         end
 
         let :get_expected do
@@ -144,7 +144,7 @@ describe FileReader::LineReader do
 
         it "initialize with LineReader and #{pattern} delimiter" do
           parser = FileReader::DelimiterParser.new(reader, error, {:delimiter_expr => Regexp.new(pattern)})
-          parser.should_not be_nil
+          expect(parser).not_to be_nil
         end
 
         context 'after initialization' do
@@ -153,7 +153,7 @@ describe FileReader::LineReader do
           end
 
           it 'forward returns one line' do
-            parser.forward.should == lines[0].split(pattern)
+            expect(parser.forward).to eq(lines[0].split(pattern))
           end
 
           let :get_expected do
@@ -198,7 +198,7 @@ describe FileReader::LineReader do
 
         it "initialize with LineReader" do
           parser = parser_class.new(reader, error, {})
-          parser.should_not be_nil
+          expect(parser).not_to be_nil
         end
 
         context 'after initialization' do
@@ -207,7 +207,7 @@ describe FileReader::LineReader do
           end
 
           it 'forward returns one line' do
-            parser.forward.should == output[0]
+            expect(parser.forward).to eq(output[0])
           end
 
           let :get_expected do