td 0.16.7 → 0.16.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0e8a82e003b9c2d6667499b111602cdca509d2c7
-  data.tar.gz: db9933625ee612d1916878a0d8af9e6b7b533136
+  metadata.gz: dc8c50e05e0ed05da8a93a7b7e39f5c7577e2115
+  data.tar.gz: a5d08d636a3d28cf2c2a35047e3e7fe415e268d8
 SHA512:
-  metadata.gz: e565c67246ce1ae52d3ce2c7417e6c656a09166c00878cedb9bb9b5d54025566c222944907bf8503e4589aac29a06062d222c26375995bcdab1bfd2a889b99ad
-  data.tar.gz: caf6024add00f26ccce68cd37665bbb5b24783cc1b47d430f9ee06e4ec65fec3a57a64ce9f96bb39179eb6e4242101024e45ff83250174b436096c37f7dbd68c
+  metadata.gz: 7536bf0689d4f7fd05bbb1d3c7a42d8e712cfeed53dda3d3f896f3803f2aff7cc036f41c7c6cf20e9f1203e538dbad0b81791d12f346f9f0b5c7ce478066131f
+  data.tar.gz: 2b5872b1402f8f5302f8814e9100e8e129b65637fc66d330fc439071cc458a5e956d065307107deafe55cd168e921a663d2824789bf708f46daa43b2abe70c47

data/.travis.yml

@@ -1,8 +1,5 @@
 language: ruby
 
-jdk:
-  - openjdk8
-
 rvm:
   - 2.1.10
   - 2.2.10

data/ChangeLog

@@ -1,3 +1,7 @@
+== 2020-01-16 version 0.16.8
+
+* Update default maven repository #230
+
 == 2019-12-16 version 0.16.7
 
 * Disable direct download in Private Connect #225

data/README.rdoc

@@ -69,7 +69,7 @@ to process and Bulk load data in the Treasure Data Cloud.
 
 The Bulk Import CLI is downloaded automatically at the first call to any of the command that require it; the use will
 need internet connectivity in order to fetch the Bulk Import CLI JAR file from the
-{Central Maven repository}[http://central.maven.org/maven2/com/treasuredata/td-import/]
+{Central Maven repository}[https://repo1.maven.org/maven2/com/treasuredata/td-import/]
 and take advantage of these advanced features. If you need to setup a proxy, please consult this
 {documentation}[https://support.treasuredata.com/hc/en-us/articles/360001502527-Legacy-Bulk-Import-Tips-and-Tricks#How%20to%20use%20a%20proxy%20server] page.
 
@@ -144,9 +144,9 @@ These are the available hooks:
 
     $ TD_TOOLBELT_UPDATE_ROOT="http://toolbelt.treasuredata.com"
 
-* Specify an alternative endpoint to use updating the JAR file (default: http://central.maven.org):
+* Specify an alternative endpoint to use updating the JAR file (default: https://repo1.maven.org):
 
-    $ TD_TOOLBELT_JARUPDATE_ROOT="http://central.maven.org"
+    $ TD_TOOLBELT_JARUPDATE_ROOT="https://repo1.maven.org"
 
 
 = Copyright

data/lib/td/command/common.rb

@@ -47,7 +47,7 @@ module Command
 
     # optional, if not provided a default is used from the ruby client library
     begin
-      if Config.endpoint
+      if !opts[:endpoint] && Config.endpoint
         opts[:endpoint] = Config.endpoint
       end
     rescue ConfigNotFoundError => e
@@ -67,6 +67,17 @@ module Command
     Client.new(apikey, opts)
   end
 
+  DEFAULT_IMPORT_ENDPOINT = "https://" + TreasureData::API::DEFAULT_IMPORT_ENDPOINT
+
+  def get_import_client
+    import_endpoint = begin
+                        Config.import_endpoint || DEFAULT_IMPORT_ENDPOINT
+                      rescue TreasureData::ConfigNotFoundError
+                        DEFAULT_IMPORT_ENDPOINT
+                      end
+    get_client(endpoint: import_endpoint)
+  end
+
   def get_ssl_client(opts={})
     opts[:ssl] = true
     get_client(opts)

data/lib/td/command/runner.rb

@@ -6,11 +6,12 @@ class Runner
     @config_path = nil
     @apikey = nil
     @endpoint = nil
+    @import_endpoint = nil
     @prog_name = nil
     @insecure = false
   end
 
-  attr_accessor :apikey, :endpoint, :config_path, :prog_name, :insecure
+  attr_accessor :apikey, :endpoint, :import_endpoint, :config_path, :prog_name, :insecure
 
   def run(argv=ARGV)
     require 'td/version'
@@ -73,6 +74,7 @@ EOF
     config_path = @config_path
     apikey = @apikey
     endpoint = @endpoint
+    import_endpoint = @import_endpoint || @endpoint
     insecure = nil
     $verbose = false
     #$debug = false
@@ -94,6 +96,12 @@ EOF
       endpoint = e
     }
 
+    op.on('--import-endpoint API_IMPORT_SERVER', "specify the URL for API Import server to use (default: https://api-import.treasuredata.com).") { |e|
+      require 'td/command/common'
+      Command.validate_api_endpoint(e)
+      import_endpoint = e
+    }
+
     op.on('--insecure', "Insecure access: disable SSL (enabled by default)") {|b|
       insecure = true
     }
@@ -140,6 +148,10 @@ EOF
         Config.endpoint = endpoint
         Config.cl_endpoint = true
       end
+      if import_endpoint
+        Config.import_endpoint = import_endpoint
+        Config.cl_import_endpoint = true
+      end
       if insecure
         Config.secure = false
       end

data/lib/td/command/table.rb

@@ -1,5 +1,6 @@
 require 'td/helpers'
 require 'td/command/job'
+require 'td/client/api'
 
 module TreasureData
 module Command
@@ -453,17 +454,18 @@ module Command
     import_params[:table_name] = table_name
     import_params[:paths] = paths
 
-    client = get_client
+    api_client = get_client
+    import_client = get_import_client
 
     if auto_create
-      create_database_and_table_if_not_exist(client, db_name, table_name)
+      create_database_and_table_if_not_exist(api_client, db_name, table_name)
     end
 
-    do_table_import(client, import_params)
+    do_table_import(api_client, import_client, import_params)
   end
 
   private
-  def do_table_import(client, import_params)
+  def do_table_import(api_client, import_client, import_params)
     case import_params[:format]
     when 'json', 'msgpack'
       #unless time_key
@@ -488,7 +490,7 @@ module Command
     end
 
     begin
-      db = client.database(import_params[:db_name])
+      db = api_client.database(import_params[:db_name])
     rescue ForbiddenError => e
       $stdout.puts "Warning: database and table validation skipped - #{e.message}"
     else
@@ -521,7 +523,7 @@ module Command
     #require 'thread'
 
     files.zip(import_params[:paths]).each {|file, path|
-      import_log_file(file, path, client, import_params[:db_name], import_params[:table_name], parser)
+      import_log_file(file, path, import_client, import_params[:db_name], import_params[:table_name], parser)
     }
 
     $stdout.puts "done."

data/lib/td/command/workflow.rb

@@ -246,14 +246,20 @@ EOF
         return false
       end
       if output =~ /openjdk version/ or output =~ /java version/
-        m = output.match(/version "(\d+)\.(\d+)\.(\d+)(?:_(\d+))"/)
+        m = output.match(/version "(\d+)\.(\d+)\.(\d+)(?:_(\d+))?"/)
         if not m or m.size < 4
           return false
         end
-        # Check for at least Java 8. Let digdag itself verify revision.
         major = m[1].to_i
         minor = m[2].to_i
-        if major < 1 or minor < 8
+        # Check for at least Java 8. Let digdag itself verify revision.
+        if major < 1
+          return false
+        elsif major == 1 # suppose the version style: 1.8.10_52 (JDK8 and before)
+          if minor < 8
+            return false
+          end
+        elsif major < 9 # suppose the version style: 11.0.2
           return false
         end
       end

data/lib/td/config.rb

@@ -21,6 +21,9 @@ class Config
   @@endpoint = ENV['TREASURE_DATA_API_SERVER'] || ENV['TD_API_SERVER']
   @@endpoint = nil if @@endpoint == ""
   @@cl_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line
+  @@import_endpoint = ENV['TREASURE_DATA_API_IMPORT_SERVER'] || ENV['TD_API_IMPORT_SERVER']
+  @@import_endpoint = nil if @@endpoint == ""
+  @@cl_import_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line option
   @@secure = true
   @@retry_post_requests = false
 
@@ -164,6 +167,22 @@ class Config
     @@cl_endpoint = flag
   end
 
+  def self.import_endpoint
+    @@import_endpoint || Config.read['account.import_endpoint']
+  end
+
+  def self.import_endpoint=(endpoint)
+    @@import_endpoint = endpoint
+  end
+
+  def self.cl_import_endpoint
+    @@cl_import_endpoint
+  end
+
+  def self.cl_import_endpoint=(flag)
+    @@cl_import_endpoint = flag
+  end
+
   def self.workflow_endpoint
     case self.endpoint_domain
     when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?(eu01\.)?treasuredata\.(com|co\.jp)\z/i
@@ -176,10 +195,9 @@ class Config
   # renders the apikey and endpoint options as a string for the helper commands
   def self.cl_options_string
     string = ""
-    string += "-k #{@@apikey}" if @@cl_apikey
-    string += " " unless string.empty?
-    string += "-e #{@@endpoint}" if @@cl_endpoint
-    string += " " unless string.empty?
+    string += "-k #{@@apikey} " if @@cl_apikey
+    string += "-e #{@@endpoint} " if @@cl_endpoint
+    string += "--import-endpoint #{@@import_endpoint} " if @@cl_import_endpoint
     string
   end
 

data/lib/td/updater.rb

@@ -270,7 +270,7 @@ end # module ModuleDefinition
   extend ModuleDefinition
 
   def maven_repo_root
-    ENV['TD_TOOLBELT_JARUPDATE_ROOT'] || "http://central.maven.org"
+    ENV['TD_TOOLBELT_JARUPDATE_ROOT'] || "https://repo1.maven.org"
   end
 
   private

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.16.7'
+  TOOLBELT_VERSION = '0.16.8'
 end

data/spec/td/command/table_spec.rb

@@ -302,6 +302,7 @@ module TreasureData::Command
       let(:db_name)    { 'database' }
       let(:table_name) { 'table' }
       let(:client)     { double('client') }
+      let(:import_client)     { double('import-client') }
       let(:command)    { Class.new { include TreasureData::Command }.new }
 
       describe 'auto create table' do
@@ -336,6 +337,7 @@ module TreasureData::Command
       describe 'time key' do
         before do
           allow(command).to receive(:get_client) { client }
+          allow(command).to receive(:get_import_client) { import_client }
           allow(command).to receive(:do_table_import)
         end
         let(:input_params) {{
@@ -355,7 +357,7 @@ module TreasureData::Command
             end
 
             it "with '#{tk_option}' option" do
-              expect(command).to receive(:do_table_import).with(client, input_params)
+              expect(command).to receive(:do_table_import).with(client, import_client, input_params)
               command.table_import(option)
             end
           end
@@ -371,7 +373,7 @@ module TreasureData::Command
           end
 
           it 'without \'-t / --time-key\' option' do
-            expect(command).to receive(:do_table_import).with(client, input_params)
+            expect(command).to receive(:do_table_import).with(client, import_client, input_params)
             command.table_import(option)
           end
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.16.7
+  version: 0.16.8
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-16 00:00:00.000000000 Z
+date: 2020-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack