td 0.16.6 → 0.16.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc9144137963eda9f35f262eda8dc48b5757fcb2bc5259152f500af104681ce7
-  data.tar.gz: b56b85d17b7cfd3c07a1cd140961c267e27ecd476b419b8e31384b4efeff4761
+  metadata.gz: a372eaff21645cac711c4df4fe0faeba64a8fffed0af8d9de4367f5bdf3d4dae
+  data.tar.gz: b4f0c3751f57780008be2162c78853003b138d9ed691f1c7eef756a8ce6f7237
 SHA512:
-  metadata.gz: f1c60a7bdfceea5927a3b7ffb88b71a89b7ae8de11ad848e3bb877c23e7fb0229fc0219ff757af4bb7a090530b0358bdb0b93c443817b0214fc22a0e955dc7e3
-  data.tar.gz: 9c003ea54b6aaf2dc53f85266f6107910998070c5b48f32605dc3c82001f9fe1d737ccab2fe1e6dda307b82abbc347e0980a4028f813caf1511b9971b474db00
+  metadata.gz: 3f332bc9d0cc8659e31d2f737aec26c827a6eda4c980dd0d21ddf0df749fae5595b16950a9ba3195e76361925d190bf8aa5e016dee6f25cf49bdf139e8606363
+  data.tar.gz: af1d0421633676a8938be8cf77474681f05e863006c56561db18b377853ee9f8529c168d1d176a2e4f3156a1eb4acd582241765f52a8aac6ecce0e3e158e1ddf

data/.circleci/config.yml

@@ -0,0 +1,96 @@
+version: 2.1
+
+
+orbs:
+  ruby: circleci/ruby@1.1.4 
+  win: circleci/windows@2.4.0
+
+
+commands:
+  install_windows_requirements:
+    description: "Install windows requirements"
+    steps:
+      - run:
+          name: "Install MSYS2"
+          command: choco install msys2
+      - run:
+          name: "Install Ruby devkit"
+          command: ridk install 2 3
+  bundle-install:
+    description: "Install dependencies"
+    steps:
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - run:
+          name: Bundle install
+          command: bundle install
+  run-tests:
+    description: "Run tests"
+    steps:
+      - run:
+          name: Run tests
+          command: bundle exec rake spec SPEC_OPTS="-fd"
+  run-tests-flow:
+    description: "Single flow for running tests"
+    steps:
+      - checkout
+      - bundle-install
+      - run-tests
+
+
+jobs:
+
+  ruby_25:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - ruby/install:
+          version: '2.5'
+      - run-tests-flow
+
+  ruby_26:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - ruby/install:
+          version: '2.6'
+      - run-tests-flow
+
+  ruby_27:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - ruby/install:
+          version: '2.7'
+      - run-tests-flow
+
+  ruby_30:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - ruby/install:
+          version: '3.0'
+      - run-tests-flow
+
+  win_ruby:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    steps:
+      - install_windows_requirements
+      - run:
+          name: "Install bundler"
+          shell: powershell.exe
+          command: gem install bundler
+      - run-tests-flow
+
+
+workflows:
+  tests:
+    jobs:
+      - ruby_25
+      - ruby_26
+      - ruby_27
+      - ruby_30
+      # - win_ruby # Not testing on windows for now due to old version of ridk. Testing in Appveyor instead, will reconsider when CircleCI update ridk in their windows image.

data/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @treasure-data/integrations

data/ChangeLog

@@ -1,3 +1,25 @@
+== 2020-09-16 version 0.16.10
+
+* Support proxy with username and password
+* Increase the summary_width value to fix the broken options help message format
+* Support AP03 Region
+* Change old document urls to new document urls
+* Show message to ask -w option in td query
+* Adding users commnad help
+* Allow trailing slash for endpoints
+
+== 2020-04-10 version 0.16.9
+
+* Update new workflow pattern to support ap02 #232
+
+== 2020-01-16 version 0.16.8
+
+* Update default maven repository #230
+
+== 2019-12-16 version 0.16.7
+
+* Disable direct download in Private Connect #225
+
 == 2019-11-13 version 0.16.6
 
 * Use HTTPS as default for downloading digdag module #223

data/README.rdoc

@@ -1,14 +1,14 @@
 = Treasure Data command line tool
-{<img src="https://travis-ci.org/treasure-data/td.svg?branch=master" alt="Build Status" />}[https://travis-ci.org/treasure-data/td]
+{<img src="https://circleci.com/gh/treasure-data/td.svg?style=svg" alt="Build Status" />}[https://circleci.com/gh/treasure-data/td.svg?style=svg]
 {<img src="https://ci.appveyor.com/api/projects/status/github/treasure-data/td?branch=master&svg=true" alt="appveyor" />}[https://ci.appveyor.com/project/treasure-data/td/branch/master]
 {<img src="https://coveralls.io/repos/treasure-data/td/badge.svg?branch=master&service=github" alt="Coverage Status" />}[https://coveralls.io/github/treasure-data/td?branch=master]
 
 This CUI utility wraps the {Ruby Client Library td-client-ruby}[https://github.com/treasure-data/td-client-ruby]
 to interact with the REST API in managing databases and jobs on the Treasure Data Cloud.
 
-For more info about Treasure Data, see <http://treasure-data.com/>.
+For more info about Treasure Data, see <https://www.treasuredata.com/>.
 
-For full documentation see <https://support.treasuredata.com/hc/en-us>.
+For full documentation see <https://docs.treasuredata.com/>.
 
 = Getting Started
 
@@ -69,9 +69,9 @@ to process and Bulk load data in the Treasure Data Cloud.
 
 The Bulk Import CLI is downloaded automatically at the first call to any of the command that require it; the use will
 need internet connectivity in order to fetch the Bulk Import CLI JAR file from the
-{Central Maven repository}[http://central.maven.org/maven2/com/treasuredata/td-import/]
+{Central Maven repository}[https://repo1.maven.org/maven2/com/treasuredata/td-import/]
 and take advantage of these advanced features. If you need to setup a proxy, please consult this
-{documentation}[https://support.treasuredata.com/hc/en-us/articles/360001502527-Legacy-Bulk-Import-Tips-and-Tricks#How%20to%20use%20a%20proxy%20server] page.
+{documentation}[https://docs.treasuredata.com/display/public/INT/Legacy+Bulk+Import+Tips+and+Tricks#LegacyBulkImportTipsandTricks-UsingaProxyServer] page.
 
 The log levels and properties of the Bulk Import CLI can be configured in a +logging.properties+ file. A default
 configuration is provided in a file within the gem or Toolbelt folder root, in the +java/+ folder. If you wish to
@@ -144,9 +144,9 @@ These are the available hooks:
 
     $ TD_TOOLBELT_UPDATE_ROOT="http://toolbelt.treasuredata.com"
 
-* Specify an alternative endpoint to use updating the JAR file (default: http://central.maven.org):
+* Specify an alternative endpoint to use updating the JAR file (default: https://repo1.maven.org):
 
-    $ TD_TOOLBELT_JARUPDATE_ROOT="http://central.maven.org"
+    $ TD_TOOLBELT_JARUPDATE_ROOT="https://repo1.maven.org"
 
 
 = Copyright

data/appveyor.yml

@@ -13,5 +13,5 @@ test_script:
   - bundle exec rake spec
 environment:
   matrix:
-    - ruby_version: "22"
-    - ruby_version: "22-x64"
+    - ruby_version: "26"
+    - ruby_version: "26-x64"

data/lib/td/command/account.rb

@@ -41,7 +41,7 @@ module Command
       end
     end
 
-    $stdout.puts "Enter your Treasure Data credentials. For Google SSO user, please see https://support.treasuredata.com/hc/en-us/articles/360000720048-Treasure-Data-Toolbelt-Command-line-Interface#Google%20SSO%20Users"
+    $stdout.puts "Enter your Treasure Data credentials. For Google SSO user, please see https://docs.treasuredata.com/display/public/PD/Configuring+Authentication+for+TD+Using+the+TD+Toolbelt#ConfiguringAuthenticationforTDUsingtheTDToolbelt-SettingUpGoogleSSOUsers"
     unless user_name
       begin
         $stdout.print "Email: "

data/lib/td/command/common.rb

@@ -47,7 +47,7 @@ module Command
 
     # optional, if not provided a default is used from the ruby client library
     begin
-      if Config.endpoint
+      if !opts[:endpoint] && Config.endpoint
         opts[:endpoint] = Config.endpoint
       end
     rescue ConfigNotFoundError => e
@@ -67,6 +67,15 @@ module Command
     Client.new(apikey, opts)
   end
 
+  def get_import_client
+    import_endpoint = begin
+                        Config.import_endpoint || nil
+                      rescue TreasureData::ConfigNotFoundError
+                        nil
+                      end
+    get_client(endpoint: import_endpoint)
+  end
+
   def get_ssl_client(opts={})
     opts[:ssl] = true
     get_client(opts)
@@ -96,7 +105,7 @@ module Command
       headers = options[:fields]
       csv_opts = {}
       csv_opts[:col_sep] = "\t" if format == 'tsv'
-      CSV.generate('', csv_opts) { |csv|
+      CSV.generate('', **csv_opts) { |csv|
         csv << headers
         rows.each { |row|
           r = []
@@ -357,12 +366,9 @@ EOS
 
   def self.get_http_class
     # use Net::HTTP::Proxy in place of Net::HTTP if a proxy is provided
-    http_proxy = ENV['HTTP_PROXY']
-    if http_proxy
-      http_proxy = (http_proxy =~ /\Ahttp:\/\/(.*)\z/) ? $~[1] : http_proxy
-      host, port = http_proxy.split(':', 2)
-      port = (port ? port.to_i : 80)
-      return Net::HTTP::Proxy(host, port)
+    if ENV['HTTP_PROXY']
+      uri = URI.parse(ENV['HTTP_PROXY'])
+      return Net::HTTP::Proxy(uri.host, uri.port, uri.user, uri.password)
     else
       return Net::HTTP
     end

data/lib/td/command/export.rb

@@ -67,6 +67,9 @@ module Command
     encryption = nil
     assume_role = nil
 
+    # Increase the summary_width value from default 32 into 36, to show the options help message properly.
+    op.summary_width = 36
+
     op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b|
       wait = b
     }

data/lib/td/command/query.rb

@@ -160,6 +160,8 @@ module Command
         rescue TreasureData::NotFoundError => e
         end
       end
+    elsif output
+      $stdout.puts "The output file won't be generated without additional `-w` or `--wait` option."
     end
   end
 

data/lib/td/command/runner.rb

@@ -6,11 +6,12 @@ class Runner
     @config_path = nil
     @apikey = nil
     @endpoint = nil
+    @import_endpoint = nil
     @prog_name = nil
     @insecure = false
   end
 
-  attr_accessor :apikey, :endpoint, :config_path, :prog_name, :insecure
+  attr_accessor :apikey, :endpoint, :import_endpoint, :config_path, :prog_name, :insecure
 
   def run(argv=ARGV)
     require 'td/version'
@@ -47,7 +48,9 @@ Basic commands:
   sched          # create/delete/list schedules that run a query periodically
   schema         # create/delete/modify schemas of tables
   connector      # manage connectors
+  users          # show/create/list users
   workflow       # manage workflows
+  user           # manage users  
 
 Additional commands:
 
@@ -73,6 +76,7 @@ EOF
     config_path = @config_path
     apikey = @apikey
     endpoint = @endpoint
+    import_endpoint = @import_endpoint || @endpoint
     insecure = nil
     $verbose = false
     #$debug = false
@@ -94,6 +98,12 @@ EOF
       endpoint = e
     }
 
+    op.on('--import-endpoint API_IMPORT_SERVER', "specify the URL for API Import server to use (default: https://api-import.treasuredata.com).") { |e|
+      require 'td/command/common'
+      Command.validate_api_endpoint(e)
+      import_endpoint = e
+    }
+
     op.on('--insecure', "Insecure access: disable SSL (enabled by default)") {|b|
       insecure = true
     }
@@ -140,6 +150,10 @@ EOF
         Config.endpoint = endpoint
         Config.cl_endpoint = true
       end
+      if import_endpoint
+        Config.import_endpoint = import_endpoint
+        Config.cl_import_endpoint = true
+      end
       if insecure
         Config.secure = false
       end

data/lib/td/command/table.rb

@@ -1,5 +1,6 @@
 require 'td/helpers'
 require 'td/command/job'
+require 'td/client/api'
 
 module TreasureData
 module Command
@@ -40,7 +41,7 @@ module Command
     if HIVE_RESERVED_KEYWORDS.include?(table_name.upcase)
       $stderr.puts "* WARNING *"
       $stderr.puts "  '#{table_name}' is a reserved keyword in Hive. We recommend renaming the table."
-      $stderr.puts "  For a list of all reserved keywords, see our FAQ: https://support.treasuredata.com/hc/en-us/articles/360009531173/"
+      $stderr.puts "  For a list of all reserved keywords, see our FAQ: https://docs.treasuredata.com/display/public/PD/Hive+Reserved+and+Non-Reserved+Keywords"
     end
 
     client = get_client
@@ -170,7 +171,7 @@ module Command
     if rows.empty?
       if db_name
         if databases.first.permission == :import_only
-          $stderr.puts "Database '#{db_name}' is import only, cannot list or create tables."
+          $stderr.puts "Database '#{db_name}' is import only, cannot list."
         else
           $stderr.puts "Database '#{db_name}' has no tables."
           $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create <db> <table>' to create a table."
@@ -453,17 +454,18 @@ module Command
     import_params[:table_name] = table_name
     import_params[:paths] = paths
 
-    client = get_client
+    api_client = get_client
+    import_client = get_import_client
 
     if auto_create
-      create_database_and_table_if_not_exist(client, db_name, table_name)
+      create_database_and_table_if_not_exist(api_client, db_name, table_name)
     end
 
-    do_table_import(client, import_params)
+    do_table_import(api_client, import_client, import_params)
   end
 
   private
-  def do_table_import(client, import_params)
+  def do_table_import(api_client, import_client, import_params)
     case import_params[:format]
     when 'json', 'msgpack'
       #unless time_key
@@ -488,7 +490,7 @@ module Command
     end
 
     begin
-      db = client.database(import_params[:db_name])
+      db = api_client.database(import_params[:db_name])
     rescue ForbiddenError => e
       $stdout.puts "Warning: database and table validation skipped - #{e.message}"
     else
@@ -521,7 +523,7 @@ module Command
     #require 'thread'
 
     files.zip(import_params[:paths]).each {|file, path|
-      import_log_file(file, path, client, import_params[:db_name], import_params[:table_name], parser)
+      import_log_file(file, path, import_client, import_params[:db_name], import_params[:table_name], parser)
     }
 
     $stdout.puts "done."

data/lib/td/command/workflow.rb

@@ -43,6 +43,9 @@ module TreasureData
           ].join($/) + $/)
           cmd << '-Dio.digdag.standards.td.secrets.enabled=false'
           cmd << "-Dconfig.td.default_endpoint=#{Config.endpoint_domain}"
+          if workflow_endpoint.match(/\.connect\./i)
+            cmd << '-Dclient.http.disable_direct_download=true'
+          end
         else
           # Use the digdag td.conf plugin to configure wf api and apikey.
           env['TREASURE_DATA_CONFIG_PATH'] = Config.path
@@ -243,14 +246,20 @@ EOF
         return false
       end
       if output =~ /openjdk version/ or output =~ /java version/
-        m = output.match(/version "(\d+)\.(\d+)\.(\d+)(?:_(\d+))"/)
+        m = output.match(/version "(\d+)\.(\d+)\.(\d+)(?:_(\d+))?"/)
         if not m or m.size < 4
           return false
         end
-        # Check for at least Java 8. Let digdag itself verify revision.
         major = m[1].to_i
         minor = m[2].to_i
-        if major < 1 or minor < 8
+        # Check for at least Java 8. Let digdag itself verify revision.
+        if major < 1
+          return false
+        elsif major == 1 # suppose the version style: 1.8.10_52 (JDK8 and before)
+          if minor < 8
+            return false
+          end
+        elsif major < 9 # suppose the version style: 11.0.2
           return false
         end
       end

data/lib/td/config.rb

@@ -21,6 +21,9 @@ class Config
   @@endpoint = ENV['TREASURE_DATA_API_SERVER'] || ENV['TD_API_SERVER']
   @@endpoint = nil if @@endpoint == ""
   @@cl_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line
+  @@import_endpoint = ENV['TREASURE_DATA_API_IMPORT_SERVER'] || ENV['TD_API_IMPORT_SERVER']
+  @@import_endpoint = nil if @@endpoint == ""
+  @@cl_import_endpoint = false # flag to indicate whether an endpoint has been provided through the command-line option
   @@secure = true
   @@retry_post_requests = false
 
@@ -145,7 +148,8 @@ class Config
 
 
   def self.endpoint
-    @@endpoint || Config.read['account.endpoint']
+    endpoint = @@endpoint || Config.read['account.endpoint']
+    endpoint.sub(/(\/)+$/, '') if endpoint
   end
 
   def self.endpoint=(endpoint)
@@ -164,9 +168,26 @@ class Config
     @@cl_endpoint = flag
   end
 
+  def self.import_endpoint
+    endpoint = @@import_endpoint || Config.read['account.import_endpoint']
+    endpoint.sub(/(\/)+$/, '') if endpoint
+  end
+
+  def self.import_endpoint=(endpoint)
+    @@import_endpoint = endpoint
+  end
+
+  def self.cl_import_endpoint
+    @@cl_import_endpoint
+  end
+
+  def self.cl_import_endpoint=(flag)
+    @@cl_import_endpoint = flag
+  end
+
   def self.workflow_endpoint
     case self.endpoint_domain
-    when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?(eu01\.)?treasuredata\.(com|co\.jp)\z/i
+    when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?((?:eu01|ap02|ap03)\.)?treasuredata\.(com|co\.jp)\z/i
       "https://api#{$1}-workflow#{$2}.#{$3}#{$4}treasuredata.#{$5}"
     else
       raise ConfigError, "Workflow is not supported for '#{self.endpoint}'"
@@ -176,10 +197,9 @@ class Config
   # renders the apikey and endpoint options as a string for the helper commands
   def self.cl_options_string
     string = ""
-    string += "-k #{@@apikey}" if @@cl_apikey
-    string += " " unless string.empty?
-    string += "-e #{@@endpoint}" if @@cl_endpoint
-    string += " " unless string.empty?
+    string += "-k #{@@apikey} " if @@cl_apikey
+    string += "-e #{@@endpoint} " if @@cl_endpoint
+    string += "--import-endpoint #{@@import_endpoint} " if @@cl_import_endpoint
     string
   end
 

data/lib/td/file_reader.rb

@@ -96,10 +96,10 @@ module TreasureData
         csv_opts[:quote_char] = opts[:quote_char] if opts[:quote_char]
         begin
           require 'fastercsv'
-          @io = FasterCSV.new(io, csv_opts)
+          @io = FasterCSV.new(io, **csv_opts)
         rescue LoadError => e
           require 'csv'
-          @io = CSV.new(io, csv_opts)
+          @io = CSV.new(io, **csv_opts)
         end
         @error = error
         # @escape_char = opts[:escape_char]

data/lib/td/updater.rb

@@ -270,7 +270,7 @@ end # module ModuleDefinition
   extend ModuleDefinition
 
   def maven_repo_root
-    ENV['TD_TOOLBELT_JARUPDATE_ROOT'] || "http://central.maven.org"
+    ENV['TD_TOOLBELT_JARUPDATE_ROOT'] || "https://repo1.maven.org"
   end
 
   private

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.16.6'
+  TOOLBELT_VERSION = '0.16.10'
 end

data/spec/td/command/table_spec.rb

@@ -302,6 +302,7 @@ module TreasureData::Command
       let(:db_name)    { 'database' }
       let(:table_name) { 'table' }
       let(:client)     { double('client') }
+      let(:import_client)     { double('import-client') }
       let(:command)    { Class.new { include TreasureData::Command }.new }
 
       describe 'auto create table' do
@@ -336,6 +337,7 @@ module TreasureData::Command
       describe 'time key' do
         before do
           allow(command).to receive(:get_client) { client }
+          allow(command).to receive(:get_import_client) { import_client }
           allow(command).to receive(:do_table_import)
         end
         let(:input_params) {{
@@ -355,7 +357,7 @@ module TreasureData::Command
             end
 
             it "with '#{tk_option}' option" do
-              expect(command).to receive(:do_table_import).with(client, input_params)
+              expect(command).to receive(:do_table_import).with(client, import_client, input_params)
               command.table_import(option)
             end
           end
@@ -371,7 +373,7 @@ module TreasureData::Command
           end
 
           it 'without \'-t / --time-key\' option' do
-            expect(command).to receive(:do_table_import).with(client, input_params)
+            expect(command).to receive(:do_table_import).with(client, import_client, input_params)
             command.table_import(option)
           end
         end

data/spec/td/command/workflow_spec.rb

@@ -335,6 +335,7 @@ EOF
     let(:config_path){ nil }
     let(:apikey){ nil }
     let(:endpoint){ nil }
+    let(:command_args){ Hash.new }
     before do
       # TreasureData::Command::Runner#run
       if config_path
@@ -348,6 +349,7 @@ EOF
         TreasureData::Config.endpoint = endpoint
         TreasureData::Config.cl_endpoint = true
       end
+      command_args.clear
 
       allow(Kernel).to receive(:system) do |env, *cmd|
         digdag_env.replace env
@@ -365,6 +367,7 @@ EOF
             args[nil] << x
           end
         end
+        args.each { |key, value| command_args[key] = value }
         if args['--config']
           File.foreach(args['--config']) do |line|
             k, v = line.strip.split(/\s*=\s*/, 2)
@@ -402,6 +405,16 @@ EOF
         expect(workflow_config['client.http.headers.authorization']).to eq "TD1 #{apikey}"
         expect(workflow_config['secrets.td.apikey']).to eq apikey
         expect(digdag_env['TREASURE_DATA_WORKFLOW_ENDPOINT']).to eq 'https://api-workflow.treasuredata.co.jp'
+        expect(command_args['-Dclient.http.disable_direct_download']).to be_nil
+      end
+    end
+    context 'endpoint: https://api-hoge.connect.treasuredata.com' do
+      let(:apikey){ '1/deadbeaf' }
+      let(:endpoint){ 'https://api-hoge.connect.treasuredata.com' }
+      it 'add arg to disable direct download workflow logs from Private Connect' do
+        op = List::CommandParser.new("workflow", [], [], nil, ['log', '8000111'], true)
+        command.workflow(op, false, false)
+        expect(command_args['-Dclient.http.disable_direct_download']).to be_truthy
       end
     end
   end

data/spec/td/config_spec.rb

@@ -28,6 +28,14 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api.eu01.treasuredata.com' }
       it { is_expected.to eq 'https://api-workflow.eu01.treasuredata.com' }
     end
+    context 'api.ap02.treasuredata.com' do
+      let(:api_endpoint){ 'api.ap02.treasuredata.com' }
+      it { is_expected.to eq 'https://api-workflow.ap02.treasuredata.com' }
+    end
+    context 'api.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-workflow.ap03.treasuredata.com' }
+    end
 
     context 'api-hoge.connect.treasuredata.com' do
       let(:api_endpoint){ 'api-hoge.connect.treasuredata.com' }
@@ -50,6 +58,14 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api-staging.eu01.treasuredata.com' }
       it { is_expected.to eq 'https://api-staging-workflow.eu01.treasuredata.com' }
     end
+    context 'api-staging.ap02.treasuredata.com' do
+      let(:api_endpoint){ 'api-staging.ap02.treasuredata.com' }
+      it { is_expected.to eq 'https://api-staging-workflow.ap02.treasuredata.com' }
+    end
+    context 'api-staging.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api-staging.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-staging-workflow.ap03.treasuredata.com' }
+    end
 
     context 'api-development.treasuredata.com' do
       let(:api_endpoint){ 'api-development.treasuredata.com' }
@@ -63,6 +79,14 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api-development.eu01.treasuredata.com' }
       it { is_expected.to eq 'https://api-development-workflow.eu01.treasuredata.com' }
     end
+    context 'api-development.ap02.treasuredata.com' do
+      let(:api_endpoint){ 'api-development.ap02.treasuredata.com' }
+      it { is_expected.to eq 'https://api-development-workflow.ap02.treasuredata.com' }
+    end
+    context 'api-development.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api-development.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-development-workflow.ap03.treasuredata.com' }
+    end
 
     context 'ybi.jp-east.idcfcloud.com' do
       let(:api_endpoint){ 'ybi.jp-east.idcfcloud.com' }
@@ -72,6 +96,64 @@ describe TreasureData::Config do
     end
   end
 
+  describe 'allow endpoint with trailing slash' do
+    context 'self.endpoint' do
+      before { TreasureData::Config.endpoint = api_endpoint }
+      subject { TreasureData::Config.endpoint }
+
+      context 'api.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.treasuredata.com' }
+      end
+      context 'api.treasuredata.co.jp' do
+        let(:api_endpoint) { 'https://api.treasuredata.co.jp/' }
+        it { is_expected.to eq 'https://api.treasuredata.co.jp' }
+      end
+      context 'api.eu01.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.eu01.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.eu01.treasuredata.com' }
+      end
+      context 'api.ap02.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.ap02.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.ap02.treasuredata.com' }
+      end
+      context 'api.ap03.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.ap03.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.ap03.treasuredata.com' }
+      end
+      context 'api-hoge.connect.treasuredata.com' do
+        let(:api_endpoint){ 'https://api-hoge.connect.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-hoge.connect.treasuredata.com' }
+      end
+    end
+
+    context 'self.import_endpoint' do
+      before { TreasureData::Config.import_endpoint = api_endpoint }
+      subject { TreasureData::Config.import_endpoint }
+
+      context 'api-import.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.treasuredata.com' }
+      end
+      context 'api-import.treasuredata.co.jp' do
+        let(:api_endpoint) { 'https://api-import.treasuredata.co.jp/' }
+        it { is_expected.to eq 'https://api-import.treasuredata.co.jp' }
+      end
+      context 'api-import.eu01.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.eu01.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.eu01.treasuredata.com' }
+      end
+      context 'api-import.ap02.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.ap02.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.ap02.treasuredata.com' }
+      end
+      context 'api-import.ap03.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.ap03.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.ap03.treasuredata.com' }
+      end
+    end
+  end
+
   describe '#read' do
     it 'sets @conf' do
       Tempfile.create('td.conf') do |f|

data/td.gemspec

@@ -15,18 +15,21 @@ Gem::Specification.new do |gem|
   gem.executables           = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths         = ['lib']
   gem.required_ruby_version = '>= 2.1'
+  gem.licenses              = ["Apache-2.0"]
 
   gem.add_dependency "msgpack"
-  gem.add_dependency "yajl-ruby", "~> 1.1"
+  gem.add_dependency "rexml"
+  gem.add_dependency "yajl-ruby", "~> 1.3.1"
   gem.add_dependency "hirb", ">= 0.4.5"
-  gem.add_dependency "parallel", "~> 1.8"
-  gem.add_dependency "td-client", ">= 1.0.6", "< 2"
+  gem.add_dependency "parallel", "~> 1.20.0"
+  gem.add_dependency "td-client", ">= 1.0.8", "< 2"
   gem.add_dependency "td-logger", ">= 0.3.21", "< 2"
-  gem.add_dependency "rubyzip", ">= 1.3.0"
+  gem.add_dependency "rubyzip", "~> 1.3.0"
   gem.add_dependency "zip-zip", "~> 0.3"
   gem.add_dependency "ruby-progressbar", "~> 1.7"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec"
+  gem.add_development_dependency 'webrick'
   gem.add_development_dependency "simplecov"
   gem.add_development_dependency 'coveralls'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.16.6
+  version: 0.16.10
 platform: ruby
 authors:
 - Treasure Data, Inc.
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-13 00:00:00.000000000 Z
+date: 2021-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -24,20 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.3.1
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
@@ -58,21 +72,21 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 1.20.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 1.20.0
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.6
+        version: 1.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -82,7 +96,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.6
+        version: 1.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -110,14 +124,14 @@ dependencies:
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.3.0
 - !ruby/object:Gem::Dependency
@@ -176,6 +190,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -211,9 +239,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".coveralls.yml"
+- ".github/CODEOWNERS"
 - ".gitignore"
-- ".travis.yml"
 - ChangeLog
 - Gemfile
 - README.rdoc
@@ -291,9 +320,10 @@ files:
 - spec/td/version_spec.rb
 - td.gemspec
 homepage: http://treasure-data.com/
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -308,8 +338,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:

data/.travis.yml

@@ -1,26 +0,0 @@
-language: ruby
-
-jdk:
-  - openjdk8
-
-rvm:
-  - 2.1.10
-  - 2.2.10
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-  - ruby-head
-
-gemfile:
-  - Gemfile
-
-script: bundle exec rake spec SPEC_OPTS="-fd"
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head
-
-sudo: false
-
-notifications:
-  webhooks: http://td-beda.herokuapp.com/travisci_callback