td 0.16.9 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6aea3ef4a96f7d2d1042cbf337e5f95288022f39
-  data.tar.gz: 7ab665de173e7ce66dfa7b4622cb4513dcef6426
+SHA256:
+  metadata.gz: 39aaf12ff04af9701e21625ab329b422a890edfe295f450ee3b36e7cdbc3e94b
+  data.tar.gz: c3d8a021e9b4c0b2c599882ee4ca109c691f81d2d2777e085ffcff1326628876
 SHA512:
-  metadata.gz: 114f5e1e8763474cb53cdead4f511dd30a988457c8d72e0115fe20f4a013aae7fdbee3c13e8a627381145e05db4d5bf584690bdf863bc9d6fe35bbae66be9b35
-  data.tar.gz: 61f72c29b14c70627680248852303d878a92ea571e140c4e5c60318944248367f6509df81b1c19553e4a271ee8ce4f9bc54747610f8263f7ef4cc3fa6d053581
+  metadata.gz: 55d1ffecf734507573af5d49c8e3f6a70bf9f4859465722ed9a05f07ab405660e6e4c6c395404699ea9b09a426cfe2fb15f2f59cbe0363349b293f29c70a6566
+  data.tar.gz: 36d7f0ddec937b81ed7a7f08e5f9e88638e96e1545af529b42eaae11152ce51459408363cbf96d6be7b53cf0ce9829d75a881b7d7e677a33a8fa18080200cf46

data/.circleci/config.yml

@@ -0,0 +1,88 @@
+version: 2.1
+
+
+orbs:
+  ruby: circleci/ruby@2.0.0
+  win: circleci/windows@5.0.0
+
+
+commands:
+  install_windows_requirements:
+    description: "Install windows requirements"
+    steps:
+      - run:
+          name: "Install MSYS2"
+          command: choco install msys2 -y
+      - run:
+          name: "Install Ruby devkit"
+          command: ridk install 2 3
+  bundle-install:
+    description: "Install dependencies"
+    steps:
+      - run:
+          name: Which bundler?
+          command: ruby -v; bundle -v
+      - run:
+          name: Bundle install
+          command: bundle install
+  run-tests:
+    description: "Run tests"
+    steps:
+      - run:
+          name: Run tests
+          command: bundle exec rake spec SPEC_OPTS="-fd"
+  run-tests-flow:
+    description: "Single flow for running tests"
+    steps:
+      - checkout
+      - bundle-install
+      - run-tests
+
+
+jobs:
+
+  ruby_27:
+    docker:
+      - image: cimg/ruby:2.7-browsers
+    steps:
+      - run-tests-flow
+
+  ruby_30:
+    docker:
+      - image: cimg/ruby:3.0-browsers
+    steps:
+      - run-tests-flow
+
+  ruby_31:
+    docker:
+      - image: cimg/ruby:3.1-browsers
+    steps:
+      - run-tests-flow
+
+  ruby_32:
+    docker:
+      - image: cimg/ruby:3.2-browsers
+    steps:
+      - run-tests-flow
+
+  win_ruby:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    steps:
+      - install_windows_requirements
+      - run:
+          name: "Install bundler"
+          shell: powershell.exe
+          command: gem install bundler
+      - run-tests-flow
+
+
+workflows:
+  tests:
+    jobs:
+      - ruby_27
+      - ruby_30
+      - ruby_31
+      - ruby_32
+      - win_ruby

data/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @treasure-data/integrations

data/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '29 4 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'ruby' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

data/.gitignore

@@ -1,9 +1,11 @@
 .bundle
 .DS_Store
+bundler
 build/td-import-java
 Gemfile.lock
 vendor/*
 /pkg/
 build/*
 coverage/
+dist/resources/pkg/ruby*.pkg
 *~

data/ChangeLog

@@ -1,3 +1,18 @@
+== 2023-03-28 version 0.17.0
+
+* Show job's start & end time for td job:show [job_id] #253
+* Support Ruby 3.2 by updating deprecated File.exists? to File.exist? #260
+
+== 2020-09-16 version 0.16.10
+
+* Support proxy with username and password
+* Increase the summary_width value to fix the broken options help message format
+* Support AP03 Region
+* Change old document urls to new document urls
+* Show message to ask -w option in td query
+* Adding users commnad help
+* Allow trailing slash for endpoints
+
 == 2020-04-10 version 0.16.9
 
 * Update new workflow pattern to support ap02 #232

data/README.rdoc

@@ -1,14 +1,14 @@
 = Treasure Data command line tool
-{<img src="https://travis-ci.org/treasure-data/td.svg?branch=master" alt="Build Status" />}[https://travis-ci.org/treasure-data/td]
+{<img src="https://circleci.com/gh/treasure-data/td.svg?style=svg" alt="Build Status" />}[https://circleci.com/gh/treasure-data/td.svg?style=svg]
 {<img src="https://ci.appveyor.com/api/projects/status/github/treasure-data/td?branch=master&svg=true" alt="appveyor" />}[https://ci.appveyor.com/project/treasure-data/td/branch/master]
 {<img src="https://coveralls.io/repos/treasure-data/td/badge.svg?branch=master&service=github" alt="Coverage Status" />}[https://coveralls.io/github/treasure-data/td?branch=master]
 
 This CUI utility wraps the {Ruby Client Library td-client-ruby}[https://github.com/treasure-data/td-client-ruby]
 to interact with the REST API in managing databases and jobs on the Treasure Data Cloud.
 
-For more info about Treasure Data, see <http://treasure-data.com/>.
+For more info about Treasure Data, see <https://www.treasuredata.com/>.
 
-For full documentation see <https://support.treasuredata.com/hc/en-us>.
+For full documentation see <https://docs.treasuredata.com/>.
 
 = Getting Started
 
@@ -71,7 +71,7 @@ The Bulk Import CLI is downloaded automatically at the first call to any of the
 need internet connectivity in order to fetch the Bulk Import CLI JAR file from the
 {Central Maven repository}[https://repo1.maven.org/maven2/com/treasuredata/td-import/]
 and take advantage of these advanced features. If you need to setup a proxy, please consult this
-{documentation}[https://support.treasuredata.com/hc/en-us/articles/360001502527-Legacy-Bulk-Import-Tips-and-Tricks#How%20to%20use%20a%20proxy%20server] page.
+{documentation}[https://docs.treasuredata.com/display/public/INT/Legacy+Bulk+Import+Tips+and+Tricks#LegacyBulkImportTipsandTricks-UsingaProxyServer] page.
 
 The log levels and properties of the Bulk Import CLI can be configured in a +logging.properties+ file. A default
 configuration is provided in a file within the gem or Toolbelt folder root, in the +java/+ folder. If you wish to

data/Rakefile

@@ -50,9 +50,12 @@ def install_use_gems(target_dir)
         puts "Copy local gem #{ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']} to #{Dir.pwd}"
         FileUtils.cp File.expand_path(ENV['TD_TOOLBELT_LOCAL_CLIENT_GEM']), Dir.pwd
       end
-      Gem::GemRunner.new.run ["install", gem, "--no-rdoc", "--no-ri"]
+      Gem::GemRunner.new.run ["install", gem, "--no-document"] # Use one of two commands bellow. Gem::GemRunner on ruby 3.0.5 has some weird bug
+      # sh "gem install \"#{gem}" --no-document"
+      # sh "/usr/local/td/ruby/bin/gem install \"#{gem}\" --no-document" # On OSX
     rescue Gem::SystemExitException => e
       unless e.exit_code.zero?
+        puts e, e.backtrace.join("\n")
         raise e
       end
     end

data/appveyor.yml

@@ -13,5 +13,5 @@ test_script:
   - bundle exec rake spec
 environment:
   matrix:
-    - ruby_version: "22"
-    - ruby_version: "22-x64"
+    - ruby_version: "26"
+    - ruby_version: "26-x64"

data/lib/td/command/account.rb

@@ -41,7 +41,7 @@ module Command
       end
     end
 
-    $stdout.puts "Enter your Treasure Data credentials. For Google SSO user, please see https://support.treasuredata.com/hc/en-us/articles/360000720048-Treasure-Data-Toolbelt-Command-line-Interface#Google%20SSO%20Users"
+    $stdout.puts "Enter your Treasure Data credentials. For Google SSO user, please see https://docs.treasuredata.com/display/public/PD/Configuring+Authentication+for+TD+Using+the+TD+Toolbelt#ConfiguringAuthenticationforTDUsingtheTDToolbelt-SettingUpGoogleSSOUsers"
     unless user_name
       begin
         $stdout.print "Email: "

data/lib/td/command/common.rb

@@ -67,13 +67,11 @@ module Command
     Client.new(apikey, opts)
   end
 
-  DEFAULT_IMPORT_ENDPOINT = "https://" + TreasureData::API::DEFAULT_IMPORT_ENDPOINT
-
   def get_import_client
     import_endpoint = begin
-                        Config.import_endpoint || DEFAULT_IMPORT_ENDPOINT
+                        Config.import_endpoint || nil
                       rescue TreasureData::ConfigNotFoundError
-                        DEFAULT_IMPORT_ENDPOINT
+                        nil
                       end
     get_client(endpoint: import_endpoint)
   end
@@ -107,7 +105,7 @@ module Command
       headers = options[:fields]
       csv_opts = {}
       csv_opts[:col_sep] = "\t" if format == 'tsv'
-      CSV.generate('', csv_opts) { |csv|
+      CSV.generate('', **csv_opts) { |csv|
         csv << headers
         rows.each { |row|
           r = []
@@ -368,12 +366,9 @@ EOS
 
   def self.get_http_class
     # use Net::HTTP::Proxy in place of Net::HTTP if a proxy is provided
-    http_proxy = ENV['HTTP_PROXY']
-    if http_proxy
-      http_proxy = (http_proxy =~ /\Ahttp:\/\/(.*)\z/) ? $~[1] : http_proxy
-      host, port = http_proxy.split(':', 2)
-      port = (port ? port.to_i : 80)
-      return Net::HTTP::Proxy(host, port)
+    if ENV['HTTP_PROXY']
+      uri = URI.parse(ENV['HTTP_PROXY'])
+      return Net::HTTP::Proxy(uri.host, uri.port, uri.user, uri.password)
     else
       return Net::HTTP
     end

data/lib/td/command/export.rb

@@ -67,6 +67,9 @@ module Command
     encryption = nil
     assume_role = nil
 
+    # Increase the summary_width value from default 32 into 36, to show the options help message properly.
+    op.summary_width = 36
+
     op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b|
       wait = b
     }

data/lib/td/command/job.rb

@@ -173,6 +173,8 @@ private
     $stdout.puts "Status      : #{job.status}"
     $stdout.puts "Type        : #{job.type}"
     $stdout.puts "Database    : #{job.db_name}"
+    $stdout.puts "Start At    : #{job.start_at}"
+    $stdout.puts "End At      : #{job.end_at}"
     # exclude some fields from bulk_import_perform type jobs
     if [:hive, :pig, :impala, :presto].include?(job.type)
       $stdout.puts "Priority    : #{job_priority_name_of(job.priority)}"
@@ -404,7 +406,7 @@ private
       raise "Unknown format #{format.inspect}"
     end
 
-    if tempfile && File.exists?(tempfile)
+    if tempfile && File.exist?(tempfile)
       FileUtils.mv(tempfile, output.respond_to?(:path) ? output.path : output)
     end
   end

data/lib/td/command/query.rb

@@ -160,6 +160,8 @@ module Command
         rescue TreasureData::NotFoundError => e
         end
       end
+    elsif output
+      $stdout.puts "The output file won't be generated without additional `-w` or `--wait` option."
     end
   end
 

data/lib/td/command/runner.rb

@@ -48,7 +48,9 @@ Basic commands:
   sched          # create/delete/list schedules that run a query periodically
   schema         # create/delete/modify schemas of tables
   connector      # manage connectors
+  users          # show/create/list users
   workflow       # manage workflows
+  user           # manage users  
 
 Additional commands:
 

data/lib/td/command/table.rb

@@ -41,7 +41,7 @@ module Command
     if HIVE_RESERVED_KEYWORDS.include?(table_name.upcase)
       $stderr.puts "* WARNING *"
       $stderr.puts "  '#{table_name}' is a reserved keyword in Hive. We recommend renaming the table."
-      $stderr.puts "  For a list of all reserved keywords, see our FAQ: https://support.treasuredata.com/hc/en-us/articles/360009531173/"
+      $stderr.puts "  For a list of all reserved keywords, see our FAQ: https://docs.treasuredata.com/display/public/PD/Hive+Reserved+and+Non-Reserved+Keywords"
     end
 
     client = get_client
@@ -171,7 +171,7 @@ module Command
     if rows.empty?
       if db_name
         if databases.first.permission == :import_only
-          $stderr.puts "Database '#{db_name}' is import only, cannot list or create tables."
+          $stderr.puts "Database '#{db_name}' is import only, cannot list."
         else
           $stderr.puts "Database '#{db_name}' has no tables."
           $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create <db> <table>' to create a table."

data/lib/td/command/workflow.rb

@@ -96,7 +96,7 @@ module TreasureData
     def workflow_version(op)
       op.cmd_parse # to show help
 
-      unless File.exists?(digdag_cli_path)
+      unless File.exist?(digdag_cli_path)
         $stderr.puts('Workflow module not yet installed.')
         return 1
       end
@@ -246,7 +246,7 @@ EOF
         return false
       end
       if output =~ /openjdk version/ or output =~ /java version/
-        m = output.match(/version "(\d+)\.(\d+)\.(\d+)(?:_(\d+))?"/)
+        m = output.match(/version "(\d+)\.(\d+)\.(\d+)(\.\d+)?(?:_(\d+))?"/)
         if not m or m.size < 4
           return false
         end
@@ -309,7 +309,7 @@ EOF
 
     private
     def download_java
-      if File.exists?(digdag_jre_dir)
+      if File.exist?(digdag_jre_dir)
         return
       end
 
@@ -427,7 +427,7 @@ EOF
     def check_digdag_cli
       check_system_java unless bundled_java?
 
-      unless File.exists?(digdag_cli_path)
+      unless File.exist?(digdag_cli_path)
         $stderr.puts 'Workflow module not yet installed, download now? [Y/n]'
         line = $stdin.gets
         line.strip!

data/lib/td/config.rb

@@ -148,7 +148,8 @@ class Config
 
 
   def self.endpoint
-    @@endpoint || Config.read['account.endpoint']
+    endpoint = @@endpoint || Config.read['account.endpoint']
+    endpoint.sub(/(\/)+$/, '') if endpoint
   end
 
   def self.endpoint=(endpoint)
@@ -168,7 +169,8 @@ class Config
   end
 
   def self.import_endpoint
-    @@import_endpoint || Config.read['account.import_endpoint']
+    endpoint = @@import_endpoint || Config.read['account.import_endpoint']
+    endpoint.sub(/(\/)+$/, '') if endpoint
   end
 
   def self.import_endpoint=(endpoint)
@@ -185,7 +187,7 @@ class Config
 
   def self.workflow_endpoint
     case self.endpoint_domain
-    when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?((?:eu01|ap02)\.)?treasuredata\.(com|co\.jp)\z/i
+    when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?((?:eu01|ap02|ap03)\.)?treasuredata\.(com|co\.jp)\z/i
       "https://api#{$1}-workflow#{$2}.#{$3}#{$4}treasuredata.#{$5}"
     else
       raise ConfigError, "Workflow is not supported for '#{self.endpoint}'"

data/lib/td/file_reader.rb

@@ -96,10 +96,10 @@ module TreasureData
         csv_opts[:quote_char] = opts[:quote_char] if opts[:quote_char]
         begin
           require 'fastercsv'
-          @io = FasterCSV.new(io, csv_opts)
+          @io = FasterCSV.new(io, **csv_opts)
         rescue LoadError => e
           require 'csv'
-          @io = CSV.new(io, csv_opts)
+          @io = CSV.new(io, **csv_opts)
         end
         @error = error
         # @escape_char = opts[:escape_char]

data/lib/td/helpers.rb

@@ -11,7 +11,7 @@ module TreasureData
     end
 
     def on_windows?
-      RUBY_PLATFORM =~ /mswin32|mingw32/
+      RUBY_PLATFORM =~ /mswin32|mingw32|mingw-ucrt/
     end
 
     def on_mac?

data/lib/td/updater.rb

@@ -21,7 +21,7 @@ module ModuleDefinition
   end
 
   def on_windows?
-    RUBY_PLATFORM =~ /mswin32|mingw32/
+    RUBY_PLATFORM =~ /mswin32|mingw32|mingw-ucrt/
   end
 
   def on_mac?
@@ -82,7 +82,7 @@ module ModuleDefinition
 
   def wait_for_lock(path, wait_for = 5, check_every = 0.5)
     start = Time.now.to_i
-    while File.exists?(path)
+    while File.exist?(path)
       sleep check_every
       if (Time.now.to_i - start) > wait_for
         raise_error "Unable to acquire update lock"
@@ -142,14 +142,15 @@ module ModuleDefinition
   end
 
   def version_endpoint
-    "#{endpoint_root}/latest_version_0_16.#{package_category}"
+    "#{endpoint_root}/latest_version_0_17.#{package_category}"
   end
 
   def update_package_endpoint
-    "#{endpoint_root}/td-update-#{package_category}-0-16.zip"
+    "#{endpoint_root}/td-update-#{package_category}-0-17.zip"
   end
 
   def update(autoupdate = false)
+    FileUtils.mkdir_p File.dirname(updating_lock_path)
     wait_for_lock(updating_lock_path, 5) do
       require "td"
       require 'open-uri'
@@ -305,7 +306,7 @@ end # module ModuleDefinition
     last_updated = existent_jar_updated_time
 
     if updated > last_updated
-      FileUtils.mkdir_p(Updater.jarfile_dest_path) unless File.exists?(Updater.jarfile_dest_path)
+      FileUtils.mkdir_p(Updater.jarfile_dest_path) unless File.exist?(Updater.jarfile_dest_path)
       Dir.chdir(Updater.jarfile_dest_path) do
         status = nil
         indicator = Command::TimeBasedDownloadProgressIndicator.new(
@@ -334,7 +335,7 @@ end # module ModuleDefinition
         else
           $stdout.puts "Update of td-import.jar failed."
           $stdout.puts "Please execute 'td import:jar_update' later to update td-import.jar correctly."
-          File.delete 'td-import.jar.new' if File.exists? 'td-import.jar.new'
+          File.delete 'td-import.jar.new' if File.exist? 'td-import.jar.new'
         end
       end
     else
@@ -359,7 +360,7 @@ end # module ModuleDefinition
         end
       end
 
-      if File.exists?(last_jar_autoupdate_timestamp) && \
+      if File.exist?(last_jar_autoupdate_timestamp) && \
        (Time.now - File.mtime(last_jar_autoupdate_timestamp)).to_i < (60 * 60 * 1) # every hour
         return
       end

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.16.9'
+  TOOLBELT_VERSION = '0.17.0'
 end

data/spec/td/command/job_spec.rb

@@ -271,6 +271,8 @@ text
 
     describe '#job_show' do
       let(:job_id) { "12345" }
+      let(:start_at) { Time.now.strftime('%Y-%m-%d %H:%M:%S UTC') }
+      let(:end_at) { (Time.now + 60).strftime('%Y-%m-%d %H:%M:%S UTC') }
 
       let :job_classs do
         Struct.new(:job_id,
@@ -278,6 +280,8 @@ text
                    :type,
                    :db_name,
                    :priority,
+                   :start_at,
+                   :end_at,
                    :retry_limit,
                    :result_url,
                    :query,
@@ -292,6 +296,8 @@ text
                        :hive,
                        "db_name",
                        1,
+                       start_at,
+                       end_at,
                        1,
                        "test_url",
                        "test_qury",
@@ -410,7 +416,7 @@ ACTUAL
       end
 
       let(:line_separator) {
-        if RUBY_PLATFORM =~ /mswin32|mingw32/
+        if RUBY_PLATFORM =~ /mswin32|mingw32|mingw-ucrt/
           "\r\n"
         else
           "\n"

data/spec/td/config_spec.rb

@@ -32,6 +32,10 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api.ap02.treasuredata.com' }
       it { is_expected.to eq 'https://api-workflow.ap02.treasuredata.com' }
     end
+    context 'api.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-workflow.ap03.treasuredata.com' }
+    end
 
     context 'api-hoge.connect.treasuredata.com' do
       let(:api_endpoint){ 'api-hoge.connect.treasuredata.com' }
@@ -58,6 +62,10 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api-staging.ap02.treasuredata.com' }
       it { is_expected.to eq 'https://api-staging-workflow.ap02.treasuredata.com' }
     end
+    context 'api-staging.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api-staging.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-staging-workflow.ap03.treasuredata.com' }
+    end
 
     context 'api-development.treasuredata.com' do
       let(:api_endpoint){ 'api-development.treasuredata.com' }
@@ -75,6 +83,10 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api-development.ap02.treasuredata.com' }
       it { is_expected.to eq 'https://api-development-workflow.ap02.treasuredata.com' }
     end
+    context 'api-development.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api-development.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-development-workflow.ap03.treasuredata.com' }
+    end
 
     context 'ybi.jp-east.idcfcloud.com' do
       let(:api_endpoint){ 'ybi.jp-east.idcfcloud.com' }
@@ -84,6 +96,64 @@ describe TreasureData::Config do
     end
   end
 
+  describe 'allow endpoint with trailing slash' do
+    context 'self.endpoint' do
+      before { TreasureData::Config.endpoint = api_endpoint }
+      subject { TreasureData::Config.endpoint }
+
+      context 'api.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.treasuredata.com' }
+      end
+      context 'api.treasuredata.co.jp' do
+        let(:api_endpoint) { 'https://api.treasuredata.co.jp/' }
+        it { is_expected.to eq 'https://api.treasuredata.co.jp' }
+      end
+      context 'api.eu01.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.eu01.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.eu01.treasuredata.com' }
+      end
+      context 'api.ap02.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.ap02.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.ap02.treasuredata.com' }
+      end
+      context 'api.ap03.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.ap03.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.ap03.treasuredata.com' }
+      end
+      context 'api-hoge.connect.treasuredata.com' do
+        let(:api_endpoint){ 'https://api-hoge.connect.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-hoge.connect.treasuredata.com' }
+      end
+    end
+
+    context 'self.import_endpoint' do
+      before { TreasureData::Config.import_endpoint = api_endpoint }
+      subject { TreasureData::Config.import_endpoint }
+
+      context 'api-import.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.treasuredata.com' }
+      end
+      context 'api-import.treasuredata.co.jp' do
+        let(:api_endpoint) { 'https://api-import.treasuredata.co.jp/' }
+        it { is_expected.to eq 'https://api-import.treasuredata.co.jp' }
+      end
+      context 'api-import.eu01.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.eu01.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.eu01.treasuredata.com' }
+      end
+      context 'api-import.ap02.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.ap02.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.ap02.treasuredata.com' }
+      end
+      context 'api-import.ap03.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.ap03.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.ap03.treasuredata.com' }
+      end
+    end
+  end
+
   describe '#read' do
     it 'sets @conf' do
       Tempfile.create('td.conf') do |f|

data/spec/td/fixture/testRootCA.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDBDCCAewCCQCQdq0wStTtITANBgkqhkiG9w0BAQsFADBEMR0wGwYDVQQDDBR3
+d3cudHJlYXN1cmVkYXRhLmNvbTELMAkGA1UEBhMCVVMxFjAUBgNVBAcMDVNhbiBG
+cmFuc2lzY28wHhcNMjIxMDE0MDQzNjA0WhcNMzIwNzEzMDQzNjA0WjBEMR0wGwYD
+VQQDDBR3d3cudHJlYXN1cmVkYXRhLmNvbTELMAkGA1UEBhMCVVMxFjAUBgNVBAcM
+DVNhbiBGcmFuc2lzY28wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCU
+4RbAD5pop+AUYyJT0L4r/ZwOFz72TqQwyPh8EbpPlkvErWPP07kLO1gmfPHPH6zE
+4sAd8Pl5HZ29Z3fF9zO4ZaOehO+kMHp3jp1nq6vURzMjxDj42CyP+M7DOLrov98m
+uwcZMdVjJlz9ZWt7P4x/F5cyqnzmFYxefi0xbgFFXKaWL1W3Fmq1dKMCgbCVWxlA
+cufRFQmmO+OOcKfJTpcZZXH+VKai81OLjjoBkEERqZU8ySv2zR3gLBMAU3x/VL6/
+AxcAi6ZbxQ29O28VTojQ44dA0op5wDtbI3reH9cMgALI1Swo2DyxXejYoZJYKqk3
+AH2BiM0dbPnzOoufxjm/AgMBAAEwDQYJKoZIhvcNAQELBQADggEBACLeA1a/TU4E
+8bKVEr8X9CMROG2j5lsmPLZRTDUQT50iu5MBZzCNQi2tzUC+QHWC8m/xrhsg4b4t
+r52dQuZlPSEGbzdiltrzQPODnSkA6tmgPeaea5wKG8abDgVQRbMBiamwgJruIi1s
+ynX2yD4qdo07fTAC9CyKd9kogMRMXaqVppfIrob+HL3KQJ4MQPRJb+IN3K14Bdq8
+/XL+8zH4zsWxPCkJyKlcUwGx3M0NHnYX0l8Enap65McTiaPIiCgr5TnkJmctDKBm
+Zrv+c1sb69oJtlayGKWNymxdU+1f79IVioQf4qCa7ZB1//ON60KrQ9cRsmObJtnE
+KD6Oett1Kto=
+-----END CERTIFICATE-----

data/spec/td/fixture/testServer.crt

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID2jCCAsKgAwIBAgIJALE6L1U9f459MA0GCSqGSIb3DQEBCwUAMEQxHTAbBgNV
+BAMMFHd3dy50cmVhc3VyZWRhdGEuY29tMQswCQYDVQQGEwJVUzEWMBQGA1UEBwwN
+U2FuIEZyYW5zaXNjbzAeFw0yMjEwMTQwOTU3NDdaFw0zMjEwMTEwOTU3NDdaMGwx
+CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4g
+RnJhbnNpc2NvMRYwFAYDVQQKDA1UcmVhc3VyZSBEYXRhMQwwCgYDVQQLDANFbmcx
+CjAIBgNVBAMMASowggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6TBzz
+TnMXEwQhmsF9ggvyjzhsVQRQnVafPjR45mKgN5+4QmkW+t1RUv93ysz2UOXM1XfN
+HptcrgbdsEmhDDWopsPgfJrSdN6O44oxplZOTusn20lJDG4Ogm0Kdn2k0P6Cj1bG
+PmFLgw4BeXqhMWI60rbQw/HtFDMx6MWU555wr//jJNPIPOVBzVOiGhmoWYJY8ARB
+WlplKeW0BkQ2J+650Q4dcqrrq4R6UbnyTjf9pqxSa2qP5y9ZQr+69COveX26ZSmA
+76PRzGE1oohsB37OCcQMwU+CuIHZkS9CMEGgkYfmfKaGtrsGtG52eGGwiYoAwSGJ
+BJ/v53Pp3BGKWo9tAgMBAAGjgaYwgaMwXgYDVR0jBFcwVaFIpEYwRDEdMBsGA1UE
+AwwUd3d3LnRyZWFzdXJlZGF0YS5jb20xCzAJBgNVBAYTAlVTMRYwFAYDVQQHDA1T
+YW4gRnJhbnNpc2NvggkAkHatMErU7SEwCQYDVR0TBAIwADALBgNVHQ8EBAMCBPAw
+EwYDVR0lBAwwCgYIKwYBBQUHAwEwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqG
+SIb3DQEBCwUAA4IBAQBQkwlcEc98Pj9NB9NjhwZalgI36ptmj7vxyD8GXaJQWh7P
+BKnOTxwbqhY68B7QgQ+NTusltnWsiO2IBAZR3MZGeLh4rFUUi4daww7fCLbYQD0t
+YKOjUv8C4rlp8lRs/IMDMl1i2w1R6x78+RKSy1OhIvxT3qV9Zxx8RfqINboyDlbo
+6lOEQlLYGGUJutIbkUoPCo3TO3/oaLfwlFhnxgEwf+lX/qHoqTkz16JRzil6yFOx
+qc5ugRF0S34lpnBd6qfbhV1QFY17WvSbbXmpBPONI2SLs0WSwwCTFVRg8Z+Y90oG
+oGH+JdWsxn+9e8JTB/UOpxr+vWTyQ63wLaElPOyC
+-----END CERTIFICATE-----

data/spec/td/fixture/testServer.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAukwc805zFxMEIZrBfYIL8o84bFUEUJ1Wnz40eOZioDefuEJp
+FvrdUVL/d8rM9lDlzNV3zR6bXK4G3bBJoQw1qKbD4Hya0nTejuOKMaZWTk7rJ9tJ
+SQxuDoJtCnZ9pND+go9Wxj5hS4MOAXl6oTFiOtK20MPx7RQzMejFlOeecK//4yTT
+yDzlQc1TohoZqFmCWPAEQVpaZSnltAZENifuudEOHXKq66uEelG58k43/aasUmtq
+j+cvWUK/uvQjr3l9umUpgO+j0cxhNaKIbAd+zgnEDMFPgriB2ZEvQjBBoJGH5nym
+hra7BrRudnhhsImKAMEhiQSf7+dz6dwRilqPbQIDAQABAoIBAF8IKpB2yUDRA398
+6Qz0BNIz+u1QJQZWbHSJD81IgLEIDuK4hdEiITm14/mgqxNPSxpFHnq1DT2mzHvT
+zItppgmlIDBof7WxxkIPklQnbMk/erd3JhgsTgv6vlLjBM7JibriEbrI4WrarI9V
+/5cwkNI+4OD3w3ZTopXoDroZuPnz/v0EKhDz05E4jHijfiRL3OxaAvZ2zWVKifVh
+elilbCeLR2KgKG6jPrBzub2Yll9C7Xxt7emIMvyTg5jEOkmm7IWfAZ6uIraI7pAs
+5MlwQDIb/Ypby4gHJ0CPWy2AhdybF2Y+hm5jXY1aD7xjlG8pjudidYtnrpWO1kPV
+bbqIvUUCgYEA3G5JQvcA9ZTiHculyW/6O++NXrOkznXI04JhhKN6JvdTdp3xC61k
+H+gnP0e2/FeqqZWF6KWlmPBuonofEQqyCwJdHRZESaixgw7iKqN2t11Z9EhWzoGq
+UGc1oMT90AIHxsGUBbUIZdlj1D8KDx/qymlsKp8ozTKwyCqNesiYet8CgYEA2FvQ
+94zgOauOCwfOwDipwivrJ9nblykMfcp8NCC2jLxI3rjH30EhRWBZrc0V55erjfrz
+QgJYlj4kN/3GwLVq2Agr5EczjePScRlkpPNHdfAfXKvg7dHHsgrYf9lP6j2Adz8B
+mhyYs1gfo6ekeHLoqKBlSn08AosfSTpETuIdizMCgYBFATFmCTT/rA/tC+dmW+uV
+/7PdxZb+Gtk3fUVR5GtE73/tThw7b5g8dMx0ftrFvBvs4qX84n4olnvL2TcIerSp
+xZ+oj2PpOyn2wR4EAxAS7uJOGqcyFl1etjCPl5ttFnWgvtC7yKRMXfVmaCWZ/n/d
+xYra/OAk/I1i3A9WNJ2nOQKBgD83jLZYPkf7fXRxopJ9u/RVOs+ZE1V2lATJPkNI
+763tcelJ2nS8JgmMXoeu7eCOa3z/v0YhQ1sa6yBFEWbLW12l/ZUkzMZ/s8SCI+si
+flXShIdiXUV/zzaRfrLUf0o1EC1HhqNOCbwVWqFJ4X+kK6DhxNbgAsHHfqu5z62w
+2esLAoGAbIbd5Xd+lRXbuNTyIqAONf4KgH529hU4W9tw+kR5OFqadC557Jvs33nn
+iSqGgET+DZTxaOZk8KeCKZys8IcIxvWyhq22/Pd0BKl1ozWj4w/bkDlqO2V9I9NY
+32DBk4iAHljJqkmxnJkSXgGypf4guPZlNGPbTijXw3oHaIpvPl0=
+-----END RSA PRIVATE KEY-----

data/spec/td/updater_spec.rb

@@ -8,7 +8,7 @@ require 'logger'
 
 module TreasureData::Updater
 
- %w(x86_64-darwin14 x64-mingw32).each do |platform|
+  %w(x86_64-darwin14 x64-mingw32).each do |platform|
     describe "RUBY_PLATFORM is '#{platform}'" do
       before do
         stub_const('RUBY_PLATFORM', platform)
@@ -128,7 +128,7 @@ module TreasureData::Updater
             subject
           end
           tmpfile = File.join(TreasureData::Updater.jarfile_dest_path, 'td-import.jar.new')
-          expect(File.exists?(tmpfile)).to eq false
+          expect(File.exist?(tmpfile)).to eq false
         end
       end
 
@@ -171,7 +171,7 @@ module TreasureData::Updater
       @proxy_server = WEBrick::HTTPProxyServer.new(
         :BindAddress => "localhost",
         :Logger => logger,
-        :Port => 0,
+        :Port => 1000 + rand(1000),
         :AccessLog => []
       )
       @proxy_server_thread = start_server_thread(@proxy_server)
@@ -185,13 +185,13 @@ module TreasureData::Updater
       @server = WEBrick::HTTPServer.new(
         :BindAddress => "localhost",
         :Logger => logger,
-        :Port => 0,
+        :Port => 1000 + rand(1000),
         :AccessLog => [],
         :DocumentRoot => '.',
         :SSLEnable => true,
-        :SSLCACertificateFile => fixture_file('ca.cert'),
-        :SSLCertificate => cert('server.cert'),
-        :SSLPrivateKey => key('server.key')
+        :SSLCACertificateFile => fixture_file('testRootCA.crt'),
+        :SSLCertificate => cert('testServer.crt'),
+        :SSLPrivateKey => key('testServer.key')
       )
       @serverport = @server.config[:Port]
       @server.mount(

data/td.gemspec

@@ -15,18 +15,21 @@ Gem::Specification.new do |gem|
   gem.executables           = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths         = ['lib']
   gem.required_ruby_version = '>= 2.1'
+  gem.licenses              = ["Apache-2.0"]
 
   gem.add_dependency "msgpack"
-  gem.add_dependency "yajl-ruby", "~> 1.1"
+  gem.add_dependency "rexml"
+  gem.add_dependency "yajl-ruby", ">= 1.3.1", "< 2.0"
   gem.add_dependency "hirb", ">= 0.4.5"
-  gem.add_dependency "parallel", "~> 1.8"
-  gem.add_dependency "td-client", ">= 1.0.6", "< 2"
+  gem.add_dependency "parallel", "~> 1.20.0"
+  gem.add_dependency "td-client", ">= 1.0.8", "< 2"
   gem.add_dependency "td-logger", ">= 0.3.21", "< 2"
   gem.add_dependency "rubyzip", "~> 1.3.0"
   gem.add_dependency "zip-zip", "~> 0.3"
   gem.add_dependency "ruby-progressbar", "~> 1.7"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec"
+  gem.add_development_dependency 'webrick'
   gem.add_development_dependency "simplecov"
   gem.add_development_dependency 'coveralls'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.16.9
+  version: 0.17.0
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-10 00:00:00.000000000 Z
+date: 2023-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -24,20 +24,40 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.3.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
@@ -58,21 +78,21 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 1.20.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 1.20.0
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.6
+        version: 1.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -82,7 +102,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.6
+        version: 1.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -176,6 +196,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -211,9 +245,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".coveralls.yml"
+- ".github/CODEOWNERS"
+- ".github/workflows/codeql-analysis.yml"
 - ".gitignore"
-- ".travis.yml"
 - ChangeLog
 - Gemfile
 - README.rdoc
@@ -285,13 +321,17 @@ files:
 - spec/td/fixture/ca.cert
 - spec/td/fixture/server.cert
 - spec/td/fixture/server.key
+- spec/td/fixture/testRootCA.crt
+- spec/td/fixture/testServer.crt
+- spec/td/fixture/testServer.key
 - spec/td/fixture/tmp.zip
 - spec/td/helpers_spec.rb
 - spec/td/updater_spec.rb
 - spec/td/version_spec.rb
 - td.gemspec
 homepage: http://treasure-data.com/
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -308,8 +348,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
@@ -338,6 +377,9 @@ test_files:
 - spec/td/fixture/ca.cert
 - spec/td/fixture/server.cert
 - spec/td/fixture/server.key
+- spec/td/fixture/testRootCA.crt
+- spec/td/fixture/testServer.crt
+- spec/td/fixture/testServer.key
 - spec/td/fixture/tmp.zip
 - spec/td/helpers_spec.rb
 - spec/td/updater_spec.rb

data/.travis.yml

@@ -1,23 +0,0 @@
-language: ruby
-
-rvm:
-  - 2.1.10
-  - 2.2.10
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-  - ruby-head
-
-gemfile:
-  - Gemfile
-
-script: bundle exec rake spec SPEC_OPTS="-fd"
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head
-
-sudo: false
-
-notifications:
-  webhooks: http://td-beda.herokuapp.com/travisci_callback