td 0.16.9 → 0.16.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6aea3ef4a96f7d2d1042cbf337e5f95288022f39
-  data.tar.gz: 7ab665de173e7ce66dfa7b4622cb4513dcef6426
+SHA256:
+  metadata.gz: a372eaff21645cac711c4df4fe0faeba64a8fffed0af8d9de4367f5bdf3d4dae
+  data.tar.gz: b4f0c3751f57780008be2162c78853003b138d9ed691f1c7eef756a8ce6f7237
 SHA512:
-  metadata.gz: 114f5e1e8763474cb53cdead4f511dd30a988457c8d72e0115fe20f4a013aae7fdbee3c13e8a627381145e05db4d5bf584690bdf863bc9d6fe35bbae66be9b35
-  data.tar.gz: 61f72c29b14c70627680248852303d878a92ea571e140c4e5c60318944248367f6509df81b1c19553e4a271ee8ce4f9bc54747610f8263f7ef4cc3fa6d053581
+  metadata.gz: 3f332bc9d0cc8659e31d2f737aec26c827a6eda4c980dd0d21ddf0df749fae5595b16950a9ba3195e76361925d190bf8aa5e016dee6f25cf49bdf139e8606363
+  data.tar.gz: af1d0421633676a8938be8cf77474681f05e863006c56561db18b377853ee9f8529c168d1d176a2e4f3156a1eb4acd582241765f52a8aac6ecce0e3e158e1ddf

data/.circleci/config.yml

@@ -0,0 +1,96 @@
+version: 2.1
+
+
+orbs:
+  ruby: circleci/ruby@1.1.4 
+  win: circleci/windows@2.4.0
+
+
+commands:
+  install_windows_requirements:
+    description: "Install windows requirements"
+    steps:
+      - run:
+          name: "Install MSYS2"
+          command: choco install msys2
+      - run:
+          name: "Install Ruby devkit"
+          command: ridk install 2 3
+  bundle-install:
+    description: "Install dependencies"
+    steps:
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - run:
+          name: Bundle install
+          command: bundle install
+  run-tests:
+    description: "Run tests"
+    steps:
+      - run:
+          name: Run tests
+          command: bundle exec rake spec SPEC_OPTS="-fd"
+  run-tests-flow:
+    description: "Single flow for running tests"
+    steps:
+      - checkout
+      - bundle-install
+      - run-tests
+
+
+jobs:
+
+  ruby_25:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - ruby/install:
+          version: '2.5'
+      - run-tests-flow
+
+  ruby_26:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - ruby/install:
+          version: '2.6'
+      - run-tests-flow
+
+  ruby_27:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - ruby/install:
+          version: '2.7'
+      - run-tests-flow
+
+  ruby_30:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - ruby/install:
+          version: '3.0'
+      - run-tests-flow
+
+  win_ruby:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    steps:
+      - install_windows_requirements
+      - run:
+          name: "Install bundler"
+          shell: powershell.exe
+          command: gem install bundler
+      - run-tests-flow
+
+
+workflows:
+  tests:
+    jobs:
+      - ruby_25
+      - ruby_26
+      - ruby_27
+      - ruby_30
+      # - win_ruby # Not testing on windows for now due to old version of ridk. Testing in Appveyor instead, will reconsider when CircleCI update ridk in their windows image.

data/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @treasure-data/integrations

data/ChangeLog

@@ -1,3 +1,13 @@
+== 2020-09-16 version 0.16.10
+
+* Support proxy with username and password
+* Increase the summary_width value to fix the broken options help message format
+* Support AP03 Region
+* Change old document urls to new document urls
+* Show message to ask -w option in td query
+* Adding users commnad help
+* Allow trailing slash for endpoints
+
 == 2020-04-10 version 0.16.9
 
 * Update new workflow pattern to support ap02 #232

data/README.rdoc

@@ -1,14 +1,14 @@
 = Treasure Data command line tool
-{<img src="https://travis-ci.org/treasure-data/td.svg?branch=master" alt="Build Status" />}[https://travis-ci.org/treasure-data/td]
+{<img src="https://circleci.com/gh/treasure-data/td.svg?style=svg" alt="Build Status" />}[https://circleci.com/gh/treasure-data/td.svg?style=svg]
 {<img src="https://ci.appveyor.com/api/projects/status/github/treasure-data/td?branch=master&svg=true" alt="appveyor" />}[https://ci.appveyor.com/project/treasure-data/td/branch/master]
 {<img src="https://coveralls.io/repos/treasure-data/td/badge.svg?branch=master&service=github" alt="Coverage Status" />}[https://coveralls.io/github/treasure-data/td?branch=master]
 
 This CUI utility wraps the {Ruby Client Library td-client-ruby}[https://github.com/treasure-data/td-client-ruby]
 to interact with the REST API in managing databases and jobs on the Treasure Data Cloud.
 
-For more info about Treasure Data, see <http://treasure-data.com/>.
+For more info about Treasure Data, see <https://www.treasuredata.com/>.
 
-For full documentation see <https://support.treasuredata.com/hc/en-us>.
+For full documentation see <https://docs.treasuredata.com/>.
 
 = Getting Started
 
@@ -71,7 +71,7 @@ The Bulk Import CLI is downloaded automatically at the first call to any of the
 need internet connectivity in order to fetch the Bulk Import CLI JAR file from the
 {Central Maven repository}[https://repo1.maven.org/maven2/com/treasuredata/td-import/]
 and take advantage of these advanced features. If you need to setup a proxy, please consult this
-{documentation}[https://support.treasuredata.com/hc/en-us/articles/360001502527-Legacy-Bulk-Import-Tips-and-Tricks#How%20to%20use%20a%20proxy%20server] page.
+{documentation}[https://docs.treasuredata.com/display/public/INT/Legacy+Bulk+Import+Tips+and+Tricks#LegacyBulkImportTipsandTricks-UsingaProxyServer] page.
 
 The log levels and properties of the Bulk Import CLI can be configured in a +logging.properties+ file. A default
 configuration is provided in a file within the gem or Toolbelt folder root, in the +java/+ folder. If you wish to

data/appveyor.yml

@@ -13,5 +13,5 @@ test_script:
   - bundle exec rake spec
 environment:
   matrix:
-    - ruby_version: "22"
-    - ruby_version: "22-x64"
+    - ruby_version: "26"
+    - ruby_version: "26-x64"

data/lib/td/command/account.rb

@@ -41,7 +41,7 @@ module Command
       end
     end
 
-    $stdout.puts "Enter your Treasure Data credentials. For Google SSO user, please see https://support.treasuredata.com/hc/en-us/articles/360000720048-Treasure-Data-Toolbelt-Command-line-Interface#Google%20SSO%20Users"
+    $stdout.puts "Enter your Treasure Data credentials. For Google SSO user, please see https://docs.treasuredata.com/display/public/PD/Configuring+Authentication+for+TD+Using+the+TD+Toolbelt#ConfiguringAuthenticationforTDUsingtheTDToolbelt-SettingUpGoogleSSOUsers"
     unless user_name
       begin
         $stdout.print "Email: "

data/lib/td/command/common.rb

@@ -67,13 +67,11 @@ module Command
     Client.new(apikey, opts)
   end
 
-  DEFAULT_IMPORT_ENDPOINT = "https://" + TreasureData::API::DEFAULT_IMPORT_ENDPOINT
-
   def get_import_client
     import_endpoint = begin
-                        Config.import_endpoint || DEFAULT_IMPORT_ENDPOINT
+                        Config.import_endpoint || nil
                       rescue TreasureData::ConfigNotFoundError
-                        DEFAULT_IMPORT_ENDPOINT
+                        nil
                       end
     get_client(endpoint: import_endpoint)
   end
@@ -107,7 +105,7 @@ module Command
       headers = options[:fields]
       csv_opts = {}
       csv_opts[:col_sep] = "\t" if format == 'tsv'
-      CSV.generate('', csv_opts) { |csv|
+      CSV.generate('', **csv_opts) { |csv|
         csv << headers
         rows.each { |row|
           r = []
@@ -368,12 +366,9 @@ EOS
 
   def self.get_http_class
     # use Net::HTTP::Proxy in place of Net::HTTP if a proxy is provided
-    http_proxy = ENV['HTTP_PROXY']
-    if http_proxy
-      http_proxy = (http_proxy =~ /\Ahttp:\/\/(.*)\z/) ? $~[1] : http_proxy
-      host, port = http_proxy.split(':', 2)
-      port = (port ? port.to_i : 80)
-      return Net::HTTP::Proxy(host, port)
+    if ENV['HTTP_PROXY']
+      uri = URI.parse(ENV['HTTP_PROXY'])
+      return Net::HTTP::Proxy(uri.host, uri.port, uri.user, uri.password)
     else
       return Net::HTTP
     end

data/lib/td/command/export.rb

@@ -67,6 +67,9 @@ module Command
     encryption = nil
     assume_role = nil
 
+    # Increase the summary_width value from default 32 into 36, to show the options help message properly.
+    op.summary_width = 36
+
     op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b|
       wait = b
     }

data/lib/td/command/query.rb

@@ -160,6 +160,8 @@ module Command
         rescue TreasureData::NotFoundError => e
         end
       end
+    elsif output
+      $stdout.puts "The output file won't be generated without additional `-w` or `--wait` option."
     end
   end
 

data/lib/td/command/runner.rb

@@ -48,7 +48,9 @@ Basic commands:
   sched          # create/delete/list schedules that run a query periodically
   schema         # create/delete/modify schemas of tables
   connector      # manage connectors
+  users          # show/create/list users
   workflow       # manage workflows
+  user           # manage users  
 
 Additional commands:
 

data/lib/td/command/table.rb

@@ -41,7 +41,7 @@ module Command
     if HIVE_RESERVED_KEYWORDS.include?(table_name.upcase)
       $stderr.puts "* WARNING *"
       $stderr.puts "  '#{table_name}' is a reserved keyword in Hive. We recommend renaming the table."
-      $stderr.puts "  For a list of all reserved keywords, see our FAQ: https://support.treasuredata.com/hc/en-us/articles/360009531173/"
+      $stderr.puts "  For a list of all reserved keywords, see our FAQ: https://docs.treasuredata.com/display/public/PD/Hive+Reserved+and+Non-Reserved+Keywords"
     end
 
     client = get_client
@@ -171,7 +171,7 @@ module Command
     if rows.empty?
       if db_name
         if databases.first.permission == :import_only
-          $stderr.puts "Database '#{db_name}' is import only, cannot list or create tables."
+          $stderr.puts "Database '#{db_name}' is import only, cannot list."
         else
           $stderr.puts "Database '#{db_name}' has no tables."
           $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create <db> <table>' to create a table."

data/lib/td/config.rb

@@ -148,7 +148,8 @@ class Config
 
 
   def self.endpoint
-    @@endpoint || Config.read['account.endpoint']
+    endpoint = @@endpoint || Config.read['account.endpoint']
+    endpoint.sub(/(\/)+$/, '') if endpoint
   end
 
   def self.endpoint=(endpoint)
@@ -168,7 +169,8 @@ class Config
   end
 
   def self.import_endpoint
-    @@import_endpoint || Config.read['account.import_endpoint']
+    endpoint = @@import_endpoint || Config.read['account.import_endpoint']
+    endpoint.sub(/(\/)+$/, '') if endpoint
   end
 
   def self.import_endpoint=(endpoint)
@@ -185,7 +187,7 @@ class Config
 
   def self.workflow_endpoint
     case self.endpoint_domain
-    when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?((?:eu01|ap02)\.)?treasuredata\.(com|co\.jp)\z/i
+    when /\Aapi(-(?:staging|development))?(-[a-z0-9]+)?\.(connect\.)?((?:eu01|ap02|ap03)\.)?treasuredata\.(com|co\.jp)\z/i
       "https://api#{$1}-workflow#{$2}.#{$3}#{$4}treasuredata.#{$5}"
     else
       raise ConfigError, "Workflow is not supported for '#{self.endpoint}'"

data/lib/td/file_reader.rb

@@ -96,10 +96,10 @@ module TreasureData
         csv_opts[:quote_char] = opts[:quote_char] if opts[:quote_char]
         begin
           require 'fastercsv'
-          @io = FasterCSV.new(io, csv_opts)
+          @io = FasterCSV.new(io, **csv_opts)
         rescue LoadError => e
           require 'csv'
-          @io = CSV.new(io, csv_opts)
+          @io = CSV.new(io, **csv_opts)
         end
         @error = error
         # @escape_char = opts[:escape_char]

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.16.9'
+  TOOLBELT_VERSION = '0.16.10'
 end

data/spec/td/config_spec.rb

@@ -32,6 +32,10 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api.ap02.treasuredata.com' }
       it { is_expected.to eq 'https://api-workflow.ap02.treasuredata.com' }
     end
+    context 'api.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-workflow.ap03.treasuredata.com' }
+    end
 
     context 'api-hoge.connect.treasuredata.com' do
       let(:api_endpoint){ 'api-hoge.connect.treasuredata.com' }
@@ -58,6 +62,10 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api-staging.ap02.treasuredata.com' }
       it { is_expected.to eq 'https://api-staging-workflow.ap02.treasuredata.com' }
     end
+    context 'api-staging.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api-staging.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-staging-workflow.ap03.treasuredata.com' }
+    end
 
     context 'api-development.treasuredata.com' do
       let(:api_endpoint){ 'api-development.treasuredata.com' }
@@ -75,6 +83,10 @@ describe TreasureData::Config do
       let(:api_endpoint){ 'api-development.ap02.treasuredata.com' }
       it { is_expected.to eq 'https://api-development-workflow.ap02.treasuredata.com' }
     end
+    context 'api-development.ap03.treasuredata.com' do
+      let(:api_endpoint){ 'api-development.ap03.treasuredata.com' }
+      it { is_expected.to eq 'https://api-development-workflow.ap03.treasuredata.com' }
+    end
 
     context 'ybi.jp-east.idcfcloud.com' do
       let(:api_endpoint){ 'ybi.jp-east.idcfcloud.com' }
@@ -84,6 +96,64 @@ describe TreasureData::Config do
     end
   end
 
+  describe 'allow endpoint with trailing slash' do
+    context 'self.endpoint' do
+      before { TreasureData::Config.endpoint = api_endpoint }
+      subject { TreasureData::Config.endpoint }
+
+      context 'api.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.treasuredata.com' }
+      end
+      context 'api.treasuredata.co.jp' do
+        let(:api_endpoint) { 'https://api.treasuredata.co.jp/' }
+        it { is_expected.to eq 'https://api.treasuredata.co.jp' }
+      end
+      context 'api.eu01.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.eu01.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.eu01.treasuredata.com' }
+      end
+      context 'api.ap02.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.ap02.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.ap02.treasuredata.com' }
+      end
+      context 'api.ap03.treasuredata.com' do
+        let(:api_endpoint) { 'https://api.ap03.treasuredata.com/' }
+        it { is_expected.to eq 'https://api.ap03.treasuredata.com' }
+      end
+      context 'api-hoge.connect.treasuredata.com' do
+        let(:api_endpoint){ 'https://api-hoge.connect.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-hoge.connect.treasuredata.com' }
+      end
+    end
+
+    context 'self.import_endpoint' do
+      before { TreasureData::Config.import_endpoint = api_endpoint }
+      subject { TreasureData::Config.import_endpoint }
+
+      context 'api-import.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.treasuredata.com' }
+      end
+      context 'api-import.treasuredata.co.jp' do
+        let(:api_endpoint) { 'https://api-import.treasuredata.co.jp/' }
+        it { is_expected.to eq 'https://api-import.treasuredata.co.jp' }
+      end
+      context 'api-import.eu01.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.eu01.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.eu01.treasuredata.com' }
+      end
+      context 'api-import.ap02.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.ap02.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.ap02.treasuredata.com' }
+      end
+      context 'api-import.ap03.treasuredata.com' do
+        let(:api_endpoint) { 'https://api-import.ap03.treasuredata.com/' }
+        it { is_expected.to eq 'https://api-import.ap03.treasuredata.com' }
+      end
+    end
+  end
+
   describe '#read' do
     it 'sets @conf' do
       Tempfile.create('td.conf') do |f|

data/td.gemspec

@@ -15,18 +15,21 @@ Gem::Specification.new do |gem|
   gem.executables           = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths         = ['lib']
   gem.required_ruby_version = '>= 2.1'
+  gem.licenses              = ["Apache-2.0"]
 
   gem.add_dependency "msgpack"
-  gem.add_dependency "yajl-ruby", "~> 1.1"
+  gem.add_dependency "rexml"
+  gem.add_dependency "yajl-ruby", "~> 1.3.1"
   gem.add_dependency "hirb", ">= 0.4.5"
-  gem.add_dependency "parallel", "~> 1.8"
-  gem.add_dependency "td-client", ">= 1.0.6", "< 2"
+  gem.add_dependency "parallel", "~> 1.20.0"
+  gem.add_dependency "td-client", ">= 1.0.8", "< 2"
   gem.add_dependency "td-logger", ">= 0.3.21", "< 2"
   gem.add_dependency "rubyzip", "~> 1.3.0"
   gem.add_dependency "zip-zip", "~> 0.3"
   gem.add_dependency "ruby-progressbar", "~> 1.7"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec"
+  gem.add_development_dependency 'webrick'
   gem.add_development_dependency "simplecov"
   gem.add_development_dependency 'coveralls'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.16.9
+  version: 0.16.10
 platform: ruby
 authors:
 - Treasure Data, Inc.
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-10 00:00:00.000000000 Z
+date: 2021-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -24,20 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.3.1
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
@@ -58,21 +72,21 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 1.20.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 1.20.0
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.6
+        version: 1.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -82,7 +96,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.6
+        version: 1.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -176,6 +190,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -211,9 +239,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".coveralls.yml"
+- ".github/CODEOWNERS"
 - ".gitignore"
-- ".travis.yml"
 - ChangeLog
 - Gemfile
 - README.rdoc
@@ -291,9 +320,10 @@ files:
 - spec/td/version_spec.rb
 - td.gemspec
 homepage: http://treasure-data.com/
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -308,9 +338,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:

data/.travis.yml

@@ -1,23 +0,0 @@
-language: ruby
-
-rvm:
-  - 2.1.10
-  - 2.2.10
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-  - ruby-head
-
-gemfile:
-  - Gemfile
-
-script: bundle exec rake spec SPEC_OPTS="-fd"
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head
-
-sudo: false
-
-notifications:
-  webhooks: http://td-beda.herokuapp.com/travisci_callback