td 0.15.6 → 0.15.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aeea977d9b1b808bebfbd473065044eb5cc1461a
-  data.tar.gz: cedb2ab5d9018f730218378cd64ed8970be0ead2
+  metadata.gz: 7bb4903ebbe92100825f6fb8c768a4b43135e32e
+  data.tar.gz: 6c2fce67432e0d56d536971b5b75e164d88a8bc4
 SHA512:
-  metadata.gz: 31ef757d8f8d902f7ba41eacedfb604a48178e456386ece4e13a21db375015d48d58e4eada1e5d09497492d021cc51357152685e68d292f720ce4af66194b9a3
-  data.tar.gz: 120425e60e43c3de310c8033039296d7ff5c1ad1c51e4b6ed811174ccf5ab2dcfb70b5a8970e6fdd31a54d6f85b13df314ad8930457c0413de1ad9f32e00f75c
+  metadata.gz: 39f2f0b420f772c670990e1303a037f626bf989c208a75103724fbc24359755335d68fc1623b1c8aaadbcca964280164aea34636cbde90eb653c298274bf6462
+  data.tar.gz: 0c4b3cc6037493a693d8d283160136bd2b7bda2f701aaab7705895322285bc388df348646245d96e214c5f338298904fbacca113bed86477945dccf9a7ae1126

data/ChangeLog

@@ -1,3 +1,7 @@
+== 2017-10-24 version 0.15.7
+
+* Support workflow on aws tokyo #94
+
 == 2017-09-21 version 0.15.6
 
 * Update td-client gem dependency to 1.0.4

data/lib/td/command/workflow.rb

@@ -26,13 +26,18 @@ module TreasureData
         env = {}
         digdag_config_path = File.join(wd, 'config')
         FileUtils.touch(digdag_config_path)
-        if Config.cl_apikey
+        workflow_endpoint = Config.workflow_endpoint
+
+        # In the future passing config to digdag should use environment variables
+        if Config.cl_apikey || workflow_endpoint != 'https://api-workflow.treasuredata.com'
           # If the user passes the apikey on the command line we cannot use the digdag td.conf plugin.
           # Instead, create a digdag configuration file with the endpoint and the specified apikey.
-          env['TD_CONFIG_PATH'] = nil
           apikey = TreasureData::Config.apikey
+          env['TD_CONFIG_PATH'] = nil
+          env['TREASURE_DATA_WORKFLOW_ENDPOINT'] = workflow_endpoint
+          env['TD_API_KEY'] = apikey
           File.write(digdag_config_path, [
-              'client.http.endpoint = https://api-workflow.treasuredata.com',
+              "client.http.endpoint = #{workflow_endpoint}",
               "client.http.headers.authorization = TD1 #{apikey}",
               "secrets.td.apikey = #{apikey}"
           ].join($/) + $/)

data/lib/td/config.rb

@@ -160,6 +160,21 @@ class Config
     @@cl_endpoint = flag
   end
 
+  def self.workflow_endpoint
+    case self.endpoint.to_s.sub(%r[https?://], '')
+    when '', 'api.treasuredata.com'
+      'https://api-workflow.treasuredata.com'
+    when 'api-staging.treasuredata.com'
+      'https://api-staging-workflow.treasuredata.com'
+    when 'api.treasuredata.co.jp'
+      'https://api-workflow.treasuredata.co.jp'
+    when 'api-staging.treasuredata.co.jp'
+      'https://api-staging-workflow.treasuredata.co.jp'
+    else
+      raise ConfigError, "Workflow is not supported for '#{self.endpoint}'"
+    end
+  end
+
   # renders the apikey and endpoint options as a string for the helper commands
   def self.cl_options_string
     string = ""

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.15.6'
+  TOOLBELT_VERSION = '0.15.7'
 end

data/spec/spec_helper.rb

@@ -21,6 +21,15 @@ unless ENV['APPVEYOR']
   Coveralls.wear!('rails')
 end
 
+RSpec.configure do |config|
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+end
+
 require 'td/command/runner'
 
 def execute_td(command_line)

data/spec/td/command/workflow_spec.rb

@@ -325,4 +325,84 @@ EOF
       end
     end
   end
+
+  describe 'verify argument with mock' do
+    let(:command) { Class.new { include TreasureData::Command }.new }
+    let(:empty_option) { List::CommandParser.new("workflow", [], [], nil, [], true) }
+    let(:td_config){ TreasureData::Config.new }
+    let(:workflow_config){ Hash.new }
+    let(:digdag_env){ Hash.new }
+    let(:config_path){ nil }
+    let(:apikey){ nil }
+    let(:endpoint){ nil }
+    before do
+      # TreasureData::Command::Runner#run
+      if config_path
+        TreasureData::Config.path = config_path
+      end
+      if apikey
+        TreasureData::Config.apikey = apikey
+        TreasureData::Config.cl_apikey = true
+      end
+      if endpoint
+        TreasureData::Config.endpoint = endpoint
+        TreasureData::Config.cl_endpoint = true
+      end
+
+      allow(Kernel).to receive(:system) do |env, *cmd|
+        digdag_env.replace env
+        cmd = cmd.dup
+        args = {nil => []}
+        while x = cmd.shift
+          case x
+          when /\A--\w+\z/
+            args[x] = cmd.shift
+          when /\A-[a-z]+\z/
+            args[x] = cmd.shift
+          when /\A(-[A-Z][^=]*)(?:=(.*))?\z/
+            args[$1] = $2
+          else
+            args[nil] << x
+          end
+        end
+        if args['--config']
+          File.foreach(args['--config']) do |line|
+            k, v = line.strip.split(/\s*=\s*/, 2)
+            workflow_config[k] = v
+          end
+        end
+        td_config_path =
+          env['TREASURE_DATA_CONFIG_PATH'] ||
+          env['TD_CONFIG_PATH'] ||
+          "#{Dir.home}/.config/.td/td.conf"
+        if File.exist?(td_config_path) && args['-Dio.digdag.standards.td.secrets.enabled'] != 'false'
+          td_config.read(td_config_path)
+        end
+        0
+      end
+    end
+    context 'endpoint: https://api.treasuredata.com' do
+      let (:endpoint){ 'https://api.treasuredata.com' }
+      it 'uses td.conf' do
+        apikey = '1/deadbeaf'
+        TreasureData::Config.apikey = apikey # emulate to load from config file
+        op = List::CommandParser.new("workflow", [], [], nil, ['version'], true)
+        command.workflow(op, false, false)
+        expect(workflow_config).to eq({})
+        expect(digdag_env['TREASURE_DATA_WORKFLOW_ENDPOINT']).to be_nil
+      end
+    end
+    context 'endpoint: https://api.treasuredata.co.jp' do
+      let(:apikey){ '1/deadbeaf' }
+      let (:endpoint){ 'https://api.treasuredata.co.jp' }
+      it 'writes temporary workflow conf' do
+        op = List::CommandParser.new("workflow", [], [], nil, ['version'], true)
+        command.workflow(op, false, false)
+        expect(workflow_config['client.http.endpoint']).to eq 'https://api-workflow.treasuredata.co.jp'
+        expect(workflow_config['client.http.headers.authorization']).to eq "TD1 #{apikey}"
+        expect(workflow_config['secrets.td.apikey']).to eq apikey
+        expect(digdag_env['TREASURE_DATA_WORKFLOW_ENDPOINT']).to eq 'https://api-workflow.treasuredata.co.jp'
+      end
+    end
+  end
 end

data/spec/td/config_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+require 'td/config'
+
+describe TreasureData::Config do
+  context 'workflow_endpoint' do
+    before { TreasureData::Config.endpoint = api_endpoint }
+    subject { TreasureData::Config.workflow_endpoint }
+    context 'api.treasuredata.com' do
+      context 'works without http schema' do
+        let(:api_endpoint){ 'api.treasuredata.com' }
+        it { is_expected.to eq 'https://api-workflow.treasuredata.com' }
+      end
+      context 'works with http schema' do
+        let(:api_endpoint){ 'http://api.treasuredata.com' }
+        it { is_expected.to eq 'https://api-workflow.treasuredata.com' }
+      end
+      context 'works with https schema' do
+        let(:api_endpoint){ 'https://api.treasuredata.com' }
+        it { is_expected.to eq 'https://api-workflow.treasuredata.com' }
+      end
+    end
+    context 'api.treasuredata.co.jp' do
+      let(:api_endpoint){ 'api.treasuredata.co.jp' }
+      it { is_expected.to eq 'https://api-workflow.treasuredata.co.jp' }
+    end
+    context 'api-staging.treasuredata.com' do
+      let(:api_endpoint){ 'api-staging.treasuredata.com' }
+      it { is_expected.to eq 'https://api-staging-workflow.treasuredata.com' }
+    end
+    context 'api-staging.treasuredata.co.jp' do
+      let(:api_endpoint){ 'api-staging.treasuredata.co.jp' }
+      it { is_expected.to eq 'https://api-staging-workflow.treasuredata.co.jp' }
+    end
+    context 'ybi.jp-east.idcfcloud.com' do
+      let(:api_endpoint){ 'ybi.jp-east.idcfcloud.com' }
+      it 'raise error' do
+        expect { subject }.to raise_error(TreasureData::ConfigError)
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.15.6
+  version: 0.15.7
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-21 00:00:00.000000000 Z
+date: 2017-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -280,6 +280,7 @@ files:
 - spec/td/command/workflow_spec.rb
 - spec/td/common_spec.rb
 - spec/td/compact_format_yamler_spec.rb
+- spec/td/config_spec.rb
 - spec/td/connector_config_normalizer_spec.rb
 - spec/td/fixture/bulk_load.yml
 - spec/td/fixture/ca.cert
@@ -332,6 +333,7 @@ test_files:
 - spec/td/command/workflow_spec.rb
 - spec/td/common_spec.rb
 - spec/td/compact_format_yamler_spec.rb
+- spec/td/config_spec.rb
 - spec/td/connector_config_normalizer_spec.rb
 - spec/td/fixture/bulk_load.yml
 - spec/td/fixture/ca.cert