td 0.14.1 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0fd96802c069458c8a45adde2c63044dbc6157e5
-  data.tar.gz: 4ca331507c1ec04cac6ad1519cee73aa86f5e33a
+  metadata.gz: 61447ffe419ff8f382b433975388f420a4b7d013
+  data.tar.gz: fedca79cdca59a6a91eb4968ca3d5508e3bf65b8
 SHA512:
-  metadata.gz: 9a8ffee2dc536d83831e9c7a6f8e092273d2434ab5b401dedfadd756e782a4c20b2655d44478d69cbbe15bf8167c696ac605aed84bddf0efd75676f6b60ff59a
-  data.tar.gz: efc8acd54a2df5304f6f55186fa44717d3427c4820c5bf964a8e88a05f2897980d5c2e121591de3feba47d1fb4766e7061bff9ec34f43eae10a09d8c9c7d7fac
+  metadata.gz: 0d85fb2e56cb54143505b472a18ff26a9b532db3ddf0a7033d7f8610626f564c9ec90a9dfece2cff4dee5447945d82cf481a061fc6eb88a43826205d7fc0c01e
+  data.tar.gz: ef7d72a9d251c546098ac39b2a3aa455dc933dfe224286ed4569b49f443f054c3acdb8ed1672738902b51731092354de11a1d8300f1690c3081cb89823cd50fc

data/.travis.yml

@@ -1,5 +1,8 @@
 language: ruby
 
+jdk:
+  - oraclejdk8
+
 rvm:
   - 1.9.3
   - 2.0.0

data/ChangeLog

@@ -1,3 +1,8 @@
+== 2016-09-06 version 0.15.0
+
+* Add `td workflows` command. #167
+* Add `td connector:update` options to allow configuring several additional data connector settings. #127 #155
+
 == 2016-07-11 version 0.14.1
 
 * Use td-client-ruby v0.8.82

data/appveyor.yml

@@ -14,3 +14,4 @@ test_script:
 environment:
   matrix:
     - ruby_version: "22"
+    - ruby_version: "22-x64"

data/contrib/completion/_td

@@ -81,6 +81,10 @@ subcommands=(
     "table\:tail":"Get recently imported logs"
     "table\:partial_delete":"Delete logs from the table within the specified time range"
 
+    "wf":"Run a workflow command"
+    "workflow":"Run a workflow command"
+    "workflow:reset":"Reset the workflow module"
+
     # TODO: Add ACL related commands
 )
 

data/lib/td/command/account.rb

@@ -41,7 +41,7 @@ module Command
       end
     end
 
-    $stdout.puts "Enter your Treasure Data credentials."
+    $stdout.puts "Enter your Treasure Data credentials. For Google SSO user, please see https://docs.treasuredata.com/articles/command-line#google-sso-users"
     unless user_name
       begin
         $stdout.print "Email: "

data/lib/td/command/common.rb

@@ -25,6 +25,9 @@ module Command
   class ImportError < RuntimeError
   end
 
+  class WorkflowError < RuntimeError
+  end
+
   private
   def initialize
     @render_indent = ''

data/lib/td/command/connector.rb

@@ -212,12 +212,47 @@ module Command
   end
 
   def connector_update(op)
-    name, config_file = op.cmd_parse
+    settings = {}
 
-    config = prepare_bulkload_job_config(config_file)
+    op.on('-n', '--newname NAME', 'change the schedule\'s name', String) {|n|
+      settings['name'] = n
+    }
+    op.on('-d', '--database DB_NAME', 'change the database', String) {|s|
+      settings['database'] = s
+    }
+    op.on('-t', '--table TABLE_NAME', 'change the table', String) {|s|
+      settings['table'] = s
+    }
+    op.on('-s', '--schedule [CRON]', 'change the schedule or leave blank to remove the schedule', String) {|s|
+      settings['cron'] = s || ''
+    }
+    op.on('-z', '--timezone TZ', "name of the timezone.",
+                                 "  Only extended timezones like 'Asia/Tokyo', 'America/Los_Angeles' are supported,",
+                                 "  (no 'PST', 'PDT', etc...).",
+                                 "  When a timezone is specified, the cron schedule is referred to that timezone.",
+                                 "  Otherwise, the cron schedule is referred to the UTC timezone.",
+                                 "  E.g. cron schedule '0 12 * * *' will execute daily at 5 AM without timezone option",
+                                 "  and at 12PM with the -t / --timezone 'America/Los_Angeles' timezone option", String) {|s|
+      settings['timezone'] = s
+    }
+    op.on('-D', '--delay SECONDS', 'change the delay time of the schedule', Integer) {|i|
+      settings['delay'] = i
+    }
+    op.on('-T', '--time-column COLUMN_NAME', 'change the name of the time column', String) {|s|
+      settings['time_column'] = s
+    }
+    op.on('-c', '--config CONFIG_FILE', 'update the connector configuration', String) {|s|
+      settings['config'] = s
+    }
+    op.on('--config-diff CONFIG_DIFF_FILE', "update the connector config_diff", String) { |s| settings['config_diff'] = s }
 
+    name, config_file = op.cmd_parse
+    settings['config'] = config_file if config_file
+    op.cmd_usage 'nothing to update' if settings.empty?
+    settings['config'] = prepare_bulkload_job_config(settings['config']) if settings.key?('config')
+    settings['config_diff'] = prepare_bulkload_job_config(settings['config_diff']) if settings.key?('config_diff')
     client = get_client()
-    session = client.bulk_load_update(name, config: config)
+    session = client.bulk_load_update(name, settings)
     dump_connector_session(session)
   end
 
@@ -312,6 +347,11 @@ private
 
 
   def prepare_bulkload_job_config(config_file)
+    config = prepare_bulkload_job_config_diff(config_file)
+    TreasureData::ConnectorConfigNormalizer.new(config).normalized_config
+  end
+
+  def prepare_bulkload_job_config_diff(config_file)
     unless File.exist?(config_file)
       raise ParameterConfigurationError, "configuration file: #{config_file} not found"
     end
@@ -326,8 +366,7 @@ private
     rescue => e
       raise ParameterConfigurationError, "configuration file: #{config_file} #{e.message}"
     end
-
-    TreasureData::ConnectorConfigNormalizer.new(config).normalized_config
+    config
   end
 
   def dump_connector_session(session)

data/lib/td/command/list.rb

@@ -352,11 +352,15 @@ module List
   add_list 'connector:list', %w[], 'Show list of connector sessions', ['connector:list']
   add_list 'connector:create', %w[name cron database table config], 'Create new connector session', ['connector:create connector1 "0 * * * *" connector_database connector_table td-bulkload.yml']
   add_list 'connector:show', %w[name], 'Show connector session', ['connector:show connector1']
-  add_list 'connector:update', %w[name config], 'Modify connector session', ['connector:update connector1 td-bulkload.yml']
+  add_list 'connector:update', %w[name config?], 'Modify connector session', ['connector:update connector1 -c td-bulkload.yml -s \'@daily\' ...']
   add_list 'connector:delete', %w[name], 'Delete connector session', ['connector:delete connector1']
   add_list 'connector:history', %w[name], 'Show job history of connector session', ['connector:history connector1']
   add_list 'connector:run', %w[name time?], 'Run connector with session for the specified time option', ['connector:run connector1 "2016-01-01 00:00:00"']
 
+  add_list 'workflow', %w[], 'Run a workflow command'
+  add_list 'workflow:reset', %w[], 'Reset the workflow module'
+  add_list 'workflow:version', %w[], 'Show workflow module version'
+
   # aliases
   add_alias 'db', 'db:show'
   add_alias 'dbs', 'db:list'
@@ -417,6 +421,8 @@ module List
 
   add_alias 'connector', 'connector:guess'
 
+  add_alias 'wf', 'workflow'
+
   # backward compatibility
   add_alias 'show-databases',   'db:list'
   add_alias 'show-dbs',         'db:list'

data/lib/td/command/runner.rb

@@ -47,6 +47,7 @@ Basic commands:
   sched          # create/delete/list schedules that run a query periodically
   schema         # create/delete/modify schemas of tables
   connector      # manage connectors
+  workflow       # manage workflows
 
 Additional commands:
 
@@ -162,12 +163,13 @@ EOF
       return 1
     end
 
+    status = nil
     begin
       # test the connectivity with the API endpoint
       if cmd_req_connectivity && Config.cl_endpoint
         Command.test_api_endpoint(Config.endpoint)
       end
-      method.call(argv)
+      status = method.call(argv)
     rescue ConfigError
       $stderr.puts "TreasureData account is not configured yet."
       $stderr.puts "Run '#{$prog} account' first."
@@ -185,7 +187,7 @@ EOF
       #   => NotFoundError
       #   => AuthError
       if ![ParameterConfigurationError, BulkImportExecutionError, UpdateError, ImportError,
-            APIError, ForbiddenError, NotFoundError, AuthError, AlreadyExistsError].include?(e.class) ||
+            APIError, ForbiddenError, NotFoundError, AuthError, AlreadyExistsError, WorkflowError].include?(e.class) ||
          !ENV['TD_TOOLBELT_DEBUG'].nil? || $verbose
         show_backtrace "Error #{$!.class}: backtrace:", $!.backtrace
       end
@@ -211,7 +213,7 @@ EOS
       end
       return 1
     end
-    return 0
+    return (status.is_a? Integer) ? status : 0
   end
 
   private

data/lib/td/command/workflow.rb

@@ -0,0 +1,446 @@
+require 'td/helpers'
+require 'td/updater'
+require 'open3'
+require 'pathname'
+require 'time'
+require 'yaml'
+
+module TreasureData
+  module Command
+    include TreasureData::Helpers
+
+    # The workflow entrypoint command. Invokes the digdag cli, passing on any command line arguments.
+    def workflow(op, capture_output=false, check_prereqs=true)
+      if Config.apikey.nil?
+        raise ConfigError
+      end
+      check_digdag_cli if check_prereqs
+      cmd = [
+          java_cmd,
+          '-Dio.digdag.cli.programName=td workflow',
+          '-XX:+TieredCompilation', '-XX:TieredStopAtLevel=1', '-Xverify:none'
+      ]
+
+      FileUtils.mkdir_p digdag_tmp_dir
+      Dir.mktmpdir(nil, digdag_tmp_dir) { |wd|
+        env = {}
+        digdag_config_path = File.join(wd, 'config')
+        FileUtils.touch(digdag_config_path)
+        if Config.cl_apikey
+          # If the user passes the apikey on the command line we cannot use the digdag td.conf plugin.
+          # Instead, create a digdag configuration file with the endpoint and the specified apikey.
+          env['TD_CONFIG_PATH'] = nil
+          apikey = TreasureData::Config.apikey
+          File.write(digdag_config_path, [
+              'client.http.endpoint = https://api-workflow.treasuredata.com',
+              "client.http.headers.authorization = TD1 #{apikey}",
+              "secrets.td.apikey = #{apikey}"
+          ].join($/) + $/)
+          cmd << '-Dio.digdag.standards.td.secrets.enabled=false'
+        else
+          # Use the digdag td.conf plugin to configure wf api and apikey.
+          env['TREASURE_DATA_CONFIG_PATH'] = Config.path
+          cmd << '-Dio.digdag.standards.td.client-configurator.enabled=true'
+        end
+
+        cmd << '-jar' << digdag_cli_path
+        unless op.argv.empty?
+          cmd << '--config' << digdag_config_path
+        end
+        cmd.concat(op.argv)
+
+        unless ENV['TD_TOOLBELT_DEBUG'].nil?
+          $stderr.puts cmd.to_s
+        end
+
+        if capture_output
+          # TODO: use popen3 instead?
+          stdout_str, stderr_str, status = Open3.capture3(env, *cmd)
+          $stdout.write(stdout_str)
+          $stderr.write(stderr_str)
+          return status.exitstatus
+        else
+          Kernel::system(env, *cmd)
+          return $?.exitstatus
+        end
+      }
+    end
+
+    # "Factory reset"
+    def workflow_reset(op)
+      $stdout << 'Removing workflow module...'
+      FileUtils.rm_rf digdag_dir
+      $stdout.puts ' Done.'
+      return 0
+    end
+
+    def workflow_version(op)
+      unless File.exists?(digdag_cli_path)
+        $stderr.puts('Workflow module not yet installed.')
+        return 1
+      end
+
+      $stdout.puts("Bundled Java: #{bundled_java?}")
+
+      begin
+        out, status = Open3.capture2e(java_cmd, '-version')
+        raise unless status.success?
+      rescue
+        $stderr.puts('Failed to run java')
+        return 1
+      end
+      $stdout.puts(out)
+
+      version_op = List::CommandParser.new("workflow", [], [], nil, ['--version'], true)
+      $stdout.write('Digdag version: ')
+      workflow(version_op, capture_output=true, check_prereqs=false)
+    end
+
+    private
+    def system_java_cmd
+      if td_wf_java.nil? or td_wf_java.empty?
+        'java'
+      else
+        td_wf_java
+      end
+    end
+
+    private
+    def bundled_java?
+      if not td_wf_java.empty?
+        return false
+      end
+      return Helpers.on_64bit_os?
+    end
+
+    private
+    def td_wf_java
+      ENV.fetch('TD_WF_JAVA', '').strip
+    end
+
+    def digdag_base_url
+      'http://toolbelt.treasure-data.com/digdag'
+    end
+
+    private
+    def digdag_url
+      url = ENV.fetch('TD_DIGDAG_URL', '').strip
+      return url unless url.empty?
+      user = Config.read['account.user']
+      if user.nil? or user.strip.empty?
+        raise ConfigError
+      end
+      query = URI.encode_www_form('user' => user)
+      "#{digdag_base_url}?#{query}"
+    end
+
+    private
+    def digdag_dir
+      File.join(home_directory, '.td', 'digdag')
+    end
+
+    private
+    def digdag_tmp_dir
+      File.join(home_directory, '.td', 'digdag', 'tmp')
+    end
+
+    private
+    def digdag_cli_path
+      File.join(digdag_dir, 'digdag')
+    end
+
+    private
+    def digdag_jre_dir
+      File.join(digdag_dir, 'jre')
+    end
+
+    private
+    def digdag_jre_tmp_dir
+      File.join(digdag_dir, 'jre.tmp')
+    end
+
+    private
+    def java_cmd
+      if bundled_java?
+        digdag_java_path
+      else
+        system_java_cmd
+      end
+    end
+
+    private
+    def digdag_java_path
+      File.join(digdag_jre_dir, 'bin', 'java')
+    end
+
+    private
+    def digdag_cli_tmp_path
+      File.join(digdag_dir, 'digdag.tmp')
+    end
+
+    private
+    def jre_archive
+      # XXX (dano): platform detection could be more robust
+      if Helpers.on_64bit_os?
+        if Helpers.on_windows?
+          return 'win_x64'
+        elsif Helpers.on_mac?
+          return 'mac_x64'
+        else # Assume linux
+          return 'lin_x64'
+        end
+      end
+      raise 'OS architecture not supported'
+    end
+
+    private
+    def jre_url
+      base_url = ENV.fetch('TD_WF_JRE_BASE_URL', 'http://toolbelt.treasuredata.com/digdag/jdk/')
+      "#{base_url}#{jre_archive}"
+    end
+
+    private
+    def fail_system_java
+      raise WorkflowError, <<EOF
+A suitable installed version of Java could not be found and and Java cannot be
+automatically installed for this OS.
+
+Please install at least Java 8u71.
+EOF
+    end
+
+    private
+    def detect_system_java
+      begin
+        output, status = Open3.capture2e(system_java_cmd, '-version')
+      rescue => e
+        return false
+      end
+      unless status.success?
+        return false
+      end
+      if output =~ /openjdk version/ or output =~ /java version/
+        m = output.match(/version "(\d+)\.(\d+)\.(\d+)(?:_(\d+))"/)
+        if not m or m.size < 4
+          return false
+        end
+        # Check for at least Java 8. Let digdag itself verify revision.
+        major = m[1].to_i
+        minor = m[2].to_i
+        if major < 1 or minor < 8
+          return false
+        end
+      end
+      return true
+    end
+
+    private
+    def check_system_java
+      # Trust the user if they've specified a jre to use
+      if td_wf_java.empty?
+        unless detect_system_java
+          fail_system_java
+        end
+      end
+    end
+
+    # Follow all redirects and return the resulting url
+    def resolve_url(url)
+      require 'net/http'
+      require 'openssl'
+
+      uri = URI(url)
+      http_class = Command.get_http_class
+      http = http_class.new(uri.host, uri.port)
+
+      if uri.scheme == 'https'
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      http.request_get(uri.path + (uri.query ? '?' + uri.query : '')) {|response|
+        if response.class == Net::HTTPOK
+          return url
+        elsif response.class == Net::HTTPFound || \
+              response.class == Net::HTTPRedirection
+          unless ENV['TD_TOOLBELT_DEBUG'].nil?
+            $stdout.puts "redirect '#{url}' to '#{response['Location']}'... "
+          end
+          return resolve_url(response['Location'])
+        else
+          raise_error "An error occurred when fetching from '#{uri}' " +
+                          "(#{response.class.to_s}: #{response.message})."
+          return false
+        end
+      }
+    end
+
+    private
+    def download_java
+      if File.exists?(digdag_jre_dir)
+        return
+      end
+
+      require 'net/http'
+      require 'openssl'
+
+      Dir.mktmpdir do |download_dir|
+        indicator = Command::TimeBasedDownloadProgressIndicator.new(
+            'Downloading Java...', Time.new.to_i, 2)
+        status = nil
+        real_jre_uri = URI(resolve_url(jre_url))
+        jre_filename = Pathname.new(real_jre_uri.path).basename.to_s
+        download_path = File.join(download_dir, jre_filename)
+        File.open(download_path, 'wb') do |file|
+          status = Updater.stream_fetch(jre_url, file) {
+            indicator.update
+          }
+        end
+        indicator.finish
+
+        $stdout.puts
+
+        unless status
+          raise WorkflowError, 'Failed to download Java.'
+        end
+
+        $stdout.print 'Installing Java... '
+        FileUtils.rm_rf digdag_jre_tmp_dir
+        FileUtils.mkdir_p digdag_jre_tmp_dir
+        extract_archive(download_path, digdag_jre_tmp_dir, 1)
+        FileUtils.mv digdag_jre_tmp_dir, digdag_jre_dir
+        $stdout.puts 'done'
+      end
+    end
+
+    private
+    def extract_archive(archive, destination, strip)
+      if archive.end_with? '.tar.gz'
+        extract_tarball(archive, destination, strip)
+      elsif archive.end_with? '.zip'
+        extract_zip(archive, destination, strip)
+      end
+    end
+
+    private
+    def extract_zip(zip_archive, destination, strip)
+      require 'fileutils'
+      require 'zip/zip'
+      Zip::ZipFile.open(zip_archive) { |zip_file|
+        zip_file.each { |f|
+          stripped = strip_components(f.name, strip)
+          if stripped.empty?
+            next
+          end
+          dest = File.join destination, stripped
+          FileUtils.rm_rf dest if File.exist? dest
+          FileUtils.mkdir_p(File.dirname(dest))
+          zip_file.extract(f, dest)
+        }
+      }
+    end
+
+    # http://stackoverflow.com/a/31310593
+    TAR_LONGLINK = '././@LongLink'
+    private
+    def extract_tarball(tar_gz_archive, destination, strip)
+      require 'fileutils'
+      require 'rubygems/package'
+      require 'zlib'
+
+      Zlib::GzipReader.open tar_gz_archive do |gzip_reader|
+        Gem::Package::TarReader.new(gzip_reader) do |tar|
+          filename = nil
+          tar.each do |entry|
+            # Handle LongLink
+            if entry.full_name == TAR_LONGLINK
+              filename = entry.read.strip
+              next
+            end
+            filename ||= entry.full_name
+
+            # Strip path components
+            stripped = strip_components(filename, strip)
+            filename = nil
+            if stripped.empty?
+              next
+            end
+            dest = File.join destination, stripped
+
+            if entry.directory? || (entry.header.typeflag == '' && entry.full_name.end_with?('/'))
+              File.rm_rf dest if File.file? dest
+              FileUtils.mkdir_p dest, :mode => entry.header.mode, :verbose => false
+            elsif entry.file? || (entry.header.typeflag == '' && !entry.full_name.end_with?('/'))
+              FileUtils.rm_rf dest if File.exist? dest
+              FileUtils.mkdir_p File.dirname dest
+              File.open dest, "wb" do |f|
+                f.print entry.read
+              end
+              FileUtils.chmod entry.header.mode, dest, :verbose => false
+            elsif entry.header.typeflag == '2' #Symlink!
+              File.symlink entry.header.linkname, dest
+            else
+              raise "Unkown tar entry: #{entry.full_name} type: #{entry.header.typeflag}."
+            end
+          end
+        end
+      end
+    end
+
+    def strip_components(filename, strip)
+      File.join(Pathname.new(filename).each_filename.drop(strip))
+    end
+
+    private
+    def check_digdag_cli
+      check_system_java unless bundled_java?
+
+      unless File.exists?(digdag_cli_path)
+        $stderr.puts 'Workflow module not yet installed, download now? [Y/n]'
+        line = $stdin.gets
+        line.strip!
+        if (not line.empty?) and (line !~ /^y(?:es)?$/i)
+          raise WorkflowError, 'Aborted'
+        end
+        download_digdag
+      end
+    end
+
+    def download_digdag
+      require 'net/http'
+      require 'openssl'
+
+      FileUtils.mkdir_p digdag_dir
+
+      if bundled_java?
+        download_java
+      end
+
+      Dir.mktmpdir do |download_dir|
+        indicator = Command::TimeBasedDownloadProgressIndicator.new(
+            'Downloading workflow module...', Time.new.to_i, 2)
+        status = nil
+        download_path = File.join(download_dir, 'digdag')
+        File.open(download_path, 'wb') do |file|
+          status = Updater.stream_fetch(digdag_url, file) {
+            indicator.update
+          }
+        end
+        indicator.finish
+
+        $stdout.puts
+
+        unless status
+          raise WorkflowError, 'Failed to download workflow module.'
+        end
+
+        $stdout.print 'Installing workflow module... '
+        FileUtils.rm_rf(digdag_cli_tmp_path)
+        FileUtils.cp(download_path, digdag_cli_tmp_path)
+        FileUtils.chmod('a=xr', digdag_cli_tmp_path)
+        FileUtils.mv(digdag_cli_tmp_path, digdag_cli_path)
+        $stdout.puts 'done'
+      end
+    end
+  end
+end

data/lib/td/helpers.rb

@@ -17,5 +17,19 @@ module TreasureData
     def on_mac?
       RUBY_PLATFORM =~ /-darwin\d/
     end
+
+    def on_64bit_os?
+      if on_windows?
+        if ENV.fetch('PROCESSOR_ARCHITECTURE', '').downcase.include? 'amd64'
+          return true
+        end
+        return ENV.has_key?('PROCESSOR_ARCHITEW6432')
+      else
+        require 'open3'
+        out, status = Open3.capture2('uname', '-m')
+        raise 'Failed to detect OS bitness' unless status.success?
+        return out.downcase.include? 'x86_64'
+      end
+    end
   end
 end

data/lib/td/updater.rb

@@ -258,6 +258,7 @@ module ModuleDefinition
 
   def stream_fetch(url, binfile, &progress)
     require 'net/http'
+    require 'openssl'
 
     uri = URI(url)
     http_class = Command.get_http_class
@@ -268,7 +269,7 @@ module ModuleDefinition
       http.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
 
-    http.request_get(uri.path) {|response|
+    http.request_get(uri.path + (uri.query ? '?' + uri.query : '')) {|response|
       if response.class == Net::HTTPOK
         # $stdout.print a . every tick_period seconds
         response.read_body do |chunk|

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.14.1'
+  TOOLBELT_VERSION = '0.15.0'
 end

data/spec/td/command/connector_spec.rb

@@ -353,5 +353,127 @@ module TreasureData::Command
         expect(stdout_io.string).to include table
       end
     end
+
+    describe '#connector_update' do
+      let(:name)     { 'daily_mysql_import' }
+      let(:name2)     { 'daily_mysql_import2' }
+      let(:cron)     { '10 0 * * *' }
+      let(:cron2)    { '20 0 * * *' }
+      let(:database) { 'td_sample_db' }
+      let(:table)    { 'td_sample_table' }
+      let(:config_file) {
+        Tempfile.new('config.yml').tap {|tf|
+          tf.puts({"foo" => "bar"}.to_yaml)
+          tf.close
+        }
+      }
+      let(:config) {
+        h = YAML.load_file(config_file.path)
+        TreasureData::ConnectorConfigNormalizer.new(h).normalized_config
+      }
+      let(:config_diff_file) {
+        Tempfile.new('config_diff.yml').tap {|tf|
+          tf.puts({"hoge" => "fuga"}.to_yaml)
+          tf.close
+        }
+      }
+      let(:config_diff) {
+        h = YAML.load_file(config_diff_file.path)
+        TreasureData::ConnectorConfigNormalizer.new(h).normalized_config
+      }
+      let(:option) {
+        List::CommandParser.new("connector:update", %w(name), %w(config_file), nil, argv, true)
+      }
+      let(:response) {
+        {'name' => name, 'cron' => cron, 'timezone' => 'UTC', 'delay' => 0, 'database' => database, 'table' => table,
+          'config' => config, 'config_diff' => config_diff}
+      }
+      let(:client) { double(:client) }
+
+      before do
+        allow(command).to receive(:get_client).and_return(client)
+        allow(client).to receive(:bulk_load_update) do |name, settings|
+          r = response.merge('name' => name)
+          settings.each do |key, value|
+            value = nil if key == 'cron' && value.empty?
+            r[key.to_s] = value
+          end
+          r
+        end
+      end
+
+      context 'with new_name' do
+        let (:argv){ ['--newname', name2, name] }
+        it 'show update result' do
+          expect{command.connector_update(option)}.not_to raise_error(SystemExit)
+          expect(stdout_io.string).to include name2
+          expect(stdout_io.string).to include cron
+          expect(stdout_io.string).to include database
+          expect(stdout_io.string).to include table
+          expect(YAML.load(stdout_io.string[/^Config\n---\n(.*?\n)\n/m, 1])).to eq(config)
+          expect(YAML.load(stdout_io.string[/^Config Diff\n---\n(.*?\n)\Z/m, 1])).to eq(config_diff)
+        end
+      end
+
+      context 'with config' do
+        let (:argv){ [name, config_file.path] }
+        it 'show update result' do
+          expect{command.connector_update(option)}.not_to raise_error(SystemExit)
+          expect(stdout_io.string).to include name
+          expect(stdout_io.string).to include cron
+          expect(stdout_io.string).to include database
+          expect(stdout_io.string).to include table
+          expect(YAML.load(stdout_io.string[/^Config\n---\n(.*?\n)\n/m, 1])).to eq(config)
+          expect(YAML.load(stdout_io.string[/^Config Diff\n---\n(.*?\n)\Z/m, 1])).to eq(config_diff)
+        end
+      end
+
+      context 'with config_diff' do
+        let(:argv){ ['--config-diff', config_diff_file.path, name] }
+        it 'show update result' do
+          expect{command.connector_update(option)}.not_to raise_error(SystemExit)
+          expect(stdout_io.string).to include name
+          expect(stdout_io.string).to include cron
+          expect(stdout_io.string).to include database
+          expect(stdout_io.string).to include table
+          expect(YAML.load(stdout_io.string[/^Config\n---\n(.*?\n)\n/m, 1])).to eq(config)
+          expect(YAML.load(stdout_io.string[/^Config Diff\n---\n(.*?\n)\Z/m, 1])).to eq(config_diff)
+        end
+      end
+
+      context 'with schedule' do
+        let(:argv) { ['--schedule', cron2, name] }
+        it 'can update cron' do
+          expect{command.connector_update(option)}.not_to raise_error(SystemExit)
+          expect(stdout_io.string).to include name
+          expect(stdout_io.string).to include cron2
+          expect(stdout_io.string).to include database
+          expect(stdout_io.string).to include table
+          expect(YAML.load(stdout_io.string[/^Config\n---\n(.*?\n)\n/m, 1])).to eq(config)
+          expect(YAML.load(stdout_io.string[/^Config Diff\n---\n(.*?\n)\Z/m, 1])).to eq(config_diff)
+        end
+      end
+
+      context 'with empty schedule' do
+        let(:argv) { [name, '--schedule'] }
+        it 'can update cron' do
+          expect{command.connector_update(option)}.not_to raise_error(SystemExit)
+          expect(stdout_io.string).to include name
+          expect(stdout_io.string).to include "Cron     : \n"
+          expect(stdout_io.string).to include database
+          expect(stdout_io.string).to include table
+          expect(YAML.load(stdout_io.string[/^Config\n---\n(.*?\n)\n/m, 1])).to eq(config)
+          expect(YAML.load(stdout_io.string[/^Config Diff\n---\n(.*?\n)\Z/m, 1])).to eq(config_diff)
+        end
+      end
+
+      context 'nothing to update' do
+        let (:argv) { [name] }
+        it 'show update result' do
+          expect{command.connector_update(option)}.to raise_error(SystemExit)
+          expect(stdout_io.string).to include 'Error: nothing to update'
+        end
+      end
+    end
   end
 end

data/spec/td/command/workflow_spec.rb

@@ -0,0 +1,314 @@
+require 'spec_helper'
+require 'td/command/common'
+require 'td/command/list'
+require 'td/command/workflow'
+
+def java_available?
+  begin
+    output, status = Open3.capture2e('java', '-version')
+  rescue
+    return false
+  end
+  if not status.success?
+    return false
+  end
+  if output !~ /(openjdk|java) version "1/
+    return false
+  end
+  return true
+end
+
+STDERR.puts
+STDERR.puts("RUBY_PLATFORM: #{RUBY_PLATFORM}")
+STDERR.puts("on_64bit_os?: #{TreasureData::Helpers.on_64bit_os?}")
+STDERR.puts("java_available?: #{java_available?}")
+STDERR.puts
+
+module TreasureData::Command
+
+  describe 'workflow command' do
+
+    let(:command) {
+      Class.new { include TreasureData::Command }.new
+    }
+    let(:stdout_io) { StringIO.new }
+    let(:stderr_io) { StringIO.new }
+    let(:home_env) { TreasureData::Helpers.on_windows? ? 'USERPROFILE' : 'HOME' }
+    let(:java_exe) { TreasureData::Helpers.on_windows? ? 'java.exe' : 'java' }
+
+    around do |example|
+
+      stdout = $stdout.dup
+      stderr = $stderr.dup
+
+      begin
+        $stdout = stdout_io
+        $stderr = stderr_io
+
+        Dir.mktmpdir { |home|
+          with_env(home_env, home) {
+            example.run
+          }
+        }
+      ensure
+        $stdout = stdout
+        $stderr = stderr
+      end
+    end
+
+    def with_env(name, var)
+      backup, ENV[name] = ENV[name], var
+      begin
+        yield
+      ensure
+        ENV[name] = backup
+      end
+    end
+
+    let(:tmpdir) {
+      Dir.mktmpdir
+    }
+
+    let(:project_name) {
+      'foobar'
+    }
+
+    let(:project_dir) {
+      File.join(tmpdir, project_name)
+    }
+
+    let(:workflow_name) {
+      project_name
+    }
+
+    let(:workflow_file) {
+      File.join(project_dir, workflow_name + '.dig')
+    }
+
+    let(:td_conf) {
+      File.join(tmpdir, 'td.conf')
+    }
+
+    after(:each) {
+      FileUtils.rm_rf tmpdir
+    }
+
+    describe '#workflow' do
+      let(:empty_option) {
+        List::CommandParser.new("workflow", [], [], nil, [], true)
+      }
+
+      let(:version_option) {
+        List::CommandParser.new("workflow", [], [], nil, ['version'], true)
+      }
+
+      let(:init_option) {
+        List::CommandParser.new("workflow", [], [], nil, ['init', project_dir], true)
+      }
+
+      let(:run_option) {
+        List::CommandParser.new("workflow", [], [], nil, ['run', workflow_name], true)
+      }
+
+      let(:reset_option) {
+        List::CommandParser.new("workflow:reset", [], [], nil, [], true)
+      }
+
+      let(:version_option) {
+        List::CommandParser.new("workflow:version", [], [], nil, [], true)
+      }
+
+      let (:apikey) {
+        '4711/badf00d'
+      }
+
+      before(:each) {
+        allow(TreasureData::Config).to receive(:apikey) { apikey }
+        allow(TreasureData::Config).to receive(:path) { td_conf }
+        File.write(td_conf, [
+            '[account]',
+            '  user = test@example.com',
+            "  apikey = #{apikey}",
+        ].join($/) + $/)
+      }
+
+      it 'complains about 32 bit platform if no usable java on path' do
+        allow(TreasureData::Helpers).to receive(:on_64bit_os?) { false }
+        with_env('PATH', '') do
+          expect { command.workflow(empty_option, capture_output=true) }.to raise_error(WorkflowError) { |error|
+            expect(error.message).to include(<<EOF
+A suitable installed version of Java could not be found and and Java cannot be
+automatically installed for this OS.
+
+Please install at least Java 8u71.
+EOF
+                                     )
+          }
+        end
+      end
+
+      it 'uses system java by default on 32 bit platforms' do
+        allow(TreasureData::Helpers).to receive(:on_64bit_os?) { false }
+        expect(java_available?).to be(true)
+
+        allow(TreasureData::Updater).to receive(:stream_fetch).and_call_original
+        allow($stdin).to receive(:gets) { 'Y' }
+        status = command.workflow(empty_option, capture_output=true)
+        expect(status).to be 0
+        expect(stdout_io.string).to_not include 'Downloading Java'
+        expect(stdout_io.string).to include 'Downloading workflow module'
+        expect(File).to exist(File.join(ENV[home_env], '.td', 'digdag', 'digdag'))
+        expect(TreasureData::Updater).to_not have_received(:stream_fetch).with(
+            %r{/java/}, instance_of(File))
+        expect(TreasureData::Updater).to have_received(:stream_fetch).with(
+            'http://toolbelt.treasure-data.com/digdag?user=test%40example.com', instance_of(File))
+      end
+
+      it 'installs java + digdag and can run a workflow' do
+        skip 'Requires 64 bit OS or java' unless (TreasureData::Helpers::on_64bit_os? or java_available?)
+
+        allow(TreasureData::Updater).to receive(:stream_fetch).and_call_original
+        allow($stdin).to receive(:gets) { 'Y' }
+        status = command.workflow(empty_option, capture_output=true)
+        expect(status).to be 0
+        if TreasureData::Helpers::on_64bit_os?
+          expect(stdout_io.string).to include 'Downloading Java'
+          expect(File).to exist(File.join(ENV[home_env], '.td', 'digdag', 'jre', 'bin', java_exe))
+          expect(TreasureData::Updater).to have_received(:stream_fetch).with(
+              %r{/java/}, instance_of(File))
+        end
+        expect(stdout_io.string).to include 'Downloading workflow module'
+        expect(File).to exist(File.join(ENV[home_env], '.td', 'digdag', 'digdag'))
+        expect(TreasureData::Updater).to have_received(:stream_fetch).with(
+            'http://toolbelt.treasure-data.com/digdag?user=test%40example.com', instance_of(File))
+
+        # Check that java and digdag is not re-installed
+        stdout_io.truncate(0)
+        stderr_io.truncate(0)
+        status = command.workflow(empty_option, capture_output=true)
+        expect(status).to be 0
+        expect(stdout_io.string).to_not include 'Downloading Java'
+        expect(stdout_io.string).to_not include 'Downloading workflow module'
+
+        # Generate a new project
+        expect(Dir.exist? project_dir).to be(false)
+        stdout_io.truncate(0)
+        stderr_io.truncate(0)
+        status = command.workflow(init_option, capture_output=true)
+        expect(status).to be 0
+        expect(stdout_io.string).to include('Creating')
+        expect(Dir.exist? project_dir).to be(true)
+        expect(File.exist? workflow_file).to be(true)
+
+        # Run a workflow
+        File.write(workflow_file, <<EOF
++main:
+  echo>: hello world
+EOF
+)
+        Dir.chdir(project_dir) {
+          stdout_io.truncate(0)
+          stderr_io.truncate(0)
+          status = command.workflow(run_option, capture_output=true)
+          expect(status).to be 0
+          expect(stderr_io.string).to include('Success')
+          expect(stdout_io.string).to include('hello world')
+        }
+      end
+
+      it 'uses specified java and installs digdag' do
+        with_env('TD_WF_JAVA', 'java') {
+          allow(TreasureData::Updater).to receive(:stream_fetch).and_call_original
+          allow($stdin).to receive(:gets) { 'Y' }
+          status = command.workflow(empty_option, capture_output=true)
+          expect(status).to be 0
+          expect(stdout_io.string).to_not include 'Downloading Java'
+          expect(stdout_io.string).to include 'Downloading workflow module'
+          expect(File).to exist(File.join(ENV[home_env], '.td', 'digdag', 'digdag'))
+          expect(TreasureData::Updater).to_not have_received(:stream_fetch).with(
+              %r{/java/}, instance_of(File))
+          expect(TreasureData::Updater).to have_received(:stream_fetch).with(
+              'http://toolbelt.treasure-data.com/digdag?user=test%40example.com', instance_of(File))
+
+          # Check that digdag is not re-installed
+          stdout_io.truncate(0)
+          stderr_io.truncate(0)
+          status = command.workflow(empty_option, capture_output=true)
+          expect(status).to be 0
+          expect(stdout_io.string).to_not include 'Downloading Java'
+          expect(stdout_io.string).to_not include 'Downloading workflow module'
+        }
+      end
+
+      it 'reinstalls cleanly after reset' do
+        skip 'Requires 64 bit OS or system java' unless (TreasureData::Helpers::on_64bit_os? or java_available?)
+
+        # First install
+        allow($stdin).to receive(:gets) { 'Y' }
+        status = command.workflow(empty_option, capture_output=true)
+        expect(status).to be 0
+        expect(stderr_io.string).to include 'Digdag v'
+        expect(File).to exist(File.join(ENV[home_env], '.td', 'digdag'))
+
+        # Reset
+        stdout_io.truncate(0)
+        stderr_io.truncate(0)
+        status = command.workflow_reset(reset_option)
+        expect(status).to be 0
+        expect(File).to_not exist(File.join(ENV[home_env], '.td', 'digdag'))
+        expect(File).to exist(File.join(ENV[home_env], '.td'))
+        expect(stdout_io.string).to include 'Removing workflow module...'
+        expect(stdout_io.string).to include 'Done'
+
+        # Reinstall
+        allow($stdin).to receive(:gets) { 'Y' }
+        stdout_io.truncate(0)
+        stderr_io.truncate(0)
+        status = command.workflow(empty_option, capture_output=true)
+        expect(status).to be 0
+        expect(stderr_io.string).to include 'Digdag v'
+        expect(File).to exist(File.join(ENV[home_env], '.td', 'digdag'))
+      end
+
+      it 'uses -k apikey' do
+        with_env('TD_WF_JAVA', 'echo') {
+          allow(TreasureData::Config).to receive(:cl_apikey) { true }
+          stdout_io.truncate(0)
+          stderr_io.truncate(0)
+          status = command.workflow(init_option, capture_output=true, check_prereqs=false)
+          expect(status).to be 0
+          expect(stdout_io.string).to include('--config')
+          expect(stdout_io.string).to_not include('io.digdag.standards.td.client-configurator.enabled=true')
+        }
+      end
+
+      it 'complains if there is no apikey' do
+          allow(TreasureData::Config).to receive(:apikey) { nil}
+          expect{command.workflow(version_option)}.to raise_error(TreasureData::ConfigError)
+      end
+
+      it 'prints the java and digdag versions' do
+        skip 'Requires 64 bit OS' unless TreasureData::Helpers::on_64bit_os?
+
+        # Not yet installed
+        status = command.workflow_version(version_option)
+        expect(status).to be 1
+        expect(stderr_io.string).to include 'Workflow module not yet installed.'
+
+        # Install
+        allow($stdin).to receive(:gets) { 'Y' }
+        status = command.workflow(empty_option, capture_output=true)
+        expect(status).to be 0
+
+        # Check that version is shown
+        stdout_io.truncate(0)
+        stderr_io.truncate(0)
+        status = command.workflow_version(version_option)
+        expect(status).to be 0
+        expect(stdout_io.string).to include 'Bundled Java: true'
+        expect(stdout_io.string).to include 'openjdk version'
+        expect(stdout_io.string).to include 'Digdag version:'
+      end
+    end
+  end
+end

data/spec/td/helpers_spec.rb

@@ -1,7 +1,9 @@
 require 'spec_helper'
 require 'td/helpers'
+require 'open3'
 
 module TreasureData
+
   describe 'format_with_delimiter' do
     it "delimits the number with ',' by default" do
       expect(Helpers.format_with_delimiter(0)).to eq("0")
@@ -13,4 +15,57 @@ module TreasureData
       expect(Helpers.format_with_delimiter(1000000)).to eq("1,000,000")
     end
   end
+
+  describe 'on_64bit_os?' do
+
+    def with_env(name, var)
+      backup, ENV[name] = ENV[name], var
+      begin
+        yield
+      ensure
+        ENV[name] = backup
+      end
+    end
+
+    it 'returns true for windows when PROCESSOR_ARCHITECTURE=amd64' do
+      allow(Helpers).to receive(:on_windows?) {true}
+      with_env('PROCESSOR_ARCHITECTURE', 'amd64') {
+        expect(Helpers.on_64bit_os?).to be(true)
+      }
+    end
+
+    it 'returns true for windows when PROCESSOR_ARCHITECTURE=x86 and PROCESSOR_ARCHITEW6432 is set' do
+      allow(Helpers).to receive(:on_windows?) {true}
+      with_env('PROCESSOR_ARCHITECTURE', 'x86') {
+        with_env('PROCESSOR_ARCHITEW6432', '') {
+          expect(Helpers.on_64bit_os?).to be(true)
+        }
+      }
+    end
+
+    it 'returns false for windows when PROCESSOR_ARCHITECTURE=x86 and PROCESSOR_ARCHITEW6432 is not set' do
+      allow(Helpers).to receive(:on_windows?) {true}
+      with_env('PROCESSOR_ARCHITECTURE', 'x86') {
+        with_env('PROCESSOR_ARCHITEW6432', nil) {
+          expect(Helpers.on_64bit_os?).to be(false)
+        }
+      }
+    end
+
+    it 'returns true for non-windows when uname -m prints x86_64' do
+      allow(Helpers).to receive(:on_windows?) {false}
+      allow(Open3).to receive(:capture2).with('uname', '-m') {['x86_64', double(:success? => true)]}
+      expect(Helpers.on_64bit_os?).to be(true)
+      expect(Open3).to have_received(:capture2).with('uname', '-m')
+    end
+
+    it 'returns false for non-windows when uname -m prints i686' do
+      allow(Helpers).to receive(:on_windows?) {false}
+      allow(Open3).to receive(:capture2).with('uname', '-m') {['i686', double(:success? => true)]}
+      expect(Helpers.on_64bit_os?).to be(false)
+      expect(Open3).to have_received(:capture2).with('uname', '-m')
+    end
+
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.15.0
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-11 00:00:00.000000000 Z
+date: 2016-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -269,6 +269,7 @@ files:
 - lib/td/command/table.rb
 - lib/td/command/update.rb
 - lib/td/command/user.rb
+- lib/td/command/workflow.rb
 - lib/td/compact_format_yamler.rb
 - lib/td/compat_core.rb
 - lib/td/compat_gzip_reader.rb
@@ -293,6 +294,7 @@ files:
 - spec/td/command/query_spec.rb
 - spec/td/command/sched_spec.rb
 - spec/td/command/table_spec.rb
+- spec/td/command/workflow_spec.rb
 - spec/td/common_spec.rb
 - spec/td/compact_format_yamler_spec.rb
 - spec/td/connector_config_normalizer_spec.rb
@@ -343,6 +345,7 @@ test_files:
 - spec/td/command/query_spec.rb
 - spec/td/command/sched_spec.rb
 - spec/td/command/table_spec.rb
+- spec/td/command/workflow_spec.rb
 - spec/td/common_spec.rb
 - spec/td/compact_format_yamler_spec.rb
 - spec/td/connector_config_normalizer_spec.rb