bricolage 5.21.0 → 5.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 38fa88f0cfbee0ae3d6cca03c6a50d9b148c2e3b
-  data.tar.gz: 29c84154960c9a298bb0966beaf5b37586a09636
+  metadata.gz: e6894dfe2fad736476fc5c77d3e9f271d211e9e7
+  data.tar.gz: 7121e86bc937c14b91cbcfe3035dcde4e08aa027
 SHA512:
-  metadata.gz: 6fa588561652d3108036557074222f77f8ea21e88141cfd9fa6d3cfcf6a81a32f0ec068cc05fb78eb8014ec8c925daf20b04818ff9c4ba0da42117f9d6db734e
-  data.tar.gz: 20c5afefe133f3f49da1e3bb9b9d5f31aaa86c90fa1ef190fb6cdec31926d5366da10bd0c8959bea21c94a7aaf7cbffd77028d85ae03194ae69f76171d5da1ca
+  metadata.gz: 46411897c454153c4dd2b726a6d9427fe49b3ab8eb4157e2d3054ba09e623709727ddec5124be46ae02777b0c7da3c037b2a2d253dec7a68680ced7dadf96c3b
+  data.tar.gz: 4fe3715fee1220d98eecab7cbe1e3cdd1fe89ee14f19008017dc1f8aea0ad1ca81507f25c8026551098246fce31ddd4a8af720cc9bab2de94ce8ad5c9e7ff231

data/lib/bricolage/application.rb

@@ -2,13 +2,16 @@ require 'bricolage/context'
 require 'bricolage/job'
 require 'bricolage/jobclass'
 require 'bricolage/jobresult'
+require 'bricolage/jobnet'
 require 'bricolage/variables'
 require 'bricolage/datasource'
 require 'bricolage/eventhandlers'
 require 'bricolage/postgresconnection'
+require 'bricolage/logfilepath'
 require 'bricolage/logger'
 require 'bricolage/exception'
 require 'bricolage/version'
+require 'fileutils'
 require 'pathname'
 require 'optparse'
 
@@ -27,6 +30,7 @@ module Bricolage
 
     def initialize
       @hooks = Bricolage
+      @start_time = Time.now
     end
 
     def main
@@ -60,7 +64,9 @@ module Bricolage
 
       @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(job.id, [job]))
       @hooks.run_before_job_hooks(BeforeJobEvent.new(job))
-      result = job.execute
+      result = redirect_log_to_file(opts.log_path, job) {
+        job.execute
+      }
       @hooks.run_after_job_hooks(AfterJobEvent.new(result))
       @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(result.success?, [job]))
       exit result.status
@@ -72,6 +78,30 @@ module Bricolage
       error_exit ex.message
     end
 
+    def redirect_log_to_file(log_path, job)
+      return yield unless log_path
+
+      path = log_path.format(
+        job_ref: JobNet::JobRef.new(job.subsystem, job.id, '-'),
+        jobnet_id: "#{job.subsystem}/#{job.id}",
+        job_start_time: @start_time,
+        jobnet_start_time: @start_time
+      )
+      FileUtils.mkdir_p File.dirname(path)
+      stdout_save = $stdout.dup
+      stderr_save = $stderr.dup
+      begin
+        File.open(path, 'w+') {|f|
+          $stdout.reopen f
+          $stderr.reopen f
+        }
+        return yield
+      ensure
+        $stdout.reopen stdout_save; stdout_save.close
+        $stderr.reopen stderr_save; stderr_save.close
+      end
+    end
+
     def load_job(ctx, opts)
       if opts.file_mode?
         Job.load_file(opts.job_file, ctx)
@@ -154,6 +184,7 @@ module Bricolage
       @global_variables = Variables.new
       @dry_run = false
       @explain = false
+      @log_path = LogFilePath.default
       @list_global_variables = false
       @list_variables = false
       @list_declarations = false
@@ -189,6 +220,12 @@ Global Options:
       parser.on('-E', '--explain', 'Applies EXPLAIN to the SQL.') {
         @explain = true
       }
+      parser.on('--log-dir=PATH', 'Log file prefix.') {|path|
+        @log_path = LogFilePath.new("#{path}/%{std}.log")
+      }
+      parser.on('--log-path=PATH', 'Log file path template.') {|path|
+        @log_path = LogFilePath.new(path)
+      }
       parser.on('--list-job-class', 'Lists job class name and (internal) class path.') {
         JobClass.list.each do |name|
           puts name
@@ -237,6 +274,7 @@ Global Options:
     attr_reader :global_variables
 
     attr_reader :job_file
+    attr_reader :log_path
 
     def file_mode?
       !!@job_file

data/lib/bricolage/context.rb

@@ -68,6 +68,10 @@ module Bricolage
         logger: @logger)
     end
 
+    def subsystem_name
+      @filesystem.scope
+    end
+
     extend Forwardable
     def_delegators '@filesystem',
       :scoped?,

data/lib/bricolage/filesystem.rb

@@ -36,6 +36,10 @@ module Bricolage
       false
     end
 
+    def scope
+      nil
+    end
+
     attr_reader :path
     attr_reader :environment
 
@@ -158,6 +162,10 @@ module Bricolage
       true
     end
 
+    def scope
+      @id
+    end
+
     def root
       @parent.root
     end

data/lib/bricolage/job.rb

@@ -50,6 +50,10 @@ module Bricolage
       @job_class.id
     end
 
+    def subsystem
+      @context.subsystem_name
+    end
+
     def init_global_variables
       # Context#global_variables loads file on each call, fix global variables here.
       @global_variables = @context.global_variables

data/lib/bricolage/jobnetrunner.rb

@@ -7,6 +7,7 @@ require 'bricolage/jobresult'
 require 'bricolage/datasource'
 require 'bricolage/variables'
 require 'bricolage/eventhandlers'
+require 'bricolage/logfilepath'
 require 'bricolage/logger'
 require 'bricolage/exception'
 require 'bricolage/version'
@@ -25,6 +26,7 @@ module Bricolage
       @hooks = ::Bricolage
       @jobnet_id = nil
       @jobnet_start_time = Time.now
+      @job_start_time = nil
       @log_path = nil
     end
 
@@ -123,10 +125,11 @@ module Bricolage
 
     def execute_job(ref, queue)
       logger.debug "job #{ref}"
+      @job_start_time = Time.now
       job = Job.load_ref(ref, @ctx)
       job.compile
       @hooks.run_before_job_hooks(BeforeJobEvent.new(ref))
-      result = job.execute_in_process(make_log_path(ref))
+      result = job.execute_in_process(instanciate_log_path(ref))
       @hooks.run_after_job_hooks(AfterJobEvent.new(result))
       result
     rescue Exception => ex
@@ -135,22 +138,14 @@ module Bricolage
       JobResult.error(ex)
     end
 
-    def make_log_path(job_ref)
+    def instanciate_log_path(ref)
       return nil unless @log_path
-      start_time = Time.now
-      @log_path.gsub(/%\{\w+\}/) {|var|
-        case var
-        when '%{jobnet_start_date}' then @jobnet_start_time.strftime('%Y%m%d')
-        when '%{jobnet_start_time}' then @jobnet_start_time.strftime('%Y%m%d_%H%M%S%L')
-        when '%{job_start_date}' then start_time.strftime('%Y%m%d')
-        when '%{job_start_time}' then start_time.strftime('%Y%m%d_%H%M%S%L')
-        when '%{jobnet}', '%{net}', '%{jobnet_id}', '%{net_id}', '%{flow}', '%{flow_id}' then @jobnet_id.gsub('/', '::')
-        when '%{subsystem}' then job_ref.subsystem
-        when '%{job}', '%{job_id}' then job_ref.name
-        else
-          raise ParameterError, "bad log path variable: #{var}"
-        end
-      }
+      @log_path.format(
+        job_ref: ref,
+        jobnet_id: @jobnet_id,
+        job_start_time: @job_start_time,
+        jobnet_start_time: @jobnet_start_time
+      )
     end
 
     public :puts
@@ -180,7 +175,7 @@ module Bricolage
         @app = app
         @environment = nil
         @jobnet_files = nil
-        @log_path = nil
+        @log_path = LogFilePath.default
         @queue_path = nil
         @check_only = false
         @list_jobs = false
@@ -191,8 +186,8 @@ module Bricolage
 
       attr_reader :environment
       attr_reader :jobnet_file
-      attr_reader :log_path
       attr_reader :queue_path
+      attr_reader :log_path
 
       def queue_exist?
         !!@queue_path
@@ -221,8 +216,11 @@ Options:
         parser.on('-e', '--environment=NAME', "Sets execution environment. [default: #{Context::DEFAULT_ENV}]") {|env|
           @environment = env
         }
+        parser.on('--log-dir=PATH', 'Log file prefix.') {|path|
+          @log_path = LogFilePath.new("#{path}/%{std}.log")
+        }
         parser.on('--log-path=PATH', 'Log file path template.') {|path|
-          @log_path = path
+          @log_path = LogFilePath.new(path)
         }
         parser.on('--queue=PATH', 'Use job queue.') {|path|
           @queue_path = Pathname(path)

data/lib/bricolage/logfilepath.rb

@@ -0,0 +1,76 @@
+module Bricolage
+  class LogFilePath
+    def LogFilePath.default
+      if dir = ENV['BRICOLAGE_LOG_DIR']
+        new("#{dir}/%{std}.log")
+      elsif path = ENV['BRICOLAGE_LOG_PATH']
+        new(path)
+      else
+        nil
+      end
+    end
+
+    def initialize(template)
+      @template = template
+    end
+
+    Params = Struct.new(:job_ref, :jobnet_id, :job_start_time, :jobnet_start_time)
+
+    def format(job_ref:, jobnet_id:, job_start_time:, jobnet_start_time:)
+      return nil unless @template
+      params = Params.new(job_ref, jobnet_id, job_start_time, jobnet_start_time)
+      fill_template(@template, params)
+    end
+
+    def fill_template(template, params)
+      template.gsub(/%\{\w+\}/) {|var|
+        case var
+        when '%{std}' then standard_format(params)
+        when '%{jobnet_start_date}' then jobnet_start_date(params)
+        when '%{jobnet_start_time}' then jobnet_start_time(params)
+        when '%{job_start_date}' then job_start_date(params)
+        when '%{job_start_time}' then job_start_time(params)
+        when '%{jobnet}', '%{net}', '%{jobnet_id}', '%{net_id}', '%{flow}', '%{flow_id}' then jobnet_id(params)
+        when '%{subsystem}' then subsystem_name(params)
+        when '%{job}', '%{job_id}' then job_name(params)
+        else
+          raise ParameterError, "bad log path variable: #{var}"
+        end
+      }
+    end
+
+    STD_TEMPLATE = '%{jobnet_start_date}/%{jobnet}/%{jobnet_start_time}/%{subsystem}-%{job}'
+
+    def standard_format(params)
+      fill_template(STD_TEMPLATE, params)
+    end
+
+    def jobnet_start_date(params)
+      params.jobnet_start_time.strftime('%Y%m%d')
+    end
+
+    def jobnet_start_time(params)
+      params.jobnet_start_time.strftime('%Y%m%d_%H%M%S%L')
+    end
+
+    def job_start_date(params)
+      params.start_time.strftime('%Y%m%d')
+    end
+
+    def job_start_time(params)
+      params.start_time.strftime('%Y%m%d_%H%M%S%L')
+    end
+
+    def jobnet_id(params)
+      params.jobnet_id.gsub('/', '::')
+    end
+
+    def subsystem_name(params)
+      params.job_ref.subsystem
+    end
+
+    def job_name(params)
+      params.job_ref.name
+    end
+  end
+end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.21.0'
+  VERSION = '5.22.0'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.21.0)
+    bricolage (5.22.0)
       aws-sdk (~> 2)
       mysql2
       pg (~> 0.18.0)

data/test/home/log/20170117/subsys::job1/20170117_202014082/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:20:14 +0900: INFO: development environment
+2017-01-17 20:20:14 +0900: INFO: 0.00 secs
+2017-01-17 20:20:14 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::job1/20170117_202334852/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:23:34 +0900: INFO: development environment
+2017-01-17 20:23:34 +0900: INFO: 0.00 secs
+2017-01-17 20:23:34 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::job1/20170117_202405056/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:24:05 +0900: INFO: development environment
+2017-01-17 20:24:05 +0900: INFO: 0.00 secs
+2017-01-17 20:24:05 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::migrate/20170117_202548409/subsys-migrate.log

@@ -0,0 +1,80 @@
+2017-01-17 20:25:48 +0900: INFO: development environment
+2017-01-17 20:25:48 +0900: INFO: [CMD] "/Users/minero-aoki/c/bricolage/libexec/sqldump" "--json" "127.0.0.1" "9011" "work_readonly" "main" "select * from main.search_backends;" | "gzip" "{:out=>"/tmp/bricolagedev/search_backends.json.gz"}"
+2017-01-17 20:25:48.540819891 +0900 JST: [SQL] select * from main.search_backends;
+2017-01-17 20:25:48.597928759 +0900 JST: query returned
+2017-01-17 20:25:48.598038993 +0900 JST: Total 11 records
+2017-01-17 20:25:48 +0900: INFO: aws s3 cp /tmp/bricolagedev/search_backends.json.gz s3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz
+2017-01-17 20:25:51 +0900: INFO: 3.28 secs
+2017-01-17 20:25:51 +0900: ERROR: Aws::Errors::MissingCredentialsError: unable to sign request without credentials set
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/request_signer.rb:104:in `require_credentials'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_request_signer.rb:14:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/xml/error_handler.rb:8:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/helpful_socket_errors.rb:10:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_request_signer.rb:65:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_redirects.rb:15:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/retry_errors.rb:87:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_dualstack.rb:32:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_accelerate.rb:49:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_md5s.rb:31:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_expect_100_continue.rb:21:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_bucket_name_restrictions.rb:12:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_bucket_dns.rb:31:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/rest/handler.rb:7:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/user_agent.rb:12:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/endpoint.rb:41:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/param_validator.rb:21:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/raise_response_errors.rb:14:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_sse_cpk.rb:19:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_dualstack.rb:24:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_accelerate.rb:34:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/idempotency_token.rb:18:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/param_converter.rb:20:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/response_target.rb:21:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/request.rb:70:in `send_request'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/base.rb:207:in `block (2 levels) in define_operation_methods'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:42:in `block in put_object'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:49:in `open_file'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:41:in `put_object'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:34:in `upload'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/object.rb:251:in `upload_file'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:188:in `block in run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:165:in `block in each_src_dest'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:163:in `each'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:163:in `each_src_dest'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:186:in `run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:82:in `block (2 levels) in run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:81:in `each'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:81:in `block in run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/datasource.rb:136:in `open'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/datasource.rb:140:in `open_for_batch'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:80:in `run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:47:in `block in run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:46:in `each'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:46:in `run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/job.rb:143:in `block in execute'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/logger.rb:44:in `with_elapsed_time'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/job.rb:142:in `execute'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:68:in `block in main'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:97:in `redirect_log_to_file'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:67:in `main'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:28:in `main'
+	/Users/minero-aoki/c/bricolage/bin/bricolage:6:in `<top (required)>'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/bin/bricolage:23:in `load'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/bin/bricolage:23:in `<top (required)>'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:74:in `load'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:74:in `kernel_load'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:27:in `run'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:332:in `exec'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/invocation.rb:126:in `invoke_command'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor.rb:359:in `dispatch'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:20:in `dispatch'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/base.rb:440:in `start'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:11:in `start'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/exe/bundle:34:in `block in <top (required)>'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/friendly_errors.rb:100:in `with_friendly_errors'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/exe/bundle:26:in `<top (required)>'
+	/Users/minero-aoki/.rbenv/versions/2.3/bin/bundle:23:in `load'
+	/Users/minero-aoki/.rbenv/versions/2.3/bin/bundle:23:in `<main>'
+
+2017-01-17 20:25:51 +0900: ERROR: error: Aws::Errors::MissingCredentialsError: unable to sign request without credentials set

data/test/home/log/20170117/subsys::migrate/20170117_202636115/subsys-migrate.log

@@ -0,0 +1,79 @@
+2017-01-17 20:26:36 +0900: INFO: development environment
+2017-01-17 20:26:36 +0900: INFO: [CMD] "/Users/minero-aoki/c/bricolage/libexec/sqldump" "--json" "127.0.0.1" "9011" "work_readonly" "main" "select * from main.search_backends;" | "gzip" "{:out=>"/tmp/bricolagedev/search_backends.json.gz"}"
+2017-01-17 20:26:36.214888001 +0900 JST: [SQL] select * from main.search_backends;
+2017-01-17 20:26:36.27827597 +0900 JST: query returned
+2017-01-17 20:26:36.278393544 +0900 JST: Total 11 records
+2017-01-17 20:26:36 +0900: INFO: aws s3 cp /tmp/bricolagedev/search_backends.json.gz s3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz
+rm -f /tmp/bricolagedev/search_backends.json.gz
+2017-01-17 20:26:36 +0900: INFO: command: psql --no-psqlrc --host=dwh.ckpd.co --port=5439 --username=aamine production --echo-all -v ON_ERROR_STOP=true -f /var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594 --no-password
+\timing on
+Timing is on.
+begin transaction;
+BEGIN
+Time: 8.269 ms
+drop table if exists aamine.search_backends_old cascade;
+psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:5: INFO:  Table "search_backends_old" does not exist and will be skipped
+DROP TABLE
+Time: 8.647 ms
+drop table if exists aamine.search_backends_wk cascade;
+psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:7: INFO:  Table "search_backends_wk" does not exist and will be skipped
+DROP TABLE
+Time: 8.031 ms
+/* /Users/minero-aoki/c/bricolage/test/home/subsys/search_backends.ct */
+create table aamine.search_backends_wk
+( id int
+, role varchar(10)
+, group_id smallint
+, index_type varchar(32)
+, config_key varchar(32)
+, updated_at timestamp
+)
+distkey (id)
+sortkey (id)
+;
+CREATE TABLE
+Time: 521.167 ms
+copy aamine.search_backends_wk
+from 's3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz'
+credentials 'aws_iam_role=arn:aws:iam::789035092620:role/RedshiftDWHTeamS3Access'
+json 'auto'
+statupdate false
+compupdate false
+emptyasnull
+gzip
+;
+psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:30: INFO:  Load into table 'search_backends_wk' completed, 11 record(s) loaded successfully.
+COPY
+Time: 1235.569 ms
+grant select on aamine.search_backends_wk to aamine;
+GRANT
+Time: 8.580 ms
+analyze aamine.search_backends_wk;
+ANALYZE
+Time: 43.441 ms
+create table if not exists aamine.search_backends (x int);
+psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:36: INFO:  Relation "search_backends" already exists and will be skipped
+CREATE TABLE
+Time: 7.768 ms
+alter table aamine.search_backends rename to search_backends_old;
+ALTER TABLE
+Time: 10.037 ms
+alter table aamine.search_backends_wk rename to search_backends;
+ALTER TABLE
+Time: 11.686 ms
+commit;
+COMMIT
+Time: 8482.452 ms
+drop table if exists aamine.search_backends_old cascade;
+DROP TABLE
+Time: 4801.919 ms
+;
+Time: 8.088 ms
+vacuum sort only aamine.search_backends;
+VACUUM
+Time: 36970.832 ms
+;
+Time: 8.746 ms
+2017-01-17 20:27:29 +0900: INFO: status: 0 (pid 69478 exit 0)
+2017-01-17 20:27:29 +0900: INFO: 53.05 secs
+2017-01-17 20:27:29 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:30:22 +0900: INFO: development environment
+2017-01-17 20:30:22 +0900: INFO: 0.00 secs
+2017-01-17 20:30:22 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job2.log

@@ -0,0 +1,3 @@
+2017-01-17 20:30:22 +0900: INFO: development environment
+2017-01-17 20:30:22 +0900: INFO: 0.00 secs
+2017-01-17 20:30:22 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job3.log

@@ -0,0 +1,3 @@
+2017-01-17 20:30:22 +0900: INFO: development environment
+2017-01-17 20:30:22 +0900: INFO: 0.00 secs
+2017-01-17 20:30:22 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job4.log

@@ -0,0 +1,3 @@
+2017-01-17 20:30:22 +0900: INFO: development environment
+2017-01-17 20:30:22 +0900: INFO: 0.00 secs
+2017-01-17 20:30:22 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:31:02 +0900: INFO: development environment
+2017-01-17 20:31:02 +0900: INFO: 0.00 secs
+2017-01-17 20:31:02 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job2.log

@@ -0,0 +1,3 @@
+2017-01-17 20:31:02 +0900: INFO: development environment
+2017-01-17 20:31:02 +0900: INFO: 0.00 secs
+2017-01-17 20:31:02 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job3.log

@@ -0,0 +1,3 @@
+2017-01-17 20:31:02 +0900: INFO: development environment
+2017-01-17 20:31:02 +0900: INFO: 0.00 secs
+2017-01-17 20:31:02 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job4.log

@@ -0,0 +1,3 @@
+2017-01-17 20:31:02 +0900: INFO: development environment
+2017-01-17 20:31:02 +0900: INFO: 0.00 secs
+2017-01-17 20:31:02 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:32:32 +0900: INFO: development environment
+2017-01-17 20:32:32 +0900: INFO: 0.00 secs
+2017-01-17 20:32:32 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job2.log

@@ -0,0 +1,3 @@
+2017-01-17 20:32:32 +0900: INFO: development environment
+2017-01-17 20:32:32 +0900: INFO: 0.00 secs
+2017-01-17 20:32:32 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job3.log

@@ -0,0 +1,3 @@
+2017-01-17 20:32:32 +0900: INFO: development environment
+2017-01-17 20:32:32 +0900: INFO: 0.00 secs
+2017-01-17 20:32:32 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job4.log

@@ -0,0 +1,3 @@
+2017-01-17 20:32:32 +0900: INFO: development environment
+2017-01-17 20:32:32 +0900: INFO: 0.00 secs
+2017-01-17 20:32:32 +0900: INFO: SUCCESS

data/test/home/subsys/my-import-delta.sql.job

@@ -2,7 +2,7 @@
 class: my-import-delta
 src-ds: mysql_summary
 src-tables:
-    item_pv: summarized_data.daily_recipe_pvs
+    item_pv: item_pv
 s3-ds: s3
 s3-prefix: bricolage/my-import-delta/$data_date/data.
 gzip: true
@@ -28,13 +28,13 @@ analyze: true
 */
 
 select
-    coalesce(`date`) as data_date
-    , coalesce(recipe_id) as item_id
-    , coalesce(page_view) as pv
-    , coalesce(unique_ip) as uu
+    data_date
+    , item_id
+    , pv
+    , uu
 from
     $item_pv
 where
-    `date` = date '$data_date'
+    date = '$data_date'
     and (@PARTITION_CONDITION@)
 ;

data/test/test_c_streaming_load.rb

@@ -5,7 +5,7 @@ Bricolage::JobClass.get('streaming_load')
 module Bricolage
   class TestStreamingLoadJobClass_S3Queue < Test::Unit::TestCase
     def test_compile_name_pattern
-      q = StreamingLoadJobClass::S3Queue.new(data_source: nil, queue_path: nil, persistent_path: nil, file_name: nil, logger: nil)
+      q = StreamingLoadJobClass::S3Queue.new(data_source: nil, queue_path: nil, ctl_prefix: nil, persistent_path: nil, file_name: nil, logger: nil)
       re = q.compile_name_pattern("%*%Y%m%d-%H%M_%Q.gz")
       assert_equal /\A[^\/]*(?<year>\d{4})(?<month>\d{2})(?<day>\d{2})\-(?<hour>\d{2})(?<minute>\d{2})_(?<seq>\d+)\.gz\z/, re
     end

data/test/test_logfilepath.rb

@@ -0,0 +1,19 @@
+require 'test/unit'
+require 'bricolage/logfilepath'
+require 'bricolage/jobnet'
+
+module Bricolage
+  class TestLogFilePath < Test::Unit::TestCase
+    test '#format' do
+      path = LogFilePath.new('./log/%{std}.log')
+      ref = JobNet::JobRef.new('subsys', 'somejob', '-')
+      start = Time.new(2012,3,4,12,34,56)
+      assert_equal './log/20120304/subsys::rebuild/20120304_123456000/subsys-somejob.log', path.format(
+        job_ref: ref,
+        jobnet_id: 'subsys/rebuild',
+        job_start_time: start,
+        jobnet_start_time: start
+      )
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.21.0
+  version: 5.22.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -187,6 +187,7 @@ files:
 - lib/bricolage/jobnet.rb
 - lib/bricolage/jobnetrunner.rb
 - lib/bricolage/jobresult.rb
+- lib/bricolage/logfilepath.rb
 - lib/bricolage/logger.rb
 - lib/bricolage/mysqldatasource.rb
 - lib/bricolage/parameters.rb
@@ -222,6 +223,23 @@ files:
 - test/home/data/20141002-1355_02.txt
 - test/home/data/test.txt
 - test/home/jobnet-test.rb
+- test/home/log/20170117/subsys::job1/20170117_202014082/subsys-job1.log
+- test/home/log/20170117/subsys::job1/20170117_202334852/subsys-job1.log
+- test/home/log/20170117/subsys::job1/20170117_202405056/subsys-job1.log
+- test/home/log/20170117/subsys::migrate/20170117_202548409/subsys-migrate.log
+- test/home/log/20170117/subsys::migrate/20170117_202636115/subsys-migrate.log
+- test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job1.log
+- test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job2.log
+- test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job3.log
+- test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job4.log
+- test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job1.log
+- test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job2.log
+- test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job3.log
+- test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job4.log
+- test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job1.log
+- test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job2.log
+- test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job3.log
+- test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job4.log
 - test/home/revert.sh
 - test/home/subsys/d.ct
 - test/home/subsys/insert.sql.job
@@ -257,6 +275,7 @@ files:
 - test/home/subsys/variable.yml
 - test/test_c_streaming_load.rb
 - test/test_filesystem.rb
+- test/test_logfilepath.rb
 - test/test_parameters.rb
 - test/test_s3datasource.rb
 - test/test_variables.rb