bricolage 5.23.3 → 5.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23ebf2fa74d99d152e388d57872332cd013fcd73
-  data.tar.gz: 73a60961068be073e8833cd1d73769982e25b486
+  metadata.gz: f06abf75b09a7c9188d54ca938ccca52e8f122b5
+  data.tar.gz: 59c808715776490dc3374ef141528c3dc1d61960
 SHA512:
-  metadata.gz: e7f1556d282e2ad26385bdb42c4133280208af22ab5098702384dda235a423ed15569dab1d596185654647e9e48db003191883229a092a1fd9c2f4bfc873631a
-  data.tar.gz: 36b056529ae9ea99fd4318d30ef159a8b4dfd2b2936fb62e0987e81f620b9be5665a79c114f963a9fffb3bb460df894d52946ed4122e13cc13a4024434a992f5
+  metadata.gz: ee5d6895a71111e135885180841cc2c5518cf7fc822bdad0ea576c84a5f77fccade1a3f0f8d68ab3240c3e9cc4d21e01798c685f73e3e8fc114ed6997daf69e2
+  data.tar.gz: df04ca26661704bc676ee26afcbe1d9bc2940bfbd97bd8c70f42725ee4ccf5842046903620f1e87471ffd97bdbeec9059a8b61c42de0dda37142df9692d067cb

data/lib/bricolage/application.rb

@@ -8,6 +8,7 @@ require 'bricolage/datasource'
 require 'bricolage/eventhandlers'
 require 'bricolage/postgresconnection'
 require 'bricolage/logfilepath'
+require 'bricolage/loglocatorbuilder'
 require 'bricolage/logger'
 require 'bricolage/exception'
 require 'bricolage/version'
@@ -36,15 +37,26 @@ module Bricolage
     def main
       opts = GlobalOptions.new(self)
       @hooks.run_before_option_parsing_hooks(opts)
-      opts.parse ARGV
+      opts.parse!(ARGV)
+
       @ctx = Context.for_application(opts.home, opts.job_file, environment: opts.environment, global_variables: opts.global_variables)
+      opts.merge_saved_options(@ctx.load_system_options)
+
+      if opts.dump_options?
+        opts.option_pairs.each do |name, value|
+          puts "#{name}=#{value.inspect}"
+        end
+        exit 0
+      end
       if opts.list_global_variables?
         list_variables @ctx.global_variables.resolve
         exit 0
       end
+
       job = load_job(@ctx, opts)
-      process_job_options job, opts
+      process_job_options(job, opts)
       job.compile
+
       if opts.list_declarations?
         list_declarations job.declarations
         exit 0
@@ -62,11 +74,11 @@ module Bricolage
         exit 0
       end
 
+      @log_locator_builder = LogLocatorBuilder.for_options(@ctx, opts.log_path_format, opts.log_s3_ds, opts.log_s3_key_format)
+
       @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(job.id, [job]))
       @hooks.run_before_job_hooks(BeforeJobEvent.new(job))
-      result = redirect_log_to_file(opts.log_path, job) {
-        job.execute
-      }
+      result = job.execute(log_locator: build_log_locator(job))
       @hooks.run_after_job_hooks(AfterJobEvent.new(result))
       @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(result.success?, [job]))
       exit result.status
@@ -78,28 +90,13 @@ module Bricolage
       error_exit ex.message
     end
 
-    def redirect_log_to_file(log_path, job)
-      return yield unless log_path
-
-      path = log_path.format(
+    def build_log_locator(job)
+      @log_locator_builder.build(
         job_ref: JobNet::JobRef.new(job.subsystem, job.id, '-'),
         jobnet_id: "#{job.subsystem}/#{job.id}",
         job_start_time: @start_time,
         jobnet_start_time: @start_time
       )
-      FileUtils.mkdir_p File.dirname(path)
-      stdout_save = $stdout.dup
-      stderr_save = $stderr.dup
-      begin
-        File.open(path, 'w+') {|f|
-          $stdout.reopen f
-          $stderr.reopen f
-        }
-        return yield
-      ensure
-        $stdout.reopen stdout_save; stdout_save.close
-        $stderr.reopen stderr_save; stderr_save.close
-      end
     end
 
     def load_job(ctx, opts)
@@ -175,7 +172,124 @@ module Bricolage
     end
   end
 
-  class GlobalOptions
+  class CommonApplicationOptions
+    OptionValue = Struct.new(:location, :value)
+    class OptionValue
+      def inspect
+        if location
+          "#{value.inspect} (#{location})"
+        else
+          value.inspect
+        end
+      end
+    end
+
+    def init_options
+      @opts = nil   # valid only after #parse!
+      @opts_default = opts_default()
+      @opts_saved = {}
+      @opts_env = opts_env()
+      @opts_cmdline = {}
+      @parser = OptionParser.new
+      define_options @parser
+    end
+    private :init_options
+
+    attr_reader :parser
+
+    def opts_default
+      {
+        'log-path' => OptionValue.new('default value', nil),
+        'log-dir' => OptionValue.new('default value', nil),
+        's3-log' => OptionValue.new('default value', nil)
+      }
+    end
+    private :opts_default
+
+    def opts_env
+      env = {}
+      if path = ENV['BRICOLAGE_LOG_PATH']
+        env['log-path'] = OptionValue.new('env BRICOLAGE_LOG_PATH', path)
+      end
+      if path = ENV['BRICOLAGE_LOG_DIR']
+        env['log-dir'] = OptionValue.new('env BRICOLAGE_LOG_DIR', path)
+      end
+      env
+    end
+    private :opts_env
+
+    # abstract :define_options
+
+    def define_common_options
+      @parser.on('-L', '--log-dir=PATH', 'Log file prefix.') {|path|
+        @opts_cmdline['log-dir'] = OptionValue.new('--log-dir option', path)
+      }
+      @parser.on('--log-path=PATH', 'Log file path template.') {|path|
+        @opts_cmdline['log-path'] = OptionValue.new('--log-path option', path)
+      }
+      @parser.on('--s3-log=DS_KEY', 'S3 log file. (format: "DS:KEY")') {|spec|
+        @opts_cmdline['s3-log'] = OptionValue.new('--s3-log option', spec)
+      }
+    end
+    private :define_common_options
+
+    def merge_saved_options(vars)
+      saved = {}
+      @opts_default.keys.each do |key|
+        if val = vars.get_force(key)
+          saved[key] = OptionValue.new("bricolage.yml:#{key}", val)
+        end
+      end
+      @opts_saved = saved
+      build_common_options!
+    end
+
+    def build_common_options!
+      @opts = [@opts_default, @opts_saved, @opts_env, @opts_cmdline].inject({}) {|h, opts| h.update(opts); h }
+    end
+    private :build_common_options!
+
+    #
+    # Accessors
+    #
+
+    def common_options
+      @opts
+    end
+
+    def log_path_format
+      if opt = @opts['log-dir']
+        LogFilePath.new("#{opt.value}/%{std}.log")
+      elsif opt = @opts['log-path']
+        LogFilePath.new(opt.value)
+      else
+        nil
+      end
+    end
+
+    def log_s3_ds
+      s3_log_spec.first
+    end
+
+    def log_s3_key_format
+      s3_log_spec.last
+    end
+
+    def s3_log_spec
+      @s3_log_spec ||=
+        if opt = @opts['s3-log']
+          spec = opt.value
+          ds, k = spec.split(':', 2)
+          k = k.to_s.strip
+          key = k.empty? ? nil : k
+          [ds, LogFilePath.new(key || '%{std}.log')]
+        else
+          [nil, nil]
+        end
+    end
+  end
+
+  class GlobalOptions < CommonApplicationOptions
     def initialize(app)
       @app = app
       @job_file = nil
@@ -184,16 +298,13 @@ module Bricolage
       @global_variables = Variables.new
       @dry_run = false
       @explain = false
-      @log_path = LogFilePath.default
       @list_global_variables = false
       @list_variables = false
       @list_declarations = false
-      @parser = OptionParser.new
-      define_options @parser
+      @dump_options = false
+      init_options
     end
 
-    attr_reader :parser
-
     def help
       @parser.help
     end
@@ -220,12 +331,9 @@ Global Options:
       parser.on('-E', '--explain', 'Applies EXPLAIN to the SQL.') {
         @explain = true
       }
-      parser.on('-L', '--log-dir=PATH', 'Log file prefix.') {|path|
-        @log_path = LogFilePath.new("#{path}/%{std}.log")
-      }
-      parser.on('--log-path=PATH', 'Log file path template.') {|path|
-        @log_path = LogFilePath.new(path)
-      }
+
+      define_common_options
+
       parser.on('--list-job-class', 'Lists job class name and (internal) class path.') {
         JobClass.list.each do |name|
           puts name
@@ -248,6 +356,9 @@ Global Options:
         name, value = name_value.split('=', 2)
         @global_variables[name] = value
       }
+      parser.on('--dump-options', 'Shows option parsing result and quit.') {
+        @dump_options = true
+      }
       parser.on('--help', 'Shows this message and quit.') {
         puts parser.help
         exit 0
@@ -262,19 +373,18 @@ Global Options:
       @parser.on(*args, &block)
     end
 
-    def parse(argv)
+    def parse!(argv)
       @parser.order! argv
       @rest_args = argv.dup
+      build_common_options!
     rescue OptionParser::ParseError => ex
       raise OptionError, ex.message
     end
 
     attr_reader :environment
     attr_reader :home
-    attr_reader :global_variables
 
     attr_reader :job_file
-    attr_reader :log_path
 
     def file_mode?
       !!@job_file
@@ -288,6 +398,12 @@ Global Options:
       @explain
     end
 
+    def dump_options?
+      @dump_options
+    end
+
+    attr_reader :global_variables
+
     def list_global_variables?
       @list_global_variables
     end
@@ -299,6 +415,13 @@ Global Options:
     def list_declarations?
       @list_declarations
     end
+
+    def option_pairs
+      common_options.merge({
+        'environment' => OptionValue.new(nil, @environment),
+        'home' => OptionValue.new(nil, @home)
+      })
+    end
   end
 
 end

data/lib/bricolage/commandutils.rb

@@ -27,26 +27,6 @@ module Bricolage
     def new_tmpfile_path(tmpdir = Dir.tmpdir)
       "#{tmpdir}/#{Time.now.to_i}_#{$$}_#{'%x' % Thread.current.object_id}_#{rand(2**16)}"
     end
-
-    # CLUDGE: FIXME: bricolage-jobnet command writes stderr to the file, we can find error messages from there.
-    # Using a temporary file or Ruby SQL driver is **MUCH** better.
-    def retrieve_last_match_from_stderr(re, nth = 0)
-      return unless $stderr.stat.file?
-      $stderr.flush
-      f = $stderr.dup
-      matched = nil
-      begin
-        f.seek(0)
-        f.each do |line|
-          m = line.slice(re, nth)
-          matched = m if m
-        end
-      ensure
-        f.close
-      end
-      matched = matched.to_s.strip
-      matched.empty? ? nil : matched
-    end
   end
 
 end

data/lib/bricolage/context.rb

@@ -84,6 +84,16 @@ module Bricolage
       :parameter_file,
       :parameter_file_loader
 
+    #
+    # System Parameters
+    #
+
+    SYSTEM_OPTION_FILE = 'bricolage.yml'
+
+    def load_system_options
+      load_variables_for_all_scopes(SYSTEM_OPTION_FILE)
+    end
+
     #
     # Variables
     #
@@ -103,15 +113,20 @@ module Bricolage
       }
     end
 
+    GLOBAL_VARIABLE_FILE = 'variable.yml'
+
     def load_global_variables
-      subsys_path = scoped? ? [@filesystem.relative(GLOBAL_VARIABLE_FILE)] : []
-      vars_list = (config_pathes(GLOBAL_VARIABLE_FILE) + subsys_path).map {|path|
+      load_variables_for_all_scopes(GLOBAL_VARIABLE_FILE)
+    end
+
+    def load_variables_for_all_scopes(basename)
+      subsys_path = scoped? ? [@filesystem.relative(basename)] : []
+      vars_list = (config_pathes(basename) + subsys_path).map {|path|
         path.exist? ? load_variables(path) : nil
       }
       Variables.union(*vars_list.compact)
     end
-
-    GLOBAL_VARIABLE_FILE = 'variable.yml'
+    private :load_variables_for_all_scopes
 
     def load_variables(path)
       Variables.define {|vars|

data/lib/bricolage/job.rb

@@ -55,7 +55,8 @@ module Bricolage
     end
 
     def init_global_variables
-      # Context#global_variables loads file on each call, fix global variables here.
+      # Context#global_variables loads file on each call,
+      # updating @global_variables is multi-thread safe.
       @global_variables = @context.global_variables
       @global_variables['bricolage_cwd'] = Dir.pwd
       @global_variables['bricolage_job_dir'] = @context.job_dir.to_s
@@ -135,7 +136,26 @@ module Bricolage
       @script.run_explain
     end
 
-    def execute
+    def execute(log_locator: LogLocator.empty)
+      log_locator.redirect_stdouts {
+        do_execute
+      }
+    end
+
+    def execute_in_process(log_locator:)
+      # ??? FIXME: status_path should be independent from log_path.
+      # Also, status_path should be defined regardless of log_path.
+      status_path = log_locator.path ? "#{log_locator.path}.status" : nil
+      isolate_process(status_path) {
+        log_locator.redirect_stdouts {
+          do_execute
+        }
+      }
+    end
+
+    private
+
+    def do_execute
       ENV['BRICOLAGE_PID'] = Process.pid.to_s
       logger = @context.logger
       logger.info "#{@context.environment} environment"
@@ -154,60 +174,40 @@ module Bricolage
       return JobResult.error(ex)
     end
 
-    def execute_in_process(log_path)
-      isolate(log_path) {
-        execute
-      }
-    end
-
-    private
-
-    def isolate(log_path)
+    def isolate_process(status_path)
       cpid = Process.fork {
         Process.setproctitle "bricolage [#{@id}]"
-        redirect_stdouts_to log_path if log_path
         result = yield
-        save_result result, log_path
+        save_result result, status_path
         exit result.status
       }
       _, st = Process.waitpid2(cpid)
-      restore_result(st, log_path)
-    end
-
-    def redirect_stdouts_to(path)
-      FileUtils.mkdir_p File.dirname(path)
-      # make readable for retrieve_last_match_from_stderr
-      File.open(path, 'w+') {|f|
-        $stdout.reopen f
-        $stderr.reopen f
-      }
+      restore_result(st, status_path)
     end
 
-    def save_result(result, log_path)
+    def save_result(result, status_path)
       return if result.success?
-      return unless log_path
+      return unless status_path
       begin
-        File.open(error_log_path(log_path), 'w') {|f|
+        File.open(status_path, 'w') {|f|
           f.puts result.message
         }
       rescue
       end
     end
 
-    def restore_result(st, log_path)
-      JobResult.for_process_status(st, restore_message(log_path))
-    end
-
-    def restore_message(log_path)
-      return nil unless log_path
-      msg = read_if_exist(error_log_path(log_path))
-      msg ? msg.strip : nil
-    ensure
-      FileUtils.rm_f error_log_path(log_path) if log_path
+    def restore_result(st, status_path)
+      JobResult.for_process_status(st, restore_message(status_path))
     end
 
-    def error_log_path(log_path)
-      "#{log_path}.error"
+    def restore_message(status_path)
+      return nil unless status_path
+      begin
+        msg = read_if_exist(status_path)
+        msg ? msg.strip : nil
+      ensure
+        FileUtils.rm_f status_path
+      end
     end
 
     def read_if_exist(path)