bricolage 5.23.3 → 5.24.0

data/lib/bricolage/jobfile.rb

@@ -90,7 +90,8 @@ module Bricolage
     attr_reader :path
 
     def job_id
-      @path.basename('.job').to_s
+      base = @path.basename('.job').to_s
+      File.basename(base, '.*')
     end
 
     def subsystem

data/lib/bricolage/jobnet.rb

@@ -5,27 +5,45 @@ module Bricolage
 
   # Represents "first" jobnet given by command line (e.g. bricolage-jobnet some.jobnet)
   class RootJobNet
+    def RootJobNet.load_auto(ctx, path)
+      if path.extname == '.job'
+        load_single_job(ctx, path)
+      else
+        load(ctx, path)
+      end
+    end
+
     def RootJobNet.load(ctx, path)
-      root = new(JobNet::FileLoader.new(ctx), JobNet.load(path))
+      root = new(JobNet::FileLoader.new(ctx), JobNet.load(path), path)
       root.load_recursive
       root.fix
       root
     end
 
     def RootJobNet.load_single_job(ctx, path)
-      root = new(JobNet::FileLoader.new(ctx), JobNet.load_single_job(path))
+      root = new(JobNet::FileLoader.new(ctx), JobNet.load_single_job(path), path)
       root.load_recursive
       root.fix
       root
     end
 
-    def initialize(jobnet_loader, start_jobnet)
+    def initialize(jobnet_loader, start_jobnet, path = nil)
       @jobnet_loader = jobnet_loader
       @start_jobnet = start_jobnet
+      @path = path
       @jobnets = {start_jobnet.ref => start_jobnet}
       @graph = nil
     end
 
+    def id
+      return nil unless @path
+      @id ||= begin
+        subsys = @path.dirname.basename
+        base = @path.basename.to_s.sub(/\..*\z/, '')
+        "#{subsys}/#{base}"
+      end
+    end
+
     attr_reader :start_jobnet
 
     def each_jobnet(&block)

data/lib/bricolage/jobnetrunner.rb

@@ -8,6 +8,7 @@ require 'bricolage/datasource'
 require 'bricolage/variables'
 require 'bricolage/eventhandlers'
 require 'bricolage/logfilepath'
+require 'bricolage/loglocatorbuilder'
 require 'bricolage/logger'
 require 'bricolage/exception'
 require 'bricolage/version'
@@ -26,8 +27,6 @@ module Bricolage
       @hooks = ::Bricolage
       @jobnet_id = nil
       @jobnet_start_time = Time.now
-      @job_start_time = nil
-      @log_path = nil
     end
 
     EXIT_SUCCESS = JobResult::EXIT_SUCCESS
@@ -37,24 +36,31 @@ module Bricolage
     def main
       opts = Options.new(self)
       @hooks.run_before_option_parsing_hooks(opts)
-      opts.parse ARGV
+      opts.parse!(ARGV)
+
       @ctx = Context.for_application(job_path: opts.jobnet_file, environment: opts.environment, global_variables: opts.global_variables)
-      @jobnet_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
-      @log_path = opts.log_path
-      jobnet =
-        if opts.jobnet_file.extname == '.job'
-          RootJobNet.load_single_job(@ctx, opts.jobnet_file)
-        else
-          RootJobNet.load(@ctx, opts.jobnet_file)
+      opts.merge_saved_options(@ctx.load_system_options)
+
+      jobnet = RootJobNet.load_auto(@ctx, opts.jobnet_file)
+      @jobnet_id = jobnet.id
+
+      if opts.dump_options?
+        puts "jobnet-id=#{@jobnet_id}"
+        puts "jobnet-file=#{opts.jobnet_file}"
+        opts.option_pairs.each do |key, value|
+          puts "#{key}=#{value.inspect}"
         end
+        exit EXIT_SUCCESS
+      end
+
       queue = get_queue(opts)
       if queue.locked?
         raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
       end
       unless queue.queued?
         enqueue_jobs jobnet, queue
-        logger.info "jobs are queued."
       end
+
       if opts.list_jobs?
         list_jobs queue
         exit EXIT_SUCCESS
@@ -64,6 +70,8 @@ module Bricolage
         puts "OK"
         exit EXIT_SUCCESS
       end
+
+      @log_locator_builder = LogLocatorBuilder.for_options(@ctx, opts.log_path_format, opts.log_s3_ds, opts.log_s3_key_format)
       run_queue queue
       exit EXIT_SUCCESS
     rescue OptionError => ex
@@ -144,11 +152,11 @@ module Bricolage
 
     def execute_job(ref, queue)
       logger.debug "job #{ref}"
-      @job_start_time = Time.now
+      job_start_time = Time.now
       job = Job.load_ref(ref, @ctx)
       job.compile
       @hooks.run_before_job_hooks(BeforeJobEvent.new(ref))
-      result = job.execute_in_process(instanciate_log_path(ref))
+      result = job.execute_in_process(log_locator: make_log_locator(ref, job_start_time))
       @hooks.run_after_job_hooks(AfterJobEvent.new(result))
       result
     rescue Exception => ex
@@ -157,12 +165,11 @@ module Bricolage
       JobResult.error(ex)
     end
 
-    def instanciate_log_path(ref)
-      return nil unless @log_path
-      @log_path.format(
+    def make_log_locator(ref, job_start_time)
+      @log_locator_builder.build(
         job_ref: ref,
         jobnet_id: @jobnet_id,
-        job_start_time: @job_start_time,
+        job_start_time: job_start_time,
         jobnet_start_time: @jobnet_start_time
       )
     end
@@ -189,51 +196,40 @@ module Bricolage
       File.basename($PROGRAM_NAME, '.*')
     end
 
-    class Options
+    class Options < CommonApplicationOptions
       def initialize(app)
         @app = app
         @environment = nil
+        @global_variables = Variables.new
         @jobnet_files = nil
-        @log_path = LogFilePath.default
-        @local_state_dir = Pathname('/tmp/bricolage')
-        if path = ENV['BRICOLAGE_QUEUE_PATH']
-          @enable_queue = true
-          @queue_path = Pathname(path)
-        elsif ENV['BRICOLAGE_ENABLE_QUEUE']
-          @enable_queue = true
-          @queue_path = nil
-        else
-          @enable_queue = false
-          @queue_path = nil
-        end
+
+        @dump_options = false
         @check_only = false
         @list_jobs = false
-        @global_variables = Variables.new
-        @parser = OptionParser.new
-        define_options @parser
-      end
 
-      attr_reader :environment
-      attr_reader :jobnet_file
-      attr_reader :log_path
-
-      attr_reader :local_state_dir
-
-      def enable_queue?
-        @enable_queue
+        init_options
       end
 
-      attr_reader :queue_path
-
-      def check_only?
-        @check_only
+      def opts_default
+        super.merge({
+          'local-state-dir' => OptionValue.new('default value', nil),
+          'enable-queue' => OptionValue.new('default value', false),
+          'queue-path' => OptionValue.new('default value', '/tmp/bricolage')
+        })
       end
+      private :opts_default
 
-      def list_jobs?
-        @list_jobs
+      def opts_env
+        env = super
+        if ENV['BRICOLAGE_ENABLE_QUEUE']
+          env['enable-queue'] = OptionValue.new('env BRICOLAGE_ENABLE_QUEUE', true)
+        end
+        if path = ENV['BRICOLAGE_QUEUE_PATH']
+          env['queue-path'] = OptionValue.new('env BRICOLAGE_QUEUE_PATH', path)
+        end
+        env
       end
-
-      attr_reader :global_variables
+      private :opts_env
 
       def help
         @parser.help
@@ -248,20 +244,17 @@ Options:
         parser.on('-e', '--environment=NAME', "Sets execution environment. [default: #{Context::DEFAULT_ENV}]") {|env|
           @environment = env
         }
-        parser.on('-L', '--log-dir=PATH', 'Log file prefix.') {|path|
-          @log_path = LogFilePath.new("#{path}/%{std}.log")
-        }
-        parser.on('--log-path=PATH', 'Log file path template.') {|path|
-          @log_path = LogFilePath.new(path)
-        }
+
+        define_common_options
+
         parser.on('--local-state-dir=PATH', 'Stores local state in this path.') {|path|
-          @local_state_dir = Pathname(path)
+          @opts_cmdline['local-state-dir'] = OptionValue.new('--local-state-dir option', path)
         }
         parser.on('-Q', '--enable-queue', 'Enables job queue.') {
-          @enable_queue = true
+          @opts_cmdline['enable-queue'] = OptionValue.new('--enable-queue option', true)
         }
         parser.on('--queue-path=PATH', 'Enables job queue with this path.') {|path|
-          @queue_path = Pathname(path)
+          @opts_cmdline['queue-path'] = OptionValue.new('--queue-path option', path)
         }
         parser.on('-c', '--check-only', 'Checks job parameters and quit without executing.') {
           @check_only = true
@@ -273,6 +266,9 @@ Options:
           name, value = name_value.split('=', 2)
           @global_variables[name] = value
         }
+        parser.on('--dump-options', 'Shows option parsing result and quit.') {
+          @dump_options = true
+        }
         parser.on('--help', 'Shows this message and quit.') {
           @app.puts parser.help
           @app.exit 0
@@ -287,14 +283,58 @@ Options:
         @parser.on(*args, &block)
       end
 
-      def parse(argv)
-        @parser.parse! argv
+      def parse!(argv)
+        @parser.parse!(argv)
         raise OptionError, "missing jobnet file" if argv.empty?
         raise OptionError, "too many jobnet file" if argv.size > 1
-        @jobnet_file = argv.map {|path| Pathname(path) }.first
+        @jobnet_file = Pathname(argv.first)
+        build_common_options!
       rescue OptionParser::ParseError => ex
         raise OptionError, ex.message
       end
+
+      attr_reader :environment
+      attr_reader :jobnet_file
+
+      attr_reader :global_variables
+
+      def dump_options?
+        @dump_options
+      end
+
+      def check_only?
+        @check_only
+      end
+
+      def list_jobs?
+        @list_jobs
+      end
+
+      def local_state_dir
+        @local_state_dir ||= begin
+          opt = @opts['local-state-dir']
+          Pathname(opt.value)
+        end
+      end
+
+      def enable_queue?
+        opt = @opts['enable-queue']
+        opt.value
+      end
+
+      def queue_path
+        if opt = @opts['queue-path']
+          Pathname(opt.value)
+        else
+          nil
+        end
+      end
+
+      def option_pairs
+        common_options.merge({
+          'environment' => OptionValue.new(nil, @environment)
+        })
+      end
     end
   end
 

data/lib/bricolage/logfilepath.rb

@@ -14,14 +14,14 @@ module Bricolage
       @template = template
     end
 
-    Params = Struct.new(:job_ref, :jobnet_id, :job_start_time, :jobnet_start_time)
-
     def format(job_ref:, jobnet_id:, job_start_time:, jobnet_start_time:)
       return nil unless @template
       params = Params.new(job_ref, jobnet_id, job_start_time, jobnet_start_time)
       fill_template(@template, params)
     end
 
+    Params = Struct.new(:job_ref, :jobnet_id, :job_start_time, :jobnet_start_time)
+
     def fill_template(template, params)
       template.gsub(/%\{\w+\}/) {|var|
         case var

data/lib/bricolage/loglocator.rb

@@ -0,0 +1,71 @@
+module Bricolage
+  class LogLocator
+    def LogLocator.empty
+      new(nil, nil)
+    end
+
+    def initialize(path, s3_writer)
+      @path = path
+      @s3_writer = s3_writer
+    end
+
+    attr_reader :path
+
+    def s3_url
+      return nil unless @s3_writer
+      @s3_writer.url
+    end
+
+    def redirect_stdouts
+      return yield unless @path
+      FileUtils.mkdir_p File.dirname(@path)
+      @original_stdout = $stdout.dup
+      @original_stderr = $stderr.dup
+      begin
+        # Use 'w+' to make readable for retrieve_last_match_from_stderr
+        File.open(@path, 'w+') {|f|
+          f.sync = true
+          $stdout.reopen f
+          $stderr.reopen f
+        }
+        return yield
+      ensure
+        $stdout.reopen @original_stdout; @original_stdout.close
+        $stderr.reopen @original_stderr; @original_stderr.close
+        upload
+      end
+    end
+
+    # CLUDGE: FIXME: We redirect stderr to the file, we can find error messages from there.
+    # Using a temporary file or Ruby SQL driver is **MUCH** better.
+    def self.slice_last_stderr(re, nth = 0)
+      return unless $stderr.stat.file?
+      $stderr.flush
+      f = $stderr.dup
+      matched = nil
+      begin
+        f.seek(0)
+        f.each do |line|
+          m = line.slice(re, nth)
+          matched = m if m
+        end
+      ensure
+        f.close
+      end
+      matched = matched.to_s.strip
+      matched.empty? ? nil : matched
+    end
+
+    def upload
+      return unless @path
+      return unless @s3_writer
+      # FIXME: Shows HTTP URL?
+      puts "bricolage: S3 log: #{s3_url}"
+      begin
+        @s3_writer.upload(path)
+      rescue => ex
+        puts "warning: S3 upload failed: #{s3_url}"
+      end
+    end
+  end
+end

data/lib/bricolage/loglocatorbuilder.rb

@@ -0,0 +1,29 @@
+require 'bricolage/loglocator'
+require 'bricolage/s3writer'
+
+module Bricolage
+  class LogLocatorBuilder
+    def LogLocatorBuilder.for_options(ctx, log_path_format, s3_ds, s3_key_format)
+      ds = s3_ds ? ctx.get_data_source('s3', s3_ds) : nil
+      new(log_path_format, ds, s3_key_format)
+    end
+
+    def initialize(log_path_format, s3_ds, s3_key_format)
+      @log_path_format = log_path_format
+      @s3_ds = s3_ds
+      @s3_key_format = s3_key_format
+    end
+
+    def build(**params)
+      path = @log_path_format ? @log_path_format.format(**params) : nil
+      s3_writer =
+        if @s3_ds and @s3_key_format
+          key = @s3_key_format.format(**params)
+          S3Writer.new(@s3_ds, key)
+        else
+          nil
+        end
+      LogLocator.new(path, s3_writer)
+    end
+  end
+end

data/lib/bricolage/psqldatasource.rb

@@ -71,7 +71,7 @@ module Bricolage
             '--no-password',
             *options,
             env: get_psql_env
-        msg = retrieve_last_match_from_stderr(/^psql:.*?:\d+: ERROR: (.*)/, 1) unless st.success?
+        msg = LogLocator.slice_last_stderr(/^psql:.*?:\d+: ERROR: (.*)/, 1) unless st.success?
         JobResult.for_process_status(st, msg)
       }
     end