bricolage 5.30.0 → 6.0.0beta1

data/lib/bricolage/dao/jobnet.rb

@@ -0,0 +1,158 @@
+module Bricolage
+  module DAO
+    class JobNet
+
+      include SQLUtils
+
+      Attributes = Struct.new(:id, :subsystem, :jobnet_name, keyword_init: true)
+
+      def JobNet.for_record(r)
+        Attributes.new(
+          id: r['jobnet_id']&.to_i,
+          subsystem: r['subsystem'],
+          jobnet_name: r['jobnet_name']
+        )
+      end
+
+      def JobNet.for_records(jobnets)
+        jobnets.map {|jobnet| JobNet.for_record(jobnet) }
+      end
+
+      def initialize(datasource)
+        @datasource = datasource
+      end
+
+      private def connect(&block)
+        @datasource.open_shared_connection(&block)
+      end
+
+      def find_or_create(ref)
+        connect {|conn|
+          jobnet = find(conn, ref)
+          if jobnet
+            return jobnet
+          else
+            begin
+              return create(conn, ref)
+            rescue UniqueViolationException
+              jobnet = find(conn, ref) or raise "[BUG] Could not create jobnet record: #{ref}"
+              return jobnet
+            end
+          end
+        }
+      end
+
+      private def create(conn, ref)
+        records = conn.execute_update(<<~SQL)
+          insert into jobnets
+              ( "subsystem"
+              , jobnet_name
+              )
+          values
+              ( #{s ref.subsystem}
+              , #{s ref.name}
+              )
+          returning jobnet_id
+          ;
+        SQL
+
+        Attributes.new(
+          id: records.first['jobnet_id']&.to_i,
+          subsystem: ref.subsystem,
+          jobnet_name: ref.name
+        )
+      end
+
+      private def find(conn, ref)
+        record = conn.query_row(<<~EndSQL)
+          select
+              jobnet_id
+              , "subsystem"
+              , jobnet_name
+          from
+              jobnets
+          where
+              "subsystem" = #{s ref.subsystem}
+              and jobnet_name = #{s ref.name}
+          ;
+        EndSQL
+
+        if record
+          JobNet.for_record(record)
+        else
+          nil
+        end
+      end
+
+      def locked?(ref)
+        value = connect {|conn|
+          conn.query_value(<<~EndSQL)
+            select
+                count(*)
+            from
+                jobnets
+            where
+                "subsystem" = #{s ref.subsystem}
+                and jobnet_name = #{s ref.name}
+                and executor_id is not null
+            ;
+          EndSQL
+        }
+
+        value.to_i > 0
+      end
+
+      def lock(jobnet_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobnets
+            set
+                executor_id = #{s executor_id}
+            where
+                jobnet_id = #{jobnet_id}
+                and executor_id is null
+            returning jobnet_id
+            ;
+          EndSQL
+        }
+        if records.empty?
+          raise DoubleLockError, "Could not lock jobnet: jobnet_id=#{jobnet_id}"
+        end
+      end
+
+      # Unlock jobnet lock.
+      # Returns true if unlocked successfully, otherwise false.
+      # FIXME: raise exception?
+      def unlock(jobnet_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobnets
+            set
+                executor_id = null
+            where
+                jobnet_id = #{jobnet_id}
+                and executor_id = #{s executor_id}
+            returning jobnet_id
+            ;
+          EndSQL
+        }
+
+        not records.empty?
+      end
+
+      def clear_lock(jobnet_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobnets
+            set
+                executor_id = null
+            where
+                jobnet_id = #{jobnet_id}
+            ;
+          EndSQL
+        }
+      end
+
+    end
+  end
+end

data/lib/bricolage/datasource.rb

@@ -137,7 +137,7 @@ module Bricolage
 
     attr_reader :name
     attr_reader :context
-    attr_reader :logger
+    attr_accessor :logger
 
     def open
       yield nil

data/lib/bricolage/exception.rb

@@ -24,12 +24,23 @@ module Bricolage
   # Various SQL exception, except connection problem.
   class SQLException < JobFailureByException; end
 
+  # SQL unique constraint violation
+  class UniqueViolationException < SQLException; end
+
   # Database connection problems (not established, closed unexpectedly, invalid state)
   class ConnectionError < JobFailureByException; end
 
   # Aquiring lock takes too long (e.g. VACUUM lock)
   class LockTimeout < JobFailure; end
 
+  # The executing jobnet or job is already locked.
+  # You should wait to unlock by another job execution or force to unlock manually.
+  class DoubleLockError < JobFailure; end
+
+  # Unexpected job state transition tried.
+  # This error must be fixed by a operator.
+  class IllegalJobStateException < JobFailure; end
+
   # S3 related exceptions
   class S3Exception < JobFailureByException; end
 

data/lib/bricolage/job.rb

@@ -9,7 +9,7 @@ require 'bricolage/exception'
 require 'fileutils'
 
 module Bricolage
-  
+
   class Job
     # For JobNetRunner
     def Job.load_ref(ref, jobnet_context)
@@ -41,6 +41,7 @@ module Bricolage
       @job_class = job_class
       @context = context
       @global_variables = nil
+      @option_variables = @context.option_variables
       @param_decls = @job_class.get_parameters
       @param_vals = nil      # Parameters::IntermediateValues by *.job
       @param_vals_opt = nil  # Parameters::IntermediateValues by options
@@ -87,6 +88,7 @@ module Bricolage
 
       job_file_rest_vars = @param_vals ? @param_vals.variables : Variables.new
       job_v_opt_vars = @param_vals_opt ? @param_vals_opt.variables : Variables.new
+      cmd_v_opt_vars = @option_variables ? @option_variables : Variables.new
 
       # We use different variable set for paramter expansion and
       # SQL variable expansion.  Parameter expansion uses global
@@ -94,6 +96,7 @@ module Bricolage
       base_vars = Variables.union(
         #          ^ Low precedence
         @global_variables,
+        cmd_v_opt_vars,
         job_v_opt_vars
         #          v High precedence
       )
@@ -103,11 +106,12 @@ module Bricolage
       # Then, expand SQL variables and check with declarations.
       vars = Variables.union(
         #          ^ Low precedence
-        declarations.default_variables,
-        @global_variables,
-        @params.variables,   # Like $dest_table
-        job_file_rest_vars,
-        job_v_opt_vars
+        declarations.default_variables, # default value written in *.sql
+        @global_variables,   # from yaml file
+        @params.variables,   # Like $dest_table in job file
+        job_file_rest_vars,  # custom variable at header of job file
+        cmd_v_opt_vars,      # -v option for bricolage/bricolage-jobnet command
+        job_v_opt_vars       # -v option for bricolage command using jobclass
         #          v High precedence
       )
       @variables = vars.resolve

data/lib/bricolage/jobnet.rb

@@ -51,6 +51,10 @@ module Bricolage
 
     attr_reader :start_jobnet
 
+    def ref
+      @start_jobnet.ref
+    end
+
     def each_jobnet(&block)
       @jobnets.each_value(&block)
     end
@@ -351,7 +355,7 @@ module Bricolage
         unless node_subsys
           raise ParameterError, "missing subsystem: #{ref}"
         end
-        ref_class.new(node_subsys, name, location)
+        ref_class.new(node_subsys.to_s, name.to_s, location)
       end
 
       def initialize(subsys, name, location)
@@ -389,7 +393,7 @@ module Bricolage
 
     class JobRef < Ref
       def JobRef.for_path(path)
-        new(path.parent.basename, JobRef.strip_exts(path), Location.dummy)
+        new(path.parent.basename.to_s, JobRef.strip_exts(path), Location.dummy)
       end
 
       def JobRef.strip_exts(path)
@@ -398,7 +402,7 @@ module Bricolage
         until (ext = basename.extname).empty?
           basename = basename.basename(ext)
         end
-        basename
+        basename.to_s
       end
 
       def net?
@@ -408,11 +412,11 @@ module Bricolage
 
     class JobNetRef < Ref
       def JobNetRef.for_path(path)
-        new(path.parent.basename, path.basename('.jobnet'), Location.dummy)
+        new(path.parent.basename.to_s, path.basename('.jobnet').to_s, Location.dummy)
       end
 
       def JobNetRef.for_job_path(path)
-        new(path.parent.basename, JobRef.strip_exts(path), Location.dummy)
+        new(path.parent.basename.to_s, JobRef.strip_exts(path).to_s, Location.dummy)
       end
 
       def initialize(subsys, name, location)
@@ -444,7 +448,6 @@ module Bricolage
         @end ||= JobRef.new(subsystem, "@#{name}@end", location)
       end
 
-
       def start
         @jobnet.start
       end

data/lib/bricolage/jobnetrunner.rb

@@ -15,6 +15,9 @@ require 'bricolage/version'
 require 'fileutils'
 require 'pathname'
 require 'optparse'
+require 'socket'
+require 'net/http'
+require 'json'
 
 module Bricolage
 
@@ -39,7 +42,7 @@ module Bricolage
       @hooks.run_before_option_parsing_hooks(opts)
       opts.parse!(ARGV)
 
-      @ctx = Context.for_application(job_path: opts.jobnet_files.first, environment: opts.environment, global_variables: opts.global_variables)
+      @ctx = Context.for_application(job_path: opts.jobnet_files.first, environment: opts.environment, option_variables: opts.option_variables)
       opts.merge_saved_options(@ctx.load_system_options)
 
       jobnet = RootJobNet.load_auto(@ctx, opts.jobnet_files)
@@ -54,16 +57,18 @@ module Bricolage
         exit EXIT_SUCCESS
       end
 
+      queue = make_queue(opts)
+      if queue.locked?(jobnet)
+        raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help(jobnet)}"
+      end
       if opts.clear_queue?
-        clear_queue(opts)
+        queue.cancel_jobnet(jobnet, 'cancelled by --clear-queue')
+        logger.info "queue is unlocked and cleared"
         exit EXIT_SUCCESS
       end
-      queue = get_queue(opts)
-      if queue.locked?
-        raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
-      end
-      unless queue.queued?
-        enqueue_jobs jobnet, queue
+      queue.restore_jobnet(jobnet)
+      if queue.empty?
+        queue.enqueue_jobnet(jobnet)
       end
 
       if opts.list_jobs?
@@ -91,18 +96,30 @@ module Bricolage
       @ctx.logger
     end
 
-    def clear_queue(opts)
-      if path = get_queue_file_path(opts)
-        FileUtils.rm_f path
+    def make_queue(opts)
+      if opts.db_name
+        logger.info "Enables DB queue: datasource=#{opts.db_name}"
+        datasource = @ctx.get_data_source('psql', opts.db_name)
+        executor_id = get_executor_id(opts.executor_type)
+        DatabaseTaskQueue.new(datasource: datasource, executor_id: executor_id, enable_lock: false)
+      elsif path = get_queue_file_path(opts)
+        logger.info "Enables file queue: #{path}"
+        FileTaskQueue.new(path: path)
+      else
+        MemoryTaskQueue.new
       end
     end
 
-    def get_queue(opts)
-      if path = get_queue_file_path(opts)
-        logger.info "queue path: #{path}"
-        FileTaskQueue.restore_if_exist(path)
+    def get_executor_id(executor_type)
+      # executor_id is 'TaskID:PID' or 'Hostname:PID'
+      if executor_type == 'ecs'
+        uri = URI.parse("#{ENV['ECS_CONTAINER_METADATA_URI']}/task")
+        response = Net::HTTP.get_response(uri)
+        task_id = JSON.parse(response.body)['TaskARN'].split('/').last
+        "#{task_id}:#{$$}"
       else
-        TaskQueue.new
+        hostname = Socket.gethostname
+        "#{hostname}:#{$$}"
       end
     end
 
@@ -124,45 +141,38 @@ module Bricolage
       path.basename.to_s
     end
 
-    def enqueue_jobs(jobnet, queue)
-      seq = 1
-      jobnet.sequential_jobs.each do |ref|
-        queue.enq JobTask.new(ref)
-        seq += 1
-      end
-      queue.save
-    end
-
     def list_jobs(queue)
-      queue.each do |task|
-        puts task.job
+      queue.each do |job|
+        puts job
       end
     end
 
     def check_jobs(queue)
-      queue.each do |task|
-        Job.load_ref(task.job, @ctx).compile
+      queue.each do |job|
+        Job.load_ref(job, @ctx).compile
       end
     end
 
     def run_queue(queue)
+      result = nil
+      job = nil
       @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@jobnet_id, queue))
-      queue.consume_each do |task|
-        result = execute_job(task.job, queue)
-        unless result.success?
-          logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
-          logger.error "[job #{task.job}] #{result.message}"
-          @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(false, queue))
-          exit result.status
-        end
+      queue.consume_each do |job|
+        result = execute_job(job, queue)
       end
-      @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(true, queue))
+      @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(result.success?, queue))
       logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
-      logger.info "status all green"
+
+      if result.success?
+        logger.info "status all green"
+      else
+        logger.error "[job #{job}] #{result.message}"
+        exit result.status
+      end
     end
 
     def execute_job(ref, queue)
-      logger.debug "job #{ref}"
+      logger.info "job #{ref}"
       job_start_time = Time.now
       job = Job.load_ref(ref, @ctx)
       job.compile
@@ -211,7 +221,7 @@ module Bricolage
       def initialize(app)
         @app = app
         @environment = nil
-        @global_variables = Variables.new
+        @option_variables = Variables.new
         @jobnet_files = nil
 
         @dump_options = false
@@ -226,7 +236,9 @@ module Bricolage
         super.merge({
           'local-state-dir' => OptionValue.new('default value', '/tmp/bricolage'),
           'enable-queue' => OptionValue.new('default value', false),
-          'queue-path' => OptionValue.new('default value', nil)
+          'queue-path' => OptionValue.new('default value', nil),
+          'db-name' => OptionValue.new('default value', nil),
+          'ecs-executor' => OptionValue.new('default value', false)
         })
       end
       private :opts_default
@@ -277,15 +289,22 @@ Options:
         parser.on('--queue-path=PATH', 'Enables job queue with this path.') {|path|
           @opts_cmdline['queue-path'] = OptionValue.new('--queue-path option', path)
         }
+        parser.on('--db-name=DB_NAME', 'Enables job queue with this database.') {|db_name|
+          @opts_cmdline['db-name'] = OptionValue.new('--db-name option', db_name)
+        }
+        parser.on('--ecs-executor', 'Set executor type as ECS ') {
+          @opts_cmdline['ecs-executor'] = OptionValue.new('--ecs-executor option', true)
+        }
+
         parser.on('-c', '--check-only', 'Checks job parameters and quit without executing.') {
           @check_only = true
         }
         parser.on('-l', '--list-jobs', 'Lists target jobs without executing.') {
           @list_jobs = true
         }
-        parser.on('-v', '--variable=NAME=VALUE', 'Defines global variable.') {|name_value|
+        parser.on('-v', '--variable=NAME=VALUE', 'Defines option variable.') {|name_value|
           name, value = name_value.split('=', 2)
-          @global_variables[name] = value
+          @option_variables[name] = value
         }
         parser.on('--dump-options', 'Shows option parsing result and quit.') {
           @dump_options = true
@@ -317,7 +336,7 @@ Options:
 
       attr_reader :jobnet_files
 
-      attr_reader :global_variables
+      attr_reader :option_variables
 
       def dump_options?
         @dump_options
@@ -352,6 +371,24 @@ Options:
         end
       end
 
+      def db_name
+        opt = @opts['db-name']
+        if opt.value
+          opt.value
+        else
+          nil
+        end
+      end
+
+      def executor_type
+        opt = @opts['ecs-executor']
+        if opt.value
+          'ecs'
+        else
+          'ec2'
+        end
+      end
+
       def clear_queue?
         @clear_queue
       end