bricolage 5.29.0 → 6.0.0beta2

data/lib/bricolage/dao/jobexecution.rb

@@ -0,0 +1,253 @@
+module Bricolage
+  module DAO
+
+    class JobExecution
+      include SQLUtils
+
+      STATUS_WAITING = 'waiting'.freeze
+      STATUS_SUCCEEDED = 'succeeded'.freeze
+      STATUS_RUNNING = 'running'.freeze
+      STATUS_FAILED = 'failed'.freeze
+      STATUS_CANCELED = 'canceled'.freeze
+
+      Attributes = Struct.new(:job_id, :job_execution_id, :subsystem, :job_name, keyword_init: true)
+
+      def JobExecution.for_record(r)
+        Attributes.new(
+          job_id: r['job_id']&.to_i,
+          job_execution_id: r['job_execution_id']&.to_i,
+          subsystem: r['subsystem'],
+          job_name: r['job_name']
+        )
+      end
+
+      def JobExecution.for_connection(conn)
+        new(nil, connection: conn)
+      end
+
+      def initialize(datasource, connection: nil)
+        @datasource = datasource
+        @connection = connection
+      end
+
+      private def connect
+        if @connection
+          yield @connection
+        else
+          @datasource.open_shared_connection {|conn|
+            yield conn
+          }
+        end
+      end
+
+      def enqueued_jobs(jobnet_ref)
+        records = connect {|conn|
+          conn.query_rows(<<~EndSQL)
+            select
+                e.job_execution_id
+                , e.job_id
+                , j.subsystem
+                , j.job_name
+            from
+                job_executions e
+                inner join jobs j using (job_id)
+                inner join jobnets n using (jobnet_id)
+            where
+                n.subsystem = #{s jobnet_ref.subsystem}
+                and n.jobnet_name = #{s jobnet_ref.name}
+                and e.status in (#{s STATUS_WAITING}, #{s STATUS_RUNNING}, #{s STATUS_FAILED})
+            order by
+                e.execution_sequence
+            ;
+          EndSQL
+        }
+        records.map {|r| JobExecution.for_record(r) }
+      end
+
+      def enqueue_job(job, execution_sequence)
+        record = nil
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+            insert into job_executions
+                ( job_id
+                , execution_sequence
+                , status
+                , message
+                , submitted_at
+                )
+            values
+                ( #{job.id}
+                , #{execution_sequence}
+                , #{s STATUS_WAITING}
+                , ''
+                , now()
+                )
+            returning job_execution_id, job_id
+            ;
+          EndSQL
+
+          record = records.first
+          save_state_transition(conn, record['job_execution_id'], 'submitted_at')
+        }
+
+        exec = JobExecution.for_record(record)
+        exec.subsystem = job.subsystem
+        exec.job_name = job.job_name
+        exec
+      end
+
+      def cancel_jobnet(jobnet_ref, message)
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+            update job_executions
+            set
+                status = #{s STATUS_CANCELED}
+                , message = #{s message}
+                , finished_at = now()
+            where
+                job_id in (
+                    select
+                        j.job_id
+                    from
+                        jobs j inner join jobnets n using (jobnet_id)
+                    where
+                        n.subsystem = #{s jobnet_ref.subsystem}
+                        and n.jobnet_name = #{s jobnet_ref.name}
+                )
+                and status in (#{s STATUS_WAITING}, #{s STATUS_RUNNING}, #{s STATUS_FAILED})
+            returning job_execution_id
+            ;
+          EndSQL
+
+          job_execution_ids = records.map {|r| r['job_execution_id'].to_i }
+          unless job_execution_ids.empty?
+            conn.execute_update(<<~EndSQL)
+                insert into job_execution_states
+                    ( job_execution_id
+                    , job_id
+                    , created_at
+                    , status
+                    , message
+                    )
+                select
+                    job_execution_id
+                    , job_id
+                    , finished_at
+                    , status
+                    , message
+                from
+                    job_executions
+                where
+                    job_execution_id in (#{job_execution_ids.join(', ')})
+                ;
+            EndSQL
+          end
+        }
+      end
+
+      def transition_to_running(job_execution_id)
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+              update job_executions
+              set
+                  status = #{s STATUS_RUNNING}
+                  , message = ''
+                  , started_at = now()
+                  , finished_at = null
+              where
+                  job_execution_id = #{job_execution_id}
+                  and status in (#{s STATUS_WAITING}, #{s STATUS_FAILED})
+              returning job_execution_id
+              ;
+          EndSQL
+          if records.empty?
+            raise IllegalJobStateException, "Could not run already running job: job_execution_id=#{job_execution_id}"
+          end
+
+          save_state_transition(conn, job_execution_id, 'started_at')
+        }
+      end
+
+      def transition_to_succeeded(job_execution_id)
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+              update job_executions
+              set
+                  finished_at = now()
+                  , status = #{s STATUS_SUCCEEDED}
+                  , message = ''
+              where
+                  job_execution_id = #{job_execution_id}
+                  and status = #{s STATUS_RUNNING}
+              returning job_execution_id
+              ;
+          EndSQL
+          if records.empty?
+            raise IllegalJobStateException, "could not transition to succeeded state: job_execution_id=#{job_execution_id}"
+          end
+
+          save_state_transition(conn, job_execution_id, 'finished_at')
+        }
+      end
+
+      def transition_to_failed(job_execution_id, message)
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+              update job_executions
+              set
+                  finished_at = now()
+                  , status = #{s STATUS_FAILED}
+                  , message = #{s message}
+              where
+                  job_execution_id = #{job_execution_id}
+                  and status = #{s STATUS_RUNNING}
+              returning job_execution_id
+              ;
+          EndSQL
+          if records.empty?
+            raise IllegalJobStateException, "could not transition to failed state: job_execution_id=#{job_execution_id}"
+          end
+
+          save_state_transition(conn, job_execution_id, 'finished_at')
+        }
+      end
+
+      private def save_state_transition(conn, job_execution_id, time_expr)
+        conn.execute_update(<<~EndSQL)
+            insert into job_execution_states
+                ( job_execution_id
+                , job_id
+                , created_at
+                , status
+                , message
+                )
+            select
+                job_execution_id
+                , job_id
+                , #{time_expr}
+                , status
+                , message
+            from
+                job_executions
+            where
+                job_execution_id = #{job_execution_id}
+            ;
+        EndSQL
+      end
+
+      # For tests only
+      def delete_all
+        connect {|conn|
+          conn.execute_update(<<~EndSQL)
+              delete from job_execution_states;
+              delete from job_executions;
+              delete from jobs;
+              delete from jobnets;
+          EndSQL
+        }
+      end
+
+    end   # class JobExecution
+
+  end
+end

data/lib/bricolage/dao/jobnet.rb

@@ -0,0 +1,158 @@
+module Bricolage
+  module DAO
+    class JobNet
+
+      include SQLUtils
+
+      Attributes = Struct.new(:id, :subsystem, :jobnet_name, keyword_init: true)
+
+      def JobNet.for_record(r)
+        Attributes.new(
+          id: r['jobnet_id']&.to_i,
+          subsystem: r['subsystem'],
+          jobnet_name: r['jobnet_name']
+        )
+      end
+
+      def JobNet.for_records(jobnets)
+        jobnets.map {|jobnet| JobNet.for_record(jobnet) }
+      end
+
+      def initialize(datasource)
+        @datasource = datasource
+      end
+
+      private def connect(&block)
+        @datasource.open_shared_connection(&block)
+      end
+
+      def find_or_create(ref)
+        connect {|conn|
+          jobnet = find(conn, ref)
+          if jobnet
+            return jobnet
+          else
+            begin
+              return create(conn, ref)
+            rescue UniqueViolationException
+              jobnet = find(conn, ref) or raise "[BUG] Could not create jobnet record: #{ref}"
+              return jobnet
+            end
+          end
+        }
+      end
+
+      private def create(conn, ref)
+        records = conn.execute_update(<<~SQL)
+          insert into jobnets
+              ( "subsystem"
+              , jobnet_name
+              )
+          values
+              ( #{s ref.subsystem}
+              , #{s ref.name}
+              )
+          returning jobnet_id
+          ;
+        SQL
+
+        Attributes.new(
+          id: records.first['jobnet_id']&.to_i,
+          subsystem: ref.subsystem,
+          jobnet_name: ref.name
+        )
+      end
+
+      private def find(conn, ref)
+        record = conn.query_row(<<~EndSQL)
+          select
+              jobnet_id
+              , "subsystem"
+              , jobnet_name
+          from
+              jobnets
+          where
+              "subsystem" = #{s ref.subsystem}
+              and jobnet_name = #{s ref.name}
+          ;
+        EndSQL
+
+        if record
+          JobNet.for_record(record)
+        else
+          nil
+        end
+      end
+
+      def locked?(ref)
+        value = connect {|conn|
+          conn.query_value(<<~EndSQL)
+            select
+                count(*)
+            from
+                jobnets
+            where
+                "subsystem" = #{s ref.subsystem}
+                and jobnet_name = #{s ref.name}
+                and executor_id is not null
+            ;
+          EndSQL
+        }
+
+        value.to_i > 0
+      end
+
+      def lock(jobnet_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobnets
+            set
+                executor_id = #{s executor_id}
+            where
+                jobnet_id = #{jobnet_id}
+                and executor_id is null
+            returning jobnet_id
+            ;
+          EndSQL
+        }
+        if records.empty?
+          raise DoubleLockError, "Could not lock jobnet: jobnet_id=#{jobnet_id}"
+        end
+      end
+
+      # Unlock jobnet lock.
+      # Returns true if unlocked successfully, otherwise false.
+      # FIXME: raise exception?
+      def unlock(jobnet_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobnets
+            set
+                executor_id = null
+            where
+                jobnet_id = #{jobnet_id}
+                and executor_id = #{s executor_id}
+            returning jobnet_id
+            ;
+          EndSQL
+        }
+
+        not records.empty?
+      end
+
+      def clear_lock(jobnet_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobnets
+            set
+                executor_id = null
+            where
+                jobnet_id = #{jobnet_id}
+            ;
+          EndSQL
+        }
+      end
+
+    end
+  end
+end

data/lib/bricolage/datasource.rb

@@ -137,7 +137,7 @@ module Bricolage
 
     attr_reader :name
     attr_reader :context
-    attr_reader :logger
+    attr_accessor :logger
 
     def open
       yield nil

data/lib/bricolage/exception.rb

@@ -24,12 +24,23 @@ module Bricolage
   # Various SQL exception, except connection problem.
   class SQLException < JobFailureByException; end
 
+  # SQL unique constraint violation
+  class UniqueViolationException < SQLException; end
+
   # Database connection problems (not established, closed unexpectedly, invalid state)
   class ConnectionError < JobFailureByException; end
 
   # Aquiring lock takes too long (e.g. VACUUM lock)
   class LockTimeout < JobFailure; end
 
+  # The executing jobnet or job is already locked.
+  # You should wait to unlock by another job execution or force to unlock manually.
+  class DoubleLockError < JobFailure; end
+
+  # Unexpected job state transition tried.
+  # This error must be fixed by a operator.
+  class IllegalJobStateException < JobFailure; end
+
   # S3 related exceptions
   class S3Exception < JobFailureByException; end
 

data/lib/bricolage/job.rb

@@ -6,10 +6,11 @@ require 'bricolage/variables'
 require 'bricolage/configloader'
 require 'bricolage/loglocator'
 require 'bricolage/exception'
+require 'tmpdir'
 require 'fileutils'
 
 module Bricolage
-  
+
   class Job
     # For JobNetRunner
     def Job.load_ref(ref, jobnet_context)
@@ -41,6 +42,7 @@ module Bricolage
       @job_class = job_class
       @context = context
       @global_variables = nil
+      @option_variables = @context.option_variables
       @param_decls = @job_class.get_parameters
       @param_vals = nil      # Parameters::IntermediateValues by *.job
       @param_vals_opt = nil  # Parameters::IntermediateValues by options
@@ -87,6 +89,7 @@ module Bricolage
 
       job_file_rest_vars = @param_vals ? @param_vals.variables : Variables.new
       job_v_opt_vars = @param_vals_opt ? @param_vals_opt.variables : Variables.new
+      cmd_v_opt_vars = @option_variables ? @option_variables : Variables.new
 
       # We use different variable set for paramter expansion and
       # SQL variable expansion.  Parameter expansion uses global
@@ -94,6 +97,7 @@ module Bricolage
       base_vars = Variables.union(
         #          ^ Low precedence
         @global_variables,
+        cmd_v_opt_vars,
         job_v_opt_vars
         #          v High precedence
       )
@@ -103,11 +107,12 @@ module Bricolage
       # Then, expand SQL variables and check with declarations.
       vars = Variables.union(
         #          ^ Low precedence
-        declarations.default_variables,
-        @global_variables,
-        @params.variables,   # Like $dest_table
-        job_file_rest_vars,
-        job_v_opt_vars
+        declarations.default_variables, # default value written in *.sql
+        @global_variables,   # from yaml file
+        @params.variables,   # Like $dest_table in job file
+        job_file_rest_vars,  # custom variable at header of job file
+        cmd_v_opt_vars,      # -v option for bricolage/bricolage-jobnet command
+        job_v_opt_vars       # -v option for bricolage command using jobclass
         #          v High precedence
       )
       @variables = vars.resolve
@@ -147,9 +152,7 @@ module Bricolage
     end
 
     def execute_in_process(log_locator:)
-      # ??? FIXME: status_path should be independent from log_path.
-      # Also, status_path should be defined regardless of log_path.
-      status_path = log_locator.path ? "#{log_locator.path}.status" : nil
+      status_path = "#{Dir.tmpdir}/bricolage.#{$$}.status.#{"%010x" % rand(1000000000000)}"
       isolate_process(status_path) {
         log_locator.redirect_stdouts {
           do_execute