bricolage 5.30.0 → 6.0.0beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd34b3e85926b0f85f76d05415341066025a5117c77d1b3b644ac9ce3358f0d2
-  data.tar.gz: 067eed98758a6de824a0ab9133830a637f3cc3790e26974a946d39eaa54032fc
+  metadata.gz: 111b4bb62234f5a428ccd1da5a5ee2d6d4b1a051c8fda9ddd6fac3e590356797
+  data.tar.gz: 77c3857f92a721503bd07f6e235c81f454d61e3a55e111e7a2c0ce72efa8f6ed
 SHA512:
-  metadata.gz: d4601475e0c9cbd70e2f54a68ddb06e417288fc52230b3ac553ff8a3f1ef79dafa6d83a8e543470d9c5cfd9d040a98df66217e2cd7a39aec6e1cd4b342640b0f
-  data.tar.gz: aa8d9bf658e8f36a84eacd18c3d680a18178d7612c995fbdbb8dfd695720331f953607614c1ad2d467c8a211e6fd6bf29f853257455b72700fe93172846375e3
+  metadata.gz: ca02be7fe28f3323f8dcfa1f9c88a0dd70050f7af75109ee077d0347a5f7a4420bb7f7a6b80d9ad207bd447b640089b979e319fa9ed3cf3465e11039327bc2f3
+  data.tar.gz: 7d9d77d27f31d5368a694b9514a1276532335e1f980b2dd45fa77b7a4f30cc500b4dc1fb6a99a0a2f5dd35b42c9e155201764ba3d1d9a253b52790acf1f92857

data/.gitignore

@@ -17,6 +17,4 @@ tmp
 _yardoc
 doc/
 
-/dev
-/test/home/config
-Gemfile.lock
+/Gemfile.lock

data/bricolage.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'test-unit'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'mocha'
+  s.add_development_dependency 'pry-byebug'
 end

data/jobclass/streaming_load.rb

@@ -212,7 +212,7 @@ class StreamingLoadJobClass < RubyJobClass
         end
       @logger.info "creating manifest: #{manifest_name}"
       json = make_manifest_json(objects)
-      @logger.info "manifest:\n" + json
+      @logger.debug "manifest:\n" + json
       url = @src.put_control_file(manifest_name, json, noop: @noop)
       yield url
       @src.remove_control_file(File.basename(url), noop: @noop) unless @keep_ctl
@@ -241,7 +241,7 @@ class StreamingLoadJobClass < RubyJobClass
       log_name = "load_log-#{@job_process_id}.csv"
       @logger.info "creating tmp load log: #{log_name}"
       csv = make_load_log_csv(objects)
-      @logger.info "load_log:\n" + csv
+      @logger.debug "load_log:\n" + csv
       url = @src.put_control_file(log_name, csv, noop: @noop)
       begin
         yield url

data/lib/bricolage/application.rb

@@ -39,7 +39,7 @@ module Bricolage
       @hooks.run_before_option_parsing_hooks(opts)
       opts.parse!(ARGV)
 
-      @ctx = Context.for_application(opts.home, opts.job_file, environment: opts.environment, global_variables: opts.global_variables)
+      @ctx = Context.for_application(opts.home, opts.job_file, environment: opts.environment, option_variables: opts.option_variables)
       opts.merge_saved_options(@ctx.load_system_options)
 
       if opts.dump_options?
@@ -294,7 +294,7 @@ module Bricolage
       @job_file = nil
       @environment = nil
       @home = nil
-      @global_variables = Variables.new
+      @option_variables = Variables.new
       @dry_run = false
       @explain = false
       @list_global_variables = false
@@ -351,9 +351,9 @@ Global Options:
       parser.on('-r', '--require=FEATURE', 'Requires ruby library.') {|feature|
         require feature
       }
-      parser.on('-v', '--variable=NAME=VALUE', 'Set global variable (is different from job-level -v !!).') {|name_value|
+      parser.on('-v', '--variable=NAME=VALUE', 'Set option variable.') {|name_value|
         name, value = name_value.split('=', 2)
-        @global_variables[name] = value
+        @option_variables[name] = value
       }
       parser.on('--dump-options', 'Shows option parsing result and quit.') {
         @dump_options = true
@@ -401,7 +401,7 @@ Global Options:
       @dump_options
     end
 
-    attr_reader :global_variables
+    attr_reader :option_variables
 
     def list_global_variables?
       @list_global_variables

data/lib/bricolage/context.rb

@@ -19,7 +19,7 @@ module Bricolage
       FileSystem.home_path(opt_path)
     end
 
-    def Context.for_application(home_path = nil, job_path_0 = nil, job_path: nil, environment: nil, global_variables: nil, logger: nil)
+    def Context.for_application(home_path = nil, job_path_0 = nil, job_path: nil, environment: nil, option_variables: nil, logger: nil)
       env = environment(environment)
       if (job_path ||= job_path_0)
         fs = FileSystem.for_job_path(job_path, env)
@@ -29,24 +29,28 @@ module Bricolage
       else
         fs = FileSystem.for_options(home_path, env)
       end
-      load(fs, env, global_variables: global_variables, logger: logger)
+      load(fs, env, option_variables: option_variables, logger: logger)
     end
 
-    def Context.load(fs, env, global_variables: nil, data_sources: nil, logger: nil)
-      new(fs, env, global_variables: global_variables, logger: logger).tap {|ctx|
+    def Context.load(fs, env, option_variables: nil, data_sources: nil, logger: nil)
+      new(fs, env, option_variables: option_variables, logger: logger).tap {|ctx|
         ctx.load_configurations
       }
     end
     private_class_method :load
 
-    def initialize(fs, env, global_variables: nil, data_sources: nil, logger: nil)
-      @logger = logger || Logger.default
+    def initialize(fs, env, option_variables: nil, data_sources: nil, logger: nil)
+      @logger = logger || default_logger(env)
       @filesystem = fs
       @environment = env
-      @opt_global_variables = global_variables || Variables.new
+      @option_variables = option_variables || Variables.new
       @data_sources = data_sources
     end
 
+    private def default_logger(env)
+      Logger.new(device: $stderr, level: Logger::INFO)
+    end
+
     def load_configurations
       @filesystem.config_pathes('prelude.rb').each do |path|
         EmbeddedCodeAPI.module_eval(File.read(path), path.to_s, 1) if path.exist?
@@ -56,6 +60,7 @@ module Bricolage
 
     attr_reader :environment
     attr_reader :logger
+    attr_reader :option_variables
 
     def get_data_source(type, name)
       @data_sources.get(type, name)
@@ -63,7 +68,7 @@ module Bricolage
 
     def subsystem(id)
       self.class.new(@filesystem.subsystem(id), @environment,
-        global_variables: @opt_global_variables,
+        option_variables: @option_variables,
         data_sources: @data_sources,
         logger: @logger)
     end
@@ -102,7 +107,6 @@ module Bricolage
       Variables.union(
         builtin_variables,
         load_global_variables,
-        @opt_global_variables
       )
     end
 
@@ -130,8 +134,11 @@ module Bricolage
 
     def load_variables(path)
       Variables.define {|vars|
-        @filesystem.config_file_loader.load_yaml(path).each do |name, value|
-          vars[name] = value
+        kvs = @filesystem.config_file_loader.load_yaml(path)
+        if kvs
+          kvs.each do |name, value|
+            vars[name] = value
+          end
         end
       }
     end

data/lib/bricolage/dao/job.rb

@@ -0,0 +1,184 @@
+require 'bricolage/exception'
+
+module Bricolage
+  module DAO
+
+    class Job
+      include SQLUtils
+
+      Attributes = Struct.new(:id, :subsystem, :job_name, :jobnet_id, :executor_id, keyword_init: true)
+
+      def Job.for_record(r)
+        Attributes.new(
+          id: r['job_id']&.to_i,
+          subsystem: r['subsystem'],
+          job_name: r['job_name'],
+          jobnet_id: r['jobnet_id']&.to_i,
+          executor_id: r['executor_id']
+        )
+      end
+
+      def initialize(datasource)
+        @datasource = datasource
+      end
+
+      private def connect(&block)
+        @datasource.open_shared_connection(&block)
+      end
+
+      def find_or_create(jobnet_id, job_ref)
+        connect {|conn|
+          job = find(conn, jobnet_id, job_ref)   # optimize the most frequent case
+          if job
+            job
+          else
+            begin
+              create(conn, jobnet_id, job_ref)
+            rescue UniqueViolationException
+              find(conn, jobnet_id, job_ref) or raise "[BUG] Could not find/create job record: jobnet_id=#{jobnet_id}, ref=#{job_ref}"
+            end
+          end
+        }
+      end
+
+      private def find(conn, jobnet_id, job_ref)
+        record = conn.query_row(<<~EndSQL)
+            select
+                "job_id"
+                , "subsystem"
+                , "job_name"
+                , "executor_id"
+                , jobnet_id
+            from
+                jobs
+            where
+                jobnet_id = #{jobnet_id}
+                and "subsystem" = #{s job_ref.subsystem}
+                and "job_name" = #{s job_ref.name}
+            ;
+        EndSQL
+
+        if record
+          Job.for_record(record)
+        else
+          nil
+        end
+      end
+
+      private def create(conn, jobnet_id, job_ref)
+        record = conn.query_row(<<~EndSQL)
+            insert into jobs
+                ( "subsystem"
+                , "job_name"
+                , jobnet_id
+                )
+            values
+                ( #{s job_ref.subsystem}
+                , #{s job_ref.name}
+                , #{jobnet_id}
+                )
+            returning "job_id", "subsystem", "job_name", jobnet_id
+            ;
+        EndSQL
+
+        Job.for_record(record)
+      end
+
+      def locked?(job_ids)
+        count = connect {|conn|
+          conn.query_value(<<~EndSQL)
+              select
+                  count(job_id)
+              from
+                  jobs
+              where
+                  job_id in (#{job_ids.join(',')})
+                  and executor_id is not null
+              ;
+          EndSQL
+        }
+
+        count.to_i > 0
+      end
+
+      def locked_jobs(jobnet_id)
+        records = connect {|conn|
+          conn.query_rows(<<~EndSQL)
+              select
+                  "job_id"
+                  , "subsystem"
+                  , "job_name"
+                  , jobnet_id
+                  , "executor_id"
+              from
+                  jobs
+              where
+                  jobnet_id = #{jobnet_id}
+                  and executor_id is not null
+              ;
+          EndSQL
+        }
+
+        if records.empty?
+          []
+        else
+          record.map {|r| Job.for_record(r) }
+        end
+      end
+
+      def lock(job_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+              update jobs
+              set
+                  executor_id = #{s executor_id}
+              where
+                  job_id = #{job_id}
+                  and executor_id is null
+              returning job_id
+              ;
+          EndSQL
+        }
+
+        if records.empty?
+          raise DoubleLockError, "Could not lock job: job_id=#{job_id}"
+        end
+      end
+
+      # Unlock the job.
+      # Returns true if successfully unlocked, otherwise false.
+      # FIXME: raise an exception on failing unlock?
+      def unlock(job_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobs
+            set
+                executor_id = null
+            where
+                job_id = #{job_id}
+                and executor_id = #{s executor_id}
+            returning job_id
+            ;
+          EndSQL
+        }
+
+        not records.empty?
+      end
+
+      def clear_lock_all(jobnet_id)
+        connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobs
+            set
+                executor_id = null
+            where
+                jobnet_id = #{jobnet_id}
+            ;
+          EndSQL
+        }
+      end
+
+    end   # class Job
+
+  end
+end

data/lib/bricolage/dao/jobexecution.rb

@@ -0,0 +1,253 @@
+module Bricolage
+  module DAO
+
+    class JobExecution
+      include SQLUtils
+
+      STATUS_WAITING = 'waiting'.freeze
+      STATUS_SUCCEEDED = 'succeeded'.freeze
+      STATUS_RUNNING = 'running'.freeze
+      STATUS_FAILED = 'failed'.freeze
+      STATUS_CANCELED = 'canceled'.freeze
+
+      Attributes = Struct.new(:job_id, :job_execution_id, :subsystem, :job_name, keyword_init: true)
+
+      def JobExecution.for_record(r)
+        Attributes.new(
+          job_id: r['job_id']&.to_i,
+          job_execution_id: r['job_execution_id']&.to_i,
+          subsystem: r['subsystem'],
+          job_name: r['job_name']
+        )
+      end
+
+      def JobExecution.for_connection(conn)
+        new(nil, connection: conn)
+      end
+
+      def initialize(datasource, connection: nil)
+        @datasource = datasource
+        @connection = connection
+      end
+
+      private def connect
+        if @connection
+          yield @connection
+        else
+          @datasource.open_shared_connection {|conn|
+            yield conn
+          }
+        end
+      end
+
+      def enqueued_jobs(jobnet_ref)
+        records = connect {|conn|
+          conn.query_rows(<<~EndSQL)
+            select
+                e.job_execution_id
+                , e.job_id
+                , j.subsystem
+                , j.job_name
+            from
+                job_executions e
+                inner join jobs j using (job_id)
+                inner join jobnets n using (jobnet_id)
+            where
+                n.subsystem = #{s jobnet_ref.subsystem}
+                and n.jobnet_name = #{s jobnet_ref.name}
+                and e.status in (#{s STATUS_WAITING}, #{s STATUS_RUNNING}, #{s STATUS_FAILED})
+            order by
+                e.execution_sequence
+            ;
+          EndSQL
+        }
+        records.map {|r| JobExecution.for_record(r) }
+      end
+
+      def enqueue_job(job, execution_sequence)
+        record = nil
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+            insert into job_executions
+                ( job_id
+                , execution_sequence
+                , status
+                , message
+                , submitted_at
+                )
+            values
+                ( #{job.id}
+                , #{execution_sequence}
+                , #{s STATUS_WAITING}
+                , ''
+                , now()
+                )
+            returning job_execution_id, job_id
+            ;
+          EndSQL
+
+          record = records.first
+          save_state_transition(conn, record['job_execution_id'], 'submitted_at')
+        }
+
+        exec = JobExecution.for_record(record)
+        exec.subsystem = job.subsystem
+        exec.job_name = job.job_name
+        exec
+      end
+
+      def cancel_jobnet(jobnet_ref, message)
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+            update job_executions
+            set
+                status = #{s STATUS_CANCELED}
+                , message = #{s message}
+                , finished_at = now()
+            where
+                job_id in (
+                    select
+                        j.job_id
+                    from
+                        jobs j inner join jobnets n using (jobnet_id)
+                    where
+                        n.subsystem = #{s jobnet_ref.subsystem}
+                        and n.jobnet_name = #{s jobnet_ref.name}
+                )
+                and status in (#{s STATUS_WAITING}, #{s STATUS_RUNNING}, #{s STATUS_FAILED})
+            returning job_execution_id
+            ;
+          EndSQL
+
+          job_execution_ids = records.map {|r| r['job_execution_id'].to_i }
+          unless job_execution_ids.empty?
+            conn.execute_update(<<~EndSQL)
+                insert into job_execution_states
+                    ( job_execution_id
+                    , job_id
+                    , created_at
+                    , status
+                    , message
+                    )
+                select
+                    job_execution_id
+                    , job_id
+                    , finished_at
+                    , status
+                    , message
+                from
+                    job_executions
+                where
+                    job_execution_id in (#{job_execution_ids.join(', ')})
+                ;
+            EndSQL
+          end
+        }
+      end
+
+      def transition_to_running(job_execution_id)
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+              update job_executions
+              set
+                  status = #{s STATUS_RUNNING}
+                  , message = ''
+                  , started_at = now()
+                  , finished_at = null
+              where
+                  job_execution_id = #{job_execution_id}
+                  and status in (#{s STATUS_WAITING}, #{s STATUS_FAILED})
+              returning job_execution_id
+              ;
+          EndSQL
+          if records.empty?
+            raise IllegalJobStateException, "Could not run already running job: job_execution_id=#{job_execution_id}"
+          end
+
+          save_state_transition(conn, job_execution_id, 'started_at')
+        }
+      end
+
+      def transition_to_succeeded(job_execution_id)
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+              update job_executions
+              set
+                  finished_at = now()
+                  , status = #{s STATUS_SUCCEEDED}
+                  , message = ''
+              where
+                  job_execution_id = #{job_execution_id}
+                  and status = #{s STATUS_RUNNING}
+              returning job_execution_id
+              ;
+          EndSQL
+          if records.empty?
+            raise IllegalJobStateException, "could not transition to succeeded state: job_execution_id=#{job_execution_id}"
+          end
+
+          save_state_transition(conn, job_execution_id, 'finished_at')
+        }
+      end
+
+      def transition_to_failed(job_execution_id, message)
+        connect {|conn|
+          records = conn.execute_update(<<~EndSQL)
+              update job_executions
+              set
+                  finished_at = now()
+                  , status = #{s STATUS_FAILED}
+                  , message = #{s message}
+              where
+                  job_execution_id = #{job_execution_id}
+                  and status = #{s STATUS_RUNNING}
+              returning job_execution_id
+              ;
+          EndSQL
+          if records.empty?
+            raise IllegalJobStateException, "could not transition to failed state: job_execution_id=#{job_execution_id}"
+          end
+
+          save_state_transition(conn, job_execution_id, 'finished_at')
+        }
+      end
+
+      private def save_state_transition(conn, job_execution_id, time_expr)
+        conn.execute_update(<<~EndSQL)
+            insert into job_execution_states
+                ( job_execution_id
+                , job_id
+                , created_at
+                , status
+                , message
+                )
+            select
+                job_execution_id
+                , job_id
+                , #{time_expr}
+                , status
+                , message
+            from
+                job_executions
+            where
+                job_execution_id = #{job_execution_id}
+            ;
+        EndSQL
+      end
+
+      # For tests only
+      def delete_all
+        connect {|conn|
+          conn.execute_update(<<~EndSQL)
+              delete from job_execution_states;
+              delete from job_executions;
+              delete from jobs;
+              delete from jobnets;
+          EndSQL
+        }
+      end
+
+    end   # class JobExecution
+
+  end
+end