bricolage 5.30.0 → 6.0.0beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd34b3e85926b0f85f76d05415341066025a5117c77d1b3b644ac9ce3358f0d2
-  data.tar.gz: 067eed98758a6de824a0ab9133830a637f3cc3790e26974a946d39eaa54032fc
+  metadata.gz: 97c5b2453966ea1464172b34c23e7ba3e6f44c715d745f17f07ebbddf361c437
+  data.tar.gz: 6044bb40e6dabedd831dc90dd4f61e8fc6614bd4b0885233ba2362b1e1c051b8
 SHA512:
-  metadata.gz: d4601475e0c9cbd70e2f54a68ddb06e417288fc52230b3ac553ff8a3f1ef79dafa6d83a8e543470d9c5cfd9d040a98df66217e2cd7a39aec6e1cd4b342640b0f
-  data.tar.gz: aa8d9bf658e8f36a84eacd18c3d680a18178d7612c995fbdbb8dfd695720331f953607614c1ad2d467c8a211e6fd6bf29f853257455b72700fe93172846375e3
+  metadata.gz: 421dd79e8da48871f3a6694f4ca8bc490280acd4dfad11f10c3165854e7c6a33b76b374a5939805304d2b53307c40c048c28c266f57b54514bf9d81226c21304
+  data.tar.gz: 95c5f97fdf7851d6def0e87713acf3eaae136ba8d2e5971b85b10cbd7b04b95f41e28925eac4912f63d2359e818e94d09c3fa905c8947e9b38747c2c71417f08

data/.gitignore

@@ -17,6 +17,4 @@ tmp
 _yardoc
 doc/
 
-/dev
-/test/home/config
-Gemfile.lock
+/Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+3.0

data/README.md

@@ -11,6 +11,9 @@ See LICENSES file for details.
 
 ## Running Test
 
+Create bricolage_test database and user on your local machine.
+See config/test/datasource.yml for details.  Then type:
+
 % rake test
 
 ## How to use

data/RELEASE.md

@@ -1,5 +1,27 @@
 # Bricolage Release Note
 
+## version 6.0.0 beta 5
+
+- [new] Supports Ruby 3.0.
+
+## version 6.0.0 beta 4
+
+- [fix] rebuild-rename did not work with ALTER RENAME syntax error.
+
+## version 6.0.0 beta 3
+
+- [new] Upgrades pg from 0.18 to 1.2.
+
+## version 6.0.0 beta 2
+
+- [fix] Always transmit error messages from jobs in the jobnet.
+
+## version 6.0.0 beta 1
+
+- [new] Introduces database queue.  Database queue saves job states in the PostgreSQL instance, we now can run bricolage on container environment.
+- [new] Default log level is DEBUG on development environment, INFO on production environment.
+- [new] Only updating query is logged as INFO level.  Read-only queries are logged in DEBUG level.
+
 ## version 5.30.0
 
 - [new] streaming_load: new option --ctl-ds to change S3 data source for metadata files.

data/Rakefile

@@ -1,3 +1,13 @@
+require 'rake/testtask'
+
 task :test do
-  load "#{__dir__}/test/all.rb"
+  desc 'Run tests'
+  Rake::TestTask.new do |t|
+    # To run test cases of specific file(s), Use:
+    # % rake test TEST=test/test_specified_path.rb
+    t.libs << "test"
+    t.test_files = Dir["test/**/test_*.rb"]
+    t.verbose = true
+    t.warning = true
+  end
 end

data/bricolage.gemspec

@@ -16,11 +16,13 @@ Gem::Specification.new do |s|
   s.executables = s.files.grep(%r{bin/}).map {|path| File.basename(path) }
   s.require_path = 'lib'
 
-  s.required_ruby_version = '>= 2.0.0'
-  s.add_dependency 'pg', '~> 0.18.0'
-  s.add_dependency 'aws-sdk-s3', '~> 1'
-  s.add_dependency 'aws-sdk-sns', '~> 1'
-  s.add_development_dependency 'test-unit'
-  s.add_development_dependency 'rake'
-  s.add_development_dependency 'mocha'
+  s.required_ruby_version = '>= 2.4.0'
+  s.add_dependency 'pg', '~> 1.2.3'
+  s.add_dependency 'aws-sdk-s3', '~> 1.64'
+  s.add_dependency 'aws-sdk-sns', '~> 1.23'
+  s.add_dependency 'nokogiri'   # aws-sdk-core requires this
+  s.add_development_dependency 'test-unit', '~> 3.3'
+  s.add_development_dependency 'rake', '~> 13.0'
+  s.add_development_dependency 'mocha', '~> 1.11'
+  s.add_development_dependency 'pry-byebug', '~> 3.9'
 end

data/config/test/datasource.yml

@@ -0,0 +1,9 @@
+test_db:
+  type: psql
+  host: localhost
+  port: 5432
+  database: bricolage_test
+  username: bricolage_test
+  password: bricolage_test
+  encoding: utf8
+  sql_log_level: DEBUG

data/jobclass/rebuild-rename.rb

@@ -21,18 +21,18 @@ JobClass.define('rebuild-rename') {
 
   script {|params, script|
     script.task(params['data-source']) {|task|
-      dest_table = '$dest_table'
-      prev_table = '${dest_table}_old'
-      work_table = '${dest_table}_wk'
+      dest_table = params['dest-table']
+      prev_table = TableSpec.parse("#{dest_table}_old")
+      work_table = TableSpec.parse("#{dest_table}_wk")
 
       task.transaction {
         # CREATE
         task.drop_force prev_table
         task.drop_force work_table
-        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table.to_s)
 
         # INSERT
-        task.exec params['sql-file'].replace(/\$\{?dest_table\}?\b/, work_table)
+        task.exec params['sql-file'].replace(/\$\{?dest_table\}?\b/, work_table.to_s)
 
         # GRANT
         task.grant_if params['grant'], work_table
@@ -45,8 +45,8 @@ JobClass.define('rebuild-rename') {
       # RENAME
       task.transaction {
         task.create_dummy_table dest_table
-        task.rename_table dest_table, prev_table
-        task.rename_table work_table, dest_table
+        task.rename_table dest_table, prev_table.name
+        task.rename_table work_table, dest_table.name
       }
     }
   }

data/jobclass/streaming_load.rb

@@ -144,7 +144,7 @@ class StreamingLoadJobClass < RubyJobClass
         @ds.open {|conn|
           execute_update conn, "delete #{log_table_wk};"
           execute_update conn, load_log_copy_stmt(log_table_wk, log_url, @src.credential_string)
-          loaded, not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
+          loaded, _not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
           loaded.each do |obj|
             obj.dequeue(force: true, noop: @noop)
           end
@@ -212,7 +212,7 @@ class StreamingLoadJobClass < RubyJobClass
         end
       @logger.info "creating manifest: #{manifest_name}"
       json = make_manifest_json(objects)
-      @logger.info "manifest:\n" + json
+      @logger.debug "manifest:\n" + json
       url = @src.put_control_file(manifest_name, json, noop: @noop)
       yield url
       @src.remove_control_file(File.basename(url), noop: @noop) unless @keep_ctl
@@ -241,7 +241,7 @@ class StreamingLoadJobClass < RubyJobClass
       log_name = "load_log-#{@job_process_id}.csv"
       @logger.info "creating tmp load log: #{log_name}"
       csv = make_load_log_csv(objects)
-      @logger.info "load_log:\n" + csv
+      @logger.debug "load_log:\n" + csv
       url = @src.put_control_file(log_name, csv, noop: @noop)
       begin
         yield url

data/lib/bricolage/application.rb

@@ -39,7 +39,7 @@ module Bricolage
       @hooks.run_before_option_parsing_hooks(opts)
       opts.parse!(ARGV)
 
-      @ctx = Context.for_application(opts.home, opts.job_file, environment: opts.environment, global_variables: opts.global_variables)
+      @ctx = Context.for_application(opts.home, opts.job_file, environment: opts.environment, option_variables: opts.option_variables)
       opts.merge_saved_options(@ctx.load_system_options)
 
       if opts.dump_options?
@@ -294,7 +294,7 @@ module Bricolage
       @job_file = nil
       @environment = nil
       @home = nil
-      @global_variables = Variables.new
+      @option_variables = Variables.new
       @dry_run = false
       @explain = false
       @list_global_variables = false
@@ -351,9 +351,9 @@ Global Options:
       parser.on('-r', '--require=FEATURE', 'Requires ruby library.') {|feature|
         require feature
       }
-      parser.on('-v', '--variable=NAME=VALUE', 'Set global variable (is different from job-level -v !!).') {|name_value|
+      parser.on('-v', '--variable=NAME=VALUE', 'Set option variable.') {|name_value|
         name, value = name_value.split('=', 2)
-        @global_variables[name] = value
+        @option_variables[name] = value
       }
       parser.on('--dump-options', 'Shows option parsing result and quit.') {
         @dump_options = true
@@ -401,7 +401,7 @@ Global Options:
       @dump_options
     end
 
-    attr_reader :global_variables
+    attr_reader :option_variables
 
     def list_global_variables?
       @list_global_variables

data/lib/bricolage/configloader.rb

@@ -32,7 +32,7 @@ module Bricolage
     end
 
     def eruby(text, path)
-      erb = ERB.new(text, nil, '%-')
+      erb = ERB.new(text, trim_mode: '%-')
       erb.filename = path.to_s
       push_base_dir(path) {
         erb.result(binding())

data/lib/bricolage/context.rb

@@ -19,7 +19,7 @@ module Bricolage
       FileSystem.home_path(opt_path)
     end
 
-    def Context.for_application(home_path = nil, job_path_0 = nil, job_path: nil, environment: nil, global_variables: nil, logger: nil)
+    def Context.for_application(home_path = nil, job_path_0 = nil, job_path: nil, environment: nil, option_variables: nil, logger: nil)
       env = environment(environment)
       if (job_path ||= job_path_0)
         fs = FileSystem.for_job_path(job_path, env)
@@ -29,24 +29,28 @@ module Bricolage
       else
         fs = FileSystem.for_options(home_path, env)
       end
-      load(fs, env, global_variables: global_variables, logger: logger)
+      load(fs, env, option_variables: option_variables, logger: logger)
     end
 
-    def Context.load(fs, env, global_variables: nil, data_sources: nil, logger: nil)
-      new(fs, env, global_variables: global_variables, logger: logger).tap {|ctx|
+    def Context.load(fs, env, option_variables: nil, data_sources: nil, logger: nil)
+      new(fs, env, option_variables: option_variables, logger: logger).tap {|ctx|
         ctx.load_configurations
       }
     end
     private_class_method :load
 
-    def initialize(fs, env, global_variables: nil, data_sources: nil, logger: nil)
-      @logger = logger || Logger.default
+    def initialize(fs, env, option_variables: nil, data_sources: nil, logger: nil)
+      @logger = logger || default_logger(env)
       @filesystem = fs
       @environment = env
-      @opt_global_variables = global_variables || Variables.new
+      @option_variables = option_variables || Variables.new
       @data_sources = data_sources
     end
 
+    private def default_logger(env)
+      Logger.new(device: $stderr, level: Logger::INFO)
+    end
+
     def load_configurations
       @filesystem.config_pathes('prelude.rb').each do |path|
         EmbeddedCodeAPI.module_eval(File.read(path), path.to_s, 1) if path.exist?
@@ -56,6 +60,7 @@ module Bricolage
 
     attr_reader :environment
     attr_reader :logger
+    attr_reader :option_variables
 
     def get_data_source(type, name)
       @data_sources.get(type, name)
@@ -63,7 +68,7 @@ module Bricolage
 
     def subsystem(id)
       self.class.new(@filesystem.subsystem(id), @environment,
-        global_variables: @opt_global_variables,
+        option_variables: @option_variables,
         data_sources: @data_sources,
         logger: @logger)
     end
@@ -102,7 +107,6 @@ module Bricolage
       Variables.union(
         builtin_variables,
         load_global_variables,
-        @opt_global_variables
       )
     end
 
@@ -130,8 +134,11 @@ module Bricolage
 
     def load_variables(path)
       Variables.define {|vars|
-        @filesystem.config_file_loader.load_yaml(path).each do |name, value|
-          vars[name] = value
+        kvs = @filesystem.config_file_loader.load_yaml(path)
+        if kvs
+          kvs.each do |name, value|
+            vars[name] = value
+          end
         end
       }
     end

data/lib/bricolage/dao/job.rb

@@ -0,0 +1,184 @@
+require 'bricolage/exception'
+
+module Bricolage
+  module DAO
+
+    class Job
+      include SQLUtils
+
+      Attributes = Struct.new(:id, :subsystem, :job_name, :jobnet_id, :executor_id, keyword_init: true)
+
+      def Job.for_record(r)
+        Attributes.new(
+          id: r['job_id']&.to_i,
+          subsystem: r['subsystem'],
+          job_name: r['job_name'],
+          jobnet_id: r['jobnet_id']&.to_i,
+          executor_id: r['executor_id']
+        )
+      end
+
+      def initialize(datasource)
+        @datasource = datasource
+      end
+
+      private def connect(&block)
+        @datasource.open_shared_connection(&block)
+      end
+
+      def find_or_create(jobnet_id, job_ref)
+        connect {|conn|
+          job = find(conn, jobnet_id, job_ref)   # optimize the most frequent case
+          if job
+            job
+          else
+            begin
+              create(conn, jobnet_id, job_ref)
+            rescue UniqueViolationException
+              find(conn, jobnet_id, job_ref) or raise "[BUG] Could not find/create job record: jobnet_id=#{jobnet_id}, ref=#{job_ref}"
+            end
+          end
+        }
+      end
+
+      private def find(conn, jobnet_id, job_ref)
+        record = conn.query_row(<<~EndSQL)
+            select
+                "job_id"
+                , "subsystem"
+                , "job_name"
+                , "executor_id"
+                , jobnet_id
+            from
+                jobs
+            where
+                jobnet_id = #{jobnet_id}
+                and "subsystem" = #{s job_ref.subsystem}
+                and "job_name" = #{s job_ref.name}
+            ;
+        EndSQL
+
+        if record
+          Job.for_record(record)
+        else
+          nil
+        end
+      end
+
+      private def create(conn, jobnet_id, job_ref)
+        record = conn.query_row(<<~EndSQL)
+            insert into jobs
+                ( "subsystem"
+                , "job_name"
+                , jobnet_id
+                )
+            values
+                ( #{s job_ref.subsystem}
+                , #{s job_ref.name}
+                , #{jobnet_id}
+                )
+            returning "job_id", "subsystem", "job_name", jobnet_id
+            ;
+        EndSQL
+
+        Job.for_record(record)
+      end
+
+      def locked?(job_ids)
+        count = connect {|conn|
+          conn.query_value(<<~EndSQL)
+              select
+                  count(job_id)
+              from
+                  jobs
+              where
+                  job_id in (#{job_ids.join(',')})
+                  and executor_id is not null
+              ;
+          EndSQL
+        }
+
+        count.to_i > 0
+      end
+
+      def locked_jobs(jobnet_id)
+        records = connect {|conn|
+          conn.query_rows(<<~EndSQL)
+              select
+                  "job_id"
+                  , "subsystem"
+                  , "job_name"
+                  , jobnet_id
+                  , "executor_id"
+              from
+                  jobs
+              where
+                  jobnet_id = #{jobnet_id}
+                  and executor_id is not null
+              ;
+          EndSQL
+        }
+
+        if records.empty?
+          []
+        else
+          record.map {|r| Job.for_record(r) }
+        end
+      end
+
+      def lock(job_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+              update jobs
+              set
+                  executor_id = #{s executor_id}
+              where
+                  job_id = #{job_id}
+                  and executor_id is null
+              returning job_id
+              ;
+          EndSQL
+        }
+
+        if records.empty?
+          raise DoubleLockError, "Could not lock job: job_id=#{job_id}"
+        end
+      end
+
+      # Unlock the job.
+      # Returns true if successfully unlocked, otherwise false.
+      # FIXME: raise an exception on failing unlock?
+      def unlock(job_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobs
+            set
+                executor_id = null
+            where
+                job_id = #{job_id}
+                and executor_id = #{s executor_id}
+            returning job_id
+            ;
+          EndSQL
+        }
+
+        not records.empty?
+      end
+
+      def clear_lock_all(jobnet_id)
+        connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobs
+            set
+                executor_id = null
+            where
+                jobnet_id = #{jobnet_id}
+            ;
+          EndSQL
+        }
+      end
+
+    end   # class Job
+
+  end
+end