bricolage 5.29.0 → 6.0.0beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 937a773a3be70440b08a33edea44480af5b5b4e9857081e5e9dfafeff7298cf0
-  data.tar.gz: 373f091f9ce32ade9862e6c9d114c0ed760166a050ba67b3fa25d29fa42183b2
+  metadata.gz: dd96a97704f116e03c432aaab65d8ee28e76a571415fc8a11ad6ce7c0d594530
+  data.tar.gz: 271b47cee82b89c910f5ea1c9804d2be272b1817dc1ee0c5e8d5125e194f13d7
 SHA512:
-  metadata.gz: 338d2113c8fc175a15f57fcaae506f83da82c60179bdbcf828fb68da45545f2988afec5eed0a70a79292030d93cb97d68862da2e716f6c409f80eb1cf1d0874f
-  data.tar.gz: afa065e5b744ae159a018a050a9e2bb693fce818bc799e85c762b42291597ab96e64223d8569d0c17559ed96e16fe8771b342f2b632f490a43e93e902dcd58aa
+  metadata.gz: 45cd7ee922e61980526c5b08729be5b9021ada7c8c13a32124b233d2d796a8a10d7bd23f920bcd6b46822828b7acd45b39941c8721c242f8aa127ecd062568ed
+  data.tar.gz: 26f8c7375451d2418f968fb537d6359f4bc16afb6a95ab0f489b3c73b5b02d4ffa44070771a204e9a3b6b26b63768842457fef977dd4b1a24de2b71f8d9b600b

data/.gitignore

@@ -17,6 +17,4 @@ tmp
 _yardoc
 doc/
 
-/dev
-/test/home/config
-Gemfile.lock
+/Gemfile.lock

data/README.md

@@ -13,6 +13,10 @@ See LICENSES file for details.
 
 % rake test
 
+## How to use
+
+See https://github.com/bricolages/bricolage/wiki
+
 ## Author
 
 Minero Aoki

data/RELEASE.md

@@ -1,5 +1,27 @@
 # Bricolage Release Note
 
+## version 6.0.0 beta 2
+
+- [fix] Always transmit error messages from jobs in the jobnet.
+
+## version 6.0.0 beta 1
+
+- [new] Introduces database queue.  Database queue saves job states in the PostgreSQL instance, we now can run bricolage on container environment.
+- [new] Default log level is DEBUG on development environment, INFO on production environment.
+- [new] Only updating query is logged as INFO level.  Read-only queries are logged in DEBUG level.
+
+## version 5.30.0
+
+- [new] streaming_load: new option --ctl-ds to change S3 data source for metadata files.
+
+## version 5.29.2
+
+- [new] load: Allows to set jsonpath by direct S3 URL, instead of relative S3 key.
+
+## version 5.29.1
+
+- [fix] PSQLDataSource: slice_last_stderr may fail when $stderr is not a regular file.
+
 ## version 5.29.0
 
 - [new] bricolage-jobnet command accepts multiple jobs/jobnets and executes them sequencially.

data/bricolage.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'test-unit'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'mocha'
+  s.add_development_dependency 'pry-byebug'
 end

data/jobclass/create.rb

@@ -18,7 +18,7 @@ JobClass.define('create') {
         task.drop_force_if params['drop']
         task.exec params['table-def']
         task.analyze_if params['analyze']
-        task.grant_if params['grant'], params['dest-table']
+        task.grant_if params['grant'], '$dest_table'
       }
     }
   }

data/jobclass/createview.rb

@@ -17,7 +17,7 @@ JobClass.define('createview') {
       task.transaction {
         task.drop_view_force_if params['drop']
         task.exec params['sql-file']
-        task.grant_if params['grant'], params['dest-table']
+        task.grant_if params['grant'], '$dest_table'
       }
     }
   }

data/jobclass/load.rb

@@ -35,9 +35,9 @@ JobClass.define('load') {
         task.transaction {
           task.drop_force '${dest_table}'
           task.exec params['table-def']
-          task.load params['src-ds'], params['src-file'], params['dest-table'],
+          task.load params['src-ds'], params['src-file'], '$dest_table',
               params['format'], params['jsonpath'], params['options']
-          task.grant_if params['grant'], params['dest-table']
+          task.grant_if params['grant'], '$dest_table'
         }
         # ANALYZE, VACUUM is needless for newly loaded table, skip always.
 
@@ -49,9 +49,9 @@ JobClass.define('load') {
 
         task.truncate_if params['truncate']
         task.transaction {
-          task.load params['src-ds'], params['src-file'], params['dest-table'],
+          task.load params['src-ds'], params['src-file'], '$dest_table',
               params['format'], params['jsonpath'], params['options']
-          task.grant_if params['grant'], params['dest-table']
+          task.grant_if params['grant'], '$dest_table'
         }
         # ANALYZE, VACUUM is needless for newly loaded table, skip always.
 
@@ -59,9 +59,9 @@ JobClass.define('load') {
         # load only pattern
 
         task.transaction {
-          task.load params['src-ds'], params['src-file'], params['dest-table'],
+          task.load params['src-ds'], params['src-file'], '$dest_table',
               params['format'], params['jsonpath'], params['options']
-          task.grant_if params['grant'], params['dest-table']
+          task.grant_if params['grant'], '$dest_table'
           task.analyze_if params['analyze']
         }
         # We cannot execute VACUUM in transaction

data/jobclass/rebuild-drop.rb

@@ -23,19 +23,19 @@ JobClass.define('rebuild-drop') {
     script.task(params['data-source']) {|task|
       task.transaction {
         # CREATE
-        task.drop_force params['dest-table']
+        task.drop_force '$dest_table'
         task.exec params['table-def']
 
         # INSERT
         task.exec params['sql-file']
 
         # GRANT
-        task.grant_if params['grant'], params['dest-table']
+        task.grant_if params['grant'], '$dest_table'
       }
 
       # VACUUM, ANALYZE
-      task.vacuum_if params['vacuum'], params['vacuum-sort'], params['dest-table']
-      task.analyze_if params['analyze'], params['dest-table']
+      task.vacuum_if params['vacuum'], params['vacuum-sort'], '$dest_table'
+      task.analyze_if params['analyze'], '$dest_table'
     }
   }
 }

data/jobclass/rebuild-rename.rb

@@ -21,6 +21,7 @@ JobClass.define('rebuild-rename') {
 
   script {|params, script|
     script.task(params['data-source']) {|task|
+      dest_table = '$dest_table'
       prev_table = '${dest_table}_old'
       work_table = '${dest_table}_wk'
 
@@ -43,10 +44,9 @@ JobClass.define('rebuild-rename') {
 
       # RENAME
       task.transaction {
-        task.create_dummy_table '$dest_table'
-        dest_table = params['dest-table']
-        task.rename_table dest_table.to_s, "#{dest_table.name}_old"
-        task.rename_table "#{dest_table}_wk", dest_table.name
+        task.create_dummy_table dest_table
+        task.rename_table dest_table, prev_table
+        task.rename_table work_table, dest_table
       }
     }
   }

data/jobclass/sql.rb

@@ -25,7 +25,7 @@ JobClass.define('sql') {
       task.exec params['sql-file']
       task.vacuum_if params['vacuum'], params['vacuum-sort']
       task.analyze_if params['analyze']
-      task.grant_if params['grant'], params['dest-table']
+      task.grant_if params['grant'], '$dest_table'
     }
   }
 }

data/jobclass/streaming_load.rb

@@ -18,6 +18,7 @@ class StreamingLoadJobClass < RubyJobClass
         optional: true, default: Bricolage::PSQLLoadOptions.new,
         value_handler: lambda {|value, ctx, vars| Bricolage::PSQLLoadOptions.parse(value) })
     params.add Bricolage::DataSourceParam.new('s3', 's3-ds', 'S3 data source.')
+    params.add Bricolage::DataSourceParam.new('s3', 'ctl-ds', 'S3 data source for control files. (default: $s3-ds)', optional: true)
     params.add Bricolage::StringParam.new('ctl-prefix', 'S3_PREFIX', 'S3 object key prefix for control files. (default: ${queue-path}/ctl)', optional: true)
     params.add Bricolage::OptionalBoolParam.new('keep-ctl', 'Does not delete control files if true.')
     params.add Bricolage::StringParam.new('queue-path', 'S3_PATH', 'S3 path for data file queue.')
@@ -59,8 +60,6 @@ class StreamingLoadJobClass < RubyJobClass
 
   def make_loader(params)
     ds = params['redshift-ds']
-    load_opts = params['load-options']
-    load_opts.provide_defaults(params['s3-ds'])
     RedshiftStreamingLoader.new(
       data_source: ds,
       queue: make_s3_queue(params),
@@ -68,7 +67,7 @@ class StreamingLoadJobClass < RubyJobClass
       table: string(params['dest-table']),
       work_table: string(params['work-table']),
       log_table: string(params['log-table']),
-      load_options: load_opts,
+      load_options: params['load-options'],
       sql: params['sql-file'],
       logger: ds.logger,
       noop: params['noop'],
@@ -81,6 +80,7 @@ class StreamingLoadJobClass < RubyJobClass
     ds = params['s3-ds']
     S3Queue.new(
       data_source: ds,
+      ctl_ds: (params['ctl-ds'] || params['s3-ds']),
       ctl_prefix: (params['ctl-prefix'] || "#{params['queue-path']}/ctl"),
       queue_path: params['queue-path'],
       persistent_path: params['persistent-path'],
@@ -212,7 +212,7 @@ class StreamingLoadJobClass < RubyJobClass
         end
       @logger.info "creating manifest: #{manifest_name}"
       json = make_manifest_json(objects)
-      @logger.info "manifest:\n" + json
+      @logger.debug "manifest:\n" + json
       url = @src.put_control_file(manifest_name, json, noop: @noop)
       yield url
       @src.remove_control_file(File.basename(url), noop: @noop) unless @keep_ctl
@@ -241,7 +241,7 @@ class StreamingLoadJobClass < RubyJobClass
       log_name = "load_log-#{@job_process_id}.csv"
       @logger.info "creating tmp load log: #{log_name}"
       csv = make_load_log_csv(objects)
-      @logger.info "load_log:\n" + csv
+      @logger.debug "load_log:\n" + csv
       url = @src.put_control_file(log_name, csv, noop: @noop)
       begin
         yield url
@@ -362,8 +362,9 @@ class StreamingLoadJobClass < RubyJobClass
   class S3Queue
     extend Forwardable
 
-    def initialize(data_source:, ctl_prefix:, queue_path:, persistent_path:, file_name:, logger:)
+    def initialize(data_source:, ctl_ds:, ctl_prefix:, queue_path:, persistent_path:, file_name:, logger:)
       @ds = data_source
+      @ctl_ds = ctl_ds
       @ctl_prefix = ctl_prefix
       @queue_path = queue_path
       @persistent_path = persistent_path
@@ -388,18 +389,18 @@ class StreamingLoadJobClass < RubyJobClass
     end
 
     def control_file_url(name)
-      @ds.url(control_file_path(name))
+      @ctl_ds.url(control_file_path(name))
     end
 
     def put_control_file(name, data, noop: false)
       @logger.info "s3 put: #{control_file_url(name)}"
-      @ds.object(control_file_path(name)).put(body: data) unless noop
+      @ctl_ds.object(control_file_path(name)).put(body: data) unless noop
       control_file_url(name)
     end
 
     def remove_control_file(name, noop: false)
       @logger.info "s3 delete: #{control_file_url(name)}"
-      @ds.object(control_file_path(name)).delete unless noop
+      @ctl_ds.object(control_file_path(name)).delete unless noop
     end
 
     def control_file_path(name)

data/lib/bricolage/application.rb

@@ -39,7 +39,7 @@ module Bricolage
       @hooks.run_before_option_parsing_hooks(opts)
       opts.parse!(ARGV)
 
-      @ctx = Context.for_application(opts.home, opts.job_file, environment: opts.environment, global_variables: opts.global_variables)
+      @ctx = Context.for_application(opts.home, opts.job_file, environment: opts.environment, option_variables: opts.option_variables)
       opts.merge_saved_options(@ctx.load_system_options)
 
       if opts.dump_options?
@@ -294,7 +294,7 @@ module Bricolage
       @job_file = nil
       @environment = nil
       @home = nil
-      @global_variables = Variables.new
+      @option_variables = Variables.new
       @dry_run = false
       @explain = false
       @list_global_variables = false
@@ -351,9 +351,9 @@ Global Options:
       parser.on('-r', '--require=FEATURE', 'Requires ruby library.') {|feature|
         require feature
       }
-      parser.on('-v', '--variable=NAME=VALUE', 'Set global variable (is different from job-level -v !!).') {|name_value|
+      parser.on('-v', '--variable=NAME=VALUE', 'Set option variable.') {|name_value|
         name, value = name_value.split('=', 2)
-        @global_variables[name] = value
+        @option_variables[name] = value
       }
       parser.on('--dump-options', 'Shows option parsing result and quit.') {
         @dump_options = true
@@ -401,7 +401,7 @@ Global Options:
       @dump_options
     end
 
-    attr_reader :global_variables
+    attr_reader :option_variables
 
     def list_global_variables?
       @list_global_variables

data/lib/bricolage/context.rb

@@ -19,7 +19,7 @@ module Bricolage
       FileSystem.home_path(opt_path)
     end
 
-    def Context.for_application(home_path = nil, job_path_0 = nil, job_path: nil, environment: nil, global_variables: nil, logger: nil)
+    def Context.for_application(home_path = nil, job_path_0 = nil, job_path: nil, environment: nil, option_variables: nil, logger: nil)
       env = environment(environment)
       if (job_path ||= job_path_0)
         fs = FileSystem.for_job_path(job_path, env)
@@ -29,24 +29,28 @@ module Bricolage
       else
         fs = FileSystem.for_options(home_path, env)
       end
-      load(fs, env, global_variables: global_variables, logger: logger)
+      load(fs, env, option_variables: option_variables, logger: logger)
     end
 
-    def Context.load(fs, env, global_variables: nil, data_sources: nil, logger: nil)
-      new(fs, env, global_variables: global_variables, logger: logger).tap {|ctx|
+    def Context.load(fs, env, option_variables: nil, data_sources: nil, logger: nil)
+      new(fs, env, option_variables: option_variables, logger: logger).tap {|ctx|
         ctx.load_configurations
       }
     end
     private_class_method :load
 
-    def initialize(fs, env, global_variables: nil, data_sources: nil, logger: nil)
-      @logger = logger || Logger.default
+    def initialize(fs, env, option_variables: nil, data_sources: nil, logger: nil)
+      @logger = logger || default_logger(env)
       @filesystem = fs
       @environment = env
-      @opt_global_variables = global_variables || Variables.new
+      @option_variables = option_variables || Variables.new
       @data_sources = data_sources
     end
 
+    private def default_logger(env)
+      Logger.new(device: $stderr, level: Logger::INFO)
+    end
+
     def load_configurations
       @filesystem.config_pathes('prelude.rb').each do |path|
         EmbeddedCodeAPI.module_eval(File.read(path), path.to_s, 1) if path.exist?
@@ -56,6 +60,7 @@ module Bricolage
 
     attr_reader :environment
     attr_reader :logger
+    attr_reader :option_variables
 
     def get_data_source(type, name)
       @data_sources.get(type, name)
@@ -63,7 +68,7 @@ module Bricolage
 
     def subsystem(id)
       self.class.new(@filesystem.subsystem(id), @environment,
-        global_variables: @opt_global_variables,
+        option_variables: @option_variables,
         data_sources: @data_sources,
         logger: @logger)
     end
@@ -102,7 +107,6 @@ module Bricolage
       Variables.union(
         builtin_variables,
         load_global_variables,
-        @opt_global_variables
       )
     end
 
@@ -130,8 +134,11 @@ module Bricolage
 
     def load_variables(path)
       Variables.define {|vars|
-        @filesystem.config_file_loader.load_yaml(path).each do |name, value|
-          vars[name] = value
+        kvs = @filesystem.config_file_loader.load_yaml(path)
+        if kvs
+          kvs.each do |name, value|
+            vars[name] = value
+          end
         end
       }
     end

data/lib/bricolage/dao/job.rb

@@ -0,0 +1,184 @@
+require 'bricolage/exception'
+
+module Bricolage
+  module DAO
+
+    class Job
+      include SQLUtils
+
+      Attributes = Struct.new(:id, :subsystem, :job_name, :jobnet_id, :executor_id, keyword_init: true)
+
+      def Job.for_record(r)
+        Attributes.new(
+          id: r['job_id']&.to_i,
+          subsystem: r['subsystem'],
+          job_name: r['job_name'],
+          jobnet_id: r['jobnet_id']&.to_i,
+          executor_id: r['executor_id']
+        )
+      end
+
+      def initialize(datasource)
+        @datasource = datasource
+      end
+
+      private def connect(&block)
+        @datasource.open_shared_connection(&block)
+      end
+
+      def find_or_create(jobnet_id, job_ref)
+        connect {|conn|
+          job = find(conn, jobnet_id, job_ref)   # optimize the most frequent case
+          if job
+            job
+          else
+            begin
+              create(conn, jobnet_id, job_ref)
+            rescue UniqueViolationException
+              find(conn, jobnet_id, job_ref) or raise "[BUG] Could not find/create job record: jobnet_id=#{jobnet_id}, ref=#{job_ref}"
+            end
+          end
+        }
+      end
+
+      private def find(conn, jobnet_id, job_ref)
+        record = conn.query_row(<<~EndSQL)
+            select
+                "job_id"
+                , "subsystem"
+                , "job_name"
+                , "executor_id"
+                , jobnet_id
+            from
+                jobs
+            where
+                jobnet_id = #{jobnet_id}
+                and "subsystem" = #{s job_ref.subsystem}
+                and "job_name" = #{s job_ref.name}
+            ;
+        EndSQL
+
+        if record
+          Job.for_record(record)
+        else
+          nil
+        end
+      end
+
+      private def create(conn, jobnet_id, job_ref)
+        record = conn.query_row(<<~EndSQL)
+            insert into jobs
+                ( "subsystem"
+                , "job_name"
+                , jobnet_id
+                )
+            values
+                ( #{s job_ref.subsystem}
+                , #{s job_ref.name}
+                , #{jobnet_id}
+                )
+            returning "job_id", "subsystem", "job_name", jobnet_id
+            ;
+        EndSQL
+
+        Job.for_record(record)
+      end
+
+      def locked?(job_ids)
+        count = connect {|conn|
+          conn.query_value(<<~EndSQL)
+              select
+                  count(job_id)
+              from
+                  jobs
+              where
+                  job_id in (#{job_ids.join(',')})
+                  and executor_id is not null
+              ;
+          EndSQL
+        }
+
+        count.to_i > 0
+      end
+
+      def locked_jobs(jobnet_id)
+        records = connect {|conn|
+          conn.query_rows(<<~EndSQL)
+              select
+                  "job_id"
+                  , "subsystem"
+                  , "job_name"
+                  , jobnet_id
+                  , "executor_id"
+              from
+                  jobs
+              where
+                  jobnet_id = #{jobnet_id}
+                  and executor_id is not null
+              ;
+          EndSQL
+        }
+
+        if records.empty?
+          []
+        else
+          record.map {|r| Job.for_record(r) }
+        end
+      end
+
+      def lock(job_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+              update jobs
+              set
+                  executor_id = #{s executor_id}
+              where
+                  job_id = #{job_id}
+                  and executor_id is null
+              returning job_id
+              ;
+          EndSQL
+        }
+
+        if records.empty?
+          raise DoubleLockError, "Could not lock job: job_id=#{job_id}"
+        end
+      end
+
+      # Unlock the job.
+      # Returns true if successfully unlocked, otherwise false.
+      # FIXME: raise an exception on failing unlock?
+      def unlock(job_id, executor_id)
+        records = connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobs
+            set
+                executor_id = null
+            where
+                job_id = #{job_id}
+                and executor_id = #{s executor_id}
+            returning job_id
+            ;
+          EndSQL
+        }
+
+        not records.empty?
+      end
+
+      def clear_lock_all(jobnet_id)
+        connect {|conn|
+          conn.execute_update(<<~EndSQL)
+            update jobs
+            set
+                executor_id = null
+            where
+                jobnet_id = #{jobnet_id}
+            ;
+          EndSQL
+        }
+      end
+
+    end   # class Job
+
+  end
+end