bricolage 6.0.0beta1 → 6.0.0beta6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 111b4bb62234f5a428ccd1da5a5ee2d6d4b1a051c8fda9ddd6fac3e590356797
-  data.tar.gz: 77c3857f92a721503bd07f6e235c81f454d61e3a55e111e7a2c0ce72efa8f6ed
+  metadata.gz: d057475396d2f1e47cfccc490ec4ab07467c0056f519abd5a7c9bafdfdb63714
+  data.tar.gz: 252a0d2c7cb00f1fee102b25eec0b0f0a52919652f2210e2a8c877ec9e532638
 SHA512:
-  metadata.gz: ca02be7fe28f3323f8dcfa1f9c88a0dd70050f7af75109ee077d0347a5f7a4420bb7f7a6b80d9ad207bd447b640089b979e319fa9ed3cf3465e11039327bc2f3
-  data.tar.gz: 7d9d77d27f31d5368a694b9514a1276532335e1f980b2dd45fa77b7a4f30cc500b4dc1fb6a99a0a2f5dd35b42c9e155201764ba3d1d9a253b52790acf1f92857
+  metadata.gz: 685f46050cb796689a9756387dabad8dcae318e6f7ae342f130d0b470b2375986b36aba65284d3cb516bdf9630aae8372eb47426f314dd233abea50bc087c86b
+  data.tar.gz: b50a8d23ec9b1df0e0303b0d6dee90a1c896fabb085943557158e61504107b64fbeaadfc15b83931a56db987ad524933281403ef1f0c41f56e2a0e5b7ac8c043

data/.ruby-version

@@ -0,0 +1 @@
+3.0

data/README.md

@@ -11,6 +11,9 @@ See LICENSES file for details.
 
 ## Running Test
 
+Create bricolage_test database and user on your local machine.
+See config/test/datasource.yml for details.  Then type:
+
 % rake test
 
 ## How to use

data/RELEASE.md

@@ -1,5 +1,31 @@
 # Bricolage Release Note
 
+## version 6.0.0 beta 6
+
+- [new] Now you can refer job file local variables from job parameters.
+
+## version 6.0.0 beta 5
+
+- [new] Supports Ruby 3.0.
+
+## version 6.0.0 beta 4
+
+- [fix] rebuild-rename did not work with ALTER RENAME syntax error.
+
+## version 6.0.0 beta 3
+
+- [new] Upgrades pg from 0.18 to 1.2.
+
+## version 6.0.0 beta 2
+
+- [fix] Always transmit error messages from jobs in the jobnet.
+
+## version 6.0.0 beta 1
+
+- [new] Introduces database queue.  Database queue saves job states in the PostgreSQL instance, we now can run bricolage on container environment.
+- [new] Default log level is DEBUG on development environment, INFO on production environment.
+- [new] Only updating query is logged as INFO level.  Read-only queries are logged in DEBUG level.
+
 ## version 5.30.0
 
 - [new] streaming_load: new option --ctl-ds to change S3 data source for metadata files.

data/Rakefile

@@ -1,3 +1,13 @@
+require 'rake/testtask'
+
 task :test do
-  load "#{__dir__}/test/all.rb"
+  desc 'Run tests'
+  Rake::TestTask.new do |t|
+    # To run test cases of specific file(s), Use:
+    # % rake test TEST=test/test_specified_path.rb
+    t.libs << "test"
+    t.test_files = Dir["test/**/test_*.rb"]
+    t.verbose = true
+    t.warning = true
+  end
 end

data/bricolage.gemspec

@@ -16,12 +16,13 @@ Gem::Specification.new do |s|
   s.executables = s.files.grep(%r{bin/}).map {|path| File.basename(path) }
   s.require_path = 'lib'
 
-  s.required_ruby_version = '>= 2.0.0'
-  s.add_dependency 'pg', '~> 0.18.0'
-  s.add_dependency 'aws-sdk-s3', '~> 1'
-  s.add_dependency 'aws-sdk-sns', '~> 1'
-  s.add_development_dependency 'test-unit'
-  s.add_development_dependency 'rake'
-  s.add_development_dependency 'mocha'
-  s.add_development_dependency 'pry-byebug'
+  s.required_ruby_version = '>= 2.4.0'
+  s.add_dependency 'pg', '~> 1.2.3'
+  s.add_dependency 'aws-sdk-s3', '~> 1.64'
+  s.add_dependency 'aws-sdk-sns', '~> 1.23'
+  s.add_dependency 'nokogiri'   # aws-sdk-core requires this
+  s.add_development_dependency 'test-unit', '~> 3.3'
+  s.add_development_dependency 'rake', '~> 13.0'
+  s.add_development_dependency 'mocha', '~> 1.11'
+  s.add_development_dependency 'pry-byebug', '~> 3.9'
 end

data/config/test/datasource.yml

@@ -0,0 +1,9 @@
+test_db:
+  type: psql
+  host: localhost
+  port: 5432
+  database: bricolage_test
+  username: bricolage_test
+  password: bricolage_test
+  encoding: utf8
+  sql_log_level: DEBUG

data/jobclass/rebuild-rename.rb

@@ -21,18 +21,18 @@ JobClass.define('rebuild-rename') {
 
   script {|params, script|
     script.task(params['data-source']) {|task|
-      dest_table = '$dest_table'
-      prev_table = '${dest_table}_old'
-      work_table = '${dest_table}_wk'
+      dest_table = params['dest-table']
+      prev_table = TableSpec.parse("#{dest_table}_old")
+      work_table = TableSpec.parse("#{dest_table}_wk")
 
       task.transaction {
         # CREATE
         task.drop_force prev_table
         task.drop_force work_table
-        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table.to_s)
 
         # INSERT
-        task.exec params['sql-file'].replace(/\$\{?dest_table\}?\b/, work_table)
+        task.exec params['sql-file'].replace(/\$\{?dest_table\}?\b/, work_table.to_s)
 
         # GRANT
         task.grant_if params['grant'], work_table
@@ -45,8 +45,8 @@ JobClass.define('rebuild-rename') {
       # RENAME
       task.transaction {
         task.create_dummy_table dest_table
-        task.rename_table dest_table, prev_table
-        task.rename_table work_table, dest_table
+        task.rename_table dest_table, prev_table.name
+        task.rename_table work_table, dest_table.name
       }
     }
   }

data/jobclass/streaming_load.rb

@@ -144,7 +144,7 @@ class StreamingLoadJobClass < RubyJobClass
         @ds.open {|conn|
           execute_update conn, "delete #{log_table_wk};"
           execute_update conn, load_log_copy_stmt(log_table_wk, log_url, @src.credential_string)
-          loaded, not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
+          loaded, _not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
           loaded.each do |obj|
             obj.dequeue(force: true, noop: @noop)
           end

data/lib/bricolage/configloader.rb

@@ -32,7 +32,7 @@ module Bricolage
     end
 
     def eruby(text, path)
-      erb = ERB.new(text, nil, '%-')
+      erb = ERB.new(text, trim_mode: '%-')
       erb.filename = path.to_s
       push_base_dir(path) {
         erb.result(binding())

data/lib/bricolage/dao/jobnet.rb

@@ -141,7 +141,7 @@ module Bricolage
       end
 
       def clear_lock(jobnet_id)
-        records = connect {|conn|
+        connect {|conn|
           conn.execute_update(<<~EndSQL)
             update jobnets
             set

data/lib/bricolage/filedatasource.rb

@@ -6,7 +6,7 @@ module Bricolage
   class FileDataSource < DataSource
     declare_type 'file'
 
-    def initialize(opts)
+    def initialize
     end
 
     def new_task

data/lib/bricolage/genericdatasource.rb

@@ -4,8 +4,7 @@ module Bricolage
   class GenericDataSource < DataSource
     declare_type 'generic'
 
-    # FIXME: keyword argument placeholder is required
-    def initialize(**)
+    def initialize
     end
 
     def new_task

data/lib/bricolage/job.rb

@@ -6,6 +6,7 @@ require 'bricolage/variables'
 require 'bricolage/configloader'
 require 'bricolage/loglocator'
 require 'bricolage/exception'
+require 'tmpdir'
 require 'fileutils'
 
 module Bricolage
@@ -50,6 +51,8 @@ module Bricolage
     end
 
     attr_reader :id
+    attr_reader :job_class
+    attr_reader :context
 
     def class_id
       @job_class.id
@@ -96,6 +99,7 @@ module Bricolage
       base_vars = Variables.union(
         #          ^ Low precedence
         @global_variables,
+        job_file_rest_vars,
         cmd_v_opt_vars,
         job_v_opt_vars
         #          v High precedence
@@ -151,9 +155,7 @@ module Bricolage
     end
 
     def execute_in_process(log_locator:)
-      # ??? FIXME: status_path should be independent from log_path.
-      # Also, status_path should be defined regardless of log_path.
-      status_path = log_locator.path ? "#{log_locator.path}.status" : nil
+      status_path = "#{Dir.tmpdir}/bricolage.#{$$}.status.#{"%010x" % rand(1000000000000)}"
       isolate_process(status_path) {
         log_locator.redirect_stdouts {
           do_execute

data/lib/bricolage/logger.rb

@@ -57,7 +57,7 @@ module Bricolage
       case
       when seconds > 60 * 60
         h, secs = seconds.divmod(60 * 60)
-        m, s = secs.divmod(60)
+        m, _sec = secs.divmod(60)
         "%d hours %d minutes" % [h, m]
       when seconds > 60
         "%d minutes %d seconds" % seconds.divmod(60)

data/lib/bricolage/psqldatasource.rb

@@ -117,21 +117,28 @@ module Bricolage
           raise ParameterError, "could not read password: #{path}, #{user}"
     end
 
-    def open(&block)
+    def open
       retries = (ENV['BRICOLAGE_OPEN_RETRY_LIMIT'] || DEFAULT_RETRY_LIMIT).to_i
       begin
         conn = PostgresConnection.open_data_source(self)
-        conn.execute_query('select 1'){}
+        conn.execute_query('select 1') {}
       rescue PG::ConnectionBad, PG::UnableToSend => ex
+        conn.close rescue nil
         retries -= 1
         if retries >= 0
-          logger.warn "Retry PG connection for execute query: #{ex.message}"
+          logger.warn "Could not open postgres connection; retry: #{ex.message}"
           sleep 1
           retry
+        else
+          raise
         end
       end
       if block_given?
-        yield conn
+        begin
+          yield conn
+        ensure
+          conn.close
+        end
       else
         return conn
       end
@@ -140,23 +147,35 @@ module Bricolage
     def open_shared_connection
       raise ParameterError, 'open_shared_connection require block' unless block_given?
       conn = nil
-      if @connection_pool.empty?
-        conn = open
-      else
-        begin
-          conn = @connection_pool.shift
-          conn.execute_query('select 1'){}
-        rescue
-          conn.close
+      until conn
+        if conn_tmp = @connection_pool.shift
+          begin
+            conn_tmp.query('select 1') {}
+          rescue PG::ConnectionBad, PG::UnableToSend
+            # retry
+          else
+            # no exception occured
+            conn = conn_tmp
+            conn_tmp = nil
+          ensure
+            if conn_tmp
+              conn_tmp.close
+              conn_tmp = nil
+            end
+          end
+        else
+          # Get a fresh connection instead of pooled connections.
           conn = open
         end
       end
-
-      yield conn
-    ensure
-      @connection_pool.push(conn)
+      begin
+        yield conn
+      ensure
+        @connection_pool.push(conn)
+      end
     end
 
+    # not MT-safe
     def clear_connection_pool
       @connection_pool.map(&:close)
       @connection_pool = []

data/lib/bricolage/rubyjobclass.rb

@@ -50,8 +50,7 @@ module Bricolage
   class RubyDataSource < DataSource
     declare_type 'ruby'
 
-    # FIXME: keyword argument placeholder is required
-    def initialize(**)
+    def initialize
     end
 
     def new_task

data/lib/bricolage/vacuumlock.rb

@@ -32,7 +32,7 @@ module Bricolage
 
     def psql_serialize_vacuum_end
       if enable_vacuum_lock?
-        path, timeout = vacuum_lock_parameters
+        path, _timeout = vacuum_lock_parameters
         "\\! rm -f #{path}"
       else
         ';'
@@ -88,7 +88,7 @@ module Bricolage
 
     def VacuumLock.cleanup_vacuum_lock
       return unless enable_vacuum_lock?
-      path, timeout = vacuum_lock_parameters
+      path, _timeout = vacuum_lock_parameters
       if locking?(path)
         $stderr.puts "remove VACUUM lock by #{Process.pid}"
         FileUtils.rm_f path

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '6.0.0beta1'
+  VERSION = '6.0.0beta6'
 end

data/schema/Dockerfile

@@ -1,13 +1,16 @@
-FROM ruby:2.6.5-stretch
+FROM ruby:3.0-slim
 
 RUN apt-get update -qq && apt-get install -y build-essential libpq-dev libpq5 postgresql-client
 
-COPY ./Gemfile /tmp/Gemfile
-COPY ./Gemfile.lock /tmp/Gemfile.lock
-RUN cd /tmp && bundle install -j4 --deployment --without 'development test'
-
 WORKDIR /app
-COPY . /app
-RUN cp -a /tmp/vendor /app/
+
+COPY ./Gemfile .
+COPY ./Gemfile.lock .
+RUN gem install bundler:2.2.15 && \
+    bundle config set --local deployment true && \
+    bundle config set --local without 'development test' && \
+    bundle install -j4
+
+COPY . .
 
 CMD ["bundle", "exec", "ridgepole", "-f", "Schemafile", "-c", "database.yml", "--merge", "--dry-run"]

data/schema/Gemfile

@@ -1,4 +1,5 @@
 source 'https://rubygems.org'
 
 gem 'ridgepole'
+gem 'activerecord', '~> 6.0.0'  # activerecord 6.1 did not work with check_constraint
 gem 'pg'

data/schema/Gemfile.lock

@@ -1,37 +1,38 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.4.3)
-      activesupport (= 5.2.4.3)
-    activerecord (5.2.4.3)
-      activemodel (= 5.2.4.3)
-      activesupport (= 5.2.4.3)
-      arel (>= 9.0)
-    activesupport (5.2.4.3)
+    activemodel (6.0.3.6)
+      activesupport (= 6.0.3.6)
+    activerecord (6.0.3.6)
+      activemodel (= 6.0.3.6)
+      activesupport (= 6.0.3.6)
+    activesupport (6.0.3.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    arel (9.0.0)
-    concurrent-ruby (1.1.6)
-    diffy (3.3.0)
-    i18n (1.8.2)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    concurrent-ruby (1.1.8)
+    diffy (3.4.0)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.1)
-    pg (1.1.4)
-    ridgepole (0.7.7)
-      activerecord (>= 5.0.1, < 6)
+    minitest (5.14.4)
+    pg (1.2.3)
+    ridgepole (0.9.2)
+      activerecord (>= 5.1, < 6.2)
       diffy
     thread_safe (0.3.6)
-    tzinfo (1.2.7)
+    tzinfo (1.2.9)
       thread_safe (~> 0.1)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  activerecord (~> 6.0.0)
   pg
   ridgepole
 
 BUNDLED WITH
-   1.17.2
+   2.2.15

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 6.0.0beta1
+  version: 6.0.0beta6
 platform: ruby
 authors:
 - Minero Aoki
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-27 00:00:00.000000000 Z
+date: 2021-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -16,98 +16,112 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.18.0
+        version: 1.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.18.0
+        version: 1.2.3
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.64'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.64'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.23'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.23'
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.9'
 description: Redshift-oriented Data Warehouse Batch Framework
 email: aamine@loveruby.net
 executables:
@@ -117,6 +131,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-version"
 - Gemfile
 - LICENSES
 - README.md
@@ -125,6 +140,7 @@ files:
 - bin/bricolage
 - bin/bricolage-jobnet
 - bricolage.gemspec
+- config/test/datasource.yml
 - jobclass/adhoc.rb
 - jobclass/create.rb
 - jobclass/createview.rb
@@ -194,7 +210,7 @@ homepage: https://github.com/bricolages/bricolage
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -202,15 +218,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: SQL Batch Framework
 test_files: []