bricolage 6.0.0beta4 → 6.0.0beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be51accbd747d2192ccda3a4a502df6083adb2379f459c39ee711208691a67b5
-  data.tar.gz: 86ba8e19da9a68befe60107f80e7376bd08959d5671920bf18065e80ce2fcaeb
+  metadata.gz: 97c5b2453966ea1464172b34c23e7ba3e6f44c715d745f17f07ebbddf361c437
+  data.tar.gz: 6044bb40e6dabedd831dc90dd4f61e8fc6614bd4b0885233ba2362b1e1c051b8
 SHA512:
-  metadata.gz: 6cd44c1d136b65534842dd85f58e4b08bbbf5340e3265a9a67e20e5a7f443ad7ba8eabf3db5a9ca628dbc5f083b38952d5d8379928745534f5d968049de2951d
-  data.tar.gz: '0788438ed231dbaa2e1d6312a62a717178ad0994b08cfe376d9949412c3cf74e147f03b7a1ccb91145ef2f6d26277159ce386b7b15a2110416c533de4dfa4c99'
+  metadata.gz: 421dd79e8da48871f3a6694f4ca8bc490280acd4dfad11f10c3165854e7c6a33b76b374a5939805304d2b53307c40c048c28c266f57b54514bf9d81226c21304
+  data.tar.gz: 95c5f97fdf7851d6def0e87713acf3eaae136ba8d2e5971b85b10cbd7b04b95f41e28925eac4912f63d2359e818e94d09c3fa905c8947e9b38747c2c71417f08

data/.ruby-version

@@ -0,0 +1 @@
+3.0

data/README.md

@@ -11,6 +11,9 @@ See LICENSES file for details.
 
 ## Running Test
 
+Create bricolage_test database and user on your local machine.
+See config/test/datasource.yml for details.  Then type:
+
 % rake test
 
 ## How to use

data/RELEASE.md

@@ -1,5 +1,27 @@
 # Bricolage Release Note
 
+## version 6.0.0 beta 5
+
+- [new] Supports Ruby 3.0.
+
+## version 6.0.0 beta 4
+
+- [fix] rebuild-rename did not work with ALTER RENAME syntax error.
+
+## version 6.0.0 beta 3
+
+- [new] Upgrades pg from 0.18 to 1.2.
+
+## version 6.0.0 beta 2
+
+- [fix] Always transmit error messages from jobs in the jobnet.
+
+## version 6.0.0 beta 1
+
+- [new] Introduces database queue.  Database queue saves job states in the PostgreSQL instance, we now can run bricolage on container environment.
+- [new] Default log level is DEBUG on development environment, INFO on production environment.
+- [new] Only updating query is logged as INFO level.  Read-only queries are logged in DEBUG level.
+
 ## version 5.30.0
 
 - [new] streaming_load: new option --ctl-ds to change S3 data source for metadata files.

data/Rakefile

@@ -1,3 +1,13 @@
+require 'rake/testtask'
+
 task :test do
-  load "#{__dir__}/test/all.rb"
+  desc 'Run tests'
+  Rake::TestTask.new do |t|
+    # To run test cases of specific file(s), Use:
+    # % rake test TEST=test/test_specified_path.rb
+    t.libs << "test"
+    t.test_files = Dir["test/**/test_*.rb"]
+    t.verbose = true
+    t.warning = true
+  end
 end

data/bricolage.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'pg', '~> 1.2.3'
   s.add_dependency 'aws-sdk-s3', '~> 1.64'
   s.add_dependency 'aws-sdk-sns', '~> 1.23'
+  s.add_dependency 'nokogiri'   # aws-sdk-core requires this
   s.add_development_dependency 'test-unit', '~> 3.3'
   s.add_development_dependency 'rake', '~> 13.0'
   s.add_development_dependency 'mocha', '~> 1.11'

data/config/test/datasource.yml

@@ -0,0 +1,9 @@
+test_db:
+  type: psql
+  host: localhost
+  port: 5432
+  database: bricolage_test
+  username: bricolage_test
+  password: bricolage_test
+  encoding: utf8
+  sql_log_level: DEBUG

data/jobclass/streaming_load.rb

@@ -144,7 +144,7 @@ class StreamingLoadJobClass < RubyJobClass
         @ds.open {|conn|
           execute_update conn, "delete #{log_table_wk};"
           execute_update conn, load_log_copy_stmt(log_table_wk, log_url, @src.credential_string)
-          loaded, not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
+          loaded, _not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
           loaded.each do |obj|
             obj.dequeue(force: true, noop: @noop)
           end

data/lib/bricolage/configloader.rb

@@ -32,7 +32,7 @@ module Bricolage
     end
 
     def eruby(text, path)
-      erb = ERB.new(text, nil, '%-')
+      erb = ERB.new(text, trim_mode: '%-')
       erb.filename = path.to_s
       push_base_dir(path) {
         erb.result(binding())

data/lib/bricolage/dao/jobnet.rb

@@ -141,7 +141,7 @@ module Bricolage
       end
 
       def clear_lock(jobnet_id)
-        records = connect {|conn|
+        connect {|conn|
           conn.execute_update(<<~EndSQL)
             update jobnets
             set

data/lib/bricolage/filedatasource.rb

@@ -6,7 +6,7 @@ module Bricolage
   class FileDataSource < DataSource
     declare_type 'file'
 
-    def initialize(opts)
+    def initialize
     end
 
     def new_task

data/lib/bricolage/genericdatasource.rb

@@ -4,8 +4,7 @@ module Bricolage
   class GenericDataSource < DataSource
     declare_type 'generic'
 
-    # FIXME: keyword argument placeholder is required
-    def initialize(**)
+    def initialize
     end
 
     def new_task

data/lib/bricolage/job.rb

@@ -51,6 +51,8 @@ module Bricolage
     end
 
     attr_reader :id
+    attr_reader :job_class
+    attr_reader :context
 
     def class_id
       @job_class.id

data/lib/bricolage/logger.rb

@@ -57,7 +57,7 @@ module Bricolage
       case
       when seconds > 60 * 60
         h, secs = seconds.divmod(60 * 60)
-        m, s = secs.divmod(60)
+        m, _sec = secs.divmod(60)
         "%d hours %d minutes" % [h, m]
       when seconds > 60
         "%d minutes %d seconds" % seconds.divmod(60)

data/lib/bricolage/psqldatasource.rb

@@ -117,21 +117,28 @@ module Bricolage
           raise ParameterError, "could not read password: #{path}, #{user}"
     end
 
-    def open(&block)
+    def open
       retries = (ENV['BRICOLAGE_OPEN_RETRY_LIMIT'] || DEFAULT_RETRY_LIMIT).to_i
       begin
         conn = PostgresConnection.open_data_source(self)
-        conn.execute_query('select 1'){}
+        conn.execute_query('select 1') {}
       rescue PG::ConnectionBad, PG::UnableToSend => ex
+        conn.close rescue nil
         retries -= 1
         if retries >= 0
-          logger.warn "Retry PG connection for execute query: #{ex.message}"
+          logger.warn "Could not open postgres connection; retry: #{ex.message}"
           sleep 1
           retry
+        else
+          raise
         end
       end
       if block_given?
-        yield conn
+        begin
+          yield conn
+        ensure
+          conn.close
+        end
       else
         return conn
       end
@@ -140,23 +147,35 @@ module Bricolage
     def open_shared_connection
       raise ParameterError, 'open_shared_connection require block' unless block_given?
       conn = nil
-      if @connection_pool.empty?
-        conn = open
-      else
-        begin
-          conn = @connection_pool.shift
-          conn.execute_query('select 1'){}
-        rescue
-          conn.close
+      until conn
+        if conn_tmp = @connection_pool.shift
+          begin
+            conn_tmp.query('select 1') {}
+          rescue PG::ConnectionBad, PG::UnableToSend
+            # retry
+          else
+            # no exception occured
+            conn = conn_tmp
+            conn_tmp = nil
+          ensure
+            if conn_tmp
+              conn_tmp.close
+              conn_tmp = nil
+            end
+          end
+        else
+          # Get a fresh connection instead of pooled connections.
           conn = open
         end
       end
-
-      yield conn
-    ensure
-      @connection_pool.push(conn)
+      begin
+        yield conn
+      ensure
+        @connection_pool.push(conn)
+      end
     end
 
+    # not MT-safe
     def clear_connection_pool
       @connection_pool.map(&:close)
       @connection_pool = []

data/lib/bricolage/rubyjobclass.rb

@@ -50,8 +50,7 @@ module Bricolage
   class RubyDataSource < DataSource
     declare_type 'ruby'
 
-    # FIXME: keyword argument placeholder is required
-    def initialize(**)
+    def initialize
     end
 
     def new_task

data/lib/bricolage/vacuumlock.rb

@@ -32,7 +32,7 @@ module Bricolage
 
     def psql_serialize_vacuum_end
       if enable_vacuum_lock?
-        path, timeout = vacuum_lock_parameters
+        path, _timeout = vacuum_lock_parameters
         "\\! rm -f #{path}"
       else
         ';'
@@ -88,7 +88,7 @@ module Bricolage
 
     def VacuumLock.cleanup_vacuum_lock
       return unless enable_vacuum_lock?
-      path, timeout = vacuum_lock_parameters
+      path, _timeout = vacuum_lock_parameters
       if locking?(path)
         $stderr.puts "remove VACUUM lock by #{Process.pid}"
         FileUtils.rm_f path

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '6.0.0beta4'
+  VERSION = '6.0.0beta5'
 end

data/schema/Dockerfile

@@ -1,13 +1,16 @@
-FROM ruby:2.6.5-stretch
+FROM ruby:3.0-slim
 
 RUN apt-get update -qq && apt-get install -y build-essential libpq-dev libpq5 postgresql-client
 
-COPY ./Gemfile /tmp/Gemfile
-COPY ./Gemfile.lock /tmp/Gemfile.lock
-RUN cd /tmp && bundle install -j4 --deployment --without 'development test'
-
 WORKDIR /app
-COPY . /app
-RUN cp -a /tmp/vendor /app/
+
+COPY ./Gemfile .
+COPY ./Gemfile.lock .
+RUN gem install bundler:2.2.15 && \
+    bundle config set --local deployment true && \
+    bundle config set --local without 'development test' && \
+    bundle install -j4
+
+COPY . .
 
 CMD ["bundle", "exec", "ridgepole", "-f", "Schemafile", "-c", "database.yml", "--merge", "--dry-run"]

data/schema/Gemfile

@@ -1,4 +1,5 @@
 source 'https://rubygems.org'
 
 gem 'ridgepole'
+gem 'activerecord', '~> 6.0.0'  # activerecord 6.1 did not work with check_constraint
 gem 'pg'

data/schema/Gemfile.lock

@@ -1,37 +1,38 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.4.3)
-      activesupport (= 5.2.4.3)
-    activerecord (5.2.4.3)
-      activemodel (= 5.2.4.3)
-      activesupport (= 5.2.4.3)
-      arel (>= 9.0)
-    activesupport (5.2.4.3)
+    activemodel (6.0.3.6)
+      activesupport (= 6.0.3.6)
+    activerecord (6.0.3.6)
+      activemodel (= 6.0.3.6)
+      activesupport (= 6.0.3.6)
+    activesupport (6.0.3.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    arel (9.0.0)
-    concurrent-ruby (1.1.6)
-    diffy (3.3.0)
-    i18n (1.8.2)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    concurrent-ruby (1.1.8)
+    diffy (3.4.0)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.1)
-    pg (1.1.4)
-    ridgepole (0.7.7)
-      activerecord (>= 5.0.1, < 6)
+    minitest (5.14.4)
+    pg (1.2.3)
+    ridgepole (0.9.2)
+      activerecord (>= 5.1, < 6.2)
       diffy
     thread_safe (0.3.6)
-    tzinfo (1.2.7)
+    tzinfo (1.2.9)
       thread_safe (~> 0.1)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  activerecord (~> 6.0.0)
   pg
   ridgepole
 
 BUNDLED WITH
-   1.17.2
+   2.2.15

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 6.0.0beta4
+  version: 6.0.0beta5
 platform: ruby
 authors:
 - Minero Aoki
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-02 00:00:00.000000000 Z
+date: 2021-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.23'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -117,6 +131,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-version"
 - Gemfile
 - LICENSES
 - README.md
@@ -125,6 +140,7 @@ files:
 - bin/bricolage
 - bin/bricolage-jobnet
 - bricolage.gemspec
+- config/test/datasource.yml
 - jobclass/adhoc.rb
 - jobclass/create.rb
 - jobclass/createview.rb
@@ -194,7 +210,7 @@ homepage: https://github.com/bricolages/bricolage
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -209,8 +225,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: SQL Batch Framework
 test_files: []