bricolage 6.0.0beta4 → 6.0.0beta5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: be51accbd747d2192ccda3a4a502df6083adb2379f459c39ee711208691a67b5
4
- data.tar.gz: 86ba8e19da9a68befe60107f80e7376bd08959d5671920bf18065e80ce2fcaeb
3
+ metadata.gz: 97c5b2453966ea1464172b34c23e7ba3e6f44c715d745f17f07ebbddf361c437
4
+ data.tar.gz: 6044bb40e6dabedd831dc90dd4f61e8fc6614bd4b0885233ba2362b1e1c051b8
5
5
  SHA512:
6
- metadata.gz: 6cd44c1d136b65534842dd85f58e4b08bbbf5340e3265a9a67e20e5a7f443ad7ba8eabf3db5a9ca628dbc5f083b38952d5d8379928745534f5d968049de2951d
7
- data.tar.gz: '0788438ed231dbaa2e1d6312a62a717178ad0994b08cfe376d9949412c3cf74e147f03b7a1ccb91145ef2f6d26277159ce386b7b15a2110416c533de4dfa4c99'
6
+ metadata.gz: 421dd79e8da48871f3a6694f4ca8bc490280acd4dfad11f10c3165854e7c6a33b76b374a5939805304d2b53307c40c048c28c266f57b54514bf9d81226c21304
7
+ data.tar.gz: 95c5f97fdf7851d6def0e87713acf3eaae136ba8d2e5971b85b10cbd7b04b95f41e28925eac4912f63d2359e818e94d09c3fa905c8947e9b38747c2c71417f08
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ 3.0
data/README.md CHANGED
@@ -11,6 +11,9 @@ See LICENSES file for details.
11
11
 
12
12
  ## Running Test
13
13
 
14
+ Create bricolage_test database and user on your local machine.
15
+ See config/test/datasource.yml for details. Then type:
16
+
14
17
  % rake test
15
18
 
16
19
  ## How to use
data/RELEASE.md CHANGED
@@ -1,5 +1,27 @@
1
1
  # Bricolage Release Note
2
2
 
3
+ ## version 6.0.0 beta 5
4
+
5
+ - [new] Supports Ruby 3.0.
6
+
7
+ ## version 6.0.0 beta 4
8
+
9
+ - [fix] rebuild-rename did not work with ALTER RENAME syntax error.
10
+
11
+ ## version 6.0.0 beta 3
12
+
13
+ - [new] Upgrades pg from 0.18 to 1.2.
14
+
15
+ ## version 6.0.0 beta 2
16
+
17
+ - [fix] Always transmit error messages from jobs in the jobnet.
18
+
19
+ ## version 6.0.0 beta 1
20
+
21
+ - [new] Introduces database queue. Database queue saves job states in the PostgreSQL instance, we now can run bricolage on container environment.
22
+ - [new] Default log level is DEBUG on development environment, INFO on production environment.
23
+ - [new] Only updating query is logged as INFO level. Read-only queries are logged in DEBUG level.
24
+
3
25
  ## version 5.30.0
4
26
 
5
27
  - [new] streaming_load: new option --ctl-ds to change S3 data source for metadata files.
data/Rakefile CHANGED
@@ -1,3 +1,13 @@
1
+ require 'rake/testtask'
2
+
1
3
  task :test do
2
- load "#{__dir__}/test/all.rb"
4
+ desc 'Run tests'
5
+ Rake::TestTask.new do |t|
6
+ # To run test cases of specific file(s), Use:
7
+ # % rake test TEST=test/test_specified_path.rb
8
+ t.libs << "test"
9
+ t.test_files = Dir["test/**/test_*.rb"]
10
+ t.verbose = true
11
+ t.warning = true
12
+ end
3
13
  end
data/bricolage.gemspec CHANGED
@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
20
20
  s.add_dependency 'pg', '~> 1.2.3'
21
21
  s.add_dependency 'aws-sdk-s3', '~> 1.64'
22
22
  s.add_dependency 'aws-sdk-sns', '~> 1.23'
23
+ s.add_dependency 'nokogiri' # aws-sdk-core requires this
23
24
  s.add_development_dependency 'test-unit', '~> 3.3'
24
25
  s.add_development_dependency 'rake', '~> 13.0'
25
26
  s.add_development_dependency 'mocha', '~> 1.11'
@@ -0,0 +1,9 @@
1
+ test_db:
2
+ type: psql
3
+ host: localhost
4
+ port: 5432
5
+ database: bricolage_test
6
+ username: bricolage_test
7
+ password: bricolage_test
8
+ encoding: utf8
9
+ sql_log_level: DEBUG
@@ -144,7 +144,7 @@ class StreamingLoadJobClass < RubyJobClass
144
144
  @ds.open {|conn|
145
145
  execute_update conn, "delete #{log_table_wk};"
146
146
  execute_update conn, load_log_copy_stmt(log_table_wk, log_url, @src.credential_string)
147
- loaded, not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
147
+ loaded, _not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
148
148
  loaded.each do |obj|
149
149
  obj.dequeue(force: true, noop: @noop)
150
150
  end
@@ -32,7 +32,7 @@ module Bricolage
32
32
  end
33
33
 
34
34
  def eruby(text, path)
35
- erb = ERB.new(text, nil, '%-')
35
+ erb = ERB.new(text, trim_mode: '%-')
36
36
  erb.filename = path.to_s
37
37
  push_base_dir(path) {
38
38
  erb.result(binding())
@@ -141,7 +141,7 @@ module Bricolage
141
141
  end
142
142
 
143
143
  def clear_lock(jobnet_id)
144
- records = connect {|conn|
144
+ connect {|conn|
145
145
  conn.execute_update(<<~EndSQL)
146
146
  update jobnets
147
147
  set
@@ -6,7 +6,7 @@ module Bricolage
6
6
  class FileDataSource < DataSource
7
7
  declare_type 'file'
8
8
 
9
- def initialize(opts)
9
+ def initialize
10
10
  end
11
11
 
12
12
  def new_task
@@ -4,8 +4,7 @@ module Bricolage
4
4
  class GenericDataSource < DataSource
5
5
  declare_type 'generic'
6
6
 
7
- # FIXME: keyword argument placeholder is required
8
- def initialize(**)
7
+ def initialize
9
8
  end
10
9
 
11
10
  def new_task
data/lib/bricolage/job.rb CHANGED
@@ -51,6 +51,8 @@ module Bricolage
51
51
  end
52
52
 
53
53
  attr_reader :id
54
+ attr_reader :job_class
55
+ attr_reader :context
54
56
 
55
57
  def class_id
56
58
  @job_class.id
@@ -57,7 +57,7 @@ module Bricolage
57
57
  case
58
58
  when seconds > 60 * 60
59
59
  h, secs = seconds.divmod(60 * 60)
60
- m, s = secs.divmod(60)
60
+ m, _sec = secs.divmod(60)
61
61
  "%d hours %d minutes" % [h, m]
62
62
  when seconds > 60
63
63
  "%d minutes %d seconds" % seconds.divmod(60)
@@ -117,21 +117,28 @@ module Bricolage
117
117
  raise ParameterError, "could not read password: #{path}, #{user}"
118
118
  end
119
119
 
120
- def open(&block)
120
+ def open
121
121
  retries = (ENV['BRICOLAGE_OPEN_RETRY_LIMIT'] || DEFAULT_RETRY_LIMIT).to_i
122
122
  begin
123
123
  conn = PostgresConnection.open_data_source(self)
124
- conn.execute_query('select 1'){}
124
+ conn.execute_query('select 1') {}
125
125
  rescue PG::ConnectionBad, PG::UnableToSend => ex
126
+ conn.close rescue nil
126
127
  retries -= 1
127
128
  if retries >= 0
128
- logger.warn "Retry PG connection for execute query: #{ex.message}"
129
+ logger.warn "Could not open postgres connection; retry: #{ex.message}"
129
130
  sleep 1
130
131
  retry
132
+ else
133
+ raise
131
134
  end
132
135
  end
133
136
  if block_given?
134
- yield conn
137
+ begin
138
+ yield conn
139
+ ensure
140
+ conn.close
141
+ end
135
142
  else
136
143
  return conn
137
144
  end
@@ -140,23 +147,35 @@ module Bricolage
140
147
  def open_shared_connection
141
148
  raise ParameterError, 'open_shared_connection require block' unless block_given?
142
149
  conn = nil
143
- if @connection_pool.empty?
144
- conn = open
145
- else
146
- begin
147
- conn = @connection_pool.shift
148
- conn.execute_query('select 1'){}
149
- rescue
150
- conn.close
150
+ until conn
151
+ if conn_tmp = @connection_pool.shift
152
+ begin
153
+ conn_tmp.query('select 1') {}
154
+ rescue PG::ConnectionBad, PG::UnableToSend
155
+ # retry
156
+ else
157
+ # no exception occured
158
+ conn = conn_tmp
159
+ conn_tmp = nil
160
+ ensure
161
+ if conn_tmp
162
+ conn_tmp.close
163
+ conn_tmp = nil
164
+ end
165
+ end
166
+ else
167
+ # Get a fresh connection instead of pooled connections.
151
168
  conn = open
152
169
  end
153
170
  end
154
-
155
- yield conn
156
- ensure
157
- @connection_pool.push(conn)
171
+ begin
172
+ yield conn
173
+ ensure
174
+ @connection_pool.push(conn)
175
+ end
158
176
  end
159
177
 
178
+ # not MT-safe
160
179
  def clear_connection_pool
161
180
  @connection_pool.map(&:close)
162
181
  @connection_pool = []
@@ -50,8 +50,7 @@ module Bricolage
50
50
  class RubyDataSource < DataSource
51
51
  declare_type 'ruby'
52
52
 
53
- # FIXME: keyword argument placeholder is required
54
- def initialize(**)
53
+ def initialize
55
54
  end
56
55
 
57
56
  def new_task
@@ -32,7 +32,7 @@ module Bricolage
32
32
 
33
33
  def psql_serialize_vacuum_end
34
34
  if enable_vacuum_lock?
35
- path, timeout = vacuum_lock_parameters
35
+ path, _timeout = vacuum_lock_parameters
36
36
  "\\! rm -f #{path}"
37
37
  else
38
38
  ';'
@@ -88,7 +88,7 @@ module Bricolage
88
88
 
89
89
  def VacuumLock.cleanup_vacuum_lock
90
90
  return unless enable_vacuum_lock?
91
- path, timeout = vacuum_lock_parameters
91
+ path, _timeout = vacuum_lock_parameters
92
92
  if locking?(path)
93
93
  $stderr.puts "remove VACUUM lock by #{Process.pid}"
94
94
  FileUtils.rm_f path
@@ -1,4 +1,4 @@
1
1
  module Bricolage
2
2
  APPLICATION_NAME = 'Bricolage'
3
- VERSION = '6.0.0beta4'
3
+ VERSION = '6.0.0beta5'
4
4
  end
data/schema/Dockerfile CHANGED
@@ -1,13 +1,16 @@
1
- FROM ruby:2.6.5-stretch
1
+ FROM ruby:3.0-slim
2
2
 
3
3
  RUN apt-get update -qq && apt-get install -y build-essential libpq-dev libpq5 postgresql-client
4
4
 
5
- COPY ./Gemfile /tmp/Gemfile
6
- COPY ./Gemfile.lock /tmp/Gemfile.lock
7
- RUN cd /tmp && bundle install -j4 --deployment --without 'development test'
8
-
9
5
  WORKDIR /app
10
- COPY . /app
11
- RUN cp -a /tmp/vendor /app/
6
+
7
+ COPY ./Gemfile .
8
+ COPY ./Gemfile.lock .
9
+ RUN gem install bundler:2.2.15 && \
10
+ bundle config set --local deployment true && \
11
+ bundle config set --local without 'development test' && \
12
+ bundle install -j4
13
+
14
+ COPY . .
12
15
 
13
16
  CMD ["bundle", "exec", "ridgepole", "-f", "Schemafile", "-c", "database.yml", "--merge", "--dry-run"]
data/schema/Gemfile CHANGED
@@ -1,4 +1,5 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
3
  gem 'ridgepole'
4
+ gem 'activerecord', '~> 6.0.0' # activerecord 6.1 did not work with check_constraint
4
5
  gem 'pg'
data/schema/Gemfile.lock CHANGED
@@ -1,37 +1,38 @@
1
1
  GEM
2
2
  remote: https://rubygems.org/
3
3
  specs:
4
- activemodel (5.2.4.3)
5
- activesupport (= 5.2.4.3)
6
- activerecord (5.2.4.3)
7
- activemodel (= 5.2.4.3)
8
- activesupport (= 5.2.4.3)
9
- arel (>= 9.0)
10
- activesupport (5.2.4.3)
4
+ activemodel (6.0.3.6)
5
+ activesupport (= 6.0.3.6)
6
+ activerecord (6.0.3.6)
7
+ activemodel (= 6.0.3.6)
8
+ activesupport (= 6.0.3.6)
9
+ activesupport (6.0.3.6)
11
10
  concurrent-ruby (~> 1.0, >= 1.0.2)
12
11
  i18n (>= 0.7, < 2)
13
12
  minitest (~> 5.1)
14
13
  tzinfo (~> 1.1)
15
- arel (9.0.0)
16
- concurrent-ruby (1.1.6)
17
- diffy (3.3.0)
18
- i18n (1.8.2)
14
+ zeitwerk (~> 2.2, >= 2.2.2)
15
+ concurrent-ruby (1.1.8)
16
+ diffy (3.4.0)
17
+ i18n (1.8.10)
19
18
  concurrent-ruby (~> 1.0)
20
- minitest (5.14.1)
21
- pg (1.1.4)
22
- ridgepole (0.7.7)
23
- activerecord (>= 5.0.1, < 6)
19
+ minitest (5.14.4)
20
+ pg (1.2.3)
21
+ ridgepole (0.9.2)
22
+ activerecord (>= 5.1, < 6.2)
24
23
  diffy
25
24
  thread_safe (0.3.6)
26
- tzinfo (1.2.7)
25
+ tzinfo (1.2.9)
27
26
  thread_safe (~> 0.1)
27
+ zeitwerk (2.4.2)
28
28
 
29
29
  PLATFORMS
30
30
  ruby
31
31
 
32
32
  DEPENDENCIES
33
+ activerecord (~> 6.0.0)
33
34
  pg
34
35
  ridgepole
35
36
 
36
37
  BUNDLED WITH
37
- 1.17.2
38
+ 2.2.15
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bricolage
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.0.0beta4
4
+ version: 6.0.0beta5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Minero Aoki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-10-02 00:00:00.000000000 Z
11
+ date: 2021-04-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pg
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '1.23'
55
+ - !ruby/object:Gem::Dependency
56
+ name: nokogiri
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: test-unit
57
71
  requirement: !ruby/object:Gem::Requirement
@@ -117,6 +131,7 @@ extensions: []
117
131
  extra_rdoc_files: []
118
132
  files:
119
133
  - ".gitignore"
134
+ - ".ruby-version"
120
135
  - Gemfile
121
136
  - LICENSES
122
137
  - README.md
@@ -125,6 +140,7 @@ files:
125
140
  - bin/bricolage
126
141
  - bin/bricolage-jobnet
127
142
  - bricolage.gemspec
143
+ - config/test/datasource.yml
128
144
  - jobclass/adhoc.rb
129
145
  - jobclass/create.rb
130
146
  - jobclass/createview.rb
@@ -194,7 +210,7 @@ homepage: https://github.com/bricolages/bricolage
194
210
  licenses:
195
211
  - MIT
196
212
  metadata: {}
197
- post_install_message:
213
+ post_install_message:
198
214
  rdoc_options: []
199
215
  require_paths:
200
216
  - lib
@@ -209,8 +225,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
209
225
  - !ruby/object:Gem::Version
210
226
  version: 1.3.1
211
227
  requirements: []
212
- rubygems_version: 3.0.3
213
- signing_key:
228
+ rubygems_version: 3.2.3
229
+ signing_key:
214
230
  specification_version: 4
215
231
  summary: SQL Batch Framework
216
232
  test_files: []