bricolage 5.30.0 → 6.0.0beta5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -3
  3. data/.ruby-version +1 -0
  4. data/README.md +3 -0
  5. data/RELEASE.md +22 -0
  6. data/Rakefile +11 -1
  7. data/bricolage.gemspec +9 -7
  8. data/config/test/datasource.yml +9 -0
  9. data/jobclass/rebuild-rename.rb +7 -7
  10. data/jobclass/streaming_load.rb +3 -3
  11. data/lib/bricolage/application.rb +5 -5
  12. data/lib/bricolage/configloader.rb +1 -1
  13. data/lib/bricolage/context.rb +18 -11
  14. data/lib/bricolage/dao/job.rb +184 -0
  15. data/lib/bricolage/dao/jobexecution.rb +253 -0
  16. data/lib/bricolage/dao/jobnet.rb +158 -0
  17. data/lib/bricolage/datasource.rb +1 -1
  18. data/lib/bricolage/exception.rb +11 -0
  19. data/lib/bricolage/filedatasource.rb +1 -1
  20. data/lib/bricolage/genericdatasource.rb +1 -2
  21. data/lib/bricolage/job.rb +14 -9
  22. data/lib/bricolage/jobnet.rb +9 -6
  23. data/lib/bricolage/jobnetrunner.rb +82 -45
  24. data/lib/bricolage/logger.rb +3 -3
  25. data/lib/bricolage/loglocator.rb +19 -1
  26. data/lib/bricolage/postgresconnection.rb +6 -4
  27. data/lib/bricolage/psqldatasource.rb +74 -5
  28. data/lib/bricolage/rubyjobclass.rb +1 -2
  29. data/lib/bricolage/sqlutils.rb +43 -1
  30. data/lib/bricolage/taskqueue.rb +221 -63
  31. data/lib/bricolage/vacuumlock.rb +2 -2
  32. data/lib/bricolage/version.rb +1 -1
  33. data/schema/Dockerfile +16 -0
  34. data/schema/Gemfile +5 -0
  35. data/schema/Gemfile.lock +38 -0
  36. data/schema/Schemafile +57 -0
  37. data/schema/database.yml +8 -0
  38. data/schema/ridgepole_dryrun.sh +2 -0
  39. data/schema/ridgepole_merge.sh +2 -0
  40. metadata +65 -25
data/lib/bricolage/vacuumlock.rb CHANGED
@@ -32,7 +32,7 @@ module Bricolage
32
32
 
33
33
  def psql_serialize_vacuum_end
34
34
  if enable_vacuum_lock?
35
- path, timeout = vacuum_lock_parameters
35
+ path, _timeout = vacuum_lock_parameters
36
36
  "\\! rm -f #{path}"
37
37
  else
38
38
  ';'
@@ -88,7 +88,7 @@ module Bricolage
88
88
 
89
89
  def VacuumLock.cleanup_vacuum_lock
90
90
  return unless enable_vacuum_lock?
91
- path, timeout = vacuum_lock_parameters
91
+ path, _timeout = vacuum_lock_parameters
92
92
  if locking?(path)
93
93
  $stderr.puts "remove VACUUM lock by #{Process.pid}"
94
94
  FileUtils.rm_f path
data/lib/bricolage/version.rb CHANGED
@@ -1,4 +1,4 @@
1
1
  module Bricolage
2
2
  APPLICATION_NAME = 'Bricolage'
3
- VERSION = '5.30.0'
3
+ VERSION = '6.0.0beta5'
4
4
  end
data/schema/Dockerfile ADDED
@@ -0,0 +1,16 @@
1
+ FROM ruby:3.0-slim
2
+
3
+ RUN apt-get update -qq && apt-get install -y build-essential libpq-dev libpq5 postgresql-client
4
+
5
+ WORKDIR /app
6
+
7
+ COPY ./Gemfile .
8
+ COPY ./Gemfile.lock .
9
+ RUN gem install bundler:2.2.15 && \
10
+ bundle config set --local deployment true && \
11
+ bundle config set --local without 'development test' && \
12
+ bundle install -j4
13
+
14
+ COPY . .
15
+
16
+ CMD ["bundle", "exec", "ridgepole", "-f", "Schemafile", "-c", "database.yml", "--merge", "--dry-run"]
data/schema/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'ridgepole'
4
+ gem 'activerecord', '~> 6.0.0' # activerecord 6.1 did not work with check_constraint
5
+ gem 'pg'
data/schema/Gemfile.lock ADDED
@@ -0,0 +1,38 @@
1
+ GEM
2
+ remote: https://rubygems.org/
3
+ specs:
4
+ activemodel (6.0.3.6)
5
+ activesupport (= 6.0.3.6)
6
+ activerecord (6.0.3.6)
7
+ activemodel (= 6.0.3.6)
8
+ activesupport (= 6.0.3.6)
9
+ activesupport (6.0.3.6)
10
+ concurrent-ruby (~> 1.0, >= 1.0.2)
11
+ i18n (>= 0.7, < 2)
12
+ minitest (~> 5.1)
13
+ tzinfo (~> 1.1)
14
+ zeitwerk (~> 2.2, >= 2.2.2)
15
+ concurrent-ruby (1.1.8)
16
+ diffy (3.4.0)
17
+ i18n (1.8.10)
18
+ concurrent-ruby (~> 1.0)
19
+ minitest (5.14.4)
20
+ pg (1.2.3)
21
+ ridgepole (0.9.2)
22
+ activerecord (>= 5.1, < 6.2)
23
+ diffy
24
+ thread_safe (0.3.6)
25
+ tzinfo (1.2.9)
26
+ thread_safe (~> 0.1)
27
+ zeitwerk (2.4.2)
28
+
29
+ PLATFORMS
30
+ ruby
31
+
32
+ DEPENDENCIES
33
+ activerecord (~> 6.0.0)
34
+ pg
35
+ ridgepole
36
+
37
+ BUNDLED WITH
38
+ 2.2.15
data/schema/Schemafile ADDED
@@ -0,0 +1,57 @@
1
+ create_table "jobs", primary_key: "job_id", force: :cascade do |t|
2
+ t.string "subsystem", null: false
3
+ t.string "job_name", null: false
4
+ t.integer "jobnet_id"
5
+ t.string "executor_id"
6
+ t.index ["subsystem", "job_name", "jobnet_id"], name: "job_unique", unique: true, using: :btree
7
+ end
8
+
9
+ create_table "jobnets", primary_key: "jobnet_id", force: :cascade do |t|
10
+ t.string "subsystem", null: false
11
+ t.string "jobnet_name", null: false
12
+ t.string "executor_id"
13
+ t.index ["subsystem", "jobnet_name"], name: "jobnet_unique", unique: true, using: :btree
14
+ end
15
+
16
+ create_table "job_executions", primary_key: "job_execution_id", force: :cascade do |t|
17
+ t.string "status", null: false
18
+ t.integer "execution_sequence", null: false
19
+ t.string "message"
20
+ t.datetime "submitted_at", default: -> { "now()" }, null: false
21
+ t.integer "job_id", null: false
22
+ t.datetime "started_at"
23
+ t.datetime "finished_at"
24
+ t.string "source"
25
+ end
26
+
27
+ create_table "job_execution_states", primary_key: "job_execution_state_id", force: :cascade do |t|
28
+ t.integer "job_execution_id", null: false
29
+ t.string "status", null: false
30
+ t.string "message"
31
+ t.datetime "created_at", default: -> { "now()" }, null: false
32
+ t.integer "job_id", null: false
33
+ end
34
+
35
+ add_foreign_key "job_executions",
36
+ "jobs",
37
+ column: "job_id",
38
+ primary_key: "job_id",
39
+ name: "job_execution_fk_job"
40
+
41
+ add_foreign_key "job_execution_states",
42
+ "jobs",
43
+ column: "job_id",
44
+ primary_key: "job_id",
45
+ name: "job_execution_state_fk_job"
46
+
47
+ add_foreign_key "job_execution_states",
48
+ "job_executions",
49
+ column: "job_execution_id",
50
+ primary_key: "job_execution_id",
51
+ name: "job_execution_state_fk_job_execution"
52
+
53
+ add_foreign_key "jobs",
54
+ "jobnets",
55
+ column: "jobnet_id",
56
+ primary_key: "jobnet_id",
57
+ name: "job_fk_jobnet"
data/schema/database.yml ADDED
@@ -0,0 +1,8 @@
1
+ adapter: postgresql
2
+ host: <%= ENV['PGHOST'] || "localhost" %>
3
+ port: 5432
4
+ database: <%= ENV['PGDATABASE'] || "bricolage" %>
5
+ username: <%= ENV['PGUSER'] || "bricolage" %>
6
+ password: <%= ENV['PGPASSWORD'] || "bricolage" %>
7
+ encoding: unicode
8
+ pool: 1
data/schema/ridgepole_dryrun.sh ADDED
@@ -0,0 +1,2 @@
1
+ #! /bin/sh
2
+ exec bundle exec ridgepole -f Schemafile -c database.yml --merge --dry-run
data/schema/ridgepole_merge.sh ADDED
@@ -0,0 +1,2 @@
1
+ #! /bin/sh
2
+ exec bundle exec ridgepole -f Schemafile -c database.yml --merge
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bricolage
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.30.0
4
+ version: 6.0.0beta5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Minero Aoki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-05-29 00:00:00.000000000 Z
11
+ date: 2021-04-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pg
@@ -16,84 +16,112 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: 0.18.0
19
+ version: 1.2.3
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: 0.18.0
26
+ version: 1.2.3
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: aws-sdk-s3
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1'
33
+ version: '1.64'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '1'
40
+ version: '1.64'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: aws-sdk-sns
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '1'
47
+ version: '1.23'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '1'
54
+ version: '1.23'
55
55
  - !ruby/object:Gem::Dependency
56
- name: test-unit
56
+ name: nokogiri
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
- type: :development
62
+ type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: test-unit
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '3.3'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '3.3'
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: rake
71
85
  requirement: !ruby/object:Gem::Requirement
72
86
  requirements:
73
- - - ">="
87
+ - - "~>"
74
88
  - !ruby/object:Gem::Version
75
- version: '0'
89
+ version: '13.0'
76
90
  type: :development
77
91
  prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
- - - ">="
94
+ - - "~>"
81
95
  - !ruby/object:Gem::Version
82
- version: '0'
96
+ version: '13.0'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: mocha
85
99
  requirement: !ruby/object:Gem::Requirement
86
100
  requirements:
87
- - - ">="
101
+ - - "~>"
88
102
  - !ruby/object:Gem::Version
89
- version: '0'
103
+ version: '1.11'
90
104
  type: :development
91
105
  prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
- - - ">="
108
+ - - "~>"
95
109
  - !ruby/object:Gem::Version
96
- version: '0'
110
+ version: '1.11'
111
+ - !ruby/object:Gem::Dependency
112
+ name: pry-byebug
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '3.9'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '3.9'
97
125
  description: Redshift-oriented Data Warehouse Batch Framework
98
126
  email: aamine@loveruby.net
99
127
  executables:
@@ -103,6 +131,7 @@ extensions: []
103
131
  extra_rdoc_files: []
104
132
  files:
105
133
  - ".gitignore"
134
+ - ".ruby-version"
106
135
  - Gemfile
107
136
  - LICENSES
108
137
  - README.md
@@ -111,6 +140,7 @@ files:
111
140
  - bin/bricolage
112
141
  - bin/bricolage-jobnet
113
142
  - bricolage.gemspec
143
+ - config/test/datasource.yml
114
144
  - jobclass/adhoc.rb
115
145
  - jobclass/create.rb
116
146
  - jobclass/createview.rb
@@ -132,6 +162,9 @@ files:
132
162
  - lib/bricolage/commandutils.rb
133
163
  - lib/bricolage/configloader.rb
134
164
  - lib/bricolage/context.rb
165
+ - lib/bricolage/dao/job.rb
166
+ - lib/bricolage/dao/jobexecution.rb
167
+ - lib/bricolage/dao/jobnet.rb
135
168
  - lib/bricolage/datasource.rb
136
169
  - lib/bricolage/embeddedcodeapi.rb
137
170
  - lib/bricolage/eventhandlers.rb
@@ -165,12 +198,19 @@ files:
165
198
  - lib/bricolage/variables.rb
166
199
  - lib/bricolage/version.rb
167
200
  - libexec/create-lockfile
201
+ - schema/Dockerfile
202
+ - schema/Gemfile
203
+ - schema/Gemfile.lock
204
+ - schema/Schemafile
205
+ - schema/database.yml
206
+ - schema/ridgepole_dryrun.sh
207
+ - schema/ridgepole_merge.sh
168
208
  - streaming_load_log.ct
169
209
  homepage: https://github.com/bricolages/bricolage
170
210
  licenses:
171
211
  - MIT
172
212
  metadata: {}
173
- post_install_message:
213
+ post_install_message:
174
214
  rdoc_options: []
175
215
  require_paths:
176
216
  - lib
@@ -178,15 +218,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
178
218
  requirements:
179
219
  - - ">="
180
220
  - !ruby/object:Gem::Version
181
- version: 2.0.0
221
+ version: 2.4.0
182
222
  required_rubygems_version: !ruby/object:Gem::Requirement
183
223
  requirements:
184
- - - ">="
224
+ - - ">"
185
225
  - !ruby/object:Gem::Version
186
- version: '0'
226
+ version: 1.3.1
187
227
  requirements: []
188
- rubygems_version: 3.0.3
189
- signing_key:
228
+ rubygems_version: 3.2.3
229
+ signing_key:
190
230
  specification_version: 4
191
231
  summary: SQL Batch Framework
192
232
  test_files: []