bricolage 5.29.0 → 6.0.0beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -3
- data/README.md +4 -0
- data/RELEASE.md +22 -0
- data/bricolage.gemspec +1 -0
- data/jobclass/create.rb +1 -1
- data/jobclass/createview.rb +1 -1
- data/jobclass/load.rb +6 -6
- data/jobclass/rebuild-drop.rb +4 -4
- data/jobclass/rebuild-rename.rb +4 -4
- data/jobclass/sql.rb +1 -1
- data/jobclass/streaming_load.rb +10 -9
- data/lib/bricolage/application.rb +5 -5
- data/lib/bricolage/context.rb +18 -11
- data/lib/bricolage/dao/job.rb +184 -0
- data/lib/bricolage/dao/jobexecution.rb +253 -0
- data/lib/bricolage/dao/jobnet.rb +158 -0
- data/lib/bricolage/datasource.rb +1 -1
- data/lib/bricolage/exception.rb +11 -0
- data/lib/bricolage/job.rb +12 -9
- data/lib/bricolage/jobnet.rb +9 -6
- data/lib/bricolage/jobnetrunner.rb +82 -45
- data/lib/bricolage/logger.rb +2 -2
- data/lib/bricolage/loglocator.rb +19 -1
- data/lib/bricolage/postgresconnection.rb +6 -4
- data/lib/bricolage/psqldatasource.rb +75 -7
- data/lib/bricolage/sqlutils.rb +43 -1
- data/lib/bricolage/taskqueue.rb +221 -63
- data/lib/bricolage/version.rb +1 -1
- data/schema/Dockerfile +13 -0
- data/schema/Gemfile +4 -0
- data/schema/Gemfile.lock +37 -0
- data/schema/Schemafile +57 -0
- data/schema/database.yml +8 -0
- data/schema/ridgepole_dryrun.sh +2 -0
- data/schema/ridgepole_merge.sh +2 -0
- metadata +29 -6
data/schema/Gemfile.lock
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
GEM
|
2
|
+
remote: https://rubygems.org/
|
3
|
+
specs:
|
4
|
+
activemodel (5.2.4.3)
|
5
|
+
activesupport (= 5.2.4.3)
|
6
|
+
activerecord (5.2.4.3)
|
7
|
+
activemodel (= 5.2.4.3)
|
8
|
+
activesupport (= 5.2.4.3)
|
9
|
+
arel (>= 9.0)
|
10
|
+
activesupport (5.2.4.3)
|
11
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
12
|
+
i18n (>= 0.7, < 2)
|
13
|
+
minitest (~> 5.1)
|
14
|
+
tzinfo (~> 1.1)
|
15
|
+
arel (9.0.0)
|
16
|
+
concurrent-ruby (1.1.6)
|
17
|
+
diffy (3.3.0)
|
18
|
+
i18n (1.8.2)
|
19
|
+
concurrent-ruby (~> 1.0)
|
20
|
+
minitest (5.14.1)
|
21
|
+
pg (1.1.4)
|
22
|
+
ridgepole (0.7.7)
|
23
|
+
activerecord (>= 5.0.1, < 6)
|
24
|
+
diffy
|
25
|
+
thread_safe (0.3.6)
|
26
|
+
tzinfo (1.2.7)
|
27
|
+
thread_safe (~> 0.1)
|
28
|
+
|
29
|
+
PLATFORMS
|
30
|
+
ruby
|
31
|
+
|
32
|
+
DEPENDENCIES
|
33
|
+
pg
|
34
|
+
ridgepole
|
35
|
+
|
36
|
+
BUNDLED WITH
|
37
|
+
1.17.2
|
data/schema/Schemafile
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
create_table "jobs", primary_key: "job_id", force: :cascade do |t|
|
2
|
+
t.string "subsystem", null: false
|
3
|
+
t.string "job_name", null: false
|
4
|
+
t.integer "jobnet_id"
|
5
|
+
t.string "executor_id"
|
6
|
+
t.index ["subsystem", "job_name", "jobnet_id"], name: "job_unique", unique: true, using: :btree
|
7
|
+
end
|
8
|
+
|
9
|
+
create_table "jobnets", primary_key: "jobnet_id", force: :cascade do |t|
|
10
|
+
t.string "subsystem", null: false
|
11
|
+
t.string "jobnet_name", null: false
|
12
|
+
t.string "executor_id"
|
13
|
+
t.index ["subsystem", "jobnet_name"], name: "jobnet_unique", unique: true, using: :btree
|
14
|
+
end
|
15
|
+
|
16
|
+
create_table "job_executions", primary_key: "job_execution_id", force: :cascade do |t|
|
17
|
+
t.string "status", null: false
|
18
|
+
t.integer "execution_sequence", null: false
|
19
|
+
t.string "message"
|
20
|
+
t.datetime "submitted_at", default: -> { "now()" }, null: false
|
21
|
+
t.integer "job_id", null: false
|
22
|
+
t.datetime "started_at"
|
23
|
+
t.datetime "finished_at"
|
24
|
+
t.string "source"
|
25
|
+
end
|
26
|
+
|
27
|
+
create_table "job_execution_states", primary_key: "job_execution_state_id", force: :cascade do |t|
|
28
|
+
t.integer "job_execution_id", null: false
|
29
|
+
t.string "status", null: false
|
30
|
+
t.string "message"
|
31
|
+
t.datetime "created_at", default: -> { "now()" }, null: false
|
32
|
+
t.integer "job_id", null: false
|
33
|
+
end
|
34
|
+
|
35
|
+
add_foreign_key "job_executions",
|
36
|
+
"jobs",
|
37
|
+
column: "job_id",
|
38
|
+
primary_key: "job_id",
|
39
|
+
name: "job_execution_fk_job"
|
40
|
+
|
41
|
+
add_foreign_key "job_execution_states",
|
42
|
+
"jobs",
|
43
|
+
column: "job_id",
|
44
|
+
primary_key: "job_id",
|
45
|
+
name: "job_execution_state_fk_job"
|
46
|
+
|
47
|
+
add_foreign_key "job_execution_states",
|
48
|
+
"job_executions",
|
49
|
+
column: "job_execution_id",
|
50
|
+
primary_key: "job_execution_id",
|
51
|
+
name: "job_execution_state_fk_job_execution"
|
52
|
+
|
53
|
+
add_foreign_key "jobs",
|
54
|
+
"jobnets",
|
55
|
+
column: "jobnet_id",
|
56
|
+
primary_key: "jobnet_id",
|
57
|
+
name: "job_fk_jobnet"
|
data/schema/database.yml
ADDED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bricolage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 6.0.0beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Minero Aoki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-08-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: pry-byebug
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
description: Redshift-oriented Data Warehouse Batch Framework
|
98
112
|
email: aamine@loveruby.net
|
99
113
|
executables:
|
@@ -132,6 +146,9 @@ files:
|
|
132
146
|
- lib/bricolage/commandutils.rb
|
133
147
|
- lib/bricolage/configloader.rb
|
134
148
|
- lib/bricolage/context.rb
|
149
|
+
- lib/bricolage/dao/job.rb
|
150
|
+
- lib/bricolage/dao/jobexecution.rb
|
151
|
+
- lib/bricolage/dao/jobnet.rb
|
135
152
|
- lib/bricolage/datasource.rb
|
136
153
|
- lib/bricolage/embeddedcodeapi.rb
|
137
154
|
- lib/bricolage/eventhandlers.rb
|
@@ -165,6 +182,13 @@ files:
|
|
165
182
|
- lib/bricolage/variables.rb
|
166
183
|
- lib/bricolage/version.rb
|
167
184
|
- libexec/create-lockfile
|
185
|
+
- schema/Dockerfile
|
186
|
+
- schema/Gemfile
|
187
|
+
- schema/Gemfile.lock
|
188
|
+
- schema/Schemafile
|
189
|
+
- schema/database.yml
|
190
|
+
- schema/ridgepole_dryrun.sh
|
191
|
+
- schema/ridgepole_merge.sh
|
168
192
|
- streaming_load_log.ct
|
169
193
|
homepage: https://github.com/bricolages/bricolage
|
170
194
|
licenses:
|
@@ -181,12 +205,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
181
205
|
version: 2.0.0
|
182
206
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
183
207
|
requirements:
|
184
|
-
- - "
|
208
|
+
- - ">"
|
185
209
|
- !ruby/object:Gem::Version
|
186
|
-
version:
|
210
|
+
version: 1.3.1
|
187
211
|
requirements: []
|
188
|
-
|
189
|
-
rubygems_version: 2.7.6
|
212
|
+
rubygems_version: 3.1.2
|
190
213
|
signing_key:
|
191
214
|
specification_version: 4
|
192
215
|
summary: SQL Batch Framework
|