bricolage 5.29.2 → 6.0.0beta4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -3
  3. data/README.md +4 -0
  4. data/RELEASE.md +4 -0
  5. data/bricolage.gemspec +8 -7
  6. data/jobclass/create.rb +1 -1
  7. data/jobclass/createview.rb +1 -1
  8. data/jobclass/load.rb +6 -6
  9. data/jobclass/rebuild-drop.rb +4 -4
  10. data/jobclass/rebuild-rename.rb +8 -8
  11. data/jobclass/sql.rb +1 -1
  12. data/jobclass/streaming_load.rb +10 -9
  13. data/lib/bricolage/application.rb +5 -5
  14. data/lib/bricolage/context.rb +18 -11
  15. data/lib/bricolage/dao/job.rb +184 -0
  16. data/lib/bricolage/dao/jobexecution.rb +253 -0
  17. data/lib/bricolage/dao/jobnet.rb +158 -0
  18. data/lib/bricolage/datasource.rb +1 -1
  19. data/lib/bricolage/exception.rb +11 -0
  20. data/lib/bricolage/job.rb +12 -9
  21. data/lib/bricolage/jobnet.rb +9 -6
  22. data/lib/bricolage/jobnetrunner.rb +82 -45
  23. data/lib/bricolage/logger.rb +2 -2
  24. data/lib/bricolage/loglocator.rb +19 -1
  25. data/lib/bricolage/postgresconnection.rb +6 -4
  26. data/lib/bricolage/psqldatasource.rb +54 -5
  27. data/lib/bricolage/sqlutils.rb +43 -1
  28. data/lib/bricolage/taskqueue.rb +221 -63
  29. data/lib/bricolage/version.rb +1 -1
  30. data/schema/Dockerfile +13 -0
  31. data/schema/Gemfile +4 -0
  32. data/schema/Gemfile.lock +37 -0
  33. data/schema/Schemafile +57 -0
  34. data/schema/database.yml +8 -0
  35. data/schema/ridgepole_dryrun.sh +2 -0
  36. data/schema/ridgepole_merge.sh +2 -0
  37. metadata +48 -25
data/schema/Gemfile.lock ADDED
@@ -0,0 +1,37 @@
1
+ GEM
2
+ remote: https://rubygems.org/
3
+ specs:
4
+ activemodel (5.2.4.3)
5
+ activesupport (= 5.2.4.3)
6
+ activerecord (5.2.4.3)
7
+ activemodel (= 5.2.4.3)
8
+ activesupport (= 5.2.4.3)
9
+ arel (>= 9.0)
10
+ activesupport (5.2.4.3)
11
+ concurrent-ruby (~> 1.0, >= 1.0.2)
12
+ i18n (>= 0.7, < 2)
13
+ minitest (~> 5.1)
14
+ tzinfo (~> 1.1)
15
+ arel (9.0.0)
16
+ concurrent-ruby (1.1.6)
17
+ diffy (3.3.0)
18
+ i18n (1.8.2)
19
+ concurrent-ruby (~> 1.0)
20
+ minitest (5.14.1)
21
+ pg (1.1.4)
22
+ ridgepole (0.7.7)
23
+ activerecord (>= 5.0.1, < 6)
24
+ diffy
25
+ thread_safe (0.3.6)
26
+ tzinfo (1.2.7)
27
+ thread_safe (~> 0.1)
28
+
29
+ PLATFORMS
30
+ ruby
31
+
32
+ DEPENDENCIES
33
+ pg
34
+ ridgepole
35
+
36
+ BUNDLED WITH
37
+ 1.17.2
data/schema/Schemafile ADDED
@@ -0,0 +1,57 @@
1
+ create_table "jobs", primary_key: "job_id", force: :cascade do |t|
2
+ t.string "subsystem", null: false
3
+ t.string "job_name", null: false
4
+ t.integer "jobnet_id"
5
+ t.string "executor_id"
6
+ t.index ["subsystem", "job_name", "jobnet_id"], name: "job_unique", unique: true, using: :btree
7
+ end
8
+
9
+ create_table "jobnets", primary_key: "jobnet_id", force: :cascade do |t|
10
+ t.string "subsystem", null: false
11
+ t.string "jobnet_name", null: false
12
+ t.string "executor_id"
13
+ t.index ["subsystem", "jobnet_name"], name: "jobnet_unique", unique: true, using: :btree
14
+ end
15
+
16
+ create_table "job_executions", primary_key: "job_execution_id", force: :cascade do |t|
17
+ t.string "status", null: false
18
+ t.integer "execution_sequence", null: false
19
+ t.string "message"
20
+ t.datetime "submitted_at", default: -> { "now()" }, null: false
21
+ t.integer "job_id", null: false
22
+ t.datetime "started_at"
23
+ t.datetime "finished_at"
24
+ t.string "source"
25
+ end
26
+
27
+ create_table "job_execution_states", primary_key: "job_execution_state_id", force: :cascade do |t|
28
+ t.integer "job_execution_id", null: false
29
+ t.string "status", null: false
30
+ t.string "message"
31
+ t.datetime "created_at", default: -> { "now()" }, null: false
32
+ t.integer "job_id", null: false
33
+ end
34
+
35
+ add_foreign_key "job_executions",
36
+ "jobs",
37
+ column: "job_id",
38
+ primary_key: "job_id",
39
+ name: "job_execution_fk_job"
40
+
41
+ add_foreign_key "job_execution_states",
42
+ "jobs",
43
+ column: "job_id",
44
+ primary_key: "job_id",
45
+ name: "job_execution_state_fk_job"
46
+
47
+ add_foreign_key "job_execution_states",
48
+ "job_executions",
49
+ column: "job_execution_id",
50
+ primary_key: "job_execution_id",
51
+ name: "job_execution_state_fk_job_execution"
52
+
53
+ add_foreign_key "jobs",
54
+ "jobnets",
55
+ column: "jobnet_id",
56
+ primary_key: "jobnet_id",
57
+ name: "job_fk_jobnet"
data/schema/database.yml ADDED
@@ -0,0 +1,8 @@
1
+ adapter: postgresql
2
+ host: <%= ENV['PGHOST'] || "localhost" %>
3
+ port: 5432
4
+ database: <%= ENV['PGDATABASE'] || "bricolage" %>
5
+ username: <%= ENV['PGUSER'] || "bricolage" %>
6
+ password: <%= ENV['PGPASSWORD'] || "bricolage" %>
7
+ encoding: unicode
8
+ pool: 1
data/schema/ridgepole_dryrun.sh ADDED
@@ -0,0 +1,2 @@
1
+ #! /bin/sh
2
+ exec bundle exec ridgepole -f Schemafile -c database.yml --merge --dry-run
data/schema/ridgepole_merge.sh ADDED
@@ -0,0 +1,2 @@
1
+ #! /bin/sh
2
+ exec bundle exec ridgepole -f Schemafile -c database.yml --merge
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bricolage
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.29.2
4
+ version: 6.0.0beta4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Minero Aoki
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-08-14 00:00:00.000000000 Z
11
+ date: 2020-10-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pg
@@ -16,84 +16,98 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: 0.18.0
19
+ version: 1.2.3
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: 0.18.0
26
+ version: 1.2.3
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: aws-sdk-s3
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1'
33
+ version: '1.64'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '1'
40
+ version: '1.64'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: aws-sdk-sns
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '1'
47
+ version: '1.23'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '1'
54
+ version: '1.23'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: test-unit
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ">="
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '0'
61
+ version: '3.3'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ">="
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '0'
68
+ version: '3.3'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: rake
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ">="
73
+ - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '0'
75
+ version: '13.0'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ">="
80
+ - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '0'
82
+ version: '13.0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: mocha
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - ">="
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.11'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.11'
97
+ - !ruby/object:Gem::Dependency
98
+ name: pry-byebug
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
88
102
  - !ruby/object:Gem::Version
89
- version: '0'
103
+ version: '3.9'
90
104
  type: :development
91
105
  prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
- - - ">="
108
+ - - "~>"
95
109
  - !ruby/object:Gem::Version
96
- version: '0'
110
+ version: '3.9'
97
111
  description: Redshift-oriented Data Warehouse Batch Framework
98
112
  email: aamine@loveruby.net
99
113
  executables:
@@ -132,6 +146,9 @@ files:
132
146
  - lib/bricolage/commandutils.rb
133
147
  - lib/bricolage/configloader.rb
134
148
  - lib/bricolage/context.rb
149
+ - lib/bricolage/dao/job.rb
150
+ - lib/bricolage/dao/jobexecution.rb
151
+ - lib/bricolage/dao/jobnet.rb
135
152
  - lib/bricolage/datasource.rb
136
153
  - lib/bricolage/embeddedcodeapi.rb
137
154
  - lib/bricolage/eventhandlers.rb
@@ -165,6 +182,13 @@ files:
165
182
  - lib/bricolage/variables.rb
166
183
  - lib/bricolage/version.rb
167
184
  - libexec/create-lockfile
185
+ - schema/Dockerfile
186
+ - schema/Gemfile
187
+ - schema/Gemfile.lock
188
+ - schema/Schemafile
189
+ - schema/database.yml
190
+ - schema/ridgepole_dryrun.sh
191
+ - schema/ridgepole_merge.sh
168
192
  - streaming_load_log.ct
169
193
  homepage: https://github.com/bricolages/bricolage
170
194
  licenses:
@@ -178,15 +202,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
178
202
  requirements:
179
203
  - - ">="
180
204
  - !ruby/object:Gem::Version
181
- version: 2.0.0
205
+ version: 2.4.0
182
206
  required_rubygems_version: !ruby/object:Gem::Requirement
183
207
  requirements:
184
- - - ">="
208
+ - - ">"
185
209
  - !ruby/object:Gem::Version
186
- version: '0'
210
+ version: 1.3.1
187
211
  requirements: []
188
- rubyforge_project:
189
- rubygems_version: 2.7.6
212
+ rubygems_version: 3.0.3
190
213
  signing_key:
191
214
  specification_version: 4
192
215
  summary: SQL Batch Framework