db_fuel 2.0.0 → 2.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +73 -0
- data/CHANGELOG.md +18 -0
- data/README.md +12 -8
- data/db_fuel.gemspec +3 -10
- data/lib/db_fuel/library/active_record/base.rb +22 -6
- data/lib/db_fuel/library/active_record/find_or_insert.rb +4 -1
- data/lib/db_fuel/library/active_record/insert.rb +6 -2
- data/lib/db_fuel/library/active_record/update.rb +7 -9
- data/lib/db_fuel/library/active_record/update_all.rb +7 -8
- data/lib/db_fuel/library/active_record/upsert.rb +30 -30
- data/lib/db_fuel/modeling/record_transformer.rb +99 -0
- data/lib/db_fuel/modeling.rb +1 -1
- data/lib/db_fuel/version.rb +1 -1
- metadata +24 -15
- data/.travis.yml +0 -28
- data/lib/db_fuel/modeling/attribute_renderer_set.rb +0 -70
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6153de5e6e01820bf9c5b0bb67260ab4aa71198a82b775e1f34c771d4a520140
|
4
|
+
data.tar.gz: a95690fcf11543f4fdeb247190e993efe53c05f946ec866eb8e482378fd75e68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e446d0a87845c79afbcfb1d20859b6fc363e39cf6069c10e833f8524a161f66f4ef254282294e0188193f4e41054877b18b781077287f2c19bb10d1a03cba545
|
7
|
+
data.tar.gz: 172f25cbc3503861f0f82ffa20885697b7a1b46fd04f44d80a8a565f6781e7d72f1e714a57aab98ca2245e2fcbf44667bfb692a88e69f2021aedc8eba92e6a73
|
@@ -0,0 +1,73 @@
|
|
1
|
+
version: 2.1
|
2
|
+
|
3
|
+
orbs:
|
4
|
+
status_to_ms_teams: bluemarblepayroll/status_to_ms_teams_pure_bash@1.0.0
|
5
|
+
|
6
|
+
jobs:
|
7
|
+
build:
|
8
|
+
parameters:
|
9
|
+
use-bundler-cache:
|
10
|
+
type: boolean
|
11
|
+
default: true
|
12
|
+
|
13
|
+
docker:
|
14
|
+
- image: circleci/ruby:2.6.6-buster
|
15
|
+
environment:
|
16
|
+
FORBID_FOCUSED_SPECS: 1
|
17
|
+
AR_VERSION: 5
|
18
|
+
working_directory: ~/db_fuel
|
19
|
+
steps:
|
20
|
+
- checkout
|
21
|
+
- run: cp spec/config/database.yaml.ci spec/config/database.yaml
|
22
|
+
|
23
|
+
# TODO: wrap bundler caching logic into an Orb:
|
24
|
+
- when:
|
25
|
+
condition: << parameters.use-bundler-cache >>
|
26
|
+
steps:
|
27
|
+
- restore_cache:
|
28
|
+
key: v1.0.0-build-ruby-dependency-cache-{{ checksum "db_fuel.gemspec" }}-{{ checksum "Gemfile" }}-{{ checksum ".ruby-version" }}
|
29
|
+
|
30
|
+
- run: bundle install --path vendor/bundle
|
31
|
+
|
32
|
+
- when:
|
33
|
+
condition: << parameters.use-bundler-cache >>
|
34
|
+
steps:
|
35
|
+
- save_cache:
|
36
|
+
key: v1.0.0-build-ruby-dependency-cache-{{ checksum "db_fuel.gemspec" }}-{{ checksum "Gemfile" }}-{{ checksum ".ruby-version" }}
|
37
|
+
paths:
|
38
|
+
- vendor/bundle
|
39
|
+
|
40
|
+
- store_artifacts:
|
41
|
+
path: Gemfile.lock
|
42
|
+
|
43
|
+
- run: bundle exec rubocop
|
44
|
+
|
45
|
+
- run: COVERAGE=true bundle exec rspec -r rspec_junit_formatter --format progress --format RspecJunitFormatter -o test-results/rspec/results.xml
|
46
|
+
|
47
|
+
- store_test_results:
|
48
|
+
path: test-results
|
49
|
+
|
50
|
+
- store_artifacts:
|
51
|
+
path: coverage
|
52
|
+
|
53
|
+
- status_to_ms_teams/report:
|
54
|
+
webhook_url: $MS_TEAMS_WEBHOOK_URL
|
55
|
+
|
56
|
+
workflows:
|
57
|
+
version: 2.1
|
58
|
+
build:
|
59
|
+
jobs:
|
60
|
+
- build:
|
61
|
+
context: org-global
|
62
|
+
monthly-gem-dependency-refresh-check:
|
63
|
+
triggers:
|
64
|
+
- schedule:
|
65
|
+
cron: '0 0 1 * *'
|
66
|
+
filters:
|
67
|
+
branches:
|
68
|
+
only:
|
69
|
+
- master
|
70
|
+
jobs:
|
71
|
+
- build:
|
72
|
+
context: org-global
|
73
|
+
use-bundler-cache: false
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,21 @@
|
|
1
|
+
|
2
|
+
# 2.2.0 (May 23rd, 2021)
|
3
|
+
|
4
|
+
Updates:
|
5
|
+
|
6
|
+
* Updating dependencies for dbee to include newest versions.
|
7
|
+
# 2.1.0 (May 19th, 2021)
|
8
|
+
|
9
|
+
Additions:
|
10
|
+
|
11
|
+
* Added keys_register to all db_fuel/active_record jobs.
|
12
|
+
|
13
|
+
# 2.0.1 (March 18th, 2021)
|
14
|
+
|
15
|
+
Changes:
|
16
|
+
|
17
|
+
* Updated attribute_renderer_set to avoid an evaluation time issue with acts_as_hashable.
|
18
|
+
|
1
19
|
# 2.0.0 (March 16th, 2021)
|
2
20
|
|
3
21
|
New Jobs:
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# DB Fuel
|
2
2
|
|
3
|
-
[![
|
3
|
+
[![CircleCI](https://circleci.com/bb/bluemarble-ondemand/db_fuel/tree/master.svg?style=svg&circle-token=f1b2aed3fa173235db39da671c5b5db22e061609)](https://circleci.com/bb/bluemarble-ondemand/db_fuel/tree/master)
|
4
4
|
|
5
5
|
This library is a plugin for [Burner](https://github.com/bluemarblepayroll/burner). Burner, by itself, cannot use a database. So, if you wish to use a database as a data source or as a target for mutation then you need to add a library similar to this.
|
6
6
|
|
@@ -24,11 +24,11 @@ Refer to the [Burner](https://github.com/bluemarblepayroll/burner) library for m
|
|
24
24
|
|
25
25
|
### ActiveRecord Jobs
|
26
26
|
|
27
|
-
* **db_fuel/active_record/find_or_insert** [name, table_name, attributes, debug, primary_keyed_column, register, separator, timestamps, unique_attributes]: An extension of the `db_fuel/active_record/insert` job that adds an existence check before SQL insertion. The `unique_attributes` will be converted to WHERE clauses for performing the existence check.
|
28
|
-
* **db_fuel/active_record/insert** [name, table_name, attributes, debug, primary_keyed_column, register, separator, timestamps]: This job can take the objects in a register and insert them into a database table. If primary_keyed_column is specified then its key will be set to the primary key. Note that composite primary keys are not supported. Attributes defines which object properties to convert to SQL. Refer to the class and constructor specification for more detail.
|
29
|
-
* **db_fuel/active_record/update_all** [name, table_name, attributes, debug, register, separator, timestamps, unique_attributes]: This job can take the objects in a register and updates them within a database table. Attributes defines which object properties to convert to SQL SET clauses while unique_attributes translate to WHERE clauses. One or more records may be updated at a time. Refer to the class and constructor specification for more detail.
|
30
|
-
* **db_fuel/active_record/update** [name, table_name, attributes, debug, register, primary_keyed_column, separator, timestamps, unique_attributes]: This job can take the unique objects in a register and updates them within a database table. Attributes defines which object properties to convert to SQL SET clauses while unique_attributes translate to WHERE clauses to find the records to update. The primary_keyed_column is used to update the unique record. Only one record will be updated per statement. Note that composite primary keys are not supported. Refer to the class and constructor specification for more detail.
|
31
|
-
* **db_fuel/active_record/upsert** [name, table_name, attributes, debug, primary_keyed_column, register, separator, timestamps, unique_attributes]: This job can take the objects in a register and either inserts or updates them within a database table. Attributes defines which object properties to convert to SQL SET clauses while each key in unique_attributes become a WHERE clause in order to check for the existence of a specific record. The updated record will use the primary_keyed_column specified to perform the UPDATE operation. Note that composite primary keys are not supported. Refer to the class and constructor specification for more detail.
|
27
|
+
* **db_fuel/active_record/find_or_insert** [name, table_name, attributes, debug, primary_keyed_column, keys_register, register, separator, timestamps, unique_attributes]: An extension of the `db_fuel/active_record/insert` job that adds an existence check before SQL insertion. The `unique_attributes` will be converted to WHERE clauses for performing the existence check.
|
28
|
+
* **db_fuel/active_record/insert** [name, table_name, attributes, debug, primary_keyed_column, keys_register, register, separator, timestamps]: This job can take the objects in a register and insert them into a database table. If primary_keyed_column is specified then its key will be set to the primary key. Note that composite primary keys are not supported. Attributes defines which object properties to convert to SQL. Refer to the class and constructor specification for more detail.
|
29
|
+
* **db_fuel/active_record/update_all** [name, table_name, attributes, debug, keys_register, register, separator, timestamps, unique_attributes]: This job can take the objects in a register and updates them within a database table. Attributes defines which object properties to convert to SQL SET clauses while unique_attributes translate to WHERE clauses. One or more records may be updated at a time. Refer to the class and constructor specification for more detail.
|
30
|
+
* **db_fuel/active_record/update** [name, table_name, attributes, debug, keys_register, register, primary_keyed_column, separator, timestamps, unique_attributes]: This job can take the unique objects in a register and updates them within a database table. Attributes defines which object properties to convert to SQL SET clauses while unique_attributes translate to WHERE clauses to find the records to update. The primary_keyed_column is used to update the unique record. Only one record will be updated per statement. Note that composite primary keys are not supported. Refer to the class and constructor specification for more detail.
|
31
|
+
* **db_fuel/active_record/upsert** [name, table_name, attributes, debug, primary_keyed_column, keys_register, register, separator, timestamps, unique_attributes]: This job can take the objects in a register and either inserts or updates them within a database table. Attributes defines which object properties to convert to SQL SET clauses while each key in unique_attributes become a WHERE clause in order to check for the existence of a specific record. The updated record will use the primary_keyed_column specified to perform the UPDATE operation. Note that composite primary keys are not supported. Refer to the class and constructor specification for more detail.
|
32
32
|
|
33
33
|
### Dbee Jobs
|
34
34
|
|
@@ -331,7 +331,7 @@ Each database record should have been updated with their new respective middle n
|
|
331
331
|
|
332
332
|
#### Upserting Records
|
333
333
|
|
334
|
-
Let's say we don't know if these chart_number values already exist or not.
|
334
|
+
Let's say we don't know if these chart_number values already exist or not.
|
335
335
|
So we want db_fuel to either insert a record if the chart_number doesn't exist or update the record if the chart_number already exists.
|
336
336
|
|
337
337
|
````ruby
|
@@ -378,6 +378,10 @@ Notes:
|
|
378
378
|
|
379
379
|
* The `unique_attributes` translate to WHERE clauses.
|
380
380
|
* Set `debug: true` to print out each UPDATE statement in the output (not for production use.)
|
381
|
+
|
382
|
+
#### Limiting The Columns Being Inserted/Updated
|
383
|
+
|
384
|
+
All the db_fuel/active_record/* jobs now feature an optional `keys_register` option. If this is set, the register will be read and used as a key filter for which fields to set. For example, if you configure a db_fuel/active_record/insert job to update A, B, C, and D, but your keys_register contains [A,B] then only A and B will be inserted/set. This is helpful in scenarios where you may want to outline how to update _all_ possible fields but your input set only contains a subset of those fields and you do not wish to set the others to null.
|
381
385
|
## Contributing
|
382
386
|
|
383
387
|
### Development Environment Configuration
|
@@ -417,7 +421,7 @@ Note: ensure you have proper authorization before trying to publish new versions
|
|
417
421
|
After code changes have successfully gone through the Pull Request review process then the following steps should be followed for publishing new versions:
|
418
422
|
|
419
423
|
1. Merge Pull Request into master
|
420
|
-
2. Update `
|
424
|
+
2. Update `version.rb` using [semantic versioning](https://semver.org/)
|
421
425
|
3. Install dependencies: `bundle`
|
422
426
|
4. Update `CHANGELOG.md` with release notes
|
423
427
|
5. Commit & push master to remote and ensure CI builds master successfully
|
data/db_fuel.gemspec
CHANGED
@@ -16,15 +16,7 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
17
|
s.bindir = 'exe'
|
18
18
|
s.executables = %w[]
|
19
|
-
s.homepage = 'https://github.com/bluemarblepayroll/db_fuel'
|
20
19
|
s.license = 'MIT'
|
21
|
-
s.metadata = {
|
22
|
-
'bug_tracker_uri' => 'https://github.com/bluemarblepayroll/db_fuel/issues',
|
23
|
-
'changelog_uri' => 'https://github.com/bluemarblepayroll/db_fuel/blob/master/CHANGELOG.md',
|
24
|
-
'documentation_uri' => 'https://www.rubydoc.info/gems/db_fuel',
|
25
|
-
'homepage_uri' => s.homepage,
|
26
|
-
'source_code_uri' => s.homepage
|
27
|
-
}
|
28
20
|
|
29
21
|
s.required_ruby_version = '>= 2.5'
|
30
22
|
|
@@ -43,14 +35,15 @@ Gem::Specification.new do |s|
|
|
43
35
|
s.add_dependency('activerecord', activerecord_version)
|
44
36
|
s.add_dependency('acts_as_hashable', '~>1.2')
|
45
37
|
s.add_dependency('burner', '~>1.7')
|
46
|
-
s.add_dependency('dbee', '~>
|
47
|
-
s.add_dependency('dbee-active_record', '~>2.
|
38
|
+
s.add_dependency('dbee', '~>3.0')
|
39
|
+
s.add_dependency('dbee-active_record', '~>2.2')
|
48
40
|
s.add_dependency('objectable', '~>1.0')
|
49
41
|
|
50
42
|
s.add_development_dependency('guard-rspec', '~>4.7')
|
51
43
|
s.add_development_dependency('pry', '~>0')
|
52
44
|
s.add_development_dependency('rake', '~> 13')
|
53
45
|
s.add_development_dependency('rspec', '~> 3.8')
|
46
|
+
s.add_development_dependency('rspec_junit_formatter')
|
54
47
|
s.add_development_dependency('rubocop', '~>1.7.0')
|
55
48
|
s.add_development_dependency('simplecov', '~>0.18.5')
|
56
49
|
s.add_development_dependency('simplecov-console', '~>0.7.0')
|
@@ -19,25 +19,31 @@ module DbFuel
|
|
19
19
|
NOW_TYPE = 'r/value/now'
|
20
20
|
UPDATED_AT = :updated_at
|
21
21
|
|
22
|
-
attr_reader :
|
23
|
-
:db_provider,
|
22
|
+
attr_reader :db_provider,
|
24
23
|
:debug,
|
24
|
+
:keys_register,
|
25
25
|
:resolver,
|
26
|
-
:
|
26
|
+
:record_transformer
|
27
27
|
|
28
28
|
def initialize(
|
29
29
|
table_name:,
|
30
30
|
name: '',
|
31
31
|
attributes: [],
|
32
32
|
debug: false,
|
33
|
+
keys_register: nil,
|
33
34
|
register: Burner::DEFAULT_REGISTER,
|
34
35
|
separator: ''
|
35
36
|
)
|
36
37
|
super(name: name, register: register)
|
37
38
|
|
38
|
-
@
|
39
|
-
@
|
40
|
-
|
39
|
+
@keys_register = keys_register.to_s
|
40
|
+
@resolver = Objectable.resolver(separator: separator)
|
41
|
+
|
42
|
+
@record_transformer = Modeling::RecordTransformer.new(
|
43
|
+
resolver: resolver,
|
44
|
+
attributes: attributes
|
45
|
+
)
|
46
|
+
|
41
47
|
@db_provider = DbProvider.new(table_name)
|
42
48
|
@debug = debug || false
|
43
49
|
end
|
@@ -49,6 +55,16 @@ module DbFuel
|
|
49
55
|
|
50
56
|
output.detail(message)
|
51
57
|
end
|
58
|
+
|
59
|
+
def resolve_key_set(output, payload)
|
60
|
+
return Set.new if keys_register.empty?
|
61
|
+
|
62
|
+
keys = array(payload[keys_register]).map(&:to_s).to_set
|
63
|
+
|
64
|
+
output.detail("Limiting to only keys: #{keys.to_a.join(', ')}") if keys.any?
|
65
|
+
|
66
|
+
keys
|
67
|
+
end
|
52
68
|
end
|
53
69
|
end
|
54
70
|
end
|
@@ -57,6 +57,7 @@ module DbFuel
|
|
57
57
|
name: '',
|
58
58
|
attributes: [],
|
59
59
|
debug: false,
|
60
|
+
keys_register: nil,
|
60
61
|
primary_keyed_column: nil,
|
61
62
|
register: Burner::DEFAULT_REGISTER,
|
62
63
|
separator: '',
|
@@ -66,6 +67,7 @@ module DbFuel
|
|
66
67
|
|
67
68
|
super(
|
68
69
|
name: name,
|
70
|
+
keys_register: keys_register,
|
69
71
|
table_name: table_name,
|
70
72
|
attributes: attributes,
|
71
73
|
debug: debug,
|
@@ -82,6 +84,7 @@ module DbFuel
|
|
82
84
|
total_existed = 0
|
83
85
|
|
84
86
|
payload[register] = array(payload[register])
|
87
|
+
keys = resolve_key_set(output, payload)
|
85
88
|
|
86
89
|
payload[register].each do |row|
|
87
90
|
exists = find_record(output, row, payload.time)
|
@@ -91,7 +94,7 @@ module DbFuel
|
|
91
94
|
next
|
92
95
|
end
|
93
96
|
|
94
|
-
insert_record(output, row, payload.time)
|
97
|
+
insert_record(output, row, payload.time, keys)
|
95
98
|
|
96
99
|
total_inserted += 1
|
97
100
|
end
|
@@ -45,6 +45,7 @@ module DbFuel
|
|
45
45
|
# to the current UTC timestamp.
|
46
46
|
def initialize(
|
47
47
|
table_name:,
|
48
|
+
keys_register: nil,
|
48
49
|
name: '',
|
49
50
|
attributes: [],
|
50
51
|
debug: false,
|
@@ -53,11 +54,11 @@ module DbFuel
|
|
53
54
|
separator: '',
|
54
55
|
timestamps: true
|
55
56
|
)
|
56
|
-
|
57
57
|
attributes = Burner::Modeling::Attribute.array(attributes)
|
58
58
|
|
59
59
|
super(
|
60
60
|
name: name,
|
61
|
+
keys_register: keys_register,
|
61
62
|
table_name: table_name,
|
62
63
|
attributes: attributes,
|
63
64
|
debug: debug,
|
@@ -70,8 +71,11 @@ module DbFuel
|
|
70
71
|
|
71
72
|
def perform(output, payload)
|
72
73
|
payload[register] = array(payload[register])
|
74
|
+
keys = resolve_key_set(output, payload)
|
75
|
+
|
76
|
+
output.detail("Inserting #{payload[register].count} record(s)")
|
73
77
|
|
74
|
-
payload[register].each { |row| insert_record(output, row, payload.time) }
|
78
|
+
payload[register].each { |row| insert_record(output, row, payload.time, keys) }
|
75
79
|
end
|
76
80
|
end
|
77
81
|
end
|
@@ -57,6 +57,7 @@ module DbFuel
|
|
57
57
|
attributes: [],
|
58
58
|
debug: false,
|
59
59
|
primary_keyed_column: nil,
|
60
|
+
keys_register: nil,
|
60
61
|
register: Burner::DEFAULT_REGISTER,
|
61
62
|
separator: '',
|
62
63
|
timestamps: true,
|
@@ -70,6 +71,7 @@ module DbFuel
|
|
70
71
|
table_name: table_name,
|
71
72
|
attributes: attributes,
|
72
73
|
debug: debug,
|
74
|
+
keys_register: keys_register,
|
73
75
|
primary_keyed_column: primary_keyed_column,
|
74
76
|
register: register,
|
75
77
|
separator: separator,
|
@@ -81,20 +83,16 @@ module DbFuel
|
|
81
83
|
end
|
82
84
|
|
83
85
|
def perform(output, payload)
|
84
|
-
total_rows_affected = 0
|
85
|
-
|
86
86
|
payload[register] = array(payload[register])
|
87
|
+
keys = resolve_key_set(output, payload)
|
87
88
|
|
88
|
-
payload[register].
|
89
|
-
|
90
|
-
|
91
|
-
first_record = update_record(output, row, payload.time)
|
92
|
-
|
93
|
-
rows_affected = 1 if first_record
|
89
|
+
total_rows_affected = payload[register].inject(0) do |memo, row|
|
90
|
+
first_record = update_record(output, row, payload.time, keys)
|
91
|
+
rows_affected = first_record ? 1 : 0
|
94
92
|
|
95
93
|
debug_detail(output, "Individual Rows Affected: #{rows_affected}")
|
96
94
|
|
97
|
-
|
95
|
+
memo + rows_affected
|
98
96
|
end
|
99
97
|
|
100
98
|
output.detail("Total Rows Affected: #{total_rows_affected}")
|
@@ -50,6 +50,7 @@ module DbFuel
|
|
50
50
|
attributes: [],
|
51
51
|
debug: false,
|
52
52
|
register: Burner::DEFAULT_REGISTER,
|
53
|
+
keys_register: nil,
|
53
54
|
separator: '',
|
54
55
|
timestamps: true,
|
55
56
|
unique_attributes: []
|
@@ -63,6 +64,7 @@ module DbFuel
|
|
63
64
|
attributes: attributes,
|
64
65
|
debug: debug,
|
65
66
|
primary_keyed_column: nil,
|
67
|
+
keys_register: keys_register,
|
66
68
|
register: register,
|
67
69
|
separator: separator,
|
68
70
|
timestamps: timestamps,
|
@@ -73,19 +75,16 @@ module DbFuel
|
|
73
75
|
end
|
74
76
|
|
75
77
|
def perform(output, payload)
|
76
|
-
total_rows_affected = 0
|
77
|
-
|
78
78
|
payload[register] = array(payload[register])
|
79
|
+
keys = resolve_key_set(output, payload)
|
79
80
|
|
80
|
-
payload[register].
|
81
|
-
where_object
|
82
|
-
|
83
|
-
|
84
|
-
rows_affected = update(output, row, payload.time, where_object)
|
81
|
+
total_rows_affected = payload[register].inject(0) do |memo, row|
|
82
|
+
where_object = unique_record_transformer.transform(row, payload.time)
|
83
|
+
rows_affected = update(output, row, payload.time, where_object, keys)
|
85
84
|
|
86
85
|
debug_detail(output, "Individual Rows Affected: #{rows_affected}")
|
87
86
|
|
88
|
-
|
87
|
+
memo + rows_affected
|
89
88
|
end
|
90
89
|
|
91
90
|
output.detail("Total Rows Affected: #{total_rows_affected}")
|
@@ -21,7 +21,7 @@ module DbFuel
|
|
21
21
|
# Expected Payload[register] input: array of objects
|
22
22
|
# Payload[register] output: array of objects.
|
23
23
|
class Upsert < Base
|
24
|
-
attr_reader :primary_keyed_column, :timestamps, :
|
24
|
+
attr_reader :primary_keyed_column, :timestamps, :unique_record_transformer
|
25
25
|
|
26
26
|
# Arguments:
|
27
27
|
# name: name of the job within the Burner::Pipeline.
|
@@ -59,6 +59,7 @@ module DbFuel
|
|
59
59
|
def initialize(
|
60
60
|
table_name:,
|
61
61
|
primary_keyed_column:,
|
62
|
+
keys_register: nil,
|
62
63
|
name: '',
|
63
64
|
attributes: [],
|
64
65
|
debug: false,
|
@@ -69,6 +70,7 @@ module DbFuel
|
|
69
70
|
)
|
70
71
|
super(
|
71
72
|
name: name,
|
73
|
+
keys_register: keys_register,
|
72
74
|
table_name: table_name,
|
73
75
|
attributes: attributes,
|
74
76
|
debug: debug,
|
@@ -78,8 +80,10 @@ module DbFuel
|
|
78
80
|
|
79
81
|
@primary_keyed_column = Modeling::KeyedColumn.make(primary_keyed_column, nullable: true)
|
80
82
|
|
81
|
-
@
|
82
|
-
|
83
|
+
@unique_record_transformer = Modeling::RecordTransformer.new(
|
84
|
+
resolver: resolver,
|
85
|
+
attributes: unique_attributes
|
86
|
+
)
|
83
87
|
|
84
88
|
@timestamps = timestamps
|
85
89
|
|
@@ -93,9 +97,10 @@ module DbFuel
|
|
93
97
|
total_updated = 0
|
94
98
|
|
95
99
|
payload[register] = array(payload[register])
|
100
|
+
keys = resolve_key_set(output, payload)
|
96
101
|
|
97
102
|
payload[register].each do |row|
|
98
|
-
record_updated = insert_or_update(output, row, payload.time)
|
103
|
+
record_updated = insert_or_update(output, row, payload.time, keys)
|
99
104
|
|
100
105
|
if record_updated
|
101
106
|
total_updated += 1
|
@@ -111,7 +116,7 @@ module DbFuel
|
|
111
116
|
protected
|
112
117
|
|
113
118
|
def find_record(output, row, time)
|
114
|
-
unique_row =
|
119
|
+
unique_row = unique_record_transformer.transform(row, time)
|
115
120
|
|
116
121
|
first_sql = db_provider.first_sql(unique_row)
|
117
122
|
|
@@ -128,16 +133,14 @@ module DbFuel
|
|
128
133
|
first_record
|
129
134
|
end
|
130
135
|
|
131
|
-
def insert_record(output, row, time)
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
set_object = attribute_renderers_set.transform(dynamic_attrs, row, time)
|
136
|
+
def insert_record(output, row, time, keys = Set.new)
|
137
|
+
set_object = record_transformer.transform(
|
138
|
+
row,
|
139
|
+
time,
|
140
|
+
keys: keys,
|
141
|
+
created_at: timestamps,
|
142
|
+
updated_at: timestamps
|
143
|
+
)
|
141
144
|
|
142
145
|
insert_sql = db_provider.insert_sql(set_object)
|
143
146
|
|
@@ -152,7 +155,7 @@ module DbFuel
|
|
152
155
|
end
|
153
156
|
|
154
157
|
# Updates only a single record. Lookups primary key to update the record.
|
155
|
-
def update_record(output, row, time)
|
158
|
+
def update_record(output, row, time, keys)
|
156
159
|
raise ArgumentError, 'primary_keyed_column is required' unless primary_keyed_column
|
157
160
|
|
158
161
|
first_record = find_record(output, row, time)
|
@@ -165,23 +168,20 @@ module DbFuel
|
|
165
168
|
where_object = { primary_keyed_column.column => id }
|
166
169
|
|
167
170
|
# update record using the primary key as the WHERE clause
|
168
|
-
update(output, row, time, where_object)
|
171
|
+
update(output, row, time, where_object, keys)
|
169
172
|
end
|
170
173
|
|
171
174
|
first_record
|
172
175
|
end
|
173
176
|
|
174
177
|
# Updates one or many records depending on where_object passed
|
175
|
-
def update(output, row, time, where_object)
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
end
|
183
|
-
|
184
|
-
set_object = attribute_renderers_set.transform(dynamic_attrs, row, time)
|
178
|
+
def update(output, row, time, where_object, keys)
|
179
|
+
set_object = record_transformer.transform(
|
180
|
+
row,
|
181
|
+
time,
|
182
|
+
keys: keys,
|
183
|
+
updated_at: timestamps
|
184
|
+
)
|
185
185
|
|
186
186
|
update_sql = db_provider.update_sql(set_object, where_object)
|
187
187
|
|
@@ -194,14 +194,14 @@ module DbFuel
|
|
194
194
|
|
195
195
|
private
|
196
196
|
|
197
|
-
def insert_or_update(output, row, time)
|
198
|
-
first_record = update_record(output, row, time)
|
197
|
+
def insert_or_update(output, row, time, keys)
|
198
|
+
first_record = update_record(output, row, time, keys)
|
199
199
|
|
200
200
|
if first_record
|
201
201
|
first_record
|
202
202
|
else
|
203
203
|
# create the record
|
204
|
-
insert_record(output, row, time)
|
204
|
+
insert_record(output, row, time, keys)
|
205
205
|
nil
|
206
206
|
end
|
207
207
|
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module DbFuel
|
11
|
+
module Modeling
|
12
|
+
# Creates attribute renderers based on attributes passed.
|
13
|
+
# Also constains methods to transform attribute renderers
|
14
|
+
# and include timestamp attributes if needed.
|
15
|
+
class RecordTransformer # :nodoc:
|
16
|
+
NOW_TYPE = 'r/value/now'
|
17
|
+
CREATED_AT = 'created_at'
|
18
|
+
UPDATED_AT = 'updated_at'
|
19
|
+
|
20
|
+
attr_reader :attribute_renderers, :resolver
|
21
|
+
|
22
|
+
def initialize(resolver:, attributes: [])
|
23
|
+
raise ArgumentError, 'resolver is required' unless resolver
|
24
|
+
|
25
|
+
@resolver = resolver
|
26
|
+
@attribute_renderers = make_renderers(attributes)
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
|
31
|
+
def transform(row, time, keys: Set.new, created_at: false, updated_at: false)
|
32
|
+
dynamic_attributes, all_keys = make_dynamic_attributes(
|
33
|
+
keys: keys,
|
34
|
+
created_at: created_at,
|
35
|
+
updated_at: updated_at
|
36
|
+
)
|
37
|
+
|
38
|
+
dynamic_attributes.each_with_object({}) do |attribute_renderer, memo|
|
39
|
+
next if all_keys.any? && all_keys.exclude?(attribute_renderer.key)
|
40
|
+
|
41
|
+
value = attribute_renderer.transform(row, time)
|
42
|
+
|
43
|
+
resolver.set(memo, attribute_renderer.key, value)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def make_dynamic_attributes(keys:, created_at:, updated_at:)
|
50
|
+
dynamic_attributes = attribute_renderers
|
51
|
+
all_keys = keys
|
52
|
+
|
53
|
+
if created_at
|
54
|
+
dynamic_attributes += [
|
55
|
+
Burner::Modeling::AttributeRenderer.new(created_at_timestamp_attribute, resolver)
|
56
|
+
]
|
57
|
+
|
58
|
+
all_keys += [CREATED_AT] if keys.any?
|
59
|
+
end
|
60
|
+
|
61
|
+
if updated_at
|
62
|
+
dynamic_attributes += [
|
63
|
+
Burner::Modeling::AttributeRenderer.new(updated_at_timestamp_attribute, resolver)
|
64
|
+
]
|
65
|
+
|
66
|
+
all_keys += [UPDATED_AT] if keys.any?
|
67
|
+
end
|
68
|
+
|
69
|
+
[
|
70
|
+
dynamic_attributes,
|
71
|
+
all_keys
|
72
|
+
]
|
73
|
+
end
|
74
|
+
|
75
|
+
def make_renderers(attributes)
|
76
|
+
Burner::Modeling::Attribute
|
77
|
+
.array(attributes)
|
78
|
+
.map { |a| Burner::Modeling::AttributeRenderer.new(a, resolver) }
|
79
|
+
end
|
80
|
+
|
81
|
+
def created_at_timestamp_attribute
|
82
|
+
timestamp_attribute(CREATED_AT)
|
83
|
+
end
|
84
|
+
|
85
|
+
def updated_at_timestamp_attribute
|
86
|
+
timestamp_attribute(UPDATED_AT)
|
87
|
+
end
|
88
|
+
|
89
|
+
def timestamp_attribute(key)
|
90
|
+
Burner::Modeling::Attribute.make(
|
91
|
+
key: key,
|
92
|
+
transformers: [
|
93
|
+
{ type: NOW_TYPE }
|
94
|
+
]
|
95
|
+
)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
data/lib/db_fuel/modeling.rb
CHANGED
data/lib/db_fuel/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: db_fuel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthew Ruggio
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-
|
12
|
+
date: 2021-12-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activerecord
|
@@ -65,28 +65,28 @@ dependencies:
|
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '3.0'
|
69
69
|
type: :runtime
|
70
70
|
prerelease: false
|
71
71
|
version_requirements: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
75
|
+
version: '3.0'
|
76
76
|
- !ruby/object:Gem::Dependency
|
77
77
|
name: dbee-active_record
|
78
78
|
requirement: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '2.
|
82
|
+
version: '2.2'
|
83
83
|
type: :runtime
|
84
84
|
prerelease: false
|
85
85
|
version_requirements: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '2.
|
89
|
+
version: '2.2'
|
90
90
|
- !ruby/object:Gem::Dependency
|
91
91
|
name: objectable
|
92
92
|
requirement: !ruby/object:Gem::Requirement
|
@@ -157,6 +157,20 @@ dependencies:
|
|
157
157
|
- - "~>"
|
158
158
|
- !ruby/object:Gem::Version
|
159
159
|
version: '3.8'
|
160
|
+
- !ruby/object:Gem::Dependency
|
161
|
+
name: rspec_junit_formatter
|
162
|
+
requirement: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
167
|
+
type: :development
|
168
|
+
prerelease: false
|
169
|
+
version_requirements: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">="
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
160
174
|
- !ruby/object:Gem::Dependency
|
161
175
|
name: rubocop
|
162
176
|
requirement: !ruby/object:Gem::Requirement
|
@@ -222,12 +236,12 @@ executables: []
|
|
222
236
|
extensions: []
|
223
237
|
extra_rdoc_files: []
|
224
238
|
files:
|
239
|
+
- ".circleci/config.yml"
|
225
240
|
- ".editorconfig"
|
226
241
|
- ".gitignore"
|
227
242
|
- ".rubocop.yml"
|
228
243
|
- ".ruby-version"
|
229
244
|
- ".tool-versions"
|
230
|
-
- ".travis.yml"
|
231
245
|
- CHANGELOG.md
|
232
246
|
- CODE_OF_CONDUCT.md
|
233
247
|
- Gemfile
|
@@ -251,18 +265,13 @@ files:
|
|
251
265
|
- lib/db_fuel/library/dbee/query.rb
|
252
266
|
- lib/db_fuel/library/dbee/range.rb
|
253
267
|
- lib/db_fuel/modeling.rb
|
254
|
-
- lib/db_fuel/modeling/attribute_renderer_set.rb
|
255
268
|
- lib/db_fuel/modeling/keyed_column.rb
|
269
|
+
- lib/db_fuel/modeling/record_transformer.rb
|
256
270
|
- lib/db_fuel/version.rb
|
257
|
-
homepage:
|
271
|
+
homepage:
|
258
272
|
licenses:
|
259
273
|
- MIT
|
260
|
-
metadata:
|
261
|
-
bug_tracker_uri: https://github.com/bluemarblepayroll/db_fuel/issues
|
262
|
-
changelog_uri: https://github.com/bluemarblepayroll/db_fuel/blob/master/CHANGELOG.md
|
263
|
-
documentation_uri: https://www.rubydoc.info/gems/db_fuel
|
264
|
-
homepage_uri: https://github.com/bluemarblepayroll/db_fuel
|
265
|
-
source_code_uri: https://github.com/bluemarblepayroll/db_fuel
|
274
|
+
metadata: {}
|
266
275
|
post_install_message:
|
267
276
|
rdoc_options: []
|
268
277
|
require_paths:
|
data/.travis.yml
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
env:
|
2
|
-
global:
|
3
|
-
- CC_TEST_REPORTER_ID=2d57d597269e2cb04a63d0a7262927cf811abb7ab528da7de681943b212b4134
|
4
|
-
jobs:
|
5
|
-
- AR_VERSION=5
|
6
|
-
- AR_VERSION=6
|
7
|
-
language: ruby
|
8
|
-
rvm:
|
9
|
-
# Build on the latest stable of all supported Rubies (https://www.ruby-lang.org/en/downloads/):
|
10
|
-
- 2.5.8
|
11
|
-
- 2.6.6
|
12
|
-
- 2.7.1
|
13
|
-
cache: bundler
|
14
|
-
before_script:
|
15
|
-
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
16
|
-
- chmod +x ./cc-test-reporter
|
17
|
-
- ./cc-test-reporter before-build
|
18
|
-
- cp spec/config/database.yaml.ci spec/config/database.yaml
|
19
|
-
script:
|
20
|
-
- bundle exec rubocop
|
21
|
-
- bundle exec rspec spec --format documentation
|
22
|
-
after_script:
|
23
|
-
- ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
|
24
|
-
addons:
|
25
|
-
# https://docs.travis-ci.com/user/uploading-artifacts/
|
26
|
-
artifacts:
|
27
|
-
paths:
|
28
|
-
- Gemfile.lock
|
@@ -1,70 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
#
|
4
|
-
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
-
#
|
6
|
-
# This source code is licensed under the MIT license found in the
|
7
|
-
# LICENSE file in the root directory of this source tree.
|
8
|
-
#
|
9
|
-
|
10
|
-
module DbFuel
|
11
|
-
module Modeling
|
12
|
-
# Creates attribute renderers based on attributes passed.
|
13
|
-
# Also constains methods to transform attribute renderers
|
14
|
-
# and include timestamp attributes if needed.
|
15
|
-
class AttributeRendererSet # :nodoc:
|
16
|
-
NOW_TYPE = 'r/value/now'
|
17
|
-
|
18
|
-
CREATED_AT = Burner::Modeling::Attribute.make(
|
19
|
-
key: :created_at, transformers: [{ type: NOW_TYPE }]
|
20
|
-
)
|
21
|
-
|
22
|
-
UPDATED_AT = Burner::Modeling::Attribute.make(
|
23
|
-
key: :updated_at, transformers: [{ type: NOW_TYPE }]
|
24
|
-
)
|
25
|
-
|
26
|
-
attr_reader :attribute_renderers, :resolver
|
27
|
-
|
28
|
-
def initialize(resolver:, attributes: [])
|
29
|
-
raise ArgumentError, 'resolver is required' unless resolver
|
30
|
-
|
31
|
-
@resolver = resolver
|
32
|
-
@attribute_renderers = make_renderers(attributes)
|
33
|
-
|
34
|
-
freeze
|
35
|
-
end
|
36
|
-
|
37
|
-
# Adds the attributes for created_at and updated_at to the currrent attribute renderers.
|
38
|
-
def timestamp_created_attribute_renderers
|
39
|
-
timestamp_attributes = [CREATED_AT, UPDATED_AT]
|
40
|
-
|
41
|
-
timestamp_attributes.map do |a|
|
42
|
-
Burner::Modeling::AttributeRenderer.new(a, resolver)
|
43
|
-
end + attribute_renderers
|
44
|
-
end
|
45
|
-
|
46
|
-
# Adds the attribute for updated_at to the currrent attribute renderers.
|
47
|
-
def timestamp_updated_attribute_renderers
|
48
|
-
timestamp_attributes = [UPDATED_AT]
|
49
|
-
|
50
|
-
timestamp_attributes.map do |a|
|
51
|
-
Burner::Modeling::AttributeRenderer.new(a, resolver)
|
52
|
-
end + attribute_renderers
|
53
|
-
end
|
54
|
-
|
55
|
-
def make_renderers(attributes)
|
56
|
-
Burner::Modeling::Attribute
|
57
|
-
.array(attributes)
|
58
|
-
.map { |a| Burner::Modeling::AttributeRenderer.new(a, resolver) }
|
59
|
-
end
|
60
|
-
|
61
|
-
def transform(attribute_renderers, row, time)
|
62
|
-
attribute_renderers.each_with_object({}) do |attribute_renderer, memo|
|
63
|
-
value = attribute_renderer.transform(row, time)
|
64
|
-
|
65
|
-
resolver.set(memo, attribute_renderer.key, value)
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|