db_blaster 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 31207f1a8613b779dffcf5a8c17c5220508ae6da66088611a4f66ec457e9c210
|
4
|
+
data.tar.gz: 2ff279b9cee2ce471fd99bb3f2e8f786eed3f94d42a4a6488e2248b476d71080
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9f83e6757ccdc32c57a0e64c004e9b07fb543b54fafcf3c3838969bb84853601d959f178977a14b2bf5244ba7b194a2a427ed8ba74b96af7aef557a786875f01
|
7
|
+
data.tar.gz: 01f717e65dc1def37cddf11af9ba0df201d213713a8a0d83aa9490d70e796bffabf17edf004e946cd87f35ac8afde64a09b6c9aa37b31488e4f1422329d82838
|
@@ -8,6 +8,8 @@ module DbBlaster
|
|
8
8
|
DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
|
9
9
|
DEFAULT_S3_KEY = '<batch_date>/<batch_time>/db_blaster/<table_name>/<uuid>.json'
|
10
10
|
DEFAULT_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%LZ'
|
11
|
+
ATTRIBUTE_S3_META_FORMAT = 'attribute' # { meta: {your: :value}, records: [] }
|
12
|
+
INLINE_S3_META_FORMAT = 'inline' # records.collect{|record| record.merge(meta) }
|
11
13
|
|
12
14
|
# The required configuration fields
|
13
15
|
REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region].freeze
|
@@ -32,14 +34,22 @@ module DbBlaster
|
|
32
34
|
# Applicable only when `s3_bucket' is set
|
33
35
|
# The value set here will be included in every payload pushed to S3
|
34
36
|
# example: config.s3_meta = {'infra_id' => '061', 'source_app' => 'kcp-api'}}
|
35
|
-
# The resulting JSON will include the `meta` merged into every record
|
36
37
|
attr_accessor :s3_meta
|
37
38
|
|
39
|
+
# Optional
|
40
|
+
# Options: ['attribute', 'inline']
|
41
|
+
# Defaults to 'attribute'
|
42
|
+
# 'attribute' payload: { meta: `s3_meta`, records: [source_table_records] }
|
43
|
+
# 'inline' payload: records.collect{|record| record.merge(meta) }
|
44
|
+
attr_accessor :s3_meta_format
|
45
|
+
|
38
46
|
# Optional
|
39
47
|
# Applicable only when `s3_bucket` is set
|
40
48
|
# The S3 key. The following values will get substituted:
|
41
|
-
# <
|
42
|
-
# <
|
49
|
+
# <batch_date_time> - date time when batch started
|
50
|
+
# <batch_date> - date when batch started
|
51
|
+
# <batch_time - time when batch started
|
52
|
+
# <date_time> - the datetime just before pushing to S3
|
43
53
|
# <table_name> - the name of the table associated with the S3 body
|
44
54
|
# <uuid> - a universal identifier
|
45
55
|
# '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
|
@@ -23,10 +23,12 @@ module DbBlaster
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def substitutions
|
26
|
+
date_time = DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
|
26
27
|
date, time = batch_start_time.split('T')
|
27
28
|
{ '<batch_date_time>' => batch_start_time,
|
28
29
|
'<batch_date>' => date,
|
29
30
|
'<batch_time>' => time,
|
31
|
+
'<date_time>' => date_time,
|
30
32
|
'<uuid>' => SecureRandom.uuid,
|
31
33
|
'<table_name>' => source_table_name }
|
32
34
|
end
|
data/lib/db_blaster/version.rb
CHANGED
@@ -22,8 +22,7 @@ DbBlaster.configure do |config|
|
|
22
22
|
# The S3 key path. The following values will get substituted:
|
23
23
|
# <batch_date_time> - a timestamp signifying the beginning of the batch processing
|
24
24
|
# <batch_date> - a date signifying the beginning of the batch processing
|
25
|
-
# <
|
26
|
-
# <timestamp> - the current time
|
25
|
+
# <date_time> - the datetime just before pushing to S3
|
27
26
|
# <table_name> - the name of the table associated with the S3 body
|
28
27
|
# <uuid> - a universal identifier
|
29
28
|
# config.s3_key = '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: db_blaster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Perry Hertler
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-09-
|
11
|
+
date: 2021-09-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-s3
|