db_blaster 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2de00cf222282fde0e0a3e38078ccc398a80ff19c41ccf7b79ecd043450db6b1
-  data.tar.gz: 66b93222044a44bcd7dd35a6233de5c4839a58d5106e527a7d540be2daee9821
+  metadata.gz: 31207f1a8613b779dffcf5a8c17c5220508ae6da66088611a4f66ec457e9c210
+  data.tar.gz: 2ff279b9cee2ce471fd99bb3f2e8f786eed3f94d42a4a6488e2248b476d71080
 SHA512:
-  metadata.gz: 7305baf42dff923be85d8105625bf3ccf8200988d7895c80202a32232b062a9c613863a002a7b8e57d7e57a3660725ab4ed6765d2b2d3ba49f3b216da2614dac
-  data.tar.gz: 8652430ce86f2ee275997ec35e52bf2ba99befa4c3a8630b13e0ca57f309d4b0fb013974806e52882bc37bd95a6c4e9e9d4ea51fa5a2096d2767df0d3608b796
+  metadata.gz: 9f83e6757ccdc32c57a0e64c004e9b07fb543b54fafcf3c3838969bb84853601d959f178977a14b2bf5244ba7b194a2a427ed8ba74b96af7aef557a786875f01
+  data.tar.gz: 01f717e65dc1def37cddf11af9ba0df201d213713a8a0d83aa9490d70e796bffabf17edf004e946cd87f35ac8afde64a09b6c9aa37b31488e4f1422329d82838

data/lib/db_blaster/configuration.rb

@@ -8,6 +8,8 @@ module DbBlaster
     DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
     DEFAULT_S3_KEY = '<batch_date>/<batch_time>/db_blaster/<table_name>/<uuid>.json'
     DEFAULT_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%LZ'
+    ATTRIBUTE_S3_META_FORMAT = 'attribute' # { meta: {your: :value}, records: [] }
+    INLINE_S3_META_FORMAT = 'inline' # records.collect{|record| record.merge(meta) }
 
     # The required configuration fields
     REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region].freeze
@@ -32,14 +34,22 @@ module DbBlaster
     # Applicable only when `s3_bucket' is set
     # The value set here will be included in every payload pushed to S3
     # example: config.s3_meta = {'infra_id' => '061', 'source_app' => 'kcp-api'}}
-    # The resulting JSON will include the `meta` merged into every record
     attr_accessor :s3_meta
 
+    # Optional
+    # Options: ['attribute', 'inline']
+    # Defaults to 'attribute'
+    # 'attribute' payload: { meta: `s3_meta`, records: [source_table_records] }
+    # 'inline' payload: records.collect{|record| record.merge(meta) }
+    attr_accessor :s3_meta_format
+
     # Optional
     # Applicable only when `s3_bucket` is set
     # The S3 key. The following values will get substituted:
-    # <batch_timestamp> - a timestamp signifying the beginning of the batch processing
-    # <timestamp> - the current time
+    # <batch_date_time> - date time when batch started
+    # <batch_date> - date when batch started
+    # <batch_time - time when batch started
+    # <date_time> - the datetime just before pushing to S3
     # <table_name> - the name of the table associated with the S3 body
     # <uuid> - a universal identifier
     # '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'

data/lib/db_blaster/s3_key_builder.rb

@@ -23,10 +23,12 @@ module DbBlaster
     end
 
     def substitutions
+      date_time = DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
       date, time = batch_start_time.split('T')
       { '<batch_date_time>' => batch_start_time,
         '<batch_date>' => date,
         '<batch_time>' => time,
+        '<date_time>' => date_time,
         '<uuid>' => SecureRandom.uuid,
         '<table_name>' => source_table_name }
     end

data/lib/db_blaster/s3_publisher.rb

@@ -14,7 +14,10 @@ module DbBlaster
     end
 
     def content
-      meta_records
+      return meta_records if DbBlaster.configuration.s3_meta_format == Configuration::INLINE_S3_META_FORMAT
+
+      { meta: meta,
+        records: records }
     end
 
     def tagging

data/lib/db_blaster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DbBlaster
-  VERSION = '0.1.5'
+  VERSION = '0.1.6'
 end

data/lib/generators/db_blaster/install/templates/db_blaster_config.rb

@@ -22,8 +22,7 @@ DbBlaster.configure do |config|
   # The S3 key path. The following values will get substituted:
   # <batch_date_time> - a timestamp signifying the beginning of the batch processing
   # <batch_date> - a date signifying the beginning of the batch processing
-  # <batch_time> - a time signifying the beginning of the batch processing
-  # <timestamp> - the current time
+  # <date_time> - the datetime just before pushing to S3
   # <table_name> - the name of the table associated with the S3 body
   # <uuid> - a universal identifier
   # config.s3_key = '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_blaster
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Perry Hertler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2021-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3