db_blaster 0.1.2 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d35e663180807e21555f50de79be57ebaf73a7a2effbc86fda8cb2b59ea538bb
-  data.tar.gz: fcd1df984229d196a858e7e81fc94503eabb476bb3569afc0ff66871f6ab0e6e
+  metadata.gz: 31207f1a8613b779dffcf5a8c17c5220508ae6da66088611a4f66ec457e9c210
+  data.tar.gz: 2ff279b9cee2ce471fd99bb3f2e8f786eed3f94d42a4a6488e2248b476d71080
 SHA512:
-  metadata.gz: 270045284ee3ff5032bbb5b34de7bd3ad6add7252b6c7af05dfa6e1525f6b156c2d11138a62cbd1a3630606bb52e6df25b3e050ce24262ab99d4679b2ba84dd3
-  data.tar.gz: af1f643cae181fdbfba7f6a5ec11acef540f33d5bda76ce9753e6e0065dd40ca8640af80fcc9bd530ada2645ef8b4131f4afc5cb00f5c0f439836ab700425de5
+  metadata.gz: 9f83e6757ccdc32c57a0e64c004e9b07fb543b54fafcf3c3838969bb84853601d959f178977a14b2bf5244ba7b194a2a427ed8ba74b96af7aef557a786875f01
+  data.tar.gz: 01f717e65dc1def37cddf11af9ba0df201d213713a8a0d83aa9490d70e796bffabf17edf004e946cd87f35ac8afde64a09b6c9aa37b31488e4f1422329d82838

data/README.md

@@ -2,15 +2,15 @@
 # DbBlaster
 ![Image of DB to SNS](https://lucid.app/publicSegments/view/c70feed3-2f48-46ee-8734-423474488feb/image.png)
 
-DbBlaster publishes changed database rows to AWS SNS. The first time `DbBlaster::PublishAllJob.perform_later` is ran,
-the entire database will be incrementally published to SNS. Subsequent runs will publish rows whose `updated_at` column
+DbBlaster can either publish changed database rows to AWS SNS or push the changes to S#. The first time `DbBlaster::PublishAllJob.perform_later` is ran,
+the entire database will be incrementally published. Subsequent runs will publish rows whose `updated_at` column
 is more recent than the last run.
 
 Consuming the published messages is functionality not provided by DbBlaster. 
 
 ## Usage
 
-Update `config/initializers/db_blaster_config.rb` with valid AWS credentials, topics, and options.
+Update `config/initializers/db_blaster_config.rb` with valid AWS credentials and options. Either `sns_topic` or `s3_bucket` must be set!
 
 Schedule `DbBlaster::PublishAllJob.perform_later` to run periodically with something
 like [sidekiq-cron](https://github.com/ondrejbartas/sidekiq-cron) or [whenever](https://github.com/javan/whenever)

data/app/jobs/db_blaster/publish_all_job.rb

@@ -13,8 +13,12 @@ module DbBlaster
                          .build_all(DbBlaster.configuration))
 
       DbBlaster::SourceTable.pluck(:id).each do |source_table_id|
-        PublishSourceTableJob.perform_later(source_table_id)
+        PublishSourceTableJob.perform_later(source_table_id, batch_start_time)
       end
     end
+
+    def batch_start_time
+      @batch_start_time ||= DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
+    end
   end
 end

data/app/jobs/db_blaster/publish_source_table_job.rb

@@ -6,11 +6,11 @@ module DbBlaster
   class PublishSourceTableJob < ApplicationJob
     queue_as 'default'
 
-    def perform(source_table_id)
+    def perform(source_table_id, batch_start_time)
       source_table = SourceTable.find_by(id: source_table_id)
       return unless source_table
 
-      PublishSourceTable.execute(source_table)
+      PublishSourceTable.execute(source_table: source_table, batch_start_time: batch_start_time)
     end
   end
 end

data/lib/db_blaster/base_publisher.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# publish records to SNS topic
+module DbBlaster
+  # Base class for publishing
+  class BasePublisher
+    attr_reader :source_table, :records, :batch_start_time
+
+    def initialize(source_table, records, batch_start_time)
+      @source_table = source_table
+      @records = records
+      @batch_start_time = batch_start_time
+    end
+
+    def self.publish(source_table:, records:, batch_start_time:)
+      publisher_class =
+        if DbBlaster.configuration.sns_topic
+          SnsPublisher
+        else
+          S3Publisher
+        end
+      publisher_class.new(source_table, records, batch_start_time).publish
+    end
+
+    def publish
+      raise NotImplementedError
+    end
+  end
+end

data/lib/db_blaster/configuration.rb

@@ -2,17 +2,65 @@
 
 module DbBlaster
   # Configuration class for providing credentials, topics, and customizations.
+  # Either the `sns_topic` or `s3_bucket' must be set
   class Configuration
     DEFAULT_BATCH_SIZE = 100
     DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
+    DEFAULT_S3_KEY = '<batch_date>/<batch_time>/db_blaster/<table_name>/<uuid>.json'
+    DEFAULT_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%LZ'
+    ATTRIBUTE_S3_META_FORMAT = 'attribute' # { meta: {your: :value}, records: [] }
+    INLINE_S3_META_FORMAT = 'inline' # records.collect{|record| record.merge(meta) }
 
     # The required configuration fields
-    REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region sns_topic].freeze
+    REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region].freeze
+
+    # exactly one of these fields needs to be set
+    EITHER_OR_FIELDS = %i[sns_topic s3_bucket].freeze
 
     # The topic to which messages will be published
     attr_accessor :sns_topic
+    # The s3 bucket name
+    attr_accessor :s3_bucket
     attr_accessor :aws_access_key, :aws_access_secret, :aws_region
 
+    # Optional
+    # Applicable only when `sns_topic` is set
+    # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
+    # Attributes set here will be included in every published message
+    # example: config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+    attr_accessor :extra_sns_message_attributes
+
+    # Optional
+    # Applicable only when `s3_bucket' is set
+    # The value set here will be included in every payload pushed to S3
+    # example: config.s3_meta = {'infra_id' => '061', 'source_app' => 'kcp-api'}}
+    attr_accessor :s3_meta
+
+    # Optional
+    # Options: ['attribute', 'inline']
+    # Defaults to 'attribute'
+    # 'attribute' payload: { meta: `s3_meta`, records: [source_table_records] }
+    # 'inline' payload: records.collect{|record| record.merge(meta) }
+    attr_accessor :s3_meta_format
+
+    # Optional
+    # Applicable only when `s3_bucket` is set
+    # The S3 key. The following values will get substituted:
+    # <batch_date_time> - date time when batch started
+    # <batch_date> - date when batch started
+    # <batch_time - time when batch started
+    # <date_time> - the datetime just before pushing to S3
+    # <table_name> - the name of the table associated with the S3 body
+    # <uuid> - a universal identifier
+    # '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
+    attr_accessor :s3_key
+
+    # Optional
+    # Applicable only when `s3_bucket` is set
+    # S3 Tags
+    # example: config.s3_tags = { infra_id: '001', source_app: 'kcp-api', source_table: 'meetings' }
+    attr_accessor :s3_tags
+
     # Global list of column names not to include in published SNS messages
     # example: config.ignored_column_names = ['email', 'phone_number']
     attr_accessor :ignored_column_names
@@ -34,12 +82,6 @@ module DbBlaster
     #                                { source_table_name: 'comments', ignored_column_names: ['tags'] }]
     attr_accessor :source_table_options
 
-    # Optional
-    # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
-    # Attributes set here will be included in every published message
-    # example: config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
-    attr_accessor :extra_sns_message_attributes
-
     # Optional
     # db_blaster will select and then publish `batch_size` rows at a time
     # Default value is 100
@@ -52,12 +94,22 @@ module DbBlaster
 
     # Raises error if a required field is not set
     def verify!
-      no_values = REQUIRED_FIELDS.select do |attribute|
+      verify_required
+      verify_either_or
+    end
+
+    def verify_either_or
+      either_or = EITHER_OR_FIELDS.select do |attribute|
         send(attribute).nil? || send(attribute).strip.empty?
       end
-      return if no_values.empty?
+      raise "only one of [#{either_or.join(', ')}] should be set" unless either_or.length == 1
+    end
 
-      raise "missing configuration values for [#{no_values.join(', ')}]"
+    def verify_required
+      no_values = REQUIRED_FIELDS.select do |attribute|
+        send(attribute).nil? || send(attribute).strip.empty?
+      end
+      raise "missing configuration values for [#{no_values.join(', ')}]" unless no_values.empty?
     end
   end
 end

data/lib/db_blaster/publish_source_table.rb

@@ -4,14 +4,15 @@ module DbBlaster
   # Given a `source_table` providing the table name,
   # finds rows in `batch_size` chunks that are published to SNS
   class PublishSourceTable
-    attr_reader :source_table
+    attr_reader :source_table, :batch_start_time
 
-    def initialize(source_table)
+    def initialize(source_table, batch_start_time)
       @source_table = source_table
+      @batch_start_time = batch_start_time
     end
 
-    def self.execute(source_table)
-      new(source_table).execute
+    def self.execute(source_table:, batch_start_time:)
+      new(source_table, batch_start_time).execute
     end
 
     def execute
@@ -20,7 +21,7 @@ module DbBlaster
       # pessimistically lock row for the duration
       source_table.with_lock do
         Finder.find(source_table) do |records|
-          Publisher.publish(source_table, records)
+          BasePublisher.publish(source_table: source_table, records: records, batch_start_time: batch_start_time)
           source_table.update(last_published_updated_at: records.last['updated_at'])
         end
       end

data/lib/db_blaster/s3_key_builder.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Builds the key to be used for the uploaded S3 Object
+  class S3KeyBuilder
+    attr_reader :source_table_name, :batch_start_time
+
+    def initialize(source_table_name, batch_start_time)
+      @source_table_name = source_table_name
+      @batch_start_time = batch_start_time
+    end
+
+    def self.build(source_table_name:, batch_start_time:)
+      new(source_table_name, batch_start_time).build
+    end
+
+    def build
+      key = starting_key
+      substitutions.each do |replace, value|
+        key = key.gsub(replace, value)
+      end
+      key
+    end
+
+    def substitutions
+      date_time = DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
+      date, time = batch_start_time.split('T')
+      { '<batch_date_time>' => batch_start_time,
+        '<batch_date>' => date,
+        '<batch_time>' => time,
+        '<date_time>' => date_time,
+        '<uuid>' => SecureRandom.uuid,
+        '<table_name>' => source_table_name }
+    end
+
+    def starting_key
+      DbBlaster.configuration.s3_key.presence || Configuration::DEFAULT_S3_KEY
+    end
+  end
+end

data/lib/db_blaster/s3_publisher.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-s3'
+
+module DbBlaster
+  # Pushes records to S3
+  class S3Publisher < BasePublisher
+    def publish
+      client.put_object(bucket: DbBlaster.configuration.s3_bucket,
+                        key: S3KeyBuilder.build(source_table_name: source_table.name,
+                                                batch_start_time: batch_start_time),
+                        tagging: tagging,
+                        body: content.to_json)
+    end
+
+    def content
+      return meta_records if DbBlaster.configuration.s3_meta_format == Configuration::INLINE_S3_META_FORMAT
+
+      { meta: meta,
+        records: records }
+    end
+
+    def tagging
+      URI.encode_www_form(tags_hash)
+    end
+
+    def tags_hash
+      @tags_hash ||= { source_table: source_table.name }
+                     .merge(DbBlaster.configuration.s3_tags.presence || {})
+    end
+
+    def meta
+      @meta ||= (DbBlaster.configuration.s3_meta.presence || {}).merge(source_table: source_table.name)
+    end
+
+    def meta_records
+      records.collect { |record| record.merge(meta) }
+    end
+
+    def client
+      @client ||= Aws::S3::Client.new(region: DbBlaster.configuration.aws_region,
+                                      credentials: Aws::Credentials.new(DbBlaster.configuration.aws_access_key,
+                                                                        DbBlaster.configuration.aws_access_secret))
+    end
+  end
+end

data/lib/db_blaster/{publisher.rb → sns_publisher.rb}

@@ -1,20 +1,12 @@
 # frozen_string_literal: true
 
-# publish records to SNS topic
-module DbBlaster
-  # Publishes records to AWS SNS
-  class Publisher
-    attr_reader :source_table, :records
-
-    def initialize(source_table, records)
-      @source_table = source_table
-      @records = records
-    end
+require 'aws-sdk-sns'
 
-    def self.publish(source_table, records)
-      new(source_table, records).publish
-    end
+# frozen_string_literal: true
 
+module DbBlaster
+  # Publishes records to AWS SNS
+  class SnsPublisher < BasePublisher
     def publish
       topic.publish(message_attributes: message_attributes,
                     message: records.to_json)

data/lib/db_blaster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DbBlaster
-  VERSION = '0.1.2'
+  VERSION = '0.1.6'
 end

data/lib/db_blaster.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'aws-sdk-sns'
 require 'db_blaster/version'
 require 'db_blaster/engine'
 require 'db_blaster/one_record_too_large_error'
@@ -8,7 +7,10 @@ require 'db_blaster/available_tables'
 require 'db_blaster/configuration'
 require 'db_blaster/source_table_configuration'
 require 'db_blaster/source_table_configuration_builder'
-require 'db_blaster/publisher'
+require 'db_blaster/base_publisher'
+require 'db_blaster/s3_key_builder'
+require 'db_blaster/s3_publisher'
+require 'db_blaster/sns_publisher'
 require 'db_blaster/publish_source_table'
 require 'db_blaster/chunker'
 require 'db_blaster/finder_sql'

data/lib/generators/db_blaster/install/templates/db_blaster_config.rb

@@ -1,12 +1,39 @@
 # frozen_string_literal: true
 
+# Either `sns_topic` or `s3_bucket` must be set
 DbBlaster.configure do |config|
-  # SNS topic to receive database changes
+  # SNS topic to receive database changes. Either `sns_topic` or `s3_bucket` must be set
   config.sns_topic = 'the-topic'
   config.aws_access_key = 'access-key'
   config.aws_access_secret = 'secret'
   config.aws_region = 'region'
 
+  # Optional
+  # Applicable only when `sns_topic` is set
+  # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
+  # Attributes set here will be included in every published message
+  # config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+
+  # S3 bucket where JSON will be pushed. Either `sns_topic` or `s3_bucket` must be set
+  # config.s3_bucket = 'bucket-name'
+
+  # Optional
+  # Applicable only when `s3_bucket` is set
+  # The S3 key path. The following values will get substituted:
+  # <batch_date_time> - a timestamp signifying the beginning of the batch processing
+  # <batch_date> - a date signifying the beginning of the batch processing
+  # <date_time> - the datetime just before pushing to S3
+  # <table_name> - the name of the table associated with the S3 body
+  # <uuid> - a universal identifier
+  # config.s3_key = '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
+
+  # Optional
+  # Applicable only when `s3_bucket' is set
+  # Extra meta values sent along with each payload
+  # example: config.s3_meta = {'infra_id' => '061'}
+  # The resulting JSON will include the `meta` merged into every record.
+  # config.s3_meta = {'infra_id' => '061'}
+
   # Optional
   # db_blaster will select and then publish `batch_size` rows at a time
   # config.batch_size = 100
@@ -16,11 +43,6 @@ DbBlaster.configure do |config|
   # Default value is 256
   # config.max_message_size_in_kilobytes = 256
 
-  # Optional
-  # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
-  # Attributes set here will be included in every published message
-  # config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
-
   # Global list of column names not to include in published SNS messages
   # example: config.ignored_column_names = ['email', 'phone_number']
   # config.ignored_column_names = ['email', 'phone_number']

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: db_blaster
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.6
 platform: ruby
 authors:
 - Perry Hertler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-12 00:00:00.000000000 Z
+date: 2021-09-03 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
   requirement: !ruby/object:Gem::Requirement
@@ -119,6 +133,7 @@ files:
 - db/migrate/20210727222252_create_source_tables.rb
 - lib/db_blaster.rb
 - lib/db_blaster/available_tables.rb
+- lib/db_blaster/base_publisher.rb
 - lib/db_blaster/chunker.rb
 - lib/db_blaster/configuration.rb
 - lib/db_blaster/engine.rb
@@ -126,8 +141,10 @@ files:
 - lib/db_blaster/finder_sql.rb
 - lib/db_blaster/one_record_too_large_error.rb
 - lib/db_blaster/publish_source_table.rb
-- lib/db_blaster/publisher.rb
 - lib/db_blaster/rspec.rb
+- lib/db_blaster/s3_key_builder.rb
+- lib/db_blaster/s3_publisher.rb
+- lib/db_blaster/sns_publisher.rb
 - lib/db_blaster/source_table_configuration.rb
 - lib/db_blaster/source_table_configuration_builder.rb
 - lib/db_blaster/version.rb