db_blaster 0.1.1 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d612afe45176a37521310c9f694cb2d4b0efe1d42406da82f04bf5fca3187b86
-  data.tar.gz: 25aae3fae57025cfcc72d14e192b21950c73baa804b4270c7de2c1377bc09686
+  metadata.gz: 2de00cf222282fde0e0a3e38078ccc398a80ff19c41ccf7b79ecd043450db6b1
+  data.tar.gz: 66b93222044a44bcd7dd35a6233de5c4839a58d5106e527a7d540be2daee9821
 SHA512:
-  metadata.gz: 28412dbbfae7b89fe57afd4e40fd4bc43bfb2c126a5e3fc67e632d776017f07d2f15d7d70da537344e59aa3021bff30131f905305d00c0784ab156a71185efe9
-  data.tar.gz: 49c856817e7c650bb36b5a74b3395fcdcd075439c2831aebe720eeda381fcc5e36fd08fbacdb1a4c5bd2fde2ec86e647a14a9a1975a023cc40e76019665e7bb0
+  metadata.gz: 7305baf42dff923be85d8105625bf3ccf8200988d7895c80202a32232b062a9c613863a002a7b8e57d7e57a3660725ab4ed6765d2b2d3ba49f3b216da2614dac
+  data.tar.gz: 8652430ce86f2ee275997ec35e52bf2ba99befa4c3a8630b13e0ca57f309d4b0fb013974806e52882bc37bd95a6c4e9e9d4ea51fa5a2096d2767df0d3608b796

data/README.md

@@ -2,15 +2,15 @@
 # DbBlaster
 ![Image of DB to SNS](https://lucid.app/publicSegments/view/c70feed3-2f48-46ee-8734-423474488feb/image.png)
 
-DbBlaster publishes changed database rows to AWS SNS. The first time `DbBlaster::PublishAllJob.perform_later` is ran,
-the entire database will be incrementally published to SNS. Subsequent runs will publish rows whose `updated_at` column
+DbBlaster can either publish changed database rows to AWS SNS or push the changes to S#. The first time `DbBlaster::PublishAllJob.perform_later` is ran,
+the entire database will be incrementally published. Subsequent runs will publish rows whose `updated_at` column
 is more recent than the last run.
 
 Consuming the published messages is functionality not provided by DbBlaster. 
 
 ## Usage
 
-Update `config/initializers/db_blaster_config.rb` with valid AWS credentials, topics, and options.
+Update `config/initializers/db_blaster_config.rb` with valid AWS credentials and options. Either `sns_topic` or `s3_bucket` must be set!
 
 Schedule `DbBlaster::PublishAllJob.perform_later` to run periodically with something
 like [sidekiq-cron](https://github.com/ondrejbartas/sidekiq-cron) or [whenever](https://github.com/javan/whenever)

data/app/jobs/db_blaster/publish_all_job.rb

@@ -13,8 +13,12 @@ module DbBlaster
                          .build_all(DbBlaster.configuration))
 
       DbBlaster::SourceTable.pluck(:id).each do |source_table_id|
-        PublishSourceTableJob.perform_later(source_table_id)
+        PublishSourceTableJob.perform_later(source_table_id, batch_start_time)
       end
     end
+
+    def batch_start_time
+      @batch_start_time ||= DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
+    end
   end
 end

data/app/jobs/db_blaster/publish_source_table_job.rb

@@ -6,11 +6,11 @@ module DbBlaster
   class PublishSourceTableJob < ApplicationJob
     queue_as 'default'
 
-    def perform(source_table_id)
+    def perform(source_table_id, batch_start_time)
       source_table = SourceTable.find_by(id: source_table_id)
       return unless source_table
 
-      PublishSourceTable.execute(source_table)
+      PublishSourceTable.execute(source_table: source_table, batch_start_time: batch_start_time)
     end
   end
 end

data/lib/db_blaster/base_publisher.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# publish records to SNS topic
+module DbBlaster
+  # Base class for publishing
+  class BasePublisher
+    attr_reader :source_table, :records, :batch_start_time
+
+    def initialize(source_table, records, batch_start_time)
+      @source_table = source_table
+      @records = records
+      @batch_start_time = batch_start_time
+    end
+
+    def self.publish(source_table:, records:, batch_start_time:)
+      publisher_class =
+        if DbBlaster.configuration.sns_topic
+          SnsPublisher
+        else
+          S3Publisher
+        end
+      publisher_class.new(source_table, records, batch_start_time).publish
+    end
+
+    def publish
+      raise NotImplementedError
+    end
+  end
+end

data/lib/db_blaster/configuration.rb

@@ -2,17 +2,55 @@
 
 module DbBlaster
   # Configuration class for providing credentials, topics, and customizations.
+  # Either the `sns_topic` or `s3_bucket' must be set
   class Configuration
     DEFAULT_BATCH_SIZE = 100
     DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
+    DEFAULT_S3_KEY = '<batch_date>/<batch_time>/db_blaster/<table_name>/<uuid>.json'
+    DEFAULT_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%LZ'
 
     # The required configuration fields
-    REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region sns_topic].freeze
+    REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region].freeze
+
+    # exactly one of these fields needs to be set
+    EITHER_OR_FIELDS = %i[sns_topic s3_bucket].freeze
 
     # The topic to which messages will be published
     attr_accessor :sns_topic
+    # The s3 bucket name
+    attr_accessor :s3_bucket
     attr_accessor :aws_access_key, :aws_access_secret, :aws_region
 
+    # Optional
+    # Applicable only when `sns_topic` is set
+    # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
+    # Attributes set here will be included in every published message
+    # example: config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+    attr_accessor :extra_sns_message_attributes
+
+    # Optional
+    # Applicable only when `s3_bucket' is set
+    # The value set here will be included in every payload pushed to S3
+    # example: config.s3_meta = {'infra_id' => '061', 'source_app' => 'kcp-api'}}
+    # The resulting JSON will include the `meta` merged into every record
+    attr_accessor :s3_meta
+
+    # Optional
+    # Applicable only when `s3_bucket` is set
+    # The S3 key. The following values will get substituted:
+    # <batch_timestamp> - a timestamp signifying the beginning of the batch processing
+    # <timestamp> - the current time
+    # <table_name> - the name of the table associated with the S3 body
+    # <uuid> - a universal identifier
+    # '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
+    attr_accessor :s3_key
+
+    # Optional
+    # Applicable only when `s3_bucket` is set
+    # S3 Tags
+    # example: config.s3_tags = { infra_id: '001', source_app: 'kcp-api', source_table: 'meetings' }
+    attr_accessor :s3_tags
+
     # Global list of column names not to include in published SNS messages
     # example: config.ignored_column_names = ['email', 'phone_number']
     attr_accessor :ignored_column_names
@@ -22,6 +60,11 @@ module DbBlaster
     # example: config.only_source_tables = ['posts', 'tags', 'comments']
     attr_accessor :only_source_tables
 
+    # Optional
+    # If set, ignore source tables specified.
+    # example: config.ignore_source_tables = ['active_storage_blobs']
+    attr_accessor :ignore_source_tables
+
     # Optional
     # Customize batch_size and/or ignored_columns
     # example:
@@ -29,12 +72,6 @@ module DbBlaster
     #                                { source_table_name: 'comments', ignored_column_names: ['tags'] }]
     attr_accessor :source_table_options
 
-    # Optional
-    # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
-    # Attributes set here will be included in every published message
-    # example: config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
-    attr_accessor :extra_sns_message_attributes
-
     # Optional
     # db_blaster will select and then publish `batch_size` rows at a time
     # Default value is 100
@@ -47,12 +84,22 @@ module DbBlaster
 
     # Raises error if a required field is not set
     def verify!
-      no_values = REQUIRED_FIELDS.select do |attribute|
+      verify_required
+      verify_either_or
+    end
+
+    def verify_either_or
+      either_or = EITHER_OR_FIELDS.select do |attribute|
         send(attribute).nil? || send(attribute).strip.empty?
       end
-      return if no_values.empty?
+      raise "only one of [#{either_or.join(', ')}] should be set" unless either_or.length == 1
+    end
 
-      raise "missing configuration values for [#{no_values.join(', ')}]"
+    def verify_required
+      no_values = REQUIRED_FIELDS.select do |attribute|
+        send(attribute).nil? || send(attribute).strip.empty?
+      end
+      raise "missing configuration values for [#{no_values.join(', ')}]" unless no_values.empty?
     end
   end
 end

data/lib/db_blaster/publish_source_table.rb

@@ -4,14 +4,15 @@ module DbBlaster
   # Given a `source_table` providing the table name,
   # finds rows in `batch_size` chunks that are published to SNS
   class PublishSourceTable
-    attr_reader :source_table
+    attr_reader :source_table, :batch_start_time
 
-    def initialize(source_table)
+    def initialize(source_table, batch_start_time)
       @source_table = source_table
+      @batch_start_time = batch_start_time
     end
 
-    def self.execute(source_table)
-      new(source_table).execute
+    def self.execute(source_table:, batch_start_time:)
+      new(source_table, batch_start_time).execute
     end
 
     def execute
@@ -20,7 +21,7 @@ module DbBlaster
       # pessimistically lock row for the duration
       source_table.with_lock do
         Finder.find(source_table) do |records|
-          Publisher.publish(source_table, records)
+          BasePublisher.publish(source_table: source_table, records: records, batch_start_time: batch_start_time)
           source_table.update(last_published_updated_at: records.last['updated_at'])
         end
       end

data/lib/db_blaster/s3_key_builder.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Builds the key to be used for the uploaded S3 Object
+  class S3KeyBuilder
+    attr_reader :source_table_name, :batch_start_time
+
+    def initialize(source_table_name, batch_start_time)
+      @source_table_name = source_table_name
+      @batch_start_time = batch_start_time
+    end
+
+    def self.build(source_table_name:, batch_start_time:)
+      new(source_table_name, batch_start_time).build
+    end
+
+    def build
+      key = starting_key
+      substitutions.each do |replace, value|
+        key = key.gsub(replace, value)
+      end
+      key
+    end
+
+    def substitutions
+      date, time = batch_start_time.split('T')
+      { '<batch_date_time>' => batch_start_time,
+        '<batch_date>' => date,
+        '<batch_time>' => time,
+        '<uuid>' => SecureRandom.uuid,
+        '<table_name>' => source_table_name }
+    end
+
+    def starting_key
+      DbBlaster.configuration.s3_key.presence || Configuration::DEFAULT_S3_KEY
+    end
+  end
+end

data/lib/db_blaster/s3_publisher.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-s3'
+
+module DbBlaster
+  # Pushes records to S3
+  class S3Publisher < BasePublisher
+    def publish
+      client.put_object(bucket: DbBlaster.configuration.s3_bucket,
+                        key: S3KeyBuilder.build(source_table_name: source_table.name,
+                                                batch_start_time: batch_start_time),
+                        tagging: tagging,
+                        body: content.to_json)
+    end
+
+    def content
+      meta_records
+    end
+
+    def tagging
+      URI.encode_www_form(tags_hash)
+    end
+
+    def tags_hash
+      @tags_hash ||= { source_table: source_table.name }
+                     .merge(DbBlaster.configuration.s3_tags.presence || {})
+    end
+
+    def meta
+      @meta ||= (DbBlaster.configuration.s3_meta.presence || {}).merge(source_table: source_table.name)
+    end
+
+    def meta_records
+      records.collect { |record| record.merge(meta) }
+    end
+
+    def client
+      @client ||= Aws::S3::Client.new(region: DbBlaster.configuration.aws_region,
+                                      credentials: Aws::Credentials.new(DbBlaster.configuration.aws_access_key,
+                                                                        DbBlaster.configuration.aws_access_secret))
+    end
+  end
+end

data/lib/db_blaster/{publisher.rb → sns_publisher.rb}

@@ -1,20 +1,12 @@
 # frozen_string_literal: true
 
-# publish records to SNS topic
-module DbBlaster
-  # Publishes records to AWS SNS
-  class Publisher
-    attr_reader :source_table, :records
-
-    def initialize(source_table, records)
-      @source_table = source_table
-      @records = records
-    end
+require 'aws-sdk-sns'
 
-    def self.publish(source_table, records)
-      new(source_table, records).publish
-    end
+# frozen_string_literal: true
 
+module DbBlaster
+  # Publishes records to AWS SNS
+  class SnsPublisher < BasePublisher
     def publish
       topic.publish(message_attributes: message_attributes,
                     message: records.to_json)

data/lib/db_blaster/source_table_configuration_builder.rb

@@ -45,11 +45,12 @@ module DbBlaster
     end
 
     def table_names_for_configuration
-      if configuration.only_source_tables&.length&.positive?
-        available_tables & configuration.only_source_tables
-      else
-        available_tables
-      end
+      table_names = if configuration.only_source_tables&.length&.positive?
+                      available_tables & configuration.only_source_tables
+                    else
+                      available_tables
+                    end
+      table_names - (configuration.ignore_source_tables || [])
     end
   end
 end

data/lib/db_blaster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DbBlaster
-  VERSION = '0.1.1'
+  VERSION = '0.1.5'
 end

data/lib/db_blaster.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'aws-sdk-sns'
 require 'db_blaster/version'
 require 'db_blaster/engine'
 require 'db_blaster/one_record_too_large_error'
@@ -8,7 +7,10 @@ require 'db_blaster/available_tables'
 require 'db_blaster/configuration'
 require 'db_blaster/source_table_configuration'
 require 'db_blaster/source_table_configuration_builder'
-require 'db_blaster/publisher'
+require 'db_blaster/base_publisher'
+require 'db_blaster/s3_key_builder'
+require 'db_blaster/s3_publisher'
+require 'db_blaster/sns_publisher'
 require 'db_blaster/publish_source_table'
 require 'db_blaster/chunker'
 require 'db_blaster/finder_sql'

data/lib/generators/db_blaster/install/templates/db_blaster_config.rb

@@ -1,12 +1,40 @@
 # frozen_string_literal: true
 
+# Either `sns_topic` or `s3_bucket` must be set
 DbBlaster.configure do |config|
-  # SNS topic to receive database changes
+  # SNS topic to receive database changes. Either `sns_topic` or `s3_bucket` must be set
   config.sns_topic = 'the-topic'
   config.aws_access_key = 'access-key'
   config.aws_access_secret = 'secret'
   config.aws_region = 'region'
 
+  # Optional
+  # Applicable only when `sns_topic` is set
+  # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
+  # Attributes set here will be included in every published message
+  # config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+
+  # S3 bucket where JSON will be pushed. Either `sns_topic` or `s3_bucket` must be set
+  # config.s3_bucket = 'bucket-name'
+
+  # Optional
+  # Applicable only when `s3_bucket` is set
+  # The S3 key path. The following values will get substituted:
+  # <batch_date_time> - a timestamp signifying the beginning of the batch processing
+  # <batch_date> - a date signifying the beginning of the batch processing
+  # <batch_time> - a time signifying the beginning of the batch processing
+  # <timestamp> - the current time
+  # <table_name> - the name of the table associated with the S3 body
+  # <uuid> - a universal identifier
+  # config.s3_key = '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
+
+  # Optional
+  # Applicable only when `s3_bucket' is set
+  # Extra meta values sent along with each payload
+  # example: config.s3_meta = {'infra_id' => '061'}
+  # The resulting JSON will include the `meta` merged into every record.
+  # config.s3_meta = {'infra_id' => '061'}
+
   # Optional
   # db_blaster will select and then publish `batch_size` rows at a time
   # config.batch_size = 100
@@ -14,12 +42,7 @@ DbBlaster.configure do |config|
   # Optional
   # db_blaster will publish no messages larger than this value
   # Default value is 256
-  # attr_accessor :max_message_size_in_kilobytes
-
-  # Optional
-  # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
-  # Attributes set here will be included in every published message
-  # config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+  # config.max_message_size_in_kilobytes = 256
 
   # Global list of column names not to include in published SNS messages
   # example: config.ignored_column_names = ['email', 'phone_number']
@@ -29,6 +52,10 @@ DbBlaster.configure do |config|
   # If set, only publish tables specified.
   # config.only_source_tables = ['posts', 'tags', 'comments']
 
+  # Optional
+  # If set, ignore source tables specified.
+  # config.ignore_source_tables = ['active_storage_blobs']
+
   # Optional
   # Customize batch_size and/or ignored_columns
   # example:

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: db_blaster
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - Perry Hertler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-11 00:00:00.000000000 Z
+date: 2021-09-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
   requirement: !ruby/object:Gem::Requirement
@@ -119,6 +133,7 @@ files:
 - db/migrate/20210727222252_create_source_tables.rb
 - lib/db_blaster.rb
 - lib/db_blaster/available_tables.rb
+- lib/db_blaster/base_publisher.rb
 - lib/db_blaster/chunker.rb
 - lib/db_blaster/configuration.rb
 - lib/db_blaster/engine.rb
@@ -126,8 +141,10 @@ files:
 - lib/db_blaster/finder_sql.rb
 - lib/db_blaster/one_record_too_large_error.rb
 - lib/db_blaster/publish_source_table.rb
-- lib/db_blaster/publisher.rb
 - lib/db_blaster/rspec.rb
+- lib/db_blaster/s3_key_builder.rb
+- lib/db_blaster/s3_publisher.rb
+- lib/db_blaster/sns_publisher.rb
 - lib/db_blaster/source_table_configuration.rb
 - lib/db_blaster/source_table_configuration_builder.rb
 - lib/db_blaster/version.rb