db_blaster 0.1.0 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8d31a519addfa74e6a2ca3c2d41f58b05e2b4551802948b5a4dc0fe20278dca
-  data.tar.gz: 33a1e76574a2b8e70c681beebf544af587ff143252b08d0dd1a8dbc0b3615864
+  metadata.gz: 9313d3abe0bd4dcaa756dab6b07b1d65eb6438b435c914d18f9d38964f08a29d
+  data.tar.gz: c694940facb41fb3656d011baba720ffb5f66cf124472f945d7de45661ba90b5
 SHA512:
-  metadata.gz: 378969c9223acf49c50a055b566b1f3f2bff3a0b032bbc9e1aebbb64b5031c9d1a2af6c539d55efc7e90a8254d0ca4cf413be762ab41dbdd1fd4e01efd723d33
-  data.tar.gz: afabd1c77490c8c80bdd1d0a49965026ae69a77d727ffe7da475c5683391dafc4cd9f91d7880ecc5c0818a4c5afeacd4403844a198f932c0c4d448b0c53cb2dc
+  metadata.gz: c045adce6f5a4fb70be59194f6f4630879761011be805dbed09c9be225a415a807ffe60a2b33ccf5361e49ce595b87250f8b0e67f370c29d867254928a36e41f
+  data.tar.gz: 4b509898ced3205b588248700a75a375ddc6c614611298f0af347ef1a45b35f4c3d1adc9ae0a70d357c56050f47a7225d454564c22a350a21cec45da0effe322

data/README.md

@@ -2,15 +2,15 @@
 # DbBlaster
 ![Image of DB to SNS](https://lucid.app/publicSegments/view/c70feed3-2f48-46ee-8734-423474488feb/image.png)
 
-DbBlaster publishes changed database rows to AWS SNS. The first time `DbBlaster::PublishAllJob.perform_later` is ran,
-the entire database will be incrementally published to SNS. Subsequent runs will publish rows whose `updated_at` column
+DbBlaster can either publish changed database rows to AWS SNS or push the changes to S#. The first time `DbBlaster::PublishAllJob.perform_later` is ran,
+the entire database will be incrementally published. Subsequent runs will publish rows whose `updated_at` column
 is more recent than the last run.
 
 Consuming the published messages is functionality not provided by DbBlaster. 
 
 ## Usage
 
-Update `config/initializers/db_blaster_config.rb` with valid AWS credentials, topics, and options.
+Update `config/initializers/db_blaster_config.rb` with valid AWS credentials and options. Either `sns_topic` or `s3_bucket` must be set!
 
 Schedule `DbBlaster::PublishAllJob.perform_later` to run periodically with something
 like [sidekiq-cron](https://github.com/ondrejbartas/sidekiq-cron) or [whenever](https://github.com/javan/whenever)

data/app/jobs/db_blaster/publish_all_job.rb

@@ -13,8 +13,12 @@ module DbBlaster
                          .build_all(DbBlaster.configuration))
 
       DbBlaster::SourceTable.pluck(:id).each do |source_table_id|
-        PublishSourceTableJob.perform_later(source_table_id)
+        PublishSourceTableJob.perform_later(source_table_id, batch_start_time)
       end
     end
+
+    def batch_start_time
+      @batch_start_time ||= DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
+    end
   end
 end

data/app/jobs/db_blaster/publish_source_table_job.rb

@@ -6,11 +6,11 @@ module DbBlaster
   class PublishSourceTableJob < ApplicationJob
     queue_as 'default'
 
-    def perform(source_table_id)
+    def perform(source_table_id, batch_start_time)
       source_table = SourceTable.find_by(id: source_table_id)
       return unless source_table
 
-      PublishSourceTable.execute(source_table)
+      PublishSourceTable.execute(source_table: source_table, batch_start_time: batch_start_time)
     end
   end
 end

data/config/brakeman.ignore

@@ -3,24 +3,24 @@
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
-      "fingerprint": "6f4d3da0707c3f5f5c5bf5a002a254fee246210248aafa655cb2f15adfb47aa7",
+      "fingerprint": "3fef6d99f896e29ef9346d81a1557bd3819fbc762b2aa91d44dfa25a5c095485",
       "check_name": "SQL",
       "message": "Possible SQL injection",
       "file": "lib/db_blaster/finder.rb",
-      "line": 38,
+      "line": 39,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "ActiveRecord::Base.connection.execute(\"#{select_sql} OFFSET #{offset}\")",
+      "code": "ActiveRecord::Base.connection.execute(\"#{FinderSql.sql_for_source_table(source_table)} OFFSET #{offset}\")",
       "render_path": null,
       "location": {
         "type": "method",
         "class": "DbBlaster::Finder",
         "method": "find_records_in_batches"
       },
-      "user_input": "select_sql",
+      "user_input": "FinderSql.sql_for_source_table(source_table)",
       "confidence": "Medium",
-      "note": "No SQL injection can occur"
+      "note": "no sql injection"
     }
   ],
-  "updated": "2021-08-09 11:03:06 -0600",
+  "updated": "2021-08-11 13:14:00 -0600",
   "brakeman_version": "5.1.1"
 }

data/lib/db_blaster/base_publisher.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# publish records to SNS topic
+module DbBlaster
+  # Base class for publishing
+  class BasePublisher
+    attr_reader :source_table, :records, :batch_start_time
+
+    def initialize(source_table, records, batch_start_time)
+      @source_table = source_table
+      @records = records
+      @batch_start_time = batch_start_time
+    end
+
+    def self.publish(source_table:, records:, batch_start_time:)
+      publisher_class =
+        if DbBlaster.configuration.sns_topic
+          SnsPublisher
+        else
+          S3Publisher
+        end
+      publisher_class.new(source_table, records, batch_start_time).publish
+    end
+
+    def publish
+      raise NotImplementedError
+    end
+  end
+end

data/lib/db_blaster/configuration.rb

@@ -2,17 +2,56 @@
 
 module DbBlaster
   # Configuration class for providing credentials, topics, and customizations.
+  # Either the `sns_topic` or `s3_bucket' must be set
   class Configuration
     DEFAULT_BATCH_SIZE = 100
     DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
+    DEFAULT_S3_KEY = '<batch_date>/<batch_time>/db_blaster/<table_name>/<uuid>.json'
+    DEFAULT_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%LZ'
 
     # The required configuration fields
-    REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region sns_topic].freeze
+    REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region].freeze
+
+    # exactly one of these fields needs to be set
+    EITHER_OR_FIELDS = %i[sns_topic s3_bucket].freeze
 
     # The topic to which messages will be published
     attr_accessor :sns_topic
+    # The s3 bucket name
+    attr_accessor :s3_bucket
     attr_accessor :aws_access_key, :aws_access_secret, :aws_region
 
+    # Optional
+    # Applicable only when `sns_topic` is set
+    # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
+    # Attributes set here will be included in every published message
+    # example: config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+    attr_accessor :extra_sns_message_attributes
+
+    # Optional
+    # Applicable only when `s3_bucket' is set
+    # The value set here will be included in every payload pushed to S3
+    # example: config.s3_meta = {'infra_id' => '061', 'source_app' => 'kcp-api'}}
+    # The resulting JSON:
+    # {"meta" : {"infra_id" : "061", "src_app" : "kcp-api", "src_table" : "the-table"}, "records" : [] }
+    attr_accessor :s3_meta
+
+    # Optional
+    # Applicable only when `s3_bucket` is set
+    # The S3 key. The following values will get substituted:
+    # <batch_timestamp> - a timestamp signifying the beginning of the batch processing
+    # <timestamp> - the current time
+    # <table_name> - the name of the table associated with the S3 body
+    # <uuid> - a universal identifier
+    # '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
+    attr_accessor :s3_key
+
+    # Optional
+    # Applicable only when `s3_bucket` is set
+    # S3 Tags
+    # example: config.s3_tags = { infra_id: '001', source_app: 'kcp-api', source_table: 'meetings' }
+    attr_accessor :s3_tags
+
     # Global list of column names not to include in published SNS messages
     # example: config.ignored_column_names = ['email', 'phone_number']
     attr_accessor :ignored_column_names
@@ -22,6 +61,11 @@ module DbBlaster
     # example: config.only_source_tables = ['posts', 'tags', 'comments']
     attr_accessor :only_source_tables
 
+    # Optional
+    # If set, ignore source tables specified.
+    # example: config.ignore_source_tables = ['active_storage_blobs']
+    attr_accessor :ignore_source_tables
+
     # Optional
     # Customize batch_size and/or ignored_columns
     # example:
@@ -29,12 +73,6 @@ module DbBlaster
     #                                { source_table_name: 'comments', ignored_column_names: ['tags'] }]
     attr_accessor :source_table_options
 
-    # Optional
-    # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
-    # Attributes set here will be included in every published message
-    # example: config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
-    attr_accessor :extra_sns_message_attributes
-
     # Optional
     # db_blaster will select and then publish `batch_size` rows at a time
     # Default value is 100
@@ -47,12 +85,22 @@ module DbBlaster
 
     # Raises error if a required field is not set
     def verify!
-      no_values = REQUIRED_FIELDS.select do |attribute|
+      verify_required
+      verify_either_or
+    end
+
+    def verify_either_or
+      either_or = EITHER_OR_FIELDS.select do |attribute|
         send(attribute).nil? || send(attribute).strip.empty?
       end
-      return if no_values.empty?
+      raise "only one of [#{either_or.join(', ')}] should be set" unless either_or.length == 1
+    end
 
-      raise "missing configuration values for [#{no_values.join(', ')}]"
+    def verify_required
+      no_values = REQUIRED_FIELDS.select do |attribute|
+        send(attribute).nil? || send(attribute).strip.empty?
+      end
+      raise "missing configuration values for [#{no_values.join(', ')}]" unless no_values.empty?
     end
   end
 end

data/lib/db_blaster/finder.rb

@@ -12,7 +12,7 @@ module DbBlaster
       @offset = 0
     end
 
-    delegate :batch_size, :name, :last_published_updated_at, to: :source_table, prefix: true
+    delegate :batch_size, :name, to: :source_table, prefix: true
 
     def self.find(source_table, &block)
       new(source_table, &block).find
@@ -34,6 +34,7 @@ module DbBlaster
     private
 
     def find_records_in_batches
+      select_sql = FinderSql.sql_for_source_table(source_table)
       loop do
         result = ActiveRecord::Base.connection.execute("#{select_sql} OFFSET #{offset}")
         yield(result)
@@ -54,17 +55,5 @@ module DbBlaster
     def invalid_source_table_message
       "source_table.name: '#{source_table_name}' does not exist!"
     end
-
-    def select_sql
-      "SELECT * FROM #{source_table_name} #{where} ORDER BY updated_at ASC LIMIT #{source_table_batch_size}"
-    end
-
-    def where
-      return '' unless source_table_last_published_updated_at
-
-      ActiveRecord::Base.sanitize_sql_for_conditions(
-        ['WHERE updated_at >= :updated_at', { updated_at: source_table_last_published_updated_at.to_s(:db) }]
-      )
-    end
   end
 end

data/lib/db_blaster/finder_sql.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Creates the SQL needed to find records for the provided source_table
+  class FinderSql
+    attr_reader :source_table
+
+    def initialize(source_table)
+      @source_table = source_table
+    end
+
+    def self.sql_for_source_table(source_table)
+      new(source_table).select_sql
+    end
+
+    def select_sql
+      "SELECT * FROM #{source_table.name} #{where} ORDER BY updated_at ASC LIMIT #{source_table.batch_size}"
+    end
+
+    def where
+      return '' unless from_updated_at
+
+      ActiveRecord::Base.sanitize_sql_for_conditions(
+        ['WHERE updated_at >= :updated_at', { updated_at: from_updated_at.to_s(:db) }]
+      )
+    end
+
+    def from_updated_at
+      @from_updated_at ||= source_table.last_published_updated_at
+    end
+  end
+end

data/lib/db_blaster/publish_source_table.rb

@@ -4,14 +4,15 @@ module DbBlaster
   # Given a `source_table` providing the table name,
   # finds rows in `batch_size` chunks that are published to SNS
   class PublishSourceTable
-    attr_reader :source_table
+    attr_reader :source_table, :batch_start_time
 
-    def initialize(source_table)
+    def initialize(source_table, batch_start_time)
       @source_table = source_table
+      @batch_start_time = batch_start_time
     end
 
-    def self.execute(source_table)
-      new(source_table).execute
+    def self.execute(source_table:, batch_start_time:)
+      new(source_table, batch_start_time).execute
     end
 
     def execute
@@ -20,7 +21,7 @@ module DbBlaster
       # pessimistically lock row for the duration
       source_table.with_lock do
         Finder.find(source_table) do |records|
-          Publisher.publish(source_table, records)
+          BasePublisher.publish(source_table: source_table, records: records, batch_start_time: batch_start_time)
           source_table.update(last_published_updated_at: records.last['updated_at'])
         end
       end

data/lib/db_blaster/s3_key_builder.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Builds the key to be used for the uploaded S3 Object
+  class S3KeyBuilder
+    attr_reader :source_table_name, :batch_start_time
+
+    def initialize(source_table_name, batch_start_time)
+      @source_table_name = source_table_name
+      @batch_start_time = batch_start_time
+    end
+
+    def self.build(source_table_name:, batch_start_time:)
+      new(source_table_name, batch_start_time).build
+    end
+
+    def build
+      key = starting_key
+      substitutions.each do |replace, value|
+        key = key.gsub(replace, value)
+      end
+      key
+    end
+
+    def substitutions
+      date, time = batch_start_time.split('T')
+      { '<batch_date_time>' => batch_start_time,
+        '<batch_date>' => date,
+        '<batch_time>' => time,
+        '<uuid>' => SecureRandom.uuid,
+        '<table_name>' => source_table_name }
+    end
+
+    def starting_key
+      DbBlaster.configuration.s3_key.presence || Configuration::DEFAULT_S3_KEY
+    end
+  end
+end

data/lib/db_blaster/s3_publisher.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-s3'
+
+module DbBlaster
+  # Pushes records to S3
+  class S3Publisher < BasePublisher
+    def publish
+      client.put_object(bucket: DbBlaster.configuration.s3_bucket,
+                        key: S3KeyBuilder.build(source_table_name: source_table.name,
+                                                batch_start_time: batch_start_time),
+                        tagging: tagging,
+                        body: content.to_json)
+    end
+
+    def content
+      { meta: meta,
+        records: records }
+    end
+
+    def tagging
+      URI.encode_www_form(tags_hash)
+    end
+
+    def tags_hash
+      @tags_hash ||= { source_table: source_table.name }
+                     .merge(DbBlaster.configuration.s3_tags.presence || {})
+    end
+
+    def meta
+      (DbBlaster.configuration.s3_meta.presence || {}).merge(source_table: source_table.name)
+    end
+
+    def client
+      @client ||= Aws::S3::Client.new(region: DbBlaster.configuration.aws_region,
+                                      credentials: Aws::Credentials.new(DbBlaster.configuration.aws_access_key,
+                                                                        DbBlaster.configuration.aws_access_secret))
+    end
+  end
+end

data/lib/db_blaster/{publisher.rb → sns_publisher.rb}

@@ -1,20 +1,12 @@
 # frozen_string_literal: true
 
-# publish records to SNS topic
-module DbBlaster
-  # Publishes records to AWS SNS
-  class Publisher
-    attr_reader :source_table, :records
-
-    def initialize(source_table, records)
-      @source_table = source_table
-      @records = records
-    end
+require 'aws-sdk-sns'
 
-    def self.publish(source_table, records)
-      new(source_table, records).publish
-    end
+# frozen_string_literal: true
 
+module DbBlaster
+  # Publishes records to AWS SNS
+  class SnsPublisher < BasePublisher
     def publish
       topic.publish(message_attributes: message_attributes,
                     message: records.to_json)

data/lib/db_blaster/source_table_configuration_builder.rb

@@ -45,11 +45,12 @@ module DbBlaster
     end
 
     def table_names_for_configuration
-      if configuration.only_source_tables&.length&.positive?
-        available_tables & configuration.only_source_tables
-      else
-        available_tables
-      end
+      table_names = if configuration.only_source_tables&.length&.positive?
+                      available_tables & configuration.only_source_tables
+                    else
+                      available_tables
+                    end
+      table_names - (configuration.ignore_source_tables || [])
     end
   end
 end

data/lib/db_blaster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DbBlaster
-  VERSION = '0.1.0'
+  VERSION = '0.1.4'
 end

data/lib/db_blaster.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'aws-sdk-sns'
 require 'db_blaster/version'
 require 'db_blaster/engine'
 require 'db_blaster/one_record_too_large_error'
@@ -8,9 +7,13 @@ require 'db_blaster/available_tables'
 require 'db_blaster/configuration'
 require 'db_blaster/source_table_configuration'
 require 'db_blaster/source_table_configuration_builder'
-require 'db_blaster/publisher'
+require 'db_blaster/base_publisher'
+require 'db_blaster/s3_key_builder'
+require 'db_blaster/s3_publisher'
+require 'db_blaster/sns_publisher'
 require 'db_blaster/publish_source_table'
 require 'db_blaster/chunker'
+require 'db_blaster/finder_sql'
 require 'db_blaster/finder'
 
 # Top-level module that serves as an entry point

data/lib/generators/db_blaster/install/templates/db_blaster_config.rb

@@ -1,12 +1,41 @@
 # frozen_string_literal: true
 
+# Either `sns_topic` or `s3_bucket` must be set
 DbBlaster.configure do |config|
-  # SNS topic to receive database changes
+  # SNS topic to receive database changes. Either `sns_topic` or `s3_bucket` must be set
   config.sns_topic = 'the-topic'
   config.aws_access_key = 'access-key'
   config.aws_access_secret = 'secret'
   config.aws_region = 'region'
 
+  # Optional
+  # Applicable only when `sns_topic` is set
+  # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
+  # Attributes set here will be included in every published message
+  # config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+
+  # S3 bucket where JSON will be pushed. Either `sns_topic` or `s3_bucket` must be set
+  # config.s3_bucket = 'bucket-name'
+
+  # Optional
+  # Applicable only when `s3_bucket` is set
+  # The S3 key path. The following values will get substituted:
+  # <batch_date_time> - a timestamp signifying the beginning of the batch processing
+  # <batch_date> - a date signifying the beginning of the batch processing
+  # <batch_time> - a time signifying the beginning of the batch processing
+  # <timestamp> - the current time
+  # <table_name> - the name of the table associated with the S3 body
+  # <uuid> - a universal identifier
+  # config.s3_key = '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
+
+  # Optional
+  # Applicable only when `s3_bucket' is set
+  # Extra meta values sent along with each payload
+  # example: config.s3_meta = {'infra_id' => '061'}
+  # The resulting JSON:
+  # {"meta" : {"infra_id" : "061", "source_app" : "kcp-api", "src_table" : "the-table"}, "records" : [] }
+  # config.s3_meta = {'infra_id' => '061'}
+
   # Optional
   # db_blaster will select and then publish `batch_size` rows at a time
   # config.batch_size = 100
@@ -14,12 +43,7 @@ DbBlaster.configure do |config|
   # Optional
   # db_blaster will publish no messages larger than this value
   # Default value is 256
-  # attr_accessor :max_message_size_in_kilobytes
-
-  # Optional
-  # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
-  # Attributes set here will be included in every published message
-  # config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+  # config.max_message_size_in_kilobytes = 256
 
   # Global list of column names not to include in published SNS messages
   # example: config.ignored_column_names = ['email', 'phone_number']
@@ -29,6 +53,10 @@ DbBlaster.configure do |config|
   # If set, only publish tables specified.
   # config.only_source_tables = ['posts', 'tags', 'comments']
 
+  # Optional
+  # If set, ignore source tables specified.
+  # config.ignore_source_tables = ['active_storage_blobs']
+
   # Optional
   # Customize batch_size and/or ignored_columns
   # example:

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: db_blaster
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.4
 platform: ruby
 authors:
 - Perry Hertler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-09 00:00:00.000000000 Z
+date: 2021-09-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
   requirement: !ruby/object:Gem::Requirement
@@ -119,14 +133,18 @@ files:
 - db/migrate/20210727222252_create_source_tables.rb
 - lib/db_blaster.rb
 - lib/db_blaster/available_tables.rb
+- lib/db_blaster/base_publisher.rb
 - lib/db_blaster/chunker.rb
 - lib/db_blaster/configuration.rb
 - lib/db_blaster/engine.rb
 - lib/db_blaster/finder.rb
+- lib/db_blaster/finder_sql.rb
 - lib/db_blaster/one_record_too_large_error.rb
 - lib/db_blaster/publish_source_table.rb
-- lib/db_blaster/publisher.rb
 - lib/db_blaster/rspec.rb
+- lib/db_blaster/s3_key_builder.rb
+- lib/db_blaster/s3_publisher.rb
+- lib/db_blaster/sns_publisher.rb
 - lib/db_blaster/source_table_configuration.rb
 - lib/db_blaster/source_table_configuration_builder.rb
 - lib/db_blaster/version.rb