db_blaster 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e8d31a519addfa74e6a2ca3c2d41f58b05e2b4551802948b5a4dc0fe20278dca
+  data.tar.gz: 33a1e76574a2b8e70c681beebf544af587ff143252b08d0dd1a8dbc0b3615864
+SHA512:
+  metadata.gz: 378969c9223acf49c50a055b566b1f3f2bff3a0b032bbc9e1aebbb64b5031c9d1a2af6c539d55efc7e90a8254d0ca4cf413be762ab41dbdd1fd4e01efd723d33
+  data.tar.gz: afabd1c77490c8c80bdd1d0a49965026ae69a77d727ffe7da475c5683391dafc4cd9f91d7880ecc5c0818a4c5afeacd4403844a198f932c0c4d448b0c53cb2dc

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2021 Perry Hertler
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,46 @@
+![Coverage Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/perryqh/be2fa5413124206272dbc700f3201f5a/raw/db_blaster__master.json)
+# DbBlaster
+![Image of DB to SNS](https://lucid.app/publicSegments/view/c70feed3-2f48-46ee-8734-423474488feb/image.png)
+
+DbBlaster publishes changed database rows to AWS SNS. The first time `DbBlaster::PublishAllJob.perform_later` is ran,
+the entire database will be incrementally published to SNS. Subsequent runs will publish rows whose `updated_at` column
+is more recent than the last run.
+
+Consuming the published messages is functionality not provided by DbBlaster. 
+
+## Usage
+
+Update `config/initializers/db_blaster_config.rb` with valid AWS credentials, topics, and options.
+
+Schedule `DbBlaster::PublishAllJob.perform_later` to run periodically with something
+like [sidekiq-cron](https://github.com/ondrejbartas/sidekiq-cron) or [whenever](https://github.com/javan/whenever)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'db_blaster'
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+Install Migrations:
+
+```bash
+$ rake db_blaster:install:migrations && rake db:migrate
+```
+
+Copy sample config file to rails project:
+
+```bash
+rails g db_blaster:install 
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+APP_RAKEFILE = File.expand_path('spec/dummy/Rakefile', __dir__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+require 'bundler/gem_tasks'
+
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+
+  task default: :spec
+rescue LoadError
+  # no rspec available
+end

data/app/assets/config/db_blaster_manifest.js

@@ -0,0 +1 @@
+//= link_directory ../stylesheets/db_blaster .css

data/app/assets/stylesheets/db_blaster/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/db_blaster/application_controller.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/helpers/db_blaster/application_helper.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Helper methods for views
+  module ApplicationHelper
+  end
+end

data/app/jobs/db_blaster/application_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Base ActiveJob
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/db_blaster/publish_all_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# ActiveJob
+# 1) sync configuration with DbBlaster::SourceTable
+# 2) Enqueue PublishSourceTableJob for every source_table
+module DbBlaster
+  # Enqueues PublishSourceTableJob for every source-table
+  class PublishAllJob < ApplicationJob
+    queue_as 'default'
+
+    def perform
+      SourceTable.sync(SourceTableConfigurationBuilder
+                         .build_all(DbBlaster.configuration))
+
+      DbBlaster::SourceTable.pluck(:id).each do |source_table_id|
+        PublishSourceTableJob.perform_later(source_table_id)
+      end
+    end
+  end
+end

data/app/jobs/db_blaster/publish_source_table_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# Executes PublishSourceTable for provided `source_table_id`
+module DbBlaster
+  # Publishes changed rows to SNS
+  class PublishSourceTableJob < ApplicationJob
+    queue_as 'default'
+
+    def perform(source_table_id)
+      source_table = SourceTable.find_by(id: source_table_id)
+      return unless source_table
+
+      PublishSourceTable.execute(source_table)
+    end
+  end
+end

data/app/models/concerns/db_blaster/sync.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Syncs tables derived from SourceTableConfigurationBuilder
+  # with SourceTable rows
+  module Sync
+    extend ActiveSupport::Concern
+
+    class_methods do
+      # Syncs configuration tables with db
+      def sync(table_configs)
+        SourceTable.where.not(name: table_configs.collect(&:source_table_name)).delete_all
+
+        table_configs.each do |config|
+          source_table = SourceTable.where(name: config.source_table_name).first_or_create
+          source_table.update!(config.update_params)
+        end
+      end
+    end
+  end
+end

data/app/models/db_blaster/application_record.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Base class for models
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/db_blaster/source_table.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Represents tables that should be synced
+  class SourceTable < ApplicationRecord
+    include Sync
+
+    validates :name, uniqueness: { case_sensitive: false }, presence: true
+  end
+end

data/config/brakeman.ignore

@@ -0,0 +1,26 @@
+{
+  "ignored_warnings": [
+    {
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "6f4d3da0707c3f5f5c5bf5a002a254fee246210248aafa655cb2f15adfb47aa7",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "lib/db_blaster/finder.rb",
+      "line": 38,
+      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "ActiveRecord::Base.connection.execute(\"#{select_sql} OFFSET #{offset}\")",
+      "render_path": null,
+      "location": {
+        "type": "method",
+        "class": "DbBlaster::Finder",
+        "method": "find_records_in_batches"
+      },
+      "user_input": "select_sql",
+      "confidence": "Medium",
+      "note": "No SQL injection can occur"
+    }
+  ],
+  "updated": "2021-08-09 11:03:06 -0600",
+  "brakeman_version": "5.1.1"
+}

data/config/routes.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+# DbBlaster::Engine.routes.draw do
+# end

data/db/migrate/20210727222252_create_source_tables.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Create db_blaster_source_tables
+class CreateSourceTables < ActiveRecord::Migration[6.1]
+  def change
+    create_table :db_blaster_source_tables do |t|
+      t.string :name, null: false
+      t.text :ignored_columns, array: true, default: []
+      t.datetime :last_published_updated_at # the most recent published `updated_at`
+      t.integer :batch_size, default: 100
+      t.timestamps
+    end
+  end
+end

data/lib/db_blaster.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sns'
+require 'db_blaster/version'
+require 'db_blaster/engine'
+require 'db_blaster/one_record_too_large_error'
+require 'db_blaster/available_tables'
+require 'db_blaster/configuration'
+require 'db_blaster/source_table_configuration'
+require 'db_blaster/source_table_configuration_builder'
+require 'db_blaster/publisher'
+require 'db_blaster/publish_source_table'
+require 'db_blaster/chunker'
+require 'db_blaster/finder'
+
+# Top-level module that serves as an entry point
+# into the engine gem
+module DbBlaster
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+  end
+end

data/lib/db_blaster/available_tables.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Finds all tables in current database and removes the "system tables"
+  module AvailableTables
+    SYSTEM_TABLES = %w[schema_migrations ar_internal_metadata].freeze
+
+    def available_tables
+      @available_tables ||= ActiveRecord::Base.connection.tables - SYSTEM_TABLES
+    end
+  end
+end

data/lib/db_blaster/chunker.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Chunk the records into sizes < Configuration.max_message_size_in_kilobytes
+  # yielding the chunks inline to the provided block
+  # If the records' size is already less than Configuration.max_message_size_in_kilobytes,
+  # all the records are yielded to the provided block
+  class Chunker
+    attr_reader :source_table, :records, :block_on_chunk, :current_chunk, :current_chunk_size
+
+    def initialize(source_table, records, &block_on_chunk)
+      @source_table = source_table
+      @records = records
+      @block_on_chunk = block_on_chunk
+      @current_chunk_size = 0
+      @current_chunk = []
+    end
+
+    def self.chunk(source_table, records, &block)
+      new(source_table, records, &block).chunk
+    end
+
+    def chunk
+      return if yield_if_records_acceptable?
+
+      breakup_records
+    end
+
+    def yield_if_records_acceptable?
+      return if records.to_json.size >= max_bytes
+
+      block_on_chunk.call(records)
+      true
+    end
+
+    def max_bytes
+      @max_bytes ||= 1000 * max_kilobytes
+    end
+
+    def max_kilobytes
+      DbBlaster.configuration.max_message_size_in_kilobytes ||
+        DbBlaster.configuration.class::DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES
+    end
+
+    def breakup_records
+      records.each(&method(:process_record))
+      block_on_chunk.call(current_chunk) if current_chunk.length.positive?
+    end
+
+    private
+
+    def process_record(record)
+      record_size = record.to_json.size
+      @current_chunk_size += record_size
+      if current_chunk_size < max_bytes
+        current_chunk << record
+      elsif record_size >= max_bytes
+        blow_up_one_record_too_large(record)
+      else
+        yield_chunk(record, record_size)
+      end
+    end
+
+    def yield_chunk(record, record_size)
+      block_on_chunk.call(current_chunk)
+      @current_chunk_size = record_size
+      @current_chunk = [record]
+    end
+
+    def blow_up_one_record_too_large(record)
+      block_on_chunk.call(current_chunk) if current_chunk.length.positive?
+      raise OneRecordTooLargeError.new(source_table: source_table,
+                                       record: record,
+                                       max_kilobytes: max_kilobytes)
+    end
+  end
+end

data/lib/db_blaster/configuration.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Configuration class for providing credentials, topics, and customizations.
+  class Configuration
+    DEFAULT_BATCH_SIZE = 100
+    DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
+
+    # The required configuration fields
+    REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region sns_topic].freeze
+
+    # The topic to which messages will be published
+    attr_accessor :sns_topic
+    attr_accessor :aws_access_key, :aws_access_secret, :aws_region
+
+    # Global list of column names not to include in published SNS messages
+    # example: config.ignored_column_names = ['email', 'phone_number']
+    attr_accessor :ignored_column_names
+
+    # Optional
+    # If set, only publish tables specified.
+    # example: config.only_source_tables = ['posts', 'tags', 'comments']
+    attr_accessor :only_source_tables
+
+    # Optional
+    # Customize batch_size and/or ignored_columns
+    # example:
+    # config.source_table_options = [{ source_table_name: 'posts', batch_size: 100, ignored_column_names: ['email'] },
+    #                                { source_table_name: 'comments', ignored_column_names: ['tags'] }]
+    attr_accessor :source_table_options
+
+    # Optional
+    # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
+    # Attributes set here will be included in every published message
+    # example: config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+    attr_accessor :extra_sns_message_attributes
+
+    # Optional
+    # db_blaster will select and then publish `batch_size` rows at a time
+    # Default value is 100
+    attr_accessor :batch_size
+
+    # Optional
+    # DbBlaster will publish no messages larger than this value
+    # Default value is 256
+    attr_accessor :max_message_size_in_kilobytes
+
+    # Raises error if a required field is not set
+    def verify!
+      no_values = REQUIRED_FIELDS.select do |attribute|
+        send(attribute).nil? || send(attribute).strip.empty?
+      end
+      return if no_values.empty?
+
+      raise "missing configuration values for [#{no_values.join(', ')}]"
+    end
+  end
+end

data/lib/db_blaster/engine.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Engine to isolate DbBlaster namespace
+  class Engine < ::Rails::Engine
+    isolate_namespace DbBlaster
+
+    config.generators do |g|
+      g.test_framework :rspec
+    end
+  end
+end

data/lib/db_blaster/finder.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Find records and yield them a `batch_size` at a time
+  class Finder
+    include AvailableTables
+    attr_reader :source_table, :block_on_find, :offset
+
+    def initialize(source_table, &block)
+      @source_table = source_table
+      @block_on_find = block
+      @offset = 0
+    end
+
+    delegate :batch_size, :name, :last_published_updated_at, to: :source_table, prefix: true
+
+    def self.find(source_table, &block)
+      new(source_table, &block).find
+    end
+
+    def find
+      verify_source_table_name
+
+      find_records_in_batches do |batch|
+        filtered = batch.collect(&method(:filter_columns))
+        next if filtered.blank?
+
+        Chunker.chunk(source_table, filtered) do |chunked|
+          block_on_find.call(chunked)
+        end
+      end
+    end
+
+    private
+
+    def find_records_in_batches
+      loop do
+        result = ActiveRecord::Base.connection.execute("#{select_sql} OFFSET #{offset}")
+        yield(result)
+        break if result.count != source_table_batch_size
+
+        @offset += source_table_batch_size
+      end
+    end
+
+    def filter_columns(selected_row)
+      selected_row.except(*source_table.ignored_columns)
+    end
+
+    def verify_source_table_name
+      raise invalid_source_table_message unless available_tables.include?(source_table_name)
+    end
+
+    def invalid_source_table_message
+      "source_table.name: '#{source_table_name}' does not exist!"
+    end
+
+    def select_sql
+      "SELECT * FROM #{source_table_name} #{where} ORDER BY updated_at ASC LIMIT #{source_table_batch_size}"
+    end
+
+    def where
+      return '' unless source_table_last_published_updated_at
+
+      ActiveRecord::Base.sanitize_sql_for_conditions(
+        ['WHERE updated_at >= :updated_at', { updated_at: source_table_last_published_updated_at.to_s(:db) }]
+      )
+    end
+  end
+end

data/lib/db_blaster/one_record_too_large_error.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Custom error representing a serious error.
+  # If one individual record is larger than the max bytes
+  # allowed, db_blaster "fails fast".
+  # If we ignored this record and moved on, it would never make it
+  # to SNS as its `updated_at` would end up being less than the
+  # last processed record.
+  # Possible fixes:
+  # 1) if possible ignore the column(s) that are too large
+  # 2) manually move the record to the intended destination
+  # 3) bug me to provide an S3 workaround
+  class OneRecordTooLargeError < StandardError
+    attr_reader :source_table, :record, :max_kilobytes
+
+    def initialize(params)
+      super
+      params.each_key do |key|
+        instance_variable_set("@#{key}", params[key])
+      end
+    end
+
+    def message
+      ["One individual record with ID '#{record[:id]}'",
+       " in source-table '#{source_table.name}'",
+       " is larger than #{max_kilobytes} KB!"].join(' ')
+    end
+  end
+end

data/lib/db_blaster/publish_source_table.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Given a `source_table` providing the table name,
+  # finds rows in `batch_size` chunks that are published to SNS
+  class PublishSourceTable
+    attr_reader :source_table
+
+    def initialize(source_table)
+      @source_table = source_table
+    end
+
+    def self.execute(source_table)
+      new(source_table).execute
+    end
+
+    def execute
+      DbBlaster.configuration.verify! # will raise error if required configurations are not set
+
+      # pessimistically lock row for the duration
+      source_table.with_lock do
+        Finder.find(source_table) do |records|
+          Publisher.publish(source_table, records)
+          source_table.update(last_published_updated_at: records.last['updated_at'])
+        end
+      end
+      self
+    end
+  end
+end

data/lib/db_blaster/publisher.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# publish records to SNS topic
+module DbBlaster
+  # Publishes records to AWS SNS
+  class Publisher
+    attr_reader :source_table, :records
+
+    def initialize(source_table, records)
+      @source_table = source_table
+      @records = records
+    end
+
+    def self.publish(source_table, records)
+      new(source_table, records).publish
+    end
+
+    def publish
+      topic.publish(message_attributes: message_attributes,
+                    message: records.to_json)
+    end
+
+    def topic
+      resource.topic(DbBlaster.configuration.sns_topic)
+    end
+
+    def resource
+      @resource ||= Aws::SNS::Resource.new(client: client)
+    end
+
+    def client
+      @client ||= Aws::SNS::Client.new(region: DbBlaster.configuration.aws_region,
+                                       credentials: Aws::Credentials.new(DbBlaster.configuration.aws_access_key,
+                                                                         DbBlaster.configuration.aws_access_secret))
+    end
+
+    def message_attributes
+      (DbBlaster.configuration.extra_sns_message_attributes || {})
+        .merge('source_table' => { data_type: 'String', string_value: source_table.name })
+    end
+  end
+end

data/lib/db_blaster/rspec.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+if defined?(FactoryBot)
+  FactoryBot.define do
+    factory :db_blaster_source_table, class: DbBlaster::SourceTable do
+      sequence(:name) { |n| "table_#{n}" }
+    end
+  end
+end

data/lib/db_blaster/source_table_configuration.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # PORO for source-table-configuration fields
+  class SourceTableConfiguration
+    attr_reader :source_table_name, :batch_size, :ignored_column_names
+
+    def initialize(params)
+      params.each_key do |key|
+        instance_variable_set("@#{key}", params[key])
+      end
+    end
+
+    def update_params
+      { batch_size: batch_size,
+        ignored_columns: ignored_column_names }
+    end
+  end
+end

data/lib/db_blaster/source_table_configuration_builder.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Builds source-table configurations based off the tables in the
+  # current database and the provided DbBlaster::Configuration
+  class SourceTableConfigurationBuilder
+    include AvailableTables
+    attr_reader :configuration, :source_table_configurations
+
+    def initialize(configuration)
+      @configuration = configuration
+      @source_table_configurations = []
+    end
+
+    def self.build_all(configuration)
+      new(configuration).build_all
+    end
+
+    def build_all
+      @build_all ||= table_names_for_configuration
+                     .collect(&method(:build_configuration))
+    end
+
+    def build_configuration(source_table_name)
+      SourceTableConfiguration.new(source_table_name: source_table_name,
+                                   batch_size: batch_size(source_table_name),
+                                   ignored_column_names: ignored_column_names(source_table_name))
+    end
+
+    def batch_size(source_table_name)
+      overridden_value_or_global(source_table_name, :batch_size) || configuration.class::DEFAULT_BATCH_SIZE
+    end
+
+    def ignored_column_names(source_table_name)
+      overridden_value_or_global(source_table_name, :ignored_column_names) || []
+    end
+
+    def overridden_value_or_global(source_table_name, field_name)
+      find_source_table_options(source_table_name)&.send(:[], field_name) || configuration.send(field_name)
+    end
+
+    def find_source_table_options(source_table_name)
+      (configuration.source_table_options || [])
+        .detect { |option| option[:source_table_name] == source_table_name }
+    end
+
+    def table_names_for_configuration
+      if configuration.only_source_tables&.length&.positive?
+        available_tables & configuration.only_source_tables
+      else
+        available_tables
+      end
+    end
+  end
+end

data/lib/db_blaster/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  VERSION = '0.1.0'
+end

data/lib/generators/db_blaster/install/install_generator.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  module Generators
+    # Generator to copy a sample db_blaster_config.rb to the
+    # rails app's config/initializer directory
+    class InstallGenerator < ::Rails::Generators::Base
+      source_root File.expand_path('templates', __dir__)
+
+      def copy_initializer_file
+        copy_file 'db_blaster_config.rb', 'config/initializers/db_blaster_config.rb'
+      end
+    end
+  end
+end

data/lib/generators/db_blaster/install/templates/db_blaster_config.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+DbBlaster.configure do |config|
+  # SNS topic to receive database changes
+  config.sns_topic = 'the-topic'
+  config.aws_access_key = 'access-key'
+  config.aws_access_secret = 'secret'
+  config.aws_region = 'region'
+
+  # Optional
+  # db_blaster will select and then publish `batch_size` rows at a time
+  # config.batch_size = 100
+
+  # Optional
+  # db_blaster will publish no messages larger than this value
+  # Default value is 256
+  # attr_accessor :max_message_size_in_kilobytes
+
+  # Optional
+  # Extra [SNS message_attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html)
+  # Attributes set here will be included in every published message
+  # config.extra_sns_message_attributes = {'infra_id' => {data_type: 'String', value: '061'}}
+
+  # Global list of column names not to include in published SNS messages
+  # example: config.ignored_column_names = ['email', 'phone_number']
+  # config.ignored_column_names = ['email', 'phone_number']
+
+  # Optional
+  # If set, only publish tables specified.
+  # config.only_source_tables = ['posts', 'tags', 'comments']
+
+  # Optional
+  # Customize batch_size and/or ignored_columns
+  # example:
+  # config.source_table_options = [{ source_table_name: 'posts', batch_size: 100, ignored_column_names: ['email'] },
+  #                                { source_table_name: 'comments', ignored_column_names: ['tags'] }]
+end

data/lib/tasks/db_blaster_tasks.rake

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+# desc "Explaining what the task does"
+# task :db_blaster do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: db_blaster
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Perry Hertler
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-08-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sns
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: factory_bot_rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-its
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda-matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Push db changes to AWS SNS.
+email:
+- perry@hertler.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/config/db_blaster_manifest.js
+- app/assets/stylesheets/db_blaster/application.css
+- app/controllers/db_blaster/application_controller.rb
+- app/helpers/db_blaster/application_helper.rb
+- app/jobs/db_blaster/application_job.rb
+- app/jobs/db_blaster/publish_all_job.rb
+- app/jobs/db_blaster/publish_source_table_job.rb
+- app/models/concerns/db_blaster/sync.rb
+- app/models/db_blaster/application_record.rb
+- app/models/db_blaster/source_table.rb
+- config/brakeman.ignore
+- config/routes.rb
+- db/migrate/20210727222252_create_source_tables.rb
+- lib/db_blaster.rb
+- lib/db_blaster/available_tables.rb
+- lib/db_blaster/chunker.rb
+- lib/db_blaster/configuration.rb
+- lib/db_blaster/engine.rb
+- lib/db_blaster/finder.rb
+- lib/db_blaster/one_record_too_large_error.rb
+- lib/db_blaster/publish_source_table.rb
+- lib/db_blaster/publisher.rb
+- lib/db_blaster/rspec.rb
+- lib/db_blaster/source_table_configuration.rb
+- lib/db_blaster/source_table_configuration_builder.rb
+- lib/db_blaster/version.rb
+- lib/generators/db_blaster/install/install_generator.rb
+- lib/generators/db_blaster/install/templates/db_blaster_config.rb
+- lib/tasks/db_blaster_tasks.rake
+homepage: https://github.com/perryqh/db_blaster
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/perryqh/db_blaster
+  source_code_uri: https://github.com/perryqh/db_blaster
+  changelog_uri: https://github.com/perryqh/db_blaster/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Push db changes to AWS SNS.
+test_files: []