db_blaster 0.1.4 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9313d3abe0bd4dcaa756dab6b07b1d65eb6438b435c914d18f9d38964f08a29d
-  data.tar.gz: c694940facb41fb3656d011baba720ffb5f66cf124472f945d7de45661ba90b5
+  metadata.gz: 548f2aefb696e0417a36de5169343570ca93d4d02fe04bdf13806f48be794d05
+  data.tar.gz: ac6a6829cfaa8523b7151b585a4037b602faeb87d6166f021679ab863a8c986b
 SHA512:
-  metadata.gz: c045adce6f5a4fb70be59194f6f4630879761011be805dbed09c9be225a415a807ffe60a2b33ccf5361e49ce595b87250f8b0e67f370c29d867254928a36e41f
-  data.tar.gz: 4b509898ced3205b588248700a75a375ddc6c614611298f0af347ef1a45b35f4c3d1adc9ae0a70d357c56050f47a7225d454564c22a350a21cec45da0effe322
+  metadata.gz: fff1f7224220a29d7318f8d9dd793a72e3696db4f61c7829de45d4e90ab431863893ac41abbf7f779cdcc5eab9e8ef17ab4128115ad4e327d8441fdb737035e3
+  data.tar.gz: c4bb757b9123e612ae42768a892b7a8cd2444f322ff3eb41d33b16ecd877fcdef9e33d3f7f6cd92fda72339207accb7f2ea2eb6d74f4d24a661cfe28c4c80343

data/db/migrate/20210908214439_add_last_published_id.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Adding last_published_id to avoid dupes in selecting sourcetables
+class AddLastPublishedId < ActiveRecord::Migration[6.1]
+  def change
+    add_column :db_blaster_source_tables, :last_published_id, :string, default: '0'
+  end
+end

data/lib/db_blaster/configuration.rb

@@ -8,6 +8,8 @@ module DbBlaster
     DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
     DEFAULT_S3_KEY = '<batch_date>/<batch_time>/db_blaster/<table_name>/<uuid>.json'
     DEFAULT_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%LZ'
+    ATTRIBUTE_S3_META_FORMAT = 'attribute' # { meta: {your: :value}, records: [] }
+    INLINE_S3_META_FORMAT = 'inline' # records.collect{|record| record.merge(meta) }
 
     # The required configuration fields
     REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region].freeze
@@ -32,15 +34,22 @@ module DbBlaster
     # Applicable only when `s3_bucket' is set
     # The value set here will be included in every payload pushed to S3
     # example: config.s3_meta = {'infra_id' => '061', 'source_app' => 'kcp-api'}}
-    # The resulting JSON:
-    # {"meta" : {"infra_id" : "061", "src_app" : "kcp-api", "src_table" : "the-table"}, "records" : [] }
     attr_accessor :s3_meta
 
+    # Optional
+    # Options: ['attribute', 'inline']
+    # Defaults to 'attribute'
+    # 'attribute' payload: { meta: `s3_meta`, records: [source_table_records] }
+    # 'inline' payload: records.collect{|record| record.merge(meta) }
+    attr_accessor :s3_meta_format
+
     # Optional
     # Applicable only when `s3_bucket` is set
     # The S3 key. The following values will get substituted:
-    # <batch_timestamp> - a timestamp signifying the beginning of the batch processing
-    # <timestamp> - the current time
+    # <batch_date_time> - date time when batch started
+    # <batch_date> - date when batch started
+    # <batch_time - time when batch started
+    # <date_time> - the datetime just before pushing to S3
     # <table_name> - the name of the table associated with the S3 body
     # <uuid> - a universal identifier
     # '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'

data/lib/db_blaster/finder_sql.rb

@@ -17,16 +17,26 @@ module DbBlaster
       "SELECT * FROM #{source_table.name} #{where} ORDER BY updated_at ASC LIMIT #{source_table.batch_size}"
     end
 
+    # if we just use updated_at > from_updated_at, it's possible to miss records
+    # that share the same `updated_at`
+    # if we use updated_at >= from_updated_at, we'll get redundant records on every run
+    # settled on the approach below
     def where
       return '' unless from_updated_at
 
       ActiveRecord::Base.sanitize_sql_for_conditions(
-        ['WHERE updated_at >= :updated_at', { updated_at: from_updated_at.to_s(:db) }]
+        ['WHERE updated_at > :updated_at OR (updated_at = :updated_at AND id <> :updated_id)',
+         { updated_at: from_updated_at,
+           updated_id: last_published_id }]
       )
     end
 
     def from_updated_at
       @from_updated_at ||= source_table.last_published_updated_at
     end
+
+    def last_published_id
+      @last_published_id ||= source_table.last_published_id
+    end
   end
 end

data/lib/db_blaster/publish_source_table.rb

@@ -22,7 +22,8 @@ module DbBlaster
       source_table.with_lock do
         Finder.find(source_table) do |records|
           BasePublisher.publish(source_table: source_table, records: records, batch_start_time: batch_start_time)
-          source_table.update(last_published_updated_at: records.last['updated_at'])
+          source_table.update(last_published_updated_at: records.last['updated_at'],
+                              last_published_id: records.last['id'])
         end
       end
       self

data/lib/db_blaster/s3_key_builder.rb

@@ -23,10 +23,12 @@ module DbBlaster
     end
 
     def substitutions
+      date_time = DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
       date, time = batch_start_time.split('T')
       { '<batch_date_time>' => batch_start_time,
         '<batch_date>' => date,
         '<batch_time>' => time,
+        '<date_time>' => date_time,
         '<uuid>' => SecureRandom.uuid,
         '<table_name>' => source_table_name }
     end

data/lib/db_blaster/s3_publisher.rb

@@ -14,6 +14,8 @@ module DbBlaster
     end
 
     def content
+      return meta_records if DbBlaster.configuration.s3_meta_format == Configuration::INLINE_S3_META_FORMAT
+
       { meta: meta,
         records: records }
     end
@@ -28,7 +30,11 @@ module DbBlaster
     end
 
     def meta
-      (DbBlaster.configuration.s3_meta.presence || {}).merge(source_table: source_table.name)
+      @meta ||= (DbBlaster.configuration.s3_meta.presence || {}).merge(source_table: source_table.name)
+    end
+
+    def meta_records
+      records.collect { |record| record.merge(meta) }
     end
 
     def client

data/lib/db_blaster/source_tables_schema_builder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Builds an array of tables and their columns
+  class SourceTablesSchemaBuilder
+    def self.build_schema
+      new.build_schema
+    end
+
+    def build_schema
+      ActiveRecord::Base.connection.tables.each_with_object({}) do |table_name, hash|
+        unless AvailableTables::SYSTEM_TABLES.include?(table_name)
+          hash[table_name] = build_columns_from_table_name(table_name)
+        end
+      end
+    end
+
+    def build_columns_from_table_name(table_name)
+      ActiveRecord::Base.connection.columns(table_name).collect do |column|
+        next if ignored_column?(column.name)
+
+        { name: column.name,
+          type: column.type,
+          limit: column.limit }
+      end.compact
+    end
+
+    def ignored_column?(column)
+      (DbBlaster.configuration.ignored_column_names || []).include?(column)
+    end
+  end
+end

data/lib/db_blaster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DbBlaster
-  VERSION = '0.1.4'
+  VERSION = '0.1.8'
 end

data/lib/db_blaster.rb

@@ -15,6 +15,7 @@ require 'db_blaster/publish_source_table'
 require 'db_blaster/chunker'
 require 'db_blaster/finder_sql'
 require 'db_blaster/finder'
+require 'db_blaster/source_tables_schema_builder'
 
 # Top-level module that serves as an entry point
 # into the engine gem

data/lib/generators/db_blaster/install/templates/db_blaster_config.rb

@@ -22,8 +22,7 @@ DbBlaster.configure do |config|
   # The S3 key path. The following values will get substituted:
   # <batch_date_time> - a timestamp signifying the beginning of the batch processing
   # <batch_date> - a date signifying the beginning of the batch processing
-  # <batch_time> - a time signifying the beginning of the batch processing
-  # <timestamp> - the current time
+  # <date_time> - the datetime just before pushing to S3
   # <table_name> - the name of the table associated with the S3 body
   # <uuid> - a universal identifier
   # config.s3_key = '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
@@ -32,8 +31,7 @@ DbBlaster.configure do |config|
   # Applicable only when `s3_bucket' is set
   # Extra meta values sent along with each payload
   # example: config.s3_meta = {'infra_id' => '061'}
-  # The resulting JSON:
-  # {"meta" : {"infra_id" : "061", "source_app" : "kcp-api", "src_table" : "the-table"}, "records" : [] }
+  # The resulting JSON will include the `meta` merged into every record.
   # config.s3_meta = {'infra_id' => '061'}
 
   # Optional

data/lib/tasks/db_blaster_tasks.rake

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
-# desc "Explaining what the task does"
-# task :db_blaster do
-#   # Task goes here
-# end
+
+require 'db_blaster'
+
+namespace :db_blaster do
+  desc 'generate table schema'
+  task generate_table_schema: :environment do
+    schema_name = 'kcp-api-schema.json'
+    puts "Generating #{schema_name}......."
+    built = DbBlaster::SourceTablesSchemaBuilder.build_schema
+    File.open(schema_name, 'w') { |f| f << built.to_json }
+    puts 'Success!'
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_blaster
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.8
 platform: ruby
 authors:
 - Perry Hertler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2021-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -131,6 +131,7 @@ files:
 - config/brakeman.ignore
 - config/routes.rb
 - db/migrate/20210727222252_create_source_tables.rb
+- db/migrate/20210908214439_add_last_published_id.rb
 - lib/db_blaster.rb
 - lib/db_blaster/available_tables.rb
 - lib/db_blaster/base_publisher.rb
@@ -147,6 +148,7 @@ files:
 - lib/db_blaster/sns_publisher.rb
 - lib/db_blaster/source_table_configuration.rb
 - lib/db_blaster/source_table_configuration_builder.rb
+- lib/db_blaster/source_tables_schema_builder.rb
 - lib/db_blaster/version.rb
 - lib/generators/db_blaster/install/install_generator.rb
 - lib/generators/db_blaster/install/templates/db_blaster_config.rb
@@ -173,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Push db changes to AWS SNS.