db_blaster 0.1.5 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2de00cf222282fde0e0a3e38078ccc398a80ff19c41ccf7b79ecd043450db6b1
-  data.tar.gz: 66b93222044a44bcd7dd35a6233de5c4839a58d5106e527a7d540be2daee9821
+  metadata.gz: 7e58f97f7c15cb1bb2d1bf1d4235e68e5d256adaf48f72481cb3c754f1d85aae
+  data.tar.gz: ffa443a609d6d2b10b7e7e3b65bdc45da8abfc8485b7f3fb46fa638ea1b0cd98
 SHA512:
-  metadata.gz: 7305baf42dff923be85d8105625bf3ccf8200988d7895c80202a32232b062a9c613863a002a7b8e57d7e57a3660725ab4ed6765d2b2d3ba49f3b216da2614dac
-  data.tar.gz: 8652430ce86f2ee275997ec35e52bf2ba99befa4c3a8630b13e0ca57f309d4b0fb013974806e52882bc37bd95a6c4e9e9d4ea51fa5a2096d2767df0d3608b796
+  metadata.gz: fd27fea13565a8f90a3d0b4db4fcac631453fc9579b35f5ad7e4061e7f4cc7bea01e12307efd5a865583f781cb793ceb50f6646708912edcef8742b5a3551f53
+  data.tar.gz: ca998517216be98f3c3d47d9e0f393aa230dd04a08cf54a0dfeb1fb734246f7e1d388c00b698b92eab7bded131addded3ffc413b9d029e1aa2f5ac3aadf78928

data/app/jobs/db_blaster/publish_all_job.rb

@@ -9,8 +9,7 @@ module DbBlaster
     queue_as 'default'
 
     def perform
-      SourceTable.sync(SourceTableConfigurationBuilder
-                         .build_all(DbBlaster.configuration))
+      SyncSourceTablesWithConfiguration.sync
 
       DbBlaster::SourceTable.pluck(:id).each do |source_table_id|
         PublishSourceTableJob.perform_later(source_table_id, batch_start_time)

data/db/migrate/20210908214439_add_last_published_id.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Adding last_published_id to avoid dupes in selecting sourcetables
+class AddLastPublishedId < ActiveRecord::Migration[6.1]
+  def change
+    add_column :db_blaster_source_tables, :last_published_id, :string, default: '0'
+  end
+end

data/lib/db_blaster/configuration.rb

@@ -8,6 +8,8 @@ module DbBlaster
     DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
     DEFAULT_S3_KEY = '<batch_date>/<batch_time>/db_blaster/<table_name>/<uuid>.json'
     DEFAULT_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%LZ'
+    ATTRIBUTE_S3_META_FORMAT = 'attribute' # { meta: {your: :value}, records: [] }
+    INLINE_S3_META_FORMAT = 'inline' # records.collect{|record| record.merge(meta) }
 
     # The required configuration fields
     REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region].freeze
@@ -32,14 +34,22 @@ module DbBlaster
     # Applicable only when `s3_bucket' is set
     # The value set here will be included in every payload pushed to S3
     # example: config.s3_meta = {'infra_id' => '061', 'source_app' => 'kcp-api'}}
-    # The resulting JSON will include the `meta` merged into every record
     attr_accessor :s3_meta
 
+    # Optional
+    # Options: ['attribute', 'inline']
+    # Defaults to 'attribute'
+    # 'attribute' payload: { meta: `s3_meta`, records: [source_table_records] }
+    # 'inline' payload: records.collect{|record| record.merge(meta) }
+    attr_accessor :s3_meta_format
+
     # Optional
     # Applicable only when `s3_bucket` is set
     # The S3 key. The following values will get substituted:
-    # <batch_timestamp> - a timestamp signifying the beginning of the batch processing
-    # <timestamp> - the current time
+    # <batch_date_time> - date time when batch started
+    # <batch_date> - date when batch started
+    # <batch_time - time when batch started
+    # <date_time> - the datetime just before pushing to S3
     # <table_name> - the name of the table associated with the S3 body
     # <uuid> - a universal identifier
     # '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'

data/lib/db_blaster/finder_sql.rb

@@ -17,16 +17,26 @@ module DbBlaster
       "SELECT * FROM #{source_table.name} #{where} ORDER BY updated_at ASC LIMIT #{source_table.batch_size}"
     end
 
+    # if we just use updated_at > from_updated_at, it's possible to miss records
+    # that share the same `updated_at`
+    # if we use updated_at >= from_updated_at, we'll get redundant records on every run
+    # settled on the approach below
     def where
       return '' unless from_updated_at
 
       ActiveRecord::Base.sanitize_sql_for_conditions(
-        ['WHERE updated_at >= :updated_at', { updated_at: from_updated_at.to_s(:db) }]
+        ['WHERE updated_at > :updated_at OR (updated_at = :updated_at AND id <> :updated_id)',
+         { updated_at: from_updated_at,
+           updated_id: last_published_id }]
       )
     end
 
     def from_updated_at
       @from_updated_at ||= source_table.last_published_updated_at
     end
+
+    def last_published_id
+      @last_published_id ||= source_table.last_published_id
+    end
   end
 end

data/lib/db_blaster/publish_source_table.rb

@@ -22,7 +22,8 @@ module DbBlaster
       source_table.with_lock do
         Finder.find(source_table) do |records|
           BasePublisher.publish(source_table: source_table, records: records, batch_start_time: batch_start_time)
-          source_table.update(last_published_updated_at: records.last['updated_at'])
+          source_table.update(last_published_updated_at: records.last['updated_at'],
+                              last_published_id: records.last['id'])
         end
       end
       self

data/lib/db_blaster/s3_key_builder.rb

@@ -23,10 +23,12 @@ module DbBlaster
     end
 
     def substitutions
+      date_time = DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
       date, time = batch_start_time.split('T')
       { '<batch_date_time>' => batch_start_time,
         '<batch_date>' => date,
         '<batch_time>' => time,
+        '<date_time>' => date_time,
         '<uuid>' => SecureRandom.uuid,
         '<table_name>' => source_table_name }
     end

data/lib/db_blaster/s3_publisher.rb

@@ -14,7 +14,10 @@ module DbBlaster
     end
 
     def content
-      meta_records
+      return meta_records if DbBlaster.configuration.s3_meta_format == Configuration::INLINE_S3_META_FORMAT
+
+      { meta: meta,
+        records: records }
     end
 
     def tagging

data/lib/db_blaster/source_tables_schema_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Builds an array of tables and their columns
+  class SourceTablesSchemaBuilder
+    def self.build_schema
+      new.build_schema
+    end
+
+    def build_schema
+      SyncSourceTablesWithConfiguration.sync
+
+      DbBlaster::SourceTable.all.each_with_object({}) do |source_table, hash|
+        hash[source_table.name] = build_columns_from_source_table(source_table)
+      end
+    end
+
+    def build_columns_from_source_table(source_table)
+      ActiveRecord::Base.connection.columns(source_table.name).collect do |column|
+        next if source_table.ignored_columns.include?(column.name)
+
+        { name: column.name,
+          type: column.type,
+          limit: column.limit }
+      end.compact
+    end
+  end
+end

data/lib/db_blaster/sync_source_tables_with_configuration.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Update the db-blaster source tables based on provided configuration
+  module SyncSourceTablesWithConfiguration
+    def self.sync
+      SourceTable.sync(SourceTableConfigurationBuilder
+                         .build_all(DbBlaster.configuration))
+    end
+  end
+end

data/lib/db_blaster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DbBlaster
-  VERSION = '0.1.5'
+  VERSION = '0.1.9'
 end

data/lib/db_blaster.rb

@@ -15,6 +15,8 @@ require 'db_blaster/publish_source_table'
 require 'db_blaster/chunker'
 require 'db_blaster/finder_sql'
 require 'db_blaster/finder'
+require 'db_blaster/source_tables_schema_builder'
+require 'db_blaster/sync_source_tables_with_configuration'
 
 # Top-level module that serves as an entry point
 # into the engine gem

data/lib/generators/db_blaster/install/templates/db_blaster_config.rb

@@ -22,8 +22,7 @@ DbBlaster.configure do |config|
   # The S3 key path. The following values will get substituted:
   # <batch_date_time> - a timestamp signifying the beginning of the batch processing
   # <batch_date> - a date signifying the beginning of the batch processing
-  # <batch_time> - a time signifying the beginning of the batch processing
-  # <timestamp> - the current time
+  # <date_time> - the datetime just before pushing to S3
   # <table_name> - the name of the table associated with the S3 body
   # <uuid> - a universal identifier
   # config.s3_key = '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'

data/lib/tasks/db_blaster_tasks.rake

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
-# desc "Explaining what the task does"
-# task :db_blaster do
-#   # Task goes here
-# end
+
+require 'db_blaster'
+
+namespace :db_blaster do
+  desc 'generate table schema'
+  task generate_table_schema: :environment do
+    schema_name = 'kcp-api-schema.json'
+    puts "Generating #{schema_name}......."
+    built = DbBlaster::SourceTablesSchemaBuilder.build_schema
+    File.open(schema_name, 'w') { |f| f << built.to_json }
+    puts 'Success!'
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_blaster
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.9
 platform: ruby
 authors:
 - Perry Hertler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2021-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -131,6 +131,7 @@ files:
 - config/brakeman.ignore
 - config/routes.rb
 - db/migrate/20210727222252_create_source_tables.rb
+- db/migrate/20210908214439_add_last_published_id.rb
 - lib/db_blaster.rb
 - lib/db_blaster/available_tables.rb
 - lib/db_blaster/base_publisher.rb
@@ -147,6 +148,8 @@ files:
 - lib/db_blaster/sns_publisher.rb
 - lib/db_blaster/source_table_configuration.rb
 - lib/db_blaster/source_table_configuration_builder.rb
+- lib/db_blaster/source_tables_schema_builder.rb
+- lib/db_blaster/sync_source_tables_with_configuration.rb
 - lib/db_blaster/version.rb
 - lib/generators/db_blaster/install/install_generator.rb
 - lib/generators/db_blaster/install/templates/db_blaster_config.rb