db_blaster 0.1.3 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e7b7ce6209a45251fe19c47d6248e932e042bca2ef60e4507b7365dd517555b
-  data.tar.gz: 774d0e7784134bfc8b14759cb7eb832851245d24a1afbcb7f1bd300be49f6537
+  metadata.gz: 21570da32145568b451fa60f6e48f2dbf889f30dcdb6baed94b51043d4e1d46e
+  data.tar.gz: 198f2fa9de6915d44e7c7717ed905a8357b65a3a2b10ba841b46b96e494016c1
 SHA512:
-  metadata.gz: 2ab4c82a9debbaeb34f40353d5e0fae79077920bd4509469ea3a4bf51788f504a813c20b57aced15c68025ae34de6dd3fb97e112d2240613adb33fddcd64f429
-  data.tar.gz: 600d83c0a603c961b68a5751dbfd5685c4aefd916484905abb00e4997977fed4f63c7f79adb18291a6b64d9dab3c89d48a89aa3b19b47218b5b5ab20fec3c8b0
+  metadata.gz: bfe0f16131ce6e7359129da73e24d51e8499fdbeb40ecc958d422cdb968b2ffcd944e768352d8495350fe9109f706d003158304c4b3ed058c0d71775bc15d9ef
+  data.tar.gz: 86508a7a04b99cf10d92ceb08bcb4259fb985b727229eda85d9cce191ea5724a93819fc45afe60f24b29dddfad8bf0336a92d1125eb925947c6fb09f6063dd41

data/app/jobs/db_blaster/publish_all_job.rb

@@ -18,7 +18,7 @@ module DbBlaster
     end
 
     def batch_start_time
-      @batch_start_time ||= DateTime.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+      @batch_start_time ||= DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
     end
   end
 end

data/db/migrate/20210908214439_add_last_published_id.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Adding last_published_id to avoid dupes in selecting sourcetables
+class AddLastPublishedId < ActiveRecord::Migration[6.1]
+  def change
+    add_column :db_blaster_source_tables, :last_published_id, :string, default: '0'
+  end
+end

data/lib/db_blaster/configuration.rb

@@ -6,7 +6,10 @@ module DbBlaster
   class Configuration
     DEFAULT_BATCH_SIZE = 100
     DEFAULT_MAX_MESSAGE_SIZE_IN_KILOBYTES = 256 # max size allowed by AWS SNS
-    DEFAULT_S3_KEY = '<batch_timestamp>/db_blaster/<table_name>/<uuid>.json'
+    DEFAULT_S3_KEY = '<batch_date>/<batch_time>/db_blaster/<table_name>/<uuid>.json'
+    DEFAULT_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%LZ'
+    ATTRIBUTE_S3_META_FORMAT = 'attribute' # { meta: {your: :value}, records: [] }
+    INLINE_S3_META_FORMAT = 'inline' # records.collect{|record| record.merge(meta) }
 
     # The required configuration fields
     REQUIRED_FIELDS = %i[aws_access_key aws_access_secret aws_region].freeze
@@ -30,21 +33,34 @@ module DbBlaster
     # Optional
     # Applicable only when `s3_bucket' is set
     # The value set here will be included in every payload pushed to S3
-    # example: config.s3_meta = {'infra_id' => '061', 'src_app' => 'kcp-api'}}
-    # The resulting JSON:
-    # {"meta" : {"infra_id" : "061", "src_app" : "kcp-api", "src_table" : "the-table"}, "records" : [] }
+    # example: config.s3_meta = {'infra_id' => '061', 'source_app' => 'kcp-api'}}
     attr_accessor :s3_meta
 
+    # Optional
+    # Options: ['attribute', 'inline']
+    # Defaults to 'attribute'
+    # 'attribute' payload: { meta: `s3_meta`, records: [source_table_records] }
+    # 'inline' payload: records.collect{|record| record.merge(meta) }
+    attr_accessor :s3_meta_format
+
     # Optional
     # Applicable only when `s3_bucket` is set
     # The S3 key. The following values will get substituted:
-    # <batch_timestamp> - a timestamp signifying the beginning of the batch processing
-    # <timestamp> - the current time
+    # <batch_date_time> - date time when batch started
+    # <batch_date> - date when batch started
+    # <batch_time - time when batch started
+    # <date_time> - the datetime just before pushing to S3
     # <table_name> - the name of the table associated with the S3 body
     # <uuid> - a universal identifier
     # '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
     attr_accessor :s3_key
 
+    # Optional
+    # Applicable only when `s3_bucket` is set
+    # S3 Tags
+    # example: config.s3_tags = { infra_id: '001', source_app: 'kcp-api', source_table: 'meetings' }
+    attr_accessor :s3_tags
+
     # Global list of column names not to include in published SNS messages
     # example: config.ignored_column_names = ['email', 'phone_number']
     attr_accessor :ignored_column_names

data/lib/db_blaster/finder_sql.rb

@@ -17,16 +17,26 @@ module DbBlaster
       "SELECT * FROM #{source_table.name} #{where} ORDER BY updated_at ASC LIMIT #{source_table.batch_size}"
     end
 
+    # if we just use updated_at > from_updated_at, it's possible to miss records
+    # that share the same `updated_at`
+    # if we use updated_at >= from_updated_at, we'll get redundant records on every run
+    # settled on the approach below
     def where
       return '' unless from_updated_at
 
       ActiveRecord::Base.sanitize_sql_for_conditions(
-        ['WHERE updated_at >= :updated_at', { updated_at: from_updated_at.to_s(:db) }]
+        ['WHERE updated_at > :updated_at OR (updated_at = :updated_at AND id <> :updated_id)',
+         { updated_at: from_updated_at.to_s(:db),
+           updated_id: last_published_id }]
       )
     end
 
     def from_updated_at
       @from_updated_at ||= source_table.last_published_updated_at
     end
+
+    def last_published_id
+      @last_published_id ||= source_table.last_published_id
+    end
   end
 end

data/lib/db_blaster/publish_source_table.rb

@@ -22,7 +22,8 @@ module DbBlaster
       source_table.with_lock do
         Finder.find(source_table) do |records|
           BasePublisher.publish(source_table: source_table, records: records, batch_start_time: batch_start_time)
-          source_table.update(last_published_updated_at: records.last['updated_at'])
+          source_table.update(last_published_updated_at: records.last['updated_at'],
+                              last_published_id: records.last['id'])
         end
       end
       self

data/lib/db_blaster/s3_key_builder.rb

@@ -23,7 +23,12 @@ module DbBlaster
     end
 
     def substitutions
-      { '<batch_timestamp>' => batch_start_time,
+      date_time = DateTime.now.utc.strftime(DbBlaster::Configuration::DEFAULT_DATETIME_FORMAT)
+      date, time = batch_start_time.split('T')
+      { '<batch_date_time>' => batch_start_time,
+        '<batch_date>' => date,
+        '<batch_time>' => time,
+        '<date_time>' => date_time,
         '<uuid>' => SecureRandom.uuid,
         '<table_name>' => source_table_name }
     end

data/lib/db_blaster/s3_publisher.rb

@@ -9,16 +9,32 @@ module DbBlaster
       client.put_object(bucket: DbBlaster.configuration.s3_bucket,
                         key: S3KeyBuilder.build(source_table_name: source_table.name,
                                                 batch_start_time: batch_start_time),
+                        tagging: tagging,
                         body: content.to_json)
     end
 
     def content
+      return meta_records if DbBlaster.configuration.s3_meta_format == Configuration::INLINE_S3_META_FORMAT
+
       { meta: meta,
         records: records }
     end
 
+    def tagging
+      URI.encode_www_form(tags_hash)
+    end
+
+    def tags_hash
+      @tags_hash ||= { source_table: source_table.name }
+                     .merge(DbBlaster.configuration.s3_tags.presence || {})
+    end
+
     def meta
-      (DbBlaster.configuration.s3_meta || {}).merge(source_table: source_table.name)
+      @meta ||= (DbBlaster.configuration.s3_meta.presence || {}).merge(source_table: source_table.name)
+    end
+
+    def meta_records
+      records.collect { |record| record.merge(meta) }
     end
 
     def client

data/lib/db_blaster/source_tables_schema_builder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Builds an array of tables and their columns
+  class SourceTablesSchemaBuilder
+    def self.build_schema
+      new.build_schema
+    end
+
+    def build_schema
+      ActiveRecord::Base.connection.tables.each_with_object({}) do |table_name, hash|
+        unless AvailableTables::SYSTEM_TABLES.include?(table_name)
+          hash[table_name] = build_columns_from_table_name(table_name)
+        end
+      end
+    end
+
+    def build_columns_from_table_name(table_name)
+      ActiveRecord::Base.connection.columns(table_name).collect do |column|
+        next if ignored_column?(column.name)
+
+        { name: column.name,
+          type: column.type,
+          limit: column.limit }
+      end.compact
+    end
+
+    def ignored_column?(column)
+      (DbBlaster.configuration.ignored_column_names || []).include?(column)
+    end
+  end
+end

data/lib/db_blaster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DbBlaster
-  VERSION = '0.1.3'
+  VERSION = '0.1.7'
 end

data/lib/db_blaster.rb

@@ -15,6 +15,7 @@ require 'db_blaster/publish_source_table'
 require 'db_blaster/chunker'
 require 'db_blaster/finder_sql'
 require 'db_blaster/finder'
+require 'db_blaster/source_tables_schema_builder'
 
 # Top-level module that serves as an entry point
 # into the engine gem

data/lib/generators/db_blaster/install/templates/db_blaster_config.rb

@@ -20,8 +20,9 @@ DbBlaster.configure do |config|
   # Optional
   # Applicable only when `s3_bucket` is set
   # The S3 key path. The following values will get substituted:
-  # <batch_timestamp> - a timestamp signifying the beginning of the batch processing
-  # <timestamp> - the current time
+  # <batch_date_time> - a timestamp signifying the beginning of the batch processing
+  # <batch_date> - a date signifying the beginning of the batch processing
+  # <date_time> - the datetime just before pushing to S3
   # <table_name> - the name of the table associated with the S3 body
   # <uuid> - a universal identifier
   # config.s3_key = '<batch_timestamp>/kcp-api/001/<table_name>/<uuid>.json'
@@ -30,8 +31,7 @@ DbBlaster.configure do |config|
   # Applicable only when `s3_bucket' is set
   # Extra meta values sent along with each payload
   # example: config.s3_meta = {'infra_id' => '061'}
-  # The resulting JSON:
-  # {"meta" : {"infra_id" : "061", "src_app" : "kcp-api", "src_table" : "the-table"}, "records" : [] }
+  # The resulting JSON will include the `meta` merged into every record.
   # config.s3_meta = {'infra_id' => '061'}
 
   # Optional

data/lib/tasks/db_blaster_tasks.rake

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
-# desc "Explaining what the task does"
-# task :db_blaster do
-#   # Task goes here
-# end
+
+require 'db_blaster'
+
+namespace :db_blaster do
+  desc 'generate table schema'
+  task generate_table_schema: :environment do
+    schema_name = 'kcp-api-schema.json'
+    puts "Generating #{schema_name}......."
+    built = DbBlaster::SourceTablesSchemaBuilder.build_schema
+    File.open(schema_name, 'w') { |f| f << built.to_json }
+    puts 'Success!'
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_blaster
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.7
 platform: ruby
 authors:
 - Perry Hertler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-01 00:00:00.000000000 Z
+date: 2021-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -131,6 +131,7 @@ files:
 - config/brakeman.ignore
 - config/routes.rb
 - db/migrate/20210727222252_create_source_tables.rb
+- db/migrate/20210908214439_add_last_published_id.rb
 - lib/db_blaster.rb
 - lib/db_blaster/available_tables.rb
 - lib/db_blaster/base_publisher.rb
@@ -147,6 +148,7 @@ files:
 - lib/db_blaster/sns_publisher.rb
 - lib/db_blaster/source_table_configuration.rb
 - lib/db_blaster/source_table_configuration_builder.rb
+- lib/db_blaster/source_tables_schema_builder.rb
 - lib/db_blaster/version.rb
 - lib/generators/db_blaster/install/install_generator.rb
 - lib/generators/db_blaster/install/templates/db_blaster_config.rb