db_blaster 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31207f1a8613b779dffcf5a8c17c5220508ae6da66088611a4f66ec457e9c210
-  data.tar.gz: 2ff279b9cee2ce471fd99bb3f2e8f786eed3f94d42a4a6488e2248b476d71080
+  metadata.gz: 21570da32145568b451fa60f6e48f2dbf889f30dcdb6baed94b51043d4e1d46e
+  data.tar.gz: 198f2fa9de6915d44e7c7717ed905a8357b65a3a2b10ba841b46b96e494016c1
 SHA512:
-  metadata.gz: 9f83e6757ccdc32c57a0e64c004e9b07fb543b54fafcf3c3838969bb84853601d959f178977a14b2bf5244ba7b194a2a427ed8ba74b96af7aef557a786875f01
-  data.tar.gz: 01f717e65dc1def37cddf11af9ba0df201d213713a8a0d83aa9490d70e796bffabf17edf004e946cd87f35ac8afde64a09b6c9aa37b31488e4f1422329d82838
+  metadata.gz: bfe0f16131ce6e7359129da73e24d51e8499fdbeb40ecc958d422cdb968b2ffcd944e768352d8495350fe9109f706d003158304c4b3ed058c0d71775bc15d9ef
+  data.tar.gz: 86508a7a04b99cf10d92ceb08bcb4259fb985b727229eda85d9cce191ea5724a93819fc45afe60f24b29dddfad8bf0336a92d1125eb925947c6fb09f6063dd41

data/db/migrate/20210908214439_add_last_published_id.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Adding last_published_id to avoid dupes in selecting sourcetables
+class AddLastPublishedId < ActiveRecord::Migration[6.1]
+  def change
+    add_column :db_blaster_source_tables, :last_published_id, :string, default: '0'
+  end
+end

data/lib/db_blaster/finder_sql.rb

@@ -17,16 +17,26 @@ module DbBlaster
       "SELECT * FROM #{source_table.name} #{where} ORDER BY updated_at ASC LIMIT #{source_table.batch_size}"
     end
 
+    # if we just use updated_at > from_updated_at, it's possible to miss records
+    # that share the same `updated_at`
+    # if we use updated_at >= from_updated_at, we'll get redundant records on every run
+    # settled on the approach below
     def where
       return '' unless from_updated_at
 
       ActiveRecord::Base.sanitize_sql_for_conditions(
-        ['WHERE updated_at >= :updated_at', { updated_at: from_updated_at.to_s(:db) }]
+        ['WHERE updated_at > :updated_at OR (updated_at = :updated_at AND id <> :updated_id)',
+         { updated_at: from_updated_at.to_s(:db),
+           updated_id: last_published_id }]
       )
     end
 
     def from_updated_at
       @from_updated_at ||= source_table.last_published_updated_at
     end
+
+    def last_published_id
+      @last_published_id ||= source_table.last_published_id
+    end
   end
 end

data/lib/db_blaster/publish_source_table.rb

@@ -22,7 +22,8 @@ module DbBlaster
       source_table.with_lock do
         Finder.find(source_table) do |records|
           BasePublisher.publish(source_table: source_table, records: records, batch_start_time: batch_start_time)
-          source_table.update(last_published_updated_at: records.last['updated_at'])
+          source_table.update(last_published_updated_at: records.last['updated_at'],
+                              last_published_id: records.last['id'])
         end
       end
       self

data/lib/db_blaster/source_tables_schema_builder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module DbBlaster
+  # Builds an array of tables and their columns
+  class SourceTablesSchemaBuilder
+    def self.build_schema
+      new.build_schema
+    end
+
+    def build_schema
+      ActiveRecord::Base.connection.tables.each_with_object({}) do |table_name, hash|
+        unless AvailableTables::SYSTEM_TABLES.include?(table_name)
+          hash[table_name] = build_columns_from_table_name(table_name)
+        end
+      end
+    end
+
+    def build_columns_from_table_name(table_name)
+      ActiveRecord::Base.connection.columns(table_name).collect do |column|
+        next if ignored_column?(column.name)
+
+        { name: column.name,
+          type: column.type,
+          limit: column.limit }
+      end.compact
+    end
+
+    def ignored_column?(column)
+      (DbBlaster.configuration.ignored_column_names || []).include?(column)
+    end
+  end
+end

data/lib/db_blaster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DbBlaster
-  VERSION = '0.1.6'
+  VERSION = '0.1.7'
 end

data/lib/db_blaster.rb

@@ -15,6 +15,7 @@ require 'db_blaster/publish_source_table'
 require 'db_blaster/chunker'
 require 'db_blaster/finder_sql'
 require 'db_blaster/finder'
+require 'db_blaster/source_tables_schema_builder'
 
 # Top-level module that serves as an entry point
 # into the engine gem

data/lib/tasks/db_blaster_tasks.rake

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
-# desc "Explaining what the task does"
-# task :db_blaster do
-#   # Task goes here
-# end
+
+require 'db_blaster'
+
+namespace :db_blaster do
+  desc 'generate table schema'
+  task generate_table_schema: :environment do
+    schema_name = 'kcp-api-schema.json'
+    puts "Generating #{schema_name}......."
+    built = DbBlaster::SourceTablesSchemaBuilder.build_schema
+    File.open(schema_name, 'w') { |f| f << built.to_json }
+    puts 'Success!'
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_blaster
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Perry Hertler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-03 00:00:00.000000000 Z
+date: 2021-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -131,6 +131,7 @@ files:
 - config/brakeman.ignore
 - config/routes.rb
 - db/migrate/20210727222252_create_source_tables.rb
+- db/migrate/20210908214439_add_last_published_id.rb
 - lib/db_blaster.rb
 - lib/db_blaster/available_tables.rb
 - lib/db_blaster/base_publisher.rb
@@ -147,6 +148,7 @@ files:
 - lib/db_blaster/sns_publisher.rb
 - lib/db_blaster/source_table_configuration.rb
 - lib/db_blaster/source_table_configuration_builder.rb
+- lib/db_blaster/source_tables_schema_builder.rb
 - lib/db_blaster/version.rb
 - lib/generators/db_blaster/install/install_generator.rb
 - lib/generators/db_blaster/install/templates/db_blaster_config.rb