backfiller 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3c3e9ee72c70a3b89caba10d6073a87058290680
-  data.tar.gz: 22071f36c634b6353787cac65a8c2984e8eb1ade
+SHA256:
+  metadata.gz: e03f52ddf2d8dbe1a7aada95079d1bbbeb6588e0a361122062a6c7e1c871d01d
+  data.tar.gz: d6b0f6b92fe6da8ac84338ff76435f16a6eea506ce42e762a594f2e81af05542
 SHA512:
-  metadata.gz: 3931af99bdc3771c1636c24e4ce1374ca0af0739090c6f56defd530f4fa26f6cd3ac497feb9cd2824fa91a7f8c6a2c4e3cef0b64fb51b4d76cd4f13d90c033c1
-  data.tar.gz: c6bd78c26fd968f0df688bb89b52d22dc7681e9f781825300c4d5183310db95aaab56a6e1efc3b0ed8f0419ec20dd271e2b46d4114f2fbba89aed424e266debb
+  metadata.gz: d2d62553aa42545079e829e93c58d01171047b6874112e123fe2f4bc0bb0bf5117c5334bd5b527600d216bc3997ed137b8f5ffded9327e068fde057f006b909c
+  data.tar.gz: ab0c2aa054baabd9afd9b94bdd5cf9b3313420ef41971c374b3c4571cacf551ebef04e7dc0488f389a6f179a766fea7cb15bf4133a1b58d274b79331127353df

data/README.md

@@ -1,3 +1,5 @@
+![Backfill machine](https://railsware.github.io/backfiller/assets/backfill_machine.jpg)
+
 # Backfiller
 
 The backfill machine for null database columns.
@@ -14,10 +16,9 @@ This gem maybe handly for `no-downtime` deployment especially when you need to f
 * add migration that disallow null values (null: false)
 * deploy code that starts using new column
 
-
 ## Concept
 
-Idea is to prepare all data in selection method on database server and fetch all data using CURSOR and then build simple UPDATE queries.
+The idea is to prepare all data in selection method on database server and fetch it data using CURSOR feature and then build simple UPDATE queries.
 With this way we minimize db server resources usage and we lock only one record (atomic update).
 We use two connections to database:
 * master - to creates cursor in transaction and fetch data in batches.
@@ -27,7 +28,8 @@ Even if backfill process crashes you may resolve issue and run it again to proce
 
 ## Connection adapters
 
-Curently it supports only PostgreSQL ActiveRecord adapter.
+Curently it support next ActiveRecord connection adapters:
+* PostgreSQL
 
 ## Installation
 
@@ -51,10 +53,13 @@ Assume we we want to backfill `profiles.name` column from `users.first_name`, `u
 
 Create backfill task into `db/backfill/profile_name.rb` and defined required methods:
 
+#### Single worker execution query
+
 ```ruby
 class Backfill::ProfileName
+
   def select_sql
-    <<-SQL.strip_heredoc
+    <<~SQL
       SELECT
         profile.id AS profile_id,
         CONCAT(users.first_name, ' ', users.last_name) AS profile_name
@@ -66,31 +71,102 @@ class Backfill::ProfileName
     SQL
   end
 
-  def update_sql(connection, row)
-    <<-SQL.strip_heredoc
+  def execute_sql(connection, row)
+    <<~SQL
       UPDATE profiles SET
         name = #{connection.quote(row['profile_name'])}
-     WHERE
-       id = #{connection.quote(row[:profile_id])}
+      WHERE
+       id = #{connection.quote(row['profile_id'])}
     SQL
   end
+
+end
+```
+
+#### Multiple worker execution queries
+
+```ruby
+class Backfill::ProfileName
+
+  def select_sql
+    <<~SQL
+      SELECT
+        profile.id AS profile_id,
+        CONCAT(users.first_name, ' ', users.last_name) AS profile_name
+      FROM profiles
+      INNER JOIN users ON
+        users.id = profiles.user_id
+      WHERE
+        profiles.name IS NULL
+    SQL
+  end
+
+  def execute_sql(connection, row)
+    [
+      'BEGIN',
+      <<~SQL,
+        UPDATE profiles SET
+          name = #{connection.quote(row['profile_name'])}
+        WHERE
+         id = #{connection.quote(row['profile_id'])} AND
+        (SELECT pg_try_advisory_xact_lock(12345678)') = TRUE
+      SQL
+      'COMMIT'
+    ]
+  end
+
 end
+
 ```
 
+#### Custom row processing
+
+```ruby
+class Backfill::ProfileName
+
+  def select_sql
+    <<~SQL
+      SELECT
+        profile.id AS profile_id,
+        CONCAT(users.first_name, ' ', users.last_name) AS profile_name
+      FROM profiles
+      INNER JOIN users ON
+        users.id = profiles.user_id
+      WHERE
+        profiles.name IS NULL
+    SQL
+  end
+
+  def process_row(connection, row)
+    connection.execute 'BEGIN'
+    if connection.select_value 'SELECT pg_try_advisory_xact_lock(12345678)'
+      connection.execute <<~SQL
+        INSERT INTO contacts(
+          full_name
+        )
+        VALUES(
+          #{connection.quote(row['profile_name'])},
+        )
+      SQL
+    connection.execute 'COMMIT'
+  end
+
+end
+
+```
 And then just run rake task:
 
 ```bash
 $ rails db:backfill[profile_name]
 ```
 
-
 ## Configuration
 
 For Rails application backfiller is initialized with next options
 
 * task_directory: `RAILS_ROOT/db/backfill`
 * task_namespace: `Backfill`
-* batch_size - `1_000`
+* batch_size: `1_000`
 * connection_pool: `ApplicationRecord.connection_pool`
 * logger: `ApplicationRecord.logger`
 

data/backfiller.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "backfiller"
-  spec.version       = "0.0.1"
+  spec.version       = "0.1.0"
   spec.authors       = ["Andriy Yanko"]
   spec.email         = ["andriy.yanko@railsware.com"]
 

data/lib/backfiller.rb

@@ -2,4 +2,4 @@ require 'backfiller/configuration'
 require 'backfiller/cursor'
 require 'backfiller/runner'
 
-require 'backfiller/railtie' if defined?(Rails)
+require 'backfiller/railtie' if defined?(Rails::Railtie)

data/lib/backfiller/runner.rb

@@ -4,12 +4,14 @@ module Backfiller
     attr_reader \
       :task,
       :connection_pool,
-      :batch_size
+      :batch_size,
+      :process_method
 
     def initialize(task_name)
       @task = build_task(task_name)
       @connection_pool = @task.respond_to?(:connection_pool) ? @task.connection_pool : Backfiller.connection_pool
       @batch_size = @task.respond_to?(:batch_size) ? @task.batch_size : Backfiller.batch_size
+      @process_method = @task.respond_to?(:process_row) ? @task.method(:process_row) : self.method(:process_row)
     end
 
     def run
@@ -17,7 +19,7 @@ module Backfiller
       worker_connection = acquire_connection
 
       fetch_each(master_connection) do |row|
-        update_row(worker_connection, row)
+        process_method.call(worker_connection, row)
       end
 
       release_connection(master_connection)
@@ -80,9 +82,10 @@ module Backfiller
       end
     end
 
-    def update_row(connection, row)
-      connection.execute task.update_sql(connection, row)
+    def process_row(connection, row)
+      Array(task.execute_sql(connection, row)).each do |sql|
+        connection.execute(sql)
+      end
     end
-
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backfiller
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Andriy Yanko
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-30 00:00:00.000000000 Z
+date: 2019-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -107,10 +107,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Backfiller for null database columns
 test_files: []
-has_rdoc: