backfiller 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d546e155a0790fdb0190454b6f59ccfa7847000ea93e64c785b7af932446c2aa
-  data.tar.gz: f8fb83e301f507909783ac29eb405c80d7497c318915a6af2c3df803f2a85ce2
+  metadata.gz: 18b60be56e9153bca870e126330f666518c012c885c2e3d86d5c24625b395780
+  data.tar.gz: 06e10fa60228916766f7740231add1e1acc0c3f7dec1b9a1055785d908941a29
 SHA512:
-  metadata.gz: 6741e63a19b01bbc57829817f558838ccbe31adc18ec53fcad2115b23cba122a396e4981d0b773ad46a3af3d531771efd29fbca5d20a700a9a849a31a196b2d4
-  data.tar.gz: 0bf707056834e42d4e94fcf50477d121eb4e2fb8b4b31a98e13eb5be926a7cb32d1260726fd10f14d43ed0c3631bafc5788257cc835dab551cba57d084250fc8
+  metadata.gz: 0315e6720c7ec5c15904696843caef797a5beceb2771934b01dcc61fddbbb4f65c28fb9b4a60ad94ea2c1d31161facc64f5016a8a236f506ea02f560b94e9262
+  data.tar.gz: 5a085ff3f03022926e6e188f1900f3697658ca78489e45273fce7fc6d6a78e93708c481c7bcfd09ddf049fe223c04f8ac6f625d7dc0fa103282070b1297d23f0

data/.gitignore

@@ -7,6 +7,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+log/*.log
 
 # rspec failure tracking
 .rspec_status

data/.rspec

@@ -1,2 +1,3 @@
+--require spec_helper
 --format documentation
 --color

data/.rubocop.yml

@@ -0,0 +1,16 @@
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/AccessorGrouping:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*_spec.rb'
+
+Metrics/MethodLength:
+  Max: 20

data/.travis.yml

@@ -0,0 +1,14 @@
+language: ruby
+rvm:
+  - 2.7.2
+gemfile:
+  - Gemfile
+cache: bundler
+before_install:
+  - gem install bundler
+before_script:
+  - psql -c 'create database test;' -U postgres
+script:
+  - bundle exec rake ci
+services:
+  - postgresql

data/Changelog.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.2.0
+
+* Upgrade local gems
+* Add rubocop
+* Add specs
+* Configure travis
+* Add `cursor_threshold` feature
+
 ## 0.1.1
 
 * Support Rails 6.0.0

data/Gemfile

@@ -1,6 +1,5 @@
-source "https://rubygems.org"
+# frozen_string_literal: true
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+source 'https://rubygems.org'
 
-# Specify your gem's dependencies in backfiller.gemspec
 gemspec

data/README.md

@@ -1,6 +1,6 @@
 ![Backfill machine](https://railsware.github.io/backfiller/assets/backfill_machine.jpg)
 
-# Backfiller
+# Backfiller [![Build Status](https://travis-ci.com/railsware/backfiller.svg?branch=master)](https://travis-ci.com/railsware/backfiller)
 
 The backfill machine for null database columns.
 This gem maybe handly for `no-downtime` deployment especially when you need to fill columns for table with huge amount for records without locking the table.

data/Rakefile

@@ -1,6 +1,11 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
 
-task :default => :spec
+desc 'CI build'
+task ci: %i[spec rubocop]

data/backfiller.gemspec

@@ -1,27 +1,33 @@
-# coding: utf-8
-lib = File.expand_path("../lib", __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
-  spec.name          = "backfiller"
-  spec.version       = "0.1.1"
-  spec.authors       = ["Andriy Yanko"]
-  spec.email         = ["andriy.yanko@railsware.com"]
+  spec.name = 'backfiller'
+  spec.version = '0.2.0'
+  spec.authors = ['Andriy Yanko']
+  spec.email = ['andriy.yanko@railsware.com']
+
+  spec.summary = 'Backfiller for null database columns'
+  spec.homepage = 'https://github.com/railsware/backfiller'
+  spec.license = 'MIT'
 
-  spec.summary       = %q{Backfiller for null database columns}
-  spec.homepage      = "https://github.com/railsware/backfiller"
-  spec.license       = "MIT"
+  spec.required_ruby_version = '>= 2.7.0'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activerecord', '>= 6.0.0'
 
-  spec.add_dependency "activerecord", ">= 5.0.0"
+  spec.add_development_dependency 'bundler', '~> 2.2.0'
+  spec.add_development_dependency 'rake', '~> 13.0.0'
+  spec.add_development_dependency 'rspec', '~> 3.10.0'
+  spec.add_development_dependency 'rubocop', '~> 1.18.0'
 
-  spec.add_development_dependency "bundler", "~> 1.15"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency 'pg', '~> 1.2.0'
 end

data/bin/console

@@ -1,14 +1,8 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-require "bundler/setup"
-require "backfiller"
+require 'bundler/setup'
+require 'backfiller'
 
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
+require 'irb'
 IRB.start(__FILE__)

data/lib/backfiller.rb

@@ -1,5 +1,7 @@
-require 'backfiller/configuration'
-require 'backfiller/cursor'
-require 'backfiller/runner'
+# frozen_string_literal: true
 
-require 'backfiller/railtie' if defined?(Rails::Railtie)
+require_relative 'backfiller/configuration'
+require_relative 'backfiller/cursor'
+require_relative 'backfiller/runner'
+
+require_relative 'backfiller/railtie' if defined?(Rails::Railtie)

data/lib/backfiller/configuration.rb

@@ -1,25 +1,36 @@
-module Backfiller
+# frozen_string_literal: true
 
+module Backfiller
   class << self
     def configure
       yield self
     end
 
+    # directory for backfill ruby classes
     attr_accessor :task_directory
 
+    # ruby module of backfill classes
     attr_accessor :task_namespace
 
+    # Max size of records in one cursor fetch
     attr_accessor :batch_size
 
+    # Size of processed records after which cursor will be re-opened
+    attr_accessor :cursor_threshold
+
+    # Logger
     attr_accessor :logger
 
+    # @param task_name [String] name of backfill task file
     def run(task_name)
       Backfiller::Runner.new(task_name).run
     end
 
+    # @param message [String] log message
     def log(message)
-      logger.info "[Backfiller] #{message}" if logger
+      return unless logger
+
+      logger.info "[Backfiller] #{message}"
     end
   end
-
 end

data/lib/backfiller/cursor.rb

@@ -1,8 +1,9 @@
-require 'backfiller/cursor/postgresql'
+# frozen_string_literal: true
+
+require_relative 'cursor/postgresql'
 
 module Backfiller
   module Cursor
-
     def self.new(connection, *args)
       case connection
       when ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
@@ -11,6 +12,5 @@ module Backfiller
         raise "Unsupported connection #{connection.inspect}"
       end
     end
-
   end
 end

data/lib/backfiller/cursor/postgresql.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
+
 module Backfiller
   module Cursor
     class Postgresql
-
       attr_reader :connection
 
       def initialize(connection, name, query)
@@ -10,6 +11,25 @@ module Backfiller
         @query = query
       end
 
+      # Open cursor, call black and close cursor in transaction.
+      #
+      # @return [Object] yielded block result.
+      def transaction
+        result = nil
+
+        @connection.transaction do
+          Backfiller.log 'Open cursor'
+          open
+
+          result = yield
+
+          Backfiller.log 'Close cursor'
+          close
+        end
+
+        result
+      end
+
       def open
         @connection.execute "DECLARE #{@name} NO SCROLL CURSOR WITHOUT HOLD FOR #{@query}"
       end
@@ -21,7 +41,6 @@ module Backfiller
       def close
         @connection.execute "CLOSE #{@name}"
       end
-
     end
   end
 end

data/lib/backfiller/railtie.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
+
 module Backfiller
   class Railtie < Rails::Railtie
-
     rake_tasks do
       load 'backfiller/tasks/db.rake'
     end
@@ -13,6 +14,8 @@ module Backfiller
 
         config.batch_size = 1_000
 
+        config.cursor_threshold = 100_000
+
         config.logger = defined?(ApplicationRecord) ? ApplicationRecord.logger : ActiveRecord::Base.logger
       end
     end
@@ -21,6 +24,5 @@ module Backfiller
       task_module = Backfiller.task_namespace.classify
       Object.const_set(task_module, Module.new) unless Object.const_defined?(task_module)
     end
-
   end
 end

data/lib/backfiller/runner.rb

@@ -1,24 +1,52 @@
+# frozen_string_literal: true
+
 module Backfiller
   class Runner
-
     attr_reader \
       :task,
       :connection_pool,
       :batch_size,
+      :cursor_threshold,
       :process_method
 
     def initialize(task_name)
       @task = build_task(task_name)
       @connection_pool = @task.respond_to?(:connection_pool) ? @task.connection_pool : default_connection_pool
       @batch_size = @task.respond_to?(:batch_size) ? @task.batch_size : Backfiller.batch_size
-      @process_method = @task.respond_to?(:process_row) ? @task.method(:process_row) : self.method(:process_row)
+      @cursor_threshold = @task.respond_to?(:cursor_threshold) ? @task.cursor_threshold : Backfiller.cursor_threshold
+      @process_method = @task.respond_to?(:process_row) ? @task.method(:process_row) : method(:process_row)
     end
 
+    # It uses two connections from pool:
+    # * master [M] - reads data using cursor in transaction
+    # * worker [W] - changes data based on record red from master
+    #
+    # @example
+    #   [M] BEGIN
+    #   [M] DECLARE backfill_cursor SCROLL CURSOR WITHOUT HOLD FOR SELECT * FROM users
+    #   // Start fetch and process loop:
+    #   [M] FETCH 1000 backfill_cursor
+    #     [W] UPDATE users SET full_name = '...' where id = 1
+    #     [W] ...
+    #     [W] UPDATE users SET full_name = '...' where id = 1000
+    #   [M] FETCH 1000 backfill_cursor
+    #     [W] UPDATE users SET full_name = '...' where id = 1001
+    #     [W] ...
+    #     [W] UPDATE users SET full_name = '...' where id = 2000
+    #   // Records per cursor transaction threshold reached. Reopen transaction.
+    #   [M] CLOSE backfill_cursor
+    #   [M] COMMIT
+    #   [M] BEGIN
+    #   [M] DECLARE backfill_cursor SCROLL CURSOR WITHOUT HOLD FOR SELECT * FROM users
+    #   [M] FETCH 1000 backfill_cursor
+    #   // The end of cursor reached. Break cursor loop and exit.
+    #   [M] CLOSE backfill_cursor
+    #   [M] COMMIT
     def run
       master_connection = acquire_connection
       worker_connection = acquire_connection
 
-      fetch_each(master_connection) do |row|
+      run_cursor_loop(master_connection) do |row|
         process_method.call(worker_connection, row)
       end
 
@@ -50,32 +78,34 @@ module Backfiller
 
     ###########################################################################
 
-    def build_cursor(connection)
-      Backfiller::Cursor.new(connection, 'backfill_cursor', task.select_sql)
-    end
+    # Run loop that re-open cursor transaction on threshold
+    def run_cursor_loop(connection, &block)
+      Backfiller.log 'Start cursor loop'
 
-    def fetch_each(master_connection, &block)
-      cursor = build_cursor(master_connection)
+      total_count = 0
+      cursor = build_cursor(connection)
 
-      cursor.connection.transaction do
-        Backfiller.log "Open cursor"
-        cursor.open
+      loop do
+        finished, count = cursor.transaction do
+          run_fetch_loop(cursor, &block)
+        end
 
-        Backfiller.log "Start fetch loop"
-        fetch_loop(cursor, &block)
+        total_count += count
 
-        Backfiller.log "Close cursor"
-        cursor.close
+        Backfiller.log "Total processed #{total_count}"
+        break if finished
       end
     end
 
-    def fetch_loop(cursor, &block)
+    # @return [Array<Boolean, Integer>] finished_status/processed_count
+    def run_fetch_loop(cursor, &block)
+      Backfiller.log 'Start fetch loop'
       count = 0
 
       loop do
         result = cursor.fetch(batch_size)
 
-        break if result.empty?
+        return [true, count] if result.empty?
 
         result.each do |row|
           block.call(row)
@@ -83,9 +113,19 @@ module Backfiller
         end
 
         Backfiller.log "Processed #{count}"
+
+        return [false, count] if count > cursor_threshold
       end
     end
 
+    ###########################################################################
+
+    # Build cursor object that will use master connection.
+    def build_cursor(connection)
+      Backfiller::Cursor.new(connection, 'backfill_cursor', task.select_sql)
+    end
+
+    # Process row using worker connection.
     def process_row(connection, row)
       Array(task.execute_sql(connection, row)).each do |sql|
         connection.execute(sql)

data/lib/backfiller/tasks/db.rake

@@ -1,10 +1,11 @@
-namespace :db do
+# frozen_string_literal: true
 
+namespace :db do
   desc 'Run database backfill task'
-  task :backfill, [:name] => :environment do |t, args|
+  task :backfill, [:name] => :environment do |_, args|
     raise 'Please specify backfill task name' unless args[:name]
+
     Backfiller.logger.level = :info if Backfiller.logger
     Backfiller.run(args[:name])
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backfiller
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Andriy Yanko
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-10-31 00:00:00.000000000 Z
+date: 2021-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,57 +16,85 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: 2.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 13.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 13.0.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
-description: 
+        version: 3.10.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.18.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.18.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+description:
 email:
 - andriy.yanko@railsware.com
 executables: []
@@ -75,6 +103,8 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
 - Changelog.md
 - Gemfile
 - README.md
@@ -82,6 +112,7 @@ files:
 - backfiller.gemspec
 - bin/console
 - bin/setup
+- db/backfill/.keep
 - lib/backfiller.rb
 - lib/backfiller/configuration.rb
 - lib/backfiller/cursor.rb
@@ -89,11 +120,12 @@ files:
 - lib/backfiller/railtie.rb
 - lib/backfiller/runner.rb
 - lib/backfiller/tasks/db.rake
+- log/.keep
 homepage: https://github.com/railsware/backfiller
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -101,15 +133,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Backfiller for null database columns
 test_files: []