online_migrations 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 10e5140728aecd56d8f79b52eda79f3244c50212a731db898f3fe36673082bdd
+  data.tar.gz: fa086fff375514e8779ef1d143af68037013373f1954ee4af2596ac3090f1ce9
+SHA512:
+  metadata.gz: c18e7f727443671759894c0320c050fbb789ce4015e49eeea751c07f9ec7232e4746c62554493e2118a08b2cc97a519c3645882d3a20c1091ed853a607039d54
+  data.tar.gz: e44ccdb25d5ea1da727e4c6828a55f7426322262388a7a1eb5f022bbd976116550dfbcd04efda289bc7340a5662343d0a283cb4f6fdd756aa7d07313e125dba5

data/.github/workflows/test.yml

@@ -0,0 +1,112 @@
+name: Test
+on: [push, pull_request]
+
+jobs:
+  # Run the linter first for rapid feedback if some trivial stylistic issues
+  # slipped through the cracks.
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.1.0
+          bundler-cache: true
+      - run: bundle exec rubocop
+
+  test:
+    needs: lint
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: online_migrations_test
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    strategy:
+      matrix:
+        include:
+          - ruby-version: 2.4.10
+            gemfile: gemfiles/activerecord_42.gemfile
+          - ruby-version: 2.5.9
+            gemfile: gemfiles/activerecord_42.gemfile
+          - ruby-version: 2.6.9
+            gemfile: gemfiles/activerecord_42.gemfile
+
+          - ruby-version: 2.4.10
+            gemfile: gemfiles/activerecord_50.gemfile
+          - ruby-version: 2.5.9
+            gemfile: gemfiles/activerecord_50.gemfile
+          - ruby-version: 2.6.9
+            gemfile: gemfiles/activerecord_50.gemfile
+          - ruby-version: 2.7.5
+            gemfile: gemfiles/activerecord_50.gemfile
+
+          - ruby-version: 2.4.10
+            gemfile: gemfiles/activerecord_51.gemfile
+          - ruby-version: 2.5.9
+            gemfile: gemfiles/activerecord_51.gemfile
+          - ruby-version: 2.6.9
+            gemfile: gemfiles/activerecord_51.gemfile
+          - ruby-version: 2.7.5
+            gemfile: gemfiles/activerecord_51.gemfile
+
+          - ruby-version: 2.4.10
+            gemfile: gemfiles/activerecord_52.gemfile
+          - ruby-version: 2.5.9
+            gemfile: gemfiles/activerecord_52.gemfile
+          - ruby-version: 2.6.9
+            gemfile: gemfiles/activerecord_52.gemfile
+          - ruby-version: 2.7.5
+            gemfile: gemfiles/activerecord_52.gemfile
+
+          - ruby-version: 2.5.9
+            gemfile: gemfiles/activerecord_60.gemfile
+          - ruby-version: 2.6.9
+            gemfile: gemfiles/activerecord_60.gemfile
+          - ruby-version: 2.7.5
+            gemfile: gemfiles/activerecord_60.gemfile
+          - ruby-version: 3.0.3
+            gemfile: gemfiles/activerecord_60.gemfile
+
+          - ruby-version: 2.5.9
+            gemfile: gemfiles/activerecord_61.gemfile
+          - ruby-version: 2.6.9
+            gemfile: gemfiles/activerecord_61.gemfile
+          - ruby-version: 2.7.4
+            gemfile: gemfiles/activerecord_61.gemfile
+          - ruby-version: 3.0.3
+            gemfile: gemfiles/activerecord_61.gemfile
+          - ruby-version: 3.1.0
+            gemfile: gemfiles/activerecord_61.gemfile
+
+          - ruby-version: 2.7.4
+            gemfile: gemfiles/activerecord_70.gemfile
+          - ruby-version: 3.0.3
+            gemfile: gemfiles/activerecord_70.gemfile
+          - ruby-version: 3.1.0
+            gemfile: gemfiles/activerecord_70.gemfile
+
+          - ruby-version: 2.7.4
+            gemfile: gemfiles/activerecord_head.gemfile
+          - ruby-version: 3.0.3
+            gemfile: gemfiles/activerecord_head.gemfile
+          - ruby-version: 3.1.0
+            gemfile: gemfiles/activerecord_head.gemfile
+    env:
+      BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+      - name: Run the test suite
+        run: bundle exec rake test

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+gemfiles/**.lock

data/.rubocop.yml

@@ -0,0 +1,113 @@
+AllCops:
+  NewCops: disable
+  SuggestExtensions: false
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/RescueModifier:
+  Exclude:
+    - test/**/*
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/SymbolArray:
+  EnforcedStyle: brackets
+
+Style/WordArray:
+  EnforcedStyle: brackets
+
+Style/GuardClause:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/MutableConstant:
+  Enabled: false
+
+Style/MissingRespondToMissing:
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: comma
+
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: comma
+
+Style/NumericPredicate:
+  Enabled: false
+
+Style/NegatedIf:
+  Enabled: false
+
+Style/ConditionalAssignment:
+  Enabled: false
+
+# only for ruby 2.3+
+Style/SafeNavigation:
+  Enabled: false
+
+Style/NumericLiterals:
+  Enabled: false
+
+Style/Next:
+  Enabled: false
+
+Style/GlobalVars:
+  Exclude:
+    - test/**/*
+
+Style/Lambda:
+  EnforcedStyle: literal
+
+Style/WhileUntilModifier:
+  Enabled: false
+
+Style/HashAsLastArrayItem:
+  Enabled: false
+
+# we can not use new syntax for older rubies
+Lint/ErbNewArguments:
+  Enabled: false
+
+Lint/MissingSuper:
+  Enabled: false
+
+Layout/EmptyLinesAroundAccessModifier:
+  EnforcedStyle: only_before
+
+Layout/IndentationConsistency:
+  EnforcedStyle: indented_internal_methods
+
+Layout/ArgumentAlignment:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false
+
+Layout/MultilineMethodCallIndentation:
+  Enabled: false
+
+Layout/HashAlignment:
+  Enabled: false
+
+Naming/VariableNumber:
+  Exclude:
+    - test/**/*
+
+Naming/MethodParameterName:
+  AllowedNames:
+    - of
+    - fk
+
+Naming/AccessorMethodName:
+  Exclude:
+    - test/**/*
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
+Metrics:
+  Enabled: false

data/.yardopts

@@ -0,0 +1 @@
+--no-private --markup markdown lib/**/**.rb - README.md

data/BACKGROUND_MIGRATIONS.md

@@ -0,0 +1,288 @@
+# Background Migrations
+
+When a project grows, your database starts to be heavy and changing the data through the deployment process can be very painful.
+
+Background migrations should be used to perform data migrations on large tables or when the migration will take a lot of time. For example, you can use background migrations to migrate data that’s stored in a single JSON column to a separate table instead or backfill some column's value from an API.
+
+**Note**: You probably don't need to use background migrations for smaller projects, since updating data directly on smaller databases will be perfectly fine and will not block the deployment too much.
+
+## Installation
+
+Make sure you have migration files generated when installed this gem:
+
+```sh
+$ bin/rails generate online_migrations:install
+```
+
+Start a background migrations scheduler. For example, to run it on cron using [whenever gem](https://github.com/javan/whenever) add the following lines to its `schedule.rb` file:
+
+```ruby
+every 1.minute do
+  runner "OnlineMigrations::BackgroundMigrations::Scheduler.run"
+end
+```
+
+## Creating a Background Migration
+
+A generator is provided to create background migrations. Generate a new background migration by running:
+
+```bash
+$ bin/rails generate online_migrations:background_migration backfill_project_issues_count
+```
+
+This creates the background migration file `lib/online_migrations/background_migrations/backfill_project_issues_count.rb`.
+
+The generated class is a subclass of `OnlineMigrations::BackgroundMigration` that implements:
+
+* `relation`: return an `ActiveRecord::Relation` to be iterated over
+* `process_batch`: do the work of your background migration on a batch (`ActiveRecord::Relation`)
+* `count`: return the number of rows that will be iterated over (optional, to be
+  able to show progress)
+
+Example:
+
+```ruby
+# lib/online_migrations/background_migrations/backfill_project_issues_count.rb
+
+module OnlineMigrations
+  module BackgroundMigrations
+    class BackfillProjectIssuesCount < OnlineMigrations::BackgroundMigration
+      class Project < ActiveRecord::Base; end
+
+      def relation
+        Project.all
+      end
+
+      def process_batch(projects)
+        projects.update_all(<<~SQL)
+          issues_count = (
+            SELECT COUNT(*)
+            FROM issues
+            WHERE issues.project_id = projects.id
+          )
+        SQL
+      end
+
+      def count
+        relation.count
+      end
+    end
+  end
+end
+```
+
+## Enqueueing a Background Migration
+
+You can enqueue your background migration to be run by the scheduler via:
+
+```ruby
+# db/migrate/xxxxxxxxxxxxxx_backfill_project_issues_count.rb
+# ...
+def up
+  enqueue_background_migration("BackfillProjectIssuesCount")
+end
+# ...
+```
+
+`enqueue_background_migration` accepts additional configuration options which controls how the background migration is run. Check the [source code](https://github.com/fatkodima/online_migrations/blob/master/lib/online_migrations/background_migrations/online_migrations.rb) for the list of all available configuration options.
+
+## Custom Background Migration Arguments
+
+Background migrations may need additional information, supplied via arguments, to run.
+
+Declare that the migration class is accepting additional arguments:
+
+```ruby
+class MyMigrationWithArgs < OnlineMigrations::BackgroundMigration
+  def initialize(arg1, arg2, ...)
+    @arg1 = arg1
+    @arg2 = arg2
+    ...
+  end
+  # ...
+end
+```
+
+And pass them when enqueuing:
+
+```ruby
+enqueue_background_migration("MyMigrationWithArgs", arg1, arg2, ...)
+```
+
+## Considerations when writing Background Migrations
+
+* **Isolation**: Background migrations should be isolated and not use application code (for example, models defined in `app/models`). Since these migrations can take a long time to run it's possible for new versions to be deployed while they are still running.
+* **Idempotence**: It should be safe to run `process_batch` multiple times for the same elements. It's important if the Background Migration errors and you run it again, because the same element that errored may be processed again. Make sure that in case that your migration job is going to be retried data integrity is guaranteed.
+
+## Testing
+
+At a minimum, it's recommended that the `#process_batch` method in your background migration is tested. You may also want to test the `#relation` and `#count` methods if they are sufficiently complex.
+
+Example:
+
+```ruby
+# test/online_migrations/background_migrations/backfill_project_issues_count_test.rb
+
+require "test_helper"
+
+module OnlineMigrations
+  module BackgroundMigrations
+    class BackfillProjectIssuesCountTest < ActiveSupport::TestCase
+      test "#process_batch performs a background migration iteration" do
+        rails = Project.create!(name: "rails")
+        postgres = Project.create!(name: "PostgreSQL")
+
+        2.times { rails.issues.create! }
+        _postgres_issue = postgres.issues.create!
+
+        BackfillProjectIssuesCount.new.process_batch(Project.all)
+
+        assert_equal 2, rails.issues_count
+        assert_equal 1, postgres.issues_count
+      end
+    end
+  end
+end
+```
+
+## Instrumentation
+
+Background migrations use the [ActiveSupport::Notifications](http://api.rubyonrails.org/classes/ActiveSupport/Notifications.html) API.
+
+You can subscribe to `background_migrations` events and log it, graph it, etc.
+
+To get notified about specific type of events, subscribe to the event name followed by the `background_migrations` namespace. E.g. for retries use:
+
+```ruby
+# config/initializers/online_migrations.rb
+ActiveSupport::Notifications.subscribe("retried.background_migrations") do |name, start, finish, id, payload|
+  # background migration job object is available in payload[:background_migration_job]
+
+  # Your code here
+end
+```
+
+If you want to subscribe to every `background_migrations` event, use:
+
+```ruby
+# config/initializers/online_migrations.rb
+ActiveSupport::Notifications.subscribe(/background_migrations/) do |name, start, finish, id, payload|
+  # background migration job object is available in payload[:background_migration_job]
+
+  # Your code here
+end
+```
+
+Available events:
+
+* `started.background_migrations`
+* `process_batch.background_migrations`
+* `completed.background_migrations`
+* `retried.background_migrations`
+* `throttled.background_migrations`
+
+## Monitoring Background Migrations
+
+Background Migrations can be in various states during its execution:
+
+* **enqueued**: A migration has been enqueued by the user.
+* **running**: A migration is being performed by a migration executor.
+* **paused**: A migration was paused in the middle of the run by the user.
+
+  To manually pause a migration, you can run:
+
+  ```ruby
+  migration = OnlineMigrations::BackgroundMigrations::Migration.find(id)
+  migration.paused!
+  ```
+* **finishing**: A migration is being manually finishing inline by the user.
+  For example, if you need to manually perform a background migration until it is finished, you can run:
+
+  ```ruby
+  migration = OnlineMigrations::BackgroundMigrations::Migration.find(id)
+  runner = OnlineMigrations::BackgroundMigrations::MigrationRunner.new(migration)
+  runner.finish
+  ```
+  Note: In normal circumstances, this should not be used since background migrations should be run and finished by the scheduler.
+* **failed**: A migration raises an exception when running.
+* **succeeded**: A migration finished without error.
+
+To get the progress (assuming `#count` method on background migration class was defined):
+
+```ruby
+migration = OnlineMigrations::BackgroundMigrations::Migration.find(id)
+migration.progress # value from 0 to 1.0
+```
+
+**Note**: It will be easier to work with background migrations through some kind of Web UI, but until it is implemented, we can work with them only manually.
+
+## Configuring
+
+There are a few configurable options for the Background Migrations. Custom configurations should be placed in a `online_migrations.rb` initializer.
+
+**Note**: Check the [source code](https://github.com/fatkodima/online_migrations/blob/master/lib/online_migrations/background_migrations/config.rb) for the list of all available configuration options.
+
+### Throttling
+
+Background Migrations often modify a lot of data and can be taxing on your database. There is a throttling mechanism that can be used to throttle a background migration when a given condition is met. If a migration is throttled, it will be interrupted and retried on the next Scheduler cycle run.
+
+Specify the throttle condition as a block:
+
+```ruby
+# config/initializers/online_migrations.rb
+
+OnlineMigrations.config.backround_migrations.throttler = -> { DatabaseStatus.unhealthy? }
+```
+
+Note that it's up to you to define a throttling condition that makes sense for your app. For example, you can check various PostgreSQL metrics such as replication lag, DB threads, whether DB writes are available, etc.
+
+### Customizing the error handler
+
+Exceptions raised while a Background Migration is performing are rescued and information about the error is persisted in the database.
+
+If you want to integrate with an exception monitoring service (e.g. Bugsnag), you can define an error handler:
+
+```ruby
+# config/initializers/online_migrations.rb
+
+OnlineMigrations.config.backround_migrations.error_handler = ->(error, errored_job) do
+  Bugsnag.notify(error) do |notification|
+    notification.add_metadata(:background_migration, { name: errored_job.migration_name })
+  end
+end
+```
+
+The error handler should be a lambda that accepts 2 arguments:
+
+* `error`: The exception that was raised.
+* `errored_job`: An `OnlineMigrations::BackgroundMigrations::MigrationJob` object that represents a failed batch.
+* `errored_element`: The `OnlineMigrations::BackgroundMigrations::MigrationJob` object representing a batch,
+  that was being processed when the Background Migration raised an exception.
+
+### Customizing the background migrations module
+
+`OnlineMigrations.config.background_migrations.migrations_module` can be configured to define the module in which
+background migrations will be placed.
+
+```ruby
+# config/initializers/online_migrations.rb
+
+OnlineMigrations.config.background_migrations.migrations_module = "BackgroundMigrationsModule"
+```
+
+If no value is specified, it will default to `OnlineMigrations::BackgroundMigrations`.
+
+### Customizing the backtrace cleaner
+
+`OnlineMigrations.config.background_migrations.backtrace_cleaner` can be configured to specify a backtrace cleaner to use when a Background Migration errors and the backtrace is cleaned and persisted. An `ActiveSupport::BacktraceCleaner` should be used.
+
+```ruby
+# config/initializers/online_migrations.rb
+
+cleaner = ActiveSupport::BacktraceCleaner.new
+cleaner.add_silencer { |line| line =~ /ignore_this_dir/ }
+
+OnlineMigrations.config.background_migrations.backtrace_cleaner = cleaner
+```
+
+If none is specified, the default `Rails.backtrace_cleaner` will be used to clean backtraces.

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## master (unreleased)
+
+## 0.1.0 (2022-01-17)
+
+- First release

data/Gemfile

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in online_migrations.gemspec
+gemspec
+
+gem "minitest", "~> 5.0"
+gem "rake", "~> 12.0"
+gem "yard"
+
+if defined?(@ar_gem_requirement)
+  gem "activerecord", @ar_gem_requirement
+  gem "railties", @ar_gem_requirement
+else
+  gem "activerecord" # latest
+  gem "railties" # to test generator
+
+  # Run Rubocop only on latest rubies, because it is incompatible with older versions.
+  gem "rubocop", "~> 1.24"
+end
+
+if defined?(@pg_gem_requirement)
+  gem "pg", @pg_gem_requirement
+else
+  gem "pg", "~> 1.2"
+end

data/Gemfile.lock

@@ -0,0 +1,108 @@
+PATH
+  remote: .
+  specs:
+    online_migrations (0.1.0)
+      activerecord (>= 4.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (7.0.1)
+      actionview (= 7.0.1)
+      activesupport (= 7.0.1)
+      rack (~> 2.0, >= 2.2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actionview (7.0.1)
+      activesupport (= 7.0.1)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activemodel (7.0.1)
+      activesupport (= 7.0.1)
+    activerecord (7.0.1)
+      activemodel (= 7.0.1)
+      activesupport (= 7.0.1)
+    activesupport (7.0.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    builder (3.2.4)
+    concurrent-ruby (1.1.9)
+    crass (1.0.6)
+    erubi (1.10.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    loofah (2.13.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    method_source (1.0.0)
+    mini_portile2 (2.7.1)
+    minitest (5.15.0)
+    nokogiri (1.13.0)
+      mini_portile2 (~> 2.7.0)
+      racc (~> 1.4)
+    parallel (1.21.0)
+    parser (3.1.0.0)
+      ast (~> 2.4.1)
+    pg (1.2.3)
+    racc (1.6.0)
+    rack (2.2.3)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.4.2)
+      loofah (~> 2.3)
+    railties (7.0.1)
+      actionpack (= 7.0.1)
+      activesupport (= 7.0.1)
+      method_source
+      rake (>= 12.2)
+      thor (~> 1.0)
+      zeitwerk (~> 2.5)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    regexp_parser (2.2.0)
+    rexml (3.2.5)
+    rubocop (1.24.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.15.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.15.1)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    thor (1.2.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    webrick (1.7.0)
+    yard (0.9.27)
+      webrick (~> 1.7.0)
+    zeitwerk (2.5.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  minitest (~> 5.0)
+  online_migrations!
+  pg (~> 1.2)
+  railties
+  rake (~> 12.0)
+  rubocop (~> 1.24)
+  yard
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 fatkodima
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.