rails_async_migrations 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a9ffc3ef530a9f156013b067663ddeea6b06c65e155a668c56936b6803a50945
+  data.tar.gz: e72fe2f4fe6289eff981e77653a3a1d00b97a90bd4750138a816ca38ea2a594f
+SHA512:
+  metadata.gz: 42e96d488de2c1b5d49b039e0174a193cfcbccae1fa619526e46f91b9edb840cf4f2c7041f325f43800cd705d3b5935d694b798402f7d9f3c6b34697fd721885
+  data.tar.gz: e8de2e332165ebdd35554bb1df53c74b77f5a3dd884cafe9624f522bcec133810b589e8c89263cad1097ead136f0d4e7f2ad0b2a660373c325c158d244b56f1b

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,138 @@
+# require: rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 2.5
+  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
+  # to ignore them, so only the ones explicitly set in this file are enabled.
+  DisabledByDefault: true
+  Exclude:
+    - '**/.DS_Store'
+    - 'db/**/*'
+    - 'tmp/**/*'
+    - 'vendor/**/*'
+    - 'bin/**/*'
+    - 'log/**/*'
+    - 'script/**/*'
+
+  Include:
+    - 'app/**/*'
+    - 'spec/**/*'
+    - 'lib/**/*'
+
+# Prefer &&/|| over and/or.
+Style/AndOr:
+  Enabled: true
+
+# Do not use braces for hash literals when they are the last argument of a
+# method call.
+Style/BracesAroundHashParameters:
+  Enabled: true
+
+# Align `when` with `case`.
+Layout/CaseIndentation:
+  Enabled: true
+
+# Align comments with method definitions.
+Layout/CommentIndentation:
+  Enabled: true
+
+# No extra empty lines.
+Layout/EmptyLines:
+  Enabled: true
+
+# In a regular class definition, no empty lines around the body.
+Layout/EmptyLinesAroundClassBody:
+  Enabled: true
+
+# In a regular method definition, no empty lines around the body.
+Layout/EmptyLinesAroundMethodBody:
+  Enabled: true
+
+# In a regular module definition, no empty lines around the body.
+Layout/EmptyLinesAroundModuleBody:
+  Enabled: true
+
+# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
+Style/HashSyntax:
+  Enabled: true
+
+# Two spaces, no tabs (for indentation).
+Layout/IndentationWidth:
+  Enabled: true
+
+Layout/SpaceAfterColon:
+  Enabled: true
+
+Layout/SpaceAfterComma:
+  Enabled: true
+
+Layout/SpaceAroundEqualsInParameterDefault:
+  Enabled: true
+
+Layout/SpaceAroundKeyword:
+  Enabled: true
+
+Layout/SpaceAroundOperators:
+  Enabled: true
+
+Layout/SpaceBeforeFirstArg:
+    Enabled: true
+
+# Defining a method with parameters needs parentheses.
+Style/MethodDefParentheses:
+  Enabled: true
+
+# Use `foo {}` not `foo{}`.
+Layout/SpaceBeforeBlockBraces:
+  Enabled: true
+
+# Use `foo { bar }` not `foo {bar}`.
+Layout/SpaceInsideBlockBraces:
+  Enabled: true
+
+# Use `{ a: 1 }` not `{a:1}`.
+Layout/SpaceInsideHashLiteralBraces:
+  Enabled: true
+
+Layout/SpaceInsideParens:
+  Enabled: true
+
+# Check quotes usage according to lint rule below.
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: single_quotes
+
+# Detect hard tabs, no hard tabs.
+Layout/Tab:
+  Enabled: true
+
+# Blank lines should not have any spaces.
+Layout/TrailingBlankLines:
+  Enabled: true
+
+# No trailing whitespace.
+Layout/TrailingWhitespace:
+  Enabled: true
+
+# Use quotes for string literals when they are enough.
+Style/UnneededPercentQ:
+  Enabled: true
+
+# Align `end` with the matching keyword or starting expression except for
+# assignments, where it should be aligned with the LHS.
+Layout/EndAlignment:
+  Enabled: true
+  EnforcedStyleAlignWith: variable
+
+# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
+Lint/RequireParentheses:
+  Enabled: true
+
+Lint/Debugger:
+  Enabled: true
+
+Metrics/LineLength:
+  # This will disable the rule completely, regardless what other options you put
+  Enabled: true
+  # Change the default 80 chars limit value
+  Max: 150

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.1
+before_install: gem install bundler -v 1.17.1

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'activerecord', '~> 5.2.0'
+gem 'sidekiq', '~> 5.2.3'
+gem 'delayed_job_active_record', '~> 4.1.3'

data/Gemfile.lock

@@ -0,0 +1,171 @@
+PATH
+  remote: .
+  specs:
+    rails_async_migrations (1.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (5.2.2)
+      actionpack (= 5.2.2)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailer (5.2.2)
+      actionpack (= 5.2.2)
+      actionview (= 5.2.2)
+      activejob (= 5.2.2)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (5.2.2)
+      actionview (= 5.2.2)
+      activesupport (= 5.2.2)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.2.2)
+      activesupport (= 5.2.2)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (5.2.2)
+      activesupport (= 5.2.2)
+      globalid (>= 0.3.6)
+    activemodel (5.2.2)
+      activesupport (= 5.2.2)
+    activerecord (5.2.2)
+      activemodel (= 5.2.2)
+      activesupport (= 5.2.2)
+      arel (>= 9.0)
+    activestorage (5.2.2)
+      actionpack (= 5.2.2)
+      activerecord (= 5.2.2)
+      marcel (~> 0.3.1)
+    activesupport (5.2.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
+    builder (3.2.3)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.3)
+    connection_pool (2.2.2)
+    crass (1.0.4)
+    database_cleaner (1.7.0)
+    delayed_job (4.1.5)
+      activesupport (>= 3.0, < 5.3)
+    delayed_job_active_record (4.1.3)
+      activerecord (>= 3.0, < 5.3)
+      delayed_job (>= 3.0, < 5)
+    diff-lcs (1.3)
+    erubi (1.7.1)
+    globalid (0.4.1)
+      activesupport (>= 4.2.0)
+    i18n (1.2.0)
+      concurrent-ruby (~> 1.0)
+    loofah (2.2.3)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marcel (0.3.3)
+      mimemagic (~> 0.3.2)
+    method_source (0.9.2)
+    mimemagic (0.3.2)
+    mini_mime (1.0.1)
+    mini_portile2 (2.3.0)
+    minitest (5.11.3)
+    nio4r (2.3.1)
+    nokogiri (1.8.5)
+      mini_portile2 (~> 2.3.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rack (2.0.6)
+    rack-protection (2.0.5)
+      rack
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails (5.2.2)
+      actioncable (= 5.2.2)
+      actionmailer (= 5.2.2)
+      actionpack (= 5.2.2)
+      actionview (= 5.2.2)
+      activejob (= 5.2.2)
+      activemodel (= 5.2.2)
+      activerecord (= 5.2.2)
+      activestorage (= 5.2.2)
+      activesupport (= 5.2.2)
+      bundler (>= 1.3.0)
+      railties (= 5.2.2)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
+    railties (5.2.2)
+      actionpack (= 5.2.2)
+      activesupport (= 5.2.2)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.19.0, < 2.0)
+    rake (10.5.0)
+    redis (4.1.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-sidekiq (3.0.3)
+      rspec-core (~> 3.0, >= 3.0.0)
+      sidekiq (>= 2.4.0)
+    rspec-support (3.8.0)
+    sidekiq (5.2.3)
+      connection_pool (~> 2.2, >= 2.2.2)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.5, < 5)
+    sprockets (3.7.2)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.1)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    sqlite3 (1.3.13)
+    thor (0.20.3)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    websocket-driver (0.7.0)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (~> 5.2.0)
+  bundler (~> 1.17)
+  database_cleaner
+  delayed_job_active_record (~> 4.1.3)
+  pry (~> 0.12)
+  rails (~> 5.2)
+  rails_async_migrations!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rspec-sidekiq
+  sidekiq (~> 5.2.3)
+  sqlite3
+
+BUNDLED WITH
+   1.17.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Laurent Schaffner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,162 @@
+# RailsAsyncMigrations
+
+`ActiveRecord::Migration` extension to turn your migrations asynschonous in a simple and straight forward way.
+
+## Motives
+
+This library was made with the intent to help small companies which are struggling to scale at technical level. Small projects don't need asynchronous migrations queues, and big companies build their own parallel systems when facing scaling problems, but what about medium sized companies with limited resources ?
+
+When a project grows, your database starts to be heavy and changing the data through the deployment process can be very painful. There are numerous reasons you want this process to be [at least] partially asynchronous.
+
+Most people turn heavy data changes into `rake tasks` or split workers; there are two schools of thought about this.
+
+1. Migrations should only mutate database structures and not its data, and if it's the case, it should be split and processed via other means.
+2. Migrations are everything which alter data one time, typically during a deployment of new code and structure.
+
+Turning data changes into a `rake task` can be a good idea, and it's ideal to test it out too, but sometimes you need this **fat ass loop** of **>1,000,000 records** which will be run **once, and only once** to be run fast and without locking down the deployment process itself; making a `rake task` for that is overkill. After all, it will only be used once and within a specific structure / data conxtext. This gem is here to answer this need.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rails_async_migrations'
+```
+
+And then execute:
+
+    $ bundle
+
+After the gem has been installed, use the generator to add the needed changes
+
+    $ rails generate rails_async_migrations:install
+
+This will add a new migration for the table `async_schema_migrations` which will be used by the gem. You can also add the migration yourself like so:
+
+```
+class CreateAsyncSchemaMigrations < ActiveRecord::Migration[5.2]
+  def change
+    create_table :async_schema_migrations do |t|
+      t.string :version
+      t.string :direction
+      t.string :state
+
+      t.timestamps
+    end
+  end
+end
+```
+
+## Usage
+
+To turn some of your migrations asynchronous, generate a migration as you would normally do and use the `turn_async` keyword.
+
+```
+class Test < ActiveRecord::Migration[5.2]
+  turn_async
+
+  def change
+    create_table 'tests' do |t|
+      t.string :test
+    end
+  end
+end
+```
+
+From now on, when you run this migration it'll simply run the normal queue, but the content of `#change` will be taken away and later on executed into an asynchronous queue.
+
+What is turned asynchronous is executed exactly the same way as a classical migration, which means you can use all keywords of the classic `ActiveRecord::Migration` such as `create_table`, `add_column`, etc.
+
+## Configuration
+
+**No configuration is needed to start using this library**, but some options are given to adapt to your needs.
+
+Add the following lines into your `config/initializer/` folder
+
+```
+RailsAsyncMigrations.config do |config|
+  # :verbose can be used if you want a full log of the execution
+  config.mode = :quiet
+
+  # which kind of worker do you want to use for this library
+  # for now you have two options: :delayed_job or :sidekiq
+  config.workers = :sidekiq
+
+  # when the migration is turned asynchronous
+  # it watches over some specific `ActiveRecord` methods
+  # by adding them to this array, you'll lock and turn those methods asynchronous
+  # by removing them you'll let them use the classical migration process
+  # for example, if you set the `locked_methods` to %i[async] the migration will be processed normally
+  # but the content of the `#async` method will be taken away and executed within the asynchronous queue
+  config.locked_methods =  %i[change up down]
+end
+```
+
+## Queue
+
+Each migration which is turned asynchronous follows each other, once one migration of the queue is ended without problem, it passes to the next one.
+
+If it fails, the error will be raised within the worker so it retries until it eventually works, or until it's considered dead. None of the further asynchronous migrations will be run until you fix the failed one, which is a good protection for data consistency.
+
+You can also manually launch the queue check and fire by using:
+
+    $ rake rails_async_migrations:check_queue
+
+**For now, there is no rollback mechanism authorized, even if the source code is ready for it, it complexifies the build up logic and may not be needed in asynchronous cases.**
+
+## States
+
+| State          | Description                                                                                                                  |
+| -------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| **created**    | the migration has just been added through the classical migration queue                                                      |
+| **pending**    | the migration has been spotted by the asynchronous queue and will be processed                                               |
+| **processing** | the migration is being processed right now                                                                                   |
+| **done**       | the migration was successful                                                                                                 |
+| **failed**     | the migration failed while being processed, there may be other attempts to make it pass depending your workers configuration |
+
+## Failure handling
+
+If your migration crashes, and blocks the rest of your asynchronous migrations but you want to execute them anyway, you can change the code of the migration file and push it again so it passes, or simply remove / update as `state = done` the matching `async_schema_migrations` row.
+
+The `version` value is always the same as the classic migrations ones.
+
+## Compatibility
+
+You can use this library through different adapters
+
+| Type             | Version | Documentation                                 |
+| ---------------- | ------- | --------------------------------------------- |
+| **Sidekiq**      | 5.2.3   | https://github.com/mperham/sidekiq            |
+| **Delayed::Job** | 4.1.3   | https://github.com/collectiveidea/delayed_job |
+
+If you use other technologies to setup your workers, please hit me up and I'll create additional adapters for you.
+
+The gem has been tested and is working with `ActiveRecord 5.2.2`, if you notice abnormal behavior with other versions or want it compatible with earlier versions, please hit me up.
+
+## Development
+
+I created this library as my company was struggling in its deployment process. It lacks functionalities but this is a good starting point; everything is easily extendable so don't hesitate to add your own needed methods to it.
+
+You're more than welcome to open an issue with feature requests so I can work on improving this library.
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/rails_async_migrations/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Author
+
+[Laurent Schaffner](http://www.laurentschaffner.com)
+
+## Credits
+
+This project and its idea was inspired by [Kir Shatrov article](https://kirshatrov.com/2018/04/01/async-migrations/) on the matter, it's worth a look!
+
+## License
+
+MIT License.
+
+## Changelog

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "rails_async_migrations"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/generators/rails_async_migrations/install_generator.rb

@@ -0,0 +1,22 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+
+module RailsAsyncMigrations
+  module Generators
+    class InstallGenerator < ::Rails::Generators::Base
+      include Rails::Generators::Migration
+      source_root File.expand_path('../templates', __FILE__)
+      desc "Add the migrations for AsyncSchemaMigration"
+
+      def self.next_migration_number(path)
+        next_migration_number = current_migration_number(path) + 1
+        ActiveRecord::Migration.next_migration_number(next_migration_number)
+      end
+
+      def copy_migrations
+        migration_template "create_async_schema_migrations.rb", 
+        "db/migrate/create_async_schema_migrations.rb"
+      end
+    end
+  end
+end

data/lib/generators/rails_async_migrations/templates/create_async_schema_migrations.rb

@@ -0,0 +1,11 @@
+class CreateAsyncSchemaMigrations < ActiveRecord::Migration[5.2]
+  def change
+    create_table :async_schema_migrations do |t|
+      t.string :version
+      t.string :direction
+      t.string :state
+
+      t.timestamps
+    end
+  end
+end

data/lib/rails_async_migrations/config.rb

@@ -0,0 +1,13 @@
+# configuration of the gem and
+# default values set here
+module RailsAsyncMigrations
+  class Config
+    attr_accessor :locked_methods, :mode, :workers
+
+    def initialize
+      @locked_methods = %i[change up down]
+      @mode = :quiet # verbose, quiet
+      @workers = :sidekiq # delayed_job, sidekiq
+    end
+  end
+end

data/lib/rails_async_migrations/connection/active_record.rb

@@ -0,0 +1,59 @@
+module RailsAsyncMigrations
+  module Connection
+    class ActiveRecord
+      attr_reader :current_direction
+
+      def initialize(current_direction)
+        @current_direction = current_direction
+      end
+
+      # NOTE : down isn't available
+      # from the public API of the gem
+      def current_version
+        if current_direction == :down
+          migration_context.current_version
+        elsif current_direction == :up
+          pending_migrations.first
+        end
+      end
+
+      def current_migration
+        @current_migration ||= migration_from current_version
+      end
+
+      def migration_from(version)
+        migration_context.migrations.find do |migration|
+          migration.version.to_s == version.to_s
+        end
+      end
+
+      def allowed_direction?
+        current_direction == :up
+      end
+
+      private
+
+      def pending_migrations
+        achieved_migrations - all_migrations
+      end
+
+      def achieved_migrations
+        migration_context.migrations.collect(&:version)
+      end
+
+      def all_migrations
+        migration_context.get_all_versions
+      end
+
+      def migration_context
+        connection.migration_context
+      end
+
+      # NOTE: seems at it was ActiveRecord::Migrator
+      # in anterior versions
+      def connection
+        @connection || ::ActiveRecord::Base.connection
+      end
+    end
+  end
+end