online_migrations 0.23.0 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1052d2cc58898bc561b5e755daccc137d7d6e1d35afa3af5968c2b655ebe5805
-  data.tar.gz: ad72d601c36ea9192f192fb8941d30a2cfbcf0ed87e0717fb5d4ebf47b20f169
+  metadata.gz: 99170e244f0f008e8d097222c88a96555692aa5afc136087d03341866ed8412c
+  data.tar.gz: 6068a722274c73d8a0ab3c1bed4d1f2673cd99d15ca77c24f7fff5af75e77129
 SHA512:
-  metadata.gz: 697ac61bcb41e00744cfa87965910a3e1d68eaa571ad5c6dc3b3d8a4d14deb16ae0cbc0d9771d8d57668be8529f200736a14c1769d7d8cf6c64fc083060ff74c
-  data.tar.gz: 7b4e383f7628ba5cc74c2bb86f4460890b70f3052738f0f20db3571ee770e22c49bc548f5c0ef3fd47caa456a84eb4174f031742869612ee3b27505faf49ecec
+  metadata.gz: 9d2a127aee978d36676d8bdd85352487f66a14289cca28173eb1126aee65ceeea5dda067171ffb320b7c0cb2be903712ab158af12312452180b8e56da1e7b641
+  data.tar.gz: 305f89162f87a3746d91bfdfacae00b7d0f2a374f27755c846cd4386d65694442b70fc6963a7dec8845c192eca2cc625bb020c7c4aec664320a395b124bb3e71

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## master (unreleased)
 
+## 0.24.0 (2025-01-20)
+
+- Add ability to run a separate background migrations scheduler per shard
+
+- Revert "Prevent multiple instances of schedulers from being running simultaneously"
+
+    The feature was implemented using advisory locks, but they do not always play nicely with the
+    database poolers, so the feature was reverted. It is expected now for user's code to prevent
+    multiple instances of the schedulers from being run (by wrapping it into Redis lock etc).
+
 ## 0.23.0 (2025-01-13)
 
 - Prevent multiple instances of schedulers from being running simultaneously

data/docs/background_data_migrations.md

@@ -349,3 +349,13 @@ OnlineMigrations::ApplicationRecord.connects_to database: { writing: :shard_one
 
 By default, ActiveRecord uses the database config named `:primary` (if exists) under the environment section from the `database.yml`.
 Otherwise, the first config under the environment section is used.
+
+By default, the scheduler works on a single shard on each run. To run a separate scheduler per shard:
+
+```ruby
+[:shard_one, :shard_two, :shard_three].each do |shard|
+  every 1.minute do
+    runner "OnlineMigrations.run_background_data_migrations(shard: :#{shard})"
+  end
+end
+```

data/docs/background_schema_migrations.md

@@ -192,3 +192,13 @@ OnlineMigrations::ApplicationRecord.connects_to database: { writing: :shard_one
 
 By default, ActiveRecord uses the database config named `:primary` (if exists) under the environment section from the `database.yml`.
 Otherwise, the first config under the environment section is used.
+
+By default, the scheduler works on a single shard on each run. To run a separate scheduler per shard:
+
+```ruby
+[:shard_one, :shard_two, :shard_three].each do |shard|
+  every 1.minute do
+    runner "OnlineMigrations.run_background_schema_migrations(shard: :#{shard})"
+  end
+end
+```

data/lib/online_migrations/background_migrations/scheduler.rb

@@ -9,36 +9,34 @@ module OnlineMigrations
     # successive runs has passed.
     #
     # Scheduler should be configured to run periodically, for example, via cron.
+    #
     # @example Run via whenever
     #   # add this to schedule.rb
     #   every 1.minute do
-    #     runner "OnlineMigrations.run_background_migrations"
+    #     runner "OnlineMigrations.run_background_data_migrations"
+    #   end
+    #
+    # @example Run via whenever (specific shard)
+    #   every 1.minute do
+    #     runner "OnlineMigrations.run_background_data_migrations(shard: :shard_two)"
     #   end
     #
     class Scheduler
-      def self.run
-        new.run
+      def self.run(**options)
+        new.run(**options)
       end
 
       # Runs Scheduler
-      def run
+      def run(**options)
         active_migrations = Migration.runnable.active.queue_order
+        active_migrations = active_migrations.where(shard: options[:shard]) if options.key?(:shard)
         runnable_migration = active_migrations.select(&:interval_elapsed?).first
 
         if runnable_migration
           runner = MigrationRunner.new(runnable_migration)
-
-          try_with_lock do
-            runner.run_migration_job
-          end
+          runner.run_migration_job
         end
       end
-
-      private
-        def try_with_lock(&block)
-          lock = AdvisoryLock.new(name: "online_migrations_data_scheduler")
-          lock.try_with_lock(&block)
-        end
     end
   end
 end

data/lib/online_migrations/background_schema_migrations/scheduler.rb

@@ -7,34 +7,41 @@ module OnlineMigrations
     # running migration on the same table.
     #
     # Scheduler should be configured to run periodically, for example, via cron.
+    #
     # @example Run via whenever
     #   # add this to schedule.rb
     #   every 1.minute do
     #     runner "OnlineMigrations.run_background_schema_migrations"
     #   end
     #
+    # @example Run via whenever (specific shard)
+    #   every 1.minute do
+    #     runner "OnlineMigrations.run_background_schema_migrations(shard: :shard_two)"
+    #   end
+    #
     class Scheduler
-      def self.run
-        new.run
+      def self.run(**options)
+        new.run(**options)
       end
 
       # Runs Scheduler
-      def run
-        migration = find_migration
+      def run(**options)
+        migration = find_migration(**options)
         if migration
           runner = MigrationRunner.new(migration)
-
-          try_with_lock do
-            runner.run
-          end
+          runner.run
         end
       end
 
       private
-        def find_migration
+        def find_migration(**options)
           active_migrations = Migration.running.reject(&:stuck?)
           runnable_migrations = Migration.runnable.enqueued.queue_order.to_a + Migration.retriable.queue_order.to_a
 
+          if options.key?(:shard)
+            runnable_migrations = runnable_migrations.select { |migration| migration.shard.to_s == options[:shard].to_s }
+          end
+
           runnable_migrations.find do |runnable_migration|
             active_migrations.none? do |active_migration|
               active_migration.connection_class_name == runnable_migration.connection_class_name &&
@@ -43,11 +50,6 @@ module OnlineMigrations
             end
           end
         end
-
-        def try_with_lock(&block)
-          lock = AdvisoryLock.new(name: "online_migrations_schema_scheduler")
-          lock.try_with_lock(&block)
-        end
     end
   end
 end

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.23.0"
+  VERSION = "0.24.0"
 end

data/lib/online_migrations.rb

@@ -23,7 +23,6 @@ module OnlineMigrations
 
   extend ActiveSupport::Autoload
 
-  autoload :AdvisoryLock
   autoload :ApplicationRecord
   autoload :BatchIterator
   autoload :VerboseSqlLogs
@@ -84,14 +83,22 @@ module OnlineMigrations
     end
 
     # Run background data migrations
-    def run_background_migrations
-      BackgroundMigrations::Scheduler.run
+    #
+    # @option options [String, Symbol, nil] :shard The name of the shard to run
+    #   background data migrations on. By default runs on all shards.
+    #
+    def run_background_migrations(**options)
+      BackgroundMigrations::Scheduler.run(**options)
     end
     alias run_background_data_migrations run_background_migrations
 
     # Run background schema migrations
-    def run_background_schema_migrations
-      BackgroundSchemaMigrations::Scheduler.run
+    #
+    # @option options [String, Symbol, nil] :shard The name of the shard to run
+    #   background schema migrations on. By default runs on all shards.
+    #
+    def run_background_schema_migrations(**options)
+      BackgroundSchemaMigrations::Scheduler.run(**options)
     end
 
     def deprecator

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-13 00:00:00.000000000 Z
+date: 2025-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -48,7 +48,6 @@ files:
 - lib/generators/online_migrations/templates/migration.rb.tt
 - lib/generators/online_migrations/upgrade_generator.rb
 - lib/online_migrations.rb
-- lib/online_migrations/advisory_lock.rb
 - lib/online_migrations/application_record.rb
 - lib/online_migrations/background_migration.rb
 - lib/online_migrations/background_migrations/backfill_column.rb

data/lib/online_migrations/advisory_lock.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-require "zlib"
-
-module OnlineMigrations
-  # @private
-  class AdvisoryLock
-    attr_reader :name, :connection
-
-    def initialize(name:, connection: ApplicationRecord.connection)
-      @name = name
-      @connection = connection
-    end
-
-    def try_lock
-      locked = connection.select_value("SELECT pg_try_advisory_lock(#{lock_key})")
-      Utils.to_bool(locked)
-    end
-
-    def unlock
-      connection.select_value("SELECT pg_advisory_unlock(#{lock_key})")
-    end
-
-    # Runs the given block if an advisory lock is able to be acquired.
-    def try_with_lock
-      if try_lock
-        begin
-          yield
-        ensure
-          unlock
-        end
-      end
-    end
-
-    def active?
-      objid = lock_key & 0xffffffff
-      classid = lock_key >> 32
-
-      active = connection.select_value(<<~SQL)
-        SELECT granted
-        FROM pg_locks
-        WHERE locktype = 'advisory'
-          AND pid = pg_backend_pid()
-          AND mode = 'ExclusiveLock'
-          AND classid = #{classid}
-          AND objid = #{objid}
-      SQL
-
-      Utils.to_bool(active)
-    end
-
-    private
-      SALT = 936723412
-
-      def lock_key
-        name_hash = Zlib.crc32(name)
-        SALT * name_hash
-      end
-  end
-end