combinaut_stagehand 1.2.1

data/lib/stagehand/staging/synchronizer.rb

@@ -0,0 +1,198 @@
+module Stagehand
+  module Staging
+    module Synchronizer
+      extend self
+      mattr_accessor :schemas_match
+
+      BATCH_SIZE = 1000
+      ENTRY_SYNC_ORDER = [:delete, :update, :insert].freeze
+      ENTRY_SYNC_ORDER_SQL = Arel.sql(ActiveRecord::Base.send(:sanitize_sql_for_order, [Arel.sql('FIELD(operation, ?), id ASC'), ENTRY_SYNC_ORDER])).freeze
+
+      # Immediately sync the changes from the block, preconfirming all changes
+      # The block is wrapped in a transaction to prevent changes to records while being synced
+      def sync_now!(*args, **opts, &block)
+        sync_now(*args, **opts, preconfirmed: true, &block)
+      end
+
+      # Immediately attempt to sync the changes from the block if possible
+      # The block is wrapped in a transaction to prevent changes to records while being synced
+      def sync_now(subject_record = nil, preconfirmed: false, **opts, &block)
+        raise SyncBlockRequired unless block_given?
+
+        Rails.logger.info "Syncing Now (preconfirmed: #{preconfirmed})"
+        Database.transaction do
+          commit = Commit.capture(subject_record, &block)
+          next unless commit # If the commit was empty don't continue
+          checklist = Checklist.new(commit.entries)
+          sync_checklist(checklist, **opts) if preconfirmed || !checklist.requires_confirmation?
+        end
+      end
+
+      def auto_sync(polling_delay = 5.seconds, **opts)
+        loop do
+          Rails.logger.info "Autosyncing"
+          sync(BATCH_SIZE, **opts)
+          sleep(polling_delay) if polling_delay
+        rescue Database::NoRetryError => e
+          Rails.logger.info "Autosyncing encountered a NoRetryError"
+        end
+      end
+
+      def sync(limit = nil, **opts)
+        synced_count = 0
+        deleted_count = 0
+
+        Rails.logger.info "Syncing"
+
+        iterate_autosyncable_entries do |entry|
+          sync_entry(entry, :callbacks => :sync, **opts)
+          synced_count += 1
+
+          scope = CommitEntry.matching(entry).not_in_progress
+          scope = scope.save_operations unless entry.delete_operation?
+          deleted_count += delete_without_range_locks(scope)
+
+          break if synced_count == limit
+        end
+
+        Rails.logger.info "Synced #{synced_count} entries"
+        Rails.logger.info "Removed #{deleted_count} stale entries"
+
+        return synced_count
+      end
+
+      def sync_all(**opts)
+        loop do
+          entries = CommitEntry.order(ENTRY_SYNC_ORDER_SQL).limit(BATCH_SIZE).to_a
+          break unless entries.present?
+
+          latest_entries = entries.uniq(&:key)
+          latest_entries.each {|entry| sync_entry(entry, :callbacks => :sync, **opts) }
+          Rails.logger.info "Synced #{latest_entries.count} entries"
+
+          deleted_count = delete_without_range_locks(CommitEntry.matching(latest_entries))
+          Rails.logger.info "Removed #{deleted_count - latest_entries.count} stale entries"
+        end
+      end
+
+      # Copies all the affected records from the staging database to the production database
+      def sync_record(record, **opts)
+        sync_checklist(Checklist.new(record), **opts)
+      end
+
+      def sync_checklist(checklist, **opts)
+        Database.transaction do
+          checklist.syncing_entries.each do |entry|
+            if checklist.subject_entries.include?(entry)
+              sync_entry(entry, :callbacks => [:sync, :sync_as_subject], **opts)
+            else
+              sync_entry(entry, :callbacks => [:sync, :sync_as_affected], **opts)
+            end
+          end
+
+          delete_without_range_locks(checklist.affected_entries)
+        end
+      end
+
+      private
+
+      # Lazily iterate through millions of commit entries
+      # Returns commit entries in ID descending order
+      def iterate_autosyncable_entries(&block)
+        current = CommitEntry.maximum(:id).to_i
+
+        while entries = autosyncable_entries("id <= #{current}").limit(BATCH_SIZE).order(ENTRY_SYNC_ORDER_SQL).to_a.presence do
+          with_confirmed_autosyncability(entries.uniq(&:key), &block)
+          current = entries.last.try(:id).to_i - 1
+        end
+      end
+
+      # Executes the code in the block if the record referred to by the entry is in fact, autosyncable.
+      # This confirmation is used to guard against writes to the record that occur after loading an initial list of
+      # entries that are autosyncable, but before the record is actually synced. To prevent this, a lock on the record
+      # is acquired and then the record's autosync eligibility is rechecked before calling the block.
+      def with_confirmed_autosyncability(entries, &block)
+        entries = Array.wrap(entries)
+        return unless entries.present?
+
+        Database.transaction do
+          # Lock the records so nothing can update them after we confirm autosyncability
+          acquire_record_locks(entries)
+
+          # Execute the block for each entry we've confirm autosyncability
+          confirmed_ids = Set.new(autosyncable_entries.where(:id => entries).pluck(:id))
+
+          entries.each do |entry|
+            block.call(entry) if confirmed_ids.include?(entry.id)
+          end
+        end
+      end
+
+      # Does not actually acquire a lock, instead it triggers a 'first read' so the transaction will ensure subsequent
+      # reads of the locked rows return the same value, even if modified outside of the transaction
+      def acquire_record_locks(entries)
+        entries.each(&:record)
+      end
+
+      def autosyncable_entries(scope = nil)
+        entries = CommitEntry.content_operations.where(scope)
+        if Configuration.ghost_mode?
+          entries = entries.not_in_progress
+        else
+          entries = entries.with_uncontained_keys
+        end
+        return entries
+      end
+
+      def sync_entry(entry, callbacks: false)
+        raise SchemaMismatch unless schemas_match?
+
+        run_sync_callbacks(entry, callbacks) do
+          next unless entry.content_operation? # Only sync records from content operations because those are the only rows that have changes
+          next if Configuration.single_connection? # Avoid deadlocking if the databases are the same. There is nothing to sync because there is only a single database
+
+          Rails.logger.info "Synchronizing #{entry.table_name} #{entry.record_id} (#{entry.operation})"
+
+          if entry.delete_operation?
+            Production.delete(entry)
+          elsif entry.save_operation?
+            Production.save(entry)
+          end
+
+          Rails.logger.info "Synchronized #{entry.table_name} #{entry.record_id} (#{entry.operation})"
+        end
+      end
+
+      def schemas_match?
+        return schemas_match unless schemas_match.nil?
+        self.schemas_match = Database.staging_database_versions == Database.production_database_versions
+        return schemas_match
+      end
+
+      def run_sync_callbacks(entry, callbacks, &block)
+        callbacks = Array.wrap(callbacks.presence).dup
+        return block.call unless callbacks.present? && entry.record
+
+        entry.record.run_callbacks(callbacks.shift) do
+          run_sync_callbacks(entry, callbacks, &block)
+        end
+      end
+
+      # Deletes records without acquiring range locks which have a higher likelihood of causing a deadlock.
+      # See https://dev.mysql.com/doc/refman/5.6/en/innodb-locks-set.html for info on locks set by SQL statements.
+      def delete_without_range_locks(commit_entries)
+        ids = commit_entries.pluck(:id)
+        ids.in_groups_of(1000, false) do |batch|
+          CommitEntry.delete(batch)
+        end
+
+        return ids.length
+      end
+    end
+  end
+
+  # EXCEPTIONS
+
+  class SyncBlockRequired < StandardError; end
+  class SchemaMismatch < StandardError; end
+end

data/lib/stagehand/staging.rb

@@ -0,0 +1,11 @@
+require 'stagehand/staging/commit'
+require 'stagehand/staging/commit_entry'
+require 'stagehand/staging/checklist'
+require 'stagehand/staging/controller'
+require 'stagehand/staging/model'
+require 'stagehand/staging/synchronizer'
+
+module Stagehand
+  module Staging
+  end
+end

data/lib/stagehand/version.rb

@@ -0,0 +1,3 @@
+module Stagehand
+  VERSION = "1.2.1"
+end

data/lib/stagehand.rb

@@ -0,0 +1,5 @@
+require 'rails/all'
+require 'stagehand/engine'
+
+module Stagehand
+end

data/lib/tasks/stagehand_tasks.rake

@@ -0,0 +1,43 @@
+namespace :stagehand do
+  desc "Polls the commit entries table for changes to sync to production"
+  task :auto_sync, [:delay] => :environment do |t, args|
+    delay = args[:delay].present? ? args[:delay].to_i : 5.seconds
+    Stagehand::Staging::Synchronizer.auto_sync(delay)
+  end
+
+  desc "Syncs records that don't need confirmation to production"
+  task :sync, [:limit] => :environment do |t, args|
+    limit = args[:limit].present? ? args[:limit].to_i : nil
+    Stagehand::Staging::Synchronizer.sync(limit)
+  end
+
+  desc "Syncs all records to production, including those that require confirmation"
+  task :sync_all => :environment do
+    Stagehand::Staging::Synchronizer.sync_all
+  end
+
+  # Enhance the regular tasks to run on both staging and production databases
+  def rake_both_databases(task, stagehand_task = task.gsub(':','_'))
+    task(stagehand_task => :environment) do
+      Stagehand::Database.each do |connection_name|
+        Stagehand::Connection.with_production_writes do
+          puts "#{connection_name}"
+          Rake::Task[task].reenable
+          Rake::Task[task].invoke
+        end
+      end
+      Rake::Task[task].clear
+    end
+
+    # Enhance the original task to run the stagehand_task as a prerequisite
+    Rake::Task[task].enhance(["stagehand:#{stagehand_task}"]) unless Stagehand::Configuration.single_connection?
+  end
+
+  rake_both_databases('db:create')
+  rake_both_databases('db:migrate')
+  rake_both_databases('db:migrate:up')
+  rake_both_databases('db:migrate:down')
+  rake_both_databases('db:rollback')
+  rake_both_databases('db:test:load_structure')
+  rake_both_databases('db:test:load_schema')
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: combinaut_stagehand
+version: !ruby/object:Gem::Version
+  version: 1.2.1
+platform: ruby
+authors:
+- Nicholas Jakobsen
+- Ryan Wallace
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-graphviz
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: combustion
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simplify the management of a sandbox database that can sync content to
+  a production database. Content changes can be bundled to allow partial syncs of
+  the database.
+email:
+- nicholas@combinaut.com
+- ryan@combinaut.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- Rakefile
+- lib/combinaut_stagehand.rb
+- lib/stagehand.rb
+- lib/stagehand/active_record_extensions.rb
+- lib/stagehand/auditor.rb
+- lib/stagehand/auditor/checklist_visualizer.rb
+- lib/stagehand/cache.rb
+- lib/stagehand/configuration.rb
+- lib/stagehand/connection_adapter_extensions.rb
+- lib/stagehand/controller_extensions.rb
+- lib/stagehand/database.rb
+- lib/stagehand/engine.rb
+- lib/stagehand/key.rb
+- lib/stagehand/production.rb
+- lib/stagehand/production/controller.rb
+- lib/stagehand/schema.rb
+- lib/stagehand/schema/statements.rb
+- lib/stagehand/schema_extensions.rb
+- lib/stagehand/staging.rb
+- lib/stagehand/staging/checklist.rb
+- lib/stagehand/staging/commit.rb
+- lib/stagehand/staging/commit_entry.rb
+- lib/stagehand/staging/controller.rb
+- lib/stagehand/staging/model.rb
+- lib/stagehand/staging/synchronizer.rb
+- lib/stagehand/version.rb
+- lib/tasks/stagehand_tasks.rake
+homepage: https://github.com/combinaut/stagehand
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Simplify the management of a sandbox database that can sync content to a
+  production database
+test_files: []