culturecode_stagehand 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88258f8ab5b38d62a941024ddeef50d3e66ac13b
-  data.tar.gz: 9f1571c1823366b2a5b576d9e213a7585713e883
+  metadata.gz: cd5f6596af77820e4b0fda648cf3a575cef70dd8
+  data.tar.gz: 73711aa60b690c1a9a7715d5ce26934d79e7727f
 SHA512:
-  metadata.gz: a8020df0d22695e6fe842996690431bff25595e5dc3c76f46cb3ff7054dc8e56fad5dab0dc93007d6cc81d84b9875e922f2d70f0b98a37f28a479e50b63e1c55
-  data.tar.gz: 391d375fd5dcc015f8ad867cc7495c476e2e1d652beddd18584ca2a4df262baf5aa0938f6ebb323357fd823055b73f8b32c8200b1dffb72829503e91bc32c176
+  metadata.gz: 9b685a7c6bd6f6c8638c86f8ff3a316ad2147942b7dcaa378daea28fbc31421cce4105097ac1a3a418d8a6639891e107c5663ee9a5177bdb25770f2d3cd555ac
+  data.tar.gz: 7f629766d1d1af995dc0870bf15bdc6a2c04c96105859cc5cdc4388cb104c4be11b7f0df4c17987f1b8e54a1fd6447e6847fd792196e346ee8f49da17a558cbe

data/lib/stagehand/production.rb

@@ -20,7 +20,7 @@ module Stagehand
 
       return unless attributes.present?
 
-      is_new = lookup(staging_record, table_name).update_all(attributes).zero?
+      is_new = matching(staging_record, table_name).update_all(attributes).zero?
 
       # Ensure we always return a record, even when updating instead of creating
       Record.new.tap do |record|
@@ -30,11 +30,11 @@ module Stagehand
     end
 
     def delete(staging_record, table_name = nil)
-      lookup(staging_record, table_name).delete_all
+      matching(staging_record, table_name).delete_all
     end
 
     def exists?(staging_record, table_name = nil)
-      lookup(staging_record, table_name).exists?
+      matching(staging_record, table_name).exists?
     end
 
     # Returns true if the staging record's attributes are different from the production record's attributes
@@ -44,9 +44,13 @@ module Stagehand
       production_record_attributes(staging_record, table_name) != staging_record_attributes(staging_record, table_name)
     end
 
+    def find(*args)
+      matching(*args).first
+    end
+
     # Returns a scope that limits results any occurrences of the specified record.
     # Record can be specified by passing a staging record, or an id and table_name.
-    def lookup(staging_record, table_name = nil)
+    def matching(staging_record, table_name = nil)
       table_name, id = Stagehand::Key.generate(staging_record, :table_name => table_name)
       prepare_to_modify(table_name)
       return Record.where(:id => id)
@@ -60,7 +64,7 @@ module Stagehand
     end
 
     def production_record_attributes(staging_record, table_name = nil)
-      Record.connection.select_one(lookup(staging_record, table_name))
+      Record.connection.select_one(matching(staging_record, table_name))
     end
 
     def staging_record_attributes(staging_record, table_name = nil)

data/lib/stagehand/staging/commit_entry.rb

@@ -109,7 +109,7 @@ module Stagehand
       private
 
       def build_production_record
-        production_record = Stagehand::Production.lookup(record_id, table_name).first
+        production_record = Stagehand::Production.find(record_id, table_name)
         return unless production_record
 
         production_record = record_class.new(production_record.attributes)

data/lib/stagehand/staging/synchronizer.rb

@@ -2,6 +2,9 @@ module Stagehand
   module Staging
     module Synchronizer
       extend self
+      mattr_accessor :schemas_match
+
+      BATCH_SIZE = 1000
 
       # Immediately attempt to sync the changes from the block if possible
       # The block is wrapped in a transaction to prevent changes to records while being synced
@@ -14,22 +17,33 @@ module Stagehand
         end
       end
 
-      def auto_sync(delay = 5.seconds)
-        scope = autosyncable_entries.limit(1000)
-
+      def auto_sync(polling_delay = 5.seconds)
         loop do
-          Rails.logger.info "Synced #{sync_entries(scope.reload)} entries"
-          sleep(delay) if delay
+          sync(BATCH_SIZE)
+          sleep(polling_delay) if polling_delay
         end
       end
 
       def sync(limit = nil)
-        sync_entries(autosyncable_entries.limit(limit))
+        synced_count = 0
+        deleted_count = 0
+
+        iterate_autosyncable_entries do |entry|
+          sync_entries(entry)
+          synced_count += 1
+          deleted_count += CommitEntry.matching(entry).delete_all
+          break if synced_count == limit
+        end
+
+        Rails.logger.info "Synced #{synced_count} entries"
+        Rails.logger.info "Removed #{deleted_count} stale entries"
+
+        return synced_count
       end
 
       def sync_all
         loop do
-          entries = CommitEntry.order(:id => :desc).limit(1000).to_a
+          entries = CommitEntry.order(:id => :desc).limit(BATCH_SIZE).to_a
           break unless entries.present?
 
           latest_entries = entries.uniq(&:key)
@@ -48,44 +62,78 @@ module Stagehand
 
       private
 
+      # Lazily iterate through millions of commit entries
+      # Returns commit entries in ID descending order
+      def iterate_autosyncable_entries(&block)
+        sessions = CommitEntry.order(:id => :desc).distinct.pluck(:session)
+        offset = 0
+
+        while sessions.present?
+          autosyncable_entries(:session => sessions.shift(30)).offset(offset).limit(BATCH_SIZE).each do |entry|
+            with_confirmed_autosyncability(entry, &block)
+          end
+          offset += BATCH_SIZE
+        end
+      end
+
+      # Executes the code in the block if the record referred to by the entry is in fact, autosyncable.
+      # This confirmation is used to guard against writes to the record that occur after loading an initial list of
+      # entries that are autosyncable, but before the record is actually synced. To prevent this, a lock on the record
+      # is acquired and then the record's autosync eligibility is rechecked before calling the block.
+      # NOTE: This method must be called from within a transaction
+      def with_confirmed_autosyncability(entry, &block)
+        ActiveRecord::Base.transaction do
+          CommitEntry.connection.execute("SELECT 1 FROM #{entry.table_name} WHERE id = #{entry.record_id}")
+          block.call(entry) if autosyncable_entries(:record_id => entry.record_id, :table_name => entry.table_name).exists?
+        end
+      end
+
+      # Returns commit entries in ID descending order
+      def autosyncable_entries(scope = nil)
+        entries = CommitEntry.content_operations.not_in_progress
+
+        unless Configuration.ghost_mode?
+          subquery = CommitEntry.group('record_id, table_name').having('count(commit_id) = 0').where(scope)
+          entries = entries.joins("JOIN (#{subquery.select('MAX(id) AS max_id').to_sql}) subquery ON id = max_id")
+        end
+
+        return entries.order(:id => :desc)
+      end
+
       def sync_checklist(checklist)
-        sync_entries(checklist.syncing_entries)
-        CommitEntry.delete(checklist.affected_entries)
+        ActiveRecord::Base.transaction do
+          sync_entries(checklist.syncing_entries)
+          CommitEntry.delete(checklist.affected_entries)
+        end
       end
 
       def sync_entries(entries)
         return 0 if Configuration.single_connection? # Avoid deadlocking if the databases are the same
-
         raise SchemaMismatch unless schemas_match?
 
-        ActiveRecord::Base.transaction do
-          entries.each do |entry|
-            Rails.logger.info "Synchronizing #{entry.table_name} #{entry.record_id}" if entry.content_operation?
-            if entry.delete_operation?
-              Stagehand::Production.delete(entry)
-            elsif entry.save_operation?
-              Stagehand::Production.save(entry)
-            end
+        entries = Array.wrap(entries)
+
+        entries.each do |entry|
+          Rails.logger.info "Synchronizing #{entry.table_name} #{entry.record_id}" if entry.content_operation?
+          if entry.delete_operation?
+            Stagehand::Production.delete(entry)
+          elsif entry.save_operation?
+            Stagehand::Production.save(entry)
           end
         end
 
         return entries.length
       end
 
-      def autosyncable_entries
-        if Configuration.ghost_mode?
-          CommitEntry
-        else
-          CommitEntry.where(:id =>
-            CommitEntry.select('MAX(id) AS id').content_operations.not_in_progress.group('record_id, table_name').having('count(commit_id) = 0'))
-        end
-      end
-
       def schemas_match?
+        return schemas_match unless schemas_match.nil?
+
         versions_scope = ActiveRecord::SchemaMigration.order(:version)
         staging_versions = Stagehand::Database.staging_connection.select_values(versions_scope)
         production_versions = Stagehand::Database.production_connection.select_values(versions_scope)
-        return staging_versions == production_versions
+        self.schemas_match = staging_versions == production_versions
+
+        return schemas_match
       end
     end
   end

data/lib/stagehand/version.rb

@@ -1,3 +1,3 @@
 module Stagehand
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: culturecode_stagehand
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Nicholas Jakobsen
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-16 00:00:00.000000000 Z
+date: 2016-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails