right_infrastructure_agent 1.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ca12e04c5b64e16c354a7031c39599a36d0c431d
+  data.tar.gz: 463c200718bfe3c8f01817497764942c2a44b173
+SHA512:
+  metadata.gz: 90d0a95a758ecc476cb5b78069e4a9914a1f4f61671855a9bb7f0217254a1b46c1b494bf52a9e35a6bae1a0ed01a3832a16d4f84b06f25ed8751dfe7ef8bcd69
+  data.tar.gz: 51a45fc9db03ee26f6e0b1b6d536e1e3060ff49f2682cf00565c4f17f63b7b9c4f28582276e2f3f8936646de8cb19ae763e6066b3d5c08c62a320510c4326299

data/LICENSE

@@ -0,0 +1,10 @@
+# Copyright (c) 2009-2011 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and
+# the licensee.

data/README.rdoc

@@ -0,0 +1,65 @@
+= RightInfrastructureAgent
+
+= DESCRIPTION
+
+== Synopsis
+
+RightInfrastructureAgent provides the foundation for RightScale infrastructure
+servers that connect into the RightScale system via the RabbitMQ message bus.
+It extends RightAgent in configuration, monitoring, and packet handling areas
+as needed generically by infrastructure servers.
+
+Maintained by the RightScale Teal Team
+
+== Interface
+
+A RightInfrastructureAgent exposes its services via actors and methods that are
+invoked by requests packets it receives via its message queue. The actors
+provided are specific to the given agent.
+
+RightInfrastructureAgent comes with several library modules for forming basic
+command line tools in addition to those provided in RightAgent:
+
+* <b>infrastructure_agent_deployer</b>: Build configuration file for running an agent (rad tool)
+* <b>infrastructure_agent_controller</b>: Manage an agent that has been configured (rnac tool)
+
+== Supported Configuration
+
+RightInfrastructureAgent has been tested on EC2 instances running CentOS 5.2 and Ubuntu 8.10.
+
+== Work in Progress
+
+RightInfrastructureAgent is work in progress.
+
+= ADDITIONAL RESOURCES
+
+* [1] RabbitMQ is http://www.rabbitmq.com/documentation.html
+
+= CHANGELOG
+
+* 0.15.1: The server_superuser role is now taken into account when building policies 
+
+= LICENSE
+
+<b>RightAgent</b>
+
+Copyright:: Copyright (c) 2011 RightScale, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,86 @@
+#--  -*-ruby-*-
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'rubygems'
+require 'bundler/setup'
+require 'fileutils'
+require 'rake'
+require 'rspec/core/rake_task'
+require 'rdoc/task'
+require 'rake/gempackagetask'
+require 'rake/clean'
+
+task :default => 'spec'
+
+# == Gem packaging == #
+
+desc "Package gem"
+gemtask = Rake::GemPackageTask.new(Gem::Specification.load("right_infrastructure_agent.gemspec")) do |package|
+  package.package_dir = ENV['PACKAGE_DIR'] || 'pkg'
+  package.need_zip = true
+  package.need_tar = true
+end
+
+directory gemtask.package_dir
+
+CLEAN.include(gemtask.package_dir)
+
+# == Unit tests == #
+
+RSPEC_OPTS = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
+
+desc 'Run unit tests'
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = RSPEC_OPTS
+end
+
+namespace :spec do
+  desc 'Run unit tests with RCov'
+  RSpec::Core::RakeTask.new(:rcov) do |t|
+    t.rspec_opts = RSPEC_OPTS
+    t.rcov = true
+    t.rcov_opts = %q[--exclude "spec"]
+  end
+
+  desc 'Print Specdoc for all unit tests'
+  RSpec::Core::RakeTask.new(:doc) do |t|
+    t.rspec_opts = ["--format", "documentation"]
+  end
+end
+
+# == Documentation == #
+
+desc 'Generate API documentation to doc/rdocs/index.html'
+Rake::RDocTask.new do |rd|
+  rd.rdoc_dir = 'doc/rdocs'
+  rd.main = 'README.rdoc'
+  rd.rdoc_files.include 'README.rdoc', 'lib/**/*.rb'
+end
+CLEAN.include('doc/rdocs')
+
+# == Emacs integration ==
+
+desc 'Rebuild TAGS file for emacs'
+task :tags do
+  sh 'rtags -R lib spec'
+end

data/lib/right_infrastructure_agent.rb

@@ -0,0 +1,26 @@
+# Copyright (c) 2009-2011 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and the licensee.
+
+require 'rubygems'
+require 'right_agent'
+
+unless defined?(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR)
+  RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR =
+    File.normalize_path(File.join(File.dirname(__FILE__), 'right_infrastructure_agent'))
+end
+
+require File.normalize_path(File.join(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR, 'models_helper'))
+require File.normalize_path(File.join(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR, 'login_policy_factory'))
+require File.normalize_path(File.join(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR, 'infrastructure_helpers'))
+require File.normalize_path(File.join(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR, 'infrastructure_auth_client'))
+require File.normalize_path(File.join(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR, 'rainbows_agent_controller'))
+require File.normalize_path(File.join(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR, 'global_object_replicator_sink'))
+require File.normalize_path(File.join(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR, 'global_object_replicator_source'))
+require File.normalize_path(File.join(RIGHT_INFRASTRUCTURE_AGENT_BASE_DIR, 'command_constants'))

data/lib/right_infrastructure_agent/command_constants.rb

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2009-2014 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+module RightScale
+
+  # Extend this class to support infrastructure agents
+  class CommandConstants
+
+    # Ports used for command protocol
+    BASE_TESTER_AGENT_SOCKET_PORT           = 69000
+    BASE_ROUTER_SOCKET_PORT                 = 79000
+
+  end
+end

data/lib/right_infrastructure_agent/global_object_replicator_sink.rb

@@ -0,0 +1,337 @@
+#--
+# Copyright (c) 2011-2013 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and
+# the licensee.
+#++
+
+module RightScale
+
+  # This module is for use as a mixin in an HTTP controller that handles requests from
+  # a global object replication source (e.g., library) to synchronize its replicas.
+  #
+  # This module expects the following to be defined:
+  #   - logger - variable pointing to the standard logger in use
+  #   - global_object_replicator_sink - method returning type of replication sink, e.g., 'core'
+  module GlobalObjectReplicatorSink
+
+    include RightScale::InfrastructureHelpers
+    include RightScale::ModelsHelper
+
+    # Update the specified right_site core object (class_name, id) so that its attributes
+    # match those of the global object
+    #
+    # @param :source [String] Replication source, e.g., 'library' or 'wasabi'
+    # @param :class_name [String] Name of the object class being updated
+    # @param :id [Integer] Unique identifier of the object being updated
+    # @param :schema_version [Integer] Schema version of the model (class)
+    # @param :global_object_version [String] Global object version of the model (class)
+    # @param :attrs [String] All attributes defined for the object represented by :class_name
+    #   and :id in yaml
+    #
+    # @return [NilClass] nil
+    #
+    # @raise [RightScale::Exceptions::QueryFailure] Failed to update object
+    # @raise [RightScale::Exceptions::RetryableError] Query failed but may be retried
+    def handle_global_object_change
+      source = params[:source]
+      if (object_class = replicate_sink_class(params[:class_name]))
+        logger.info("GlobalObjectReplica: Processing #{source} global object change for #{object_class.name}/#{params[:id]}")
+
+        id = to_int_or_nil(params[:id])
+        schema_version = to_int_or_nil(params[:schema_version])
+        action  = "replicator sink handle_global_object_change for #{object_class.name}/#{id} " +
+                  "(schema version #{schema_version}, global object version #{params[:global_object_version]})"
+        success = query(action, :include_backtrace_in_last_error => true) do
+          object_class.handle_global_object_change(id, schema_version, params[:global_object_version], params[:attrs])
+          true
+        end
+
+        raise RightScale::Exceptions::QueryFailure.new(@last_error) unless success
+      else
+        logger.warn("GlobalObjectReplica: Ignoring replicator sink handle_global_object_change from #{source} " +
+                    "for unknown will_replicate sink class #{params[:class_name].inspect}")
+      end
+      render_nothing
+    end
+
+    # Compare checksum parameter of a replicated table and, if they do not match,
+    # begin the synchronization process by sending verify_replica_range to replicator source
+    #
+    # @param :source [String] Replication source, e.g., 'library' or 'wasabi'
+    # @param :class_name [String] Name of a class that acts_as_global_object_replica
+    # @param :checksum_type [String] Type of checksum: only 'global_object_version_sum'
+    # @param :checksum_value [Integer, NilClass] Sum of global_object_versions for the table
+    #   in the source database
+    #
+    # @return [NilClass] nil
+    #
+    # @raise [ArgumentError] Unknown replication source
+    # @raise [RightScale::Exceptions::QueryFailure] Failed to complete verify replicas query
+    # @raise [RightScale::Exceptions::RetryableError] Query failed but may be retried
+    def verify_replicas
+      source = params[:source]
+      if (replica_class = replicate_sink_class(params[:class_name]))
+        checksum_type = params[:checksum_type]
+        checksum_value = params[:checksum_value]
+
+        success = query("verify_replicas for #{replica_class.name}", :email_errors => true) do
+          if replica_class.calculate_global_object_checksum(checksum_type) == checksum_value
+            logger.info("GlobalObjectReplica: Verified #{source} #{replica_class.name} global_object_version_sum.")
+            set_global_object_replication_status(replica_class.name, checksum_type, :completed => true)
+          else
+            logger.info("GlobalObjectReplica: Verification of #{source} #{replica_class.name} global_object_version_sum failed. " +
+                        "Beginning synchronization.")
+            set_global_object_replication_status(replica_class.name, checksum_type, :start => true)
+
+            max_id = replica_class.max_id
+            verify_next_replica_range(source, replica_class, checksum_type, max_id, 0, max_id)
+          end
+          true
+        end
+
+        unless success
+          set_global_object_replication_status(replica_class.name, checksum_type, :failed => true)
+          raise RightScale::Exceptions::QueryFailure.new(@last_error)
+        end
+      else
+        logger.warn("GlobalObjectReplica: Ignoring replicator sink verify_replicas from #{source} " +
+                    "for unknown will_replicate sink class #{params[:class_name].inspect}")
+      end
+      render_nothing
+    end
+
+    # Update rows using any received synchronization data and send a
+    # verify_replica_range request for the next range to process
+    #
+    # @param :source [String] Replication source, e.g., 'library' or 'wasabi'
+    # @param :class_name [String] Name of a class that acts_as_global_object_replica
+    # @param :checksum_type [String] Type of checksum: only 'global_object_version_sum'
+    # @param :max_id_at_start [Integer] Max ID of the replica_class when first
+    #   started synchronization process
+    # @param :begin_id [Integer, NilClass] First row ID in range verified by the source,
+    #   or nil for all rows
+    # @param :end_id [Integer, NilClass] Last row ID in range verified by the source,
+    #   or nil for all rows
+    # @param :checksum_matched [Boolean] Whether checksum matched
+    # @param :records_to_synchronize [Array, NilClass] Blank array or nil (if the last
+    #   verification check was positive) or an array of hashes to be used to update the
+    #   replicated table with each hash containing the keys used in handle_global_object_change:
+    #   :id, :schema_version, :global_object_version, :attrs
+    # @param :has_more [Boolean] Indicates at least one more range should be verified
+    #
+    # @return [NilClass] nil
+    #
+    # @raise [ArgumentError] Unknown replication source
+    # @raise [RightScale::Exceptions::QueryFailure] Failed to complete synchronization query
+    # @raise [RightScale::Exceptions::RetryableError] Query failed but may be retried
+    def synchronize_replica_range
+      source = params[:source]
+      if (replica_class = replicate_sink_class(params[:class_name]))
+        checksum_type = params[:checksum_type]
+        max_id_at_start = to_int_or_nil(params[:max_id_at_start])
+        begin_id = to_int_or_nil(params[:begin_id])
+        end_id = to_int_or_nil(params[:end_id])
+        checksum_matched = to_bool(params[:checksum_matched])
+        records = params[:records_to_synchronize] || []
+        has_more = to_bool(params[:has_more])
+
+        success = query("synchronize_replica_range for #{replica_class.name}", :email_errors => true) do
+          logger.info("GlobalObjectReplica: Synchronizing #{source} #{replica_class.name} range #{begin_id}-#{end_id} " +
+                      "(#{checksum_type} #{checksum_matched ? 'match' : 'mismatch'}) #{records.size} rows received.")
+          # Shuffle the records here so that collisions are less likely if two processes are trying to sync at once
+          records.shuffle.each do |h|
+            h = RightScale::SerializationHelper.symbolize_keys(h)
+            replica_class.handle_global_object_change(h[:id], h[:schema_version], h[:global_object_version], h[:attrs])
+          end
+
+          in_sync = checksum_matched || !(records.nil? || records.empty?)
+
+          if has_more || !in_sync
+            set_global_object_replication_status(replica_class.name, checksum_type, :percent_complete =>
+                                                 (100 * ([begin_id, 1].max / [max_id_at_start, 1].max.to_f)).floor) if in_sync
+
+            next_begin_id, next_end_id = calculate_next_range_for_binary_sync(source, max_id_at_start,
+                                                                              replica_class.will_replicate_initialization_chunk_size,
+                                                                              begin_id, end_id, in_sync)
+            verify_next_replica_range(source, replica_class, checksum_type, max_id_at_start, next_begin_id, next_end_id)
+          else
+            logger.info("GlobalObjectReplica: Synchronization of #{source} #{replica_class.name} complete at row #{end_id}")
+            set_global_object_replication_status(replica_class.name, checksum_type, :completed => true)
+          end
+          true
+        end
+
+        unless success
+          set_global_object_replication_status(replica_class.name, checksum_type, :failed => true)
+          raise RightScale::Exceptions::QueryFailure.new(@last_error)
+        end
+      else
+        logger.warn("GlobalObjectReplica: Ignoring replicator sink synchronize_replica_range from #{source} " +
+                    "for unknown will_replicate sink class #{params[:class_name].inspect}")
+      end
+      render_nothing
+    end
+
+    protected
+
+    # Calculate the checksum for the next synchronization range and send to the replication source
+    #
+    # @param :source [String] Replication source, e.g., 'library' or 'wasabi'
+    # @param replica_class [Class] Class that acts_as_global_object_replica
+    # @param checksum_type [String] Type of checksum to perform: only 'global_object_version_sum'
+    # @param max_id_at_start [Integer] Max ID of the replica_class when first
+    #   started synchronization process
+    # @param begin_id [Integer, NilClass] First ID in the checksum range, or nil for all rows
+    # @param end_id [Integer, NilClass] Last ID in the checksum range, or nil for all rows
+    #
+    # @return [TrueClass] Always return true
+    #
+    # @raise [ArgumentError] Unknown replication source
+    def verify_next_replica_range(source, replica_class, checksum_type, max_id_at_start, begin_id, end_id)
+      replicator = case source # TODO Replace this with "#{source}_replicator" once all replicator actors gone
+                   when "library" then "library_replicator"
+                   when "wasabi"  then "wasabi_replicator_source"
+                   else raise ArgumentError, "Unknown replication source: #{source.inspect}"
+                   end
+      payload = {
+        :sink => global_object_replicator_sink,
+        :class_name => replica_class.name,
+        :schema_version => replica_class.will_replicate_current_schema_version,
+        :checksum_type => checksum_type,
+        :max_id_at_start => max_id_at_start,
+        :begin_id => begin_id,
+        :end_id => end_id,
+        :send_records_on_checksum_mismatch => calc_size(begin_id, end_id) <= replica_class.will_replicate_initialization_chunk_size,
+        :checksum_value => replica_class.calculate_global_object_checksum(checksum_type, begin_id, end_id),
+        # Assume global if this sink has no associated shard
+        :shard_id => (RightSharding::ShardHandler.fetch_this_shard_id(SystemConfig) rescue RightScale::Packet::GLOBAL) }
+
+      EM.next_tick do
+        # Execute this request on next_tick since, depending on the configuration, the router could route
+        # this request directly via HTTP and there would be no break in the chain of replication requests
+        begin
+          RightScale::RightHttpClient.push("/#{replicator}/verify_replica_range", payload)
+        rescue Exception => e
+          logger.error(format_error("Failed to verify_replica_range for class #{replica_class.name}", e))
+        end
+      end
+      true
+    end
+
+    # Calculate the begin_id and end_id for the next range to compare given the current range & status
+    # Basically successive calls to this method will result in a binary depth-first search finding out
+    # of sync ranges by using the following rules:
+    #   1. If the current range is not in_sync, return the left half as the next range
+    #   2. If the current range is a left half (of some greater range) and it is in_sync, return the
+    #      corresponding right half as the next range
+    #   3. If the current range is a right half and it is in_sync, move it up until it is part of a left
+    #      half, then return the corresponding right half as the next range.
+    #
+    # @param max_id [Integer] Max ID when the search started, which is needed to calculate the left/right
+    #   halves consistently during the whole search
+    # @param min_size [Integer] Minimum chunk size that is used to retrieve records
+    # @param last_begin_id [Integer] First ID in the current checksum range
+    # @param last_end_id [Integer] Last ID in the current checksum range
+    # @param in_sync [Boolean] true if the current checksum range is synchronized, i.e.,
+    #   if do not need to go deeper here, otherwise false
+    #
+    # @return [Array] Next range to search in the form [next_begin_id, next_end_id]
+    #
+    # @raise [StandardError] Unexpected state
+    def calculate_next_range_for_binary_sync(source, max_id, min_size, last_begin_id, last_end_id, in_sync)
+      last_size, last_center = calc_size_and_center(last_begin_id, last_end_id)
+
+      next_begin_id = nil
+      next_end_id = nil
+
+      if !in_sync
+        if last_size <= min_size
+          raise StandardError.new("Unexpected state: Not in sync, but last_size #{last_size} is less " +
+            "then min_size #{min_size}. This shouldn't happen because should have received sync records " +
+            "from the #{source} database and applied them.")
+        else
+          # Check the left half next (i.e. depth-first search)
+          next_begin_id = last_begin_id
+          next_end_id = last_center
+        end
+      else
+        if last_end_id >= max_id
+          # Past the end, so just do the next initialization chunk size
+          next_begin_id = last_end_id + 1
+          next_end_id = last_end_id + min_size
+        else
+          # In sync in the last range, so figure out current location in the depth first search and
+          # check the next right branch
+          cur_begin_id = 0
+          cur_end_id = max_id
+          cur_size, cur_center = calc_size_and_center(cur_begin_id, cur_end_id)
+
+          while cur_size > last_size
+            if last_center < cur_center
+              # About to move down a left branch, so save the corresponding right branch and
+              # if don't go down anymore lefts, this is the branch to sync next
+              next_begin_id = cur_center + 1
+              next_end_id = cur_end_id
+
+              cur_end_id = cur_center
+            else
+              cur_begin_id = cur_center + 1
+            end
+
+            cur_size, cur_center = calc_size_and_center(cur_begin_id, cur_end_id)
+          end
+        end
+      end
+
+      if next_begin_id.nil?  || next_end_id.nil?
+        raise StandardError.new("Unexpected state: Nil result when calculating next range for binary sync: max_id=#{max_id}, " +
+          "min_size=#{min_size}, last_begin=#{last_begin_id}, last_end=#{last_end_id}, in_sync=#{in_sync}")
+      end
+
+      [next_begin_id, next_end_id]
+    end
+
+    def calc_size(begin_id, end_id)
+      end_id - begin_id + 1
+    end
+
+    def calc_size_and_center(begin_id, end_id)
+      [calc_size(begin_id, end_id), ((begin_id + end_id) / 2)]
+    end
+
+    def replicate_sink_class(class_name)
+      klass = constantize(class_name) rescue nil
+      (klass.respond_to?(:will_replicate_sink?) && klass.will_replicate_sink?) ? klass : nil
+    end
+
+    def set_global_object_replication_status(class_name, checksum_type, options = {})
+      status = GlobalObjectReplicationStatus.find_or_initialize_by_name(class_name)
+      status.last_sync_at = Time.now
+      status.last_sync_status = if options[:failed]
+        status.last_sync_status = "failed" + status.last_sync_status.sub("in progress", "")
+      elsif options[:completed]
+        'completed'
+      elsif options[:start]
+        "in progress 0%"
+      elsif options[:percent_complete] && options[:percent_complete] > 100
+        "in progress unknown %"
+      else
+        "in progress #{options[:percent_complete]}%"
+      end
+      status.last_sync_checksum_type = checksum_type
+      status.last_sync_start_at = Time.now if options[:start]
+      status.save!
+    rescue Exception => e
+      logger.error(format_error("Failed to update global object replication status", e, :trace))
+    end
+
+  end # GlobalObjectReplicatorSink
+
+end # RightScale