right_infrastructure_agent 1.1.2

data/right_infrastructure_agent.gemspec

@@ -0,0 +1,54 @@
+# -*-ruby-*-
+# Copyright: Copyright (c) 2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'rubygems'
+
+Gem::Specification.new do |spec|
+  spec.name      = 'right_infrastructure_agent'
+  spec.version   = '1.1.2'
+  spec.date      = '2014-07-24'
+  spec.authors   = ['Lee Kirchhoff', 'Raphael Simon']
+  spec.email     = 'lee@rightscale.com'
+  spec.homepage  = 'https://github.com/rightscale/right_infrastructure_agent'
+  spec.platform  = Gem::Platform::RUBY
+  spec.summary   = 'RightAgent extension for use by RightScale infrastructure servers'
+  spec.has_rdoc  = true
+  spec.rdoc_options = ["--main", "README.rdoc", "--title", "RightInfrastructureAgent"]
+  spec.extra_rdoc_files = ["README.rdoc"]
+  spec.required_ruby_version = '>= 1.8.7'
+  spec.require_path = 'lib'
+
+  spec.add_dependency('right_support', '~> 2.1')
+  spec.add_dependency('right_amqp', '~> 0.4')
+  spec.add_dependency('right_agent', '~> 2.0')
+
+  spec.description = <<-EOF
+  RightInfrastructureAgent provides the foundation for RightScale infrastructure
+  servers that connect into the RightScale system via the RabbitMQ message bus.
+  It extends RightAgent in configuration, monitoring, and packet handling areas
+  as needed generically by infrastructure servers.
+EOF
+
+  candidates = Dir.glob("{lib,spec}/**/*") +
+               ["LICENSE", "README.rdoc", "Rakefile", "right_infrastructure_agent.gemspec"]
+  spec.files = candidates.sort
+end

data/spec/global_object_replicator_sink_spec.rb

@@ -0,0 +1,305 @@
+#--
+# Copyright (c) 2011-2013 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and the licensee.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
+
+describe RightScale::GlobalObjectReplicatorSink do
+
+  include FlexMock::ArgumentTypes
+
+  class SystemConfig; end
+
+  module RightSharding
+    module ShardHandler
+      def self.fetch_this_shard_id(system_config)
+        9
+      end
+    end
+  end
+
+  # A pseudo HTTP controller applying GlobalObjectReplicatorSink
+  class GlobalObjectReplicatorSinkController
+    include RightScale::GlobalObjectReplicatorSink
+
+    attr_reader :logger
+    attr_reader :params
+
+    def initialize(params = {})
+      @params = params
+      @logger = RightScale::Log
+    end
+
+    def global_object_replicator_sink
+      "core"
+    end
+
+    def render(hash)
+      nil
+    end
+  end
+
+  class FakeGlobalSinkObject
+
+    attr_accessor :id, :name, :foo, :global_object_version
+
+    def initialize
+      @id = 71
+      @name = "fake"
+      @foo = "bar"
+      @global_object_version = 999
+      self.class.store(self)
+    end
+
+    def attributes
+      {"name" => @name, "foo" => @foo}
+    end
+
+    def self.will_replicate_sink?
+      true
+    end
+
+    def self.will_replicate_current_schema_version
+      3
+    end
+
+    def self.calculate_global_object_checksum(checksum_type, begin_id = nil, end_id = nil)
+      123
+    end
+
+    def self.will_replicate_initialization_chunk_size
+      2
+    end
+
+    def self.max_id
+      101
+    end
+
+    def self.handle_global_object_change(id, schema_version, global_object_version, attrs)
+      object = (@objects ||= {})[id]
+      raise ArgumentError("Invalid id: #{id}") unless object
+      raise ArgumentError("Invalid schema verson: #{schema_version}") unless schema_version == FakeGlobalSinkObject.will_replicate_current_schema_version
+      object.global_object_version = global_object_version
+      YAML.load(attrs).each { |name, value| object.instance_variable_set(eval(":@#{name}"), value) }
+    end
+
+    def self.store(object)
+      (@objects ||= {})[object.id] = object
+    end
+
+    def self.purge
+      @objects = nil
+    end
+  end
+
+  before(:each) do
+    @request_params = {}
+    @controller = GlobalObjectReplicatorSinkController.new(@request_params)
+    flexmock(@controller).should_receive(:query).and_yield.by_default
+    @checksum_type = "global_object_version_sum"
+  end
+
+  context "handle_global_object_change" do
+
+    before(:each) do
+      @object = FakeGlobalSinkObject.new
+      @updated_name = "Updated Name"
+      @request_params.merge!({
+        :source                => "library",
+        :api_version           => "1.5",
+        :request_token         => "1234",
+        :class_name            => FakeGlobalSinkObject.to_s,
+        :id                    => @object.id,
+        :schema_version        => FakeGlobalSinkObject.will_replicate_current_schema_version,
+        :global_object_version => @object.global_object_version + 1,
+        :attrs                 => @object.attributes.merge("name" => @updated_name).to_yaml })
+      flexmock(RightScale::RightHttpClient).should_receive(:push).by_default
+    end
+
+    it "should succeed for a valid global object" do
+      @controller.handle_global_object_change.should be_true
+      @object.name.should == @updated_name
+    end
+
+    it "should ignore requests for unknown objects" do
+      flexmock(@controller.logger).should_receive(:warn).with(/GlobalObjectReplica: Ignoring/).once
+      @request_params[:class_name] = "Server"
+      @controller.handle_global_object_change.should be_true
+    end
+  end
+
+  context "verify_next_replica_range" do
+
+    it "should send a request to the library_replicator" do
+      flexmock(EM).should_receive(:next_tick).and_yield.once
+      flexmock(RightScale::RightHttpClient).should_receive(:push).with("/library_replicator/verify_replica_range", hsh(:sink => "core")).once
+      @controller.send(:verify_next_replica_range, "library", FakeGlobalSinkObject, @checksum_type, 20, 1, 10).should be_true
+    end
+
+    it "logs exception if synchronize_replica_range request fails" do
+      flexmock(@controller.logger).should_receive(:error).with(/Failed to verify_replica_range.*failure/).once
+      flexmock(EM).should_receive(:next_tick).and_yield.once
+      flexmock(RightScale::RightHttpClient).should_receive(:push).and_raise(Exception, "failure").once
+      @controller.send(:verify_next_replica_range, "library", FakeGlobalSinkObject, @checksum_type, 20, 1, 10).should be_true
+    end
+  end
+
+  context "verify_replicas" do
+
+    before(:each) do
+      @request_params.merge!({
+        :source         => "library",
+        :api_version    => "1.5",
+        :request_token  => "1234",
+        :class_name     => "FakeGlobalSinkObject",
+        :checksum_type  => @checksum_type,
+        :checksum_value => 123 })
+    end
+
+    it "should not start the synchronization process if the received sum matches the calculated" do
+      flexmock(@controller).should_receive(:verify_next_replica_range).never
+      @controller.verify_replicas.should be_true
+    end
+
+    it "should start the synchronization process if the received sum does not match the calculated" do
+      flexmock(@controller).should_receive(:verify_next_replica_range).once
+      @request_params[:checksum_value] = 999
+      @controller.verify_replicas.should be_true
+    end
+
+    it "should ignore requests for unknown objects" do
+      flexmock(@controller.logger).should_receive(:warn).with(/GlobalObjectReplica: Ignoring/).once
+      @request_params[:class_name] = "Server"
+      @controller.verify_replicas.should be_true
+    end
+  end
+
+  context "synchronize_replica_range" do
+
+    before(:each) do
+      @request_params.merge!({
+        :source                 => "library",
+        :api_version            => "1.5",
+        :request_token          => "1234",
+        :class_name             => "FakeGlobalSinkObject",
+        :checksum_type          => @checksum_type,
+        :max_id_at_start        => 14114,
+        :begin_id               => 1,
+        :end_id                 => 100,
+        :checksum_matched       => true,
+        :has_more               => false,
+        :records_to_synchronize => [] })
+    end
+
+    it "should update any received records" do
+      flexmock(FakeGlobalSinkObject).should_receive(:handle_global_object_change).with(1, 2, 3, {}).once
+      @request_params[:records_to_synchronize] = [{"id" => 1, "schema_version" => 2, "global_object_version" => 3, "attrs" => {}}]
+      @controller.synchronize_replica_range.should be_true
+    end
+
+    it "should continue synchronizing if has_more is true" do
+      flexmock(@controller).should_receive(:verify_next_replica_range).once
+      @request_params[:has_more] = true
+      @controller.synchronize_replica_range.should be_true
+    end
+
+    it "should stop synchronizing if has_more is false" do
+      flexmock(@replicator).should_receive(:verify_next_replica_range).never
+      @controller.synchronize_replica_range.should be_true
+    end
+
+    it "should ignore requests for unknown objects" do
+      flexmock(@controller.logger).should_receive(:warn).with(/GlobalObjectReplica: Ignoring/).once
+      @request_params[:class_name] = "Server"
+      @controller.synchronize_replica_range.should be_true
+    end
+  end
+
+  context "calculate_next_range_for_binary_sync" do
+
+    before(:each) do
+      @min_size = 250
+    end
+
+    context "context when max_id > min_size" do
+
+      before(:each) do
+        @max_id = 31021
+        @source = "library"
+      end
+
+      it "should return next min_size range if top level is in sync" do
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, 0, @max_id, true).
+            should == [@max_id + 1, @max_id + @min_size]
+      end
+
+      it "should return next min_size range if start is past max_id" do
+        start_id = @max_id + 50
+        end_id = @max_id + 100
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, start_id, end_id, true).
+            should == [end_id + 1, end_id + @min_size]
+      end
+
+      it "should return the 1st level left branch if top is not in sync" do
+        size, center = @controller.send(:calc_size_and_center, 0, @max_id)
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, 0, @max_id, false).
+            should == [0, center]
+      end
+
+      it "should return the 1st level right branch if 1st level left is in sync" do
+        size, center = @controller.send(:calc_size_and_center, 0, @max_id)
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, 0, center, true).
+            should == [center + 1, @max_id]
+      end
+
+      it "should return the 2nd level left-left branch if 1st level left is not in sync" do
+        size, center = @controller.send(:calc_size_and_center, 0, @max_id)
+        size, left_center = @controller.send(:calc_size_and_center, 0, center)
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, 0, center, false).
+            should == [0, left_center]
+      end
+
+      it "should return the 2nd level left-right branch if 2nd left-left is in sync" do
+        size, center = @controller.send(:calc_size_and_center, 0, @max_id)
+        size, left_center = @controller.send(:calc_size_and_center, 0, center)
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, 0, left_center, true).
+            should == [left_center + 1, center]
+      end
+
+      it "should return the 1st level right branch if 2nd left-right is in sync" do
+        size, center = @controller.send(:calc_size_and_center, 0, @max_id)
+        size, left_center = @controller.send(:calc_size_and_center, 0, center)
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, left_center + 1, center, true).
+            should == [center + 1, @max_id]
+      end
+
+      it "should return the 1st level right branch if 3rd left-right-right is in sync" do
+        size, center = @controller.send(:calc_size_and_center, 0, @max_id)
+        size, left_center = @controller.send(:calc_size_and_center, 0, center)
+        size, left_right_center = @controller.send(:calc_size_and_center, left_center + 1, center)
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, left_right_center + 1, center, true).
+            should == [center + 1, @max_id]
+      end
+
+      it "should return nils if 1st level right branch is in sync" do
+        size, center = @controller.send(:calc_size_and_center, 0, @max_id)
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, center + 1, @max_id, true).
+            should == [@max_id + 1, @max_id + @min_size]
+      end
+
+      it "should return nils if 2nd level right-right branch is in sync" do
+        size, center = @controller.send(:calc_size_and_center, 0, @max_id)
+        size, right_center = @controller.send(:calc_size_and_center, center + 1, @max_id)
+        @controller.send(:calculate_next_range_for_binary_sync, @source, @max_id, @min_size, right_center + 1, @max_id, true).should == [@max_id + 1, @max_id + @min_size]
+      end
+    end
+  end
+end

data/spec/global_object_replicator_source_spec.rb

@@ -0,0 +1,113 @@
+#--
+# Copyright (c) 2011-2013 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and the licensee.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
+
+describe RightScale::GlobalObjectReplicatorSource do
+
+  include FlexMock::ArgumentTypes
+
+  # A pseudo HTTP controller applying GlobalObjectReplicatorSource
+  class GlobalObjectReplicatorSourceController
+    include RightScale::GlobalObjectReplicatorSource
+
+    attr_reader :logger
+    attr_reader :params
+
+    def initialize(params)
+      @params = params
+      @logger = RightScale::Log
+    end
+
+    def global_object_replicator_source
+      "library"
+    end
+
+    def render(hash)
+      nil
+    end
+  end
+
+  class FakeGlobalSourceObject
+    def self.calculate_global_object_checksum(checksum_type, schema_version, begin_id, end_id)
+      123
+    end
+
+    def self.max_id
+      101
+    end
+  end
+
+  context "verify_replica_range" do
+
+    before(:each) do
+      @request_params = {
+        :sink => "core",
+        :class_name => "FakeGlobalSourceObject",
+        :schema_version => 1,
+        :max_id_at_start => 100,
+        :begin_id => 1,
+        :end_id => 2,
+        :send_records_on_checksum_mismatch => true,
+        :checksum_value => FakeGlobalSourceObject.calculate_global_object_checksum("x", 1, 1, 2),
+        :shard_id => 9}
+      @controller = GlobalObjectReplicatorSourceController.new(@request_params)
+      flexmock(@controller).should_receive(:query).and_yield.by_default
+      flexmock(RightScale::RightHttpClient).should_receive(:push).by_default
+    end
+
+    it "gets records to synchronize if the received sum does not match the calculated sum" do
+      flexmock(FakeGlobalSourceObject).should_receive(:get_initialization_hashes).and_return([{:a => :b}]).once
+      @request_params[:checksum_value] = 999
+      @controller.verify_replica_range.should be_true
+    end
+
+    it "sends synchronize_replica_range request to sink with records to synchronize" do
+      flexmock(EM).should_receive(:next_tick).and_yield.once
+      flexmock(FakeGlobalSourceObject).should_receive(:get_initialization_hashes).and_return([{:a => :b}]).once
+      flexmock(RightScale::RightHttpClient).should_receive(:push).with("/replicator/synchronize_replica_range",
+          on { |arg| arg[:records_to_synchronize] == [{:a => :b}] }, :scope => {:shard => 9}).once
+      @request_params[:checksum_value] = 999
+      @controller.verify_replica_range.should be_true
+    end
+
+    it "does not get records to synchronize if send_records_on_checksum_mismatch is false" do
+      flexmock(EM).should_receive(:next_tick).and_yield.once
+      flexmock(FakeGlobalSourceObject).should_receive(:get_initialization_hashes).never
+      flexmock(RightScale::RightHttpClient).should_receive(:push).with("/replicator/synchronize_replica_range",
+          on { |arg| arg[:records_to_synchronize] == [] }, :scope => {:shard => 9}).once
+      @request_params[:checksum_value] = 999
+      @request_params[:send_records_on_checksum_mismatch] = false
+      @controller.verify_replica_range.should be_true
+    end
+
+    it "does not get records to synchronize if the received checksum matches the calculated checksum" do
+      flexmock(EM).should_receive(:next_tick).and_yield.once
+      flexmock(FakeGlobalSourceObject).should_receive(:get_initialization_hashes).never
+      flexmock(RightScale::RightHttpClient).should_receive(:push).with("/replicator/synchronize_replica_range",
+          on { |arg| arg[:records_to_synchronize] == [] }, :scope => {:shard => 9}).once
+      @controller.verify_replica_range.should be_true
+    end
+
+    it "logs exception if synchronize_replica_range request fails" do
+      flexmock(@controller.logger).should_receive(:error).with(/Failed to synchronize_replica_range.*failure/).once
+      flexmock(EM).should_receive(:next_tick).and_yield.once
+      flexmock(RightScale::RightHttpClient).should_receive(:push).and_raise(Exception, "failure").once
+      @controller.verify_replica_range.should be_true
+    end
+
+    it "raises exception if query is unsuccessful" do
+      flexmock(@controller).should_receive(:query).and_return(false).once
+      lambda { @controller.verify_replica_range }.should raise_error(RightScale::Exceptions::QueryFailure)
+    end
+  end
+end