right_infrastructure_agent 1.1.2

data/lib/right_infrastructure_agent/global_object_replicator_source.rb

@@ -0,0 +1,117 @@
+#--
+# Copyright (c) 2011-2013 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and the licensee.
+#++
+
+module RightScale
+
+  # This module is for use as a mixin in an HTTP controller that handles requests from
+  # global object replication sink to check whether replication is required for a global
+  # object owned by this server.
+  #
+  # This module expects the following to be defined:
+  #   - logger - variable pointing to the standard logger in use
+  #   - global_object_replicator_source - method returning type of replication source, e.g., 'library'
+  module GlobalObjectReplicatorSource
+
+    include RightScale::InfrastructureHelpers
+    include RightScale::ModelsHelper
+
+    # Compares checksum parameter with local state of a replicated table and sends backs a
+    # replicator/synchronize_replica_range with records that may need to be updated in the core DB
+    #
+    # @param :sink [String] Type of replication sink making request, e.g., "core" or "library"
+    # @param :class_name [String] Name of a class that acts_as_global_object
+    # @param :schema_version [Integer] Requested schema version for any synchronization records
+    # @param :checksum_type [String] Type of checksum: only 'global_object_version_sum'
+    # @param :max_id_at_start [Integer] State passed back to the core in the resulting
+    #   synchronize_replica_range request
+    # @param :begin_id [Integer, NilClass] First row ID in the checksum range,
+    #   or nil for all rows
+    # @param :end_id [Integer, NilClass] Last row ID in the checksum range,
+    #   or nil for all rows
+    # @param :send_records_on_checksum_mismatch [Boolean] Whether any records should be
+    #   returned in the resulting synchronize_replica_range request
+    # @param :checksum_value [Integer, NilClass] Checksum value calculated using the
+    #   specific :checksum_type
+    # @param :shard_id [Integer] Shard ID of the sender
+    #
+    # @return [NilClass] nil
+    #
+    # @raise [ArgumentError] Unknown replication sink
+    # @raise [RightScale::Exceptions::QueryFailure] Failed to complete synchronization query
+    # @raise [RightScale::Exceptions::RetryableError] Query failed but may be retried
+    def verify_replica_range
+      global_object_class = constantize(params[:class_name])
+      schema_version = to_int_or_nil(params[:schema_version])
+      checksum_type = params[:checksum_type]
+      max_id_at_start = to_int_or_nil(params[:max_id_at_start])
+      begin_id = to_int_or_nil(params[:begin_id])
+      end_id = to_int_or_nil(params[:end_id])
+      send_records_on_checksum_mismatch = params[:send_records_on_checksum_mismatch]
+      checksum_value = params[:checksum_value]
+      shard_id = to_int_or_nil(params[:shard_id])
+
+      success = query("verify_replica_range for #{global_object_class.name} rows #{begin_id} " +
+                      " - #{end_id}", :email_errors => true) do
+        records_to_synchronize = []
+
+        if (global_object_class.calculate_global_object_checksum(checksum_type, schema_version, begin_id, end_id) == checksum_value)
+          checksum_matched = true
+          logger.debug("GlobalObjectReplica: Verified #{global_object_class.name} global_object_version_sum " +
+            "for rows #{begin_id} - #{end_id}")
+        else
+          checksum_matched = false
+          logger.debug("GlobalObjectReplica: Verification of #{global_object_class.name} global_object_version_sum " +
+            "rows #{begin_id} - #{end_id} failed. Sending synchronization records.")
+          records_to_synchronize = if send_records_on_checksum_mismatch
+            global_object_class.get_initialization_hashes(schema_version, begin_id, end_id)
+          else
+            []
+          end
+        end
+
+        replicator = case params[:sink] # TODO Replace this with "#{params[:sink]}_replicator" once all replicator actors gone
+                     when "core" then (global_object_replicator_source == "library") ? "replicator" : "wasabi_replicator_sink"
+                     when "library" then "wasabi_replicator_sink"
+                     else raise ArgumentError, "Unknown replication sink: #{params[:sink].inspect}"
+                     end
+        payload = {
+          :source                 => global_object_replicator_source,
+          :class_name             => global_object_class.name,
+          :checksum_type          => checksum_type,
+          :max_id_at_start        => max_id_at_start,
+          :begin_id               => begin_id,
+          :end_id                 => end_id,
+          :checksum_matched       => checksum_matched,
+          :records_to_synchronize => records_to_synchronize,
+          :has_more               => global_object_class.max_id > end_id }
+
+        EM.next_tick do
+          # Execute this request on next_tick since, depending on the configuration, the router could route
+          # this request directly via HTTP and there would be no break in the chain of replication requests
+          begin
+            RightScale::RightHttpClient.push("/#{replicator}/synchronize_replica_range", payload, :scope => {:shard => shard_id})
+          rescue Exception => e
+            logger.error(format_error("Failed to synchronize_replica_range for class #{global_object_class.name}", e))
+          end
+        end
+
+        true
+      end
+
+      raise RightScale::Exceptions::QueryFailure.new(@last_error) unless success
+
+      render_nothing
+    end
+
+  end # GlobalObjectReplicatorSource
+
+end # RightScale

data/lib/right_infrastructure_agent/infrastructure_auth_client.rb

@@ -0,0 +1,88 @@
+#--
+## Copyright (c) 2014 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and
+# the licensee.
+#++
+
+require 'global_session'
+
+module RightScale
+
+  # Authorization client for infrastructure agents
+  class InfrastructureAuthClient < AuthClient
+
+    include RightScale::InfrastructureHelpers
+
+    # Initialized authorization client
+    #
+    # @param [String] client_name of application using this client
+    # @param [String] router_url including base path for accessing RightNet router
+    # @param [String] config_dir path to global session configuration information
+    #
+    # @options option [String] :agent_id identifying this agent in AgentIdentity format
+    def initialize(client_name, router_url, config_dir, options = {})
+      @client_name = client_name
+      @router_url = router_url
+      @agent_id = options[:agent_id]
+
+      config = GlobalSession::Configuration.new(File.join(config_dir, "global_session.yml"), ENV["RAILS_ENV"])
+      @global_session_dir = constantize(config["directory"]).new(config, File.join(config_dir, "authorities"))
+      @global_session_timeout = (config["timeout"] * 8) / 10
+
+      @state = :authorized
+      reset_stats
+    end
+
+    # Headers to be added to HTTP request
+    # Include authorization header by default
+    #
+    # @return [Hash] headers to be added to request header
+    #
+    # @raise [Exceptions::Unauthorized] not authorized
+    # @raise [Exceptions::RetryableError] authorization expired, but retry may succeed
+    def headers
+      check_authorized
+      auth_header
+    end
+
+    # Headers to be added to HTTP request
+    #
+    # @return [Hash] headers to be added to request header
+    #
+    # @raise [Exceptions::Unauthorized] not authorized
+    # @raise [Exceptions::RetryableError] authorization expired, but retry may succeed
+    def auth_header
+      check_authorized
+      {"Authorization" => "Bearer #{infrastructure_session}"}
+    end
+
+    protected
+
+    # Retrieve or create global session for infrastructure agent
+    # Cache session and only recreate when times out
+    #
+    # @return [String] infrastructure session
+    def infrastructure_session
+      now = Time.now
+      if @cached_infrastructure_session && (now - @cached_infrastructure_session[:created_at]) < @global_session_timeout
+        @cached_infrastructure_session[:session]
+      else
+        global_session = GlobalSession::Session.new(@global_session_dir)
+        global_session["infrastructure"] = @client_name
+        global_session["agent"] = @agent_id if @agent_id
+        session = global_session.to_s
+        @cached_infrastructure_session = {:session => session, :created_at => now}
+        session
+      end
+    end
+
+  end # InfrastructureAuthClient
+
+end # RightScale

data/lib/right_infrastructure_agent/infrastructure_helpers.rb

@@ -0,0 +1,85 @@
+#--
+# Copyright (c) 2011-2013 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and the licensee.
+#++
+
+module RightScale
+
+  module InfrastructureHelpers
+
+    # Render result as nothing
+    #
+    # @return [TrueClass] Always return true
+    def render_nothing
+      render(:nothing => true, :status => :no_content)
+      true
+    end
+
+    # Format log message for an error
+    #
+    # @param description [String] Error description that is placed first in output
+    # @param exception [Exception, String] Exception or exception message
+    # @param backtrace [Symbol] Exception backtrace extent: :no_trace, :caller, or :trace,
+    #   defaults to :caller
+    #
+    # @return [String] Formatted error
+    def format_error(description, exception, backtrace = :caller)
+      RightScale::Log.format(description, exception, backtrace)
+    end
+
+    # Convert string parameter to integer if not nil or empty
+    #
+    # @param param [String, NilClass] Parameter value
+    #
+    # @return [Integer, NilClass] Integer form of parameter, or nil if not a string
+    def to_int_or_nil(param)
+      (param.nil? || (param.respond_to?(:empty?) && param.empty?)) ? nil : param.to_i
+    end
+
+    # Convert string parameter to boolean
+    #
+    # @param param [String, TrueClass, FalseClass, NilClass] Parameter value
+    #
+    # @return [TrueClass, FalseClass] Boolean form of parameter
+    def to_bool(param)
+      ![nil, false, "false"].include?(param)
+    end
+
+    # Tries to find a constant with the name specified in the argument string:
+    #
+    #   "Module".constantize     # => Module
+    #   "Test::Unit".constantize # => Test::Unit
+    #
+    # The name is assumed to be the one of a top-level constant, no matter whether
+    # it starts with "::" or not. No lexical context is taken into account:
+    #
+    #   C = 'outside'
+    #   module M
+    #     C = 'inside'
+    #     C               # => 'inside'
+    #     "C".constantize # => 'outside', same as ::C
+    #   end
+    #
+    # NameError is raised when the name is not in CamelCase or the constant is
+    # unknown.
+    def constantize(camel_cased_word)
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+      end
+      constant
+    end
+
+  end # InfrastructureHelpers
+
+end # RightScale

data/lib/right_infrastructure_agent/login_policy_factory.rb

@@ -0,0 +1,137 @@
+# Copyright (c) 2009-2014 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and the licensee.
+
+module RightScale
+
+  class LoginPolicyFactory
+    # Regexp used to substitute sequences of non-username-friendly characters with a humble underscore.
+    # Technically we are more strict than the POSIX standard allows (a username could have any character
+    # in his name that is a valid filename character), but we want to have sane *and* legal usernames.
+    INVALID_USERNAME_CHARS = /[^a-z0-9_]+/
+
+    # Number of seconds before a public key is considered old
+    OLD_PUBLIC_KEY_BOUNDARY_AGE = 24 * 60 * 60
+
+    # Create LoginPolicy for given account, including all users that are authorized
+    # to login, their public keys, etc. The policy is based on Permissions, UserCredentials
+    # and Settings of the account and is uniform across all instances of the account.
+    # Optionally omit public keys that have not been updated recently with the expectation
+    # that the receiver of this policy instead uses the fingerprint to determine
+    # whether it needs to request the update.
+    #
+    # === Parameters
+    # account(Account):: Account that login policy should be built for
+    # terse(Boolean):: Whether to omit a public key from the policy if it was created
+    #   more than OLD_PUBLIC_KEY_BOUNDARY_AGE seconds ago
+    #
+    # === Return
+    # policy(RightScale::LoginPolicy) Policy specifying which users may login & some related metadata
+    def self.policy_for_account(account, terse = false)
+      # Find all users who are allowed and able to do managed login. Simultaneously build some indices
+      # of timestamps authorizing perms, which will be used presently to create the policy object.
+
+      # Build the policy and its list of users
+      old_cred_boundary = Time.now - OLD_PUBLIC_KEY_BOUNDARY_AGE
+      timestamps_max = Time.at(0)
+      policy = LoginPolicy.new
+      policy.exclusive  = account.setting('managed_login_mandatory')
+
+      # Do this as a closure around the timestamp and cred boundaries instead
+      # of a method to avoid having to pass them back and forth constantly.
+      create_login_user = Proc.new do |u, is_superuser|
+        cred_updated_at = Time.parse(u.cred_updated_at)
+        timestamps_max = [timestamps_max, Time.parse(u.perm_updated_at), cred_updated_at].max
+
+        # Currently there is only one public key per user but in the future there may be more
+        # There should always be the exact number of fingerprints
+        public_keys = [u.cred_public_value]
+        processed_keys = []
+        public_key_fingerprints = [u.cred_public_fingerprint]
+        processed_fingerprints = []
+
+        if terse && cred_updated_at < old_cred_boundary
+          processed_keys = public_keys.map { |k| nil }
+          processed_fingerprints = public_key_fingerprints
+        else
+          # Post-process the public keys to make sure they are well-formed AND contain a comment
+          # at the end of the line. Technically the comment is optional, but a bug in RightImage
+          # 5.1.1 - 5.6 makes the comment mandatory else the instance has heinous problems
+          # when trying to update its managed login policy.
+          public_keys.zip(public_key_fingerprints).each do |(k, f)|
+            if (components = LoginPolicy.parse_public_key(k))
+              components[3] ||= u.email
+              processed_keys << components.compact.join(' ')
+              processed_fingerprints << f
+            else
+              # In case this is some malformed or other weird key, omit its value
+            end
+          end
+        end
+
+        uuid        = u.id.to_s
+        username    = u.email.split('@', 2).first.downcase.gsub(INVALID_USERNAME_CHARS,'_')
+        common_name = u.email
+        superuser   = is_superuser
+        expires_at  = u.perm_deleted_at.nil? ? nil : Time.parse(u.perm_deleted_at)
+
+        LoginUser.new(uuid, username, public_key = nil, common_name, superuser, expires_at, processed_keys,
+                      profiled_data = nil, processed_fingerprints)
+      end
+
+      # Get list of all super users
+      superusers = users_with_role(account, 'server_superuser')
+
+      # Build all users conditionally on inclusion of superusers list
+      policy.users = users_with_role(account, 'server_login').map do |u|
+        create_login_user.call(u, (superusers && superusers.include?(u)) || false )
+      end
+
+      policy.created_at = timestamps_max
+
+      return policy
+    end
+
+    # Create LoginPolicy for given instance, including all users that are authorized
+    # to login, their public keys, etc. Currently this just delegates to #policy_for_account
+    # since all instances of a given account have the same policy at a given time. This may
+    # change in the future, however.
+    #
+    # === Parameters
+    # instance(Ec2Instance):: Instance that login policy should be built for
+    # audit_id(Integer):: Audit identifier
+    # terse(Boolean):: Whether to omit a public key from the policy if it was created
+    #   more than OLD_PUBLIC_KEY_BOUNDARY_AGE seconds ago
+    #
+    # === Return
+    # policy(RightScale::LoginPolicy) Policy specifying which users may login & some related metadata
+    def self.policy_for_instance(instance, audit_id, terse = false)
+      policy = policy_for_account(instance.account, terse)
+      policy.audit_id = audit_id
+      return policy      
+    end
+
+    def self.users_with_role(account, role)
+      User.all( :select     => "`users`.*, `user_credentials`.public_value as cred_public_value, " +
+                               "`user_credentials`.updated_at as cred_updated_at, " +
+                               "`user_credentials`.public_value_fingerprint as cred_public_fingerprint, " +
+                               "`permissions`.updated_at as perm_updated_at, " +
+                               "`permissions`.deleted_at as perm_deleted_at",
+                :joins      => [:user_credential, :permissions],
+                :conditions => ["permissions.account_id = ?  " +
+                                  "AND permissions.role_id = ?  " +
+                                  "AND (permissions.deleted_at IS NULL OR permissions.deleted_at > ?) " +
+                                  "AND user_credentials.public_value IS NOT NULL " +
+                                  "AND user_credentials.public_value <> ''",
+                               account.id, Role[role], Time.now.utc ])
+    end
+
+  end # LoginPolicyFactory
+
+end # RightScale

data/lib/right_infrastructure_agent/models_helper.rb

@@ -0,0 +1,483 @@
+# Copyright (c) 2009-2011 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and
+# the licensee.
+
+module RightScale
+
+  # Helper methods for accessing ActiveRecord models
+  # They are only usable when executing in a Rails environment
+  module ModelsHelper
+
+    # Pattern in exception message from trigger in core when account is not in current shard
+    WRONG_SHARD_ERROR = "ERROR_UPDATE_NOT_ALLOWED does not exist"
+
+    # Retry exception message for when get wrong shard exception
+    WRONG_SHARD_RETRY_MESSAGE = "Account temporarily unavailable"
+
+    # Retry exception message for when account disabled due to shard migration
+    DISABLED_SHARD_RETRY_MESSAGE = "Account temporarily unavailable"
+
+    # Default retry exception message
+    DEFAULT_RETRY_MESSAGE = "RightScale database temporarily unavailable"
+
+    # Mysql and ActiveRecord case-insensitive exception message content that if present causes the
+    # error to be propagated to the requester as retryable after max retries is exceeded internally
+    RETRYABLE_ERRORS = ["deadlock found", "lock wait timeout", "can't connect to", "has gone away", WRONG_SHARD_ERROR]
+
+    # Query database using retry on failure and reconnect handling
+    # Audit and/or log error if given block returns nil or raises, return block result otherwise
+    # Store any error message in @last_error
+    #
+    # === Parameters
+    # description(String):: Description of query action that is used in error messages
+    # options(Hash):: Query options:
+    #   :audit(AuditEntry):: Audit entry used to append error message if any
+    #   :include_backtrace_in_last_error(Boolean):: Whether to pass :trace to Log.format for exceptions
+    #   :email_errors(Boolean):: Whether to send email for errors
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Block
+    # Accesses MySQL and returns result, required
+    #
+    # === Return
+    # (Object|nil):: Value returned by block, or nil if failed
+    def query(description, options = {}, &blk)
+      begin
+        @last_error = nil
+        run_query(options, &blk)
+      rescue Exception => e
+        retryable = e.is_a?(RightScale::Exceptions::RetryableError)
+        description = "Failed to #{description}" + (retryable ? " but retryable" : "")
+        Log.error(description, e, :trace)
+        if options[:include_backtrace_in_last_error]
+          @last_error = Log.format(description, e, :trace)
+        else
+          @last_error = Log.format(description, e)
+        end
+        options[:audit].append(AuditFormatter.error(@last_error)) if options[:audit]
+
+        if(options[:email_errors])
+          ExceptionMailer.deliver_notification(description, e.message, e)
+        end
+
+        raise if retryable
+        nil
+      end
+    end
+
+    # Run database query block
+    # When catch retryable MySQL and ActiveRecord errors, rerun block, retry up to 3 times
+    #
+    # === Parameters
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Block
+    # Accesses MySQL and returns result, required
+    #
+    # === Return
+    # res(Object|nil):: Value returned by given block, or nil if desired data was not found
+    #
+    # === Raise
+    # RuntimeError:: Block is missing
+    # RetryableError: If exceed max retries and :retryable_error option enabled
+    # Also re-raises any query block exceptions
+    def run_query(options = {})
+      raise 'Missing block' unless block_given?
+      res = nil
+      disconnected = true
+      while disconnected do
+        retries = 0
+        begin
+          res = yield
+          disconnected = false
+        rescue ActiveRecord::RecordNotFound
+          res = nil
+          disconnected = false
+        rescue Exception => e
+          if is_retryable_error?(e, local = true)
+            if retries >= 3
+              Log.warning("Aborting query after 3 failed retries")
+              if options[:retryable_error] && is_retryable_error?(e)
+                if e.message =~ /#{WRONG_SHARD_ERROR}/i
+                  raise RightScale::Exceptions::RetryableError.new(WRONG_SHARD_RETRY_MESSAGE, e)
+                else
+                  raise RightScale::Exceptions::RetryableError.new(DEFAULT_RETRY_MESSAGE, e)
+                end
+              else
+                raise # re-raise the exception
+              end
+            else
+              retries += 1
+              Log.error("Failed running MySQL query", e, :trace)
+              Log.info("Retrying query...")
+              retry
+            end
+          else
+            raise # re-raise the exception
+          end
+        end
+      end
+      res
+    end
+
+    # Is given exception a MySQL exception worth retrying, e.g., a deadlock or timeout?
+    #
+    # === Parameter
+    # e(Exception):: Exception to be tested
+    # local(Boolean):: Whether making this decision for local or external consumption
+    #
+    # === Return
+    # (Boolean):: true if worth retrying, otherwise false or nil
+    def is_retryable_error?(e, local = false)
+      (e.is_a?(MysqlError) || e.is_a?(ActiveRecord::ActiveRecordError)) && (local ||
+      (RETRYABLE_ERRORS + ActiveRecord::ConnectionAdapters::MysqlAdapter::LOST_CONNECTION_ERROR_MESSAGES).find { |m| e.message =~ /#{m}/i })
+    end
+
+    # Retrieve existing audit or create new one
+    #
+    # === Parameters
+    # instance(Instance):: Instance used as auditee
+    # summary(String):: New audit summary, default to empty string
+    # detail(String):: New audit detail, default to empty string
+    # account(Account):: Account associated with audit, default to instance's account
+    # user(User):: User who caused the activity
+    # options(Hash):: Query options:
+    #   :audit_id(Integer):: Audit entry id if audit is to be retrieved
+    #   :agent_id(String):: Serialized agent identity if audit is to be created (:agent_identity
+    #     is an alternative deprecated option name)
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (AuditEntry):: Audit entry model
+    #
+    # === Raise
+    # ArgumentError:: If neither :audit_id nor :agent_id is specified
+    def retrieve_or_create_audit(instance, summary = '', detail = '', account = instance.account, user = nil, options = {})
+      if options[:audit_id] && options[:audit_id] != -1
+        retrieve("audit with id #{options[:audit_id]}") { audit_entry(options[:audit_id], options) }
+      elsif (agent_id = options[:agent_id] || options[:agent_identity])
+        query("create audit for instance agent #{agent_id}", options) do
+          AuditEntry.create!( {:auditee => instance, :summary => summary, :detail => detail,
+                               :account => account, :user => user} )
+        end
+      else
+        raise ArgumentError, "Must specify audit ID or agent ID"
+      end
+    end
+
+    # Retrieve existing user or get default user stub
+    # Store any error message in @last_error
+    #
+    # === Parameters
+    # instance(Instance):: Instance for account
+    # options(Hash):: Query options:
+    #   :user_id(Integer):: User id or 0 meaning use stub
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # current_user(User):: User retrieved or stubbed
+    def retrieve_or_default_user(instance, options = {})
+      user_id = (options[:user_id] || 0).to_i  # ensure user id is non-nil integer
+      if user_id == 0
+        current_user = User.new(:email => 'alerter@rightscale.com')
+        current_user.id = 0
+      else
+        account = instance.account
+        current_user = query("User in account #{account} with id #{user_id}", options) do
+          account.users.detect { |u| u.id == user_id }
+        end
+      end
+      current_user
+    end
+
+    # Retrieve database object using the ModelsHelper functions below that themselves use #run_query,
+    # e.g., retrieve("recipe for instance", :audit => audit) { recipe(id, :retryable_error => true }
+    # Audit and/or log error if block returns nil or raises exception, otherwise return block result
+    # Store any error message in @last_error
+    #
+    # === Parameters
+    # description(String):: Description of object that is used in error messages
+    # options(Hash):: Query options:
+    #   :audit(AuditEntry):: Audit entry used to append error message if any
+    #   :log(Boolean):: Whether to log message when object does not exist
+    #
+    # === Block
+    # Performs query to retrieve object, required
+    # The block should retrieve using other ModelsHelper functions like #instance or #account
+    # so that #run_query is applied rather than accessing models directly in the block
+    #
+    # === Return
+    # item(Object):: Value returned by block, or nil if not found or failed
+    #
+    # === Raise
+    # RuntimeError:: Block is missing
+    def retrieve(description, options = {})
+      raise 'Missing block' unless block_given?
+      retryable = nil
+      begin
+        @last_error = nil
+        unless item = yield
+          @last_error = "Could not find #{description}"
+          Log.warning(@last_error) if options[:log]
+        end
+      rescue Exception => e
+        retryable = e if e.is_a?(RightScale::Exceptions::RetryableError)
+        description = "Failed to retrieve #{description}" + (retryable ? " but retryable" : "")
+        Log.error(description, e, e.is_a?(RightScale::Exceptions) ? :caller : :trace)
+        @last_error = Log.format(description, e)
+        item = nil
+      end
+      options[:audit].append(AuditFormatter.error(@last_error)) if options[:audit] && item.nil? && @last_error
+      raise retryable if retryable
+      item
+    end
+
+    # Retrieve InstanceApiToken model with given id
+    #
+    # === Parameters
+    # id(Integer):: Id of InstanceApiToken to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (InstanceApiToken|nil):: Corresponding API token, or nil if no token with this id exists
+    def instance_token(id, options = {})
+      run_query(options) { InstanceApiToken.find(id) }
+    end
+
+    # Retrieve instance model with given API token id
+    #
+    # === Parameters
+    # token_id(Integer):: Id of InstanceApiToken of instance to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # instance(Ec2Instance|Instance|nil):: Corresponding instance, or nil if no token with
+    #   this id exists
+    #
+    # === Raise
+    # RetryableError:: If account disabled for shard migration, or if exceed maximum retries
+    def instance(token_id, options = {})
+      token = instance_token(token_id, options)
+      token && (instance = token.instance)
+      if instance && instance.account.disabled_in_every_shard?
+        raise RightScale::Exceptions::RetryableError.new(DISABLED_SHARD_RETRY_MESSAGE)
+      end
+      instance
+    end
+
+    # Get instance from API token id
+    # Cache all tokens retrieved
+    #
+    # === Parameters
+    # token_id(Integer):: API token id
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # instance(Instance):: Corresponding instance
+    #
+    # === Raise
+    # RetryableError:: If account disabled for shard migration, or if exceed maximum retries
+    # RightScale::Exceptions::Application:: If InstanceApiToken or Instance not found
+    def instance_from_token_id(token_id, options = {})
+      @tokens ||= {}
+      if instance = @tokens[token_id]
+        run_query(options) { instance.reload }
+      else
+        # Not in cache, look it up
+        instance_api_token = instance_token(token_id)
+        raise RightScale::Exceptions::Application, "Instance token with id '#{token_id}' not found" unless instance_api_token
+        instance = instance_api_token.instance
+        raise RightScale::Exceptions::Application, "Instance with token id '#{token_id}' not found" unless instance
+        @tokens[token_id] = instance
+      end
+      if instance && instance.account.disabled_in_every_shard?
+        raise RightScale::Exceptions::RetryableError.new(DISABLED_SHARD_RETRY_MESSAGE)
+      end
+      instance
+    end
+
+    # Get instance model corresponding to instance agent with given identity
+    #
+    # === Parameters
+    # identity(String):: Serialized instance agent identity
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (Instance):: Corresponding instance
+    #
+    # === Raise
+    # ArgumentError:: Invalid agent identity
+    def instance_from_agent_id(agent_id, options = {})
+      raise ArgumentError, "Invalid agent identity" unless AgentIdentity.valid?(agent_id)
+      instance_from_token_id(AgentIdentity.parse(agent_id).base_id, options)
+    end
+
+    # Retrieve account with given id
+    #
+    # === Parameters
+    # id(Integer):: Id of account to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (Account|nil):: Corresponding account, or nil if no account with this id exists
+    def account(id, options = {})
+      run_query(options) { Account.find(id) }
+    end
+
+    # Retrieve AuditEntry with given id
+    #
+    # === Parameters
+    # id(Integer):: Id of AuditEntry to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (AuditEntry|nil):: Corresponding setting, or nil if no setting with this id exists
+    def audit_entry(id, options = {})
+      run_query(options) { AuditEntry.find(id) }
+    end
+
+    # Retrieve RightScript with given id
+    #
+    # === Parameters
+    # id(Integer):: Id of RightScript to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (RightScript|nil):: Corresponding RightScript or nil if no RightScript with this id exists
+    def right_script(id, options = {})
+      run_query(options) { RightScript.find(id) }
+    end
+
+    # Retrieve RightScript with given name on given instance
+    #
+    # === Parameters
+    # name(String):: Name of RightScript that should be retrieved
+    # instance(Instance):: Instance on which RightScript is defined
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # script(RightScript):: Corresponding RightScript
+    def right_script_from_name(name, instance, options = {})
+      script = nil
+      template = run_query(options) { instance.server_template }
+      if template
+        script = run_query(options) { template.right_scripts.find_by_name(name) }
+      end
+      script
+    end
+
+    # Retrieve Chef recipe with given id
+    #
+    # === Parameters
+    # id(Integer):: Id of ServerTemplateChefRecipe to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (ServerTemplateChefRecipe|nil):: Corresponding recipe, or nil if no recipe with this id exists
+    def recipe(id, options = {})
+      run_query(options) { ServerTemplateChefRecipe.find(id) }
+    end
+
+    # Retrieve recipe with given name on given instance
+    #
+    # === Parameters
+    # name(String):: Name of recipe that should be retrieved
+    # instance(Instance):: Instance on which recipe is defined
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # recipe(ServerTemplateChefRecipe):: Corresponding recipe
+    def recipe_from_name(name, instance, options = {})
+      recipe = nil
+      template = run_query(options) { instance.server_template }
+      if template
+        recipe = run_query(options) { template.server_template_chef_recipes.find_by_recipe(name) }
+      end
+      recipe
+    end
+
+    # Retrieve Permission with given id
+    #
+    # === Parameters
+    # id(Integer):: Id of Permission to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (Permission|nil):: Corresponding permission, or nil if no permission with this id exists
+    def permission(id, options = {})
+      run_query(options) { Permission.find(id) }
+    end
+
+    # Retrieve UserCredential with given id
+    #
+    # === Parameters
+    # id(Integer):: Id of UserCredential to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (UserCredential|nil):: Corresponding user credential, or nil if no credential with this id exists
+    def user_credential(id, options = {})
+      run_query(options) { UserCredential.find(id) }
+    end
+
+    # Retrieve UserCredential with given public value fingerprint
+    #
+    # === Parameters
+    # public_value_fingerprint(String):: Public value fingerprint of UserCredential to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (UserCredential|nil):: Corresponding user credential, or nil if no credential with this fingerprint exists
+    def user_credential_from_fingerprint(public_value_fingerprint, options = {})
+      run_query(options) { UserCredential.find_by_public_value_fingerprint(public_value_fingerprint) }
+    end
+
+    # Retrieve Setting with given id
+    #
+    # === Parameters
+    # id(Integer):: Id of Setting to be retrieved
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (Setting|nil):: Corresponding setting, or nil if no setting with this id exists
+    def setting(id, options = {})
+      run_query(options) { Setting.find(id) }
+    end
+
+    # Retrieve all software repositories
+    #
+    # === Parameters
+    # options(Hash):: Query options:
+    #   :retryable_error(Boolean):: Whether to raise RetryableError if exceed maximum retries
+    #
+    # === Return
+    # (Array):: Array of Repository
+    def repositories(options = {})
+      run_query(options) { Repository.find(:all) }
+    end
+
+  end # ModelHelpers
+
+end # RightScale