right_link 5.9.0

data/lib/instance/reenroll_manager.rb

@@ -0,0 +1,104 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Tracks reenroll votes and trigger reenroll as necessary
+  class ReenrollManager
+
+    # Number of votes required to trigger re-enroll
+    REENROLL_THRESHOLD = 3
+
+    # Additional number of seconds that should be spent in offline mode before
+    # triggering a re-enroll after vote threshold has been reached
+    MAXIMUM_REENROLL_DELAY = 900 # 15 minutes
+
+    # Delay in seconds until votes count is reset if no more votes occur
+    # This value should be more than two hours as this is the period at which
+    # votes will get generated in offline mode
+    RESET_DELAY = 7200 # 2 hours
+
+    # Are we currently reenrolling?
+    #
+    # === Return
+    # true:: If we are currently re-enrolling (note: process is going away)
+    # false:: Otherwise
+    def self.reenrolling?
+      @reenrolling
+    end
+
+    # Set reenrolling flag
+    #
+    # === Return
+    # true:: Always return true
+    def self.set_reenrolling
+      @reenrolling = true
+    end
+
+    # Vote for re-enrolling, if threshold is reached re-enroll
+    # If no vote occurs in the next two hours, then reset counter
+    #
+    # === Return
+    # true:: Always return true
+    def self.vote
+      @total_votes ||= 0
+      @reenrolling ||= false
+      @total_votes += 1
+      @reset_timer.cancel if @reset_timer
+      @reset_timer = EM::Timer.new(RESET_DELAY) { reset_votes }
+      if @total_votes >= REENROLL_THRESHOLD && !@reenrolling
+        delay = rand(MAXIMUM_REENROLL_DELAY)
+        Log.info("[re-enroll] Re-enroll threshold reached, shutting down and re-enrolling in #{delay} seconds")
+        set_reenrolling
+        EM::Timer.new(delay) { reenroll! }
+      end
+      true
+    end
+
+    # Reset votes count
+    #
+    # === Return
+    # true:: Always return true
+    def self.reset_votes
+      @total_votes = 0
+      @reset_timer = nil
+      true
+    end
+
+    # Trigger reenroll
+    #
+    # === Parameters
+    # args(String):: Arguments to be passed through to rs_reenroll
+    #
+    # === Return
+    # true:: If command ran successfully
+    # false:: Otherwise
+    def self.reenroll!(args=nil)
+      cmd = "rs_reenroll #{args}"
+      cmd += '&' unless RightScale::Platform.windows?
+      AMQP.stop { EM.stop } rescue nil
+      system(cmd)
+    end
+
+  end
+
+end

data/lib/instance/right_scripts_cookbook.rb

@@ -0,0 +1,181 @@
+#
+# Copyright (c) 2009-2012 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'fileutils'
+
+module RightScale
+
+  # Generate recipes dynamically for RightScripts
+  # Usage is:
+  #   1. Call 'recipe_from_right_script' for each RightScript that should be converted to a recipe
+  #   2. Call 'save' before running Chef, 'recipe_from_right_script' cannot be called after 'save'
+  #   3. Use 'repo_dir' to retrieve the Chef cookbook repo path (can be called at any time)
+  class RightScriptsCookbook
+
+    # Name of cookbook containing RightScript recipes
+    COOKBOOK_NAME = 'right_scripts_cookbook'
+
+    # Path to generated cookbook repo
+    attr_reader :repo_dir
+
+    # Wheter 'save' has been called
+    attr_reader :saved
+
+    # Setup temporary directory for cookbook repo containing
+    # recipes generated from RightScripts
+    def initialize(thread_name)
+      @saved        = false
+      @recipes      = {}
+      @repo_dir     = File.join(AgentConfig.right_scripts_repo_dir, thread_name)
+      @cookbook_dir = File.join(@repo_dir, COOKBOOK_NAME)
+      @recipes_dir  = File.join(@cookbook_dir, 'recipes')
+      FileUtils.rm_rf(@cookbook_dir)
+      FileUtils.mkdir_p(@recipes_dir)
+    end
+
+    # Add RightScript instantiation to cookbook
+    #
+    # === Parameters
+    # script(RightScale::RightScriptInstantiation):: RightScript to be added
+    #
+    # === Return
+    # recipe(RightScale::RecipeInstantiation):: Recipe that wraps RightScript
+    #
+    # === Raise
+    # (RightScale::Exceptions::Application):: If 'save' has been called
+    def recipe_from_right_script(script)
+      raise RightScale::Exceptions::Application, 'cannot create recipe after cookbook repo has been saved' if @saved
+      path = script_path(script.nickname)
+      recipe_name = File.basename(path)
+      @recipes[recipe_name] = script.nickname
+      recipe_content = <<-EOS
+right_script '#{script.nickname.gsub("'", "\\\\'")}' do
+  parameters(node["#{script.nickname}"]["parameters"])
+  cache_dir  '#{cache_dir(script)}'
+  source_file '#{path}'
+  display_version '#{script.display_version.to_s.strip.gsub("'", "\\\\'")}'
+end
+      EOS
+      File.open(path, 'w') { |f| f.puts script.source }
+      File.chmod(0744, path)
+      recipe_path = "#{path}.rb"
+      File.open(recipe_path, 'w') { |f| f.puts recipe_content }
+
+      recipe = RecipeInstantiation.new("#{COOKBOOK_NAME}::#{recipe_name}",
+                                       { script.nickname => { "parameters" => script.parameters } },
+                                       script.id, script.ready)
+
+    end
+
+    # Produce file name for given script nickname
+    #
+    # === Parameters
+    # nickname(String):: Script nick name
+    #
+    # === Return
+    # path(String):: Path to corresponding recipe
+    def script_path(nickname)
+      base_path = nickname.gsub(/[^0-9a-zA-Z_]/,'_')
+      base_path = File.join(@recipes_dir, base_path)
+      candidate_path = RightScale::Platform.shell.format_script_file_name(base_path)
+      i = 1
+      path = candidate_path
+      path = candidate_path + (i += 1).to_s while File.exists?(path)
+      path
+    end
+
+    # Save cookbook repo
+    #
+    # === Return
+    # true:: Always return true
+    def save
+      unless empty?
+        recipes = @recipes.keys.sort
+        metadata_content = <<-EOS
+description "Automatically generated repo, do not modify"
+#{recipes.map { |r| "recipe \"#{COOKBOOK_NAME}::#{r}\", \"RightScript < #{@recipes[r]} >\"" }.join("\n")}
+        EOS
+        metadata_path = File.join(@cookbook_dir, 'metadata.rb')
+        File.open(metadata_path, 'w') { |f| f.puts metadata_content }
+      end
+      @saved = true
+    end
+
+    # Whether given recipe name corresponds to a converted RightScript
+    #
+    # === Parameters
+    # nickname(String):: Recipe nickname
+    #
+    # === Return
+    # true:: If recipe was created from converting a RightScript
+    # false:: Otherwise
+    def self.right_script?(nickname)
+      # the RightScripts deserialized from core appear to not have the
+      # RightScript cookbook in their nickname. actual Chef recipes will have
+      # their cookbook name, so assume missing cookbook name means RightScript.
+      regex = /^(.*)::/
+      match = regex.match(nickname)
+      return match.nil? || COOKBOOK_NAME == match[1]
+    end
+
+    # Human friendly title for given recipe instantiation
+    #
+    # === Parameters
+    # recipe(String):: Recipe nickname
+    #
+    # === Return
+    # title(String):: Recipe title to be used in audits
+    def self.recipe_title(recipe)
+      title = right_script?(recipe) ? 'RightScript' : 'recipe'
+      title = "#{title} < #{recipe} >"
+    end
+
+    # Path to cache directory for given script
+    #
+    # === Parameters
+    # script(Object):: script object of some kind (e.g. RightScale::RightScriptInstantiation)
+    #
+    # === Return
+    # path(String):: Path to directory used for attachments and source
+    def cache_dir(script)
+      # prefix object ID with a text constant to make a legal directory name
+      # in case object id is negative (Ubuntu, etc.). this method will be called
+      # more than once and must return the same directory each time for a given
+      # script instantiation.
+      path = File.normalize_path(File.join(AgentConfig.cache_dir, 'right_scripts_content', "rs_attach" + script.object_id.to_s))
+
+      # convert to native format for ease of scripting in Windows, etc. the
+      # normalized path is normal for Ruby but not necessarily for native FS.
+      return RightScale::Platform.filesystem.pretty_path(path, true)
+    end
+
+    # Is there no RightScript recipe in repo?
+    #
+    # === Return
+    # true:: If +recipe_from_right_script+ was never called
+    # false:: Otherwise
+    def empty?
+      @recipes.empty?
+    end
+
+  end
+end

data/lib/instance/shutdown_request.rb

@@ -0,0 +1,221 @@
+#
+# Copyright (c) 2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Mixin for defining a common interface for a shutdown request class and its
+  # proxy class.
+  module ShutdownRequestMixin
+
+    # initial kind; no shutdown requested.
+    CONTINUE = 'continue'
+
+    # requested a reboot.
+    REBOOT = 'reboot'
+
+    # requested a stop (keep boot volume after shutdown).
+    STOP = 'stop'
+
+    # requested a terminate (discard boot volume after shutdown).
+    TERMINATE = 'terminate'
+
+    # levels.
+    LEVELS = [CONTINUE, REBOOT, STOP, TERMINATE]
+
+    # exceptions.
+    class NotInitialized < Exception; end
+    class InvalidLevel < Exception; end
+
+
+    # true if no shutdown was requested, false if shutdown was requested.
+    def continue?; CONTINUE == @level; end
+
+    # true if any requested shutdown will interrupt sequence of running scripts
+    # immediately (current script is allowed to complete).
+    # false to defer shutdown until all outstanding scripts have run.
+    def immediately?; @immediately; end
+    def immediately!
+      raise InvalidLevel.new("Immediately is unexpected for current shutdown state") if continue?
+      @immediately = true
+    end
+
+    # Shutdown request level.
+    def level; @level; end
+    def level=(value)
+      value = value.to_s
+      raise InvalidLevel.new("Invalid shutdown level: #{value.inspect}") unless LEVELS.include?(value)
+
+      # strictly escalate to higher level and ignore lower level requests.
+      @level = value if LEVELS.index(value) > LEVELS.index(@level)
+    end
+
+    # Stringizer.
+    def to_s
+      # note that printing 'deferred' would seem strange at the time when the
+      # deferred shutdown is actually being processed, so only say immediately.
+      immediacy = if @immediately; ' immediately'; else; ''; end
+      return "#{@level}#{immediacy}"
+    end
+
+    protected
+
+    def initialize
+      @level = CONTINUE
+      @immediately = false
+    end
+
+  end
+
+  # Represents outstanding request(s) for reboot, stop or terminate instance.
+  # Requests are cumulative and implicitly non-decreasing in level (e.g. reboot
+  # never superceeds terminate).
+  class ShutdownRequest
+
+    include ShutdownRequestMixin
+
+    # Class initializer.
+    #
+    # === Parameters
+    # scheduler(InstanceScheduler):: scheduler for shutdown requests
+    #
+    # === Return
+    # always true
+    def self.init(scheduler)
+      @@instance = ShutdownRequest.new
+      @@scheduler = scheduler
+      true
+    end
+
+    # Factory method
+    #
+    # === Return
+    # (ShutdownRequest):: the singleton for this class
+    def self.instance
+      raise NotInitialized.new("ShutdownRequest.init has not been called") unless defined?(@@instance)
+      return @@instance
+    end
+
+    # Submits a new shutdown request state which may be superceded by a
+    # previous, higher-priority shutdown level.
+    #
+    # === Parameters
+    # request[:level](String):: shutdown level
+    # request[:immediately](Boolean):: shutdown immediacy or nil
+    #
+    # === Returns
+    # result(ShutdownRequest):: the updated instance
+    def self.submit(request)
+      # RightNet protocols use kind instead of level, so be a little flexible.
+      result = instance
+      result.level = request[:kind] || request[:level]
+      result.immediately! if request[:immediately]
+      @@scheduler.schedule_shutdown unless result.continue?
+      return result
+    end
+
+    # Processes shutdown requests by communicating the need to shutdown an
+    # instance with the core agent, if necessary.
+    #
+    # === Parameters
+    # errback(Proc):: error handler or nil
+    # audit(Audit):: audit for shutdown action, if needed, or nil.
+    #
+    # === Block
+    # block(Proc):: continuation block for successful handling of shutdown or nil
+    #
+    # === Return
+    # always true
+    def process(errback = nil, audit = nil, &block)
+      # yield if not shutting down (continuing) or if already requested shutdown.
+      if continue? || @shutdown_scheduled
+        block.call if block
+        return true
+      end
+
+      # ensure we have an audit, creating a temporary audit if necessary.
+      sender = Sender.instance
+      agent_identity = sender.identity
+      if audit
+        case @level
+        when REBOOT, STOP, TERMINATE
+          operation = "/forwarder/shutdown"
+          payload = {:agent_identity => agent_identity, :kind => @level}
+        else
+          raise InvalidLevel.new("Unexpected shutdown level: #{@level.inspect}")
+        end
+
+        # request shutdown (kind indicated by operation and/or payload).
+        audit.append_info("Shutdown requested: #{self}")
+        sender.send_persistent_request(operation, payload) do |r|
+          res = OperationResult.from_results(r)
+          if res.success?
+            @shutdown_scheduled = true
+            block.call if block
+          else
+            fail(errback, audit, "Failed to shutdown instance", res)
+          end
+        end
+      else
+        AuditProxy.create(agent_identity, "Shutdown requested: #{self}") do |new_audit|
+          process(errback, new_audit, &block)
+        end
+      end
+      true
+    rescue Exception => e
+      fail(errback, audit, e)
+    end
+
+    protected
+
+    def initialize
+      super
+      @shutdown_scheduled = false
+    end
+
+    # Handles any shutdown failure.
+    #
+    # === Parameters
+    # audit(Audit):: Audit or nil
+    # errback(Proc):: error handler or nil
+    # msg(String):: Error message that will be audited and logged
+    # res(RightScale::OperationResult):: Operation result with additional information
+    #
+    # === Return
+    # always true
+    def fail(errback, audit, msg, res = nil)
+      if msg.kind_of?(Exception)
+        e = msg
+        detailed = Log.format("Could not process shutdown state #{self}", e, :trace)
+        msg = e.message
+      else
+        detailed = nil
+      end
+      msg += ": #{res.content}" if res && res.content
+      audit.append_error(msg, :category => RightScale::EventCategories::CATEGORY_ERROR) if audit
+      Log.error(detailed) if detailed
+      errback.call if errback
+      true
+    end
+
+  end  # ShutdownRequest
+
+end  # RightScale

data/lib/instance/single_thread_bundle_queue.rb

@@ -0,0 +1,189 @@
+#
+# Copyright (c) 2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  class SingleThreadBundleQueue < BundleQueue
+
+    attr_reader :thread_name
+
+    # Set continuation block to be called after 'close' is called
+    #
+    # === Block
+    # continuation block
+    def initialize(thread_name = ::RightScale::AgentConfig.default_thread_name, &continuation)
+      super(&continuation)
+      @active = false
+      @thread = nil
+      @thread_name = thread_name
+      @pid = nil
+      @mutex = Mutex.new
+      @queue = Queue.new
+      @sequence_finished = ConditionVariable.new
+    end
+
+    # Determines if queue is active
+    #
+    # === Return
+    # active(Boolean):: true if queue is active
+    def active?
+      active = false
+      @mutex.synchronize { active = @active }
+      return active
+    end
+
+    # Activate queue for execution, idempotent
+    # Any pending bundle will be run sequentially in order
+    #
+    # === Return
+    # true:: Always return true
+    def activate
+      @mutex.synchronize do
+        unless @active
+          @thread = Thread.new { run }
+          @active = true
+        end
+      end
+      true
+    end
+
+    # Determines if queue is busy
+    #
+    # === Return
+    # active(Boolean):: true if queue is busy
+    def busy?
+      busy = false
+      @mutex.synchronize { busy = !!@busy }
+      busy
+    end
+
+    # Push new context to bundle queue and run next bundle
+    #
+    # === Return
+    # true:: Always return true
+    def push(context)
+      @queue << context
+      true
+    end
+
+    # Clear queue content
+    #
+    # === Return
+    # true:: Always return true
+    def clear
+      @queue.clear
+      true
+    end
+
+    # Close queue so that further call to 'push' will be ignored
+    #
+    # === Return
+    # true:: Always return true
+    def close
+      push(FINAL_BUNDLE)
+    end
+
+    protected
+
+    # Run next bundle in the queue if active
+    # If bundle is FINAL_BUNDLE then call continuation block and deactivate
+    #
+    # === Return
+    # true:: Always return true
+    def run
+      loop do
+        context = @queue.shift
+        if context == FINAL_BUNDLE
+          break
+        elsif context == SHUTDOWN_BUNDLE
+          # process shutdown request.
+          ShutdownRequest.instance.process
+          # continue in queue in the expectation that the decommission bundle will
+          # shutdown the instance and its agent normally.
+        elsif !context.decommission? && ShutdownRequest.instance.immediately?
+          # immediate shutdown pre-empts any futher attempts to run operational
+          # scripts but still allows the decommission bundle to run.
+          context.audit.update_status("Skipped bundle due to immediate shutdown of #{@thread_name} thread: #{context.payload}")
+          # proceed ignoring bundles until final or shutdown are encountered.
+        else
+          sequence = create_sequence(context)
+          sequence.callback { audit_status(sequence) }
+          sequence.errback  { audit_status(sequence) }
+
+          # wait until sequence is finished using a ruby mutex conditional.
+          # need to synchronize before run to ensure we are waiting before any
+          # immediate signalling occurs (under test conditions, etc.).
+          @mutex.synchronize do
+            @busy = true
+            sequence.run
+            @sequence_finished.wait(@mutex)
+            @pid = nil
+            @busy = false
+          end
+        end
+      end
+      true
+    rescue Exception => e
+      Log.error(Log.format("SingleThreadBundleQueue.run failed for #{@thread_name} thread", e, :trace))
+    ensure
+      # invoke continuation (off of this thread which is going away).
+      @mutex.synchronize { @active = false }
+      run_continuation
+      @thread = nil
+    end
+
+    # Factory method for a new sequence.
+    #
+    # context(RightScale::OperationContext)
+    def create_sequence(context)
+      pid_callback = lambda do |sequence|
+        # TODO preserve cook PIDs per thread in InstanceState and recover
+        # orphaned cook in case of agent crash.
+        @mutex.synchronize { @pid = sequence.pid }
+      end
+      return RightScale::ExecutableSequenceProxy.new(context, :pid_callback => pid_callback )
+    end
+
+    # Audit executable sequence status after it ran
+    #
+    # === Parameters
+    # sequence(RightScale::ExecutableSequence):: finished sequence being audited
+    #
+    # === Return
+    # true:: Always return true
+    def audit_status(sequence)
+      context = sequence.context
+      title = context.decommission? ? 'decommission ' : ''
+      title += context.succeeded ? 'completed' : 'failed'
+      context.audit.update_status("#{title}: #{context.payload}")
+      true
+    rescue Exception => e
+      Log.error(Log.format("SingleThreadBundleQueue.audit_status failed for #{@thread_name} thread", e, :trace))
+    ensure
+      # release queue thread to wait on next bundle in queue. we must ensure
+      # that we are not currently on the queue thread so next-tick the signal.
+      EM.next_tick { @mutex.synchronize { @sequence_finished.signal } }
+    end
+
+  end
+
+end