right_link 5.9.0

data/lib/instance/downloader.rb

@@ -0,0 +1,208 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'fileutils'
+
+module RightScale
+
+  # Abstract download capabilities
+  class Downloader
+
+    DEFAULT_MAX_DOWNLOAD_RETRIES = 10
+
+    # Initialize downloader with given retry period
+    # When using backoff algorithm the retry period specifies the initial value, the value then gets
+    # incremented after each retry exponentially until it reaches the specified maximum retry period
+    #
+    # === Parameters
+    # retry_period(Integer):: Retry period in seconds - defaults to 10 seconds
+    # use_backoff(Boolean):: Whether download should use a backoff algorithm to space retries - defaults to true
+    # max_retry_period(Integer):: Maximum retry period in seconds, only meaningful when using backoff algorithm -
+    #                             defaults to 5 minutes
+    def initialize(retry_period = 10, use_backoff = true, max_retry_period = 5 * 60)
+      @retry_period = retry_period
+      @use_backoff = use_backoff
+      @max_retry_period = max_retry_period if use_backoff
+      platform = RightScale::Platform
+      @found_curl = platform.filesystem.has_executable_in_path('curl')
+    end
+
+    # (Integer) Retry period in seconds
+    attr_accessor :retry_period
+
+    # (Boolean) Whether download should use a backoff algorithm to space retries
+    attr_accessor :use_backoff
+
+    # (Integer) Maximum retry period in seconds, only meaningful when using backoff algorithm
+    attr_accessor :max_retry_period
+
+    # (Integer) Size in bytes of last successful download (nil if none)
+    attr_reader :size
+
+    # (Integer) Speed in bytes/seconds of last successful download (nil if none)
+    attr_reader :speed
+
+    # Error message associated with last failure (nil if none)
+    #
+    # === Return
+    # error(String):: Error message
+    # nil:: No error occured during last download
+    def error
+      error = (@errors.nil? ||  @errors.empty?) ? nil : @errors.join("\n")
+    end
+
+    # Was last download successful?
+    #
+    # === Return
+    # true:: If last download was successful or there was no download yet
+    # false:: Otherwise
+    def successful?
+      error.nil?
+    end
+
+    # Download file synchronously and report on success, download size and download speed.
+    # Use successful, size and speed to query about last download.
+    # If last download failed, use error to retrieve error message.
+    # Requires 'curl' to be available on PATH.
+    #
+    # === Parameters
+    # url(String):: URL to downloaded file
+    # dest(String):: Path where file should be saved on disk
+    # username(String):: Optional HTTP basic authentication username
+    # password(String):: Optional HTTP basic authentication password
+    # max_retries(Integer):: Maximum number of retries - defaults to DEFAULT_MAX_DOWNLOAD_RETRIES
+    #
+    # === Block
+    # Call (optional) passed in block after each unsuccessful download attempt.
+    # Block must accept one argument corresponding to the last returned http code.
+    #
+    # === Return
+    # true:: Download was successful
+    # false:: Download failed
+    def download(url, dest, username=nil, password=nil, max_retries=DEFAULT_MAX_DOWNLOAD_RETRIES)
+      @errors = []
+      retry_count = error_code = 0
+      success = false
+      reset_wait_time_span
+
+      @errors << 'curl is not installed' unless @found_curl
+
+      @errors << "destination file '#{dest}' is a directory" if File.directory?(dest)
+      begin
+        FileUtils.mkdir_p(File.dirname(dest)) unless File.directory?(File.dirname(dest))
+      rescue Exception => e
+        @errors << e.message
+      end
+
+      return false unless @errors.empty?
+
+      # format curl command and redirect stderr away.
+      #
+      # note: ensure we use double-quotes (") to surround arguments on command
+      # line because single-quotes (') are literals in windows.
+      platform = RightScale::Platform
+      user_opt = username && password ? "--user \"#{username}:#{password}\"" : ""
+      dest = platform.filesystem.long_path_to_short_path(dest)
+      cmd = "curl --fail --silent --show-error --insecure --location --connect-timeout 300 --max-time 3600 --write-out \"%{http_code} %{size_download} %{speed_download}\" #{user_opt} --output \"#{dest}\" \"#{url}\""
+      cmd = platform.shell.format_redirect_stderr(cmd)
+      begin
+        out = `#{cmd}`
+        out = out.split
+        success = $?.success? && out.size == 3
+        if success
+          @size = out[1].to_i
+          @speed = out[2].to_i
+          @last_url = url
+          return true
+        else
+          retry_count += 1
+          error_code = out[0].to_i
+          yield error_code if block_given?
+          sleep wait_time_span
+        end
+      end until success || retry_count >= max_retries
+      unless success
+        @errors << "#{retry_count} download attempts failed, last HTTP response code was #{error_code}"
+        return false
+      end
+      true
+    end
+
+    # Message summarizing last successful download details
+    #
+    # === Return
+    # details(String):: Message with last url, download size and speed
+    def details
+      "Downloaded #{sanitize_uri(@last_url)} (#{ scale(size.to_i).join(' ') }) at #{ scale(speed.to_i).join(' ') }/s"
+    end
+
+    protected
+
+    # Calculate wait time span before next download retry, takes into account retry period and whether backoff
+    # algorithm should be used
+    #
+    # === Return
+    # time_span(Integer):: Number of seconds algorithm should wait before proceeding with next download attempt
+    def wait_time_span
+      return @retry_period unless @use_backoff
+      time_span = [ 2 ** @iteration * @retry_period, @max_retry_period ].min
+      @iteration += 1
+      time_span
+    end
+
+    # Reset backoff algorithm
+    #
+    # === Return
+    # true:: Always return true
+    def reset_wait_time_span
+      @iteration = 0
+      true
+    end
+
+    # Return scale and scaled value from given argument
+    # Scale can be B, KB, MB or GB
+    #
+    # === Return
+    # scaled(Array):: First element is scaled value, second element is scale ('B', 'KB', 'MB' or 'GB')
+    def scale(value)
+      scaled = case value
+        when 0..1023
+          [value, 'B']
+        when 1024..1024**2 - 1
+          [value / 1024, 'KB']
+        when 1024^2..1024**3 - 1
+          [value / 1024**2, 'MB']
+        else
+          [value / 1024**3, 'GB']
+      end
+    end
+
+    def sanitize_uri(uri)
+      begin
+        uri = URI.parse(uri)
+        return "#{uri.scheme}://#{uri.host}#{uri.path}" 
+      rescue Exception => e
+        return "file"
+      end
+    end
+  end
+end

data/lib/instance/duplicable.rb

@@ -0,0 +1,67 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# Code taken from RAILS which allows checking whether '.dup' can be called
+# on an arbitrary object
+class Object
+  # Can you safely .dup this object?
+  # False for nil, false, true, symbols, numbers, and class objects; true otherwise.
+  def duplicable?
+    true
+  end
+end
+
+class NilClass #:nodoc:
+  def duplicable?
+    false
+  end
+end
+
+class FalseClass #:nodoc:
+  def duplicable?
+    false
+  end
+end
+
+class TrueClass #:nodoc:
+  def duplicable?
+    false
+  end
+end
+
+class Symbol #:nodoc:
+  def duplicable?
+    false
+  end
+end
+
+class Numeric #:nodoc:
+  def duplicable?
+    false
+  end
+end
+
+class Class #:nodoc:
+  def duplicable?
+    false
+  end
+end

data/lib/instance/exceptions.rb

@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+begin
+  require 'chef/exceptions'
+rescue LoadError => e
+  # Make sure we're dealing with a legitimate missing-file LoadError
+  raise e unless e.message =~ /^no such file to load/
+
+  # Do not require the chef gem to be installed just to load this code
+  module Chef
+    class Exceptions
+      class Exec < RuntimeError; end
+    end
+  end
+end
+
+module RightScale
+  # Extend the set of RightScale extensions
+  class Exceptions
+    class WrongState < RuntimeError; end
+    class Exec < Chef::Exceptions::Exec
+      def initialize(msg, cwd=nil)
+        super(msg)
+        @path = cwd
+      end
+      attr_reader :path    # Path where command was run
+    end
+  end
+end

data/lib/instance/executable_sequence_proxy.rb

@@ -0,0 +1,278 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'right_popen'
+
+module RightScale
+
+  # Bundle sequence proxy, create child process to execute bundle
+  # Use right_popen gem to control child process asynchronously
+  class ExecutableSequenceProxy
+    DEFAULT_OPTIONS = {
+      :tag_query_timeout => 120
+    }
+
+    include EM::Deferrable
+
+    # Wait up to 20 seconds to process pending audits after child process exited
+    AUDIT_CLOSE_TIMEOUT = 20
+
+    # (Hash) Inputs patch to be forwarded to core after each converge
+    attr_accessor :inputs_patch
+
+    # (::RightScale::OperationContext) operation context containing bundle
+    attr_reader :context
+
+    # PID for created process or nil
+    attr_reader :pid
+
+    # Execution thread name or default.
+    attr_reader :thread_name
+
+    # Initialize sequence
+    #
+    # === Parameters
+    # context(RightScale::OperationContext):: Bundle to be run and associated audit
+    #
+    # === Options
+    # :pid_callback(Proc):: proc that will be called, passing self, when the PID of the child process becomes known
+    # :tag_query_timeout(Proc):: default 120 -- how many seconds to wait for the agent tag query to complete, before giving up and continuing
+    def initialize(context, options = {})
+      options = DEFAULT_OPTIONS.merge(options)
+      @context = context
+      @thread_name = get_thread_name_from_context(context)
+      @pid_callback = options[:pid_callback]
+      @tag_query_timeout = options[:tag_query_timeout]
+
+      AuditCookStub.instance.setup_audit_forwarding(@thread_name, context.audit)
+      AuditCookStub.instance.on_close(@thread_name) { @audit_closed = true; check_done }
+    end
+    
+    # FIX: thread_name should never be nil from the core in future, but
+    # temporarily we must supply the default thread_name before if nil. in
+    # future we should fail execution when thread_name is reliably present and
+    # for any reason does not match ::RightScale::AgentConfig.valid_thread_name
+    # see also ExecutableSequenceProxy#initialize
+    #
+    # === Parameters
+    # bundle(OperationalContext):: An operational context
+    #
+    # === Return
+    # result(String):: Thread name of this context
+    def get_thread_name_from_context(context) 
+      thread_name = nil
+      thread_name = context.thread_name if context.respond_to?(:thread_name)
+      Log.warn("Encountered a nil thread name unexpectedly, defaulting to '#{RightScale::AgentConfig.default_thread_name}'") unless thread_name
+      thread_name ||= RightScale::AgentConfig.default_thread_name
+      unless thread_name =~ RightScale::AgentConfig.valid_thread_name
+        raise ArgumentError, "Invalid thread name #{thread_name.inspect}"
+      end
+      thread_name
+    end
+
+    # Run given executable bundle
+    # Asynchronous, set deferrable object's disposition
+    #
+    # === Return
+    # true:: Always return true
+    def run
+      @succeeded = true
+
+      @context.audit.create_new_section('Querying tags')
+
+      # update CookState with the latest instance before launching Cook
+      RightScale::AgentTagManager.instance.tags(:timeout=>@tag_query_timeout) do |tags|
+        if tags.is_a?(String)
+          # AgentTagManager could give us a String (error message)
+          Log.error("Failed to query tags before running executable sequence: #{tags}")
+
+          @context.audit.append_error('Could not discover tags due to an error or timeout.')
+        else
+          # or, it could give us anything else -- generally an array) -- which indicates success
+          CookState.update(InstanceState, :startup_tags=>tags)
+
+          if tags.empty?
+            @context.audit.append_info('No tags discovered.')
+          else
+            @context.audit.append_info("Tags discovered: '#{tags.join("', '")}'")
+          end
+        end
+
+        input_text = "#{MessageEncoder.for_agent(InstanceState.identity).encode(@context.payload)}\n"
+
+        # TEAL FIX: we have an issue with the Windows EM implementation not
+        # allowing both sockets and named pipes to share the same file/socket
+        # id. sending the input on the command line is a temporary workaround.
+        platform = RightScale::Platform
+        if platform.windows?
+          input_path = File.normalize_path(File.join(platform.filesystem.temp_dir, "rs_executable_sequence#{@thread_name}.txt"))
+          File.open(input_path, "w") { |f| f.write(input_text) }
+          input_text = nil
+          cmd_exe_path = File.normalize_path(ENV['ComSpec']).gsub("/", "\\")
+          ruby_exe_path = File.normalize_path(AgentConfig.ruby_cmd).gsub("/", "\\")
+          input_path = input_path.gsub("/", "\\")
+          cmd = "#{cmd_exe_path} /C type \"#{input_path}\" | #{ruby_exe_path} #{cook_path_and_arguments}"
+        else
+          # WARNING - always ensure cmd is a String, never an Array of command parts.
+          #
+          # right_popen handles single-String arguments using "sh -c #{cmd}" which ensures
+          # we are invoked through a shell which will parse shell config files and ensure that
+          # changes to system PATH, etc are freshened on every converge.
+          #
+          # If we pass cmd as an Array, right_popen uses the Array form of exec without an
+          # intermediate shell, and system config changes will not be picked up.
+          cmd = "#{AgentConfig.ruby_cmd} #{cook_path_and_arguments}"
+        end
+
+        EM.next_tick do
+          # prepare env vars for child process.
+          environment = {
+            ::RightScale::OptionsBag::OPTIONS_ENV =>
+              ::ENV[::RightScale::OptionsBag::OPTIONS_ENV]
+          }
+          if @context.decommission?
+            environment['RS_DECOM_REASON'] = @context.decommission_type
+          end
+
+          # spawn
+          RightScale::RightPopen.popen3_async(
+            cmd,
+            :input          => input_text,
+            :target         => self,
+            :environment    => environment,
+            :stdout_handler => :on_read_stdout,
+            :stderr_handler => :on_read_stderr,
+            :pid_handler    => :on_pid,
+            :exit_handler   => :on_exit)
+        end
+      end
+    end
+
+    protected
+
+    # Path to 'cook_runner' ruby script
+    #
+    # === Return
+    # path(String):: Path to ruby script used to run Chef
+    def cook_path
+      relative_path = File.join(File.dirname(__FILE__), '..', '..', 'bin', 'cook_runner')
+      return File.normalize_path(relative_path)
+    end
+
+    # Command line fragment for the cook script path and any arguments.
+    #
+    # === Return
+    # path_and_arguments(String):: Cook path plus any arguments properly quoted.
+    def cook_path_and_arguments
+      return "\"#{cook_path}\""
+    end
+
+    # Handle cook standard output, should not get called
+    #
+    # === Parameters
+    # data(String):: Standard output content
+    #
+    # === Return
+    # true:: Always return true
+    def on_read_stdout(data)
+      Log.error("Unexpected output from execution: #{data.inspect}")
+    end
+
+    # Handle cook error output
+    #
+    # === Parameters
+    # data(String):: Error output content
+    #
+    # === Return
+    # true:: Always return true
+    def on_read_stderr(data)
+      @context.audit.append_info(data)
+    end
+
+    # Receives the pid for the created process.
+    def on_pid(pid)
+      @pid = pid
+      @pid_callback.call(self) if @pid_callback
+    end
+
+    # Handle runner process exited event
+    #
+    # === Parameters
+    # status(Process::Status):: Exit status
+    #
+    # === Return
+    # true:: Always return true
+    def on_exit(status)
+      @exit_status = status
+      check_done
+    end
+
+    # Check whether child process exited *and* all audits were processed
+    # Do not proceed until both these conditions are true
+    # If the child process exited start a timer so that if there was a failure
+    # and the child process was not able to properly close the auditing we will
+    # still proceed and be able to handle other scripts/recipes
+    #
+    # Note: success and failure reports are handled by the cook process for normal
+    # scenarios. We only handle cook process execution failures here.
+    #
+    # === Return
+    # true:: Always return true
+    def check_done
+      if @exit_status && @audit_closed
+        if @audit_close_timeout
+          @audit_close_timeout.cancel
+          @audit_close_timeout = nil
+        end
+        if !@exit_status.success?
+          RightScale::PolicyManager.fail(@context.payload)
+          report_failure("Subprocess #{SubprocessFormatting.reason(@exit_status)}")
+        else
+          @context.succeeded = true
+          RightScale::PolicyManager.success(@context.payload)
+          succeed
+        end
+      elsif @exit_status
+        @audit_close_timeout = EM::Timer.new(AUDIT_CLOSE_TIMEOUT) { AuditCookStub.instance.close(@thread_name) }
+      end
+      true
+    end
+
+    # Report cook process execution failure
+    #
+    # === Parameters
+    # title(String):: Title used to update audit status
+    # msg(String):: Optional, extended failure message
+    #
+    # === Return
+    # true:: Always return true
+    def report_failure(title, msg=nil)
+      @context.audit.append_error(title, :category => RightScale::EventCategories::CATEGORY_ERROR)
+      @context.audit.append_error(msg) unless msg.nil?
+      @context.succeeded = false
+      fail
+      true
+    end
+
+  end
+
+end