right_link 5.9.0

data/lib/instance/cook/cook_state.rb

@@ -0,0 +1,298 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Manages and persists cook process state
+  class CookState
+    # Dev tags namespace
+    DEV_TAG_NAMESPACE = 'rs_agent_dev:'
+
+    # Cookbook path dev tag namespace and prefix
+    COOKBOOK_PATH_TAG = "#{DEV_TAG_NAMESPACE}cookbooks_path"
+
+    # Breakpoint dev tag namespace and prefix
+    BREAKPOINT_TAG    = "#{DEV_TAG_NAMESPACE}break_point"
+
+    # Download once dev tag namespace and prefix
+    DOWNLOAD_ONCE_TAG = "#{DEV_TAG_NAMESPACE}download_cookbooks_once"
+
+    # Log level dev tag namespace and prefix
+    LOG_LEVEL_TAG     = "#{DEV_TAG_NAMESPACE}log_level"
+
+    # Path to JSON file where dev state is serialized
+    STATE_DIR         = RightScale::AgentConfig.agent_state_dir
+    STATE_FILE        = File.join(STATE_DIR, 'cook_state.js')
+
+    class << self
+      # Reset class state and load persisted state if any
+      #
+      # === Return
+      # true:: Always return true
+      def init(reset=false)
+        if @state.nil? || reset
+          @state = CookState.new
+        end
+        true
+      end
+
+      private
+      # not sure if method_missing is the best way to go to expose
+      # our state interface as class methods.  If this is not the best
+      # approach, then we should look for another...
+      def method_missing(method, *args, &block)
+        # ensure the state is initialized before calling any methods
+        init unless initialized?
+        if @state.respond_to?(method)
+          @state.send(method, *args, &block)
+        else
+          super
+        end
+      end
+
+      # Was cook state initialized?
+      #
+      # === Return
+      # true:: if logger has been initialized
+      # false:: Otherwise
+      def initialized?
+        defined?(@state) && @state
+      end
+    end
+
+    # Reset class state and load persisted state if any
+    #
+    # === Return
+    # true:: Always return true
+    def initialize
+      # set some defaults
+      @has_downloaded_cookbooks = false
+      @reboot                   = false
+      @startup_tags             = []
+      @log_level                = Logger::INFO
+      @log_file                 = nil
+
+      # replace defaults with state on disk
+      load_state
+
+      true
+    end
+
+    attr_reader :log_level, :log_file, :startup_tags
+    attr_writer :has_downloaded_cookbooks
+
+    def has_downloaded_cookbooks?
+      !!@has_downloaded_cookbooks
+    end
+
+    # Are we rebooting? (needed for RightScripts)
+    #
+    # === Return
+    # res(Boolean):: Whether this instance was rebooted
+    def reboot?
+      !!@reboot
+    end
+
+    # Is the instance running in dev mode?
+    # dev mode tweaks the behavior of the RightLink agent to help
+    # the development of Chef recipes.
+    # In dev mode, the log level is always debug.
+    #
+    # === Return
+    # true:: If dev tags are defined on this instance
+    # false:: Otherwise
+    def dev_mode_enabled?
+      !!tag_value(DEV_TAG_NAMESPACE)
+    end
+
+    # Determines the developer log level, if any, which forces and supercedes
+    # all other log level configurations.
+    #
+    # === Return
+    # level(Token):: developer log level or nil
+    def dev_log_level
+      if value = tag_value(LOG_LEVEL_TAG)
+        value = value.downcase.to_sym
+        value = nil unless [:debug, :info, :warn, :error, :fatal].include?(value)
+      end
+      value
+    end
+
+    # Path to cookbooks repos directory. Cookbooks are downloaded to
+    # this location if and only if it doesn't exist.
+    #
+    # === Return
+    # path(Array):: Dev cookbooks repositories path
+    # nil:: Use default cookbook download algorithm
+    def cookbooks_path
+      path = tag_value(COOKBOOK_PATH_TAG)
+      path.split(/, */) if path
+    end
+
+    # Name of first recipe in run list that should not be run
+    #
+    # === Return
+    # recipe(String):: Name of recipe to break execution of sequence on
+    def breakpoint
+      recipe = tag_value(BREAKPOINT_TAG)
+    end
+
+    # Whether dev cookbooks path should be used instead of standard
+    # cookbooks repositories location
+    # True if in dev mode and all dev cookbooks repos directories are not empty
+    #
+    # === Return
+    # true:: If dev cookbooks repositories path should be used
+    # false:: Otherwise
+    def use_cookbooks_path?
+      res = !!(paths = cookbooks_path)
+      return false unless res
+      paths.each do |path|
+        res = path && File.directory?(path) && Dir.entries(path) != ['.', '..']
+        break unless res
+      end
+      res
+    end
+
+    # Whether cookbooks should be downloaded
+    # False if either a dev cookbooks path is used or the download once
+    # flag is set and cookbooks have already been downloaded
+    # Note: we know a cookbook is downloaded when this method is called.
+    #       Make sure this stays true.
+    #
+    # === Return
+    # true:: If cookbooks should be downloaded
+    # false:: Otherwise
+    def download_cookbooks?
+      # always download unless machine is tagged with a valid cookbooks path or
+      # machine is tagged with download once and cookbooks have already been downloaded.
+      res = !(use_cookbooks_path? || (has_downloaded_cookbooks? && download_once?))
+    end
+
+    # Whether cookbooks should be downloaded only once
+    #
+    # === Return
+    # true:: If cookbooks should be downloaded only once for this instance
+    # false:: Otherwise
+    def download_once?
+      tag_value(DOWNLOAD_ONCE_TAG) == 'true'
+    end
+
+    # Current logger severity
+    #
+    # === Return
+    # level(Integer):: one of Logger::INFO ... Logger::FATAL
+    def log_level
+      @log_level
+    end
+
+    # Re-initialize then merge given state
+    #
+    # === Parameters
+    # state_to_merge(RightScale::InstanceState):: InstanceState to be passed on to Cook
+    # overrides(Hash):: Hash keyed by state name that will override state_to_merge
+    #
+    # === Return
+    # true:: Always
+    def update(state_to_merge, overrides = {})
+      # only merge state if state to be merged has values
+      @startup_tags = state_to_merge.startup_tags if state_to_merge.respond_to?(:startup_tags)
+      @reboot       = state_to_merge.reboot?      if state_to_merge.respond_to?(:reboot?)
+      @log_level    = state_to_merge.log_level    if state_to_merge.respond_to?(:log_level)
+      if state_to_merge.respond_to?(:log_file) && state_to_merge.respond_to?(:value)
+        @log_file = state_to_merge.log_file(state_to_merge.value)
+      end
+
+      @startup_tags = overrides[:startup_tags] if overrides.has_key?(:startup_tags)
+      @reboot       = overrides[:reboot]       if overrides.has_key?(:reboot)
+      @log_file     = overrides[:log_file]     if overrides.has_key?(:log_file)
+
+      # check the log level again after the startup_tags have been updated or
+      # overridden.
+      if overrides.has_key?(:log_level)
+        @log_level = overrides[:log_level]
+      elsif tagged_log_level = dev_log_level
+        @log_level = tagged_log_level
+      end
+
+      save_state
+
+      true
+    end
+
+    protected
+
+    # Extract tag value for tag with given namespace and prefix
+    #
+    # === Parameters
+    # prefix(String):: Tag namespace and prefix
+    #
+    # === Return
+    # value(String):: Corresponding tag value
+    def tag_value(prefix)
+      tag = @startup_tags.detect { |t| t =~ /^#{prefix}/ }
+      value = tag[prefix.size + 1..-1] if tag
+    end
+
+    # Save dev state to file
+    #
+    # === Return
+    # true:: Always return true
+    def save_state
+      # start will al state to be saved
+      state_to_save = { 'startup_tags' => startup_tags,
+                      'reboot' => reboot?,
+                      'log_level' => log_level }
+
+      # only save a log file one is defined
+      if log_file
+        state_to_save['log_file'] = log_file
+      end
+
+      # only save persist the fact we downloaded cookbooks if we are in dev mode
+      if download_once?
+        state_to_save['has_downloaded_cookbooks'] = has_downloaded_cookbooks?
+      end
+
+      RightScale::JsonUtilities::write_json(RightScale::CookState::STATE_FILE, state_to_save)
+      true
+    end
+
+    # load dev state from disk
+    #
+    # === Return
+    # true:: Always return true
+    def load_state
+      if File.file?(STATE_FILE)
+        state = RightScale::JsonUtilities::read_json(STATE_FILE)
+        @log_level = state['log_level'] || Logger::INFO
+        Log.info("Initializing CookState from  #{STATE_FILE} with #{state.inspect}") if @log_level == Logger::DEBUG
+
+        @has_downloaded_cookbooks = state['has_downloaded_cookbooks']
+        @startup_tags = state['startup_tags'] || []
+        @reboot = state['reboot']
+        @log_file = state['log_file'] # nil if not in state loaded from disk
+      end
+      true
+    end
+  end
+end

data/lib/instance/cook/cookbook_path_mapping.rb

@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+  # helpers for constructing various paths related to coookbooks
+  class CookbookPathMapping
+    # Constructs a repose download path for a given cookbook
+    #
+    # === Parameters
+    # repose_root (String):: the root for all cookbooks downloaded with repose
+    # repo_sha (String):: unique identifier for the repo containig the cookbook
+    # position (String):: the relative path of the cookbook within the repo
+    #
+    # === Return
+    #  (String):: full path to the repose download location for the given cookbook
+    def self.repose_path(repose_root, repo_sha, position)
+      build_path(repose_root, repo_sha, position)
+    end
+
+    # Constructs a cookbook checkout path for a given cookbook
+    #
+    # === Parameters
+    # checkout_root (String):: the root of all checked out cookbooks
+    # repo_dir (String):: root directory for all cookbooks in a the repo
+    # position (String):: the relative path of the cookbook within the repo
+    #
+    # === Return
+    #  (String):: full path to the checked out location for the given cookbook
+    def self.checkout_path(repo_dir, position)
+      build_path(repo_dir, position)
+    end
+
+    private
+    # Combines the given path segments removing un needed separators and missing intermediate
+    #
+    # === Parameters
+    # root (String):: the root of the path
+    # sub (String):: middle section of the path
+    # leaf (String):: end of the path
+    #
+    # === Return
+    #  (String):: cleaned path, empty string is all params are nil
+    def self.build_path(root="", sub="", leaf="")
+      File.join(*([root || ""] + "#{sub || ""}".split('/') + "#{leaf || ""}".split('/')))
+    end
+  end
+end

data/lib/instance/cook/cookbook_repo_retriever.rb

@@ -0,0 +1,190 @@
+#
+# Copyright (c) 2009-2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'right_scraper'
+
+module RightScale
+  #
+  class CookbookRepoRetriever
+    attr_reader :checkout_root
+
+    # Maps the given DevRepository to a has that can be consumed by the RightScraper gem
+    # TODO: May make sense for  this to a member of the DevRepository class as an instance method.
+    #           Also, may want to take a version as a parameter in the future (or not...).
+    #
+    # === Parameters
+    # dev_repo (DevRepository):: core representation of a resource repository
+    #
+    # === Returns
+    # (Hash)::
+    #   :repo_type (Symbol):: Type of repository: one of :git, :svn, :download or :local
+    #     * :git denotes a 'git' repository that should be retrieved via 'git clone'
+    #     * :svn denotes a 'svn' repository that should be retrieved via 'svn checkout'
+    #     * :download denotes a tar ball that should be retrieved via HTTP GET (HTTPS if uri starts with https://)
+    #     * :local denotes cookbook that is already local and doesn't need to be retrieved
+    #   :url (String):: URL to repository (e.g. git://github.com/opscode/chef-repo.git)
+    #   :tag (String):: git commit or svn branch that should be used to retrieve repository
+    #                       Optional, use 'master' for git and 'trunk' for svn if tag is nil.
+    #                       Not used for raw repositories.
+    #   :cookbooks_path (Array):: Path to cookbooks inside repostory
+    #                                             Optional (use location of repository as cookbook path if nil)
+    #   :first_credential (String):: Either the Private SSH key used to retrieve git repositories, or the Username used to retrieve svn and raw repositories
+    #   :second_credential (String):: Password used to retrieve svn and raw repositories
+    def self.to_scraper_hash(dev_repo)
+      repo = {}
+      repo[:repo_type]            = dev_repo.repo_type.to_sym unless dev_repo.repo_type.nil?
+      repo[:url]                  = dev_repo.url
+      repo[:tag]                  = dev_repo.tag
+      repo[:resources_path]       = dev_repo.cookbooks_path
+      if !dev_repo.ssh_key.nil?
+        repo[:first_credential]   = dev_repo.ssh_key
+      elsif !(dev_repo.username.nil? && dev_repo.password.nil?)
+        repo[:first_credential]   = dev_repo.username
+        repo[:second_credential]  = dev_repo.password
+      end
+
+      repo
+    end
+
+    # Initialize...
+    #
+    # === Parameters
+    # repose_root (String):: root of all repose downloaded cookbooks
+    # dev_cookbooks (Hash):: collection of repos to be checked out see  RightScale::DevRepositories for hash content
+    def initialize(repose_root, dev_cookbooks)
+      @checkout_root  = AgentConfig.dev_cookbook_checkout_dir
+      @repose_root    = repose_root
+      @dev_cookbooks  = (dev_cookbooks.nil? || dev_cookbooks.repositories.nil?) ? {} : dev_cookbooks.repositories
+      @scraper        = RightScraper::Scraper.new(:kind => :cookbook, :basedir => @checkout_root)
+      @registered_checkouts = {}
+    end
+
+    # Are there any cookbooks to be checked out?
+    #
+    # === Returns
+    # true if there are cookbooks to be checked out, false otherwise
+    def has_cookbooks?
+      !@dev_cookbooks.empty?
+    end
+
+    # Should there be a link created for this cookbook?
+    #
+    # === Parameters
+    # repo_sha (String) :: unique identifier of the cookbook repository
+    # position (String) :: repo relative ppath of the cookbook
+    #
+    # === Returns
+    # true if there is a cookbook in the given repo that should be checekd out
+    def should_be_linked?(repo_sha, position)
+      @dev_cookbooks.has_key?(repo_sha) &&
+      @dev_cookbooks[repo_sha].positions &&
+      @dev_cookbooks[repo_sha].positions.detect { |dev_position| dev_position.position == position }
+    end
+
+    # Was the cookbook checked out?
+    #
+    # === Parameters
+    # repo_sha (String) :: unique identifier of the cookbook repository
+    #
+    # === Returns
+    # true if the given repo is checekd out
+    def is_checked_out?(repo_sha)
+      !!repo_dir_for(repo_sha)
+    end
+
+    # Checkout directory for given repository hash.
+    #
+    # === Parameters
+    # @param [String] repo_sha for lookup
+    #
+    # === Return
+    # @return [String] checked-out directory or nil
+    def repo_dir_for(repo_sha)
+      @registered_checkouts[repo_sha]
+    end
+
+    # Checkout the given repo and link each dev cookbook to it's matching repose path
+    #
+    # === Parameters
+    # callback (Proc) :: to be called for each repo checked out see RightScraper::Scraper.scrape for details
+    #
+    # === Return
+    #  true
+    def checkout_cookbook_repos(&callback)
+      @dev_cookbooks.each_pair do |repo_sha, dev_repo|
+        repo = self.class.to_scraper_hash(dev_repo)
+
+        # get the root dir this repo should be, or was, checked out to
+        repo_dir = @scraper.repo_dir(repo)
+
+        if File.directory?(repo_dir)
+          # repo was already checked out on this machine; leave it alone
+          # synthesize a scraper callback so our progress listener knows what's up
+          if callback
+            callback.call(:commit, :initialize, "Skipping checkout -- repository already exists in #{repo_dir}", nil)
+          end
+          @registered_checkouts[repo_sha] = repo_dir
+        else
+          # repo wasn't checked out successfully yet; check it out now
+          success = false
+          begin
+            success = @scraper.scrape(repo, &callback)
+          ensure
+            if success
+              @registered_checkouts[repo_sha] = repo_dir
+            else
+              # nuke the repo dir if checkout fails, so we try again next time
+              FileUtils.rm_rf(repo_dir) unless success
+
+              # scraper logger is an odd duck, so just transfer any errors to
+              # the normal logger.
+              @scraper.errors.each { |e| ::RightScale::Log.error(e) }
+              ::RightScale::Log.error("Failed to checkout from #{repo[:url].inspect}")
+            end
+          end
+        end
+      end
+      true
+    end
+
+    # Creates a symlink from the checked out cookbook to the expected repose download location
+    #
+    # === Parameters
+    # repo_sha (String) :: unique identifier of the cookbook repository
+    # position (String) :: repo relative ppath of the cookbook
+    #
+    # === Returns
+    # true if link was created, false otherwise
+    def link(repo_sha, position)
+      # symlink to the checked out cookbook only if it was actually checked out
+      if repo_dir = @registered_checkouts[repo_sha]
+        checkout_path = CookbookPathMapping.checkout_path(repo_dir, position)
+        raise ArgumentError.new("Missing directory cannot be linked: #{checkout_path}") unless File.directory?(checkout_path)
+        repose_path   = CookbookPathMapping.repose_path(@repose_root, repo_sha, position)
+        FileUtils.mkdir_p(File.dirname(repose_path))
+        Platform.filesystem.create_symlink(checkout_path, repose_path)
+        return true
+      end
+      false
+    end
+  end
+end