chef-cli 1.0.3

data/lib/chef-cli/policyfile/git_lock_fetcher.rb

@@ -0,0 +1,265 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "../policyfile_lock"
+require_relative "../exceptions"
+require_relative "../helpers"
+require "mixlib/shellout" unless defined?(Mixlib::ShellOut)
+require "tmpdir" unless defined?(Dir.mktmpdir)
+
+module ChefCLI
+  module Policyfile
+
+    # A Policyfile lock fetcher that can read a lock file from a git repository.
+    #
+    # @author Ryan Hass
+    # @author Daniel DeLeo
+    #
+    # @since 3.0
+    #
+    class GitLockFetcher
+      attr_accessor :name
+      attr_accessor :source_options
+      attr_accessor :storage_config
+
+      attr_reader :uri
+      attr_reader :revision
+      attr_reader :path
+      attr_reader :branch
+      attr_reader :tag
+      attr_reader :ref
+
+      # Initialize a GitLockFetcher
+      #
+      # @param name [String] The name of the policyfile
+      # @param source_options [Hash] A hash with a :path key pointing at the location
+      #                              of the lock
+      def initialize(name, source_options, storage_config)
+        @name           = name
+        @storage_config = storage_config
+        @source_options = symbolize_keys(source_options)
+        @revision = @source_options[:revision]
+        @path     = @source_options[:path] || @source_options[:rel]
+        @uri      = @source_options[:git]
+        @branch   = @source_options[:branch]
+        @tag      = @source_options[:tag]
+        @ref      = @source_options[:ref]
+
+        # The revision to parse
+        @rev_parse = @source_options[:ref] || @source_options[:branch] || @source_options[:tag] || "master"
+      end
+
+      # @return [True] if there were no errors with the provided source_options
+      # @return [False] if there were errors with the provided source_options
+      def valid?
+        errors.empty?
+      end
+
+      # Check the options provided when craeting this class for errors
+      #
+      # @return [Array<String>] A list of errors found
+      def errors
+        error_messages = []
+        [:git].each do |key|
+          error_messages << "include_policy for #{name} is missing key #{key}" unless source_options[key]
+        end
+
+        error_messages
+      end
+
+      # @return [Hash] The source_options that describe how to fetch this exact lock again
+      def source_options_for_lock
+        source_options.merge({
+                               revision: revision,
+                             })
+      end
+
+      # Applies source options from a lock file. This is used to make sure that the same
+      # policyfile lock is loaded that was locked
+      #
+      # @param options_from_lock [Hash] The source options loaded from a policyfile lock
+      def apply_locked_source_options(options_from_lock)
+        options = options_from_lock.inject({}) do |acc, (key, value)|
+          acc[key.to_sym] = value
+          acc
+        end
+        source_options.merge!(options)
+        raise ChefCLI::InvalidLockfile, "Invalid source_options provided from lock data: #{options_from_lock_file.inspect}" if !valid?
+      end
+
+      # @return [Hash] of the policyfile lock data
+      def lock_data
+        @lock_data ||= fetch_lock_data.tap do |data|
+          data["cookbook_locks"].each do |cookbook_name, cookbook_lock|
+            if cookbook_lock["source_options"].key?("path")
+              cookbook_lock["source_options"].tap do |opt|
+                opt["git"]      = uri unless opt.key?("git")
+                opt["revision"] = revision unless opt.key?("revision")
+                opt["branch"]   = branch unless opt.key?("branch") || branch.nil?
+                opt["tag"]      = tag unless opt.key?("tag") || branch.nil?
+                opt["ref"]      = ref unless opt.key?("ref") || ref.nil?
+
+                path_keys = %w{path rel}.map { |path_key| path_key if opt.key?(path_key) }.compact
+
+                path_keys.each do |name|
+                  # We can safely grab the entire cookbook when the Policyfile defines a cookbook path of itself (".")
+                  if opt[name] == "."
+                    opt.delete(name)
+                    next
+                  end
+
+                  # Mutate the path key to a rel key so that we identify the source_type
+                  # as a git repo and not a local directory. Git also doesn't like paths
+                  # prefixed with `./` and cannot use relative paths outside the repo.
+                  # http://rubular.com/r/JYpdYHT19p
+                  pattern = /(^..\/)|(^.\/)/
+                  opt["rel"] = opt[name].gsub(pattern, "")
+                end
+
+                # Delete the path key if present to ensure we use the git source_type
+                opt.delete("path")
+              end
+            end # cookbook_lock["source_options"]
+          end # data["cookbook_locks"].each
+        end # fetch_lock_data.tap
+
+        @lock_data
+      end
+
+      private
+
+      # Helper method to normalize data.
+      #
+      # @param [Hash] hash Hash with symbols and/or strings as keys.
+      # @return [Hash] Hash with only symbols as keys.
+      def symbolize_keys(hash)
+        hash.inject({}) { |memo, (k, v)| memo[k.to_sym] = v; memo }
+      end
+
+      def fetch_lock_data
+        install unless installed?
+        FFI_Yajl::Parser.parse(
+          show_file(rev_parse, lockfile_path)
+        )
+      end
+
+      # COPYPASTA from CookbookOmnifetch
+      def installed?
+        !!(revision && cache_path.exist?)
+      end
+
+      # COPYPASTA from CookbookOmnifetch::GitLocation and Berkshelf::GitLocation
+      # then munged since we do not have Policyfile validation in scope.
+      # Install into the chefcli cookbook store. This method leverages a cached
+      # git copy.
+      def install
+        if cached?
+          Dir.chdir(cache_path) do
+            git %{fetch --force --tags #{uri} "refs/heads/*:refs/heads/*"}
+          end
+        else
+          git %{clone #{uri} "#{cache_path}" --bare --no-hardlinks}
+        end
+
+        Dir.chdir(cache_path) do
+          @revision ||= git %{rev-parse #{rev_parse}}
+        end
+      end
+
+      def rev_parse
+        source_options[:revision] || @rev_parse
+      end
+
+      # Shows contents of a file from a shallow or full clone repository for a
+      # given git version.
+      #
+      # This method was originally made before I slammed a bunch of copypasta
+      # code in which is generally more tied to a specific git ref.
+      #
+      # @param version Git version as a tag, branch, or ref.
+      # @param file Full path to file including filename in repository
+      #
+      # @return [String] Content of specified file for a given revision.
+      def show_file(version, file)
+        git("show #{version}:#{file}", cwd: cache_path)
+      end
+
+      # COPYPASTA from CookbookOmnifetch
+      # Location an executable in the current user's $PATH
+      #
+      # @return [String, nil]
+      #   the path to the executable, or +nil+ if not present
+      def which(executable)
+        if File.file?(executable) && File.executable?(executable)
+          executable
+        elsif ENV["PATH"]
+          path = ENV["PATH"].split(File::PATH_SEPARATOR).find do |p|
+            File.executable?(File.join(p, executable))
+          end
+          path && File.expand_path(executable, path)
+        end
+      end
+
+      # COPYPASTA from CookbookOmnifetch::Git
+      # Perform a git command.
+      #
+      # @param [String] command
+      #   the command to run
+      # @param [Boolean] error
+      #   whether to raise error if the command fails
+      #
+      # @raise [String]
+      #   the +$stdout+ from the command
+      def git(command, options = {})
+        error = options[:error] || true
+        unless which("git") || which("git.exe") || which("git.bat")
+          raise GitNotInstalled
+        end
+
+        response = Mixlib::ShellOut.new(%{git #{command}}, options)
+        response.run_command
+
+        if error && response.error?
+          raise GitError.new "#{command} #{cache_path}: #{response.stderr}"
+        end
+
+        response.stdout.strip
+      end
+
+      # COPYPASTA from CookbookOmnifetch::Git (then munged by me)
+      # The path where this git repository is cached.
+      #
+      # @return [Pathname]
+      def cache_path
+        Pathname.new(File.expand_path(File.join(ChefCLI::Helpers.package_home, "cache")))
+          .join(".cache", "git", Digest::SHA1.hexdigest(uri))
+      end
+
+      # COPYPASTA from CookbookOmnifetch::Git
+      # Determine if this git repo has already been downloaded.
+      #
+      # @return [Boolean]
+      def cached?
+        cache_path.exist?
+      end
+
+      def lockfile_path
+        @path.nil? ? "Policyfile.lock.json" : @path
+      end
+    end
+  end
+end

data/lib/chef-cli/policyfile/included_policies_cookbook_source.rb

@@ -0,0 +1,156 @@
+#
+# Copyright:: Copyright (c) 2014-2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "../ui"
+
+module ChefCLI
+  module Policyfile
+    class IncludedPoliciesCookbookSource
+
+      class ConflictingCookbookVersions < StandardError
+      end
+
+      class ConflictingCookbookDependencies < StandardError
+      end
+
+      class ConflictingCookbookSources < StandardError
+      end
+
+      # Why do we need this class?
+      # If we rely on default sources, we may not have the the universe of cookbooks
+      # provided in the included policies
+      #
+      # This is not meant to be used from the DSL
+
+      # A list of included policies
+      attr_reader :included_policy_location_specs
+      # UI object for output
+      attr_accessor :ui
+
+      # Constructor
+      #
+      def initialize(included_policy_location_specs)
+        @included_policy_location_specs = included_policy_location_specs
+        @ui = UI.new
+        @preferred_cookbooks = []
+        yield self if block_given?
+      end
+
+      def default_source_args
+        [:included_policies, []]
+      end
+
+      def check_for_conflicts!
+        source_options
+        universe_graph
+      end
+
+      # All are preferred here
+      def preferred_source_for?(cookbook_name)
+        universe_graph.include?(cookbook_name)
+      end
+
+      def preferred_cookbooks
+        universe_graph.keys
+      end
+
+      def ==(other)
+        other.kind_of?(self.class) && other.included_policy_location_specs == included_policy_location_specs
+      end
+
+      # Calls the slurp_metadata! helper once to calculate the @universe_graph
+      # and @cookbook_version_paths metadata.  Returns the @universe_graph.
+      #
+      # @return [Hash] universe_graph
+      def universe_graph
+        @universe_graph ||= build_universe
+      end
+
+      # Returns the metadata (path and version) for an individual cookbook
+      #
+      # @return [Hash] metadata for a single cookbook version
+      def source_options_for(cookbook_name, cookbook_version)
+        source_options[cookbook_name][cookbook_version]
+      end
+
+      def null?
+        false
+      end
+
+      def desc
+        "included_policies()"
+      end
+
+      private
+
+      def build_universe
+        included_policy_location_specs.inject({}) do |acc, policy_spec|
+          lock = policy_spec.policyfile_lock
+          cookbook_dependencies = lock.solution_dependencies.cookbook_dependencies
+          cookbook_dependencies.each do |(cookbook, deps)|
+            name = cookbook.name
+            version = cookbook.version
+            mapped_deps = deps.map do |dep|
+              [dep[0], dep[1].to_s]
+            end
+            if acc[name]
+              if acc[name][version]
+                if acc[name][version] != mapped_deps
+                  raise ConflictingCookbookDependencies.new("Conflicting dependencies provided for cookbook #{name}")
+                end
+              else
+                raise ConflictingCookbookVersions.new("Multiple versions provided for cookbook #{name}")
+              end
+            else
+              acc[name] = {}
+              acc[name][version] = mapped_deps
+            end
+          end
+          acc
+        end
+      end
+
+      def source_options
+        @source_options ||= build_source_options
+      end
+
+      ## Collect all the source options
+      def build_source_options
+        included_policy_location_specs.inject({}) do |acc, policy_spec|
+          lock = policy_spec.policyfile_lock
+          lock.cookbook_locks.each do |(name, cookbook_lock)|
+            version = cookbook_lock.version
+            if acc[name]
+              if acc[name][version]
+                if acc[name][version] != cookbook_lock.source_options
+                  raise ConflictingCookbookSources.new("Conflicting sources provided for cookbook #{name}")
+                end
+              else
+                raise ConflictingCookbookVersions.new("Multiple sources provided for cookbook #{name}")
+              end
+            else
+              acc[name] = {}
+              acc[name][version] = cookbook_lock.source_options
+            end
+          end
+          acc
+        end
+      end
+
+    end
+  end
+end

data/lib/chef-cli/policyfile/lister.rb

@@ -0,0 +1,229 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "set" unless defined?(Set)
+
+require "chef/server_api"
+require_relative "../service_exceptions"
+
+module ChefCLI
+  module Policyfile
+
+    class RevIDLockDataMap
+
+      attr_reader :policy_name
+      attr_reader :lock_info_by_rev_id
+
+      def initialize(policy_name, lock_info_by_rev_id)
+        @policy_name = policy_name
+        @lock_info_by_rev_id = lock_info_by_rev_id
+      end
+
+      def cb_info_for(rev_id, cookbook_name)
+        lock = lock_info_by_rev_id[rev_id]
+        cookbook_lock = lock["cookbook_locks"][cookbook_name]
+
+        if cookbook_lock
+          [cookbook_lock["version"], cookbook_lock["identifier"] ]
+        else
+          nil
+        end
+      end
+
+      def cbs_with_differing_ids
+        cbs_with_differing_ids = Set.new
+        all_cookbook_names.each do |cookbook_name|
+          all_identifiers = lock_info_by_rev_id.inject(Set.new) do |id_set, (_rev_id, rev_info)|
+            cookbook_lock = rev_info["cookbook_locks"][cookbook_name]
+            identifier = cookbook_lock && cookbook_lock["identifier"]
+            id_set << identifier
+          end
+          cbs_with_differing_ids << cookbook_name if all_identifiers.size > 1
+        end
+        cbs_with_differing_ids
+      end
+
+      def all_cookbook_names
+        lock_info_by_rev_id.inject(Set.new) do |cb_set, (_rev_id, rev_info)|
+          cb_set.merge(rev_info["cookbook_locks"].keys)
+        end
+      end
+    end
+
+    class PolicyGroupRevIDMap
+
+      include Enumerable
+
+      attr_reader :policy_name
+      attr_reader :revision_ids_by_group
+
+      def initialize(policy_name, revision_ids_by_group)
+        @policy_name = policy_name
+        @revision_ids_by_group = revision_ids_by_group
+      end
+
+      def unique_revision_ids
+        revision_ids_by_group.values.uniq
+      end
+
+      def policy_group_names
+        revision_ids_by_group.keys
+      end
+
+      def max_group_name_length
+        policy_group_names.map(&:size).max
+      end
+
+      def format_revision_ids
+        revision_ids_by_group.inject({}) do |map, (group_name, rev_id)|
+          map[group_name] = yield rev_id
+          map
+        end
+      end
+
+      def empty?
+        policy_group_names.empty?
+      end
+
+      def each
+        revision_ids_by_group.each do |group_name, rev_id|
+          yield group_name, rev_id
+        end
+      end
+    end
+
+    class Lister
+
+      attr_accessor :policy_lock_content
+
+      attr_reader :config
+
+      def initialize(config: nil)
+        @config = config
+        @policies_by_name = nil
+        @policies_by_group = nil
+        @policy_lock_content = {}
+        @active_revisions = nil
+      end
+
+      # A Hash with the following format
+      #   {
+      #     "appserver" => {
+      #       "1111111111111111111111111111111111111111111111111111111111111111" => {},
+      #       "2222222222222222222222222222222222222222222222222222222222222222" => {}
+      #     },
+      def policies_by_name
+        @policies_by_name || fetch_policy_lists
+        @policies_by_name
+      end
+
+      # A Hash with the following format:
+      #   "dev" => {
+      #     "appserver" => "1111111111111111111111111111111111111111111111111111111111111111",
+      #     "load-balancer" => "5555555555555555555555555555555555555555555555555555555555555555",
+      #     "db" => "9999999999999999999999999999999999999999999999999999999999999999"
+      #   }
+      def policies_by_group
+        @policies_by_group || fetch_policy_lists
+        @policies_by_group
+      end
+
+      def revision_info_for(policy_name, _revision_id_list)
+        RevIDLockDataMap.new(policy_name, policy_lock_content[policy_name])
+      end
+
+      def revision_ids_by_group_for_each_policy
+        policies_by_name.each do |policy_name, _policies|
+          rev_id_by_group = revision_ids_by_group_for(policy_name)
+          yield policy_name, rev_id_by_group
+        end
+      end
+
+      def revision_ids_by_group_for(policy_name)
+        map = policies_by_group.inject({}) do |rev_id_map, (group_name, rev_id_map_for_group)|
+          rev_id_map[group_name] = rev_id_map_for_group[policy_name]
+          rev_id_map
+        end
+        PolicyGroupRevIDMap.new(policy_name, map)
+      end
+
+      def orphaned_revisions(policy_name)
+        orphans = []
+        policies_by_name[policy_name].each do |rev_id, _data|
+          orphans << rev_id unless active_revisions.include?(rev_id)
+        end
+        orphans
+      end
+
+      def active_revisions
+        @active_revisions ||= policies_by_group.inject(Set.new) do |set, (_group, policy_name_rev_id_map)|
+          policy_name_rev_id_map.each do |policy_name, rev_id|
+            set << rev_id
+          end
+          set
+        end
+      end
+
+      def empty?
+        policies_by_name.empty? && policies_by_group.empty?
+      end
+
+      def http_client
+        @http_client ||= Chef::ServerAPI.new(config.chef_server_url,
+                                                       signing_key_filename: config.client_key,
+                                                       client_name: config.node_name)
+      end
+
+      # @api private
+      # Sets internal copy of policyfile data to policies_by_name and
+      # policies_by_group. Used for internal testing.
+      def set!(policies_by_name, policies_by_group)
+        @policies_by_name = policies_by_name
+        @policies_by_group = policies_by_group
+        @active_revisions = nil
+      end
+
+      private
+
+      def fetch_policy_lists
+        policy_list_data = http_client.get("policies")
+        set_policies_by_name_from_api(policy_list_data)
+
+        policy_group_data = http_client.get("policy_groups")
+        set_policies_by_group_from_api(policy_group_data)
+      end
+
+      def set_policies_by_name_from_api(policy_list_data)
+        @policies_by_name = policy_list_data.inject({}) do |map, (policy_name, policy_info)|
+          map[policy_name] = policy_info["revisions"]
+          map
+        end
+      end
+
+      def set_policies_by_group_from_api(policy_group_data)
+        @policies_by_group = policy_group_data.inject({}) do |map, (policy_group, policy_info)|
+          map[policy_group] = (policy_info["policies"] || []).inject({}) do |rev_map, (policy_name, rev_info)|
+            rev_map[policy_name] = rev_info["revision_id"]; rev_map
+          end
+
+          map
+        end
+      end
+
+    end
+  end
+end