chef-dk 2.3.4 → 2.4.17

data/lib/chef-dk/policyfile/chef_server_cookbook_source.rb

@@ -18,7 +18,7 @@
 require "ffi_yajl"
 require "chef-dk/exceptions"
 require "chef-dk/policyfile/source_uri"
-require "chef/server_api"
+require "chef-dk/chef_server_api_multi"
 
 module ChefDK
   module Policyfile
@@ -82,9 +82,10 @@ module ChefDK
       private
 
       def http_connection_for(base_url)
-        @http_connections[base_url] ||= Chef::ServerAPI.new(base_url,
-                                                       signing_key_filename: chef_config.client_key,
-                                                       client_name: chef_config.node_name)
+        @http_connections[base_url] ||=
+          ChefServerAPIMulti.new(base_url,
+                                 signing_key_filename: chef_config.client_key,
+                                 client_name: chef_config.node_name)
       end
 
       def full_chef_server_graph

data/lib/chef-dk/policyfile/chef_server_lock_fetcher.rb

@@ -0,0 +1,164 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef-dk/policyfile_lock"
+require "chef-dk/exceptions"
+
+module ChefDK
+  module Policyfile
+
+    # A policyfile lock fetcher that can read a lock from a chef server
+    class ChefServerLockFetcher
+
+      attr_accessor :name
+      attr_accessor :source_options
+      attr_accessor :chef_config
+
+      # Initialize a LocalLockFetcher
+      #
+      # @param name [String] The name of the policyfile
+      # @param source_options [Hash] A hash with a :server key pointing at the chef server,
+      # along with :policy_name and either :policy_group or :policy_revision_id. If :policy_name
+      # is not provided, name is used.
+      # @param storage_config [StorageConfig]
+      #
+      # @example ChefServerLockFetcher for a policyfile with a specific revision id
+      #   ChefServerLockFetcher.new("foo",
+      #     {server: "http://example.com", policy_revision_id: "abcdabcdabcd"},
+      #     chef_config)
+      #
+      #   ChefServerLockFetcher.new("foo",
+      #     {server: "http://example.com", policy_name: "foo", policy_revision_id: "abcdabcdabcd"},
+      #     chef_config)
+      #
+      # @example ChefServerLockFetcher for a policyfile with the latest revision_id for a policy group
+       #   ChefServerLockFetcher.new("foo",
+      #     {server: "http://example.com", policy_group: "dev"},
+      #     chef_config)
+      #
+      #   ChefServerLockFetcher.new("foo",
+      #     {server: "http://example.com", policy_name: "foo", policy_group: "dev"},
+      #     chef_config)
+      def initialize(name, source_options, chef_config)
+        @name = name
+        @source_options = source_options
+        @chef_config = chef_config
+
+        @source_options[:policy_name] ||= name
+      end
+
+      # @return [True] if there were no errors with the provided source_options
+      # @return [False] if there were errors with the provided source_options
+      def valid?
+        errors.empty?
+      end
+
+      # Check the options provided when craeting this class for errors
+      #
+      # @return [Array<String>] A list of errors found
+      def errors
+        error_messages = []
+
+        [:server, :policy_name].each do |key|
+          error_messages << "include_policy for #{name} is missing key #{key}" unless source_options[key]
+        end
+
+        if [:policy_revision_id, :policy_group].all? { |key| source_options[key].nil? }
+          error_messages << "include_policy for #{name} must specify policy_revision_id or policy_group"
+        end
+
+        error_messages
+      end
+
+      # @return [Hash] The source_options that describe how to fetch this exact lock again
+      def source_options_for_lock
+        source_options.merge({
+          policy_revision_id: lock_data["revision_id"],
+        })
+      end
+
+      # Applies source options from a lock file. This is used to make sure that the same
+      # policyfile lock is loaded that was locked
+      #
+      # @param options_from_lock [Hash] The source options loaded from a policyfile lock
+      def apply_locked_source_options(options_from_lock)
+        options = options_from_lock.inject({}) do |acc, (key, value)|
+          acc[key.to_sym] = value
+          acc
+        end
+        source_options.merge!(options)
+        raise ChefDK::InvalidLockfile, "Invalid source_options provided from lock data: #{options_from_lock_file.inspect}" if !valid?
+      end
+
+      # @return [String] of the policyfile lock data
+      def lock_data
+        @lock_data ||= fetch_lock_data.tap do |data|
+          data["cookbook_locks"].each do |cookbook_name, cookbook_lock|
+            cookbook_lock["source_options"] = {
+              "chef_server_artifact" => server,
+              "identifier" => cookbook_lock["identifier"],
+            }
+          end
+        end
+      end
+
+      private
+
+      def fetch_lock_data
+        if revision
+          http_client.get("policies/#{policy_name}/revisions/#{revision}")
+        elsif policy_group
+          http_client.get("policy_groups/#{policy_group}/policies/#{policy_name}")
+        else
+          raise ChefDK::BUG.new("The source_options should have been validated: #{source_options.inspect}")
+        end
+      rescue Net::ProtocolError => e
+        if e.respond_to?(:response) && e.response.code.to_s == "404"
+          raise ChefDK::PolicyfileLockDownloadError.new("No policyfile lock named '#{policy_name}' found with revision '#{revision}' at #{http_client.url}") if revision
+          raise ChefDK::PolicyfileLockDownloadError.new("No policyfile lock named '#{policy_name}' found with policy group '#{policy_group}' at #{http_client.url}") if policy_group
+        else
+          raise ChefDK::PolicyfileLockDownloadError.new("HTTP error attempting to fetch policyfile lock from #{http_client.url}")
+        end
+      rescue => e
+        raise e
+      end
+
+      def policy_name
+        source_options[:policy_name]
+      end
+
+      def revision
+        source_options[:policy_revision_id]
+      end
+
+      def policy_group
+        source_options[:policy_group]
+      end
+
+      def server
+        source_options[:server]
+      end
+
+      def http_client
+        @http_client ||= Chef::ServerAPI.new(source_options[:server],
+                                             signing_key_filename: chef_config.client_key,
+                                             client_name: chef_config.node_name)
+      end
+
+    end
+  end
+end

data/lib/chef-dk/policyfile/cookbook_location_specification.rb

@@ -29,7 +29,7 @@ module ChefDK
       # API contract
       include StorageConfigDelegation
 
-      SOURCE_TYPES = [:git, :github, :path, :artifactserver, :chef_server, :artifactory]
+      SOURCE_TYPES = [:git, :github, :path, :artifactserver, :chef_server, :chef_server_artifact, :artifactory]
 
       #--
       # Required by CookbookOmnifetch API contract
@@ -69,7 +69,7 @@ module ChefDK
       end
 
       def mirrors_canonical_upstream?
-        [:git, :github, :artifactserver, :chef_server, :artifactory].include?(source_type)
+        [:git, :github, :artifactserver, :chef_server, :chef_server_artifact, :artifactory].include?(source_type)
       end
 
       def installed?
@@ -112,7 +112,7 @@ module ChefDK
       end
 
       def version_fixed?
-        [:git, :github, :path].include?(@source_type)
+        [:git, :github, :path, :chef_server_artifact].include?(@source_type)
       end
 
       def version

data/lib/chef-dk/policyfile/dsl.rb

@@ -18,6 +18,7 @@
 require "chef-dk/policyfile/cookbook_sources"
 require "chef-dk/policyfile/cookbook_location_specification"
 require "chef-dk/policyfile/storage_config"
+require "chef-dk/policyfile/policyfile_location_specification"
 
 require "chef/node/attribute"
 require "chef/run_list/run_list_item"
@@ -36,6 +37,7 @@ module ChefDK
       attr_reader :run_list
       attr_reader :default_source
       attr_reader :cookbook_location_specs
+      attr_reader :included_policies
 
       attr_reader :named_run_lists
       attr_reader :node_attributes
@@ -48,6 +50,7 @@ module ChefDK
         @errors = []
         @run_list = []
         @named_run_lists = {}
+        @included_policies = []
         @default_source = [ NullCookbookSource.new ]
         @cookbook_location_specs = {}
         @storage_config = storage_config
@@ -121,6 +124,19 @@ module ChefDK
         end
       end
 
+      def include_policy(name, source_options = {})
+        if existing = included_policies.find { |p| p.name == name }
+          err = "Included policy '#{name}' assigned conflicting locations or was already specified\n\n"
+          err << "Previous source: #{existing.source_options.inspect}\n"
+          err << "Conflicts with: #{source_options.inspect}\n"
+          @errors << err
+        else
+          spec = PolicyfileLocationSpecification.new(name, source_options, storage_config, chef_config)
+          included_policies << spec
+          @errors += spec.errors
+        end
+      end
+
       def default
         @node_attributes.default
       end

data/lib/chef-dk/policyfile/included_policies_cookbook_source.rb

@@ -0,0 +1,156 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef-dk/ui"
+
+module ChefDK
+  module Policyfile
+    class IncludedPoliciesCookbookSource
+
+      class ConflictingCookbookVersions < StandardError
+      end
+
+      class ConflictingCookbookDependencies < StandardError
+      end
+
+      class ConflictingCookbookSources < StandardError
+      end
+
+      # Why do we need this class?
+      # If we rely on default sources, we may not have the the universe of cookbooks
+      # provided in the included policies
+      #
+      # This is not meant to be used from the DSL
+
+      # A list of included policies
+      attr_reader :included_policy_location_specs
+      # UI object for output
+      attr_accessor :ui
+
+      # Constructor
+      #
+      def initialize(included_policy_location_specs)
+        @included_policy_location_specs = included_policy_location_specs
+        @ui = UI.new
+        @preferred_cookbooks = []
+        yield self if block_given?
+      end
+
+      def default_source_args
+        [:included_policies, []]
+      end
+
+      def check_for_conflicts!
+        source_options
+        universe_graph
+      end
+
+      # All are preferred here
+      def preferred_source_for?(cookbook_name)
+        universe_graph.include?(cookbook_name)
+      end
+
+      def preferred_cookbooks
+        universe_graph.keys
+      end
+
+      def ==(other)
+        other.kind_of?(self.class) && other.included_policy_location_specs == included_policy_location_specs
+      end
+
+      # Calls the slurp_metadata! helper once to calculate the @universe_graph
+      # and @cookbook_version_paths metadata.  Returns the @universe_graph.
+      #
+      # @return [Hash] universe_graph
+      def universe_graph
+        @universe_graph ||= build_universe
+      end
+
+      # Returns the metadata (path and version) for an individual cookbook
+      #
+      # @return [Hash] metadata for a single cookbook version
+      def source_options_for(cookbook_name, cookbook_version)
+        source_options[cookbook_name][cookbook_version]
+      end
+
+      def null?
+        false
+      end
+
+      def desc
+        "included_policies()"
+      end
+
+      private
+
+      def build_universe
+        included_policy_location_specs.inject({}) do |acc, policy_spec|
+          lock = policy_spec.policyfile_lock
+          cookbook_dependencies = lock.solution_dependencies.cookbook_dependencies
+          cookbook_dependencies.each do |(cookbook, deps)|
+            name = cookbook.name
+            version = cookbook.version
+            mapped_deps = deps.map do |dep|
+              [dep[0], dep[1].to_s]
+            end
+            if acc[name]
+              if acc[name][version]
+                if acc[name][version] != mapped_deps
+                  raise ConflictingCookbookDependencies.new("Conflicting dependencies provided for cookbook #{name}")
+                end
+              else
+                raise ConflictingCookbookVersions.new("Multiple versions provided for cookbook #{name}")
+              end
+            else
+              acc[name] = {}
+              acc[name][version] = mapped_deps
+            end
+          end
+          acc
+        end
+      end
+
+      def source_options
+        @source_options ||= build_source_options
+      end
+
+      ## Collect all the source options
+      def build_source_options
+        included_policy_location_specs.inject({}) do |acc, policy_spec|
+          lock = policy_spec.policyfile_lock
+          lock.cookbook_locks.each do |(name, cookbook_lock)|
+            version = cookbook_lock.version
+            if acc[name]
+              if acc[name][version]
+                if acc[name][version] != cookbook_lock.source_options
+                  raise ConflictingCookbookSources.new("Conflicting sources provided for cookbook #{name}")
+                end
+              else
+                raise ConflictingCookbookVersions.new("Multiple sources provided for cookbook #{name}")
+              end
+            else
+              acc[name] = {}
+              acc[name][version] = cookbook_lock.source_options
+            end
+          end
+          acc
+        end
+      end
+
+    end
+  end
+end

data/lib/chef-dk/policyfile/local_lock_fetcher.rb

@@ -0,0 +1,122 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef-dk/policyfile_lock"
+require "chef-dk/exceptions"
+
+module ChefDK
+  module Policyfile
+
+    # A policyfile lock fetcher that can read a lock from a local disk
+    class LocalLockFetcher
+
+      attr_reader :name
+      attr_reader :source_options
+      attr_reader :storage_config
+
+      # Initialize a LocalLockFetcher
+      #
+      # @param name [String] The name of the policyfile
+      # @param source_options [Hash] A hash with a :path key pointing at the location
+      #                              of the lock
+      # @param storage_config [StorageConfig]
+      def initialize(name, source_options, storage_config)
+        @name = name
+        @source_options = source_options
+        @storage_config = storage_config
+      end
+
+      # @return [True] if there were no errors with the provided source_options
+      # @return [False] if there were errors with the provided source_options
+      def valid?
+        errors.empty?
+      end
+
+      # Check the options provided when craeting this class for errors
+      #
+      # @return [Array<String>] A list of errors found
+      def errors
+        error_messages = []
+
+        [:path].each do |key|
+          error_messages << "include_policy for #{name} is missing key #{key}" unless source_options[key]
+        end
+
+        error_messages
+      end
+
+      # @return [Hash] The source_options that describe how to fetch this exact lock again
+      def source_options_for_lock
+        source_options
+      end
+
+      # Applies source options from a lock file. This is used to make sure that the same
+      # policyfile lock is loaded that was locked
+      #
+      # @param options_from_lock [Hash] The source options loaded from a policyfile lock
+      def apply_locked_source_options(options_from_lock)
+        # There are no options the lock could provide
+      end
+
+      # @return [String] of the policyfile lock data
+      def lock_data
+        FFI_Yajl::Parser.new.parse(content).tap do |data|
+          data["cookbook_locks"].each do |cookbook_name, cookbook_lock|
+            cookbook_path = cookbook_lock["source_options"]["path"]
+            if !cookbook_path.nil?
+              cookbook_lock["source_options"]["path"] = transform_path(cookbook_path)
+            end
+          end
+        end
+      end
+
+      private
+
+      def transform_path(path_to_transform)
+        cur_path = Pathname.new(storage_config.relative_paths_root)
+        include_path = Pathname.new(path).dirname
+        include_path.relative_path_from(cur_path).join(path_to_transform).to_s
+      end
+
+      def content
+        IO.read(path)
+      end
+
+      def path
+        @path ||= begin
+          path = Pathname.new(source_options[:path])
+          if path.directory?
+            path = path.join("#{name}.lock.json")
+            if !path.file?
+              raise ChefDK::LocalPolicyfileLockNotFound.new(
+                "Expected to find file #{name}.lock.json inside #{source_options[:path]}. If the file name is different than this, provide the file name as part of the path.")
+            end
+          else
+            if !path.file?
+              raise ChefDK::LocalPolicyfileLockNotFound.new(
+                "The provided path #{source_options[:path]} does not exist.")
+            end
+          end
+          if !path.absolute?
+            path = Pathname.new(storage_config.relative_paths_root).join(path)
+          end
+          path
+        end
+      end
+    end
+  end
+end