chef 18.6.2-universal-mingw-ucrt

data/lib/chef/cookbook_loader.rb

@@ -0,0 +1,223 @@
+#
+# Author:: Adam Jacob (<adam@chef.io>)
+# Author:: Christopher Walters (<cw@chef.io>)
+# Author:: Daniel DeLeo (<dan@kallistec.com>)
+# Copyright:: Copyright (c) Chef Software Inc.
+# Copyright:: Copyright 2009-2016, Daniel DeLeo
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative "config"
+require_relative "exceptions"
+require_relative "cookbook/cookbook_version_loader"
+require_relative "cookbook_version"
+require_relative "cookbook/chefignore"
+require_relative "cookbook/metadata"
+
+class Chef
+  # This class is used by knife, cheffs and legacy chef-solo modes.  It is not used by the server mode
+  # of chef-client or zolo/zero modes.
+  #
+  # This class implements orchestration around producing a single cookbook_version for a cookbook or
+  # loading a Mash of all cookbook_versions, using the cookbook_version_loader class, and doing
+  # lazy-access and memoization to only load each cookbook once on demand.
+  #
+  # This implements a key-value style each which makes it appear to be a Hash of String => CookbookVersion
+  # pairs where the String is the cookbook name.  The use of Enumerable combined with the Hash-style
+  # each is likely not entirely sane.
+  #
+  # This object is also passed and injected into the CookbookCollection object where it is converted
+  # to a Mash that looks almost exactly like the cookbook_by_name Mash in this object.
+  #
+  class CookbookLoader
+    # @return [Array<String>] the array of repo paths containing cookbook dirs
+    attr_reader :repo_paths
+
+    # XXX: this is highly questionable combined with the Hash-style each method
+    include Enumerable
+
+    # @param repo_paths [Array<String>] the array of repo paths containing cookbook dirs
+    def initialize(*repo_paths)
+      @repo_paths = repo_paths.flatten.compact.map { |p| File.expand_path(p) }
+      raise ArgumentError, "You must specify at least one cookbook repo path" if @repo_paths.empty?
+    end
+
+    # The primary function of this class is to build this Mash mapping cookbook names as a string to
+    # the CookbookVersion objects for them.  Callers must call "load_cookbooks" first.
+    #
+    # @return [Mash<String, Chef::CookbookVersion>]
+    def cookbooks_by_name
+      @cookbooks_by_name ||= Mash.new
+    end
+
+    # This class also builds a mapping of cookbook names to their Metadata objects.  Callers must call
+    # "load_cookbooks" first.
+    #
+    # @return [Mash<String, Chef::Cookbook::Metadata>]
+    def metadata
+      @metadata ||= Mash.new
+    end
+
+    # Loads all cookbooks across all repo_paths
+    #
+    # @return [Mash<String, Chef::CookbookVersion>] the cookbooks_by_name Mash
+    def load_cookbooks
+      cookbook_version_loaders.each_key do |cookbook_name|
+        load_cookbook(cookbook_name)
+      end
+      cookbooks_by_name
+    end
+
+    # Loads a single cookbook by its name.
+    #
+    # @param [String]
+    # @return [Chef::CookbookVersion]
+    def load_cookbook(cookbook_name)
+      unless cookbook_version_loaders.key?(cookbook_name)
+        raise Exceptions::CookbookNotFoundInRepo, "Cannot find a cookbook named #{cookbook_name}; did you forget to add metadata to a cookbook? (https://docs.chef.io/config_rb_metadata/)"
+      end
+
+      return cookbooks_by_name[cookbook_name] if cookbooks_by_name.key?(cookbook_name)
+
+      loader = cookbook_version_loaders[cookbook_name]
+
+      loader.load!
+
+      cookbook_version = loader.cookbook_version
+      cookbooks_by_name[cookbook_name] = cookbook_version
+      metadata[cookbook_name] = cookbook_version.metadata unless cookbook_version.nil?
+      cookbook_version
+    end
+
+    def [](cookbook)
+      load_cookbook(cookbook)
+    end
+
+    alias :fetch :[]
+
+    def has_key?(cookbook_name)
+      not self[cookbook_name.to_sym].nil?
+    end
+
+    alias :cookbook_exists? :has_key?
+    alias :key? :has_key?
+
+    def each
+      cookbooks_by_name.keys.sort_by(&:to_s).each do |cname|
+        yield(cname, cookbooks_by_name[cname])
+      end
+    end
+
+    def each_key(&block)
+      cookbook_names.each(&block)
+    end
+
+    def each_value(&block)
+      values.each(&block)
+    end
+
+    def cookbook_names
+      cookbooks_by_name.keys.sort
+    end
+
+    def values
+      cookbooks_by_name.values
+    end
+
+    # This method creates tmp directory and copies all cookbooks into it and creates cookbook loader object which points to tmp directory
+    def self.copy_to_tmp_dir_from_array(cookbooks)
+      Dir.mktmpdir do |tmp_dir|
+        cookbooks.each do |cookbook|
+          checksums_to_on_disk_paths = cookbook.checksums
+          cookbook.each_file do |manifest_record|
+            path_in_cookbook = manifest_record[:path]
+            on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
+            dest = File.join(tmp_dir, cookbook.name.to_s, path_in_cookbook)
+            FileUtils.mkdir_p(File.dirname(dest))
+            FileUtils.cp_r(on_disk_path, dest)
+          end
+        end
+        tmp_cookbook_loader ||= begin
+          Chef::Cookbook::FileVendor.fetch_from_disk(tmp_dir)
+          CookbookLoader.new(tmp_dir)
+        end
+        yield tmp_cookbook_loader
+      end
+    end
+
+    # generates metadata.json adds it in the manifest
+    def compile_metadata
+      each do |cookbook_name, cookbook|
+        compiled_metadata = cookbook.compile_metadata
+        if compiled_metadata
+          cookbook.all_files << compiled_metadata
+          cookbook.cookbook_manifest.send(:generate_manifest)
+        end
+      end
+    end
+
+    # freeze versions of all the cookbooks
+    def freeze_versions
+      each do |cookbook_name, cookbook|
+        cookbook.freeze_version
+      end
+    end
+
+    alias :cookbooks :values
+
+    private
+
+    # Helper method to lazily create and remember the chefignore object
+    # for a given repo_path.
+    #
+    # @param [String] repo_path the full path to the cookbook directory of the repo
+    # @return [Chef::Cookbook::Chefignore] the chefignore object for the repo_path
+    def chefignore(repo_path)
+      @chefignores ||= {}
+      @chefignores[repo_path] ||= Cookbook::Chefignore.new(repo_path)
+    end
+
+    def all_directories_in_repo_paths
+      @all_directories_in_repo_paths ||=
+        all_files_in_repo_paths.select { |path| File.directory?(path) }
+    end
+
+    def all_files_in_repo_paths
+      @all_files_in_repo_paths ||=
+        repo_paths.inject([]) do |all_children, repo_path|
+          all_children + Dir[File.join(Chef::Util::PathHelper.escape_glob_dir(repo_path), "*")]
+        end
+    end
+
+    # This method creates a Mash of the CookbookVersionLoaders for each cookbook.
+    #
+    # @return [Mash<String, Cookbook::CookbookVersionLoader>]
+    def cookbook_version_loaders
+      @cookbook_version_loaders ||=
+        begin
+          mash = Mash.new
+          all_directories_in_repo_paths.each do |cookbook_path|
+            loader = Cookbook::CookbookVersionLoader.new(cookbook_path, chefignore(cookbook_path))
+            cookbook_name = loader.cookbook_name
+            if mash.key?(cookbook_name)
+              raise Chef::Exceptions::CookbookMergingError, "Cookbook merging is no longer supported, the cookbook named #{cookbook_name} can only appear once in the cookbook_path"
+            end
+
+            mash[cookbook_name] = loader
+          end
+          mash
+        end
+    end
+  end
+end

data/lib/chef/cookbook_manifest.rb

@@ -0,0 +1,331 @@
+# Author:: Daniel DeLeo (<dan@chef.io>)
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "forwardable" unless defined?(Forwardable)
+require_relative "mixin/versioned_api"
+require_relative "util/path_helper"
+require_relative "cookbook/manifest_v0"
+require_relative "cookbook/manifest_v2"
+require_relative "log"
+
+class Chef
+
+  # Handles the details of representing a cookbook in JSON form for uploading
+  # to a Chef Server.
+  class CookbookManifest
+
+    extend Forwardable
+
+    attr_reader :cookbook_version
+
+    def_delegator :@cookbook_version, :root_paths
+    def_delegator :@cookbook_version, :name
+    def_delegator :@cookbook_version, :identifier
+    def_delegator :@cookbook_version, :metadata
+    def_delegator :@cookbook_version, :full_name
+    def_delegator :@cookbook_version, :version
+    def_delegator :@cookbook_version, :frozen_version?
+
+    # Create a new CookbookManifest object for the given `cookbook_version`.
+    # You can subsequently call #to_h to get a Hash representation of the
+    # cookbook_version in the "manifest" format, or #to_json to get a JSON
+    # representation of the cookbook_version.
+    #
+    # The interface for this behavior is expected to change as we implement new
+    # manifest formats. The entire class should be considered a private API for
+    # now.
+    #
+    # @api private
+    # @param policy_mode [Boolean] whether to convert cookbooks to Hash/JSON in
+    #   the format used by the `cookbook_artifacts` endpoint (for policyfiles).
+    #   Setting this option also changes the behavior of #save_url and
+    #   #force_save_url such that CookbookVersions will be uploaded to the new
+    #   `cookbook_artifacts` API.
+    def initialize(cookbook_version, policy_mode: false)
+      @cookbook_version = cookbook_version
+      @policy_mode = !!policy_mode
+
+      reset!
+    end
+
+    # Resets all lazily computed values.
+    def reset!
+      @manifest = nil
+      @checksums = nil
+      @manifest_records_by_path = nil
+      true
+    end
+
+    # Returns a 'manifest' data structure that can be uploaded to a Chef
+    # Server.
+    #
+    # The format is as follows:
+    #
+    #     {
+    #       :cookbook_name  => name,            # String
+    #       :metadata       => metadata,        # Chef::Cookbook::Metadata
+    #       :version        => version,         # Chef::Version
+    #       :name           => full_name,       # String of "#{name}-#{version}"
+    #
+    #       :recipes        => Array<FileSpec>,
+    #       :definitions    => Array<FileSpec>,
+    #       :libraries      => Array<FileSpec>,
+    #       :attributes     => Array<FileSpec>,
+    #       :files          => Array<FileSpec>,
+    #       :templates      => Array<FileSpec>,
+    #       :resources      => Array<FileSpec>,
+    #       :providers      => Array<FileSpec>,
+    #       :root_files     => Array<FileSpec>
+    #     }
+    #
+    # Where a `FileSpec` is a Hash of the form:
+    #
+    #     {
+    #       :name         => file_name,
+    #       :path         => path,
+    #       :checksum     => csum,
+    #       :specificity  => specificity
+    #     }
+    #
+    def manifest
+      @manifest || generate_manifest
+      @manifest
+    end
+
+    def checksums
+      @manifest || generate_manifest
+      @checksums
+    end
+
+    def manifest_records_by_path
+      @manifest || generate_manifest
+      @manifest_records_by_path
+    end
+
+    def policy_mode?
+      @policy_mode
+    end
+
+    def to_h
+      CookbookManifestVersions.to_h(self)
+    end
+
+    alias_method :to_hash, :to_h
+
+    def to_json(*a)
+      result = to_h
+      result["json_class"] = "Chef::CookbookVersion"
+      Chef::JSONCompat.to_json(result, *a)
+    end
+
+    # Return the URL to save (PUT) this object to the server via the
+    # REST api. If there is an existing document on the server and it
+    # is marked frozen, a PUT will result in a 409 Conflict.
+    def save_url
+      if policy_mode?
+        "#{named_cookbook_url}/#{identifier}"
+      else
+        "#{named_cookbook_url}/#{version}"
+      end
+    end
+
+    def named_cookbook_url
+      "#{cookbook_url_path}/#{name}"
+    end
+
+    # Adds the `force=true` parameter to the upload URL. This allows
+    # the user to overwrite a frozen cookbook (a PUT against the
+    # normal #save_url raises a 409 Conflict in this case).
+    def force_save_url
+      "#{save_url}?force=true"
+    end
+
+    # Update this CookbookManifest from the contents of another manifest, and
+    # make the corresponding changes to the cookbook_version object. Required
+    # to provide backward compatibility with CookbookVersion#manifest= method.
+    def update_from(new_manifest)
+      @manifest = Chef::CookbookManifestVersions.from_hash(new_manifest)
+      @checksums = extract_checksums_from_manifest(@manifest)
+      @manifest_records_by_path = extract_manifest_records_by_path(@manifest)
+    end
+
+    # @api private
+    # takes a list of hashes
+    def add_files_to_manifest(files)
+      manifest[:all_files].concat(Array(files))
+      @checksums = extract_checksums_from_manifest(@manifest)
+      @manifest_records_by_path = extract_manifest_records_by_path(@manifest)
+    end
+
+    def files_for(part)
+      return root_files if part.to_s == "root_files"
+
+      part_match = "#{part}/"
+      manifest[:all_files].select do |file|
+        file[:name].start_with?(part_match)
+      end
+    end
+
+    def each_file(excluded_parts: [], &block)
+      excluded_parts = Array(excluded_parts).map(&:to_s)
+
+      manifest[:all_files].each do |file|
+        seg = file[:name].split("/")[0]
+        next if excluded_parts.include?(seg)
+
+        yield file if block_given?
+      end
+    end
+
+    def by_parent_directory
+      @by_parent_directory ||=
+        manifest[:all_files].inject({}) do |memo, file|
+          parts = file[:name].split("/")
+          parent = if parts.length == 1
+                     "root_files"
+                   else
+                     parts[0]
+                   end
+
+          memo[parent] ||= []
+          memo[parent] << file
+          memo
+        end
+    end
+
+    def root_files
+      manifest[:all_files].select do |file|
+        segment, name = file[:name].split("/")
+        name.nil? || segment == "root_files"
+      end
+    end
+
+    private
+
+    def cookbook_url_path
+      policy_mode? ? "cookbook_artifacts" : "cookbooks"
+    end
+
+    # See #manifest for a description of the manifest return value.
+    # See #preferred_manifest_record for a description an individual manifest record.
+    def generate_manifest
+      manifest = Mash.new({
+        all_files: [],
+      })
+      @checksums = {}
+
+      if !root_paths || root_paths.size == 0
+        Chef::Log.error("Cookbook #{name} does not have root_paths! Cannot generate manifest.")
+        raise "Cookbook #{name} does not have root_paths! Cannot generate manifest."
+      end
+
+      @cookbook_version.all_files.each do |file|
+        next if File.directory?(file)
+
+        name, path, specificity = parse_file_from_root_paths(file)
+
+        csum = checksum_cookbook_file(file)
+        @checksums[csum] = file
+        rs = Mash.new({
+          name: name,
+          path: path,
+          checksum: csum,
+          specificity: specificity,
+          # full_path is not a part of the normal manifest, but is very useful to keep around.
+          # uploaders should strip this out.
+          full_path: file,
+        })
+
+        manifest[:all_files] << rs
+      end
+
+      manifest[:metadata] = metadata
+      manifest[:version] = metadata.version
+
+      if policy_mode?
+        manifest[:name] = name.to_s
+        manifest[:identifier] = identifier
+      else
+        manifest[:name] = full_name
+        manifest[:cookbook_name] = name.to_s
+      end
+
+      @manifest_records_by_path = extract_manifest_records_by_path(manifest)
+      @manifest = manifest
+    end
+
+    def parse_file_from_root_paths(file)
+      root_paths.each do |root_path|
+        pathname = Chef::Util::PathHelper.relative_path_from(root_path, file)
+
+        parts = pathname.each_filename.take(2)
+        # Check if path is actually under root_path
+        next if parts[0] == ".."
+
+        # if we have a root_file, such as metadata.rb, the first part will be "."
+        return [ "root_files/#{pathname}", pathname.to_s, "default" ] if parts.length == 1
+
+        segment = parts[0]
+
+        name = File.join(segment, pathname.basename.to_s)
+
+        if %w{templates files}.include?(segment)
+          # Check if pathname looks like files/foo or templates/foo (unscoped)
+          if pathname.each_filename.to_a.length == 2
+            # Use root_default in case the same path exists at root_default and default
+            return [ name, pathname.to_s, "root_default" ]
+          else
+            return [ name, pathname.to_s, parts[1] ]
+          end
+        else
+          return [ name, pathname.to_s, "default" ]
+        end
+      end
+      Chef::Log.error("Cookbook file #{file} not under cookbook root paths #{root_paths.inspect}.")
+      raise "Cookbook file #{file} not under cookbook root paths #{root_paths.inspect}."
+    end
+
+    def extract_checksums_from_manifest(manifest)
+      manifest[:all_files].inject({}) do |memo, manifest_record|
+        memo[manifest_record[:checksum]] = nil
+        memo
+      end
+    end
+
+    def checksum_cookbook_file(filepath)
+      CookbookVersion.checksum_cookbook_file(filepath)
+    end
+
+    def extract_manifest_records_by_path(manifest)
+      manifest[:all_files].inject({}) do |memo, manifest_record|
+        memo[manifest_record[:path]] = manifest_record
+        memo
+      end
+    end
+
+  end
+
+  class CookbookManifestVersions
+
+    extend Chef::Mixin::VersionedAPIFactory
+    add_versioned_api_class Chef::Cookbook::ManifestV0
+    add_versioned_api_class Chef::Cookbook::ManifestV2
+
+    def_versioned_delegator :from_hash
+    def_versioned_delegator :to_hash
+    def_versioned_delegator :to_h
+  end
+end

data/lib/chef/cookbook_uploader.rb

@@ -0,0 +1,157 @@
+
+autoload :Set, "set"
+require_relative "exceptions"
+require_relative "digester"
+require_relative "cookbook_manifest"
+require_relative "cookbook_version"
+require_relative "cookbook/syntax_check"
+require_relative "cookbook/file_system_file_vendor"
+require_relative "util/threaded_job_queue"
+require_relative "sandbox"
+require_relative "server_api"
+
+class Chef
+  class CookbookUploader
+
+    attr_reader :cookbooks
+    attr_reader :path
+    attr_reader :opts
+    attr_reader :rest
+    attr_reader :concurrency
+
+    # Creates a new CookbookUploader.
+    # ===Arguments:
+    # * cookbooks::: A Chef::CookbookVersion or array of them describing the
+    #                cookbook(s) to be uploaded
+    # * path::: A String or Array of Strings representing the base paths to the
+    #           cookbook repositories.
+    # * opts::: (optional) An options Hash
+    # ===Options:
+    # * :force  indicates that the uploader should set the force option when
+    #           uploading the cookbook. This allows frozen CookbookVersion
+    #           documents on the server to be overwritten (otherwise a 409 is
+    #           returned by the server)
+    # * :rest   A Chef::ServerAPI object that you have configured the way you like it.
+    #           If you don't provide this, one will be created using the values
+    #           in Chef::Config.
+    # * :concurrency   An integer that decided how many threads will be used to
+    #           perform concurrent uploads
+    def initialize(cookbooks, opts = {})
+      @opts = opts
+      @cookbooks = Array(cookbooks)
+      @rest = opts[:rest] || Chef::ServerAPI.new(Chef::Config[:chef_server_url], version_class: Chef::CookbookManifestVersions)
+      @concurrency = opts[:concurrency] || 10
+      @policy_mode = opts[:policy_mode] || false
+    end
+
+    def upload_cookbooks
+      # Syntax Check
+      validate_cookbooks unless opts[:skip_syntax_check]
+      # generate checksums of cookbook files and create a sandbox
+      checksum_files = {}
+      cookbooks.each do |cb|
+        Chef::Log.info("Saving #{cb.name}")
+        checksum_files.merge!(cb.checksums)
+      end
+
+      checksums = checksum_files.inject({}) { |memo, elt| memo[elt.first] = nil; memo }
+      new_sandbox = rest.post("sandboxes", { checksums: checksums })
+
+      Chef::Log.info("Uploading files")
+
+      queue = Chef::Util::ThreadedJobQueue.new
+
+      checksums_to_upload = Set.new
+
+      # upload the new checksums and commit the sandbox
+      new_sandbox["checksums"].each do |checksum, info|
+        if info["needs_upload"] == true
+          checksums_to_upload << checksum
+          Chef::Log.info("Uploading #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info["url"]}")
+          queue << uploader_function_for(checksum_files[checksum], checksum, info["url"], checksums_to_upload)
+        else
+          Chef::Log.trace("#{checksum_files[checksum]} has not changed")
+        end
+      end
+
+      queue.process(@concurrency)
+
+      sandbox_url = new_sandbox["uri"]
+      Chef::Log.trace("Committing sandbox")
+      # Retry if S3 is claims a checksum doesn't exist (the eventual
+      # in eventual consistency)
+      retries = 0
+      begin
+        rest.put(sandbox_url, { is_completed: true })
+      rescue Net::HTTPClientException => e
+        if e.message =~ /^400/ && (retries += 1) <= 5
+          sleep 2
+          retry
+        else
+          raise
+        end
+      end
+
+      # files are uploaded, so save the manifest
+      cookbooks.each do |cb|
+
+        manifest = Chef::CookbookManifest.new(cb, policy_mode: policy_mode?)
+
+        save_url = opts[:force] ? manifest.force_save_url : manifest.save_url
+        begin
+          rest.put(save_url, manifest)
+        rescue Net::HTTPClientException => e
+          case e.response.code
+          when "409"
+            raise Chef::Exceptions::CookbookFrozen, "Version #{cb.version} of cookbook #{cb.name} is frozen. Use --force to override."
+          else
+            raise
+          end
+        end
+      end
+
+      Chef::Log.info("Upload complete!")
+    end
+
+    def uploader_function_for(file, checksum, url, checksums_to_upload)
+      lambda do
+        # Checksum is the hexadecimal representation of the md5,
+        # but we need the base64 encoding for the content-md5
+        # header
+        checksum64 = Base64.encode64([checksum].pack("H*")).strip
+        file_contents = File.binread(file)
+
+        # Custom headers. 'content-type' disables JSON serialization of the request body.
+        headers = { "content-type" => "application/x-binary", "content-md5" => checksum64, "accept" => "application/json" }
+
+        begin
+          rest.put(url, file_contents, headers)
+          checksums_to_upload.delete(checksum)
+        rescue Net::HTTPClientException, Net::HTTPFatalError, Errno::ECONNREFUSED, Timeout::Error, Errno::ETIMEDOUT, SocketError => e
+          error_message = "Failed to upload #{file} (#{checksum}) to #{url} : #{e.message}"
+          error_message << "\n#{e.response.body}" if e.respond_to?(:response)
+          Chef::Knife.ui.error(error_message)
+          raise
+        end
+      end
+    end
+
+    def validate_cookbooks
+      cookbooks.each do |cb|
+        next if cb.nil?
+
+        syntax_checker = Chef::Cookbook::SyntaxCheck.new(cb.root_dir)
+        Chef::Log.info("Validating ruby files")
+        exit(1) unless syntax_checker.validate_ruby_files
+        Chef::Log.info("Validating templates")
+        exit(1) unless syntax_checker.validate_templates
+        Chef::Log.info("Syntax OK")
+      end
+    end
+
+    def policy_mode?
+      @policy_mode
+    end
+
+  end
+end