chef 12.18.31 → 13.0.118

data/lib/chef/cookbook/synchronizer.rb

@@ -62,18 +62,11 @@ class Chef
   # Synchronizes the locally cached copies of cookbooks with the files on the
   # server.
   class CookbookSynchronizer
-    CookbookFile = Struct.new(:cookbook, :segment, :manifest_record)
+    CookbookFile = Struct.new(:cookbook, :manifest_record)
 
     attr_accessor :remove_obsoleted_files
 
     def initialize(cookbooks_by_name, events)
-      @eager_segments = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup
-      unless Chef::Config[:no_lazy_load]
-        @eager_segments.delete(:files)
-        @eager_segments.delete(:templates)
-      end
-      @eager_segments.freeze
-
       @cookbooks_by_name, @events = cookbooks_by_name, events
 
       @cookbook_full_file_paths = {}
@@ -101,15 +94,19 @@ class Chef
     end
 
     def cookbook_segment(cookbook_name, segment)
-      @cookbooks_by_name[cookbook_name].manifest[segment]
+      @cookbooks_by_name[cookbook_name].files_for(segment)
     end
 
     def files
+      exclude = unless Chef::Config[:no_lazy_load]
+                  [ :files, :templates ]
+                else
+                  []
+                end
+
       @files ||= cookbooks.inject([]) do |memo, cookbook|
-        @eager_segments.each do |segment|
-          cookbook.manifest[segment].each do |manifest_record|
-            memo << CookbookFile.new(cookbook, segment, manifest_record)
-          end
+        cookbook.each_file(excluded_parts: exclude) do |manifest_record|
+          memo << CookbookFile.new(cookbook, manifest_record)
         end
         memo
       end
@@ -162,8 +159,10 @@ class Chef
 
       @events.cookbook_sync_start(cookbook_count)
       queue.process(Chef::Config[:cookbook_sync_threads])
+      # Ensure that cookbooks know where they're rooted at, for manifest purposes.
+      ensure_cookbook_paths
       # Update the full file paths in the manifest
-      update_cookbook_filenames()
+      update_cookbook_filenames
 
     rescue Exception => e
       @events.cookbook_sync_failed(cookbooks, e)
@@ -176,9 +175,8 @@ class Chef
     # Saves the full_path to the file of the cookbook to be updated
     # in the manifest later
     def save_full_file_path(file, full_path)
-      @cookbook_full_file_paths[file.cookbook] ||= {}
-      @cookbook_full_file_paths[file.cookbook][file.segment] ||= [ ]
-      @cookbook_full_file_paths[file.cookbook][file.segment] << full_path
+      @cookbook_full_file_paths[file.cookbook] ||= []
+      @cookbook_full_file_paths[file.cookbook] << full_path
     end
 
     # remove cookbooks that are not referenced in the expanded run_list at all
@@ -229,10 +227,15 @@ class Chef
     end
 
     def update_cookbook_filenames
-      @cookbook_full_file_paths.each do |cookbook, file_segments|
-        file_segments.each do |segment, full_paths|
-          cookbook.replace_segment_filenames(segment, full_paths)
-        end
+      @cookbook_full_file_paths.each do |cookbook, full_paths|
+        cookbook.all_files = full_paths
+      end
+    end
+
+    def ensure_cookbook_paths
+      cookbooks.each do |cookbook|
+        cb_dir = File.join(Chef::Config[:file_cache_path], "cookbooks", cookbook.name)
+        cookbook.root_paths = Array(cb_dir)
       end
     end
 

data/lib/chef/cookbook/syntax_check.rb

@@ -110,7 +110,7 @@ class Chef
       end
 
       def remove_uninteresting_ruby_files(file_list)
-        file_list.reject { |f| f =~ %r{#{cookbook_path}/(files|templates)/} }
+        file_list.reject { |f| f =~ %r{#{Regexp.quote(cookbook_path)}/(files|templates)/} }
       end
 
       def ruby_files

data/lib/chef/cookbook_manifest.rb

@@ -15,7 +15,10 @@
 # limitations under the License.
 
 require "forwardable"
+require "chef/mixin/versioned_api"
 require "chef/util/path_helper"
+require "chef/cookbook/manifest_v0"
+require "chef/cookbook/manifest_v2"
 require "chef/log"
 
 class Chef
@@ -24,17 +27,11 @@ class Chef
   # to a Chef Server.
   class CookbookManifest
 
-    # Duplicates the same constant in CookbookVersion. We cannot remove it
-    # there because it is treated by other code as part of CookbookVersion's
-    # public API (also used in some deprecated methods).
-    COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ].freeze
-
     extend Forwardable
 
     attr_reader :cookbook_version
 
     def_delegator :@cookbook_version, :root_paths
-    def_delegator :@cookbook_version, :segment_filenames
     def_delegator :@cookbook_version, :name
     def_delegator :@cookbook_version, :identifier
     def_delegator :@cookbook_version, :metadata
@@ -56,10 +53,7 @@ class Chef
     #   the format used by the `cookbook_artifacts` endpoint (for policyfiles).
     #   Setting this option also changes the behavior of #save_url and
     #   #force_save_url such that CookbookVersions will be uploaded to the new
-    #   `cookbook_artifacts` API. This endpoint is currently under active
-    #   development and the format is expected to change frequently, therefore
-    #   the result of #manifest, #to_hash, and #to_json will not be stable when
-    #   `policy_mode` is enabled.
+    #   `cookbook_artifacts` API.
     def initialize(cookbook_version, policy_mode: false)
       @cookbook_version = cookbook_version
       @policy_mode = !!policy_mode
@@ -126,10 +120,7 @@ class Chef
     end
 
     def to_hash
-      result = manifest.dup
-      result["frozen?"] = frozen_version?
-      result["chef_type"] = "cookbook_version"
-      result.to_hash
+      CookbookManifestVersions.to_hash(self)
     end
 
     def to_json(*a)
@@ -164,15 +155,48 @@ class Chef
     # make the corresponding changes to the cookbook_version object. Required
     # to provide backward compatibility with CookbookVersion#manifest= method.
     def update_from(new_manifest)
-      @manifest = Mash.new new_manifest
+      @manifest = Chef::CookbookManifestVersions.from_hash(new_manifest)
       @checksums = extract_checksums_from_manifest(@manifest)
       @manifest_records_by_path = extract_manifest_records_by_path(@manifest)
+    end
+
+    def files_for(part)
+      return root_files if part.to_s == "root_files"
+      manifest[:all_files].select do |file|
+        seg = file[:name].split("/")[0]
+        part.to_s == seg
+      end
+    end
+
+    def each_file(excluded_parts: [], &block)
+      excluded_parts = Array(excluded_parts).map { |p| p.to_s }
 
-      COOKBOOK_SEGMENTS.each do |segment|
-        next unless @manifest.has_key?(segment)
-        filenames = @manifest[segment].map { |manifest_record| manifest_record["name"] }
+      manifest[:all_files].each do |file|
+        seg = file[:name].split("/")[0]
+        next if excluded_parts.include?(seg)
+        yield file if block_given?
+      end
+    end
+
+    def by_parent_directory
+      @by_parent_directory ||=
+        manifest[:all_files].inject({}) do |memo, file|
+          parts = file[:name].split("/")
+          parent = if parts.length == 1
+                     "root_files"
+                   else
+                     parts[0]
+                   end
+
+          memo[parent] ||= []
+          memo[parent] << file
+          memo
+        end
+    end
 
-        cookbook_version.replace_segment_filenames(segment, filenames)
+    def root_files
+      manifest[:all_files].select do |file|
+        file[:name].split("/").length == 1
       end
     end
 
@@ -186,15 +210,7 @@ class Chef
     # See #preferred_manifest_record for a description an individual manifest record.
     def generate_manifest
       manifest = Mash.new({
-        :recipes => Array.new,
-        :definitions => Array.new,
-        :libraries => Array.new,
-        :attributes => Array.new,
-        :files => Array.new,
-        :templates => Array.new,
-        :resources => Array.new,
-        :providers => Array.new,
-        :root_files => Array.new,
+        all_files: Array.new,
       })
       @checksums = {}
 
@@ -203,24 +219,24 @@ class Chef
         raise "Cookbook #{name} does not have root_paths! Cannot generate manifest."
       end
 
-      COOKBOOK_SEGMENTS.each do |segment|
-        segment_filenames(segment).each do |segment_file|
-          next if File.directory?(segment_file)
+      @cookbook_version.all_files.each do |file|
+        next if File.directory?(file)
 
-          path, specificity = parse_segment_file_from_root_paths(segment, segment_file)
-          file_name = File.basename(path)
+        name, path, specificity = parse_file_from_root_paths(file)
 
-          csum = checksum_cookbook_file(segment_file)
-          @checksums[csum] = segment_file
-          rs = Mash.new({
-            :name => file_name,
-            :path => path,
-            :checksum => csum,
-            :specificity => specificity,
-          })
+        csum = checksum_cookbook_file(file)
+        @checksums[csum] = file
+        rs = Mash.new({
+          :name => name,
+          :path => path,
+          :checksum => csum,
+          :specificity => specificity,
+          # full_path is not a part of the normal manifest, but is very useful to keep around.
+          # uploaders should strip this out.
+          :full_path => file,
+        })
 
-          manifest[segment] << rs
-        end
+        manifest[:all_files] << rs
       end
 
       manifest[:metadata] = metadata
@@ -238,38 +254,42 @@ class Chef
       @manifest = manifest
     end
 
-    def parse_segment_file_from_root_paths(segment, segment_file)
+    def parse_file_from_root_paths(file)
       root_paths.each do |root_path|
-        pathname = Chef::Util::PathHelper.relative_path_from(root_path, segment_file)
+        pathname = Chef::Util::PathHelper.relative_path_from(root_path, file)
 
         parts = pathname.each_filename.take(2)
         # Check if path is actually under root_path
         next if parts[0] == ".."
-        if segment == :templates || segment == :files
+
+        # if we have a root_file, such as metadata.rb, the first part will be "."
+        return [ pathname.to_s, pathname.to_s, "default" ] if parts.length == 1
+
+        segment = parts[0]
+
+        name = File.join(segment, pathname.basename.to_s)
+
+        if segment == "templates" || segment == "files"
           # Check if pathname looks like files/foo or templates/foo (unscoped)
           if pathname.each_filename.to_a.length == 2
             # Use root_default in case the same path exists at root_default and default
-            return [ pathname.to_s, "root_default" ]
+            return [ name, pathname.to_s, "root_default" ]
           else
-            return [ pathname.to_s, parts[1] ]
+            return [ name, pathname.to_s, parts[1] ]
           end
         else
-          return [ pathname.to_s, "default" ]
+          return [ name, pathname.to_s, "default" ]
         end
       end
-      Chef::Log.error("Cookbook file #{segment_file} not under cookbook root paths #{root_paths.inspect}.")
-      raise "Cookbook file #{segment_file} not under cookbook root paths #{root_paths.inspect}."
+      Chef::Log.error("Cookbook file #{file} not under cookbook root paths #{root_paths.inspect}.")
+      raise "Cookbook file #{file} not under cookbook root paths #{root_paths.inspect}."
     end
 
     def extract_checksums_from_manifest(manifest)
-      checksums = {}
-      COOKBOOK_SEGMENTS.each do |segment|
-        next unless manifest.has_key?(segment)
-        manifest[segment].each do |manifest_record|
-          checksums[manifest_record[:checksum]] = nil
-        end
+      manifest[:all_files].inject({}) do |memo, manifest_record|
+        memo[manifest_record[:checksum]] = nil
+        memo
       end
-      checksums
     end
 
     def checksum_cookbook_file(filepath)
@@ -277,14 +297,20 @@ class Chef
     end
 
     def extract_manifest_records_by_path(manifest)
-      manifest_records_by_path = {}
-      COOKBOOK_SEGMENTS.each do |segment|
-        next unless manifest.has_key?(segment)
-        manifest[segment].each do |manifest_record|
-          manifest_records_by_path[manifest_record[:path]] = manifest_record
-        end
+      manifest[:all_files].inject({}) do |memo, manifest_record|
+        memo[manifest_record[:path]] = manifest_record
+        memo
       end
-      manifest_records_by_path
     end
+
+  end
+  class CookbookManifestVersions
+
+    extend Chef::Mixin::VersionedAPIFactory
+    add_versioned_api_class Chef::Cookbook::ManifestV0
+    add_versioned_api_class Chef::Cookbook::ManifestV2
+
+    def_versioned_delegator :from_hash
+    def_versioned_delegator :to_hash
   end
 end

data/lib/chef/cookbook_site_streaming_uploader.rb

@@ -43,15 +43,13 @@ class Chef
         FileUtils.mkdir_p(tmp_cookbook_dir)
         Chef::Log.debug("Staging at #{tmp_cookbook_dir}")
         checksums_to_on_disk_paths = cookbook.checksums
-        Chef::CookbookVersion::COOKBOOK_SEGMENTS.each do |segment|
-          cookbook.manifest[segment].each do |manifest_record|
-            path_in_cookbook = manifest_record[:path]
-            on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
-            dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook)
-            FileUtils.mkdir_p(File.dirname(dest))
-            Chef::Log.debug("Staging #{on_disk_path} to #{dest}")
-            FileUtils.cp(on_disk_path, dest)
-          end
+        cookbook.each_file do |manifest_record|
+          path_in_cookbook = manifest_record[:path]
+          on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
+          dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook)
+          FileUtils.mkdir_p(File.dirname(dest))
+          Chef::Log.debug("Staging #{on_disk_path} to #{dest}")
+          FileUtils.cp(on_disk_path, dest)
         end
 
         # First, generate metadata

data/lib/chef/cookbook_uploader.rb

@@ -40,7 +40,7 @@ class Chef
     def initialize(cookbooks, opts = {})
       @opts = opts
       @cookbooks = Array(cookbooks)
-      @rest = opts[:rest] || Chef::ServerAPI.new(Chef::Config[:chef_server_url])
+      @rest = opts[:rest] || Chef::ServerAPI.new(Chef::Config[:chef_server_url], version_class: Chef::CookbookManifestVersions)
       @concurrency = opts[:concurrency] || 10
       @policy_mode = opts[:policy_mode] || false
     end