chef 15.6.10 → 15.7.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e02fb7120c329ebf6233690e2e110b68b0291947e3adea86c70ceb66985294ec
-  data.tar.gz: 66e70b92fb39c17aa0f6a4603e086375c889efa627d1b4698019e864d7ccb64a
+  metadata.gz: 21a65c380c16265bc74c76cb525f691c84c26327e067a6149805219a6424c793
+  data.tar.gz: a0b987c14a4f944998856755ce3c29079437537656a23677c990354d4f412ad9
 SHA512:
-  metadata.gz: 1e7af12ed22f92f4ebf512820822fed9fcdd28aeb63b68da368083bc3425fe79fd230183d13ae97961ab0734e56601f3f5947cf5e272f032b761eef81fac12f1
-  data.tar.gz: a5c843217dd59bdfe0d8712838507b422cc89f8fadd0d1b10fae3914f60726b6d956ce0c44c4138e37a8808cac95513811d9584f698b9d32a59f0fbcf1be2ba0
+  metadata.gz: 5dd7d0407f6ab4162bb06dba39338ff1559a124eed0254e7621e727bcf20793e4c4ba024e56b111d1ffdacaf7df70d1c05e66668d270ab46f0aca0eb6f0d2e01
+  data.tar.gz: 90d03863587712d956ae1f1115f5f1d0d3ea09ef9f52c0f4c1adfd7820c75869428edf8cf614babd6b02f46a39a0d56d6baaa7ad5d8a8bcd8640bcd4252313dd

data/Gemfile

@@ -8,7 +8,7 @@ source "https://rubygems.org"
 gem "chef", path: "."
 
 # necessary until we release ohai 15
-gem "ohai", git: "https://github.com/chef/ohai.git", branch: "master"
+gem "ohai", git: "https://github.com/chef/ohai.git", branch: "15-stable"
 
 gem "chef-utils", path: File.expand_path("../chef-utils", __FILE__) if File.exist?(File.expand_path("../chef-utils", __FILE__))
 gem "chef-config", path: File.expand_path("../chef-config", __FILE__) if File.exist?(File.expand_path("../chef-config", __FILE__))

data/README.md

@@ -1,6 +1,6 @@
 # Chef Infra
 [![Code Climate](https://codeclimate.com/github/chef/chef.svg)](https://codeclimate.com/github/chef/chef)
-[![Build Status](https://badge.buildkite.com/c82093430ceec7d27af05febb9dcafe3aa331fff9d74c0ab9d.svg?branch=master)](https://buildkite.com/chef-oss/chef-chef-master-verify)
+[![Build status](https://badge.buildkite.com/13b36bf2d9116e3527ff9893e7edd82cb183027941fef3d24e.svg)](https://buildkite.com/chef-oss/chef-chef-chef-15-verify)
 [![Gem Version](https://badge.fury.io/rb/chef.svg)](https://badge.fury.io/rb/chef)
 [![](https://img.shields.io/badge/Release%20Policy-Cadence%20Release-brightgreen.svg)](https://github.com/chef/chef/blob/v15.2.21/docs/dev/design_documents/client_release_cadence.md)
 
@@ -32,7 +32,7 @@ Other useful resources for Chef Infra users:
 
 Issues can be reported by using [GitHub Issues](https://github.com/chef/chef/issues).
 
-Note that this repository is primarily for reporting issues in the chef-client itself. For reporting issues against other Chef projects, please look up the appropriate repository. If you're unsure where to submit an issue, please ask in the #chef-dev channel in [Chef Community Slack](https://community-slack.chef.io/).
+Note that this repository is primarily for reporting issues in the chef-client itself.  For reporting issues against other Chef projects, please look up the appropriate repository. If you're unsure where to submit an issue, please ask in the #chef-dev channel in [Chef Community Slack](https://community-slack.chef.io/).
 
 ## How We Build & Release Chef
 

data/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb

@@ -72,12 +72,29 @@ class Chef
           end
 
           def upload_cookbook(other, options)
-            cookbook_to_upload = other.chef_object
-            cookbook_to_upload.freeze_version if options[:freeze]
-            uploader = Chef::CookbookUploader.new(cookbook_to_upload, force: options[:force], rest: chef_rest)
+            cookbook = other.chef_object if other.chef_object
+            raise Chef::Exceptions::MetadataNotFound.new(cookbook.root_paths[0], cookbook.name) unless cookbook.has_metadata_file?
 
-            with_actual_cookbooks_dir(other.parent.file_path) do
-              uploader.upload_cookbooks
+            if cookbook
+              begin
+                tmp_cl = Chef::CookbookLoader.copy_to_tmp_dir_from_array([cookbook])
+                tmp_cl.load_cookbooks
+                tmp_cl.compile_metadata
+                tmp_cl.freeze_versions if options[:freeze]
+                cookbook_for_upload = []
+                tmp_cl.each do |cookbook_name, cookbook|
+                  cookbook_for_upload << cookbook
+                end
+
+                uploader = Chef::CookbookUploader.new(cookbook_for_upload, force: options[:force], rest: chef_rest)
+
+                with_actual_cookbooks_dir(other.parent.file_path) do
+                  uploader.upload_cookbooks
+                end
+
+              ensure
+                tmp_cl.unlink!
+              end
             end
           end
 

data/lib/chef/cookbook/cookbook_version_loader.rb

@@ -65,19 +65,24 @@ class Chef
       # Load the cookbook. Raises an error if the cookbook_path given to the
       # constructor doesn't point to a valid cookbook.
       def load!
-        file_paths_map = load
+        metadata # force lazy evaluation to occur
+
+        # re-raise any exception that occurred when reading the metadata
+        raise_metadata_error!
+
+        load_all_files
+
+        remove_ignored_files
 
         if empty?
           raise Exceptions::CookbookNotFoundInRepo, "The directory #{cookbook_path} does not contain a cookbook"
         end
 
-        file_paths_map
+        cookbook_settings
       end
 
-      # Load the cookbook. Does not raise an error if given a non-cookbook
-      # directory as the cookbook_path. This behavior is provided for
-      # compatibility, it is recommended to use #load! instead.
       def load
+        Chef::Log.warn "load method is deprecated. Use load! instead"
         metadata # force lazy evaluation to occur
 
         # re-raise any exception that occurred when reading the metadata
@@ -88,8 +93,9 @@ class Chef
         remove_ignored_files
 
         if empty?
-          Chef::Log.warn "Found a directory #{cookbook_name} in the cookbook path, but it contains no cookbook files. skipping."
+          raise Exceptions::CookbookNotFoundInRepo, "The directory #{cookbook_path} does not contain a cookbook"
         end
+
         cookbook_settings
       end
 

data/lib/chef/cookbook_loader.rb

@@ -44,11 +44,14 @@ class Chef
     # @return [Array<String>] the array of repo paths containing cookbook dirs
     attr_reader :repo_paths
 
+    attr_accessor :tmp_working_dir_path
+
     # XXX: this is highly questionable combined with the Hash-style each method
     include Enumerable
 
     # @param repo_paths [Array<String>] the array of repo paths containing cookbook dirs
     def initialize(*repo_paths)
+      @tmp_working_dir_path = nil
       @repo_paths = repo_paths.flatten.compact.map { |p| File.expand_path(p) }
       raise ArgumentError, "You must specify at least one cookbook repo path" if @repo_paths.empty?
     end
@@ -92,11 +95,11 @@ class Chef
 
       loader = cookbook_version_loaders[cookbook_name]
 
-      loader.load
+      loader.load!
 
       cookbook_version = loader.cookbook_version
       cookbooks_by_name[cookbook_name] = cookbook_version
-      metadata[cookbook_name] = cookbook_version.metadata
+      metadata[cookbook_name] = cookbook_version.metadata unless cookbook_version.nil?
       cookbook_version
     end
 
@@ -135,6 +138,56 @@ class Chef
       cookbooks_by_name.values
     end
 
+    # This method creates tmp directory and copies all cookbooks into it and creates cookbook loder object which points to tmp directory
+    def self.copy_to_tmp_dir_from_array(cookbooks)
+      tmp_cookbook_file = Tempfile.new("tmp_working_dir_path")
+      tmp_cookbook_file.close
+      @tmp_working_dir_path = tmp_cookbook_file.path
+      File.unlink(@tmp_working_dir_path)
+      FileUtils.mkdir_p(@tmp_working_dir_path)
+      cookbooks.each do |cookbook|
+        checksums_to_on_disk_paths = cookbook.checksums
+        cookbook.each_file do |manifest_record|
+          path_in_cookbook = manifest_record[:path]
+          on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
+          dest = File.join(@tmp_working_dir_path, cookbook.name.to_s, path_in_cookbook)
+          FileUtils.mkdir_p(File.dirname(dest))
+          FileUtils.cp_r(on_disk_path, dest)
+        end
+      end
+      tmp_cookbook_loader ||= begin
+        Chef::Cookbook::FileVendor.fetch_from_disk(@tmp_working_dir_path)
+        CookbookLoader.new(@tmp_working_dir_path)
+      end
+      tmp_cookbook_loader.tmp_working_dir_path = @tmp_working_dir_path
+      tmp_cookbook_loader
+    end
+
+    # generates metadata.json adds it in the manifest
+    def compile_metadata
+      each do |cookbook_name, cookbook|
+        compiled_metadata = cookbook.compile_metadata
+        if compiled_metadata
+          cookbook.all_files << compiled_metadata
+          cookbook.cookbook_manifest.send(:generate_manifest)
+        end
+      end
+    end
+
+    # freeze versions of all the cookbooks
+    def freeze_versions
+      each do |cookbook_name, cookbook|
+        cookbook.freeze_version
+      end
+    end
+
+    # removes the tmp_dir_path
+    def unlink!
+      raise "Invalid directory path." if @tmp_working_dir_path.nil?
+
+      FileUtils.rm_rf(@tmp_working_dir_path)
+    end
+
     alias :cookbooks :values
 
     private

data/lib/chef/cookbook_uploader.rb

@@ -139,6 +139,8 @@ class Chef
 
     def validate_cookbooks
       cookbooks.each do |cb|
+        next if cb.nil?
+
         syntax_checker = Chef::Cookbook::SyntaxCheck.new(cb.root_dir)
         Chef::Log.info("Validating ruby files")
         exit(1) unless syntax_checker.validate_ruby_files

data/lib/chef/cookbook_version.rb

@@ -445,6 +445,10 @@ class Chef
       end
     end
 
+    def has_metadata_file?
+      all_files.include?(metadata_json_file) || all_files.include?(metadata_rb_file)
+    end
+
     ##
     # REST API
     ##
@@ -513,6 +517,19 @@ class Chef
       @cookbook_manifest ||= CookbookManifest.new(self)
     end
 
+    def compile_metadata(path = root_dir)
+      json_file = "#{path}/metadata.json"
+      rb_file = "#{path}/metadata.rb"
+      return nil if File.exist?(json_file)
+
+      md = Chef::Cookbook::Metadata.new
+      md.from_file(rb_file)
+      f = File.open(json_file, "w")
+      f.write(Chef::JSONCompat.to_json_pretty(md))
+      f.close
+      f.path
+    end
+
     private
 
     def find_preferred_manifest_record(node, segment, filename)

data/lib/chef/dist.rb

@@ -9,13 +9,13 @@ class Chef
 
     # A short designation for the product, used in Windows event logs
     # and some nomenclature.
-    SHORT = "chef".freeze
+    SHORT = ChefConfig::Dist::SHORT.freeze
 
     # The name of the server product
     SERVER_PRODUCT = "Chef Infra Server".freeze
 
     # The client's alias (chef-client)
-    CLIENT = "chef-client".freeze
+    CLIENT = ChefConfig::Dist::CLIENT.freeze
 
     # name of the automate product
     AUTOMATE = "Chef Automate".freeze
@@ -52,7 +52,7 @@ class Chef
     CONF_DIR = ChefConfig::Config.etc_chef_dir.freeze
 
     # The user's configuration directory
-    USER_CONF_DIR = ".chef".freeze
+    USER_CONF_DIR = ChefConfig::Dist::USER_CONF_DIR.freeze
 
     # The server's configuration directory
     SERVER_CONF_DIR = "/etc/chef-server".freeze

data/lib/chef/event_loggers/windows_eventlog.rb

@@ -36,7 +36,7 @@ class Chef
       LOG_CATEGORY_ID = 11001
 
       # Since we must install the event logger, this is not really configurable
-      SOURCE = "#{Chef::Dist::PRODUCT}".freeze
+      SOURCE = Chef::Dist::SHORT.freeze
 
       def self.available?
         ChefUtils.windows?

data/lib/chef/knife/bootstrap.rb

@@ -296,6 +296,11 @@ class Chef
         description: "URL to a custom installation script.",
         proc: Proc.new { |u| Chef::Config[:knife][:bootstrap_url] = u }
 
+      option :bootstrap_product,
+        long: "--bootstrap-product PRODUCT",
+        description: "Product to install.",
+        default: "chef"
+
       option :msi_url, # Windows target only
         short: "-m URL",
         long: "--msi-url URL",

data/lib/chef/knife/bootstrap/templates/chef-full.erb

@@ -177,7 +177,7 @@ do_download() {
   else
     echo "-----> Installing Chef Omnibus (<%= @config[:channel] %>/<%= version_to_install %>)"
     do_download ${install_sh} $tmp_dir/install.sh
-    sh $tmp_dir/install.sh -P chef -c <%= @config[:channel] %> -v <%= version_to_install %>
+    sh $tmp_dir/install.sh -P <%= @config[:bootstrap_product] %> -c <%= @config[:channel] %> -v <%= version_to_install %>
   fi
 <% end %>
 
@@ -185,55 +185,55 @@ if test "x$tmp_dir" != "x"; then
   rm -r "$tmp_dir"
 fi
 
-mkdir -p <%= Chef::Dist::CONF_DIR %>
+mkdir -p /etc/chef
 
 <% if client_pem -%>
-(umask 077 && (cat > <%= Chef::Dist::CONF_DIR %>/client.pem <<'EOP'
+(umask 077 && (cat > /etc/chef/client.pem <<'EOP'
 <%= ::File.read(::File.expand_path(client_pem)) %>
 EOP
 )) || exit 1
 <% end -%>
 
 <% if validation_key -%>
-(umask 077 && (cat > <%= Chef::Dist::CONF_DIR %>/validation.pem <<'EOP'
+(umask 077 && (cat > /etc/chef/validation.pem <<'EOP'
 <%= validation_key %>
 EOP
 )) || exit 1
 <% end -%>
 
 <% if encrypted_data_bag_secret -%>
-(umask 077 && (cat > <%= Chef::Dist::CONF_DIR %>/encrypted_data_bag_secret <<'EOP'
+(umask 077 && (cat > /etc/chef/encrypted_data_bag_secret <<'EOP'
 <%= encrypted_data_bag_secret %>
 EOP
 )) || exit 1
 <% end -%>
 
 <% unless trusted_certs.empty? -%>
-mkdir -p <%= Chef::Dist::CONF_DIR %>/trusted_certs
+mkdir -p /etc/chef/trusted_certs
 <%= trusted_certs %>
 <% end -%>
 
 <%# Generate Ohai Hints -%>
 <% unless @chef_config[:knife][:hints].nil? || @chef_config[:knife][:hints].empty? -%>
-mkdir -p <%= Chef::Dist::CONF_DIR %>/ohai/hints
+mkdir -p /etc/chef/ohai/hints
 
 <% @chef_config[:knife][:hints].each do |name, hash| -%>
-cat > <%= Chef::Dist::CONF_DIR %>/ohai/hints/<%= name %>.json <<'EOP'
+cat > /etc/chef/ohai/hints/<%= name %>.json <<'EOP'
 <%= Chef::JSONCompat.to_json(hash) %>
 EOP
 <% end -%>
 <% end -%>
 
-cat > <%= Chef::Dist::CONF_DIR %>/client.rb <<'EOP'
+cat > /etc/chef/client.rb <<'EOP'
 <%= config_content %>
 EOP
 
-cat > <%= Chef::Dist::CONF_DIR %>/first-boot.json <<'EOP'
+cat > /etc/chef/first-boot.json <<'EOP'
 <%= Chef::JSONCompat.to_json(first_boot) %>
 EOP
 
 <% unless client_d.empty? -%>
-mkdir -p <%= Chef::Dist::CONF_DIR %>/client.d
+mkdir -p /etc/chef/client.d
 <%= client_d %>
 <% end -%>
 

data/lib/chef/knife/cookbook_upload.rb

@@ -20,11 +20,14 @@
 
 require_relative "../knife"
 require_relative "../cookbook_uploader"
+require_relative "../mixin/file_class"
 
 class Chef
   class Knife
     class CookbookUpload < Knife
 
+      include Chef::Mixin::FileClass
+
       CHECKSUM = "checksum".freeze
       MATCH_CHECKSUM = /[0-9a-f]{32,}/.freeze
 
@@ -93,57 +96,84 @@ class Chef
         # to check for the existence of a cookbook's dependencies.
         @server_side_cookbooks = Chef::CookbookVersion.list_all_versions
         justify_width = @server_side_cookbooks.map(&:size).max.to_i + 2
-        if config[:all]
-          cookbook_repo.load_cookbooks
-          cookbooks_for_upload = []
-          cookbook_repo.each do |cookbook_name, cookbook|
-            cookbooks_for_upload << cookbook
-            cookbook.freeze_version if config[:freeze]
-            version_constraints_to_update[cookbook_name] = cookbook.version
-          end
-          if cookbooks_for_upload.any?
-            begin
-              upload(cookbooks_for_upload, justify_width)
-            rescue Exceptions::CookbookFrozen
-              ui.warn("Not updating version constraints for some cookbooks in the environment as the cookbook is frozen.")
-            end
-            ui.info("Uploaded all cookbooks.")
-          else
-            cookbook_path = config[:cookbook_path].respond_to?(:join) ? config[:cookbook_path].join(", ") : config[:cookbook_path]
-            ui.warn("Could not find any cookbooks in your cookbook path: #{cookbook_path}. Use --cookbook-path to specify the desired path.")
+
+        cookbooks = []
+        cookbooks_to_upload.each do |cookbook_name, cookbook|
+          raise Chef::Exceptions::MetadataNotFound.new(cookbook.root_paths[0], cookbook_name) unless cookbook.has_metadata_file?
+
+          if cookbook.metadata.name.nil?
+            message = "Cookbook loaded at path [#{cookbook.root_paths[0]}] has invalid metadata: #{cookbook.metadata.errors.join("; ")}"
+            raise Chef::Exceptions::MetadataNotValid, message
           end
+
+          cookbooks << cookbook
+        end
+
+        if cookbooks.empty?
+          cookbook_path = config[:cookbook_path].respond_to?(:join) ? config[:cookbook_path].join(", ") : config[:cookbook_path]
+          ui.warn("Could not find any cookbooks in your cookbook path: #{cookbook_path}. Use --cookbook-path to specify the desired path.")
         else
-          cookbooks_to_upload.each do |cookbook_name, cookbook|
-            cookbook.freeze_version if config[:freeze]
-            begin
-              upload([cookbook], justify_width)
-              upload_ok += 1
+          begin
+            tmp_cl = Chef::CookbookLoader.copy_to_tmp_dir_from_array(cookbooks)
+            tmp_cl.load_cookbooks
+            tmp_cl.compile_metadata
+            tmp_cl.freeze_versions if config[:freeze]
+
+            cookbooks_for_upload = []
+            tmp_cl.each do |cookbook_name, cookbook|
+              cookbooks_for_upload << cookbook
               version_constraints_to_update[cookbook_name] = cookbook.version
-            rescue Exceptions::CookbookNotFoundInRepo => e
-              upload_failures += 1
-              ui.error("Could not find cookbook #{cookbook_name} in your cookbook path, skipping it")
-              Log.debug(e)
-              upload_failures += 1
-            rescue Exceptions::CookbookFrozen
-              ui.warn("Not updating version constraints for #{cookbook_name} in the environment as the cookbook is frozen.")
-              upload_failures += 1
             end
-          end
+            if config[:all]
+              if cookbooks_for_upload.any?
+                begin
+                  upload(cookbooks_for_upload, justify_width)
+                rescue Chef::Exceptions::CookbookFrozen
+                  ui.warn("Not updating version constraints for some cookbooks in the environment as the cookbook is frozen.")
+                  ui.error("Uploading of some of the cookbooks must be failed. Remove cookbook whose version is frozen from your cookbooks repo OR use --force option.")
+                  upload_failures += 1
+                rescue SystemExit => e
+                  tmp_cl.unlink!
+                  raise exit e.status
+                end
+                ui.info("Uploaded all cookbooks.") if upload_failures == 0
+              end
+            else
+              tmp_cl.each do |cookbook_name, cookbook|
+                begin
+                  upload([cookbook], justify_width)
+                  upload_ok += 1
+                rescue Exceptions::CookbookNotFoundInRepo => e
+                  upload_failures += 1
+                  ui.error("Could not find cookbook #{cookbook_name} in your cookbook path, skipping it")
+                  Log.debug(e)
+                  upload_failures += 1
+                rescue Exceptions::CookbookFrozen
+                  ui.warn("Not updating version constraints for #{cookbook_name} in the environment as the cookbook is frozen.")
+                  upload_failures += 1
+                rescue SystemExit => e
+                  tmp_cl.unlink!
+                  raise exit e.status
+                end
+              end
 
-          if upload_failures == 0
-            ui.info "Uploaded #{upload_ok} cookbook#{upload_ok == 1 ? "" : "s"}."
-          elsif upload_failures > 0 && upload_ok > 0
-            ui.warn "Uploaded #{upload_ok} cookbook#{upload_ok == 1 ? "" : "s"} ok but #{upload_failures} " +
-              "cookbook#{upload_failures == 1 ? "" : "s"} upload failed."
-          elsif upload_failures > 0 && upload_ok == 0
-            ui.error "Failed to upload #{upload_failures} cookbook#{upload_failures == 1 ? "" : "s"}."
-            exit 1
+              if upload_failures == 0
+                ui.info "Uploaded #{upload_ok} cookbook#{upload_ok == 1 ? "" : "s"}."
+              elsif upload_failures > 0 && upload_ok > 0
+                ui.warn "Uploaded #{upload_ok} cookbook#{upload_ok == 1 ? "" : "s"} ok but #{upload_failures} " +
+                  "cookbook#{upload_failures == 1 ? "" : "s"} upload failed."
+              elsif upload_failures > 0 && upload_ok == 0
+                ui.error "Failed to upload #{upload_failures} cookbook#{upload_failures == 1 ? "" : "s"}."
+                exit 1
+              end
+            end
+            unless version_constraints_to_update.empty?
+              update_version_constraints(version_constraints_to_update) if config[:environment]
+            end
+          ensure
+            tmp_cl.unlink!
           end
         end
-
-        unless version_constraints_to_update.empty?
-          update_version_constraints(version_constraints_to_update) if config[:environment]
-        end
       end
 
       def cookbooks_to_upload
@@ -161,7 +191,7 @@ class Chef
                   end
                 end
               rescue Exceptions::CookbookNotFoundInRepo => e
-                ui.error("Could not find cookbook #{cookbook_name} in your cookbook path, skipping it")
+                ui.error(e.message)
                 Log.debug(e)
               end
             end