chef 12.0.3-x86-mingw32 → 12.1.0.rc.0-x86-mingw32

data/lib/chef/audit/rspec_formatter.rb

@@ -0,0 +1,37 @@
+#
+# Author:: Serdar Sutay (<serdar@chef.io>)
+# Copyright:: Copyright (c) 2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'rspec/core'
+
+class Chef
+  class Audit
+    class RspecFormatter < RSpec::Core::Formatters::DocumentationFormatter
+      RSpec::Core::Formatters.register self, :close
+
+      # @api public
+      #
+      # Invoked at the very end, `close` allows the formatter to clean
+      # up resources, e.g. open streams, etc.
+      #
+      # @param _notification [NullNotification] (Ignored)
+      def close(_notification)
+        # Normally Rspec closes the streams it's given. We don't want it for Chef.
+      end
+    end
+  end
+end

data/lib/chef/audit/runner.rb

@@ -0,0 +1,178 @@
+#
+# Author:: Claire McQuin (<claire@getchef.com>)
+# Copyright:: Copyright (c) 2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Chef
+  class Audit
+    class Runner
+
+      attr_reader :run_context
+      private :run_context
+
+      def initialize(run_context)
+        @run_context = run_context
+      end
+
+      def run
+        setup
+        register_control_groups
+        do_run
+      end
+
+      def failed?
+        RSpec.world.reporter.failed_examples.size > 0
+      end
+
+      def num_failed
+        RSpec.world.reporter.failed_examples.size
+      end
+
+      def num_total
+        RSpec.world.reporter.examples.size
+      end
+
+      private
+      # Prepare to run audits:
+      #  - Require files
+      #  - Configure RSpec
+      #  - Configure Specinfra/Serverspec
+      def setup
+        require_deps
+        configure_rspec
+        configure_specinfra
+      end
+
+      # RSpec uses a global configuration object, RSpec.configuration. We found
+      # there was interference between the configuration for audit-mode and
+      # the configuration for our own spec tests in these cases:
+      #   1. Specinfra and Serverspec modify RSpec.configuration when loading.
+      #   2. Setting output/error streams.
+      #   3. Adding formatters.
+      #   4. Defining example group aliases.
+      #
+      # Moreover, Serverspec loads its DSL methods into the global namespace,
+      # which causes conflicts with the Chef namespace for resources and packages.
+      #
+      # We wait until we're in the audit-phase of the chef-client run to load
+      # these files. This helps with the namespacing problems we saw, and
+      # prevents Specinfra and Serverspec from modifying the RSpec configuration
+      # used by our spec tests.
+      def require_deps
+        require 'rspec'
+        require 'rspec/its'
+        require 'specinfra'
+        require 'serverspec/helper'
+        require 'serverspec/matcher'
+        require 'serverspec/subject'
+        require 'chef/audit/audit_event_proxy'
+        require 'chef/audit/rspec_formatter'
+      end
+
+      # Configure RSpec just the way we like it:
+      #   - Set location of error and output streams
+      #   - Add custom audit-mode formatters
+      #   - Explicitly disable :should syntax
+      #   - Set :color option according to chef config
+      #   - Disable exposure of global DSL
+      def configure_rspec
+        set_streams
+        add_formatters
+        disable_should_syntax
+
+        RSpec.configure do |c|
+          c.color = Chef::Config[:color]
+          c.expose_dsl_globally = false
+        end
+      end
+
+      # Set the error and output streams which audit-mode will use to report
+      # human-readable audit information.
+      #
+      # This should always be called before #add_formatters. RSpec won't allow
+      # the output stream to be changed for a formatter once the formatter has
+      # been added.
+      def set_streams
+        RSpec.configuration.output_stream = Chef::Config[:log_location]
+        RSpec.configuration.error_stream = Chef::Config[:log_location]
+      end
+
+      # Add formatters which we use to
+      #   1. Output human-readable data to the output stream,
+      #   2. Collect JSON data to send back to the analytics server.
+      def add_formatters
+        RSpec.configuration.add_formatter(Chef::Audit::AuditEventProxy)
+        RSpec.configuration.add_formatter(Chef::Audit::RspecFormatter)
+        Chef::Audit::AuditEventProxy.events = run_context.events
+      end
+
+      # Audit-mode uses RSpec 3. :should syntax is deprecated by default in
+      # RSpec 3, so we explicitly disable it here.
+      #
+      # This can be removed once :should is removed from RSpec.
+      def disable_should_syntax
+        RSpec.configure do |config|
+          config.expect_with :rspec do |c|
+            c.syntax = :expect
+          end
+        end
+      end
+
+      # Set up the backend for Specinfra/Serverspec.  :exec is the local system.
+      def configure_specinfra
+        Specinfra.configuration.backend = :exec
+      end
+
+      # Iterates through the control groups registered to this run_context, builds an
+      # example group (RSpec::Core::ExampleGroup) object per control group, and
+      # registers the group with the RSpec.world.
+      #
+      # We could just store an array of example groups and not use RSpec.world,
+      # but it may be useful later if we decide to apply our own ordering scheme
+      # or use example group filters.
+      def register_control_groups
+        add_example_group_methods
+        run_context.audits.each do |name, group|
+          ctl_grp = RSpec::Core::ExampleGroup.__control_group__(*group.args, &group.block)
+          RSpec.world.register(ctl_grp)
+        end
+      end
+
+      # Add example group method aliases to RSpec.
+      #
+      # __control_group__: Used internally to create example groups from the control
+      #               groups saved in the run_context.
+      #      control: Used within the context of a control group block, like RSpec's
+      #               describe or context.
+      def add_example_group_methods
+        RSpec::Core::ExampleGroup.define_example_group_method :__control_group__
+        RSpec::Core::ExampleGroup.define_example_group_method :control
+      end
+
+      # Run the audits!
+      def do_run
+        # RSpec::Core::Runner wants to be initialized with an
+        # RSpec::Core::ConfigurationOptions object, which is used to process
+        # command line configuration arguments. We directly fiddle with the
+        # internal RSpec configuration object, so we give nil here and let
+        # RSpec pick up its own configuration and world.
+        runner = RSpec::Core::Runner.new(nil)
+        runner.run_specs(RSpec.world.ordered_example_groups)
+      end
+
+    end
+  end
+end

data/lib/chef/chef_fs/chef_fs_data_store.rb

@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 
+require 'chef/cookbook_manifest'
 require 'chef_zero/data_store/memory_store'
 require 'chef_zero/data_store/data_already_exists_error'
 require 'chef_zero/data_store/data_not_found_error'
@@ -147,7 +148,7 @@ class Chef
               # get /cookbooks/NAME/version
               result = nil
               begin
-                result = entry.chef_object.to_hash
+                result = Chef::CookbookManifest.new(entry.chef_object).to_hash
               rescue Chef::ChefFS::FileSystem::NotFoundError => e
                 raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
               end
@@ -369,6 +370,11 @@ class Chef
           if path.length >= 3
             path[2] = "#{path[2]}.json"
           end
+        elsif path[0] == 'policies'
+          path = path.dup
+          if path.length >= 3
+            path[2] = "#{path[2]}.json"
+          end
         elsif path[0] == 'cookbooks'
           if path.length == 2
             raise ChefZero::DataStore::DataNotFoundError.new(path)
@@ -445,10 +451,13 @@ class Chef
       def with_dir(path)
         # Do not automatically create data bags
         create = !(path[0] == 'data' && path.size >= 2)
+
         begin
           yield get_dir(_to_chef_fs_path(path), create)
         rescue Chef::ChefFS::FileSystem::NotFoundError => e
-          raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+          err = ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+          err.set_backtrace(e.backtrace)
+          raise err
         end
       end
 

data/lib/chef/chef_fs/config.rb

@@ -26,6 +26,25 @@ class Chef
     # objects representing the server and local repository, respectively).
     #
     class Config
+
+      # Not all of our object types pluralize by adding an 's', so we map them
+      # out here:
+      INFLECTIONS = {
+        "acls" => "acl",
+        "clients" => "client",
+        "cookbooks" => "cookbook",
+        "containers" => "container",
+        "data_bags" => "data_bag",
+        "environments" => "environment",
+        "groups" => "group",
+        "nodes" => "node",
+        "roles" => "role",
+        "users" => "user",
+        "policies" => "policy"
+      }
+      INFLECTIONS.each { |k,v| k.freeze; v.freeze }
+      INFLECTIONS.freeze
+
       #
       # Create a new Config object which can produce a chef_fs and local_fs.
       #
@@ -215,14 +234,16 @@ class Chef
           result = {}
           case @chef_config[:repo_mode]
           when 'static'
-            object_names = %w(cookbooks data_bags environments roles)
+            object_names = %w(cookbooks data_bags environments roles policies)
           when 'hosted_everything'
-            object_names = %w(acls clients cookbooks containers data_bags environments groups nodes roles)
+            object_names = %w(acls clients cookbooks containers data_bags environments groups nodes roles policies)
           else
-            object_names = %w(clients cookbooks data_bags environments nodes roles users)
+            object_names = %w(clients cookbooks data_bags environments nodes roles users policies)
           end
           object_names.each do |object_name|
-            variable_name = "#{object_name[0..-2]}_path" # cookbooks -> cookbook_path
+            # cookbooks -> cookbook_path
+            singular_name = INFLECTIONS[object_name] or raise "Unknown object name #{object_name}"
+            variable_name = "#{singular_name}_path"
             paths = Array(@chef_config[variable_name]).flatten
             result[object_name] = paths.map { |path| File.expand_path(path) }
           end

data/lib/chef/chef_fs/data_handler/policy_data_handler.rb

@@ -0,0 +1,15 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class PolicyDataHandler < DataHandlerBase
+
+        def normalize(policy, entry)
+          policy
+        end
+      end
+    end
+  end
+end
+

data/lib/chef/chef_fs/data_handler/user_data_handler.rb

@@ -8,6 +8,7 @@ class Chef
           normalize_hash(user, {
             'name' => remove_dot_json(entry.name),
             'username' => remove_dot_json(entry.name),
+            'display_name' => remove_dot_json(entry.name),
             'admin' => false,
             'json_class' => 'Chef::WebUIUser',
             'chef_type' => 'webui_user',

data/lib/chef/chef_fs/file_system.rb

@@ -379,7 +379,7 @@ class Chef
                   should_copy = true
                   src_value = nil
                 else
-                  are_same, src_value, dest_value = compare(src_entry, dest_entry)
+                  are_same, src_value, _dest_value = compare(src_entry, dest_entry)
                   should_copy = !are_same
                 end
                 if should_copy

data/lib/chef/chef_fs/file_system/base_fs_dir.rb

@@ -40,6 +40,11 @@ class Chef
           true
         end
 
+        # An empty children array is an empty dir
+        def empty?
+          children.empty?
+        end
+
         # Abstract: children
       end
     end

data/lib/chef/chef_fs/file_system/chef_repository_file_system_cookbook_entry.rb

@@ -53,7 +53,7 @@ class Chef
 
           # Check chefignore
           ignorer = parent
-          begin
+          loop do
             if ignorer.is_a?(ChefRepositoryFileSystemCookbooksDir)
               # Grab the path from entry to child
               path_to_child = name
@@ -66,7 +66,8 @@ class Chef
               return !ignorer.chefignore || !ignorer.chefignore.ignored?(path_to_child)
             end
             ignorer = ignorer.parent
-          end while ignorer
+            break unless ignorer
+          end
 
           true
         end

data/lib/chef/{shef/ext.rb → chef_fs/file_system/chef_repository_file_system_policies_dir.rb}

@@ -1,5 +1,5 @@
-#--
-# Author:: Joshua Timberman (<joshua@opscode.com>)
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
 # Copyright:: Copyright (c) 2012 Opscode, Inc.
 # License:: Apache License, Version 2.0
 #
@@ -16,4 +16,23 @@
 # limitations under the License.
 #
 
-require 'chef/shell/ext'
+require 'chef/chef_fs/file_system/chef_repository_file_system_entry'
+require 'chef/chef_fs/data_handler/policy_data_handler'
+
+class Chef
+  module ChefFS
+    module FileSystem
+
+      class ChefRepositoryFileSystemPoliciesDir < ChefRepositoryFileSystemEntry
+        def initialize(name, parent, path = nil)
+          super(name, parent, path, Chef::ChefFS::DataHandler::PolicyDataHandler.new)
+        end
+
+        def can_have_child?(name, is_dir)
+          is_dir && !name.start_with?('.')
+        end
+      end
+    end
+  end
+end
+

data/lib/chef/chef_fs/file_system/chef_repository_file_system_root_dir.rb

@@ -21,6 +21,7 @@ require 'chef/chef_fs/file_system/chef_repository_file_system_entry'
 require 'chef/chef_fs/file_system/chef_repository_file_system_acls_dir'
 require 'chef/chef_fs/file_system/chef_repository_file_system_cookbooks_dir'
 require 'chef/chef_fs/file_system/chef_repository_file_system_data_bags_dir'
+require 'chef/chef_fs/file_system/chef_repository_file_system_policies_dir'
 require 'chef/chef_fs/file_system/multiplexed_dir'
 require 'chef/chef_fs/data_handler/client_data_handler'
 require 'chef/chef_fs/data_handler/environment_data_handler'
@@ -33,6 +34,7 @@ require 'chef/chef_fs/data_handler/container_data_handler'
 class Chef
   module ChefFS
     module FileSystem
+
       #
       # Represents the root of a local Chef repository, with directories for
       # nodes, cookbooks, roles, etc. under it.
@@ -157,6 +159,8 @@ class Chef
             dirs = paths.map { |path| ChefRepositoryFileSystemCookbooksDir.new(name, self, path) }
           elsif name == 'data_bags'
             dirs = paths.map { |path| ChefRepositoryFileSystemDataBagsDir.new(name, self, path) }
+          elsif name == 'policies'
+            dirs = paths.map { |path| ChefRepositoryFileSystemPoliciesDir.new(name, self, path) }
           elsif name == 'acls'
             dirs = paths.map { |path| ChefRepositoryFileSystemAclsDir.new(name, self, path) }
           else

data/lib/chef/client.rb

@@ -25,6 +25,7 @@ require 'chef/log'
 require 'chef/rest'
 require 'chef/api_client'
 require 'chef/api_client/registration'
+require 'chef/audit/runner'
 require 'chef/node'
 require 'chef/role'
 require 'chef/file_cache'
@@ -38,11 +39,13 @@ require 'chef/cookbook/remote_file_vendor'
 require 'chef/event_dispatch/dispatcher'
 require 'chef/event_loggers/base'
 require 'chef/event_loggers/windows_eventlog'
+require 'chef/exceptions'
 require 'chef/formatters/base'
 require 'chef/formatters/doc'
 require 'chef/formatters/minimal'
 require 'chef/version'
 require 'chef/resource_reporter'
+require 'chef/audit/audit_reporter'
 require 'chef/run_lock'
 require 'chef/policy_builder'
 require 'chef/request_id'
@@ -209,6 +212,17 @@ class Chef
       end
     end
 
+    # Resource repoters send event information back to the chef server for processing.
+    # Can only be called after we have a @rest object
+    def register_reporters
+      [
+        Chef::ResourceReporter.new(rest),
+        Chef::Audit::AuditReporter.new(rest)
+      ].each do |r|
+        events.register(r)
+      end
+    end
+
     # Instantiates a Chef::Node object, possibly loading the node's prior state
     # when using chef-client. Delegates to policy_builder
     #
@@ -246,7 +260,6 @@ class Chef
       @policy_builder ||= Chef::PolicyBuilder.strategy.new(node_name, ohai.data, json_attribs, @override_runlist, events)
     end
 
-
     def save_updated_node
       if Chef::Config[:solo]
         # nothing to do
@@ -260,6 +273,7 @@ class Chef
 
     def run_ohai
       ohai.all_plugins
+      @events.ohai_completed(node)
     end
 
     def node_name
@@ -295,8 +309,7 @@ class Chef
       end
       # We now have the client key, and should use it from now on.
       @rest = Chef::REST.new(config[:chef_server_url], client_name, config[:client_key])
-      @resource_reporter = Chef::ResourceReporter.new(@rest)
-      @events.register(@resource_reporter)
+      register_reporters
     rescue Exception => e
       # TODO: munge exception so a semantic failure message can be given to the
       # user
@@ -307,18 +320,56 @@ class Chef
     # Converges the node.
     #
     # === Returns
-    # true:: Always returns true
+    # The thrown exception, if there was one.  If this returns nil the converge was successful.
     def converge(run_context)
-      @events.converge_start(run_context)
-      Chef::Log.debug("Converging node #{node_name}")
-      @runner = Chef::Runner.new(run_context)
-      runner.converge
-      @events.converge_complete
-      true
-    rescue Exception
-      # TODO: should this be a separate #converge_failed(exception) method?
-      @events.converge_complete
-      raise
+      converge_exception = nil
+      catch(:end_client_run_early) do
+        begin
+          @events.converge_start(run_context)
+          Chef::Log.debug("Converging node #{node_name}")
+          @runner = Chef::Runner.new(run_context)
+          runner.converge
+          @events.converge_complete
+        rescue Exception => e
+          Chef::Log.error("Converge failed with error message #{e.message}")
+          @events.converge_failed(e)
+          converge_exception = e
+        end
+      end
+      converge_exception
+    end
+
+    # We don't want to change the old API on the `converge` method to have it perform
+    # saving.  So we wrap it in this method.
+    def converge_and_save(run_context)
+      converge_exception = converge(run_context)
+      unless converge_exception
+        begin
+          save_updated_node
+        rescue Exception => e
+          converge_exception = e
+        end
+      end
+      converge_exception
+    end
+
+    def run_audits(run_context)
+      audit_exception = nil
+      begin
+        @events.audit_phase_start(run_status)
+        Chef::Log.info("Starting audit phase")
+        auditor = Chef::Audit::Runner.new(run_context)
+        auditor.run
+        if auditor.failed?
+          raise Chef::Exceptions::AuditsFailed.new(auditor.num_failed, auditor.num_total)
+        end
+        @events.audit_phase_complete
+      rescue Exception => e
+        Chef::Log.error("Audit phase failed with error message: #{e.message}")
+        @events.audit_phase_failed(e)
+        audit_exception = e
+      end
+      audit_exception
     end
 
     # Expands the run list. Delegates to the policy_builder.
@@ -333,7 +384,6 @@ class Chef
       policy_builder.expand_run_list
     end
 
-
     def do_windows_admin_check
       if Chef::Platform.windows?
         Chef::Log.debug("Checking for administrator privileges....")
@@ -370,8 +420,6 @@ class Chef
       begin
         runlock.save_pid
 
-        check_ssl_config
-
         request_id = Chef::RequestID.instance.request_id
         run_context = nil
         @events.run_start(Chef::VERSION)
@@ -380,7 +428,7 @@ class Chef
         Chef::Log.debug("Chef-client request_id: #{request_id}")
         enforce_path_sanity
         run_ohai
-        @events.ohai_completed(node)
+
         register unless Chef::Config[:solo]
 
         load_node
@@ -396,11 +444,22 @@ class Chef
 
         run_context = setup_run_context
 
-        catch(:end_client_run_early) do
-          converge(run_context)
+        if Chef::Config[:audit_mode] != :audit_only
+          converge_error = converge_and_save(run_context)
+        end
+
+        if Chef::Config[:why_run] == true
+          # why_run should probably be renamed to why_converge
+          Chef::Log.debug("Not running audits in 'why_run' mode - this mode is used to see potential converge changes")
+        elsif Chef::Config[:audit_mode] != :disabled
+          audit_error = run_audits(run_context)
         end
 
-        save_updated_node
+        if converge_error || audit_error
+          e = Chef::Exceptions::RunFailedWrappingError.new(converge_error, audit_error)
+          e.fill_backtrace
+          raise e
+        end
 
         run_status.stop_clock
         Chef::Log.info("Chef Run complete in #{run_status.elapsed_time} seconds")
@@ -411,6 +470,7 @@ class Chef
         Chef::Platform::Rebooter.reboot_if_needed!(node)
 
         true
+
       rescue Exception => e
         # CHEF-3336: Send the error first in case something goes wrong below and we don't know why
         Chef::Log.debug("Re-raising exception: #{e.class} - #{e.message}\n#{e.backtrace.join("\n  ")}")
@@ -468,37 +528,6 @@ class Chef
       Chef::ReservedNames::Win32::Security.has_admin_privileges?
     end
 
-    def check_ssl_config
-      if Chef::Config[:ssl_verify_mode] == :verify_none and !Chef::Config[:verify_api_cert]
-        Chef::Log.warn(<<-WARN)
-
-* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
-SSL validation of HTTPS requests is disabled. HTTPS connections are still
-encrypted, but chef is not able to detect forged replies or man in the middle
-attacks.
-
-To fix this issue add an entry like this to your configuration file:
-
-```
-  # Verify all HTTPS connections (recommended)
-  ssl_verify_mode :verify_peer
-
-  # OR, Verify only connections to chef-server
-  verify_api_cert true
-```
-
-To check your SSL configuration, or troubleshoot errors, you can use the
-`knife ssl check` command like so:
-
-```
-  knife ssl check -c #{Chef::Config.config_file}
-```
-
-* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
-WARN
-      end
-    end
-
   end
 end