chef-cli 1.0.3

data/lib/chef-cli/policyfile/cookbook_sources.rb

@@ -0,0 +1,23 @@
+#
+# Copyright:: Copyright (c) 2014-2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "null_cookbook_source"
+require_relative "community_cookbook_source"
+require_relative "chef_server_cookbook_source"
+require_relative "chef_repo_cookbook_source"
+require_relative "delivery_supermarket_source"
+require_relative "artifactory_cookbook_source"

data/lib/chef-cli/policyfile/delivery_supermarket_source.rb

@@ -0,0 +1,89 @@
+#
+# Copyright:: Copyright (c) 2014-2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "forwardable" unless defined?(Forwardable)
+
+require "semverse"
+
+require_relative "community_cookbook_source"
+
+module ChefCLI
+  module Policyfile
+
+    # Fetches cookbooks from a supermarket, similar to CommunityCookbookSource
+    # (which it delegates to), except that only the latest versions of any
+    # cookbook can be used.
+    #
+    # This is intended to be used in an environment where the team wants to
+    # make only the newest version of a given cookbook available in order to
+    # force developers to integrate continuously at the component artifact
+    # (cookbook) level. To achieve this goal, two constraints must be imposed:
+    #
+    # * Cookbook changes pass through a Ci pipeline and are ultimately uploaded
+    #   to a private supermarket (or equivalent, i.e. mini-mart) after final
+    #   approval (which can be automated or not)
+    # * Version numbers for cookbooks that pass through the Ci pipeline always
+    #   increase over time (so that largest version number == newest)
+    #
+    # In the future, alternative approaches may be persued to achieve the goal
+    # of continuously integrating at the cookbook level without imposing those
+    # constraints.
+    #
+    class DeliverySupermarketSource
+
+      extend Forwardable
+
+      def_delegator :@community_source, :uri
+      def_delegator :@community_source, :source_options_for
+      def_delegator :@community_source, :null?
+      def_delegator :@community_source, :preferred_cookbooks
+      def_delegator :@community_source, :preferred_source_for?
+      def_delegator :@community_source, :preferred_for
+
+      def initialize(uri)
+        @community_source = CommunityCookbookSource.new(uri)
+        yield self if block_given?
+      end
+
+      def ==(other)
+        other.kind_of?(self.class) && other.uri == uri
+      end
+
+      def default_source_args
+        [:delivery_supermarket, uri]
+      end
+
+      def universe_graph
+        @universe_graph ||= begin
+          @community_source.universe_graph.inject({}) do |truncated, (cookbook_name, version_and_deps_list)|
+            sorted_versions = version_and_deps_list.keys.sort_by do |version_string|
+              Semverse::Version.new(version_string)
+            end
+            greatest_version = sorted_versions.last
+            truncated[cookbook_name] = { greatest_version => version_and_deps_list[greatest_version] }
+            truncated
+          end
+        end
+      end
+
+      def desc
+        "delivery_supermarket(#{uri})"
+      end
+
+    end
+  end
+end

data/lib/chef-cli/policyfile/differ.rb

@@ -0,0 +1,263 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "diff/lcs"
+require "diff/lcs/hunk"
+require "paint"
+require "ffi_yajl" unless defined?(FFI_Yajl)
+
+module ChefCLI
+  module Policyfile
+    class Differ
+
+      POLICY_SECTIONS = %w{ revision_id run_list named_run_lists cookbook_locks default_attributes override_attributes }.freeze
+      LINES_OF_CONTEXT = 3
+      INITIAL_FILE_LENGTH_DIFFERENCE = 0
+      FORMAT = :unified
+
+      attr_reader :old_lock
+      attr_reader :old_name
+      attr_reader :new_lock
+      attr_reader :new_name
+      attr_reader :ui
+
+      def initialize(old_name: nil, old_lock: nil, new_name: nil, new_lock: nil, ui: nil)
+        @old_lock = old_lock
+        @new_lock = new_lock
+        @old_name = old_name
+        @new_name = new_name
+        @ui = ui
+
+        @added_cookbooks    = nil
+        @removed_cookbooks  = nil
+        @modified_cookbooks = nil
+      end
+
+      def lock_name
+        old_lock["name"]
+      end
+
+      def old_cookbook_locks
+        old_lock["cookbook_locks"]
+      end
+
+      def new_cookbook_locks
+        new_lock["cookbook_locks"]
+      end
+
+      def updated_sections
+        @updated_sections ||= POLICY_SECTIONS.select do |key|
+          old_lock[key] != new_lock[key]
+        end
+      end
+
+      def different?
+        !updated_sections.empty?
+      end
+
+      def run_report
+        unless different?
+          ui.err("No changes for policy lock '#{lock_name}' between '#{old_name}' and '#{new_name}'")
+          return true
+        end
+
+        ui.print("Policy lock '#{lock_name}' differs between '#{old_name}' and '#{new_name}':\n\n")
+
+        report_rev_id_changes
+        report_run_list_changes
+        report_added_cookbooks
+        report_removed_cookbooks
+        report_modified_cookbooks
+        report_default_attribute_changes
+        report_override_attribute_changes
+      end
+
+      def report_rev_id_changes
+        h1("REVISION ID CHANGED")
+        old_rev = old_lock["revision_id"]
+        new_rev = new_lock["revision_id"]
+        diff_lines([ old_rev ], [ new_rev ])
+      end
+
+      def report_run_list_changes
+        return nil unless updated_sections.include?("run_list")
+        h1("RUN LIST CHANGED")
+
+        old_run_list = old_lock["run_list"]
+        new_run_list = new_lock["run_list"]
+
+        diff_lines(old_run_list, new_run_list)
+      end
+
+      def report_removed_cookbooks
+        return nil if removed_cookbooks.empty?
+        h1("REMOVED COOKBOOKS")
+        removed_cookbooks.each do |cb_name|
+          ui.print("\n")
+          old_lock = pretty_json(old_cookbook_locks[cb_name])
+          new_lock = []
+          h2(cb_name)
+          diff_lines(old_lock, new_lock)
+        end
+      end
+
+      def report_added_cookbooks
+        return nil if added_cookbooks.empty?
+        h1("ADDED COOKBOOKS")
+        added_cookbooks.each do |cb_name|
+          ui.print("\n")
+          old_lock = []
+          new_lock = pretty_json(new_cookbook_locks[cb_name])
+          h2(cb_name)
+          diff_lines(old_lock, new_lock)
+        end
+      end
+
+      def report_modified_cookbooks
+        return nil if modified_cookbooks.empty?
+        h1("MODIFIED COOKBOOKS")
+        modified_cookbooks.each do |cb_name|
+          ui.print("\n")
+          old_lock = pretty_json(old_cookbook_locks[cb_name])
+          new_lock = pretty_json(new_cookbook_locks[cb_name])
+          h2(cb_name)
+          diff_lines(old_lock, new_lock)
+        end
+      end
+
+      def report_default_attribute_changes
+        return nil unless updated_sections.include?("default_attributes")
+
+        h1("DEFAULT ATTRIBUTES CHANGED")
+
+        old_default = pretty_json(old_lock["default_attributes"])
+        new_default = pretty_json(new_lock["default_attributes"])
+        diff_lines(old_default, new_default)
+      end
+
+      def report_override_attribute_changes
+        return nil unless updated_sections.include?("override_attributes")
+
+        h1("OVERRIDE ATTRIBUTES CHANGED")
+
+        old_override = pretty_json(old_lock["override_attributes"])
+        new_override = pretty_json(new_lock["override_attributes"])
+        diff_lines(old_override, new_override)
+      end
+
+      def added_cookbooks
+        detect_cookbook_changes if @added_cookbooks.nil?
+        @added_cookbooks
+      end
+
+      def removed_cookbooks
+        detect_cookbook_changes if @removed_cookbooks.nil?
+        @removed_cookbooks
+      end
+
+      def modified_cookbooks
+        detect_cookbook_changes if @modified_cookbooks.nil?
+        @modified_cookbooks
+      end
+
+      private
+
+      def h1(str)
+        ui.msg(str)
+        ui.msg("=" * str.size)
+      end
+
+      def h2(str)
+        ui.msg(str)
+        ui.msg("-" * str.size)
+      end
+
+      def diff_lines(old_lines, new_lines)
+        file_length_difference = INITIAL_FILE_LENGTH_DIFFERENCE
+
+        previous_hunk = nil
+
+        diffs = Diff::LCS.diff(old_lines, new_lines)
+
+        ui.print("\n")
+
+        diffs.each do |piece|
+          hunk = Diff::LCS::Hunk.new(old_lines, new_lines, piece, LINES_OF_CONTEXT, file_length_difference)
+
+          file_length_difference = hunk.file_length_difference
+
+          maybe_contiguous_hunks = (previous_hunk.nil? || hunk.merge(previous_hunk))
+
+          if !maybe_contiguous_hunks
+            print_color_diff("#{previous_hunk.diff(FORMAT)}\n")
+          end
+          previous_hunk = hunk
+        end
+        print_color_diff("#{previous_hunk.diff(FORMAT)}\n") unless previous_hunk.nil?
+        ui.print("\n")
+      end
+
+      def print_color_diff(hunk)
+        hunk.to_s.each_line do |line|
+          ui.print(Paint[line, color_for_line(line)])
+        end
+      end
+
+      def color_for_line(line)
+        case line[0].chr
+        when "+"
+          :green
+        when "-"
+          :red
+        when "@"
+          line[1].chr == "@" ? :blue : nil
+        else
+          nil
+        end
+      end
+
+      def pretty_json(data)
+        FFI_Yajl::Encoder.encode(data, pretty: true).lines.map { |l| l.chomp }
+      end
+
+      def detect_cookbook_changes
+        all_locked_cookbooks = old_cookbook_locks.keys | new_cookbook_locks.keys
+
+        @added_cookbooks = []
+        @removed_cookbooks = []
+        @modified_cookbooks = []
+
+        all_locked_cookbooks.each do |cb_name|
+          if old_cookbook_locks.key?(cb_name) && new_cookbook_locks.key?(cb_name)
+            old_cb_lock = old_cookbook_locks[cb_name]
+            new_cb_lock = new_cookbook_locks[cb_name]
+            if old_cb_lock != new_cb_lock
+              @modified_cookbooks << cb_name
+            end
+          elsif old_cookbook_locks.key?(cb_name)
+            @removed_cookbooks << cb_name
+          elsif new_cookbook_locks.key?(cb_name)
+            @added_cookbooks << cb_name
+          else
+            raise "Bug: cookbook lock #{cb_name} cannot be determined as new/removed/modified/unmodified"
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/chef-cli/policyfile/dsl.rb

@@ -0,0 +1,288 @@
+#
+# Copyright:: Copyright (c) 2014-2019, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "cookbook_sources"
+require_relative "cookbook_location_specification"
+require_relative "storage_config"
+require_relative "policyfile_location_specification"
+
+require "chef/node/attribute"
+require "chef/run_list/run_list_item"
+
+module ChefCLI
+  module Policyfile
+    class DSL
+
+      RUN_LIST_ITEM_COMPONENT = %r{^[.[:alnum:]_-]+$}.freeze
+
+      include StorageConfigDelegation
+
+      attr_writer :name
+
+      attr_reader :errors
+      attr_writer :run_list
+      attr_writer :default_source
+      attr_reader :cookbook_location_specs
+      attr_reader :included_policies
+
+      attr_reader :named_run_lists
+      attr_reader :node_attributes
+
+      attr_reader :storage_config
+
+      attr_reader :chef_config
+      def initialize(storage_config, chef_config: nil)
+        @name = nil
+        @errors = []
+        @run_list = []
+        @named_run_lists = {}
+        @included_policies = []
+        @default_source = [ NullCookbookSource.new ]
+        @cookbook_location_specs = {}
+        @storage_config = storage_config
+        @chef_config = chef_config
+
+        @node_attributes = Chef::Node::Attribute.new({}, {}, {}, {})
+      end
+
+      def name(name = nil)
+        unless name.nil?
+          @name = name
+        end
+        @name
+      end
+
+      def run_list(*run_list_items)
+        run_list_items = run_list_items.flatten
+        unless run_list_items.empty?
+          validate_run_list_items(run_list_items)
+          @run_list = run_list_items
+        end
+        @run_list
+      end
+
+      def named_run_list(name, *run_list_items)
+        run_list_items = run_list_items.flatten
+        unless run_list_items.empty?
+          validate_run_list_items(run_list_items, name)
+          @named_run_lists[name] = run_list_items
+        end
+        @named_run_lists[name]
+      end
+
+      def default_source(source_type = nil, source_argument = nil, &block)
+        return @default_source if source_type.nil?
+        case source_type
+        when :community, :supermarket
+          set_default_community_source(source_argument, &block)
+        when :delivery_supermarket
+          set_default_delivery_supermarket_source(source_argument, &block)
+        when :chef_server
+          set_default_chef_server_source(source_argument, &block)
+        when :chef_repo
+          set_default_chef_repo_source(source_argument, &block)
+        when :artifactory
+          set_default_artifactory_source(source_argument, &block)
+        else
+          @errors << "Invalid default_source type '#{source_type.inspect}'"
+        end
+      end
+
+      def cookbook(name, *version_and_source_opts)
+        source_options =
+          if version_and_source_opts.last.is_a?(Hash)
+            version_and_source_opts.pop
+          else
+            {}
+          end
+
+        constraint = version_and_source_opts.first || ">= 0.0.0"
+        spec = CookbookLocationSpecification.new(name, constraint, source_options, storage_config)
+
+        if ( existing_source = @cookbook_location_specs[name] )
+          err = "Cookbook '#{name}' assigned to conflicting sources\n\n"
+          err << "Previous source: #{existing_source.source_options.inspect}\n"
+          err << "Conflicts with: #{source_options.inspect}\n"
+          @errors << err
+        else
+          @cookbook_location_specs[name] = spec
+          @errors += spec.errors
+        end
+      end
+
+      def include_policy(name, source_options = {})
+        if ( existing = included_policies.find { |p| p.name == name } )
+          err = "Included policy '#{name}' assigned conflicting locations or was already specified\n\n"
+          err << "Previous source: #{existing.source_options.inspect}\n"
+          err << "Conflicts with: #{source_options.inspect}\n"
+          @errors << err
+        else
+          spec = PolicyfileLocationSpecification.new(name, source_options, storage_config, chef_config)
+          included_policies << spec
+          @errors += spec.errors
+        end
+      end
+
+      def default
+        @node_attributes.default
+      end
+
+      def override
+        @node_attributes.override
+      end
+
+      def eval_policyfile(policyfile_string)
+        @policyfile_filename = policyfile_filename
+        instance_eval(policyfile_string, policyfile_filename)
+        validate!
+        self
+      rescue SyntaxError => e
+        @errors << "Invalid Ruby syntax in Policyfile '#{policyfile_filename}':\n\n#{e.message}"
+      rescue SignalException, SystemExit
+        # allow signal from kill, ctrl-C, etc. to bubble up:
+        raise
+      rescue Exception => e
+        error_message = "Evaluation of policyfile '#{policyfile_filename}' raised an exception\n"
+        error_message << "  Exception: #{e.class.name} \"#{e}\"\n\n"
+        trace = filtered_bt(policyfile_filename, e)
+        error_message << "  Relevant Code:\n"
+        error_message << "    #{error_context(policyfile_string, policyfile_filename, e)}\n\n"
+        unless trace.empty?
+          error_message << "  Backtrace:\n"
+          # TODO: need a way to disable filtering
+          error_message << filtered_bt(policyfile_filename, e).inject("") { |formatted_trace, line| formatted_trace << "    #{line}\n" }
+        end
+        @errors << error_message
+      end
+
+      private
+
+      def set_default_community_source(source_uri, &block)
+        set_default_source(CommunityCookbookSource.new(source_uri, &block))
+      end
+
+      def set_default_delivery_supermarket_source(source_uri, &block)
+        if source_uri.nil?
+          @errors << "You must specify the server's URI when using a default_source :delivery_supermarket"
+        else
+          set_default_source(DeliverySupermarketSource.new(source_uri, &block))
+        end
+      end
+
+      def set_default_chef_server_source(source_uri, &block)
+        if source_uri.nil?
+          @errors << "You must specify the server's URI when using a default_source :chef_server"
+        else
+          set_default_source(ChefServerCookbookSource.new(source_uri, chef_config: chef_config, &block))
+        end
+      end
+
+      def set_default_artifactory_source(source_uri, &block)
+        if source_uri.nil?
+          @errors << "You must specify the server's URI when using a default_source :artifactory"
+        else
+          set_default_source(ArtifactoryCookbookSource.new(source_uri, chef_config: chef_config, &block))
+        end
+      end
+
+      def set_default_chef_repo_source(path, &block)
+        if path.nil?
+          @errors << "You must specify the path to the chef-repo when using a default_source :chef_repo"
+        else
+          set_default_source(ChefRepoCookbookSource.new(File.expand_path(path, storage_config.relative_paths_root), &block))
+        end
+      end
+
+      def set_default_source(source)
+        @default_source.delete_at(0) if @default_source[0].null?
+        @default_source << source
+      end
+
+      def validate!
+        if @run_list.empty?
+          @errors << "Invalid run_list. run_list cannot be empty"
+        end
+
+        handle_preferred_cookbooks_conflicts
+      end
+
+      def validate_run_list_items(items, run_list_name = nil)
+        items.each do |item|
+
+          run_list_desc = run_list_name.nil? ? "Run List Item '#{item}'" : "Named Run List '#{run_list_name}' Item '#{item}'"
+
+          item_name = Chef::RunList::RunListItem.new(item).name
+          cookbook, separator, recipe = item_name.partition("::")
+
+          if RUN_LIST_ITEM_COMPONENT.match(cookbook).nil?
+            message = "#{run_list_desc} has invalid cookbook name '#{cookbook}'.\nCookbook names can only contain alphanumerics, hyphens, and underscores."
+
+            # Special case when there's only one colon instead of two:
+            if cookbook =~ /[^:]:[^:]/
+              message << "\nDid you mean '#{item.sub(":", "::")}'?"
+            end
+
+            @errors << message
+          end
+          unless separator.empty?
+            # we have a cookbook and recipe
+            if RUN_LIST_ITEM_COMPONENT.match(recipe).nil?
+              @errors << "#{run_list_desc} has invalid recipe name '#{recipe}'.\nRecipe names can only contain alphanumerics, hyphens, and underscores."
+            end
+          end
+        end
+      end
+
+      def handle_preferred_cookbooks_conflicts
+        conflicting_source_messages = []
+        default_source.combination(2).each do |source_a, source_b|
+          conflicting_preferences = source_a.preferred_cookbooks & source_b.preferred_cookbooks
+          next if conflicting_preferences.empty?
+          conflicting_source_messages << "#{source_a.desc} and #{source_b.desc} are both set as the preferred source for cookbook(s) '#{conflicting_preferences.join(', ')}'"
+        end
+        unless conflicting_source_messages.empty?
+          msg = "Multiple sources are marked as the preferred source for some cookbooks. Only one source can be preferred for a cookbook.\n"
+          msg << conflicting_source_messages.join("\n") << "\n"
+          @errors << msg
+        end
+      end
+
+      def error_context(policyfile_string, policyfile_filename, exception)
+        if ( line_number_to_show = culprit_line_number(policyfile_filename, exception) )
+          code = policyfile_string.lines.to_a[line_number_to_show - 1].strip
+          "#{line_number_to_show}: #{code}"
+        else
+          "Could not find relevant code from backtrace"
+        end
+      end
+
+      def culprit_line_number(policyfile_filename, exception)
+        if ( most_proximate_backtrace_line = filtered_bt(policyfile_filename, exception).first )
+          most_proximate_backtrace_line[/^(?:.\:)?[^:]+:([\d]+)/, 1].to_i
+        else
+          nil
+        end
+      end
+
+      def filtered_bt(policyfile_filename, exception)
+        policyfile_filename_matcher = /^#{Regexp.escape(policyfile_filename)}/
+        exception.backtrace.select { |line| line =~ policyfile_filename_matcher }
+      end
+
+    end
+  end
+end