chef-dk 0.5.0.rc.1 → 0.5.0

data/lib/chef-dk/policyfile/cookbook_sources.rb

@@ -18,3 +18,4 @@
 require 'chef-dk/policyfile/null_cookbook_source'
 require 'chef-dk/policyfile/community_cookbook_source'
 require 'chef-dk/policyfile/chef_server_cookbook_source'
+require 'chef-dk/policyfile/chef_repo_cookbook_source'

data/lib/chef-dk/policyfile/differ.rb

@@ -0,0 +1,266 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'diff/lcs'
+require 'diff/lcs/hunk'
+require 'paint'
+require 'ffi_yajl'
+
+module ChefDK
+  module Policyfile
+    class Differ
+
+      POLICY_SECTIONS = %w{ revision_id run_list named_run_lists cookbook_locks default_attributes override_attributes }.freeze
+      LINES_OF_CONTEXT = 3
+      INITIAL_FILE_LENGTH_DIFFERENCE = 0
+      FORMAT = :unified
+
+
+      attr_reader :old_lock
+      attr_reader :old_name
+      attr_reader :new_lock
+      attr_reader :new_name
+      attr_reader :ui
+
+      def initialize(old_name: nil, old_lock: nil, new_name: nil, new_lock: nil, ui: nil)
+        @old_lock = old_lock
+        @new_lock = new_lock
+        @old_name = old_name
+        @new_name = new_name
+        @ui = ui
+
+        @added_cookbooks    = nil
+        @removed_cookbooks  = nil
+        @modified_cookbooks = nil
+      end
+
+      def lock_name
+        old_lock["name"]
+      end
+
+      def old_cookbook_locks
+        old_lock["cookbook_locks"]
+      end
+
+      def new_cookbook_locks
+        new_lock["cookbook_locks"]
+      end
+
+      def updated_sections
+        @updated_sections ||= POLICY_SECTIONS.select do |key|
+          old_lock[key] != new_lock[key]
+        end
+      end
+
+      def different?
+        !updated_sections.empty?
+      end
+
+      def run_report
+        unless different?
+          ui.err("No changes for policy lock '#{lock_name}' between '#{old_name}' and '#{new_name}'")
+          return true
+        end
+
+        ui.print("Policy lock '#{lock_name}' differs between '#{old_name}' and '#{new_name}':\n\n")
+
+        report_rev_id_changes
+        report_run_list_changes
+        report_added_cookbooks
+        report_removed_cookbooks
+        report_modified_cookbooks
+        report_default_attribute_changes
+        report_override_attribute_changes
+      end
+
+      def report_rev_id_changes
+        h1('REVISION ID CHANGED')
+        old_rev = old_lock["revision_id"]
+        new_rev = new_lock["revision_id"]
+        diff_lines([ old_rev ], [ new_rev ])
+      end
+
+      def report_run_list_changes
+        return nil unless updated_sections.include?("run_list")
+        h1("RUN LIST CHANGED")
+
+        old_run_list = old_lock["run_list"]
+        new_run_list = new_lock["run_list"]
+
+        diff_lines(old_run_list, new_run_list)
+      end
+
+      def report_removed_cookbooks
+        return nil if removed_cookbooks.empty?
+        h1("REMOVED COOKBOOKS")
+        removed_cookbooks.each do |cb_name|
+          ui.print("\n")
+          old_lock = pretty_json(old_cookbook_locks[cb_name])
+          new_lock = []
+          h2(cb_name)
+          diff_lines(old_lock, new_lock)
+        end
+      end
+
+      def report_added_cookbooks
+        return nil if added_cookbooks.empty?
+        h1("ADDED COOKBOOKS")
+        added_cookbooks.each do |cb_name|
+          ui.print("\n")
+          old_lock = []
+          new_lock = pretty_json(new_cookbook_locks[cb_name])
+          h2(cb_name)
+          diff_lines(old_lock, new_lock)
+        end
+      end
+
+      def report_modified_cookbooks
+        return nil if modified_cookbooks.empty?
+        h1("MODIFIED COOKBOOKS")
+        modified_cookbooks.each do |cb_name|
+          ui.print("\n")
+          old_lock = pretty_json(old_cookbook_locks[cb_name])
+          new_lock = pretty_json(new_cookbook_locks[cb_name])
+          h2(cb_name)
+          diff_lines(old_lock, new_lock)
+        end
+      end
+
+      def report_default_attribute_changes
+        return nil unless updated_sections.include?("default_attributes")
+
+        h1("DEFAULT ATTRIBUTES CHANGED")
+
+        old_default = pretty_json(old_lock["default_attributes"])
+        new_default = pretty_json(new_lock["default_attributes"])
+        diff_lines(old_default, new_default)
+      end
+
+      def report_override_attribute_changes
+        return nil unless updated_sections.include?("override_attributes")
+
+        h1("OVERRIDE ATTRIBUTES CHANGED")
+
+        old_override = pretty_json(old_lock["override_attributes"])
+        new_override = pretty_json(new_lock["override_attributes"])
+        diff_lines(old_override, new_override)
+      end
+
+      def added_cookbooks
+        detect_cookbook_changes if @added_cookbooks.nil?
+        @added_cookbooks
+      end
+
+      def removed_cookbooks
+        detect_cookbook_changes if @removed_cookbooks.nil?
+        @removed_cookbooks
+      end
+
+      def modified_cookbooks
+        detect_cookbook_changes if @modified_cookbooks.nil?
+        @modified_cookbooks
+      end
+
+      private
+
+      def h1(str)
+        ui.msg(str)
+        ui.msg('=' * str.size)
+      end
+
+      def h2(str)
+        ui.msg(str)
+        ui.msg('-' * str.size)
+      end
+
+      def diff_lines(old_lines, new_lines)
+        file_length_difference = INITIAL_FILE_LENGTH_DIFFERENCE
+
+        previous_hunk = nil
+
+        diffs = Diff::LCS.diff(old_lines, new_lines)
+
+        ui.print("\n")
+
+        diffs.each do |piece|
+          hunk = Diff::LCS::Hunk.new(old_lines, new_lines, piece, LINES_OF_CONTEXT, file_length_difference)
+
+          file_length_difference = hunk.file_length_difference
+
+          maybe_contiguous_hunks = (previous_hunk.nil? || hunk.merge(previous_hunk))
+
+          if !maybe_contiguous_hunks
+            print_color_diff("#{previous_hunk.diff(FORMAT)}\n")
+          end
+          previous_hunk = hunk
+        end
+        print_color_diff("#{previous_hunk.diff(FORMAT)}\n") unless previous_hunk.nil?
+        ui.print("\n")
+      end
+
+      def print_color_diff(hunk)
+        hunk.to_s.each_line do |line|
+          ui.print(Paint[line, color_for_line(line)])
+        end
+      end
+
+      def color_for_line(line)
+        case line[0].chr
+        when "+"
+          :green
+        when "-"
+          :red
+        when "@"
+          line[1].chr == "@" ? :blue : nil
+        else
+          nil
+        end
+      end
+
+      def pretty_json(data)
+        FFI_Yajl::Encoder.encode(data, pretty: true).lines.map { |l| l.chomp }
+      end
+
+      def detect_cookbook_changes
+        all_locked_cookbooks = old_cookbook_locks.keys | new_cookbook_locks.keys
+
+        @added_cookbooks = []
+        @removed_cookbooks = []
+        @modified_cookbooks = []
+
+        all_locked_cookbooks.each do |cb_name|
+          if old_cookbook_locks.key?(cb_name) && new_cookbook_locks.key?(cb_name)
+            old_cb_lock = old_cookbook_locks[cb_name]
+            new_cb_lock = new_cookbook_locks[cb_name]
+            if old_cb_lock != new_cb_lock
+              @modified_cookbooks << cb_name
+            end
+          elsif old_cookbook_locks.key?(cb_name)
+            @removed_cookbooks << cb_name
+          elsif new_cookbook_locks.key?(cb_name)
+            @added_cookbooks << cb_name
+          else
+            raise "Bug: cookbook lock #{cb_name} cannot be determined as new/removed/modified/unmodified"
+          end
+        end
+      end
+
+
+    end
+  end
+end
+

data/lib/chef-dk/policyfile/dsl.rb

@@ -19,6 +19,8 @@ require 'chef-dk/policyfile/cookbook_sources'
 require 'chef-dk/policyfile/cookbook_location_specification'
 require 'chef-dk/policyfile/storage_config'
 
+require 'chef/node/attribute'
+
 module ChefDK
   module Policyfile
     class DSL
@@ -33,6 +35,7 @@ module ChefDK
       attr_reader :cookbook_location_specs
 
       attr_reader :named_run_lists
+      attr_reader :node_attributes
 
       attr_reader :storage_config
 
@@ -44,6 +47,8 @@ module ChefDK
         @default_source = NullCookbookSource.new
         @cookbook_location_specs = {}
         @storage_config = storage_config
+
+        @node_attributes = Chef::Node::Attribute.new({}, {}, {}, {})
       end
 
       def name(name = nil)
@@ -63,13 +68,15 @@ module ChefDK
         @named_run_lists[name] = run_list_items.flatten
       end
 
-      def default_source(source_type = nil, source_uri = nil)
+      def default_source(source_type = nil, source_argument = nil)
         return @default_source if source_type.nil?
         case source_type
         when :community
-          set_default_community_source(source_uri)
+          set_default_community_source(source_argument)
         when :chef_server
-          set_default_chef_server_source(source_uri)
+          set_default_chef_server_source(source_argument)
+        when :chef_repo
+          set_default_chef_repo_source(source_argument)
         else
           @errors << "Invalid default_source type '#{source_type.inspect}'"
         end
@@ -98,6 +105,14 @@ module ChefDK
         end
       end
 
+      def default
+        @node_attributes.default
+      end
+
+      def override
+        @node_attributes.override
+      end
+
       def eval_policyfile(policyfile_string)
         @policyfile_filename = policyfile_filename
         instance_eval(policyfile_string, policyfile_filename)
@@ -136,6 +151,14 @@ module ChefDK
         end
       end
 
+      def set_default_chef_repo_source(path)
+        if path.nil?
+          @errors << "You must specify the path to the chef-repo when using a default_source :chef_repo"
+        else
+          @default_source = ChefRepoCookbookSource.new(path)
+        end
+      end
+
       def validate!
         if @run_list.empty?
           @errors << "Invalid run_list. run_list cannot be empty"

data/lib/chef-dk/policyfile/uploader.rb

@@ -51,13 +51,12 @@ module ChefDK
       def upload
         ui.msg("Uploading policy to policy group #{policy_group}")
 
-        if using_policy_document_native_api?
+        if !using_policy_document_native_api?
           ui.msg(<<-DRAGONS)
-WARN: Using native policy API preview mode. You may be required to delete and
-re-upload this data when upgrading to the final release version of the feature.
+WARN: Uploading policy to policy group #{policy_group} in compatibility mode.
+Cookbooks will be uploaded with very large version numbers, which may be picked
+up by existing nodes.
 DRAGONS
-        else
-          ui.msg("WARN: Uploading policy to policy group #{policy_group} in compatibility mode")
         end
 
         upload_cookbooks

data/lib/chef-dk/policyfile_compiler.rb

@@ -97,6 +97,14 @@ module ChefDK
       end
     end
 
+    def default_attributes
+      dsl.node_attributes.combined_default.to_hash
+    end
+
+    def override_attributes
+      dsl.node_attributes.combined_override.to_hash
+    end
+
     def lock
       @policyfile_lock ||= PolicyfileLock.build_from_compiler(self, storage_config)
     end

data/lib/chef-dk/policyfile_lock.rb

@@ -1,3 +1,4 @@
+# -*- coding: UTF-8 -*-
 #
 # Copyright:: Copyright (c) 2014 Chef Software Inc.
 # License:: Apache License, Version 2.0
@@ -15,7 +16,7 @@
 # limitations under the License.
 #
 
-require 'digest/sha1'
+require 'digest/sha2'
 
 require 'chef-dk/policyfile/storage_config'
 require 'chef-dk/policyfile/cookbook_locks'
@@ -31,7 +32,7 @@ module ChefDK
       attr_reader :ui
       attr_reader :policyfile_lock
 
-      def initialize(ui: ui, policyfile_lock: nil)
+      def initialize(ui: nil, policyfile_lock: nil)
         @ui = ui
         @policyfile_lock = policyfile_lock
 
@@ -83,6 +84,8 @@ module ChefDK
     attr_accessor :name
     attr_accessor :run_list
     attr_accessor :named_run_lists
+    attr_accessor :default_attributes
+    attr_accessor :override_attributes
 
     attr_reader :solution_dependencies
 
@@ -101,6 +104,9 @@ module ChefDK
       @storage_config = storage_config
       @ui = ui || UI.null
 
+      @default_attributes = {}
+      @override_attributes = {}
+
       @solution_dependencies = Policyfile::SolutionDependencies.new
       @install_report = InstallReport.new(ui: @ui, policyfile_lock: self)
     end
@@ -132,6 +138,8 @@ module ChefDK
         lock["run_list"] = run_list
         lock["named_run_lists"] = named_run_lists unless named_run_lists.empty?
         lock["cookbook_locks"] = cookbook_locks_for_lockfile
+        lock["default_attributes"] = default_attributes
+        lock["override_attributes"] = override_attributes
         lock["solution_dependencies"] = solution_dependencies.to_lock
       end
     end
@@ -139,7 +147,7 @@ module ChefDK
     # Returns a fingerprint of the PolicyfileLock by computing the SHA1 hash of
     # #canonical_revision_string
     def revision_id
-      Digest::SHA1.new.hexdigest(canonical_revision_string)
+      Digest::SHA256.new.hexdigest(canonical_revision_string)
     end
 
     # Generates a string representation of the lock data in a specialized
@@ -172,6 +180,10 @@ module ChefDK
         canonical_rev_text << "cookbook:#{name};id:#{lock["identifier"]}\n"
       end
 
+      canonical_rev_text << "default_attributes:#{canonicalize(default_attributes)}\n"
+
+      canonical_rev_text << "override_attributes:#{canonicalize(override_attributes)}\n"
+
       canonical_rev_text
     end
 
@@ -232,6 +244,9 @@ module ChefDK
         end
       end
 
+      @default_attributes = compiler.default_attributes
+      @override_attributes = compiler.override_attributes
+
       @solution_dependencies = compiler.solution_dependencies
 
       self
@@ -241,6 +256,7 @@ module ChefDK
       set_name_from_lock_data(lock_data)
       set_run_list_from_lock_data(lock_data)
       set_cookbook_locks_from_lock_data(lock_data)
+      set_attributes_from_lock_data(lock_data)
       set_solution_dependencies_from_lock_data(lock_data)
       self
     end
@@ -264,6 +280,97 @@ module ChefDK
 
     private
 
+    # Generates a canonical JSON representation of the attributes. Based on
+    # http://wiki.laptop.org/go/Canonical_JSON but not quite as strict, yet.
+    #
+    # In particular:
+    # - String encoding stuff isn't normalized
+    # - We allow floats that fit within the range/precision requirements of
+    #   IEEE 754-2008 binary64 (double precision) numbers.
+    # - +/- Infinity and NaN are banned, but float/numeric size aren't checked.
+    #   numerics should be in range [-(2**53)+1, (2**53)-1] to comply with
+    #   IEEE 754-2008
+    #
+    # Recursive, so absurd nesting levels could cause a SystemError. Invalid
+    # input will cause an InvalidPolicyfileAttribute exception.
+    def canonicalize(attributes)
+      unless attributes.kind_of?(Hash)
+        raise "Top level attributes must be a Hash (you gave: #{attributes})"
+      end
+      canonicalize_elements(attributes)
+    end
+
+    def canonicalize_elements(item)
+      case item
+      when Hash
+        # Hash keys will sort differently based on the encoding, but after a
+        # JSON round trip everything will be UTF-8, so we have to normalize the
+        # keys to UTF-8 first so that the sort order uses the UTF-8 strings.
+        item_with_normalized_keys = item.inject({}) do |normalized_item, (key, value)|
+          validate_attr_key(key)
+          normalized_item[key.encode('utf-8')] = value
+          normalized_item
+        end
+        elements = item_with_normalized_keys.keys.sort.map do |key|
+          k = '"' << key << '":'
+          v = canonicalize_elements(item_with_normalized_keys[key])
+          k << v
+        end
+        "{" << elements.join(',') << "}"
+      when String
+        '"' << item.encode('utf-8') << '"'
+      when Array
+        elements = item.map { |i| canonicalize_elements(i) }
+        '[' << elements.join(',') << ']'
+      when Integer
+        item.to_s
+      when Float
+        unless item.finite?
+          raise InvalidPolicyfileAttribute, "Floating point numbers cannot be infinite or NaN. You gave #{item.inspect}"
+        end
+        # Support for floats assumes that any implementation of our JSON
+        # canonicalization routine will use IEEE-754 doubles. In decimal terms,
+        # doubles give 15-17 digits of precision, so we err on the safe side
+        # and only use 15 digits in the string conversion. We use the `g`
+        # format, which is a documented-enough "do what I mean" where floats
+        # >= 0.1 and < precsion are represented as floating point literals, and
+        # other numbers use the exponent notation with a lowercase 'e'. Note
+        # that both Ruby and Erlang document what their `g` does but have some
+        # differences both subtle and non-subtle:
+        #
+        # ```ruby
+        # format("%.15g", 0.1) #=> "0.1"
+        # format("%.15g", 1_000_000_000.0) #=> "1000000000"
+        # ```
+        #
+        # Whereas:
+        #
+        # ```erlang
+        # lists:flatten(io_lib:format("~.15g", [0.1])). %=> "0.100000000000000"
+        # lists:flatten(io_lib:format("~.15e", [1000000000.0])). %=> "1.00000000000000e+9"
+        # ```
+        #
+        # Other implementations should normalize to ruby's %.15g behavior.
+        Kernel.format("%.15g", item)
+      when NilClass
+        "null"
+      when TrueClass
+        "true"
+      when FalseClass
+        "false"
+      else
+        raise InvalidPolicyfileAttribute,
+          "Invalid type in attributes. Only Hash, Array, String, Integer, Float, true, false, and nil are accepted. You gave #{item.inspect} (#{item.class})"
+      end
+    end
+
+    def validate_attr_key(key)
+      unless key.kind_of?(String)
+        raise InvalidPolicyfileAttribute,
+          "Attribute keys must be Strings (other types are not allowed in JSON). You gave: #{key.inspect} (#{key.class})"
+      end
+    end
+
     def set_name_from_lock_data(lock_data)
       name_attribute = lock_data["name"]
 
@@ -316,6 +423,31 @@ module ChefDK
       end
     end
 
+    def set_attributes_from_lock_data(lock_data)
+      default_attr_data = lock_data["default_attributes"]
+
+      if default_attr_data.nil?
+        raise InvalidLockfile, "lockfile does not have a `default_attributes` attribute"
+      end
+
+      unless default_attr_data.kind_of?(Hash)
+        raise InvalidLockfile, "lockfile's `default_attributes` attribute must be a Hash (JSON object). (got: #{default_attr_data.inspect})"
+      end
+
+      override_attr_data = lock_data["override_attributes"]
+
+      if override_attr_data.nil?
+        raise InvalidLockfile, "lockfile does not have a `override_attributes` attribute"
+      end
+
+      unless override_attr_data.kind_of?(Hash)
+        raise InvalidLockfile, "lockfile's `override_attributes` attribute must be a Hash (JSON object). (got: #{override_attr_data.inspect})"
+      end
+
+      @default_attributes   = default_attr_data
+      @override_attributes  = override_attr_data
+    end
+
     def set_solution_dependencies_from_lock_data(lock_data)
       soln_deps = lock_data["solution_dependencies"]