chef-dk 0.6.2 → 0.7.0

data/lib/chef-dk/command/verify.rb

@@ -93,7 +93,13 @@ module ChefDK
         c.base_dir = "test-kitchen"
         c.unit_test { sh("bundle exec rake unit") }
         c.integration_test { sh("bundle exec rake features") }
-        c.smoke_test { run_in_tmpdir("kitchen init") }
+
+        # NOTE: By default, kitchen tries to be helpful and install a driver
+        # gem for you. This causes a race condition when running the tests
+        # concurrently, because rubygems breaks when there are partially
+        # installed gems in the gem repository. Instructing kitchen to create a
+        # gemfile instead avoids the gem installation.
+        c.smoke_test { run_in_tmpdir("kitchen init --create-gemfile") }
       end
 
       add_component "chef-client" do |c|
@@ -115,6 +121,57 @@ module ChefDK
         c.smoke_test { run_in_tmpdir("chef generate cookbook example") }
       end
 
+      # entirely possible this needs to be driven by a utility method in chef-provisioning.
+      add_component "chef-provisioning" do |c|
+        c.base_dir = "chef-dk"
+
+        c.smoke_test do
+          # ------------
+          # we want to avoid hard-coding driver names, but calling Gem::Specification produces a warning; this
+          # seems to be the best way to silence it.
+          verbose = $VERBOSE
+          $VERBOSE = nil
+
+          drivers = Gem::Specification.all.map { |gs| gs.name }.
+                                           select { |n| n =~ /^chef-provisioning-/ }.
+                                           uniq
+
+          versions = Gem::Specification.find_all_by_name("chef-provisioning").map { |s| s.version }
+          $VERBOSE = verbose
+          # ------------
+          failures = []
+
+          tmpdir do |cwd|
+            versions.each do |provisioning_version|
+              gemfile = "chef-provisioning-#{provisioning_version}-chefdk-test.gemfile"
+
+              # write out the gemfile for this chef-provisioning version, and see if Bundler can make it go.
+              with_file(File.join(cwd, gemfile)) do |f|
+                f.puts %Q(gem "chef-provisioning", "= #{provisioning_version}")
+                drivers.each { |d| f.puts %Q(gem "#{d}") }
+              end
+
+              result = sh("bundle install --local --quiet", cwd: cwd, env: {"BUNDLE_GEMFILE" => gemfile })
+
+              if result.exitstatus != 0
+                failures << result
+              end
+
+            end  # end provisioning versions.
+
+            if failures.size > 0
+              failures.each { |fail| puts fail.stdout }
+              puts "\nDriver list (no version restrictions):\n  #{drivers.join("\n  ")}"
+            end
+
+            # dubious on Windows.
+            # this is weird, but we seem to require a Mixlib::ShellOut as the return value. suggestions
+            # welcome.
+            sh(failures.size > 0 ? "false" : "true")
+          end
+        end
+      end
+
       add_component "chefspec" do |c|
         c.gem_base_dir = "chefspec"
         c.unit_test { sh("rake unit") }

data/lib/chef-dk/cookbook_omnifetch.rb

@@ -18,13 +18,14 @@
 require 'cookbook-omnifetch'
 require 'chef-dk/shell_out'
 require 'chef-dk/cookbook_metadata'
+require 'chef-dk/helpers'
 
 require 'chef/http/simple'
 
 # Configure CookbookOmnifetch's dependency injection settings to use our classes and config.
 CookbookOmnifetch.configure do |c|
-  c.cache_path = File.expand_path('~/.chefdk/cache')
-  c.storage_path = Pathname.new(File.expand_path('~/.chefdk/cache/cookbooks'))
+  c.cache_path = File.expand_path(File.join(ChefDK::Helpers.chefdk_home, 'cache'))
+  c.storage_path = Pathname.new(File.expand_path(File.join(ChefDK::Helpers.chefdk_home, 'cache', 'cookbooks')))
   c.shell_out_class = ChefDK::ShellOut
   c.cached_cookbook_class = ChefDK::CookbookMetadata
 end

data/lib/chef-dk/exceptions.rb

@@ -71,4 +71,31 @@ module ChefDK
   class BUG < RuntimeError
   end
 
+  class CookbookSourceConflict < StandardError
+
+    attr_reader :conflicting_cookbooks
+
+    attr_reader :cookbook_sources
+
+    def initialize(conflicting_cookbooks, cookbook_sources)
+      @conflicting_cookbooks = conflicting_cookbooks
+      @cookbook_sources = cookbook_sources
+      super(compute_message)
+    end
+
+    private
+
+    def compute_message
+      conflicting_cookbook_sets = cookbook_sources.combination(2).map do |source_a, source_b|
+        overlapping_cookbooks = conflicting_cookbooks.select do |cookbook_name|
+          source_a.universe_graph.key?(cookbook_name) && source_b.universe_graph.key?(cookbook_name)
+        end
+        "Source #{source_a.desc} and #{source_b.desc} contain conflicting cookbooks:\n" +
+          overlapping_cookbooks.sort.map {|c| "- #{c}"}.join("\n")
+      end
+      conflicting_cookbook_sets.join("\n") + "\n"
+    end
+
+  end
+
 end

data/lib/chef-dk/helpers.rb

@@ -20,6 +20,7 @@ require 'chef-dk/exceptions'
 
 module ChefDK
   module Helpers
+    extend self
 
     #
     # Runs given commands using mixlib-shellout
@@ -74,6 +75,17 @@ module ChefDK
       @omnibus_chefdk_location ||= File.expand_path('embedded/apps/chef-dk', expected_omnibus_root)
     end
 
+    def chefdk_home
+      @chefdk_home ||= begin
+                         chefdk_home_set = !([nil, ''].include? ENV['CHEFDK_HOME'])
+                         if chefdk_home_set
+                           ENV['CHEFDK_HOME']
+                         else
+                           default_chefdk_home
+                         end
+                       end
+    end
+
     private
 
     def omnibus_expand_path(*paths)
@@ -86,6 +98,14 @@ module ChefDK
       File.expand_path(File.join(Gem.ruby, "..", "..", ".."))
     end
 
+    def default_chefdk_home
+      if Chef::Platform.windows?
+        File.join(ENV['LOCALAPPDATA'], 'chefdk')
+      else
+        File.expand_path('~/.chefdk')
+      end
+    end
+
     #
     # environment vars for omnibus
     #
@@ -109,5 +129,14 @@ module ChefDK
     def with_file(path, mode='wb+', &block)
       File.open(path, mode, &block)
     end
+
+    #@api private
+    # This method resets all the instance variables used. It
+    # should only be used for testing
+    def reset!
+      self.instance_variables.each do |ivar|
+        self.instance_variable_set(ivar, nil)
+      end
+    end
   end
 end

data/lib/chef-dk/policyfile/chef_repo_cookbook_source.rb

@@ -57,6 +57,14 @@ module ChefDK
         { path: cookbook_version_paths[cookbook_name][cookbook_version], version: cookbook_version }
       end
 
+      def null?
+        false
+      end
+
+      def desc
+        "chef_repo(#{path})"
+      end
+
       private
 
       # Setter for setting the path.  It may either be a full chef-repo with

data/lib/chef-dk/policyfile/chef_server_cookbook_source.rb

@@ -39,6 +39,14 @@ module ChefDK
         raise UnsupportedFeature, 'ChefDK does not support chef-server cookbook default sources at this time'
       end
 
+      def null?
+        false
+      end
+
+      def desc
+        "chef_server(#{uri})"
+      end
+
     end
   end
 end

data/lib/chef-dk/policyfile/community_cookbook_source.rb

@@ -56,6 +56,14 @@ module ChefDK
         { artifactserver: base_uri, version: cookbook_version }
       end
 
+      def null?
+        false
+      end
+
+      def desc
+        "supermarket(#{uri})"
+      end
+
       private
 
       def http_connection_for(base_url)

data/lib/chef-dk/policyfile/cookbook_locks.rb

@@ -15,6 +15,8 @@
 # limitations under the License.
 #
 
+require 'forwardable'
+
 require 'chef-dk/exceptions'
 
 require 'chef-dk/cookbook_profiler/null_scm'
@@ -98,11 +100,16 @@ module ChefDK
       end
 
       def cookbook_path
-        raise NotImplementedError, "#{self.class} must override #to_lock with a specific implementation"
+        raise NotImplementedError, "#{self.class} must override #cookbook_path with a specific implementation"
       end
 
       def to_lock
-        raise NotImplementedError, "#{self.class} must override #to_lock with a specific implementation"
+        validate!
+        lock_data
+      end
+
+      def lock_data
+        raise NotImplementedError, "#{self.class} must override #lock_data a specific implementation"
       end
 
       def build_from_lock_data(lock_data)
@@ -227,8 +234,7 @@ module ChefDK
         @source_options = symbolize_source_options_keys(lock_data["source_options"])
       end
 
-      def to_lock
-        validate!
+      def lock_data
         {
           "version" => version,
           "identifier" => identifier,
@@ -280,6 +286,7 @@ module ChefDK
         @identifier_updated = false
         @version_updated = false
         @cookbook_in_git_repo = nil
+        @scm_info = nil
       end
 
       def cookbook_path
@@ -295,11 +302,15 @@ module ChefDK
       end
 
       def scm_info
-        scm_profiler.profile_data
+        @scm_info
       end
 
       def to_lock
-        validate!
+        refresh_scm_info
+        super
+      end
+
+      def lock_data
         {
           "version" => version,
           "identifier" => identifier,
@@ -319,6 +330,7 @@ module ChefDK
         @dotted_decimal_identifier = lock_data["dotted_decimal_identifier"]
         @source = lock_data["source"]
         @source_options = symbolize_source_options_keys(lock_data["source_options"])
+        @scm_info = lock_data["scm_info"]
       end
 
       def validate!
@@ -361,6 +373,10 @@ module ChefDK
 
       private
 
+      def refresh_scm_info
+        @scm_info = scm_profiler.profile_data
+      end
+
       def assert_required_keys_valid!(lock_data)
         super
 
@@ -392,5 +408,59 @@ module ChefDK
       end
 
     end
+
+    class ArchivedCookbook < CookbookLock
+
+      extend Forwardable
+
+      def_delegator :@archived_lock, :name
+      def_delegator :@archived_lock, :source_options
+      def_delegator :@archived_lock, :identifier
+      def_delegator :@archived_lock, :dotted_decimal_identifier
+      def_delegator :@archived_lock, :version
+      def_delegator :@archived_lock, :source
+
+      # #to_lock calls #validate! which will typically ensure that the cookbook
+      # is present at the correct path for the lock type. For an archived
+      # cookbook, the cookbook is located in the archive, so it probably won't
+      # exist there. Therefore, we bypass #validate! and get the lock data
+      # directly.
+      def_delegator :@archived_lock, :lock_data, :to_lock
+
+      def initialize(archived_lock, storage_config)
+        @archived_lock = archived_lock
+        @storage_config = storage_config
+      end
+
+      def build_from_lock_data(lock_data)
+        raise NotImplementedError, "ArchivedCookbook cannot be built from lock data, it can only wrap an existing lock object"
+      end
+
+      def installed?
+        File.exist?(cookbook_path) && File.directory?(cookbook_path)
+      end
+
+      # The cookbook is assumed to be stored in a Chef Zero compatible repo as
+      # created by `chef export`. Currently that only creates "compatibility
+      # mode" repos since Chef Zero doesn't yet support cookbook_artifact APIs.
+      # So the cookbook will be located in a path like:
+      #   cookbooks/nginx-111.222.333
+      def cookbook_path
+        File.join(relative_paths_root, "cookbooks", "#{name}-#{dotted_decimal_identifier}")
+      end
+
+      # We trust that archived cookbooks haven't been modified, so just return
+      # true for #validate!
+      def validate!
+        true
+      end
+
+      # We trust that archived cookbooks haven't been modified, so just return
+      # true for #refresh!
+      def refresh!
+        true
+      end
+    end
+
   end
 end

data/lib/chef-dk/policyfile/dsl.rb

@@ -44,7 +44,7 @@ module ChefDK
         @errors = []
         @run_list = []
         @named_run_lists = {}
-        @default_source = NullCookbookSource.new
+        @default_source = [ NullCookbookSource.new ]
         @cookbook_location_specs = {}
         @storage_config = storage_config
 
@@ -71,7 +71,7 @@ module ChefDK
       def default_source(source_type = nil, source_argument = nil)
         return @default_source if source_type.nil?
         case source_type
-        when :community
+        when :community, :supermarket
           set_default_community_source(source_argument)
         when :chef_server
           set_default_chef_server_source(source_argument)
@@ -140,14 +140,14 @@ module ChefDK
       private
 
       def set_default_community_source(source_uri)
-        @default_source = CommunityCookbookSource.new(source_uri)
+        set_default_source(CommunityCookbookSource.new(source_uri))
       end
 
       def set_default_chef_server_source(source_uri)
         if source_uri.nil?
           @errors << "You must specify the server's URI when using a default_source :chef_server"
         else
-          @default_source = ChefServerCookbookSource.new(source_uri)
+          set_default_source(ChefServerCookbookSource.new(source_uri))
         end
       end
 
@@ -155,10 +155,15 @@ module ChefDK
         if path.nil?
           @errors << "You must specify the path to the chef-repo when using a default_source :chef_repo"
         else
-          @default_source = ChefRepoCookbookSource.new(path)
+          set_default_source(ChefRepoCookbookSource.new(path))
         end
       end
 
+      def set_default_source(source)
+        @default_source.delete_at(0) if @default_source[0].null?
+        @default_source << source
+      end
+
       def validate!
         if @run_list.empty?
           @errors << "Invalid run_list. run_list cannot be empty"

data/lib/chef-dk/policyfile/lister.rb

@@ -0,0 +1,230 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'set'
+
+require 'chef-dk/authenticated_http'
+require 'chef-dk/service_exceptions'
+
+module ChefDK
+  module Policyfile
+
+    class RevIDLockDataMap
+
+      attr_reader :policy_name
+      attr_reader :lock_info_by_rev_id
+
+      def initialize(policy_name, lock_info_by_rev_id)
+        @policy_name = policy_name
+        @lock_info_by_rev_id = lock_info_by_rev_id
+      end
+
+
+      def cb_info_for(rev_id, cookbook_name)
+        lock = lock_info_by_rev_id[rev_id]
+        cookbook_lock = lock["cookbook_locks"][cookbook_name]
+
+        if cookbook_lock
+          [cookbook_lock["version"], cookbook_lock["identifier"] ]
+        else
+          nil
+        end
+      end
+
+      def cbs_with_differing_ids
+        cbs_with_differing_ids = Set.new
+        all_cookbook_names.each do |cookbook_name|
+          all_identifiers = lock_info_by_rev_id.inject(Set.new) do |id_set, (_rev_id, rev_info)|
+            cookbook_lock = rev_info["cookbook_locks"][cookbook_name]
+            identifier = cookbook_lock && cookbook_lock["identifier"]
+            id_set << identifier
+          end
+          cbs_with_differing_ids << cookbook_name if all_identifiers.size > 1
+        end
+        cbs_with_differing_ids
+      end
+
+      def all_cookbook_names
+        lock_info_by_rev_id.inject(Set.new) do |cb_set, (_rev_id, rev_info)|
+          cb_set.merge(rev_info["cookbook_locks"].keys)
+        end
+      end
+    end
+
+    class PolicyGroupRevIDMap
+
+      attr_reader :policy_name
+      attr_reader :revision_ids_by_group
+
+      def initialize(policy_name, revision_ids_by_group)
+        @policy_name = policy_name
+        @revision_ids_by_group = revision_ids_by_group
+      end
+
+      def unique_revision_ids
+        revision_ids_by_group.values.uniq
+      end
+
+      def policy_group_names
+        revision_ids_by_group.keys
+      end
+
+      def max_group_name_length
+        policy_group_names.map(&:size).max
+      end
+
+      def format_revision_ids
+        revision_ids_by_group.inject({}) do |map, (group_name, rev_id)|
+          map[group_name] = yield rev_id
+          map
+        end
+      end
+
+      def empty?
+        policy_group_names.empty?
+      end
+
+      def each
+        revision_ids_by_group.each do |group_name, rev_id|
+          yield group_name, rev_id
+        end
+      end
+    end
+
+    class Lister
+
+      attr_accessor :policy_lock_content
+
+      attr_reader :config
+
+      def initialize(config: nil)
+        @config = config
+        @policies_by_name = nil
+        @policies_by_group = nil
+        @policy_lock_content = {}
+        @active_revisions = nil
+      end
+
+      # A Hash with the following format
+      #   {
+      #     "appserver" => {
+      #       "1111111111111111111111111111111111111111111111111111111111111111" => {},
+      #       "2222222222222222222222222222222222222222222222222222222222222222" => {}
+      #     },
+      def policies_by_name
+        @policies_by_name || fetch_policy_lists
+        @policies_by_name
+      end
+
+      # A Hash with the following format:
+      #   "dev" => {
+      #     "appserver" => "1111111111111111111111111111111111111111111111111111111111111111",
+      #     "load-balancer" => "5555555555555555555555555555555555555555555555555555555555555555",
+      #     "db" => "9999999999999999999999999999999999999999999999999999999999999999"
+      #   }
+      def policies_by_group
+        @policies_by_group || fetch_policy_lists
+        @policies_by_group
+      end
+
+      def revision_info_for(policy_name, _revision_id_list)
+        RevIDLockDataMap.new(policy_name, policy_lock_content[policy_name])
+      end
+
+      def revision_ids_by_group_for_each_policy
+        policies_by_name.each do |policy_name, _policies|
+          rev_id_by_group = revision_ids_by_group_for(policy_name)
+          yield policy_name, rev_id_by_group
+        end
+      end
+
+      def revision_ids_by_group_for(policy_name)
+        map = policies_by_group.inject({}) do |rev_id_map, (group_name, rev_id_map_for_group)|
+          rev_id_map[group_name] = rev_id_map_for_group[policy_name]
+          rev_id_map
+        end
+        PolicyGroupRevIDMap.new(policy_name, map)
+      end
+
+      def orphaned_revisions(policy_name)
+        orphans = []
+        policies_by_name[policy_name].each do |rev_id, _data|
+          orphans << rev_id unless active_revisions.include?(rev_id)
+        end
+        orphans
+      end
+
+      def active_revisions
+        @active_revisions ||= policies_by_group.inject(Set.new) do |set, (_group, policy_name_rev_id_map)|
+          policy_name_rev_id_map.each do |policy_name, rev_id|
+            set << rev_id
+          end
+          set
+        end
+      end
+
+      def empty?
+        policies_by_name.empty? && policies_by_group.empty?
+      end
+
+      def http_client
+        @http_client ||= ChefDK::AuthenticatedHTTP.new(config.chef_server_url,
+                                                       signing_key_filename: config.client_key,
+                                                       client_name: config.node_name)
+      end
+
+      # @api private
+      # Sets internal copy of policyfile data to policies_by_name and
+      # policies_by_group. Used for internal testing.
+      def set!(policies_by_name, policies_by_group)
+        @policies_by_name = policies_by_name
+        @policies_by_group = policies_by_group
+        @active_revisions = nil
+      end
+
+      private
+
+      def fetch_policy_lists
+        policy_list_data = http_client.get("policies")
+        set_policies_by_name_from_api(policy_list_data)
+
+        policy_group_data = http_client.get("policy_groups")
+        set_policies_by_group_from_api(policy_group_data)
+      end
+
+      def set_policies_by_name_from_api(policy_list_data)
+        @policies_by_name = policy_list_data.inject({}) do |map, (policy_name, policy_info)|
+          map[policy_name] = policy_info["revisions"]
+          map
+        end
+      end
+
+      def set_policies_by_group_from_api(policy_group_data)
+        @policies_by_group = policy_group_data.inject({}) do |map, (policy_group, policy_info)|
+          map[policy_group] = policy_info["policies"].inject({}) do |rev_map, (policy_name, rev_info)|
+            rev_map[policy_name] = rev_info["revision_id"]; rev_map
+          end
+
+          map
+        end
+      end
+
+    end
+  end
+end
+
+