chef-dk 0.2.0 → 0.2.1

data/lib/chef-dk/policyfile_compiler.rb

@@ -18,7 +18,7 @@
 require 'forwardable'
 
 require 'solve'
-require 'chef/run_list/run_list_item'
+require 'chef/run_list'
 
 require 'chef-dk/policyfile/dsl'
 require 'chef-dk/policyfile_lock'
@@ -40,16 +40,19 @@ module ChefDK
       compiler
     end
 
+    def_delegator :@dsl, :name
     def_delegator :@dsl, :run_list
     def_delegator :@dsl, :errors
     def_delegator :@dsl, :default_source
-    def_delegator :@dsl, :policyfile_cookbook_specs
+    def_delegator :@dsl, :cookbook_location_specs
 
     attr_reader :dsl
+    attr_reader :storage_config
 
     def initialize
-      @dsl = Policyfile::DSL.new
-      @artifact_server_cookbook_specs = {}
+      @storage_config = Policyfile::StorageConfig.new
+      @dsl = Policyfile::DSL.new(storage_config)
+      @artifact_server_cookbook_location_specs = {}
     end
 
     def error!
@@ -58,23 +61,29 @@ module ChefDK
       end
     end
 
-    def cookbook_spec_for(cookbook_name)
-      policyfile_cookbook_specs[cookbook_name]
+    def cookbook_location_spec_for(cookbook_name)
+      cookbook_location_specs[cookbook_name]
     end
 
     def expanded_run_list
-      run_list
+      # doesn't support roles yet...
+      Chef::RunList.new(*run_list)
+    end
+
+    # copy of the expanded_run_list, properly formatted for use in a lockfile
+    def normalized_run_list
+      expanded_run_list.map { |i| normalize_recipe(i) }
     end
 
     def lock
-      @policyfile_lock ||= PolicyfileLock.build_from_compiler(self, cache_path: cache_path)
+      @policyfile_lock ||= PolicyfileLock.build_from_compiler(self, storage_config)
     end
 
     def install
       ensure_cache_dir_exists
 
       graph_solution.each do |cookbook_name, version|
-        spec = cookbook_spec_for(cookbook_name)
+        spec = cookbook_location_spec_for(cookbook_name)
         if spec.nil? or !spec.version_fixed?
           spec = create_spec_for_cookbook(cookbook_name, version)
           spec.ensure_cached
@@ -84,17 +93,17 @@ module ChefDK
 
     def create_spec_for_cookbook(cookbook_name, version)
       source_options = default_source.source_options_for(cookbook_name, version)
-      spec = Policyfile::CookbookSpec.new(cookbook_name, "= #{version}", source_options, dsl)
-      @artifact_server_cookbook_specs[cookbook_name] = spec
+      spec = Policyfile::CookbookLocationSpecification.new(cookbook_name, "= #{version}", source_options, storage_config)
+      @artifact_server_cookbook_location_specs[cookbook_name] = spec
     end
 
-    def all_cookbook_specs
-      # in the installation proces, we create "artifact_server_cookbook_specs"
+    def all_cookbook_location_specs
+      # in the installation proces, we create "artifact_server_cookbook_location_specs"
       # for any cookbook that isn't sourced from a single-version source (e.g.,
       # path and git only support one version at a time), but we might have
       # specs for them to track additional version constraint demands. Merging
-      # in this order ensures the artifact_server_cookbook_specs "win".
-      policyfile_cookbook_specs.merge(@artifact_server_cookbook_specs)
+      # in this order ensures the artifact_server_cookbook_location_specs "win".
+      cookbook_location_specs.merge(@artifact_server_cookbook_location_specs)
     end
 
     ##
@@ -120,9 +129,23 @@ module ChefDK
       end
     end
 
+    def solution_dependencies
+      solution_deps = Policyfile::SolutionDependencies.new
+
+      all_cookbook_location_specs.each do |name, spec|
+        solution_deps.add_policyfile_dep(name, spec.version_constraint)
+      end
+
+      graph_solution.each do |name, version|
+        transitive_deps = artifacts_graph[name][version]
+        solution_deps.add_cookbook_dep(name, version, transitive_deps)
+      end
+      solution_deps
+    end
+
     def graph_demands
       cookbooks_for_demands.map do |cookbook_name|
-        spec = cookbook_spec_for(cookbook_name)
+        spec = cookbook_location_spec_for(cookbook_name)
         if spec.nil?
           [ cookbook_name, DEFAULT_DEMAND_CONSTRAINT ]
         elsif spec.version_fixed?
@@ -145,9 +168,9 @@ module ChefDK
     # version number. To accomodate this, the local_artifacts_graph should be
     # merged over the upstream's artifacts graph.
     def local_artifacts_graph
-      policyfile_cookbook_specs.inject({}) do |local_artifacts, (cookbook_name, cookbook_spec)|
-        if cookbook_spec.version_fixed?
-          local_artifacts[cookbook_name] = { cookbook_spec.version => cookbook_spec.dependencies }
+      cookbook_location_specs.inject({}) do |local_artifacts, (cookbook_name, cookbook_location_spec)|
+        if cookbook_location_spec.version_fixed?
+          local_artifacts[cookbook_name] = { cookbook_location_spec.version => cookbook_location_spec.dependencies }
         end
         local_artifacts
       end
@@ -158,8 +181,8 @@ module ChefDK
     end
 
     def version_constraint_for(cookbook_name)
-      if (cookbook_spec = cookbook_spec_for(cookbook_name)) and cookbook_spec.version_fixed?
-        version = cookbook_spec.version
+      if (cookbook_location_spec = cookbook_location_spec_for(cookbook_name)) and cookbook_location_spec.version_fixed?
+        version = cookbook_location_spec.version
         "= #{version}"
       else
         DEFAULT_DEMAND_CONSTRAINT
@@ -167,15 +190,16 @@ module ChefDK
     end
 
     def cookbook_version_fixed?(cookbook_name)
-      if cookbook_spec = cookbook_spec_for(cookbook_name)
-        cookbook_spec.version_fixed?
+      if cookbook_location_spec = cookbook_location_spec_for(cookbook_name)
+        cookbook_location_spec.version_fixed?
       else
         false
       end
     end
 
     def cookbooks_in_run_list
-      run_list.map {|item_spec| Chef::RunList::RunListItem.new(item_spec).name }
+      recipes = expanded_run_list.map {|recipe| recipe.name }
+      recipes.map { |r| r[/^([^:]+)/, 1] }
     end
 
     def build
@@ -184,21 +208,28 @@ module ChefDK
     end
 
     def evaluate_policyfile(policyfile_string, policyfile_filename)
-      @dsl.eval_policyfile(policyfile_string, policyfile_filename)
+      storage_config.use_policyfile(policyfile_filename)
+      @dsl.eval_policyfile(policyfile_string)
       self
     end
 
     private
 
+    def normalize_recipe(run_list_item)
+      name = run_list_item.name
+      name = "#{name}::default" unless name.include?("::")
+      "recipe[#{name}]"
+    end
+
     def cookbooks_for_demands
-      (cookbooks_in_run_list + policyfile_cookbook_specs.keys).uniq
+      (cookbooks_in_run_list + cookbook_location_specs.keys).uniq
     end
 
     def cache_fixed_version_cookbooks
       ensure_cache_dir_exists
 
-      policyfile_cookbook_specs.each do |_cookbook_name, cookbook_spec|
-        cookbook_spec.ensure_cached if cookbook_spec.version_fixed?
+      cookbook_location_specs.each do |_cookbook_name, cookbook_location_spec|
+        cookbook_location_spec.ensure_cached if cookbook_location_spec.version_fixed?
       end
     end
 
@@ -212,6 +243,5 @@ module ChefDK
       CookbookOmnifetch.storage_path
     end
 
-
   end
 end

data/lib/chef-dk/policyfile_lock.rb

@@ -15,229 +15,173 @@
 # limitations under the License.
 #
 
-require 'chef-dk/cookbook_profiler/identifiers'
-require 'chef-dk/cookbook_profiler/null_scm'
-require 'chef-dk/cookbook_profiler/git'
+require 'chef-dk/policyfile/storage_config'
+require 'chef-dk/policyfile/cookbook_locks'
+require 'chef-dk/policyfile/solution_dependencies'
 
 module ChefDK
   class PolicyfileLock
 
-    # CachedCookbook objects represent a cookbook that has been fetched from an
-    # upstream canonical source and stored (presumed unmodified).
-    class CachedCookbook
-
-      # The cookbook name (without any version or other info suffixed)
-      attr_reader :name
-
-      # The directory name in the cookbook cache where the cookbook is stored.
-      # By convention, this should be the name of the cookbook followed by a
-      # hyphen and then some sort of version identifier (depending on the
-      # cookbook source).
-      attr_accessor :cache_key
-
-      # A URI pointing to the canonical source of the cookbook.
-      attr_accessor :origin
-
-      # A string that uniquely identifies the cookbook version. If not
-      # explicitly set, an identifier is generated based on the cookbook's
-      # content.
-      attr_writer :identifier
-
-      # A string in "X.Y.Z" version number format that uniquely identifies the
-      # cookbook version. This is for compatibility with Chef Server 11.x,
-      # where cookbooks are stored by x.y.z version numbers.
-      attr_writer :dotted_decimal_identifier
-
-      # The root of the cookbook cache.
-      attr_reader :cache_path
-
-      def initialize(name, cache_path)
-        @name = name
-        @cache_path = cache_path
-        @origin = nil
-        @cache_key = nil
-        @identifier = nil
-        @dotted_decimal_identifier = nil
-      end
-
-      def cookbook_path
-        File.join(cache_path, cache_key)
-      end
-
-      def identifier
-        @identifier || identifiers.content_identifier
-      end
-
-      def dotted_decimal_identifier
-        @dotted_decimal_identifier || identifiers.dotted_decimal_identifier
-      end
-
-      def to_lock
-        validate!
-        {
-          "version" => identifiers.semver_version,
-          "identifier" => identifier,
-          "dotted_decimal_identifier" => dotted_decimal_identifier,
-          "cache_key" => cache_key,
-          "origin" => origin
-        }
-      end
-
-      def identifiers
-        @identifiers ||= CookbookProfiler::Identifiers.new(cookbook_path)
-      end
-
-      def validate!
-        unless File.exist?(cookbook_path)
-          raise CachedCookbookNotFound, "Cookbook `#{name}' not found at expected cache location `#{cookbook_path}'"
-        end
-      end
-
-    end
-
-    # LocalCookbook objects represent cookbooks that are sourced from the local
-    # filesystem and are assumed to be under active development.
-    class LocalCookbook
-
-      # A relative or absolute path to the cookbook. If a relative path is
-      # given, it is resolved relative to #relative_paths_root
-      attr_accessor :source
-
-      # A string that uniquely identifies the cookbook version. If not
-      # explicitly set, an identifier is generated based on the cookbook's
-      # content.
-      attr_writer :identifier
-
-      # A string in "X.Y.Z" version number format that uniquely identifies the
-      # cookbook version. This is for compatibility with Chef Server 11.x,
-      # where cookbooks are stored by x.y.z version numbers.
-      attr_writer :dotted_decimal_identifier
-
-      # The root path from which source is expanded.
-      attr_accessor :relative_paths_root
-
-      def initialize(name, relative_paths_root)
-        @name = name
-        @identifier = nil
-        @relative_paths_root = relative_paths_root
-      end
-
-      def cookbook_path
-        File.expand_path(source, relative_paths_root)
-      end
-
-      def scm_profiler
-        if File.exist?(File.join(cookbook_path, ".git"))
-          CookbookProfiler::Git.new(cookbook_path)
-        else
-          CookbookProfiler::NullSCM.new(cookbook_path)
-        end
-      end
-
-      def identifier
-        @identifier || identifiers.content_identifier
-      end
-
-      def dotted_decimal_identifier
-        @dotted_decimal_identifier || identifiers.dotted_decimal_identifier
-      end
-
-      def to_lock
-
-        {
-          "version" => identifiers.semver_version,
-          "identifier" => identifier,
-          "dotted_decimal_identifier" => dotted_decimal_identifier,
-          "source" => source,
-          "cache_key" => nil,
-          "scm_info" => scm_profiler.profile_data
-        }
-      end
 
-      def identifiers
-        @identifiers ||= CookbookProfiler::Identifiers.new(cookbook_path)
-      end
-
-    end
-
-    def self.build(options = {})
-      lock = new(options)
+    def self.build(storage_config)
+      lock = new(storage_config)
       yield lock
       lock
     end
 
-    def self.build_from_compiler(compiler, options = {})
-      lock = new(options)
+    def self.build_from_compiler(compiler, storage_config)
+      lock = new(storage_config)
       lock.build_from_compiler(compiler)
       lock
     end
 
+    include Policyfile::StorageConfigDelegation
+
     attr_accessor :name
     attr_accessor :run_list
 
+    attr_reader :solution_dependencies
+
+    attr_reader :storage_config
+
     attr_reader :cookbook_locks
-    attr_reader :cache_path
-    attr_reader :relative_paths_root
 
-    def initialize(options = {})
+    def initialize(storage_config)
       @name = nil
       @run_list = []
       @cookbook_locks = {}
       @relative_paths_root = Dir.pwd
-      handle_options(options)
+      @storage_config = storage_config
+
+      @solution_dependencies = Policyfile::SolutionDependencies.new
+    end
+
+    def lock_data_for(cookbook_name)
+      @cookbook_locks[cookbook_name]
     end
 
     def cached_cookbook(name)
-      cached_cookbook = CachedCookbook.new(name, cache_path)
-      yield cached_cookbook
+      cached_cookbook = Policyfile::CachedCookbook.new(name, storage_config)
+      yield cached_cookbook if block_given?
       @cookbook_locks[name] = cached_cookbook
     end
 
     def local_cookbook(name)
-      local_cookbook = LocalCookbook.new(name, relative_paths_root)
-      yield local_cookbook
+      local_cookbook = Policyfile::LocalCookbook.new(name, storage_config)
+      yield local_cookbook if block_given?
       @cookbook_locks[name] = local_cookbook
     end
 
+    def dependencies
+      yield solution_dependencies
+    end
+
     def to_lock
       {}.tap do |lock|
         lock["name"] = name
         lock["run_list"] = run_list
         lock["cookbook_locks"] = cookbook_locks_for_lockfile
+        lock["solution_dependencies"] = solution_dependencies.to_lock
       end
     end
 
     def cookbook_locks_for_lockfile
-      cookbook_locks.inject({}) do |locks_map, (name, cookbook_spec)|
-        locks_map[name] = cookbook_spec.to_lock
+      cookbook_locks.inject({}) do |locks_map, (name, location_spec)|
+        location_spec.validate!
+        location_spec.gather_profile_data
+        locks_map[name] = location_spec.to_lock
         locks_map
       end
     end
 
+    def validate_cookbooks!
+      cookbook_locks.each do |name, cookbook_lock|
+        cookbook_lock.validate!
+        cookbook_lock.refresh!
+      end
+
+      # Check that versions and dependencies are still valid. First we need to
+      # refresh the dependency info for everything that has changed, then we
+      # check that the new versions and dependencies are valid for the working
+      # set of cookbooks. We can't do this in a single loop because the user
+      # may have modified two cookbooks such that the versions and constraints
+      # are only valid when both changes are considered together.
+      cookbook_locks.each do |name, cookbook_lock|
+        if cookbook_lock.updated?
+          solution_dependencies.update_cookbook_dep(name, cookbook_lock.version, cookbook_lock.dependencies)
+        end
+      end
+      cookbook_locks.each do |name, cookbook_lock|
+        if cookbook_lock.updated?
+          solution_dependencies.test_conflict!(cookbook_lock.name, cookbook_lock.version)
+        end
+      end
+
+      true
+    end
+
     def build_from_compiler(compiler)
-      @run_list = compiler.expanded_run_list
+      @name = compiler.name
+
+      @run_list = compiler.normalized_run_list
 
-      compiler.all_cookbook_specs.each do |cookbook_name, spec|
+      compiler.all_cookbook_location_specs.each do |cookbook_name, spec|
         if spec.mirrors_canonical_upstream?
           cached_cookbook(cookbook_name) do |cached_cb|
             cached_cb.cache_key = spec.cache_key
             cached_cb.origin = spec.uri
+            cached_cb.source_options = spec.source_options_for_lock
           end
         else
           local_cookbook(cookbook_name) do |local_cb|
             local_cb.source = spec.relative_path
-            local_cb.relative_paths_root = spec.relative_paths_root
+            local_cb.source_options = spec.source_options_for_lock
           end
         end
       end
+
+      @solution_dependencies = compiler.solution_dependencies
+
+      self
+    end
+
+    def build_from_lock_data(lock_data)
+      @name = lock_data["name"]
+      @run_list = lock_data["run_list"]
+      lock_data["cookbook_locks"].each do |name, lock_info|
+        build_cookbook_lock_from_lock_data(name, lock_info)
+      end
+
+      s = Policyfile::SolutionDependencies.from_lock(lock_data["solution_dependencies"])
+      @solution_dependencies = s
       self
     end
 
+    def install_cookbooks
+      # note: duplicates PolicyfileCompiler#ensure_cache_dir_exists
+      ensure_cache_dir_exists
+
+      cookbook_locks.each do |cookbook_name, cookbook_lock|
+        cookbook_lock.install_locked
+      end
+    end
+
+    def ensure_cache_dir_exists
+      # note: duplicates PolicyfileCompiler#ensure_cache_dir_exists
+      unless File.exist?(cache_path)
+        FileUtils.mkdir_p(cache_path)
+      end
+    end
+
     private
 
-    def handle_options(options)
-      @cache_path = options[:cache_path]
-      @relative_paths_root = options[:relative_paths_root] if options.key?(:relative_paths_root)
+    def build_cookbook_lock_from_lock_data(name, lock_info)
+      if lock_info["cache_key"].nil?
+        local_cookbook(name).build_from_lock_data(lock_info)
+      else
+        cached_cookbook(name).build_from_lock_data(lock_info)
+      end
     end
+
   end
 end