chef-dk 0.2.0

data/lib/chef-dk/policyfile/chef_server_cookbook_source.rb

@@ -0,0 +1,46 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef-dk/exceptions'
+
+module ChefDK
+  module Policyfile
+    class ChefServerCookbookSource
+
+      attr_reader :uri
+
+      def initialize(uri)
+        @uri = uri
+      end
+
+      def ==(other)
+        other.kind_of?(self.class) && other.uri == uri
+      end
+
+      def universe_graph
+        raise UnsupportedFeature, 'ChefDK does not support chef-server cookbook default sources at this time'
+      end
+
+      def source_options_for(cookbook_name, cookbook_version)
+        raise UnsupportedFeature, 'ChefDK does not support chef-server cookbook default sources at this time'
+      end
+
+    end
+  end
+end
+
+

data/lib/chef-dk/policyfile/community_cookbook_source.rb

@@ -0,0 +1,84 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'json'
+require 'chef-dk/cookbook_omnifetch'
+require 'chef-dk/exceptions'
+# TODO: chef bug. Chef::HTTP::Simple needs to require this itself.
+# Fixed in 2ed829e661f9a357fc9a8cdf316c84f077dad7f9 waiting for that to be
+# released...
+require 'tempfile'
+require 'chef/platform/query_helpers' # should be handled by http/simple
+require 'chef/http/cookie_manager' # should be handled by http/simple
+require 'chef/http/validate_content_length' # should be handled by http/simple
+require 'chef/http/simple'
+
+# TODO: fix hardcoding
+Chef::Config.ssl_verify_mode = :verify_peer
+
+module ChefDK
+  module Policyfile
+
+    class CommunityCookbookSource
+
+      attr_reader :uri
+
+      def initialize(uri = nil)
+        @uri = uri || "https://supermarket.getchef.com"
+        @http_connections = {}
+      end
+
+      def ==(other)
+        other.kind_of?(self.class) && other.uri == uri
+      end
+
+      def universe_graph
+        @universe_graph ||= begin
+          full_community_graph.inject({}) do |normalized_graph, (cookbook_name, metadata_by_version)|
+            normalized_graph[cookbook_name] = metadata_by_version.inject({}) do |deps_by_version, (version, metadata)|
+              deps_by_version[version] = metadata["dependencies"]
+              deps_by_version
+            end
+            normalized_graph
+          end
+        end
+      end
+
+      def source_options_for(cookbook_name, cookbook_version)
+        base_uri = full_community_graph[cookbook_name][cookbook_version]["download_url"]
+        { artifactserver: base_uri, version: cookbook_version }
+      end
+
+      private
+
+      def http_connection_for(base_url)
+        @http_connections[base_url] ||= Chef::HTTP::Simple.new(base_url)
+      end
+
+
+      def full_community_graph
+        @full_community_graph ||=
+          begin
+            graph_json = http_connection_for(uri).get("/universe")
+            JSON.parse(graph_json)
+          end
+      end
+
+    end
+  end
+end
+

data/lib/chef-dk/policyfile/cookbook_sources.rb

@@ -0,0 +1,20 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef-dk/policyfile/null_cookbook_source'
+require 'chef-dk/policyfile/community_cookbook_source'
+require 'chef-dk/policyfile/chef_server_cookbook_source'

data/lib/chef-dk/policyfile/cookbook_spec.rb

@@ -0,0 +1,96 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'semverse'
+require 'cookbook-omnifetch'
+
+module ChefDK
+  module Policyfile
+
+    class CookbookSpec
+
+      SOURCE_TYPES = [:git, :github, :path, :artifactserver]
+
+      attr_reader :version_constraint
+      attr_reader :name
+      attr_reader :source_options
+      attr_reader :source_type
+
+      def initialize(name, version_constraint, source_options, policyfile_filename)
+        @name = name
+        @version_constraint = Semverse::Constraint.new(version_constraint)
+        @source_options = source_options
+        @policyfile_filename = policyfile_filename
+        @source_type = SOURCE_TYPES.find { |type| source_options.key?(type) }
+      end
+
+      def ==(other)
+        other.kind_of?(self.class) and
+          other.name == name and
+          other.version_constraint == version_constraint and
+          other.source_options == source_options
+      end
+
+      def mirrors_canonical_upstream?
+        [:git, :github, :artifactserver].include?(source_type)
+      end
+
+      def ensure_cached
+        unless installer.installed?
+          installer.install
+        end
+      end
+
+      def installer
+        @installer ||= CookbookOmnifetch.init(self, source_options)
+      end
+
+      def cache_key
+        installer.cache_key
+      end
+
+      def relative_path
+        installer.relative_path.to_s
+      end
+
+      def uri
+        installer.uri
+      end
+
+      def version_fixed?
+        [:git, :github, :path].include?(@source_type)
+      end
+
+      def version
+        cached_cookbook.version
+      end
+
+      def dependencies
+        cached_cookbook.dependencies
+      end
+
+      def cached_cookbook
+        installer.cached_cookbook
+      end
+
+      def relative_paths_root
+        File.dirname(@policyfile_filename)
+      end
+
+    end
+  end
+end

data/lib/chef-dk/policyfile/dsl.rb

@@ -0,0 +1,148 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef-dk/policyfile/cookbook_sources'
+require 'chef-dk/policyfile/cookbook_spec'
+
+module ChefDK
+  module Policyfile
+    class DSL
+
+      attr_reader :errors
+      attr_reader :run_list
+      attr_reader :default_source
+      attr_reader :policyfile_cookbook_specs
+
+      attr_accessor :policyfile_filename
+
+      def initialize
+        @errors = []
+        @run_list = []
+        @default_source = NullCookbookSource.new
+        @policyfile_cookbook_specs = {}
+        @policyfile_filename = nil
+      end
+
+      def run_list(*run_list_items)
+        run_list_items = run_list_items.flatten
+        @run_list = run_list_items unless run_list_items.empty?
+        @run_list
+      end
+
+      def default_source(source_type = nil, source_uri = nil)
+        return @default_source if source_type.nil?
+        case source_type
+        when :community
+          set_default_community_source(source_uri)
+        when :chef_server
+          set_default_chef_server_source(source_uri)
+        else
+          @errors << "Invalid default_source type '#{source_type.inspect}'"
+        end
+      end
+
+      def cookbook(name, *version_and_source_opts)
+        source_options =
+          if version_and_source_opts.last.is_a?(Hash)
+            version_and_source_opts.pop
+          else
+            {}
+          end
+
+        constraint = version_and_source_opts.first || ">= 0.0.0"
+        spec = CookbookSpec.new(name, constraint, source_options, policyfile_filename)
+
+
+        if existing_source = @policyfile_cookbook_specs[name]
+          err = "Cookbook '#{name}' assigned to conflicting sources\n\n"
+          err << "Previous source: #{existing_source.source_options.inspect}\n"
+          err << "Conflicts with: #{source_options.inspect}\n"
+          @errors << err
+        else
+          @policyfile_cookbook_specs[name] = spec
+        end
+      end
+
+      def eval_policyfile(policyfile_string, policyfile_filename)
+        @policyfile_filename = policyfile_filename
+        instance_eval(policyfile_string, policyfile_filename)
+        validate!
+        self
+      rescue SyntaxError => e
+        @errors << "Invalid ruby syntax in policyfile '#{policyfile_filename}':\n\n#{e.message}"
+      rescue SignalException, SystemExit
+        # allow signal from kill, ctrl-C, etc. to bubble up:
+        raise
+      rescue Exception => e
+        error_message = "Evaluation of policyfile '#{policyfile_filename}' raised an exception\n"
+        error_message << "  Exception: #{e.class.name} \"#{e.to_s}\"\n\n"
+        trace = filtered_bt(policyfile_filename, e)
+        error_message << "  Relevant Code:\n"
+        error_message << "    #{error_context(policyfile_string, policyfile_filename, e)}\n\n"
+        unless trace.empty?
+          error_message << "  Backtrace:\n"
+          error_message << filtered_bt(policyfile_filename, e).inject("") { |formatted_trace, line| formatted_trace << "    #{line}" }
+          error_message << "\n"
+        end
+        @errors << error_message
+      end
+
+      private
+
+      def set_default_community_source(source_uri)
+        @default_source = CommunityCookbookSource.new(source_uri)
+      end
+
+      def set_default_chef_server_source(source_uri)
+        if source_uri.nil?
+          @errors << "You must specify the server's URI when using a default_source :chef_server"
+        else
+          @default_source = ChefServerCookbookSource.new(source_uri)
+        end
+      end
+
+      def validate!
+        if @run_list.empty?
+          @errors << "Invalid run_list. run_list cannot be empty"
+        end
+      end
+
+      def error_context(policyfile_string, policyfile_filename, exception)
+        if line_number_to_show = culprit_line_number(policyfile_filename, exception)
+          code = policyfile_string.lines.to_a[line_number_to_show - 1].strip
+          "#{line_number_to_show}: #{code}"
+        else
+          "Could not find relevant code from backtrace"
+        end
+      end
+
+      def culprit_line_number(policyfile_filename, exception)
+        if most_proximate_backtrace_line = filtered_bt(policyfile_filename, exception).first
+          most_proximate_backtrace_line[/^(?:.\:)?[^:]+:([\d]+)/,1].to_i
+        else
+          nil
+        end
+      end
+
+      def filtered_bt(policyfile_filename, exception)
+        policyfile_filename_matcher = /^#{Regexp.escape(policyfile_filename)}/
+        exception.backtrace.select {|line| line =~ policyfile_filename_matcher }
+      end
+
+    end
+  end
+end

data/lib/chef-dk/policyfile/null_cookbook_source.rb

@@ -0,0 +1,37 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef-dk/exceptions'
+
+module ChefDK
+  module Policyfile
+    class NullCookbookSource
+
+      def initialize(_uri=nil)
+      end
+
+      def universe_graph
+        {}
+      end
+
+      def source_options_for(cookbook_name, cookbook_version)
+        raise UnsupportedFeature, 'You must set a default_source in your Policyfile to download cookbooks without explicit sources'
+      end
+
+    end
+  end
+end

data/lib/chef-dk/policyfile_compiler.rb

@@ -0,0 +1,217 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'forwardable'
+
+require 'solve'
+require 'chef/run_list/run_list_item'
+
+require 'chef-dk/policyfile/dsl'
+require 'chef-dk/policyfile_lock'
+
+module ChefDK
+
+  class PolicyfileCompiler
+
+    extend Forwardable
+
+    DEFAULT_DEMAND_CONSTRAINT = '>= 0.0.0'.freeze
+
+    # Cookbooks from these sources lock that cookbook to exactly one version
+    SOURCE_TYPES_WITH_FIXED_VERSIONS = [:git, :path].freeze
+
+    def self.evaluate(policyfile_string, policyfile_filename)
+      compiler = new
+      compiler.evaluate_policyfile(policyfile_string, policyfile_filename)
+      compiler
+    end
+
+    def_delegator :@dsl, :run_list
+    def_delegator :@dsl, :errors
+    def_delegator :@dsl, :default_source
+    def_delegator :@dsl, :policyfile_cookbook_specs
+
+    attr_reader :dsl
+
+    def initialize
+      @dsl = Policyfile::DSL.new
+      @artifact_server_cookbook_specs = {}
+    end
+
+    def error!
+      unless errors.empty?
+        raise PolicyfileError, errors.join("\n")
+      end
+    end
+
+    def cookbook_spec_for(cookbook_name)
+      policyfile_cookbook_specs[cookbook_name]
+    end
+
+    def expanded_run_list
+      run_list
+    end
+
+    def lock
+      @policyfile_lock ||= PolicyfileLock.build_from_compiler(self, cache_path: cache_path)
+    end
+
+    def install
+      ensure_cache_dir_exists
+
+      graph_solution.each do |cookbook_name, version|
+        spec = cookbook_spec_for(cookbook_name)
+        if spec.nil? or !spec.version_fixed?
+          spec = create_spec_for_cookbook(cookbook_name, version)
+          spec.ensure_cached
+        end
+      end
+    end
+
+    def create_spec_for_cookbook(cookbook_name, version)
+      source_options = default_source.source_options_for(cookbook_name, version)
+      spec = Policyfile::CookbookSpec.new(cookbook_name, "= #{version}", source_options, dsl)
+      @artifact_server_cookbook_specs[cookbook_name] = spec
+    end
+
+    def all_cookbook_specs
+      # in the installation proces, we create "artifact_server_cookbook_specs"
+      # for any cookbook that isn't sourced from a single-version source (e.g.,
+      # path and git only support one version at a time), but we might have
+      # specs for them to track additional version constraint demands. Merging
+      # in this order ensures the artifact_server_cookbook_specs "win".
+      policyfile_cookbook_specs.merge(@artifact_server_cookbook_specs)
+    end
+
+    ##
+    # Compilation Methods
+    ##
+
+    def graph_solution
+      return @solution if @solution
+      cache_fixed_version_cookbooks
+      @solution = Solve.it!(graph, graph_demands)
+    end
+
+    def graph
+      @graph ||= Solve::Graph.new.tap do |g|
+        artifacts_graph.each do |name, dependencies_by_version|
+          dependencies_by_version.each do |version, dependencies|
+            artifact = g.artifact(name, version)
+            dependencies.each do |dep_name, constraint|
+              artifact.dependency(dep_name, constraint)
+            end
+          end
+        end
+      end
+    end
+
+    def graph_demands
+      cookbooks_for_demands.map do |cookbook_name|
+        spec = cookbook_spec_for(cookbook_name)
+        if spec.nil?
+          [ cookbook_name, DEFAULT_DEMAND_CONSTRAINT ]
+        elsif spec.version_fixed?
+          [ cookbook_name, "= #{spec.version}" ]
+        else
+          [ cookbook_name, spec.version_constraint.to_s ]
+        end
+      end
+    end
+
+    def artifacts_graph
+      remote_artifacts_graph.merge(local_artifacts_graph)
+    end
+
+    # Gives a dependency graph for cookbooks that are source from an alternate
+    # location. These cookbooks could have a different set of dependencies
+    # compared to an unmodified copy upstream. For example, the community site
+    # may have a cookbook "apache2" at version "1.10.4", which the user has
+    # forked on github and modified the dependencies without changing the
+    # version number. To accomodate this, the local_artifacts_graph should be
+    # merged over the upstream's artifacts graph.
+    def local_artifacts_graph
+      policyfile_cookbook_specs.inject({}) do |local_artifacts, (cookbook_name, cookbook_spec)|
+        if cookbook_spec.version_fixed?
+          local_artifacts[cookbook_name] = { cookbook_spec.version => cookbook_spec.dependencies }
+        end
+        local_artifacts
+      end
+    end
+
+    def remote_artifacts_graph
+      default_source.universe_graph
+    end
+
+    def version_constraint_for(cookbook_name)
+      if (cookbook_spec = cookbook_spec_for(cookbook_name)) and cookbook_spec.version_fixed?
+        version = cookbook_spec.version
+        "= #{version}"
+      else
+        DEFAULT_DEMAND_CONSTRAINT
+      end
+    end
+
+    def cookbook_version_fixed?(cookbook_name)
+      if cookbook_spec = cookbook_spec_for(cookbook_name)
+        cookbook_spec.version_fixed?
+      else
+        false
+      end
+    end
+
+    def cookbooks_in_run_list
+      run_list.map {|item_spec| Chef::RunList::RunListItem.new(item_spec).name }
+    end
+
+    def build
+      yield @dsl
+      self
+    end
+
+    def evaluate_policyfile(policyfile_string, policyfile_filename)
+      @dsl.eval_policyfile(policyfile_string, policyfile_filename)
+      self
+    end
+
+    private
+
+    def cookbooks_for_demands
+      (cookbooks_in_run_list + policyfile_cookbook_specs.keys).uniq
+    end
+
+    def cache_fixed_version_cookbooks
+      ensure_cache_dir_exists
+
+      policyfile_cookbook_specs.each do |_cookbook_name, cookbook_spec|
+        cookbook_spec.ensure_cached if cookbook_spec.version_fixed?
+      end
+    end
+
+    def ensure_cache_dir_exists
+      unless File.exist?(cache_path)
+        FileUtils.mkdir_p(cache_path)
+      end
+    end
+
+    def cache_path
+      CookbookOmnifetch.storage_path
+    end
+
+
+  end
+end