berkshelf 5.2.0 → 8.0.15

data/lib/berkshelf/berksfile.rb

@@ -1,5 +1,8 @@
 require_relative "packager"
 
+require "chef/cookbook/chefignore"
+require "chef/util/path_helper"
+
 module Berkshelf
   class Berksfile
     class << self
@@ -9,7 +12,8 @@ module Berkshelf
       # @param (see Berksfile#initialize)
       def from_options(options = {})
         options[:berksfile] ||= File.join(Dir.pwd, Berkshelf::DEFAULT_FILENAME)
-        from_file(options[:berksfile], options.slice(:except, :only, :delete))
+        symbolized = Hash[options.map { |k, v| [k.to_sym, v] }]
+        from_file(options[:berksfile], symbolized.select { |k,| %i{except only delete}.include? k })
       end
 
       # @param [#to_s] file
@@ -31,7 +35,7 @@ module Berkshelf
 
     # Don't vendor VCS files.
     # Reference GNU tar --exclude-vcs: https://www.gnu.org/software/tar/manual/html_section/tar_49.html
-    EXCLUDED_VCS_FILES_WHEN_VENDORING = ['.arch-ids', '{arch}', '.bzr', '.bzrignore', '.bzrtags', 'CVS', '.cvsignore', '_darcs', '.git', '.hg', '.hgignore', '.hgrags', 'RCS', 'SCCS', '.svn', '**/.git'].freeze
+    EXCLUDED_VCS_FILES_WHEN_VENDORING = [".arch-ids", "{arch}", ".bzr", ".bzrignore", ".bzrtags", "CVS", ".cvsignore", "_darcs", ".git", ".hg", ".hgignore", ".hgrags", "RCS", "SCCS", ".svn", "**/.git", "**/.svn"].freeze
 
     include Mixin::Logging
     include Cleanroom
@@ -62,8 +66,8 @@ module Berkshelf
     #   group to be installed and all others to be ignored
     def initialize(path, options = {})
       @filepath         = File.expand_path(path)
-      @dependencies     = Hash.new
-      @sources          = Hash.new
+      @dependencies     = {}
+      @sources          = {}
       @delete           = options[:delete]
 
       # defaults for what solvers to use
@@ -71,7 +75,7 @@ module Berkshelf
       @preferred_solver = :gecode
 
       if options[:except] && options[:only]
-        raise ArgumentError, 'Cannot specify both :except and :only!'
+        raise ArgumentError, "Cannot specify both :except and :only!"
       elsif options[:except]
         except = Array(options[:except]).collect(&:to_sym)
         @filter = ->(dependency) { (except & dependency.groups).empty? }
@@ -114,7 +118,7 @@ module Berkshelf
     #   cookbook 'artifact', path: '/Users/reset/code/artifact'
     #
     # @example a cookbook dependency that will be retrieved from a Git server
-    #   cookbook 'artifact', git: 'git://github.com/RiotGames/artifact-cookbook.git'
+    #   cookbook 'artifact', git: 'https://github.com/chef/artifact-cookbook.git'
     #
     # @overload cookbook(name, version_constraint, options = {})
     #   @param [#to_s] name
@@ -142,7 +146,7 @@ module Berkshelf
     #   @see PathLocation
     #   @see GitLocation
     def cookbook(*args)
-      options = args.last.is_a?(Hash) ? args.pop : Hash.new
+      options = args.last.is_a?(Hash) ? args.pop : {}
       name, constraint = args
 
       options[:path] &&= File.expand_path(options[:path], File.dirname(filepath))
@@ -152,7 +156,7 @@ module Berkshelf
         options[:group] += @active_group
       end
 
-      add_dependency(name, constraint, options)
+      add_dependency(name, constraint, **options)
     end
     expose :cookbook
 
@@ -172,9 +176,12 @@ module Berkshelf
     # @option options [String] :path
     #   path to the metadata file
     def metadata(options = {})
-      path          = options[:path] || File.dirname(filepath)
-      metadata_path = File.expand_path(File.join(path, 'metadata.rb'))
-      metadata      = Ridley::Chef::Cookbook::Metadata.from_file(metadata_path)
+      path = options[:path] || File.dirname(filepath)
+
+      loader = Chef::Cookbook::CookbookVersionLoader.new(path)
+      loader.load!
+      cookbook_version = loader.cookbook_version
+      metadata = cookbook_version.metadata
 
       add_dependency(metadata.name, nil, path: path, metadata: true)
     end
@@ -191,11 +198,15 @@ module Berkshelf
     # @param [String] api_url
     #   url for the api to add
     #
+    # @param [Hash] options
+    #   extra source options
+    #
     # @raise [InvalidSourceURI]
     #
     # @return [Array<Source>]
-    def source(api_url)
-      @sources[api_url] = Source.new(api_url)
+    def source(api_url, **options)
+      source = Source.new(self, api_url, **options)
+      @sources[source.uri.to_s] = source
     end
     expose :source
 
@@ -249,35 +260,6 @@ module Berkshelf
       sources.find { |source| source.cookbook(name, version) }
     end
 
-    # @todo remove in Berkshelf 4.0
-    #
-    # @raise [DeprecatedError]
-    def site(*args)
-      if args.first == :opscode
-        Berkshelf.formatter.deprecation "Your Berksfile contains a site location pointing to the Opscode Community " +
-          "Site (site :opscode). Site locations have been replaced by the source location. Change this to: " +
-          "'source \"https://supermarket.chef.io\"' to remove this warning. For more information visit " +
-          "https://github.com/berkshelf/berkshelf/wiki/deprecated-locations"
-        source(DEFAULT_API_URL)
-        return
-      end
-
-      raise DeprecatedError.new "Your Berksfile contains a site location. Site locations have been " +
-        " replaced by the source location. Please remove your site location and try again. For more information " +
-        " visit https://github.com/berkshelf/berkshelf/wiki/deprecated-locations"
-    end
-    expose :site
-
-    # @todo remove in Berkshelf 4.0
-    #
-    # @raise [DeprecatedError]
-    def chef_api(*args)
-      raise DeprecatedError.new "Your Berksfile contains a chef_api location. Chef API locations have " +
-        " been replaced by the source location. Please remove your site location and try again. For more " +
-        " information visit https://github.com/berkshelf/berkshelf/wiki/deprecated-locations"
-    end
-    expose :chef_api
-
     # Add a dependency of the given name and constraint to the array of dependencies.
     #
     # @param [String] name
@@ -300,14 +282,15 @@ module Berkshelf
       if @dependencies[name]
         # Only raise an exception if the dependency is a true duplicate
         groups = (options[:group].nil? || options[:group].empty?) ? [:default] : options[:group]
-        if !(@dependencies[name].groups & groups).empty?
+        unless (@dependencies[name].groups & groups).empty?
           raise DuplicateDependencyDefined.new(name)
         end
       end
 
-      if options[:path]
-        metadata_file = File.join(options[:path], 'metadata.rb')
-      end
+      # this appears to be dead code
+      # if options[:path]
+      #  metadata_file = File.join(options[:path], "metadata.rb")
+      # end
 
       options[:constraint] = constraint
 
@@ -444,7 +427,7 @@ module Berkshelf
       end
 
       # NOTE: We intentionally do NOT pass options to the installer
-      self.install
+      install
     end
 
     # Retrieve information about a given cookbook that is installed by this Berksfile.
@@ -485,6 +468,10 @@ module Berkshelf
     # List of all the cookbooks which have a newer version found at a source
     # that satisfies the constraints of your dependencies.
     #
+    # @param [Boolean] include_non_satisfying
+    #   include cookbooks that would not satisfy the given constraints in the
+    #   +Berksfile+. Defaults to false.
+    #
     # @return [Hash]
     #   a hash of cached cookbooks and their latest version grouped by their
     #   remote API source. The hash will be empty if there are no newer
@@ -500,7 +487,7 @@ module Berkshelf
     #       }
     #     }
     #   }
-    def outdated(*names)
+    def outdated(*names, include_non_satisfying: false)
       validate_lockfile_present!
       validate_lockfile_trusted!
       validate_dependencies_installed!
@@ -511,16 +498,16 @@ module Berkshelf
           cookbooks = source.versions(name)
 
           latest = cookbooks.select do |cookbook|
-            dependency.version_constraint.satisfies?(cookbook.version) &&
-            Semverse::Version.coerce(cookbook.version) > dependency.locked_version
-          end.sort_by { |cookbook| cookbook.version }.last
+            (include_non_satisfying || dependency.version_constraint.satisfies?(cookbook.version)) &&
+              Semverse::Version.coerce(cookbook.version) > dependency.locked_version
+          end.max_by(&:version)
 
           unless latest.nil?
             hash[name] ||= {
-              'local' => dependency.locked_version,
-              'remote' => {
-                source => Semverse::Version.coerce(latest.version)
-              }
+              "local" => dependency.locked_version,
+              "remote" => {
+                source => Semverse::Version.coerce(latest.version),
+              },
             }
           end
         end
@@ -609,7 +596,7 @@ module Berkshelf
       packager.validate!
 
       outdir = Dir.mktmpdir do |temp_dir|
-        Berkshelf.ui.mute { vendor(File.join(temp_dir, 'cookbooks')) }
+        Berkshelf.ui.mute { vendor(File.join(temp_dir, "cookbooks")) }
         packager.run(temp_dir)
       end
 
@@ -617,6 +604,25 @@ module Berkshelf
       outdir
     end
 
+    # backcompat with ridley lookup of chefignore
+    def find_chefignore(path)
+      filename = "chefignore"
+
+      Pathname.new(path).ascend do |dir|
+        next unless dir.directory?
+
+        [
+          dir.join(filename),
+          dir.join("cookbooks", filename),
+          dir.join(".chef",     filename),
+        ].each do |possible|
+          return possible.expand_path.to_s if possible.exist?
+        end
+      end
+
+      nil
+    end
+
     # Install the Berksfile or Berksfile.lock and then sync the cached cookbooks
     # into directories within the given destination matching their name.
     #
@@ -626,32 +632,42 @@ module Berkshelf
     # @return [String, nil]
     #   the expanded path cookbooks were vendored to or nil if nothing was vendored
     def vendor(destination)
-      Dir.mktmpdir('vendor') do |scratch|
-        chefignore       = nil
+      Dir.mktmpdir("vendor") do |scratch|
         cached_cookbooks = install
-        raw_metadata_files = []
 
         return nil if cached_cookbooks.empty?
 
         cached_cookbooks.each do |cookbook|
           Berkshelf.formatter.vendor(cookbook, destination)
+
           cookbook_destination = File.join(scratch, cookbook.cookbook_name)
           FileUtils.mkdir_p(cookbook_destination)
 
           # Dir.glob does not support backslash as a File separator
-          src   = cookbook.path.to_s.gsub('\\', '/')
-          files = FileSyncer.glob(File.join(src, '*'))
+          src   = cookbook.path.to_s.tr("\\", "/")
+          files = FileSyncer.glob(File.join(src, "**/*"))
 
-          chefignore = Ridley::Chef::Chefignore.new(cookbook.path.to_s) rescue nil
-          chefignore.apply!(files) if chefignore
+          # strip directories
+          files.reject! { |file_path| File.directory?(file_path) }
 
-          unless cookbook.compiled_metadata?
-            cookbook.compile_metadata(cookbook_destination)
-          end
+          # convert to relative Pathname objects for chefignore
+          files.map! { |file_path| Chef::Util::PathHelper.relative_path_from(cookbook.path.to_s, file_path) }
+
+          chefignore = Chef::Cookbook::Chefignore.new(find_chefignore(cookbook.path.to_s) || cookbook.path.to_s)
+
+          # apply chefignore
+          files.reject! { |file_path| chefignore.ignored?(file_path) }
 
-          raw_metadata_files << File::join(cookbook.cookbook_name, 'metadata.rb')
+          # convert Pathname objects back to strings
+          files.map!(&:to_s)
 
-          FileUtils.cp_r(files, cookbook_destination)
+          # copy each file to destination
+          files.each do |rpath|
+            FileUtils.mkdir_p( File.join(cookbook_destination, File.dirname(rpath)) )
+            FileUtils.cp( File.join(cookbook.path.to_s, rpath), File.join(cookbook_destination, rpath) )
+          end
+
+          cookbook.compile_metadata(cookbook_destination)
         end
 
         # Don't vendor the raw metadata (metadata.rb). The raw metadata is
@@ -669,7 +685,7 @@ module Berkshelf
         #
         #   * https://tickets.opscode.com/browse/CHEF-4811
         #   * https://tickets.opscode.com/browse/CHEF-4810
-        FileSyncer.sync(scratch, destination, exclude: raw_metadata_files + EXCLUDED_VCS_FILES_WHEN_VENDORING, delete: @delete)
+        FileSyncer.sync(scratch, destination, exclude: EXCLUDED_VCS_FILES_WHEN_VENDORING, delete: @delete)
       end
 
       destination
@@ -694,12 +710,21 @@ module Berkshelf
     #
     # @return [String] path
     #   the path where the image was written
-    def viz(outfile = nil)
-      outfile = File.join(Dir.pwd, outfile || 'graph.png')
+    def viz(outfile = nil, format = "png")
+      outfile = File.join(Dir.pwd, outfile || "graph.png")
 
       validate_lockfile_present!
       validate_lockfile_trusted!
-      Visualizer.from_lockfile(lockfile).to_png(outfile)
+      vizualiser = Visualizer.from_lockfile(lockfile)
+
+      case format
+      when "dot"
+        vizualiser.to_dot_file(outfile)
+      when "png"
+        vizualiser.to_png(outfile)
+      else
+        raise ConfigurationError, "Vizualiser format #{format} not recognised."
+      end
     end
 
     # Get the lockfile corresponding to this Berksfile. This is necessary because
@@ -715,62 +740,64 @@ module Berkshelf
 
     private
 
-      # Ensure the lockfile is present on disk.
-      #
-      # @raise [LockfileNotFound]
-      #   if the lockfile does not exist on disk
-      #
-      # @return [true]
-      def validate_lockfile_present!
-        raise LockfileNotFound unless lockfile.present?
-        true
-      end
+    # Ensure the lockfile is present on disk.
+    #
+    # @raise [LockfileNotFound]
+    #   if the lockfile does not exist on disk
+    #
+    # @return [true]
+    def validate_lockfile_present!
+      raise LockfileNotFound unless lockfile.present?
 
-      # Ensure that all dependencies defined in the Berksfile exist in this
-      # lockfile.
-      #
-      # @raise [LockfileOutOfSync]
-      #   if there are dependencies specified in the Berksfile which do not
-      #   exist (or are not satisifed by) the lockfile
-      #
-      # @return [true]
-      def validate_lockfile_trusted!
-        raise LockfileOutOfSync unless lockfile.trusted?
-        true
-      end
+      true
+    end
 
-      # Ensure that all dependencies in the lockfile are installed on this
-      # system. You should validate that the lockfile can be trusted before
-      # using this method.
-      #
-      # @raise [DependencyNotInstalled]
-      #   if the dependency in the lockfile is not in the Berkshelf shelf on
-      #   this system
-      #
-      # @return [true]
-      def validate_dependencies_installed!
-        lockfile.graph.locks.each do |_, dependency|
-          unless dependency.installed?
-            raise DependencyNotInstalled.new(dependency)
-          end
-        end
+    # Ensure that all dependencies defined in the Berksfile exist in this
+    # lockfile.
+    #
+    # @raise [LockfileOutOfSync]
+    #   if there are dependencies specified in the Berksfile which do not
+    #   exist (or are not satisifed by) the lockfile
+    #
+    # @return [true]
+    def validate_lockfile_trusted!
+      raise LockfileOutOfSync unless lockfile.trusted?
 
-        true
+      true
+    end
+
+    # Ensure that all dependencies in the lockfile are installed on this
+    # system. You should validate that the lockfile can be trusted before
+    # using this method.
+    #
+    # @raise [DependencyNotInstalled]
+    #   if the dependency in the lockfile is not in the Berkshelf shelf on
+    #   this system
+    #
+    # @return [true]
+    def validate_dependencies_installed!
+      lockfile.graph.locks.each do |_, dependency|
+        unless dependency.installed?
+          raise DependencyNotInstalled.new(dependency)
+        end
       end
 
-      # Determine if any cookbooks were specified that aren't in our shelf.
-      #
-      # @param [Array<String>] names
-      #   a list of cookbook names
-      #
-      # @raise [DependencyNotFound]
-      #   if a cookbook name is given that does not exist
-      def validate_cookbook_names!(names)
-        missing = names - lockfile.graph.locks.keys
+      true
+    end
 
-        unless missing.empty?
-          raise DependencyNotFound.new(missing)
-        end
+    # Determine if any cookbooks were specified that aren't in our shelf.
+    #
+    # @param [Array<String>] names
+    #   a list of cookbook names
+    #
+    # @raise [DependencyNotFound]
+    #   if a cookbook name is given that does not exist
+    def validate_cookbook_names!(names)
+      missing = names - lockfile.graph.locks.keys
+
+      unless missing.empty?
+        raise DependencyNotFound.new(missing)
       end
+    end
   end
 end

data/lib/berkshelf/cached_cookbook.rb

@@ -1,5 +1,10 @@
+require "chef/cookbook/cookbook_version_loader"
+require "chef/cookbook/syntax_check"
+require_relative "errors"
+require "chef/json_compat"
+
 module Berkshelf
-  class CachedCookbook < Ridley::Chef::Cookbook
+  class CachedCookbook
     class << self
       # @param [#to_s] path
       #   a path on disk to the location of a Cookbook downloaded by the Downloader
@@ -15,38 +20,108 @@ module Berkshelf
         loaded_cookbooks[path.to_s] ||= from_path(path)
       end
 
+      # Creates a new instance of Berkshelf::CachedCookbook from a path on disk containing
+      # a Cookbook.
+      #
+      # The name of the Cookbook is determined by the value of the name attribute set in
+      # the cookbooks' metadata. If the name attribute is not present the name of the loaded
+      # cookbook is determined by directory containing the cookbook.
+      #
+      # @param [#to_s] path
+      #   a path on disk to the location of a Cookbook
+      #
+      # @raise [IOError] if the path does not contain a metadata.rb or metadata.json file
+      #
+      # @return [Ridley::Chef::Cookbook]
+      def from_path(path)
+        path = Pathname.new(path)
+
+        new(path)
+      end
+
+      def checksum(filepath)
+        Chef::Digester.generate_md5_checksum_for_file(filepath)
+      end
+
       private
 
-        # @return [Hash<String, CachedCookbook>]
-        def loaded_cookbooks
-          @loaded_cookbooks ||= {}
+      # @return [Hash<String, CachedCookbook>]
+      def loaded_cookbooks
+        @loaded_cookbooks ||= {}
+      end
+    end
+
+    attr_writer :metadata
+    attr_accessor :path
+    attr_writer :cookbook_version
+
+    def initialize(path)
+      @path = path
+      # eagerly load to force throwing on bad metadata while constructing
+      cookbook_name
+      metadata
+    end
+
+    def loader
+      @loader ||=
+        begin
+          loader = Chef::Cookbook::CookbookVersionLoader.new(@path)
+          loader.load!
+          loader
         end
     end
 
+    def cookbook_version
+      @cookbook_version ||= loader.cookbook_version
+    end
+
+    def cookbook_name
+      @cookbook_name ||= cookbook_version.name
+    end
+
+    def metadata
+      @metadata ||= cookbook_version.metadata
+    end
+
+    def reload
+      @metadata = nil
+      @cookbook_name = nil
+      @cookbook_version = nil
+      @loader = nil
+    end
+
+    def <=>(other)
+      [cookbook_name, version] <=> [other.cookbook_name, other.version]
+    end
+
     DIRNAME_REGEXP = /^(.+)-(.+)$/.freeze
 
     extend Forwardable
+
+    def_delegator :cookbook_version, :version
+    def_delegator :metadata, :name, :cookbook_name
     def_delegator :metadata, :description
     def_delegator :metadata, :maintainer
     def_delegator :metadata, :maintainer_email
     def_delegator :metadata, :license
     def_delegator :metadata, :platforms
+    def_delegator :metadata, :name
 
     # @return [Hash]
     def dependencies
-      metadata.recommendations.merge(metadata.dependencies)
+      metadata.dependencies
     end
 
     def pretty_print
       [].tap do |a|
-        a.push "        Name: #{cookbook_name}" unless name.blank?
-        a.push "     Version: #{version}" unless version.blank?
-        a.push " Description: #{metadata.description}" unless metadata.description.blank?
-        a.push "      Author: #{metadata.maintainer}" unless metadata.maintainer.blank?
-        a.push "       Email: #{metadata.maintainer_email}" unless metadata.maintainer_email.blank?
-        a.push "     License: #{metadata.license}" unless metadata.license.blank?
-        a.push "   Platforms: #{pretty_map(metadata.platforms, 14)}" unless metadata.platforms.blank?
-        a.push "Dependencies: #{pretty_map(dependencies, 14)}" unless dependencies.blank?
+        a.push "        Name: #{cookbook_name}" if name && name =~ /\S/
+        a.push "     Version: #{version}" if version && version =~ /\S/
+        a.push " Description: #{metadata.description}" if metadata.description && metadata.description =~ /\S/
+        a.push "      Author: #{metadata.maintainer}" if metadata.maintainer && metadata.maintainer =~ /\S/
+        a.push "       Email: #{metadata.maintainer_email}" if metadata.maintainer_email && metadata.maintainer_email =~ /\S/
+        a.push "     License: #{metadata.license}" if metadata.license && metadata.license =~ /\S/
+        a.push "   Platforms: #{pretty_map(metadata.platforms, 14)}" if metadata.platforms && !metadata.platforms.empty?
+        a.push "Dependencies: #{pretty_map(dependencies, 14)}" if dependencies && !dependencies.empty?
       end.join("\n")
     end
 
@@ -62,21 +137,49 @@ module Berkshelf
     # @return [Hash]
     def pretty_hash
       {}.tap do |h|
-        h[:name]          = cookbook_name unless cookbook_name.blank?
-        h[:version]       = version unless version.blank?
-        h[:description]   = description unless description.blank?
-        h[:author]        = maintainer unless maintainer.blank?
-        h[:email]         = maintainer_email unless maintainer_email.blank?
-        h[:license]       = license unless license.blank?
-        h[:platforms]     = platforms.to_hash unless platforms.blank?
-        h[:dependencies]  = dependencies.to_hash unless dependencies.blank?
+        h[:name]          = cookbook_name if cookbook_name && cookbook_name =~ /\S/
+        h[:version]       = version if version && version =~ /\S/
+        h[:description]   = description if description && description =~ /\S/
+        h[:author]        = maintainer if maintainer && maintainer =~ /\S/
+        h[:email]         = maintainer_email if maintainer_email && maintainer_email =~ /\S/
+        h[:license]       = license if license && license =~ /\S/
+        h[:platforms]     = platforms.to_hash if platforms && !platforms.empty?
+        h[:dependencies]  = dependencies.to_hash if dependencies && !dependencies.empty?
       end
     end
 
-    private
+    def validate
+      raise IOError, "No Cookbook found at: #{path}" unless path.exist?
+
+      syntax_checker = Chef::Cookbook::SyntaxCheck.new(path.to_path)
 
-      def pretty_map(hash, padding)
-        hash.map { |k,v| "#{k} (#{v})" }.join("\n" + ' '*padding)
+      unless syntax_checker.validate_ruby_files
+        raise Berkshelf::Errors::CookbookSyntaxError, "Invalid ruby files in cookbook: #{cookbook_name} (#{version})."
       end
+      unless syntax_checker.validate_templates
+        raise Berkshelf::Errors::CookbookSyntaxError, "Invalid template files in cookbook: #{cookbook_name} (#{version})."
+      end
+
+      true
+    end
+
+    def compile_metadata(path = self.path)
+      json_file = "#{path}/metadata.json"
+      rb_file = "#{path}/metadata.rb"
+      return nil if File.exist?(json_file)
+
+      md = Chef::Cookbook::Metadata.new
+      md.from_file(rb_file)
+      f = File.open(json_file, "w")
+      f.write(Chef::JSONCompat.to_json_pretty(md))
+      f.close
+      f.path
+    end
+
+    private
+
+    def pretty_map(hash, padding)
+      hash.map { |k, v| "#{k} (#{v})" }.join("\n" + " " * padding)
+    end
   end
 end