berkshelf 6.3.4 → 7.0.0

data/lib/berkshelf/dependency.rb

@@ -22,7 +22,7 @@ module Berkshelf
     # @return [String]
     attr_reader :name
     # @return [Array<String,Symbol>]
-    attr_reader :groups
+    attr_writer :groups
     # @return [Berkshelf::Location]
     attr_reader :location
     # @return [Semverse::Version]
@@ -151,15 +151,6 @@ module Berkshelf
       groups.include?(group.to_sym)
     end
 
-    # The location for this dependency, such as a remote Chef Server, the
-    # community API, :git, or a :path location. By default, this will be the
-    # community API.
-    #
-    # @return [Berkshelf::Location]
-    def location
-      @location
-    end
-
     # The list of groups this dependency belongs to.
     #
     # @return [Array<Symbol>]

data/lib/berkshelf/downloader.rb

@@ -33,11 +33,12 @@ module Berkshelf
     #
     # @return [String]
     def download(*args, &block)
-      options = args.last.is_a?(Hash) ? args.pop : Hash.new
+      # options are ignored
+      #options = args.last.is_a?(Hash) ? args.pop : Hash.new
       dependency, version = args
 
       sources.each do |source|
-        if result = try_download(source, dependency, version)
+        if ( result = try_download(source, dependency, version) )
           if block_given?
             value = yield result
             FileUtils.rm_rf(result)
@@ -57,7 +58,7 @@ module Berkshelf
     #
     # @return [String]
     def try_download(source, name, version)
-      unless remote_cookbook = source.cookbook(name, version)
+      unless ( remote_cookbook = source.cookbook(name, version) )
         return nil
       end
 
@@ -69,6 +70,8 @@ module Berkshelf
         end
         CommunityREST.new(remote_cookbook.location_path, options).download(name, version)
       when :chef_server
+        tmp_dir      = Dir.mktmpdir
+        unpack_dir   = Pathname.new(tmp_dir) + "#{name}-#{version}"
         # @todo Dynamically get credentials for remote_cookbook.location_path
         credentials = {
           server_url: remote_cookbook.location_path,
@@ -76,10 +79,19 @@ module Berkshelf
           client_key: source.options[:client_key] || Berkshelf::Config.instance.chef.client_key,
           ssl: source.options[:ssl],
         }
-        # @todo  Something scary going on here - getting an instance of Kitchen::Logger from test-kitchen
-        # https://github.com/opscode/test-kitchen/blob/master/lib/kitchen.rb#L99
-        Celluloid.logger = nil unless ENV["DEBUG_CELLULOID"]
-        Ridley.open(credentials) { |r| r.cookbook.download(name, version) }
+        RidleyCompat.new_client(credentials) do |conn|
+          cookbook = Chef::CookbookVersion.load(name, version)
+          manifest = cookbook.cookbook_manifest
+          manifest.by_parent_directory.each do |segment, files|
+            files.each do |segment_file|
+              dest = File.join(unpack_dir, segment_file["path"].gsub("/", File::SEPARATOR))
+              FileUtils.mkdir_p(File.dirname(dest))
+              tempfile = conn.streaming_request(segment_file["url"])
+              FileUtils.mv(tempfile.path, dest)
+            end
+          end
+        end
+        unpack_dir
       when :github
         require "octokit"
 

data/lib/berkshelf/errors.rb

@@ -527,6 +527,9 @@ module Berkshelf
     alias_method :message, :to_s
   end
 
+  class CookbookSyntaxError < BerkshelfError; end
+  class RedirectLimitReached < BerkshelfError; end
+
   class MissingLockfileCookbookVersion < CookbookNotFound
     set_status_code(149)
 

data/lib/berkshelf/location.rb

@@ -19,7 +19,7 @@ module Berkshelf
       #
       # @return [~BaseLocation, nil]
       def init(dependency, options = {})
-        if klass = klass_from_options(options)
+        if ( klass = klass_from_options(options) )
           klass.new(dependency, options)
         else
           nil

data/lib/berkshelf/locations/base.rb

@@ -61,7 +61,7 @@ module Berkshelf
 
       begin
         cookbook = CachedCookbook.from_path(path)
-      rescue Ridley::Errors::RidleyError => e
+      rescue => e
         raise InternalError, "The following error occurred while reading the " \
           "cookbook `#{dependency.name}':\n#{e.class}: #{e.message}"
       end

data/lib/berkshelf/lockfile.rb

@@ -1,4 +1,5 @@
 require_relative "dependency"
+require "chef/environment"
 
 module Berkshelf
   class Lockfile
@@ -57,7 +58,7 @@ module Berkshelf
       @dependencies = {}
       @graph        = Graph.new(self)
 
-      parse if File.exists?(@filepath)
+      parse if File.exist?(@filepath)
     end
 
     # Parse the lockfile.
@@ -75,7 +76,7 @@ module Berkshelf
     # @return [Boolean]
     #   true if this lockfile exists on the disk, false otherwise
     def present?
-      File.exists?(filepath) && !File.read(filepath).strip.empty?
+      File.exist?(filepath) && !File.read(filepath).strip.empty?
     end
 
     # Determine if we can "trust" this lockfile. A lockfile is trustworthy if:
@@ -113,7 +114,7 @@ module Berkshelf
           return false
         end
 
-        if cookbook = locked.cached_cookbook
+        if ( cookbook = locked.cached_cookbook )
           Berkshelf.log.debug "  Detected there is a cached cookbook"
 
           unless (cookbook.dependencies.keys - graphed.dependencies.keys).empty?
@@ -203,8 +204,8 @@ module Berkshelf
     #   if you are locking cookbooks with an invalid or not-specified client
     #   configuration
     def apply(name, options = {})
-      locks = graph.locks.inject({}) do |hash, (name, dependency)|
-        hash[name] = "= #{dependency.locked_version}"
+      locks = graph.locks.inject({}) do |hash, (dep_name, dependency)|
+        hash[dep_name] = "= #{dependency.locked_version}"
         hash
       end
 
@@ -212,11 +213,14 @@ module Berkshelf
         update_environment_file(options[:envfile], locks) if options[:envfile]
       else
         Berkshelf.ridley_connection(options) do |connection|
-          environment = connection.environment.find(name)
-
-          raise EnvironmentNotFound.new(name) if environment.nil?
+          environment =
+            begin
+              Chef::Environment.from_hash(connection.get("environments/#{name}"))
+            rescue Berkshelf::APIClient::ServiceNotFound
+              raise EnvironmentNotFound.new(name)
+            end
 
-          environment.cookbook_versions = locks
+          environment.cookbook_versions locks
           environment.save unless options[:envfile]
         end
       end
@@ -314,7 +318,7 @@ module Berkshelf
     # @raise [EnvironmentFileNotFound]
     #   If environment file doesn't exist
     def update_environment_file(environment_file, locks)
-      unless File.exists?(environment_file)
+      unless File.exist?(environment_file)
         raise EnvironmentFileNotFound.new(environment_file)
       end
 
@@ -431,7 +435,7 @@ module Berkshelf
         # constraints are satisfied by it.
         dependency.locked_version = graphed.version
 
-        if cookbook = dependency.cached_cookbook
+        if ( cookbook = dependency.cached_cookbook )
           Berkshelf.log.debug "    Cached cookbook exists"
           Berkshelf.log.debug "    Updating cookbook dependencies if required"
           graphed.set_dependencies(cookbook.dependencies)
@@ -799,8 +803,8 @@ module Berkshelf
           out << "  #{name} (#{item.version})\n"
 
           unless item.dependencies.empty?
-            item.dependencies.sort.each do |name, constraint|
-              out << "    #{name} (#{constraint})\n"
+            item.dependencies.sort.each do |dep_name, constraint|
+              out << "    #{dep_name} (#{constraint})\n"
             end
           end
         end

data/lib/berkshelf/logger.rb

@@ -1,5 +1,66 @@
 module Berkshelf
-  class Logger < Ridley::Logging::Logger
+  class Logger < Logger
+    def initialize(device = STDOUT)
+      super
+      self.level = Logger::WARN
+      @filter_params = Array.new
+    end
+
+    # Reimplements Logger#add adding message filtering. The info,
+    # warn, debug, error, and fatal methods all call add.
+    #
+    # @param [Fixnum] severity
+    #   an integer measuing the severity - Logger::INFO, etc.
+    # @param [String] message = nil
+    #   the message to log
+    # @param [String] progname = nil
+    #   the program name performing the logging
+    # @param  &block
+    #  a block that will be evaluated (for complicated logging)
+    #
+    # @example
+    #   log.filter_param("hello")
+    #   log.info("hello world!") => "FILTERED world!"
+    #
+    # @return [Boolean]
+    def add(severity, message = nil, progname = nil, &block)
+      severity ||= Logger::UNKNOWN
+      if @logdev.nil? || severity < (@level)
+        return true
+      end
+      progname ||= @progname
+      if message.nil?
+        if block_given?
+          message = yield
+        else
+          message = progname
+          progname = @progname
+        end
+      end
+      @logdev.write(
+        format_message(format_severity(severity), Time.now, progname, filter(message)))
+      true
+    end
+
+    def filter_params
+      @filter_params.dup
+    end
+
+    def filter_param(param)
+      @filter_params << param unless filter_params.include?(param)
+    end
+
+    def clear_filter_params
+      @filter_params.clear
+    end
+
+    def filter(message)
+      filter_params.each do |param|
+        message.gsub!(param.to_s, "FILTERED")
+      end
+      message
+    end
+
     alias_method :fatal, :error
 
     def deprecate(message)

data/lib/berkshelf/packager.rb

@@ -15,7 +15,7 @@ module Berkshelf
   class Packager
     class << self
       def validate_destination(path)
-        path = path.to_s
+        path.to_s
       end
     end
 

data/lib/berkshelf/resolver.rb

@@ -93,7 +93,7 @@ module Berkshelf
     # @return [Dependency]
     def [](demand)
       name = demand.respond_to?(:name) ? demand.name : demand.to_s
-      demands.find { |demand| demand.name == name }
+      demands.find { |d| d.name == name }
     end
     alias_method :get_demand, :[]
 

data/lib/berkshelf/ridley_compat.rb

@@ -26,6 +26,11 @@ module Berkshelf
       super(opts[:server_url].to_s, **chef_opts)
     end
 
+    # for compat with Ridley::Connection
+    def server_url
+      url
+    end
+
     def get(url)
       super(url)
     rescue Net::HTTPExceptions => e
@@ -42,10 +47,24 @@ module Berkshelf
     rescue Errno::EHOSTUNREACH, Errno::ECONNREFUSED => e
       raise Berkshelf::APIClient::ServiceUnavailable, e
     end
+
+    module ClassMethods
+      def new_client(**opts, &block)
+        client = new(**opts)
+        yield client
+        # ensure
+        # FIXME: does Chef::HTTP support close anywhere?  this will just leak open fds
+      end
+    end
+
+    def self.included(klass)
+      klass.extend ClassMethods
+    end
+
   end
 
   # This is for simple HTTP
-  class RidleyCompatSimple < Chef::ServerAPI
+  class RidleyCompatSimple < ::Chef::ServerAPI
     use Chef::HTTP::Decompressor
     use Chef::HTTP::CookieManager
     use Chef::HTTP::ValidateContentLength
@@ -54,7 +73,7 @@ module Berkshelf
   end
 
   # This is for JSON-REST
-  class RidleyCompatJSON < Chef::HTTP::SimpleJSON
+  class RidleyCompatJSON < ::Chef::HTTP::SimpleJSON
     use Chef::HTTP::JSONInput
     use Chef::HTTP::JSONOutput
     use Chef::HTTP::CookieManager
@@ -67,7 +86,7 @@ module Berkshelf
 
   # RidleyCompat is the ServerAPI, but we inherit from Chef::HTTP::Simple and then include all our middlewares
   # and then need to include our own CompatAPI.  The end result is a ridley-esque way of talking to a chef server.
-  class RidleyCompat < Chef::HTTP::Simple
+  class RidleyCompat < ::Chef::HTTP::Simple
     use Chef::HTTP::JSONInput
     use Chef::HTTP::JSONOutput
     use Chef::HTTP::CookieManager

data/lib/berkshelf/uploader.rb

@@ -1,3 +1,8 @@
+require "chef/cookbook/chefignore"
+require "chef/cookbook/cookbook_version_loader"
+require "chef/cookbook_uploader"
+require "chef/exceptions"
+
 module Berkshelf
   class Uploader
     attr_reader :berksfile
@@ -8,7 +13,7 @@ module Berkshelf
     def initialize(berksfile, *args)
       @berksfile = berksfile
       @lockfile  = berksfile.lockfile
-      opts       = args.last.respond_to?(:to_hash) ? args.pop.to_hash.symbolize_keys : {}
+      opts       = args.last.respond_to?(:to_hash) ? args.pop.to_hash.each_with_object({}) { |(k, v), m| m[k.to_sym] = v } : {}
 
       @options = {
         force:          false,
@@ -48,69 +53,92 @@ module Berkshelf
       Berkshelf.log.info "Starting upload"
 
       Berkshelf.ridley_connection(options) do |connection|
-        cookbooks.each do |cookbook|
-          Berkshelf.log.debug "  Uploading #{cookbook}"
+        # this is a hack to work around a bug in chef 13.0-13.2 protocol negotiation on POST requests, its only
+        # use is to force protocol negotiation via a GET request -- it doesn't matter if it 404s.  once we do not
+        # support those early 13.x versions this line can be safely deleted.
+        connection.get("users/#{Berkshelf.config.chef.node_name}") rescue nil
 
+        cookbooks.map do |cookbook|
           begin
-            connection.cookbook.upload(cookbook.path,
-              name:     cookbook.cookbook_name,
-              force:    options[:force],
-              freeze:   options[:freeze],
-              validate: options[:validate]
-            )
-
-            Berkshelf.formatter.uploaded(cookbook, connection)
-          rescue Ridley::Errors::FrozenCookbook
-            if options[:halt_on_frozen]
-              raise FrozenCookbook.new(cookbook)
+            compiled_metadata = cookbook.compile_metadata
+            cookbook.reload if compiled_metadata
+            cookbook_version = cookbook.cookbook_version
+            Berkshelf.log.debug "  Uploading #{cookbook}"
+            cookbook_version.freeze_version if options[:freeze]
+
+            # another two lines that are necessary for chef < 13.2 support (affects 11.x/12.x as well)
+            cookbook_version.metadata.maintainer "" if cookbook_version.metadata.maintainer.nil?
+            cookbook_version.metadata.maintainer_email "" if cookbook_version.metadata.maintainer_email.nil?
+
+            begin
+              Chef::CookbookUploader.new(
+                [ cookbook_version ],
+                force: options[:force],
+                concurrency: 1, # sadly
+                rest: connection
+              ).upload_cookbooks
+              Berkshelf.formatter.uploaded(cookbook, connection)
+            rescue Chef::Exceptions::CookbookFrozen
+              if options[:halt_on_frozen]
+                raise FrozenCookbook.new(cookbook)
+              end
+              Berkshelf.formatter.skipping(cookbook, connection)
+            end
+          ensure
+            if compiled_metadata
+              # this is necessary on windows to clean up the ruby object that was pointing at the file
+              # so that we can reliably delete it.  windows is terrible.
+              GC.start
+              File.unlink(compiled_metadata)
             end
-
-            Berkshelf.formatter.skipping(cookbook, connection)
           end
         end
       end
     end
 
-      # Filter cookbooks based off the list of dependencies in the Berksfile.
-      #
-      # This method is secretly recursive. It iterates over each dependency in
-      # the Berksfile (using {Berksfile#dependencies} to account for filters)
-      # and retrieves that cookbook, it's dependencies, and the recusive
-      # dependencies, but iteratively.
-      #
-      # @return [Array<CachedCookbook>]
-      #
+    # Lookup dependencies in a cookbook and iterate to return dependencies of dependencies.
+    #
+    # This method is recursive. It iterates over a cookbook's dependencies
+    # and their dependencies in order to return an array of cookbooks, starting
+    # with the cookbook passed and followed by it's dependencies.
+    #
+    # @return [Array<CachedCookbook>]
+    #
+    def lookup_dependencies(cookbook, dependencies = [], checked = {})
+      Berkshelf.log.debug "  Looking up dependencies for #{cookbook}"
+      lockfile.graph.find(cookbook).dependencies.each do |name, _|
+        next if checked[name]
+
+        dependencies << name
+
+        checked[name] = true
+
+        lookup_dependencies(name, dependencies, checked) unless lockfile.graph.find(name).dependencies.empty?
+      end
+
+      dependencies
+    end
+
+    # Filter cookbooks based off the list of dependencies in the Berksfile.
+    #
+    # This method is secretly recursive. It iterates over each dependency in
+    # the Berksfile (using {Berksfile#dependencies} to account for filters)
+    # and retrieves that cookbook, it's dependencies, and the recusive
+    # dependencies, but iteratively.
+    #
+    # @return [Array<CachedCookbook>]
+    #
     def filtered_cookbooks
       # Create a copy of the dependencies. We need to make a copy, or else
       # we would be adding dependencies directly to the Berksfile object, and
       # that would be a bad idea...
       dependencies = berksfile.dependencies.map(&:name)
 
-      checked   = {}
-      cookbooks = {}
-
-      dependencies.each do |dependency|
-        next if checked[dependency]
-
-        lockfile.graph.find(dependency).dependencies.each do |name, _|
-          cookbooks[name] ||= lockfile.retrieve(name)
-          dependencies << name
-        end
-
-        checked[dependency] = true
-        cookbooks[dependency] ||= lockfile.retrieve(dependency)
+      cookbook_order = dependencies.each do |dependency|
+        lookup_dependencies(dependency, dependencies)
       end
 
-      # This is a temporary change and will be removed in a future release. We should
-      # add the ability to define a custom uploader which would allow the authors of Chef-Guard
-      # to define their upload strategy instead of using Ridley.
-      #
-      # See https://github.com/berkshelf/berkshelf/pull/1316 for details.
-      if Berkshelf.chef_config.knife[:chef_guard] == true
-        cookbooks.values
-      else
-        cookbooks.values.sort
-      end
+      cookbook_order.reverse.map { |dependency| lockfile.retrieve(dependency) }.uniq
     end
   end
 end