berkshelf 1.2.0.rc1 → 1.2.1

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+# 1.2.0
+- Remove Vagrant as a gem dependency
+- Remove Chef as a gem dependency
+- Add retries to downloads/uploads
+- Speed optimizations to resolver
+- Speed optimizations to downloading cookbooks
+- Speed optimizations to uploading cookbooks
+
 # 1.1.0
 ## new/improved commands
 - `berks show` command: display the file path for the given cookbook's current version resolved by your Berksfile

data/Gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source 'https://rubygems.org'
 
 gemspec
 

data/README.md

@@ -1,6 +1,8 @@
 Berkshelf
 =========
+[![Gem Version](https://badge.fury.io/rb/berkshelf.png)](http://badge.fury.io/rb/berkshelf)
 [![Build Status](https://travis-ci.org/RiotGames/berkshelf.png?branch=master)](https://travis-ci.org/RiotGames/berkshelf)
+[![Dependency Status](https://gemnasium.com/RiotGames/berkshelf.png)](https://gemnasium.com/RiotGames/berkshelf)
 [![Code Climate](https://codeclimate.com/github/RiotGames/berkshelf.png)](https://codeclimate.com/github/RiotGames/berkshelf)
 
 Manage a Cookbook or an Application's Cookbook dependencies

data/berkshelf.gemspec

@@ -32,14 +32,15 @@ Gem::Specification.new do |s|
   s.add_dependency 'mixlib-shellout'
   s.add_dependency 'mixlib-config'
   s.add_dependency 'faraday', '>= 0.8.5'
-  s.add_dependency 'ridley', '>= 0.7.0.rc4'
-  s.add_dependency 'chozo', '>= 0.5.0'
-  s.add_dependency 'hashie'
+  s.add_dependency 'ridley', '>= 0.8.3'
+  s.add_dependency 'chozo', '>= 0.6.1'
+  s.add_dependency 'hashie', '>= 2.0.2'
   s.add_dependency 'minitar'
   s.add_dependency 'json', '>= 1.5.0'
   s.add_dependency 'multi_json', '~> 1.5'
   s.add_dependency 'solve', '>= 0.4.2'
   s.add_dependency 'thor', '~> 0.16.0'
+  s.add_dependency 'retryable'
 
   # Vagrant 1-0-stable compatability locks
   s.add_dependency 'moneta', '~> 0.6.0'

data/features/step_definitions/filesystem_steps.rb

@@ -106,7 +106,6 @@ Then /^I should have a new cookbook skeleton "(.*?)"$/ do |name|
     end
     file "Gemfile" do
       contains "gem 'berkshelf'"
-      contains "gem 'vagrant'"
     end
     file "metadata.rb"
     file "README.md"

data/generator_files/Gemfile.erb

@@ -7,6 +7,3 @@ gem 'thor-foodcritic'
 <% if options[:scmversion] -%>
 gem 'thor-scmversion'
 <% end -%>
-<% unless options[:skip_vagrant] -%>
-gem 'vagrant', '~> 1.0.5'
-<% end -%>

data/generator_files/Vagrantfile.erb

@@ -1,4 +1,16 @@
-require 'berkshelf/vagrant'
+begin
+  require 'berkshelf/vagrant'
+rescue LoadError
+  puts "[WARNING] Berkshelf not found in your Vagrant's RubyGems but your Vagrantfile is attempting"
+  puts "[WARNING] to require the Berkshelf Vagrant plugin! Install the Berkshelf Vagrant plugin or"
+  puts "[WARNING] remove the 'require \"berkshelf/vagrant\"' line from the top of your Vagrantfile."
+  puts ""
+  puts "If you installed Vagrant by RubyGems:"
+  puts "  Install Berkshelf by running: \"gem install berkshelf\""
+  puts "If you installed Vagrant by one of the pre-packaged installers:"
+  puts "  Install Berkshelf by running: \"vagrant gem install berkshelf\""
+  puts ""
+end
 
 Vagrant::Config.run do |config|
   # All Vagrant configuration is done here. The most common configuration

data/lib/berkshelf.rb

@@ -48,7 +48,6 @@ module Berkshelf
   autoload :Mixin, 'berkshelf/mixin'
   autoload :Resolver, 'berkshelf/resolver'
   autoload :UI, 'berkshelf/ui'
-  autoload :Uploader, 'berkshelf/uploader'
 
   require 'berkshelf/location'
 

data/lib/berkshelf/berksfile.rb

@@ -183,7 +183,7 @@ module Berkshelf
         raise CookbookNotFound, "No 'metadata.rb' found at #{path}"
       end
 
-      metadata = Berkshelf::Chef::Cookbook::Metadata.from_file(metadata_file.to_s)
+      metadata = Ridley::Chef::Cookbook::Metadata.from_file(metadata_file.to_s)
 
       name = if metadata.name.empty? || metadata.name.nil?
         File.basename(File.dirname(metadata_file))
@@ -478,12 +478,17 @@ module Berkshelf
     #
     # @raise [UploadFailure] if you are uploading cookbooks with an invalid or not-specified client key
     def upload(options = {})
-      uploader = Uploader.new(options)
+      conn     = Ridley.new(options)
       solution = resolve(options)
+
       solution.each do |cb|
-        Berkshelf.formatter.upload cb.cookbook_name, cb.version, options[:server_url]
-        uploader.upload(cb, options)
+        upload_opts = options.dup
+        upload_opts[:name] = cb.cookbook_name
+
+        Berkshelf.formatter.upload cb.cookbook_name, cb.version, upload_opts[:server_url]
+        conn.cookbook.upload(cb.path, upload_opts)
       end
+
       if options[:skip_dependencies]
         missing_cookbooks = options.fetch(:cookbooks, nil) - solution.map(&:cookbook_name)
         unless missing_cookbooks.empty?
@@ -496,6 +501,8 @@ module Berkshelf
       msg = "Could not upload cookbooks: Missing Chef client key: '#{Berkshelf::Config.instance.chef.client_key}'."
       msg << " Generate or update your Berkshelf configuration that contains a valid path to a Chef client key."
       raise UploadFailure, msg
+    ensure
+      conn.terminate if conn && conn.alive?
     end
 
     # Finds a solution for the Berksfile and returns an array of CachedCookbooks.

data/lib/berkshelf/cached_cookbook.rb

@@ -1,29 +1,7 @@
 module Berkshelf
   # @author Jamie Winsor <reset@riotgames.com>
-  class CachedCookbook
+  class CachedCookbook < Ridley::Chef::Cookbook
     class << self
-      include Berkshelf::Mixin::Checksum
-
-      # Creates a new instance of Berkshelf::CachedCookbook from a path on disk that
-      # contains a Cookbook. The name of the Cookbook will be determined first by the
-      # name attribute of the metadata.rb file if it is present. If the name attribute
-      # has not been set the Cookbook name will be determined by the basename of the
-      # given filepath.
-      #
-      # @param [#to_s] path
-      #   a path on disk to the location of a Cookbook
-      #
-      # @return [Berkshelf::CachedCookbook]
-      def from_path(path)
-        path     = Pathname.new(path)
-        metadata = Berkshelf::Chef::Cookbook::Metadata.from_file(path.join('metadata.rb'))
-
-        name = metadata.name.empty? ? File.basename(path) : metadata.name
-        metadata.name(name) if metadata.name.empty?
-
-        new(name, path, metadata)
-      end
-
       # @param [#to_s] path
       #   a path on disk to the location of a Cookbook downloaded by the Downloader
       #
@@ -35,248 +13,15 @@ module Berkshelf
         cached_name = File.basename(path.to_s).slice(DIRNAME_REGEXP, 1)
         return nil if cached_name.nil?
 
-        metadata = Berkshelf::Chef::Cookbook::Metadata.from_file(path.join('metadata.rb'))
-        metadata.name(cached_name) if metadata.name.empty?
-
-        new(cached_name, path, metadata)
-      end
-
-      # @param [String] filepath
-      #   a path on disk to the location of a file to checksum
-      #
-      # @return [String]
-      #   a checksum that can be used to uniquely identify the file understood
-      #   by a Chef Server.
-      def checksum(filepath)
-        Berkshelf::Chef::Digester.md5_checksum_for_file(filepath)
+        from_path(path, name: cached_name)      
       end
     end
 
     DIRNAME_REGEXP = /^(.+)-(.+)$/
-    CHEF_TYPE = "cookbook_version".freeze
-    CHEF_JSON_CLASS = "Chef::CookbookVersion".freeze
-
-    extend Forwardable
-
-    attr_reader :cookbook_name
-    attr_reader :path
-    attr_reader :metadata
-
-    # @return [Hashie::Mash]
-    #   a Hashie::Mash containing Cookbook file category names as keys and an Array of Hashes
-    #   containing metadata about the files belonging to that category. This is used
-    #   to communicate what a Cookbook looks like when uploading to a Chef Server.
-    #
-    #   example:
-    #     {
-    #       :recipes => [
-    #         {
-    #           name: "default.rb",
-    #           path: "recipes/default.rb",
-    #           checksum: "fb1f925dcd5fc4ebf682c4442a21c619",
-    #           specificity: "default"
-    #         }
-    #       ]
-    #       ...
-    #       ...
-    #     }
-    attr_reader :manifest
-
-    def_delegator :@metadata, :version
-
-    def initialize(name, path, metadata)
-      @cookbook_name = name
-      @path = Pathname.new(path)
-      @metadata = metadata
-      @files = Array.new
-      @manifest = Hashie::Mash.new(
-        recipes: Array.new,
-        definitions: Array.new,
-        libraries: Array.new,
-        attributes: Array.new,
-        files: Array.new,
-        templates: Array.new,
-        resources: Array.new,
-        providers: Array.new,
-        root_files: Array.new
-      )
-
-      load_files
-    end
-
-    # @return [String]
-    #   the name of the cookbook and the version number separated by a dash (-).
-    #
-    #   example:
-    #     "nginx-0.101.2"
-    def name
-      "#{cookbook_name}-#{version}"
-    end
 
     # @return [Hash]
     def dependencies
       metadata.recommendations.merge(metadata.dependencies)
     end
-
-    # @return [Hash]
-    #   an hash containing the checksums and expanded file paths of all of the
-    #   files found in the instance of CachedCookbook
-    #
-    #   example:
-    #     {
-    #       "da97c94bb6acb2b7900cbf951654fea3" => "/Users/reset/.berkshelf/nginx-0.101.2/README.md"
-    #     }
-    def checksums
-      {}.tap do |checksums|
-        files.each do |file|
-          checksums[self.class.checksum(file)] = file
-        end
-      end
-    end
-
-    # @param [Symbol] category
-    #   the category of file to generate metadata about
-    # @param [String] target
-    #   the filepath to the file to get metadata information about
-    #
-    # @return [Hash]
-    #   a Hash containing a name, path, checksum, and specificity key representing the
-    #   metadata about a file contained in a Cookbook. This metadata is used when
-    #   uploading a Cookbook's files to a Chef Server.
-    #
-    #   example:
-    #     {
-    #       name: "default.rb",
-    #       path: "recipes/default.rb",
-    #       checksum: "fb1f925dcd5fc4ebf682c4442a21c619",
-    #       specificity: "default"
-    #     }
-    def file_metadata(category, target)
-      target = Pathname.new(target)
-
-      {
-        name: target.basename.to_s,
-        path: target.relative_path_from(path).to_s,
-        checksum: self.class.checksum(target),
-        specificity: file_specificity(category, target)
-      }
-    end
-
-    # Validates that this instance of CachedCookbook points to a valid location on disk that
-    # contains a cookbook which passes a Ruby and template syntax check. Raises an error if
-    # these assertions are not true.
-    #
-    # @return [Boolean]
-    #   returns true if Cookbook is valid
-    def validate!
-      raise CookbookNotFound, "No Cookbook found at: #{path}" unless path.exist?
-
-      unless quietly { syntax_checker.validate_ruby_files }
-        raise CookbookSyntaxError, "Invalid ruby files in cookbook: #{name} (#{version})."
-      end
-      unless quietly { syntax_checker.validate_templates }
-        raise CookbookSyntaxError, "Invalid template files in cookbook: #{name} (#{version})."
-      end
-
-      true
-    end
-
-    def to_hash
-      result = manifest.dup
-      result['chef_type'] = 'cookbook_version'
-      result['name'] = name
-      result['cookbook_name'] = cookbook_name
-      result['version'] = version
-      result['metadata'] = metadata
-      result['chef_type']
-      result
-    end
-
-    def to_json(*a)
-      result = self.to_hash
-      result['json_class'] = chef_json_class
-      result['frozen?'] = false
-      result.to_json(*a)
-    end
-
-    def to_s
-      "#{cookbook_name} (#{version}) '#{path}'"
-    end
-
-    def <=>(other_cookbook)
-      [self.cookbook_name, self.version] <=> [other_cookbook.cookbook_name, other_cookbook.version]
-    end
-
-    private
-
-      attr_reader :files
-
-      def chef_type
-        CHEF_TYPE
-      end
-
-      def chef_json_class
-        CHEF_JSON_CLASS
-      end
-
-      def syntax_checker
-        @syntax_checker ||= Berkshelf::Chef::Cookbook::SyntaxCheck.new(path.to_s)
-      end
-
-      def load_files
-        load_shallow(:recipes, 'recipes', '*.rb')
-        load_shallow(:definitions, 'definitions', '*.rb')
-        load_shallow(:libraries, 'libraries', '*.rb')
-        load_shallow(:attributes, 'attributes', '*.rb')
-        load_recursively(:files, "files", "*")
-        load_recursively(:templates, "templates", "*")
-        load_recursively(:resources, "resources", "*.rb")
-        load_recursively(:providers, "providers", "*.rb")
-        load_root
-      end
-
-      def load_root
-        [].tap do |files|
-          Dir.glob(path.join('*'), File::FNM_DOTMATCH).each do |file|
-            next if File.directory?(file)
-            @files << file
-            @manifest[:root_files] << file_metadata(:root_files, file)
-          end
-        end
-      end
-
-      def load_recursively(category, category_dir, glob)
-        [].tap do |files|
-          file_spec = path.join(category_dir, '**', glob)
-          Dir.glob(file_spec, File::FNM_DOTMATCH).each do |file|
-            next if File.directory?(file)
-            @files << file
-            @manifest[category] << file_metadata(category, file)
-          end
-        end
-      end
-
-      def load_shallow(category, *path_glob)
-        [].tap do |files|
-          Dir[path.join(*path_glob)].each do |file|
-            @files << file
-            @manifest[category] << file_metadata(category, file)
-          end
-        end
-      end
-
-      # @param [Symbol] category
-      # @param [Pathname] target
-      #
-      # @return [String]
-      def file_specificity(category, target)
-        case category
-        when :files, :templates
-          relpath = target.relative_path_from(path).to_s
-          relpath.slice(/(.+)\/(.+)\/.+/, 2)
-        else
-          'default'
-        end
-      end
   end
 end

data/lib/berkshelf/chef.rb

@@ -6,6 +6,5 @@ module Berkshelf
   module Chef
     autoload :Config, 'berkshelf/chef/config'
     autoload :Cookbook, 'berkshelf/chef/cookbook'
-    autoload :Digester, 'berkshelf/chef/digester'
   end
 end

data/lib/berkshelf/chef/config.rb

@@ -1,4 +1,5 @@
 require 'socket'
+require 'tmpdir'
 require 'berkshelf/mixin'
 require 'mixlib/config'
 
@@ -81,6 +82,8 @@ module Berkshelf::Chef
     cookbook_email          "YOUR_EMAIL"
     cookbook_license        "reserved"
 
+    knife                   Hash.new
+
     # history: prior to Chef 11, the cache implementation was based on
     # moneta and configured via cache_options[:path]. Knife configs
     # generated with Chef 11 will have `syntax_check_cache_path`, but older

data/lib/berkshelf/chef/cookbook.rb

@@ -2,7 +2,5 @@ module Berkshelf::Chef
   # @author Jamie Winsor <reset@riotgames.com>
   module Cookbook
     autoload :Chefignore, 'berkshelf/chef/cookbook/chefignore'
-    autoload :Metadata, 'berkshelf/chef/cookbook/metadata'
-    autoload :SyntaxCheck, 'berkshelf/chef/cookbook/syntax_check'
   end
 end

data/lib/berkshelf/community_rest.rb

@@ -1,4 +1,5 @@
 require 'open-uri'
+require 'retryable'
 
 module Berkshelf
   # @author Jamie Winsor <reset@riotgames.com>
@@ -32,13 +33,35 @@ module Berkshelf
 
     V1_API = 'http://cookbooks.opscode.com/api/v1/cookbooks'.freeze
 
+    # @return [String]
     attr_reader :api_uri
-
-    def initialize(uri = V1_API)
-      @api_uri = Addressable::URI.parse(uri)
+    # @return [Integer]
+    #   how many retries to attempt on HTTP requests
+    attr_reader :retries
+    # @return [Float]
+    #   time to wait between retries
+    attr_reader :retry_interval
+
+    # @param [String] uri (CommunityREST::V1_API)
+    #   location of community site to connect to
+    #
+    # @option options [Integer] :retries (5)
+    #   retry requests on 5XX failures
+    # @option options [Float] :retry_interval (0.5)
+    #   how often we should pause between retries
+    def initialize(uri = V1_API, options = {})
+      options         = options.reverse_merge(retries: 5, retry_interval: 0.5)
+      @api_uri        = Addressable::URI.parse(uri)
+      @retries        = options[:retries]
+      @retry_interval = options[:retry_interval]
 
       builder = Faraday::Builder.new do |b|
         b.response :json
+        b.request :retry,
+          max: @retries,
+          interval: @retry_interval,
+          exceptions: [Faraday::Error::TimeoutError]
+
         b.adapter :net_http
       end
 
@@ -123,10 +146,12 @@ module Berkshelf
       local = Tempfile.new('community-rest-stream')
       local.binmode
 
-      open(target, 'rb', headers) do |remote|
-        local.write(remote.read)
+      retryable(tries: retries, on: OpenURI::HTTPError, sleep: retry_interval) do
+        open(target, 'rb', headers) do |remote|
+          local.write(remote.read)
+        end
       end
-      
+
       local
     ensure
       local.close(false) unless local.nil?