librarian 0.0.25 → 0.0.26

data/.gitignore

@@ -1,5 +1,9 @@
 *.gem
+bin
+!bin/librarian-chef
+!bin/librarian-mock
 .bundle
 Gemfile.lock
 pkg/*
 tmp
+vendor

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Change Log
 
+## 0.0.26
+
+* \#112. Prevent the git source being confused in certain cases by colorized
+  output from git. HT: @ericpp.
+
+* \#115, \#116. Accommodate cookbook archives generated by `git archive`.
+  HT: @taqtiqa-mark.
+
+* \#119. Support NO_PROXY, a modifier on HTTP_PROXY and friends. HT: @databus23.
+
+* \#121. A github pseudosource. HT: @alno.
+
+* \#122. Follow http redirect responses in the chef site source. HT: @Fleurer.
+
+* Improve the resolver performance by detecting conflicts earlier and
+  backtracking more quickly. This will help avoid certain cases of long
+  resolutions caused by conflicts; the conflicts will still be there, but will
+  be detected more quickly and the resolution terminated more quickly. This
+  changes the resolution algorithm, and new resolutions may differ from older
+  resolutions.
+
 ## 0.0.25
 
 * \#71. Fix an error, given certain locale settings, with reading cookbook

data/README.md

@@ -1,4 +1,4 @@
-Librarian [![Build Status](https://secure.travis-ci.org/applicationsonline/librarian.png)](http://travis-ci.org/applicationsonline/librarian)
+Librarian [![Build Status](https://secure.travis-ci.org/applicationsonline/librarian.png)](http://travis-ci.org/applicationsonline/librarian) [![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/applicationsonline/librarian)
 =========
 
 Librarian is a framework for writing bundlers, which are tools that resolve,
@@ -136,6 +136,11 @@ we vendored it in our repository into the `cookbooks/` directory for us.
 The `:path =>` source won't be confused with the `:git =>` source's `:path =>`
 option.
 
+Also, there is shortcut for cookbooks hosted on GitHub, so we may write:
+
+    cookbook "rvm",
+      :github => "fnichol/chef-rvm"
+
 ### How to Use
 
 Install Librarian-Chef:

data/lib/librarian/action/persist_resolution_mixin.rb

@@ -0,0 +1,51 @@
+require "librarian/error"
+require "librarian/spec_change_set"
+
+module Librarian
+  module Action
+    module PersistResolutionMixin
+
+    private
+
+      def persist_resolution(resolution)
+        resolution.correct? or raise Error,
+          "Could not resolve the dependencies."
+
+        lockfile_text = lockfile.save(resolution)
+        debug { "Bouncing #{lockfile_name}" }
+        bounced_lockfile_text = lockfile.save(lockfile.load(lockfile_text))
+        unless bounced_lockfile_text == lockfile_text
+          debug { "lockfile_text: \n#{lockfile_text}" }
+          debug { "bounced_lockfile_text: \n#{bounced_lockfile_text}" }
+          raise Error, "Cannot bounce #{lockfile_name}!"
+        end
+        lockfile_path.open('wb') { |f| f.write(lockfile_text) }
+      end
+
+      def specfile_name
+        environment.specfile_name
+      end
+
+      def lockfile_name
+        environment.lockfile_name
+      end
+
+      def specfile_path
+        environment.specfile_path
+      end
+
+      def lockfile_path
+        environment.lockfile_path
+      end
+
+      def specfile
+        environment.specfile
+      end
+
+      def lockfile
+        environment.lockfile
+      end
+
+    end
+  end
+end

data/lib/librarian/action/resolve.rb

@@ -1,11 +1,12 @@
-require "librarian/error"
 require "librarian/resolver"
 require "librarian/spec_change_set"
 require "librarian/action/base"
+require "librarian/action/persist_resolution_mixin"
 
 module Librarian
   module Action
     class Resolve < Base
+      include PersistResolutionMixin
 
       def run
         if force? || !lockfile_path.exist?
@@ -23,19 +24,7 @@ module Librarian
         end
 
         resolution = resolver.resolve(spec, manifests)
-        unless resolution.correct?
-          raise Error, "Could not resolve the dependencies."
-        else
-          lockfile_text = lockfile.save(resolution)
-          debug { "Bouncing #{lockfile_name}" }
-          bounced_lockfile_text = lockfile.save(lockfile.load(lockfile_text))
-          unless bounced_lockfile_text == lockfile_text
-            debug { "lockfile_text: \n#{lockfile_text}"}
-            debug { "bounced_lockfile_text: \n#{bounced_lockfile_text}"}
-            raise Error, "Cannot bounce #{lockfile_name}!"
-          end
-          lockfile_path.open('wb') { |f| f.write(lockfile_text) }
-        end
+        persist_resolution(resolution)
       end
 
     private
@@ -44,30 +33,6 @@ module Librarian
         options[:force]
       end
 
-      def specfile_name
-        environment.specfile_name
-      end
-
-      def lockfile_name
-        environment.lockfile_name
-      end
-
-      def specfile_path
-        environment.specfile_path
-      end
-
-      def lockfile_path
-        environment.lockfile_path
-      end
-
-      def specfile
-        environment.specfile
-      end
-
-      def lockfile
-        environment.lockfile
-      end
-
       def resolver
         Resolver.new(environment)
       end

data/lib/librarian/action/update.rb

@@ -1,12 +1,13 @@
-require "librarian/error"
 require "librarian/manifest_set"
 require "librarian/resolver"
 require "librarian/spec_change_set"
 require "librarian/action/base"
+require "librarian/action/persist_resolution_mixin"
 
 module Librarian
   module Action
     class Update < Base
+      include PersistResolutionMixin
 
       def run
         unless lockfile_path.exist?
@@ -19,20 +20,9 @@ module Librarian
         partial_manifests = ManifestSet.deep_strip(manifests, dependency_names)
         unpinnable_sources = previous_resolution.sources - partial_manifests.map(&:source)
         unpinnable_sources.each(&:unpin!)
+
         resolution = resolver.resolve(spec, partial_manifests)
-        unless resolution.correct?
-          raise Error, "Could not resolve the dependencies."
-        else
-          lockfile_text = lockfile.save(resolution)
-          debug { "Bouncing #{lockfile_name}" }
-          bounced_lockfile_text = lockfile.save(lockfile.load(lockfile_text))
-          unless bounced_lockfile_text == lockfile_text
-            debug { "lockfile_text: \n#{lockfile_text}"}
-            debug { "bounced_lockfile_text: \n#{bounced_lockfile_text}"}
-            raise Error, "Cannot bounce #{lockfile_name}!"
-          end
-          lockfile_path.open('wb') { |f| f.write(lockfile_text) }
-        end
+        persist_resolution(resolution)
       end
 
     private
@@ -41,30 +31,6 @@ module Librarian
         options[:names]
       end
 
-      def specfile_name
-        environment.specfile_name
-      end
-
-      def lockfile_name
-        environment.lockfile_name
-      end
-
-      def specfile_path
-        environment.specfile_path
-      end
-
-      def lockfile_path
-        environment.lockfile_path
-      end
-
-      def specfile
-        environment.specfile
-      end
-
-      def lockfile
-        environment.lockfile
-      end
-
       def resolver
         Resolver.new(environment)
       end

data/lib/librarian/chef/dsl.rb

@@ -9,6 +9,7 @@ module Librarian
 
       source :site => Source::Site
       source :git => Source::Git
+      source :github => Source::Github
       source :path => Source::Path
     end
   end

data/lib/librarian/chef/source.rb

@@ -1,3 +1,4 @@
 require 'librarian/chef/source/path'
 require 'librarian/chef/source/git'
+require 'librarian/chef/source/github'
 require 'librarian/chef/source/site'

data/lib/librarian/chef/source/github.rb

@@ -0,0 +1,27 @@
+require 'librarian/chef/source/git'
+
+module Librarian
+  module Chef
+    module Source
+      class Github
+
+        class << self
+
+          def lock_name
+            Git.lock_name
+          end
+
+          def from_lock_options(environment, options)
+            Git.from_lock_options(environment, options)
+          end
+
+          def from_spec_args(environment, uri, options)
+            Git.from_spec_args(environment, "https://github.com/#{uri}", options)
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/librarian/chef/source/site.rb

@@ -8,6 +8,7 @@ require 'zlib'
 require 'securerandom'
 require 'archive/tar/minitar'
 
+require 'librarian/source/basic_api'
 require 'librarian/chef/manifest_reader'
 
 module Librarian
@@ -76,8 +77,7 @@ module Librarian
           end
 
           def version_uri_metadata(version_uri)
-            @version_uri_metadata ||= { }
-            @version_uri_metadata[version_uri.to_s] ||= begin
+            memo(__method__, version_uri.to_s) do
               cache_version_uri_metadata! version_uri
               parse_local_json(version_uri_metadata_cache_path(version_uri))
             end
@@ -89,8 +89,7 @@ module Librarian
           end
 
           def version_uri_manifest(version_uri)
-            @version_uri_manifest ||= { }
-            @version_uri_manifest[version_uri.to_s] ||= begin
+            memo(__method__, version_uri.to_s) do
               cache_version_uri_unpacked! version_uri
               unpacked_path = version_uri_unpacked_cache_path(version_uri)
               manifest_path = ManifestReader.manifest_path(unpacked_path)
@@ -106,8 +105,7 @@ module Librarian
           end
 
           def to_version_uri(version)
-            @to_version_uri ||= { }
-            @to_version_uri[version.to_s] ||= begin
+            memo(__method__, version.to_s) do
               cache_version! version
               version_cache_path(version).read
             end
@@ -126,15 +124,13 @@ module Librarian
           end
 
           def version_cache_path(version)
-            @version_cache_path ||= { }
-            @version_cache_path[version.to_s] ||= begin
+            memo(__method__, version.to_s) do
               cache_path.join("version").join(version.to_s)
             end
           end
 
           def version_uri_cache_path(version_uri)
-            @version_uri_cache_path ||= { }
-            @version_uri_cache_path[version_uri.to_s] ||= begin
+            memo(__method__, version_uri.to_s) do
               cache_path.join("version-uri").join(hexdigest(version_uri))
             end
           end
@@ -145,8 +141,7 @@ module Librarian
           end
 
           def version_uri_metadata_cache_path(version_uri)
-            @version_uri_metadata_cache_path ||= { }
-            @version_uri_metadata_cache_path[version_uri.to_s] ||= begin
+            memo(__method__, version_uri.to_s) do
               version_uri_cache_path(version_uri).join("metadata.json")
             end
           end
@@ -157,8 +152,7 @@ module Librarian
           end
 
           def version_uri_package_cache_path(version_uri)
-            @version_uri_package_cache_path ||= { }
-            @version_uri_package_cache_path[version_uri.to_s] ||= begin
+            memo(__method__, version_uri.to_s) do
               version_uri_cache_path(version_uri).join("package.tar.gz")
             end
           end
@@ -169,8 +163,7 @@ module Librarian
           end
 
           def version_uri_unpacked_cache_path(version_uri)
-            @version_uri_unpacked_cache_path ||= { }
-            @version_uri_unpacked_cache_path[version_uri.to_s] ||= begin
+            memo(__method__, version_uri.to_s) do
               version_uri_cache_path(version_uri).join("package")
             end
           end
@@ -243,9 +236,6 @@ module Librarian
             debug { "Caching #{uri} to #{path}" }
 
             response = http_get(uri)
-            unless Net::HTTPSuccess === response
-              raise Error, "Could not get #{uri} because #{response.code} #{response.message}!"
-            end
 
             object = response.body
             case type
@@ -273,9 +263,11 @@ module Librarian
             end
 
             # Cookbook files, as pulled from Opscode Community Site API, are
-            # embedded in a subdirectory of the tarball.
+            # embedded in a subdirectory of the tarball. If created by git archive they
+            # can include the subfolder `pax_global_header`, which is ignored.
             subtemps = temp.children
             subtemps.empty? and raise "The package archive was empty!"
+            subtemps.delete_if{|pth| pth.to_s[/pax_global_header/]}
             subtemps.size > 1 and raise "The package archive has too many children!"
             subtemp = subtemps.first
             debug { "Moving #{relative_path_to(subtemp)} to #{relative_path_to(path)}" }
@@ -306,40 +298,55 @@ module Librarian
           end
 
           def http(uri)
-            environment.net_http_class.new(uri.host, uri.port)
+            environment.net_http_class(uri.host).new(uri.host, uri.port)
           end
 
           def http_get(uri)
-            http = http(uri)
-            request = Net::HTTP::Get.new(uri.path)
-            response = http.start{|http| http.request(request)}
-
-            response
+            max_redirects = 10
+            redirects = []
+
+            loop do
+              debug { "Performing http-get for #{uri}" }
+              http = http(uri)
+              request = Net::HTTP::Get.new(uri.path)
+              response = http.start{|http| http.request(request)}
+
+              case response
+              when Net::HTTPSuccess
+                debug { "Responded with success" }
+                return response
+              when Net::HTTPRedirection
+                location = response["Location"]
+                debug { "Responded with redirect to #{uri}" }
+                redirects.size > max_redirects and raise Error,
+                  "Could not get #{uri} because too many redirects!"
+                redirects.include?(location) and raise Error,
+                  "Could not get #{uri} because redirect cycle!"
+                redirects << location
+                uri = URI.parse(location)
+                # continue the loop
+              else
+                raise Error, "Could not get #{uri} because #{response.code} #{response.message}!"
+              end
+            end
           end
 
-        end
-
-        class << self
-
-          LOCK_NAME = 'SITE'
-
-          def lock_name
-            LOCK_NAME
-          end
+          def memo(method, *path)
+            ivar = "@#{method}".to_sym
+            unless memo = instance_variable_get(ivar)
+              memo = instance_variable_set(ivar, { })
+            end
 
-          def from_lock_options(environment, options)
-            new(environment, options[:remote], options.reject{|k, v| k == :remote})
+            memo.key?(path) or memo[path] = yield
+            memo[path]
           end
 
-          def from_spec_args(environment, uri, options)
-            recognized_options = []
-            unrecognized_options = options.keys - recognized_options
-            unrecognized_options.empty? or raise Error, "unrecognized options: #{unrecognized_options.join(", ")}"
+        end
 
-            new(environment, uri, options)
-          end
+        include Librarian::Source::BasicApi
 
-        end
+        lock_name 'SITE'
+        spec_options []
 
         attr_accessor :environment, :uri
         private :environment=, :uri=
@@ -395,13 +402,6 @@ module Librarian
           line(name).manifests
         end
 
-        def manifest(name, version, dependencies)
-          manifest = Manifest.new(self, name)
-          manifest.version = version
-          manifest.dependencies = dependencies
-          manifest
-        end
-
         def cache_path
           @cache_path ||= begin
             dir = Digest::MD5.hexdigest(uri)