librarian-puppet 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1984f0b83258ba54c8f1d62a2d6804dd29d75065
-  data.tar.gz: e64114a525b0f92e74d4d09277b9f0dd5f76e0c6
+  metadata.gz: dbe00d37667721e7a61be8fb75277e88f75dec0d
+  data.tar.gz: 6655c8212fe01704c529ea35fcbc7ada5ed14a1b
 SHA512:
-  metadata.gz: c5a3794a4677ae67769338e469d2eb9c646b1497cdf4b6ef6ad81a11e525350e0e397f7bf8d9a81202e33aa24c311aa6e81f3d400e1eb1729725287aceafd289
-  data.tar.gz: afb5509325ef53d0d24334c9ddb3c2607ba35823024a96f1ce285b2306002ea0ba46fcb7a59cb937512c047b131ccd0813c71b0c4966f80819d762f7480e6f8c
+  metadata.gz: 41f48efc10bb44b6ab924bc2040d686264a1e74ee58f1a179bf23552e4947df3f2b94251fc92d96ad3ada0ad2fe3bada5c765a5ff7f174c2da67b1fe69f9a44e
+  data.tar.gz: 59f9a5784cf4c0f801806c6821cf5c7ca33fbb577022ef6fcdae428d932d24ff918e0692bee4a8aebe9cab60de901de9d2f40a90587aaf9a008475d41c3c7f4f

data/README.md

@@ -6,11 +6,11 @@
 
 Librarian-puppet is a bundler for your puppet infrastructure.  You can use
 librarian-puppet to manage the puppet modules your infrastructure depends on,
-whether the modules come from the [Puppet Forge](http://forge.puppetlabs.com/),
+whether the modules come from the [Puppet Forge](https://forge.puppetlabs.com/),
 Git repositories or a just a path.
 
 * Librarian-puppet can reuse the dependencies listed in your Modulefile
-* Forge modules can be installed from [Puppetlabs Forge](http://forge.puppetlabs.com/) or an internal Forge such as [Pulp](http://www.pulpproject.org/)
+* Forge modules can be installed from [Puppetlabs Forge](https://forge.puppetlabs.com/) or an internal Forge such as [Pulp](http://www.pulpproject.org/)
 * Git modules can be installed from a branch, tag or specific commit, optionally using a path inside the repository
 * Modules can be installed from GitHub using tarballs, without needing Git installed
 * Module dependencies are resolved transitively without needing to list all the modules explicitly
@@ -38,14 +38,14 @@ for which modules your puppet infrastructure repository  depends goes in here.
 
 This Puppetfile will download all the dependencies listed in your Modulefile from the Puppet Forge
 
-    forge "http://forge.puppetlabs.com"
+    forge "https://forge.puppetlabs.com"
 
     modulefile
 
 
 ### Example Puppetfile
 
-    forge "http://forge.puppetlabs.com"
+    forge "https://forge.puppetlabs.com"
 
     mod "puppetlabs/razor"
     mod "puppetlabs/ntp", "0.0.3"
@@ -67,7 +67,7 @@ When fetching a module all dependencies specified in its
 
 ### Puppetfile Breakdown
 
-    forge "http://forge.puppetlabs.com"
+    forge "https://forge.puppetlabs.com"
 
 This declares that we want to use the official Puppet Labs Forge as our default
 source when pulling down modules.  If you run your own local forge, you may

data/lib/librarian/puppet/source/forge.rb

@@ -1,6 +1,6 @@
-require 'json'
-require 'open-uri'
+require 'uri'
 require 'librarian/puppet/util'
+require 'librarian/puppet/source/forge/repo'
 
 module Librarian
   module Puppet
@@ -8,204 +8,6 @@ module Librarian
       class Forge
         include Librarian::Puppet::Util
 
-        class Repo
-          include Librarian::Puppet::Util
-
-          attr_accessor :source, :name
-          private :source=, :name=
-
-          def initialize(source, name)
-            self.source = source
-            self.name = name
-            # API returned data for this module including all versions and dependencies, indexed by module name
-            # from http://forge.puppetlabs.com/api/v1/releases.json?module=#{name}
-            @api_data = nil
-            # API returned data for this module and a specific version, indexed by version
-            # from http://forge.puppetlabs.com/api/v1/releases.json?module=#{name}&version=#{version}
-            @api_version_data = {}
-          end
-
-          def versions
-            return @versions if @versions
-            @versions = api_data(name).map { |r| r['version'] }.reverse
-            if @versions.empty?
-              info { "No versions found for module #{name}" }
-            else
-              debug { "  Module #{name} found versions: #{@versions.join(", ")}" }
-            end
-            @versions
-          end
-
-          def dependencies(version)
-            api_version_data(name, version)['dependencies']
-          end
-
-          def manifests
-            versions.map do |version|
-              Manifest.new(source, name, version)
-            end
-          end
-
-          def install_version!(version, install_path)
-            if environment.local? && !vendored?(name, version)
-              raise Error, "Could not find a local copy of #{name} at #{version}."
-            end
-
-            if environment.vendor?
-              vendor_cache(name, version) unless vendored?(name, version)
-            end
-
-            cache_version_unpacked! version
-
-            if install_path.exist?
-              install_path.rmtree
-            end
-
-            unpacked_path = version_unpacked_cache_path(version).join(name.split('/').last)
-
-            unless unpacked_path.exist?
-              raise Error, "#{unpacked_path} does not exist, something went wrong. Try removing it manually"
-            else
-              cp_r(unpacked_path, install_path)
-            end
-
-          end
-
-          def environment
-            source.environment
-          end
-
-          def cache_path
-            @cache_path ||= source.cache_path.join(name)
-          end
-
-          def version_unpacked_cache_path(version)
-            cache_path.join('version').join(hexdigest(version.to_s))
-          end
-
-          def hexdigest(value)
-            Digest::MD5.hexdigest(value)
-          end
-
-          def cache_version_unpacked!(version)
-            path = version_unpacked_cache_path(version)
-            return if path.directory?
-
-            # The puppet module command is only available from puppet versions >= 2.7.13
-            #
-            # Specifying the version in the gemspec would force people to upgrade puppet while it's still usable for git
-            # So we do some more clever checking
-            #
-            # Executing older versions or via puppet-module tool gives an exit status = 0 .
-            #
-            check_puppet_module_options
-
-            path.mkpath
-
-            target = vendored?(name, version) ? vendored_path(name, version).to_s : name
-
-            # TODO can't pass the default forge url (http://forge.puppetlabs.com) to clients that use the v3 API (https://forgeapi.puppetlabs.com)
-            module_repository = source.to_s
-            if Forge.client_api_version() > 1 and source =~ %r{^http(s)?://forge\.puppetlabs\.com}
-              module_repository = "https://forgeapi.puppetlabs.com"
-              warn("Your Puppet client uses the Forge API v3, you should use this Forge URL: #{module_repository}")
-            end
-
-            command = %W{puppet module install --version #{version} --target-dir}
-            command.push(*[path.to_s, "--module_repository", module_repository, "--modulepath", path.to_s, "--module_working_dir", path.to_s, "--ignore-dependencies", target])
-            debug { "Executing puppet module install for #{name} #{version}" }
-
-            begin
-              Librarian::Posix.run!(command)
-            rescue Posix::CommandFailure => e
-              # Rollback the directory if the puppet module had an error
-              begin
-                path.unlink
-              rescue => u
-                warn("Unable to rollback path #{path}: #{u}")
-              end
-              tar = Dir[File.join(path.to_s, "**/*.tar.gz")]
-              msg = ""
-              if e.message =~ /Unexpected EOF in archive/ and !tar.empty?
-                file = tar.first
-                msg = " (looks like an incomplete download of #{file})"
-              end
-              raise Error, "Error executing puppet module install#{msg}:\n#{command.join(" ")}\nError:\n#{e.message}"
-            end
-
-          end
-
-          def check_puppet_module_options
-            min_version    = Gem::Version.create('2.7.13')
-
-            if Librarian::Puppet.puppet_gem_version < min_version
-              raise Error, "To get modules from the forge, we use the puppet faces module command. For this you need at least puppet version 2.7.13 and you have #{puppet_version}"
-            end
-          end
-
-          def vendored?(name, version)
-            vendored_path(name, version).exist?
-          end
-
-          def vendored_path(name, version)
-            environment.vendor_cache.join("#{name.sub("/", "-")}-#{version}.tar.gz")
-          end
-
-          def vendor_cache(name, version)
-            info = api_version_data(name, version)
-            url = "#{source}#{info[name].first['file']}"
-            path = vendored_path(name, version).to_s
-            debug { "Downloading #{url} into #{path}"}
-            File.open(path, 'wb') do |f|
-              open(url, "rb") do |input|
-                f.write(input.read)
-              end
-            end
-          end
-
-        private
-
-          # get and cache the API data for a specific module with all its versions and dependencies
-          def api_data(module_name)
-            return @api_data[module_name] if @api_data
-            # call API and cache data
-            @api_data = api_call(module_name)
-            if @api_data.nil?
-              raise Error, "Unable to find module '#{name}' on #{source}"
-            end
-            @api_data[module_name]
-          end
-
-          # get and cache the API data for a specific module and version
-          def api_version_data(module_name, version)
-            # if we already got all the versions, find in cached data
-            return @api_data[module_name].detect{|x| x['version'] == version.to_s} if @api_data
-            # otherwise call the api for this version if not cached already
-            @api_version_data[version] = api_call(name, version) if @api_version_data[version].nil?
-            @api_version_data[version]
-          end
-
-          def api_call(module_name, version=nil)
-            base_url = source.uri
-            path = "api/v1/releases.json?module=#{module_name}"
-            path = "#{path}&version=#{version}" unless version.nil?
-            url = "#{base_url}/#{path}"
-            debug { "Querying Forge API for module #{name}#{" and version #{version}" unless version.nil?}: #{url}" }
-
-            begin
-              data = open(url) {|f| f.read}
-              JSON.parse(data)
-            rescue OpenURI::HTTPError => e
-              case e.io.status[0].to_i
-              when 404,410
-                nil
-              else
-                raise e, "Error requesting #{base_url}/#{path}: #{e.to_s}"
-              end
-            end
-          end
-        end
-
         class << self
           LOCK_NAME = 'FORGE'
 
@@ -231,8 +33,11 @@ module Librarian
             version = 1
             pe_version = Librarian::Puppet.puppet_version.match(/\(Puppet Enterprise (.+)\)/)
 
+            # Puppet 3.6.0+ uses api v3
+            if Librarian::Puppet::puppet_gem_version >= Gem::Version.create('3.6.0.a')
+              version = 3
             # Puppet enterprise 3.2.0+ uses api v3
-            if pe_version and Gem::Version.create(pe_version[1].strip) >= Gem::Version.create('3.2.0')
+            elsif pe_version and Gem::Version.create(pe_version[1].strip) >= Gem::Version.create('3.2.0')
               version = 3
             end
             return version
@@ -246,12 +51,12 @@ module Librarian
 
         def initialize(environment, uri, options = {})
           self.environment = environment
-          @uri = uri
+          @uri = URI::parse(uri)
           @cache_path = nil
         end
 
         def to_s
-          uri
+          clean_uri(uri).to_s
         end
 
         def ==(other)
@@ -267,11 +72,11 @@ module Librarian
         end
 
         def to_spec_args
-          [uri, {}]
+          [clean_uri(uri).to_s, {}]
         end
 
         def to_lock_options
-          {:remote => uri}
+          {:remote => clean_uri(uri).to_s}
         end
 
         def pinned?
@@ -284,6 +89,8 @@ module Librarian
         def install!(manifest)
           manifest.source == self or raise ArgumentError
 
+          debug { "Installing #{manifest}" }
+
           name = manifest.name
           version = manifest.version
           install_path = install_path(name)
@@ -301,7 +108,7 @@ module Librarian
 
         def cache_path
           @cache_path ||= begin
-            dir = Digest::MD5.hexdigest(uri)
+            dir = "#{uri.host}#{uri.path}".gsub(/[^0-9a-z\-_]/i, '_')
             environment.cache_path.join("source/puppet/forge/#{dir}")
           end
         end

data/lib/librarian/puppet/source/forge/repo.rb

@@ -0,0 +1,182 @@
+require 'json'
+require 'open-uri'
+require 'librarian/puppet/util'
+require 'librarian/puppet/source/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class Repo < Librarian::Puppet::Source::Repo
+          include Librarian::Puppet::Util
+
+          def initialize(source, name)
+            super(source, name)
+            # API returned data for this module including all versions and dependencies, indexed by module name
+            # from http://forge.puppetlabs.com/api/v1/releases.json?module=#{name}
+            @api_data = nil
+            # API returned data for this module and a specific version, indexed by version
+            # from http://forge.puppetlabs.com/api/v1/releases.json?module=#{name}&version=#{version}
+            @api_version_data = {}
+          end
+
+          def versions
+            return @versions if @versions
+            @versions = api_data(name).map { |r| r['version'] }.reverse
+            if @versions.empty?
+              info { "No versions found for module #{name}" }
+            else
+              debug { "  Module #{name} found versions: #{@versions.join(", ")}" }
+            end
+            @versions
+          end
+
+          def dependencies(version)
+            api_version_data(name, version)['dependencies']
+          end
+
+          def manifests
+            versions.map do |version|
+              Manifest.new(source, name, version)
+            end
+          end
+
+          def install_version!(version, install_path)
+            if environment.local? && !vendored?(name, version)
+              raise Error, "Could not find a local copy of #{name} at #{version}."
+            end
+
+            if environment.vendor?
+              vendor_cache(name, version) unless vendored?(name, version)
+            end
+
+            cache_version_unpacked! version
+
+            if install_path.exist?
+              install_path.rmtree
+            end
+
+            unpacked_path = version_unpacked_cache_path(version).join(name.split('/').last)
+
+            unless unpacked_path.exist?
+              raise Error, "#{unpacked_path} does not exist, something went wrong. Try removing it manually"
+            else
+              cp_r(unpacked_path, install_path)
+            end
+
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            # The puppet module command is only available from puppet versions >= 2.7.13
+            #
+            # Specifying the version in the gemspec would force people to upgrade puppet while it's still usable for git
+            # So we do some more clever checking
+            #
+            # Executing older versions or via puppet-module tool gives an exit status = 0 .
+            #
+            check_puppet_module_options
+
+            path.mkpath
+
+            target = vendored?(name, version) ? vendored_path(name, version).to_s : name
+
+            # TODO can't pass the default forge url (http://forge.puppetlabs.com) to clients that use the v3 API (https://forgeapi.puppetlabs.com)
+            module_repository = source.to_s
+            if Forge.client_api_version() > 1 and module_repository =~ %r{^http(s)?://forge\.puppetlabs\.com}
+              module_repository = "https://forgeapi.puppetlabs.com"
+            end
+
+            command = %W{puppet module install --version #{version} --target-dir}
+            command.push(*[path.to_s, "--module_repository", module_repository, "--modulepath", path.to_s, "--module_working_dir", path.to_s, "--ignore-dependencies", target])
+            debug { "Executing puppet module install for #{name} #{version}" }
+
+            begin
+              Librarian::Posix.run!(command)
+            rescue Posix::CommandFailure => e
+              # Rollback the directory if the puppet module had an error
+              begin
+                path.unlink
+              rescue => u
+                debug("Unable to rollback path #{path}: #{u}")
+              end
+              tar = Dir[File.join(path.to_s, "**/*.tar.gz")]
+              msg = ""
+              if e.message =~ /Unexpected EOF in archive/ and !tar.empty?
+                file = tar.first
+                msg = " (looks like an incomplete download of #{file})"
+              end
+              raise Error, "Error executing puppet module install#{msg}. Check that this command succeeds:\n#{command.join(" ")}\nError:\n#{e.message}"
+            end
+
+          end
+
+          def check_puppet_module_options
+            min_version    = Gem::Version.create('2.7.13')
+
+            if Librarian::Puppet.puppet_gem_version < min_version
+              raise Error, "To get modules from the forge, we use the puppet faces module command. For this you need at least puppet version 2.7.13 and you have #{puppet_version}"
+            end
+          end
+
+          def vendor_cache(name, version)
+            info = api_version_data(name, version)
+            url = "#{source}#{info[name].first['file']}"
+            path = vendored_path(name, version).to_s
+            debug { "Downloading #{url} into #{path}"}
+            environment.vendor!
+            File.open(path, 'wb') do |f|
+              open(url, "rb") do |input|
+                f.write(input.read)
+              end
+            end
+          end
+
+        private
+
+          # get and cache the API data for a specific module with all its versions and dependencies
+          def api_data(module_name)
+            return @api_data[module_name] if @api_data
+            # call API and cache data
+            @api_data = api_call(module_name)
+            if @api_data.nil?
+              raise Error, "Unable to find module '#{name}' on #{source}"
+            end
+            @api_data[module_name]
+          end
+
+          # get and cache the API data for a specific module and version
+          def api_version_data(module_name, version)
+            # if we already got all the versions, find in cached data
+            return @api_data[module_name].detect{|x| x['version'] == version.to_s} if @api_data
+            # otherwise call the api for this version if not cached already
+            @api_version_data[version] = api_call(name, version) if @api_version_data[version].nil?
+            @api_version_data[version]
+          end
+
+          def api_call(module_name, version=nil)
+            url = source.uri.clone
+            url.path += "#{'/' if url.path.empty? or url.path[-1] != '/'}api/v1/releases.json"
+            url.query = "module=#{module_name}"
+            url.query += "&version=#{version}" unless version.nil?
+            debug { "Querying Forge API for module #{name}#{" and version #{version}" unless version.nil?}: #{url}" }
+
+            begin
+              data = open(url) {|f| f.read}
+              JSON.parse(data)
+            rescue OpenURI::HTTPError => e
+              case e.io.status[0].to_i
+              when 404,410
+                nil
+              else
+                raise e, "Error requesting #{url}: #{e.to_s}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/git.rb

@@ -36,8 +36,14 @@ module Librarian
           cache_in_vendor(repository.path) if environment.vendor?
         end
 
+        private
+
+        def vendor_tar
+          environment.vendor_source.join("#{sha}.tar")
+        end
+
         def vendor_tgz
-          environment.vendor_source + "#{sha}.tar.gz"
+          environment.vendor_source.join("#{sha}.tar.gz")
         end
 
         def vendor_cached?
@@ -48,13 +54,14 @@ module Librarian
           repository.path.rmtree if repository.path.exist?
           repository.path.mkpath
 
-          run!(%W{tar xzf #{vendor_tgz}}, :chdir => repository.path.to_s)
+          Librarian::Posix.run!(%W{tar xzf #{vendor_tgz}}, :chdir => repository.path.to_s)
 
           repository_cached!
         end
 
         def cache_in_vendor(tmp_path)
-          run!(%W{git archive #{sha} | gzip > #{vendor_tgz}}, :chdir => tmp_path.to_s)
+          Librarian::Posix.run!(%W{git archive -o #{vendor_tar} #{sha}}, :chdir => tmp_path.to_s)
+          Librarian::Posix.run!(%W{gzip #{vendor_tar}}, :chdir => tmp_path.to_s)
         end
 
       end

data/lib/librarian/puppet/source/githubtarball.rb

@@ -1,9 +1,6 @@
 require 'uri'
-require 'net/https'
-require 'open-uri'
-require 'json'
-
-require 'librarian/puppet/version'
+require 'librarian/puppet/util'
+require 'librarian/puppet/source/githubtarball/repo'
 
 module Librarian
   module Puppet
@@ -11,158 +8,6 @@ module Librarian
       class GitHubTarball
         include Librarian::Puppet::Util
 
-        class Repo
-          include Librarian::Puppet::Util
-
-          TOKEN_KEY = 'GITHUB_API_TOKEN'
-
-          attr_accessor :source, :name
-          private :source=, :name=
-
-          def initialize(source, name)
-            self.source = source
-            self.name = name
-          end
-
-          def versions
-            return @versions if @versions
-            data = api_call("/repos/#{source.uri}/tags")
-            if data.nil?
-              raise Error, "Unable to find module '#{source.uri}' on https://github.com"
-            end
-
-            all_versions = data.map { |r| r['name'].gsub(/^v/, '') }.sort.reverse
-
-            all_versions.delete_if do |version|
-              version !~ /\A\d\.\d(\.\d.*)?\z/
-            end
-
-            @versions = all_versions.compact
-            debug { "  Module #{name} found versions: #{@versions.join(", ")}" }
-            @versions
-          end
-
-          def manifests
-            versions.map do |version|
-              Manifest.new(source, name, version)
-            end
-          end
-
-          def install_version!(version, install_path)
-            if environment.local? && !vendored?(source.uri, version)
-              raise Error, "Could not find a local copy of #{source.uri} at #{version}."
-            end
-
-            vendor_cache(source.uri, version) unless vendored?(source.uri, version)
-
-            cache_version_unpacked! version
-
-            if install_path.exist?
-              install_path.rmtree
-            end
-
-            unpacked_path = version_unpacked_cache_path(version).children.first
-            cp_r(unpacked_path, install_path)
-          end
-
-          def environment
-            source.environment
-          end
-
-          def cache_path
-            @cache_path ||= source.cache_path.join(name)
-          end
-
-          def version_unpacked_cache_path(version)
-            cache_path.join('version').join(hexdigest(version.to_s))
-          end
-
-          def hexdigest(value)
-            Digest::MD5.hexdigest(value)
-          end
-
-          def cache_version_unpacked!(version)
-            path = version_unpacked_cache_path(version)
-            return if path.directory?
-
-            path.mkpath
-
-            target = vendored?(source.uri, version) ? vendored_path(source.uri, version) : name
-
-            Librarian::Posix.run!(%W{tar xzf #{target} -C #{path}})
-          end
-
-          def vendored?(name, version)
-            vendored_path(name, version).exist?
-          end
-
-          def vendored_path(name, version)
-            environment.vendor_cache.mkpath
-            environment.vendor_cache.join("#{name.sub("/", "-")}-#{version}.tar.gz")
-          end
-
-          def vendor_cache(name, version)
-            clean_up_old_cached_versions(name)
-
-            url = "https://api.github.com/repos/#{name}/tarball/#{version}"
-            url << "?access_token=#{ENV['GITHUB_API_TOKEN']}" if ENV['GITHUB_API_TOKEN']
-
-            File.open(vendored_path(name, version).to_s, 'wb') do |f|
-              begin
-                debug { "Downloading <#{url}> to <#{f.path}>" }
-                open(url,
-                  "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}") do |res|
-                  while buffer = res.read(8192)
-                    f.write(buffer)
-                  end
-                end
-              rescue OpenURI::HTTPError => e
-                raise e, "Error requesting <#{url}>: #{e.to_s}"
-              end
-            end
-          end
-
-          def clean_up_old_cached_versions(name)
-            Dir["#{environment.vendor_cache}/#{name.sub('/', '-')}*.tar.gz"].each do |old_version|
-              FileUtils.rm old_version
-            end
-          end
-
-        private
-
-          def api_call(path)
-            url = "https://api.github.com#{path}"
-            url << "?access_token=#{ENV[TOKEN_KEY]}" if ENV[TOKEN_KEY]
-            code, data = http_get(url, :headers => {
-              "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}"
-            })
-
-            if code == 200
-              JSON.parse(data)
-            elsif code == 403
-              begin
-                message = JSON.parse(data)['message']
-                if message && message.include?('API rate limit exceeded')
-                  raise Error, message + " -- increase limit by authenticating via #{TOKEN_KEY}=your-token"
-                end
-              rescue JSON::ParserError
-                # 403 response does not return json, skip.
-              end
-            end
-          end
-
-          def http_get(url, options)
-            uri = URI.parse(url)
-            http = Net::HTTP.new(uri.host, uri.port)
-            http.use_ssl = true
-            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-            request = Net::HTTP::Get.new(uri.request_uri)
-            options[:headers].each { |k, v| request.add_field k, v }
-            resp = http.request(request)
-            [resp.code.to_i, resp.body]
-          end
-        end
-
         class << self
           LOCK_NAME = 'GITHUBTARBALL'
 
@@ -191,12 +36,12 @@ module Librarian
 
         def initialize(environment, uri, options = {})
           self.environment = environment
-          @uri = uri
+          @uri = URI::parse(uri)
           @cache_path = nil
         end
 
         def to_s
-          uri
+          clean_uri(uri).to_s
         end
 
         def ==(other)
@@ -212,11 +57,11 @@ module Librarian
         end
 
         def to_spec_args
-          [uri, {}]
+          [clean_uri(uri).to_s, {}]
         end
 
         def to_lock_options
-          {:remote => uri}
+          {:remote => clean_uri(uri).to_s}
         end
 
         def pinned?
@@ -229,6 +74,8 @@ module Librarian
         def install!(manifest)
           manifest.source == self or raise ArgumentError
 
+          debug { "Installing #{manifest}" }
+
           name = manifest.name
           version = manifest.version
           install_path = install_path(name)
@@ -246,8 +93,7 @@ module Librarian
 
         def cache_path
           @cache_path ||= begin
-            dir = Digest::MD5.hexdigest(uri)
-            environment.cache_path.join("source/puppet/githubtarball/#{dir}")
+            environment.cache_path.join("source/puppet/githubtarball/#{uri.host}#{uri.path}")
           end
         end
 

data/lib/librarian/puppet/source/githubtarball/repo.rb

@@ -0,0 +1,149 @@
+require 'uri'
+require 'net/https'
+require 'open-uri'
+require 'json'
+
+require 'librarian/puppet/version'
+require 'librarian/puppet/source/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class GitHubTarball
+        class Repo < Librarian::Puppet::Source::Repo
+          include Librarian::Puppet::Util
+
+          TOKEN_KEY = 'GITHUB_API_TOKEN'
+
+          def versions
+            return @versions if @versions
+            data = api_call("/repos/#{source.uri}/tags")
+            if data.nil?
+              raise Error, "Unable to find module '#{source.uri}' on https://github.com"
+            end
+
+            all_versions = data.map { |r| r['name'].gsub(/^v/, '') }.sort.reverse
+
+            all_versions.delete_if do |version|
+              version !~ /\A\d\.\d(\.\d.*)?\z/
+            end
+
+            @versions = all_versions.compact
+            debug { "  Module #{name} found versions: #{@versions.join(", ")}" }
+            @versions
+          end
+
+          def manifests
+            versions.map do |version|
+              Manifest.new(source, name, version)
+            end
+          end
+
+          def install_version!(version, install_path)
+            if environment.local? && !vendored?(source.uri.to_s, version)
+              raise Error, "Could not find a local copy of #{source.uri} at #{version}."
+            end
+
+            vendor_cache(source.uri.to_s, version) unless vendored?(source.uri.to_s, version)
+
+            cache_version_unpacked! version
+
+            if install_path.exist?
+              install_path.rmtree
+            end
+
+            unpacked_path = version_unpacked_cache_path(version).children.first
+            cp_r(unpacked_path, install_path)
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            path.mkpath
+
+            target = vendored?(source.uri.to_s, version) ? vendored_path(source.uri.to_s, version) : name
+
+            Librarian::Posix.run!(%W{tar xzf #{target} -C #{path}})
+          end
+
+          def vendor_cache(name, version)
+            clean_up_old_cached_versions(name)
+
+            url = "https://api.github.com/repos/#{name}/tarball/#{version}"
+            url << "?access_token=#{ENV['GITHUB_API_TOKEN']}" if ENV['GITHUB_API_TOKEN']
+
+            environment.vendor!
+            File.open(vendored_path(name, version).to_s, 'wb') do |f|
+              begin
+                debug { "Downloading <#{url}> to <#{f.path}>" }
+                open(url,
+                  "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}") do |res|
+                  while buffer = res.read(8192)
+                    f.write(buffer)
+                  end
+                end
+              rescue OpenURI::HTTPError => e
+                raise e, "Error requesting <#{url}>: #{e.to_s}"
+              end
+            end
+          end
+
+          def clean_up_old_cached_versions(name)
+            Dir["#{environment.vendor_cache}/#{name.sub('/', '-')}*.tar.gz"].each do |old_version|
+              FileUtils.rm old_version
+            end
+          end
+
+        private
+
+          def api_call(path)
+            tags = []
+            url = "https://api.github.com#{path}?page=1&per_page=100"
+            while true do
+              debug { "  Module #{name} getting tags at: #{url}" }
+              url << "&access_token=#{ENV[TOKEN_KEY]}" if ENV[TOKEN_KEY]
+              response = http_get(url, :headers => {
+                "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}"
+              })
+
+              code, data = response.code.to_i, response.body
+
+              if code == 200
+                tags.concat JSON.parse(data)
+              else
+                begin
+                  message = JSON.parse(data)['message']
+                  if code == 403 && message && message.include?('API rate limit exceeded')
+                    raise Error, message + " -- increase limit by authenticating via #{TOKEN_KEY}=your-token"
+                  elsif message
+                    raise Error, "Error fetching #{url}: [#{code}] #{message}"
+                  end
+                rescue JSON::ParserError
+                  # response does not return json
+                end
+                raise Error, "Error fetching #{url}: [#{code}] #{response.body}"
+              end
+
+              # next page
+              break if response["link"].nil?
+              next_link = response["link"].split(",").select{|l| l.match /rel=.*next.*/}
+              break if next_link.empty?
+              url = next_link.first.match(/<(.*)>/)[1]
+            end
+            return tags
+          end
+
+          def http_get(url, options)
+            uri = URI.parse(url)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = true
+            request = Net::HTTP::Get.new(uri.request_uri)
+            options[:headers].each { |k, v| request.add_field k, v }
+            http.request(request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/local.rb

@@ -83,7 +83,11 @@ module Librarian
             ::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
           rescue ArgumentError, SyntaxError => error
             warn { "Unable to parse #{modulefile}, ignoring: #{error}" }
-            metadata.version = '0.0.1'
+            if metadata.respond_to? :version=
+              metadata.version = '0.0.1' # puppet < 3.6
+            else
+              metadata.update({'version' => '0.0.1'}) # puppet >= 3.6
+            end
           end
           metadata
         end

data/lib/librarian/puppet/source/repo.rb

@@ -0,0 +1,38 @@
+# parent class for githubtarball and forge source Repos
+module Librarian
+  module Puppet
+    module Source
+      class Repo
+
+        attr_accessor :source, :name
+        private :source=, :name=
+
+        def initialize(source, name)
+          self.source = source
+          self.name = name
+        end
+
+        def environment
+          source.environment
+        end
+
+        def cache_path
+          @cache_path ||= source.cache_path.join(name)
+        end
+
+        def version_unpacked_cache_path(version)
+          cache_path.join(version.to_s)
+        end
+
+        def vendored?(name, version)
+          vendored_path(name, version).exist?
+        end
+
+        def vendored_path(name, version)
+          environment.vendor_cache.join("#{name.sub("/", "-")}-#{version}.tar.gz")
+        end
+
+      end
+    end
+  end
+end

data/lib/librarian/puppet/util.rb

@@ -25,6 +25,14 @@ module Librarian
           FileUtils.cp_r(src, dest)
         end
       end
+
+      # Remove user and password from a URI object
+      def clean_uri(uri)
+        new_uri = uri.clone
+        new_uri.user = nil
+        new_uri.password = nil
+        new_uri
+      end
     end
   end
 end

data/lib/librarian/puppet/version.rb

@@ -1,5 +1,5 @@
 module Librarian
   module Puppet
-    VERSION = "1.0.1"
+    VERSION = "1.0.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: librarian-puppet
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Tim Sharpe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-12 00:00:00.000000000 Z
+date: 2014-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: librarian
@@ -160,10 +160,13 @@ files:
 - lib/librarian/puppet/requirement.rb
 - lib/librarian/puppet/source.rb
 - lib/librarian/puppet/source/forge.rb
+- lib/librarian/puppet/source/forge/repo.rb
 - lib/librarian/puppet/source/git.rb
 - lib/librarian/puppet/source/githubtarball.rb
+- lib/librarian/puppet/source/githubtarball/repo.rb
 - lib/librarian/puppet/source/local.rb
 - lib/librarian/puppet/source/path.rb
+- lib/librarian/puppet/source/repo.rb
 - lib/librarian/puppet/templates/Puppetfile
 - lib/librarian/puppet/util.rb
 - lib/librarian/puppet/version.rb