librarian-puppet 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef4fd59f2a6e41ed320ea0bb94eb6374af121092
-  data.tar.gz: a8575338727993713ef495723986d28e71007244
+  metadata.gz: 3904f177c8c656c97186e7ffebf578db9fde4569
+  data.tar.gz: 14b95b8c41e5848b109c0b9c2759ba8017a20986
 SHA512:
-  metadata.gz: d006b91e46d7c964aacbeed2a88e4e385e9c8357e1a9f2920dced67d008ee0acba6d868f7dffccca7acc5ddc0642de1e43f680a96a7379b8f0b767090fe3222a
-  data.tar.gz: 666b2ee1a3d469fd23272495d228a9d6c64b8df70a2e7ae9a29ef8eee64bee06fca3bfdb495e06b23757413263c93d993bb637309ea292c127f8845c48097396
+  metadata.gz: fdbac9e14ea288dac3d92f2dbfcc5fa019bacdbcf63f54b237359ee41cc8f291b8dbc740e748cfa8d236384a85eee5c7b4f0e277a771a01703480897756070da
+  data.tar.gz: d3e6ff6c0f5211516e21cb6c656120638a720822ddb2acded53d456831cead0feed152ccd7b22e903781d5345aae285071d707173181d500904fcc96a568ed22

data/README.md

@@ -38,7 +38,7 @@ for which modules your puppet infrastructure repository  depends goes in here.
 
 This Puppetfile will download all the dependencies listed in your Modulefile from the Puppet Forge
 
-    forge "https://forge.puppetlabs.com"
+    forge "https://forgeapi.puppetlabs.com"
 
     modulefile
 

data/lib/librarian/puppet/source/forge.rb

@@ -1,6 +1,7 @@
 require 'uri'
 require 'librarian/puppet/util'
-require 'librarian/puppet/source/forge/repo'
+require 'librarian/puppet/source/forge/repo_v1'
+# require 'librarian/puppet/source/forge/repo_v3'
 
 module Librarian
   module Puppet
@@ -51,6 +52,12 @@ module Librarian
 
         def initialize(environment, uri, options = {})
           self.environment = environment
+
+          # if uri =~ %r{^http(s)?://forge\.puppetlabs\.com}
+          #   uri = "https://forgeapi.puppetlabs.com"
+          #   warn { "Replacing Puppet Forge API URL to use v3 #{uri}. You should update your Puppetfile" }
+          # end
+
           @uri = URI::parse(uri)
           @cache_path = nil
         end
@@ -141,7 +148,15 @@ module Librarian
 
         def repo(name)
           @repo ||= {}
-          @repo[name] ||= Repo.new(self, name)
+          unless @repo[name]
+            # if we are using the official Forge then use API v3, otherwise stick to v1 for now
+            # if uri.hostname =~ /\.puppetlabs\.com$/
+            #   @repo[name] = RepoV3.new(self, name)
+            # else
+              @repo[name] = RepoV1.new(self, name)
+            # end
+          end
+          @repo[name]
         end
       end
     end

data/lib/librarian/puppet/source/forge/repo.rb

@@ -10,19 +10,9 @@ module Librarian
         class Repo < Librarian::Puppet::Source::Repo
           include Librarian::Puppet::Util
 
-          def initialize(source, name)
-            super(source, name)
-            # API returned data for this module including all versions and dependencies, indexed by module name
-            # from http://forge.puppetlabs.com/api/v1/releases.json?module=#{name}
-            @api_data = nil
-            # API returned data for this module and a specific version, indexed by version
-            # from http://forge.puppetlabs.com/api/v1/releases.json?module=#{name}&version=#{version}
-            @api_version_data = {}
-          end
-
           def versions
             return @versions if @versions
-            @versions = api_data(name).map { |r| r['version'] }.reverse
+            @versions = get_versions
             if @versions.empty?
               info { "No versions found for module #{name}" }
             else
@@ -31,8 +21,21 @@ module Librarian
             @versions
           end
 
+          # fetch list of versions ordered for newer to older
+          def get_versions
+            # implement in subclasses
+          end
+
+          # return map with dependencies in the form {module_name => version,...}
+          # version: Librarian::Manifest::Version
           def dependencies(version)
-            api_version_data(name, version)['dependencies']
+            # implement in subclasses
+          end
+
+          # return the url for a specific version tarball
+          # version: Librarian::Manifest::Version
+          def url(name, version)
+            # implement in subclasses
           end
 
           def manifests
@@ -83,10 +86,25 @@ module Librarian
 
             target = vendored?(name, version) ? vendored_path(name, version).to_s : name
 
-            # TODO can't pass the default forge url (http://forge.puppetlabs.com) to clients that use the v3 API (https://forgeapi.puppetlabs.com)
+            # can't pass the default v3 forge url (http://forgeapi.puppetlabs.com)
+            # to clients that use the v1 API (https://forge.puppetlabs.com)
+            # nor the other way around
             module_repository = source.to_s
+
             if Forge.client_api_version() > 1 and module_repository =~ %r{^http(s)?://forge\.puppetlabs\.com}
               module_repository = "https://forgeapi.puppetlabs.com"
+              warn { "Replacing Puppet Forge API URL to use v3 #{module_repository} as required by your client version #{Librarian::Puppet.puppet_version}" }
+            end
+
+            m = module_repository.match(%r{^http(s)?://forgeapi\.puppetlabs\.com})
+            if Forge.client_api_version() == 1 and m
+              ssl = m[1]
+              # Puppet 2.7 can't handle the 302 returned by the https url, so stick to http
+              if ssl and Librarian::Puppet::puppet_gem_version < Gem::Version.create('3.0.0')
+                warn { "Using plain http as your version of Puppet #{Librarian::Puppet::puppet_gem_version} can't download from forge.puppetlabs.com using https" }
+                ssl = nil
+              end
+              module_repository = "http#{ssl}://forge.puppetlabs.com"
             end
 
             command = %W{puppet module install --version #{version} --target-dir}
@@ -122,8 +140,7 @@ module Librarian
           end
 
           def vendor_cache(name, version)
-            info = api_version_data(name, version)
-            url = "#{source}#{info[name].first['file']}"
+            url = url(name, version)
             path = vendored_path(name, version).to_s
             debug { "Downloading #{url} into #{path}"}
             environment.vendor!
@@ -134,70 +151,6 @@ module Librarian
             end
           end
 
-        private
-
-          # Issue #223 dependencies may be duplicated
-          def clear_duplicated_dependencies(data)
-            return nil if data.nil?
-            data.each do |m,versions|
-              versions.each do |v|
-                if v["dependencies"] and !v["dependencies"].empty?
-                  dependency_names = v["dependencies"].map {|d| d[0]}
-                  duplicated = dependency_names.select{ |e| dependency_names.count(e) > 1 }
-                  unless duplicated.empty?
-                    duplicated.uniq.each do |module_duplicated|
-                      to_remove = []
-                      v["dependencies"].each_index{|i| to_remove << i if module_duplicated == v["dependencies"][i][0]}
-                      warn { "Module #{m}@#{v["version"]} contains duplicated dependencies for #{module_duplicated}, ignoring all but the first of #{to_remove.map {|i| v["dependencies"][i]}}" }
-                      to_remove.slice(1..-1).reverse.each {|i| v["dependencies"].delete_at(i) }
-                      v["dependencies"] = v["dependencies"] - to_remove.slice(1..-1)
-                    end
-                  end
-                end
-              end
-            end
-            data
-          end
-
-          # get and cache the API data for a specific module with all its versions and dependencies
-          def api_data(module_name)
-            return @api_data[module_name] if @api_data
-            # call API and cache data
-            @api_data = clear_duplicated_dependencies(api_call(module_name))
-            if @api_data.nil?
-              raise Error, "Unable to find module '#{name}' on #{source}"
-            end
-            @api_data[module_name]
-          end
-
-          # get and cache the API data for a specific module and version
-          def api_version_data(module_name, version)
-            # if we already got all the versions, find in cached data
-            return @api_data[module_name].detect{|x| x['version'] == version.to_s} if @api_data
-            # otherwise call the api for this version if not cached already
-            @api_version_data[version] = clear_duplicated_dependencies(api_call(name, version)) if @api_version_data[version].nil?
-            @api_version_data[version]
-          end
-
-          def api_call(module_name, version=nil)
-            url = source.uri.clone
-            url.path += "#{'/' if url.path.empty? or url.path[-1] != '/'}api/v1/releases.json"
-            url.query = "module=#{module_name}"
-            url.query += "&version=#{version}" unless version.nil?
-            debug { "Querying Forge API for module #{name}#{" and version #{version}" unless version.nil?}: #{url}" }
-
-            begin
-              data = open(url) {|f| f.read}
-              JSON.parse(data)
-            rescue OpenURI::HTTPError => e
-              case e.io.status[0].to_i
-              when 404,410
-                nil
-              else
-                raise e, "Error requesting #{url}: #{e.to_s}"
-              end
-            end
-          end
         end
       end
     end

data/lib/librarian/puppet/source/forge/repo_v1.rb

@@ -0,0 +1,102 @@
+require 'json'
+require 'open-uri'
+require 'librarian/puppet/source/forge/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class RepoV1 < Librarian::Puppet::Source::Forge::Repo
+
+          def initialize(source, name)
+            super(source, name)
+            # API returned data for this module including all versions and dependencies, indexed by module name
+            # from http://forge.puppetlabs.com/api/v1/releases.json?module=#{name}
+            @api_data = nil
+            # API returned data for this module and a specific version, indexed by version
+            # from http://forge.puppetlabs.com/api/v1/releases.json?module=#{name}&version=#{version}
+            @api_version_data = {}
+          end
+
+          def get_versions
+            api_data(name).map { |r| r['version'] }.reverse
+          end
+
+          def dependencies(version)
+            api_version_data(name, version)['dependencies']
+          end
+
+          def url(name, version)
+            info = api_version_data(name, version)
+            "#{source}#{info[name].first['file']}"
+          end
+
+        private
+
+          # Issue #223 dependencies may be duplicated
+          def clear_duplicated_dependencies(data)
+            return nil if data.nil?
+            data.each do |m,versions|
+              versions.each do |v|
+                if v["dependencies"] and !v["dependencies"].empty?
+                  dependency_names = v["dependencies"].map {|d| d[0]}
+                  duplicated = dependency_names.select{ |e| dependency_names.count(e) > 1 }
+                  unless duplicated.empty?
+                    duplicated.uniq.each do |module_duplicated|
+                      to_remove = []
+                      v["dependencies"].each_index{|i| to_remove << i if module_duplicated == v["dependencies"][i][0]}
+                      warn { "Module #{m}@#{v["version"]} contains duplicated dependencies for #{module_duplicated}, ignoring all but the first of #{to_remove.map {|i| v["dependencies"][i]}}" }
+                      to_remove.slice(1..-1).reverse.each {|i| v["dependencies"].delete_at(i) }
+                      v["dependencies"] = v["dependencies"] - to_remove.slice(1..-1)
+                    end
+                  end
+                end
+              end
+            end
+            data
+          end
+
+          # get and cache the API data for a specific module with all its versions and dependencies
+          def api_data(module_name)
+            return @api_data[module_name] if @api_data
+            # call API and cache data
+            @api_data = clear_duplicated_dependencies(api_call(module_name))
+            if @api_data.nil?
+              raise Error, "Unable to find module '#{name}' on #{source}"
+            end
+            @api_data[module_name]
+          end
+
+          # get and cache the API data for a specific module and version
+          def api_version_data(module_name, version)
+            # if we already got all the versions, find in cached data
+            return @api_data[module_name].detect{|x| x['version'] == version.to_s} if @api_data
+            # otherwise call the api for this version if not cached already
+            @api_version_data[version] = clear_duplicated_dependencies(api_call(name, version)) if @api_version_data[version].nil?
+            @api_version_data[version]
+          end
+
+          def api_call(module_name, version=nil)
+            url = source.uri.clone
+            url.path += "#{'/' if url.path.empty? or url.path[-1] != '/'}api/v1/releases.json"
+            url.query = "module=#{module_name}"
+            url.query += "&version=#{version}" unless version.nil?
+            debug { "Querying Forge API for module #{name}#{" and version #{version}" unless version.nil?}: #{url}" }
+
+            begin
+              data = open(url) {|f| f.read}
+              JSON.parse(data)
+            rescue OpenURI::HTTPError => e
+              case e.io.status[0].to_i
+              when 404,410
+                nil
+              else
+                raise e, "Error requesting #{url}: #{e.to_s}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/forge/repo_v3.rb

@@ -0,0 +1,54 @@
+require 'librarian/puppet/source/forge/repo'
+require 'puppet_forge'
+require 'librarian/puppet/version'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class RepoV3 < Librarian::Puppet::Source::Forge::Repo
+
+          PuppetForge.user_agent = "librarian-puppet/#{Librarian::Puppet::VERSION}"
+
+          def get_versions
+            get_module.releases.map{|r| r.version}
+          end
+
+          def dependencies(version)
+            array = get_release(version).metadata[:dependencies].map{|d| [d['name'], d['version_requirement']]}
+            Hash[*array.flatten(1)]
+          end
+
+          def url(name, version)
+            if name == "#{get_module().owner.username}/#{get_module().name}"
+              release = get_release(version)
+            else
+              # should never get here as we use one repo object for each module (to be changed in the future)
+              debug { "Looking up url for #{name}@#{version}" }
+              release = PuppetForge::Release.find("#{name.sub('/','-')}-#{version}")
+            end
+            "#{source}#{release.file_uri}"
+          end
+
+        private
+
+          def get_module
+            @module ||= PuppetForge::Module.find(name.sub('/','-'))
+            raise(Error, "Unable to find module '#{name}' on #{source}") unless @module
+            @module
+          end
+
+          def get_release(version)
+            release = get_module.releases.find{|r| r.version == version.to_s}
+            if release.nil?
+              versions = get_module.releases.map{|r| r.version}
+              raise Error, "Unable to find version '#{version}' for module '#{name}' on #{source} amongst #{versions}"
+            end
+            release
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/githubtarball/repo.rb

@@ -25,7 +25,7 @@ module Librarian
             all_versions = data.map { |r| r['name'].gsub(/^v/, '') }.sort.reverse
 
             all_versions.delete_if do |version|
-              version !~ /\A\d\.\d(\.\d.*)?\z/
+              version !~ /\A\d+\.\d+(\.\d+.*)?\z/
             end
 
             @versions = all_versions.compact
@@ -71,7 +71,7 @@ module Librarian
             clean_up_old_cached_versions(name)
 
             url = "https://api.github.com/repos/#{name}/tarball/#{version}"
-            url << "?access_token=#{ENV['GITHUB_API_TOKEN']}" if ENV['GITHUB_API_TOKEN']
+            add_api_token_to_url(url)
 
             environment.vendor!
             File.open(vendored_path(name, version).to_s, 'wb') do |f|
@@ -95,6 +95,23 @@ module Librarian
             end
           end
 
+          def token_key_value
+            ENV[TOKEN_KEY]
+          end
+
+          def token_key_nil?
+            token_key_value.nil? || token_key_value.empty?
+          end
+
+          def add_api_token_to_url url
+            if token_key_nil?
+              debug { "#{TOKEN_KEY} environment value is empty or missing" }
+            else
+              url << "?access_token=#{ENV[TOKEN_KEY]}"
+            end
+            url
+          end
+
         private
 
           def api_call(path)
@@ -102,7 +119,7 @@ module Librarian
             url = "https://api.github.com#{path}?page=1&per_page=100"
             while true do
               debug { "  Module #{name} getting tags at: #{url}" }
-              url << "&access_token=#{ENV[TOKEN_KEY]}" if ENV[TOKEN_KEY]
+              add_api_token_to_url(url)
               response = http_get(url, :headers => {
                 "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}"
               })

data/lib/librarian/puppet/templates/Puppetfile

@@ -1,16 +1,22 @@
 #!/usr/bin/env ruby
 #^syntax detection
 
-forge "http://forge.puppetlabs.com"
+forge "https://forgeapi.puppetlabs.com"
 
 # use dependencies defined in Modulefile
 modulefile
 
+# A module from the Puppet Forge
 # mod 'puppetlabs/stdlib'
 
-# mod 'ntp',
+# A module from git
+# mod 'puppetlabs/ntp',
 #   :git => 'git://github.com/puppetlabs/puppetlabs-ntp.git'
 
-# mod 'apt',
+# A module from a git branch/tag
+# mod 'puppetlabs/apt',
 #   :git => 'https://github.com/puppetlabs/puppetlabs-apt.git',
-#   :ref => 'feature/master/dans_refactor'
+#   :ref => '1.4.x'
+
+# A module from Github pre-packaged tarball
+# mod 'puppetlabs/apache', '0.6.0', :github_tarball => 'puppetlabs/puppetlabs-apache'

data/lib/librarian/puppet/version.rb

@@ -1,5 +1,5 @@
 module Librarian
   module Puppet
-    VERSION = "1.0.3"
+    VERSION = "1.0.4"
   end
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: librarian-puppet
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Tim Sharpe
+- Carlos Sanchez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-07 00:00:00.000000000 Z
+date: 2014-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: librarian
@@ -142,6 +143,7 @@ description: |-
     a single command.
 email:
 - tim@sharpe.id.au
+- carlos@apache.org
 executables:
 - librarian-puppet
 extensions: []
@@ -161,6 +163,8 @@ files:
 - lib/librarian/puppet/source.rb
 - lib/librarian/puppet/source/forge.rb
 - lib/librarian/puppet/source/forge/repo.rb
+- lib/librarian/puppet/source/forge/repo_v1.rb
+- lib/librarian/puppet/source/forge/repo_v3.rb
 - lib/librarian/puppet/source/git.rb
 - lib/librarian/puppet/source/githubtarball.rb
 - lib/librarian/puppet/source/githubtarball/repo.rb