anthill-librarian-puppet 3.0.0

data/lib/librarian/puppet/source/forge/repo.rb

@@ -0,0 +1,158 @@
+require 'json'
+require 'open-uri'
+require 'librarian/puppet/util'
+require 'librarian/puppet/source/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class Repo < Librarian::Puppet::Source::Repo
+          include Librarian::Puppet::Util
+
+          def versions
+            return @versions if @versions
+            @versions = get_versions
+            if @versions.empty?
+              info { "No versions found for module #{name}" }
+            else
+              debug { "  Module #{name} found versions: #{@versions.join(", ")}" }
+            end
+            @versions
+          end
+
+          # fetch list of versions ordered for newer to older
+          def get_versions
+            # implement in subclasses
+          end
+
+          # return map with dependencies in the form {module_name => version,...}
+          # version: Librarian::Manifest::Version
+          def dependencies(version)
+            # implement in subclasses
+          end
+
+          # return the url for a specific version tarball
+          # version: Librarian::Manifest::Version
+          def url(name, version)
+            # implement in subclasses
+          end
+
+          def manifests
+            versions.map do |version|
+              Manifest.new(source, name, version)
+            end
+          end
+
+          def install_version!(version, install_path)
+            if environment.local? && !vendored?(name, version)
+              raise Error, "Could not find a local copy of #{name} at #{version}."
+            end
+
+            if environment.vendor?
+              vendor_cache(name, version) unless vendored?(name, version)
+            end
+
+            cache_version_unpacked! version
+
+            if install_path.exist? && rsync? != true
+              install_path.rmtree
+            end
+
+            unpacked_path = version_unpacked_cache_path(version).join(module_name(name))
+
+            unless unpacked_path.exist?
+              raise Error, "#{unpacked_path} does not exist, something went wrong. Try removing it manually"
+            else
+              cp_r(unpacked_path, install_path)
+            end
+
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            # The puppet module command is only available from puppet versions >= 2.7.13
+            #
+            # Specifying the version in the gemspec would force people to upgrade puppet while it's still usable for git
+            # So we do some more clever checking
+            #
+            # Executing older versions or via puppet-module tool gives an exit status = 0 .
+            #
+            check_puppet_module_options
+
+            path.mkpath
+
+            target = vendored?(name, version) ? vendored_path(name, version).to_s : name
+
+            # can't pass the default v3 forge url (http://forgeapi.puppetlabs.com)
+            # to clients that use the v1 API (https://forge.puppet.com)
+            # nor the other way around
+            module_repository = source.to_s
+
+            if Forge.client_api_version() > 1 and module_repository =~ %r{^http(s)?://forge\.puppetlabs\.com}
+              module_repository = "https://forgeapi.puppetlabs.com"
+              warn { "Replacing Puppet Forge API URL to use v3 #{module_repository} as required by your client version #{Librarian::Puppet.puppet_version}" }
+            end
+
+            m = module_repository.match(%r{^http(s)?://forge(api)?\.puppetlabs\.com})
+            if Forge.client_api_version() == 1 and m
+              ssl = m[1]
+              # Puppet 2.7 can't handle the 302 returned by the https url, so stick to http
+              if ssl and Librarian::Puppet::puppet_gem_version < Gem::Version.create('3.0.0')
+                warn { "Using plain http as your version of Puppet #{Librarian::Puppet::puppet_gem_version} can't download from forge.puppetlabs.com using https" }
+                ssl = nil
+              end
+              module_repository = "http#{ssl}://forge.puppetlabs.com"
+            end
+
+            command = %W{puppet module install --version #{version} --target-dir}
+            command.push(*[path.to_s, "--module_repository", module_repository, "--modulepath", path.to_s, "--module_working_dir", path.to_s, "--ignore-dependencies", target])
+            debug { "Executing puppet module install for #{name} #{version}: #{command.join(" ")}" }
+
+            begin
+              Librarian::Posix.run!(command)
+            rescue Posix::CommandFailure => e
+              # Rollback the directory if the puppet module had an error
+              begin
+                path.unlink
+              rescue => u
+                debug("Unable to rollback path #{path}: #{u}")
+              end
+              tar = Dir[File.join(path.to_s, "**/*.tar.gz")]
+              msg = ""
+              if e.message =~ /Unexpected EOF in archive/ and !tar.empty?
+                file = tar.first
+                msg = " (looks like an incomplete download of #{file})"
+              end
+              raise Error, "Error executing puppet module install#{msg}. Check that this command succeeds:\n#{command.join(" ")}\nError:\n#{e.message}"
+            end
+
+          end
+
+          def check_puppet_module_options
+            min_version    = Gem::Version.create('2.7.13')
+
+            if Librarian::Puppet.puppet_gem_version < min_version
+              raise Error, "To get modules from the forge, we use the puppet faces module command. For this you need at least puppet version 2.7.13 and you have #{Librarian::Puppet.puppet_version}"
+            end
+          end
+
+          def vendor_cache(name, version)
+            url = url(name, version)
+            path = vendored_path(name, version).to_s
+            debug { "Downloading #{url} into #{path}"}
+            environment.vendor!
+            File.open(path, 'wb') do |f|
+              open(url, "rb") do |input|
+                f.write(input.read)
+              end
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/forge/repo_v1.rb

@@ -0,0 +1,92 @@
+require 'json'
+require 'open-uri'
+require 'librarian/puppet/source/forge/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class RepoV1 < Librarian::Puppet::Source::Forge::Repo
+
+          def initialize(source, name)
+            super(source, name)
+            # API returned data for this module including all versions and dependencies, indexed by module name
+            # from https://forge.puppetlabs.com/api/v1/releases.json?module=#{name}
+            @api_data = nil
+            # API returned data for this module and a specific version, indexed by version
+            # from https://forge.puppetlabs.com/api/v1/releases.json?module=#{name}&version=#{version}
+            @api_version_data = {}
+          end
+
+          def get_versions
+            api_data(name).map { |r| r['version'] }.reverse
+          end
+
+          def dependencies(version)
+            api_version_data(name, version)['dependencies']
+          end
+
+          def url(name, version)
+            info = api_version_data(name, version)
+            "#{source}#{info[name].first['file']}"
+          end
+
+        private
+
+          # convert organization/modulename to organization-modulename
+          def normalize_dependencies(data)
+            return nil if data.nil?
+            # convert organization/modulename to organization-modulename
+            data.keys.each do |m|
+              if m =~ %r{.*/.*}
+                data[normalize_name(m)] = data[m]
+                data.delete(m)
+              end
+            end
+            data
+          end
+
+          # get and cache the API data for a specific module with all its versions and dependencies
+          def api_data(module_name)
+            return @api_data[module_name] if @api_data
+            # call API and cache data
+            @api_data = normalize_dependencies(api_call(module_name))
+            if @api_data.nil?
+              raise Error, "Unable to find module '#{name}' on #{source}"
+            end
+            @api_data[module_name]
+          end
+
+          # get and cache the API data for a specific module and version
+          def api_version_data(module_name, version)
+            # if we already got all the versions, find in cached data
+            return @api_data[module_name].detect{|x| x['version'] == version.to_s} if @api_data
+            # otherwise call the api for this version if not cached already
+            @api_version_data[version] = normalize_dependencies(api_call(name, version)) if @api_version_data[version].nil?
+            @api_version_data[version]
+          end
+
+          def api_call(module_name, version=nil)
+            url = source.uri.clone
+            url.path += "#{'/' if url.path.empty? or url.path[-1] != '/'}api/v1/releases.json"
+            url.query = "module=#{module_name.sub('-','/')}" # v1 API expects "organization/module"
+            url.query += "&version=#{version}" unless version.nil?
+            debug { "Querying Forge API for module #{name}#{" and version #{version}" unless version.nil?}: #{url}" }
+
+            begin
+              data = open(url) {|f| f.read}
+              JSON.parse(data)
+            rescue OpenURI::HTTPError => e
+              case e.io.status[0].to_i
+              when 404,410
+                nil
+              else
+                raise e, "Error requesting #{url}: #{e.to_s}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/forge/repo_v3.rb

@@ -0,0 +1,62 @@
+require 'librarian/puppet/source/forge/repo'
+require 'puppet_forge'
+require 'librarian/puppet/version'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class RepoV3 < Librarian::Puppet::Source::Forge::Repo
+
+          PuppetForge.user_agent = "librarian-puppet/#{Librarian::Puppet::VERSION}"
+
+          def initialize(source, name)
+            PuppetForge.host = source.uri.clone
+            super(source, name)
+          end
+
+          def get_versions
+            get_module.releases.select{|r| r.deleted_at.nil?}.map{|r| r.version}
+          end
+
+          def dependencies(version)
+            array = get_release(version).metadata[:dependencies].map{|d| [d[:name], d[:version_requirement]]}
+            Hash[*array.flatten(1)]
+          end
+
+          def url(name, version)
+            if name == "#{get_module().owner.username}/#{get_module().name}"
+              release = get_release(version)
+            else
+              # should never get here as we use one repo object for each module (to be changed in the future)
+              debug { "Looking up url for #{name}@#{version}" }
+              release = PuppetForge::V3::Release.find("#{name}-#{version}")
+            end
+            "#{source}#{release.file_uri}"
+          end
+
+        private
+
+          def get_module
+            begin
+              @module ||= PuppetForge::V3::Module.find(name)
+            rescue Faraday::ResourceNotFound => e
+              raise(Error, "Unable to find module '#{name}' on #{source}")
+            end
+            @module
+          end
+
+          def get_release(version)
+            release = get_module.releases.find{|r| r.version == version.to_s}
+            if release.nil?
+              versions = get_module.releases.map{|r| r.version}
+              raise Error, "Unable to find version '#{version}' for module '#{name}' on #{source} amongst #{versions}"
+            end
+            release
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/git.rb

@@ -0,0 +1,74 @@
+require 'librarian/source/git'
+require 'librarian/puppet/source/local'
+
+module Librarian
+  module Source
+    class Git
+      class Repository
+        def hash_from(remote, reference)
+          branch_names = remote_branch_names[remote]
+          if branch_names.include?(reference)
+            reference = "#{remote}/#{reference}"
+          end
+
+          command = %W(rev-parse #{reference}^{commit} --quiet)
+          run!(command, :chdir => true).strip
+        end
+      end
+    end
+  end
+
+  module Puppet
+    module Source
+      class Git < Librarian::Source::Git
+        include Local
+        include Librarian::Puppet::Util
+
+        def cache!
+          return vendor_checkout! if vendor_cached?
+
+          if environment.local?
+            raise Error, "Could not find a local copy of #{uri}#{" at #{sha}" unless sha.nil?}."
+          end
+
+          begin
+            super
+          rescue Librarian::Posix::CommandFailure => e
+            raise Error, "Could not checkout #{uri}#{" at #{sha}" unless sha.nil?}: #{e}"
+          end
+
+          cache_in_vendor(repository.path) if environment.vendor?
+        end
+
+        private
+
+        def vendor_tar
+          environment.vendor_source.join("#{sha}.tar")
+        end
+
+        def vendor_tgz
+          environment.vendor_source.join("#{sha}.tar.gz")
+        end
+
+        def vendor_cached?
+          vendor_tgz.exist?
+        end
+
+        def vendor_checkout!
+          repository.path.rmtree if repository.path.exist?
+          repository.path.mkpath
+
+          Librarian::Posix.run!(%W{tar xzf #{vendor_tgz}}, :chdir => repository.path.to_s)
+
+          repository_cached!
+        end
+
+        def cache_in_vendor(tmp_path)
+          Librarian::Posix.run!(%W{git archive -o #{vendor_tar} #{sha}}, :chdir => tmp_path.to_s)
+          Librarian::Posix.run!(%W{gzip #{vendor_tar}}, :chdir => tmp_path.to_s)
+        end
+
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/githubtarball.rb

@@ -0,0 +1,130 @@
+require 'uri'
+require 'librarian/puppet/util'
+require 'librarian/puppet/source/githubtarball/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class GitHubTarball
+        include Librarian::Puppet::Util
+
+        class << self
+          LOCK_NAME = 'GITHUBTARBALL'
+
+          def lock_name
+            LOCK_NAME
+          end
+
+          def from_lock_options(environment, options)
+            new(environment, options[:remote], options.reject { |k, v| k == :remote })
+          end
+
+          def from_spec_args(environment, uri, options)
+            recognised_options = []
+            unrecognised_options = options.keys - recognised_options
+            unless unrecognised_options.empty?
+              raise Error, "unrecognised options: #{unrecognised_options.join(", ")}"
+            end
+
+            new(environment, uri, options)
+          end
+        end
+
+        attr_accessor :environment
+        private :environment=
+        attr_reader :uri
+
+        def initialize(environment, uri, options = {})
+          self.environment = environment
+          @uri = URI::parse(uri)
+          @cache_path = nil
+        end
+
+        def to_s
+          clean_uri(uri).to_s
+        end
+
+        def ==(other)
+          other &&
+          self.class == other.class &&
+          self.uri == other.uri
+        end
+
+        alias :eql? :==
+
+        def hash
+          self.to_s.hash
+        end
+
+        def to_spec_args
+          [clean_uri(uri).to_s, {}]
+        end
+
+        def to_lock_options
+          {:remote => clean_uri(uri).to_s}
+        end
+
+        def pinned?
+          false
+        end
+
+        def unpin!
+        end
+
+        def install!(manifest)
+          manifest.source == self or raise ArgumentError
+
+          debug { "Installing #{manifest}" }
+
+          name = manifest.name
+          version = manifest.version
+          install_path = install_path(name)
+          repo = repo(name)
+
+          repo.install_version! version, install_path
+        end
+
+        def manifest(name, version, dependencies)
+          manifest = Manifest.new(self, name)
+          manifest.version = version
+          manifest.dependencies = dependencies
+          manifest
+        end
+
+        def cache_path
+          @cache_path ||= begin
+            environment.cache_path.join("source/puppet/githubtarball/#{uri.host}#{uri.path}")
+          end
+        end
+
+        def install_path(name)
+          environment.install_path.join(module_name(name))
+        end
+
+        def fetch_version(name, version_uri)
+          versions = repo(name).versions
+          if versions.include? version_uri
+            version_uri
+          else
+            versions.first
+          end
+        end
+
+        def fetch_dependencies(name, version, version_uri)
+          {}
+        end
+
+        def manifests(name)
+          repo(name).manifests
+        end
+
+      private
+
+        def repo(name)
+          @repo ||= {}
+          @repo[name] ||= Repo.new(self, name)
+        end
+      end
+    end
+  end
+end