librarian-puppet-rethinc 3.0.1.1

data/lib/librarian/puppet/source/forge.rb

@@ -0,0 +1,180 @@
+require 'uri'
+require 'librarian/puppet/util'
+require 'librarian/puppet/source/forge/repo_v1'
+require 'librarian/puppet/source/forge/repo_v3'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        include Librarian::Puppet::Util
+
+        class << self
+          LOCK_NAME = 'FORGE'
+
+          def default=(source)
+            @@default = source
+          end
+
+          def default
+            @@default
+          end
+
+          def lock_name
+            LOCK_NAME
+          end
+
+          def from_lock_options(environment, options)
+            new(environment, options[:remote], options.reject { |k, v| k == :remote })
+          end
+
+          def from_spec_args(environment, uri, options)
+            recognised_options = []
+            unrecognised_options = options.keys - recognised_options
+            unless unrecognised_options.empty?
+              raise Error, "unrecognised options: #{unrecognised_options.join(", ")}"
+            end
+
+            new(environment, uri, options)
+          end
+
+          def client_api_version()
+            version = 1
+            pe_version = Librarian::Puppet.puppet_version.match(/\(Puppet Enterprise (.+)\)/)
+
+            # Puppet 3.6.0+ uses api v3
+            if Librarian::Puppet::puppet_gem_version >= Gem::Version.create('3.6.0.a')
+              version = 3
+            # Puppet enterprise 3.2.0+ uses api v3
+            elsif pe_version and Gem::Version.create(pe_version[1].strip) >= Gem::Version.create('3.2.0')
+              version = 3
+            end
+            return version
+          end
+
+        end
+
+        attr_accessor :environment
+        private :environment=
+        attr_reader :uri
+
+        def initialize(environment, uri, options = {})
+          self.environment = environment
+
+          if uri =~ %r{^http(s)?://forge\.puppetlabs\.com}
+            uri = "https://forgeapi.puppetlabs.com"
+            warn { "Replacing Puppet Forge API URL to use v3 #{uri}. You should update your Puppetfile" }
+          end
+
+          @uri = URI::parse(uri)
+          @cache_path = nil
+        end
+
+        def to_s
+          clean_uri(uri).to_s
+        end
+
+        def ==(other)
+          other &&
+          self.class == other.class &&
+          self.uri == other.uri
+        end
+
+        alias :eql? :==
+
+        def hash
+          self.to_s.hash
+        end
+
+        def to_spec_args
+          [clean_uri(uri).to_s, {}]
+        end
+
+        def to_lock_options
+          {:remote => uri.to_s}
+        end
+
+        def pinned?
+          false
+        end
+
+        def unpin!
+        end
+
+        def install!(manifest)
+          manifest.source == self or raise ArgumentError
+
+          debug { "Installing #{manifest}" }
+
+          name = manifest.name
+          version = manifest.version
+          install_path = install_path(name)
+          repo = repo(name)
+
+          repo.install_version! version, install_path
+        end
+
+        def manifest(name, version, dependencies)
+          manifest = Manifest.new(self, name)
+          manifest.version = version
+          manifest.dependencies = dependencies
+          manifest
+        end
+
+        def cache_path
+          @cache_path ||= begin
+            if environment.use_short_cache_path
+              # Take only the first 7 digits of the SHA1 checksum of the forge URI
+              # (short Git commit hash approach)
+              dir = Digest::SHA1.hexdigest("#{uri.host}#{uri.path}")[0..6]
+            else
+              dir = "#{uri.host}#{uri.path}".gsub(/[^0-9a-z\-_]/i, '_')
+            end
+            environment.cache_path.join("source/p/f/#{dir}")
+          end
+        end
+
+        def install_path(name)
+          environment.install_path.join(module_name(name))
+        end
+
+        def fetch_version(name, version_uri)
+          versions = repo(name).versions
+          if versions.include? version_uri
+            version_uri
+          else
+            versions.first
+          end
+        end
+
+        def fetch_dependencies(name, version, version_uri)
+          repo(name).dependencies(version).map do |k, v|
+            v = Librarian::Dependency::Requirement.new(v).to_gem_requirement
+            Dependency.new(k, v, nil, name)
+          end
+        end
+
+        def manifests(name)
+          repo(name).manifests
+        end
+
+      private
+
+        def repo(name)
+          @repo ||= {}
+
+          unless @repo[name]
+            # If we are using the official Forge then use API v3, otherwise use the preferred api 
+            # as defined by the CLI option use_v1_api
+            if uri.hostname =~ /\.puppetlabs\.com$/ || !environment.use_v1_api
+              @repo[name] = RepoV3.new(self, name)
+            else
+              @repo[name] = RepoV1.new(self, name)
+            end
+          end
+          @repo[name]
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/git.rb

@@ -0,0 +1,74 @@
+require 'librarian/source/git'
+require 'librarian/puppet/source/local'
+
+module Librarian
+  module Source
+    class Git
+      class Repository
+        def hash_from(remote, reference)
+          branch_names = remote_branch_names[remote]
+          if branch_names.include?(reference)
+            reference = "#{remote}/#{reference}"
+          end
+
+          command = %W(rev-parse #{reference}^{commit} --quiet)
+          run!(command, :chdir => true).strip
+        end
+      end
+    end
+  end
+
+  module Puppet
+    module Source
+      class Git < Librarian::Source::Git
+        include Local
+        include Librarian::Puppet::Util
+
+        def cache!
+          return vendor_checkout! if vendor_cached?
+
+          if environment.local?
+            raise Error, "Could not find a local copy of #{uri}#{" at #{sha}" unless sha.nil?}."
+          end
+
+          begin
+            super
+          rescue Librarian::Posix::CommandFailure => e
+            raise Error, "Could not checkout #{uri}#{" at #{sha}" unless sha.nil?}: #{e}"
+          end
+
+          cache_in_vendor(repository.path) if environment.vendor?
+        end
+
+        private
+
+        def vendor_tar
+          environment.vendor_source.join("#{sha}.tar")
+        end
+
+        def vendor_tgz
+          environment.vendor_source.join("#{sha}.tar.gz")
+        end
+
+        def vendor_cached?
+          vendor_tgz.exist?
+        end
+
+        def vendor_checkout!
+          repository.path.rmtree if repository.path.exist?
+          repository.path.mkpath
+
+          Librarian::Posix.run!(%W{tar xzf #{vendor_tgz}}, :chdir => repository.path.to_s)
+
+          repository_cached!
+        end
+
+        def cache_in_vendor(tmp_path)
+          Librarian::Posix.run!(%W{git archive -o #{vendor_tar} #{sha}}, :chdir => tmp_path.to_s)
+          Librarian::Posix.run!(%W{gzip #{vendor_tar}}, :chdir => tmp_path.to_s)
+        end
+
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/githubtarball/repo.rb

@@ -0,0 +1,172 @@
+require 'uri'
+require 'net/https'
+require 'open-uri'
+require 'json'
+
+require 'librarian/puppet/version'
+require 'librarian/puppet/source/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class GitHubTarball
+        class Repo < Librarian::Puppet::Source::Repo
+          include Librarian::Puppet::Util
+
+          TOKEN_KEY = 'GITHUB_API_TOKEN'
+
+          def versions
+            return @versions if @versions
+            data = api_call("/repos/#{source.uri}/tags")
+            if data.nil?
+              raise Error, "Unable to find module '#{source.uri}' on https://github.com"
+            end
+
+            all_versions = data.map { |r| r['name'].gsub(/^v/, '') }.sort.reverse
+
+            all_versions.delete_if do |version|
+              version !~ /\A\d+\.\d+(\.\d+.*)?\z/
+            end
+
+            @versions = all_versions.compact
+            debug { "  Module #{name} found versions: #{@versions.join(", ")}" }
+            @versions
+          end
+
+          def manifests
+            versions.map do |version|
+              Manifest.new(source, name, version)
+            end
+          end
+
+          def install_version!(version, install_path)
+            if environment.local? && !vendored?(vendored_name, version)
+              raise Error, "Could not find a local copy of #{source.uri} at #{version}."
+            end
+
+            vendor_cache(source.uri.to_s, version) unless vendored?(vendored_name, version)
+
+            cache_version_unpacked! version
+
+            if install_path.exist? && rsync? != true
+              install_path.rmtree
+            end
+
+            unpacked_path = version_unpacked_cache_path(version).children.first
+            cp_r(unpacked_path, install_path)
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            path.mkpath
+
+            target = vendored?(vendored_name, version) ? vendored_path(vendored_name, version) : name
+
+            Librarian::Posix.run!(%W{tar xzf #{target} -C #{path}})
+          end
+
+          def vendor_cache(name, version)
+            clean_up_old_cached_versions(vendored_name(name))
+
+            url = "https://api.github.com/repos/#{name}/tarball/#{version}"
+            add_api_token_to_url(url)
+
+            environment.vendor!
+            File.open(vendored_path(vendored_name(name), version).to_s, 'wb') do |f|
+              begin
+                debug { "Downloading <#{url}> to <#{f.path}>" }
+                URI.open(url,
+                  "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}") do |res|
+                  while buffer = res.read(8192)
+                    f.write(buffer)
+                  end
+                end
+              rescue OpenURI::HTTPError => e
+                raise e, "Error requesting <#{url}>: #{e.to_s}"
+              end
+            end
+          end
+
+          def clean_up_old_cached_versions(name)
+            Dir["#{environment.vendor_cache}/#{name}*.tar.gz"].each do |old_version|
+              FileUtils.rm old_version
+            end
+          end
+
+          def token_key_value
+            ENV[TOKEN_KEY]
+          end
+
+          def token_key_nil?
+            token_key_value.nil? || token_key_value.empty?
+          end
+
+          def add_api_token_to_url url
+            if token_key_nil?
+              debug { "#{TOKEN_KEY} environment value is empty or missing" }
+            elsif url.include? "?"
+              url << "&access_token=#{ENV[TOKEN_KEY]}"
+            else
+              url << "?access_token=#{ENV[TOKEN_KEY]}"
+            end
+            url
+          end
+
+        private
+
+          def api_call(path)
+            tags = []
+            url = "https://api.github.com#{path}?page=1&per_page=100"
+            while true do
+              debug { "  Module #{name} getting tags at: #{url}" }
+              add_api_token_to_url(url)
+              response = http_get(url, :headers => {
+                "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}"
+              })
+
+              code, data = response.code.to_i, response.body
+
+              if code == 200
+                tags.concat JSON.parse(data)
+              else
+                begin
+                  message = JSON.parse(data)['message']
+                  if code == 403 && message && message.include?('API rate limit exceeded')
+                    raise Error, message + " -- increase limit by authenticating via #{TOKEN_KEY}=your-token"
+                  elsif message
+                    raise Error, "Error fetching #{url}: [#{code}] #{message}"
+                  end
+                rescue JSON::ParserError
+                  # response does not return json
+                end
+                raise Error, "Error fetching #{url}: [#{code}] #{response.body}"
+              end
+
+              # next page
+              break if response["link"].nil?
+              next_link = response["link"].split(",").select{|l| l.match /rel=.*next.*/}
+              break if next_link.empty?
+              url = next_link.first.match(/<(.*)>/)[1]
+            end
+            return tags
+          end
+
+          def http_get(url, options)
+            uri = URI.parse(url)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = true
+            request = Net::HTTP::Get.new(uri.request_uri)
+            options[:headers].each { |k, v| request.add_field k, v }
+            http.request(request)
+          end
+
+          def vendored_name(name = source.uri.to_s)
+            name.sub('/','-')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/githubtarball.rb

@@ -0,0 +1,137 @@
+require 'uri'
+require 'librarian/puppet/util'
+require 'librarian/puppet/source/githubtarball/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class GitHubTarball
+        include Librarian::Puppet::Util
+
+        class << self
+          LOCK_NAME = 'GITHUBTARBALL'
+
+          def lock_name
+            LOCK_NAME
+          end
+
+          def from_lock_options(environment, options)
+            new(environment, options[:remote], options.reject { |k, v| k == :remote })
+          end
+
+          def from_spec_args(environment, uri, options)
+            recognised_options = []
+            unrecognised_options = options.keys - recognised_options
+            unless unrecognised_options.empty?
+              raise Error, "unrecognised options: #{unrecognised_options.join(", ")}"
+            end
+
+            new(environment, uri, options)
+          end
+        end
+
+        attr_accessor :environment
+        private :environment=
+        attr_reader :uri
+
+        def initialize(environment, uri, options = {})
+          self.environment = environment
+          @uri = URI::parse(uri)
+          @cache_path = nil
+        end
+
+        def to_s
+          clean_uri(uri).to_s
+        end
+
+        def ==(other)
+          other &&
+          self.class == other.class &&
+          self.uri == other.uri
+        end
+
+        alias :eql? :==
+
+        def hash
+          self.to_s.hash
+        end
+
+        def to_spec_args
+          [clean_uri(uri).to_s, {}]
+        end
+
+        def to_lock_options
+          {:remote => clean_uri(uri).to_s}
+        end
+
+        def pinned?
+          false
+        end
+
+        def unpin!
+        end
+
+        def install!(manifest)
+          manifest.source == self or raise ArgumentError
+
+          debug { "Installing #{manifest}" }
+
+          name = manifest.name
+          version = manifest.version
+          install_path = install_path(name)
+          repo = repo(name)
+
+          repo.install_version! version, install_path
+        end
+
+        def manifest(name, version, dependencies)
+          manifest = Manifest.new(self, name)
+          manifest.version = version
+          manifest.dependencies = dependencies
+          manifest
+        end
+
+        def cache_path
+          @cache_path ||= begin
+            if environment.use_short_cache_path
+              # Take only the first 7 digits of the SHA1 checksum of the forge URI
+              # (short Git commit hash approach)
+              dir = Digest::SHA1.hexdigest("#{uri.host}#{uri.path}")[0..6]
+            else
+              dir = "#{uri.host}#{uri.path}"
+            end
+            environment.cache_path.join("source/p/gh/#{dir}")
+          end
+        end
+
+        def install_path(name)
+          environment.install_path.join(module_name(name))
+        end
+
+        def fetch_version(name, version_uri)
+          versions = repo(name).versions
+          if versions.include? version_uri
+            version_uri
+          else
+            versions.first
+          end
+        end
+
+        def fetch_dependencies(name, version, version_uri)
+          {}
+        end
+
+        def manifests(name)
+          repo(name).manifests
+        end
+
+      private
+
+        def repo(name)
+          @repo ||= {}
+          @repo[name] ||= Repo.new(self, name)
+        end
+      end
+    end
+  end
+end