librarian-puppet-maestrodev 0.9.7

data/lib/librarian/puppet/source.rb

@@ -0,0 +1,4 @@
+require 'librarian/puppet/source/path'
+require 'librarian/puppet/source/git'
+require 'librarian/puppet/source/forge'
+require 'librarian/puppet/source/githubtarball'

data/lib/librarian/puppet/source/forge.rb

@@ -0,0 +1,279 @@
+require 'uri'
+require 'net/http'
+require 'json'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class Repo
+
+          attr_accessor :source, :name
+          private :source=, :name=
+
+          def initialize(source, name)
+            self.source = source
+            self.name = name
+          end
+
+          def versions
+            data = api_call("#{name}.json")
+            if data.nil?
+              raise Error, "Unable to find module '#{name}' on #{source}"
+            end
+
+            data['releases'].map { |r| r['version'] }.sort.reverse
+          end
+
+          def dependencies(version)
+            data = api_call("api/v1/releases.json?module=#{name}&version=#{version}")
+            data[name].first['dependencies']
+          end
+
+          def manifests
+            versions.map do |version|
+              Manifest.new(source, name, version)
+            end
+          end
+
+          def install_version!(version, install_path)
+            if environment.local? && !vendored?(name, version)
+              raise Error, "Could not find a local copy of #{name} at #{version}."
+            end
+
+            if environment.vendor?
+              vendor_cache(name, version) unless vendored?(name, version)
+            end
+
+            cache_version_unpacked! version
+
+            if install_path.exist?
+              install_path.rmtree
+            end
+
+            unpacked_path = version_unpacked_cache_path(version).join(name.split('/').last)
+
+            unless unpacked_path.exist?
+              raise Error, "#{unpacked_path} does not exist, something went wrong. Try removing it manually"
+            else
+              FileUtils.cp_r(unpacked_path, install_path)
+            end
+
+          end
+
+          def environment
+            source.environment
+          end
+
+          def cache_path
+            @cache_path ||= source.cache_path.join(name)
+          end
+
+          def version_unpacked_cache_path(version)
+            cache_path.join('version').join(hexdigest(version.to_s))
+          end
+
+          def hexdigest(value)
+            Digest::MD5.hexdigest(value)
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            # The puppet module command is only available from puppet versions >= 2.7.13
+            #
+            # Specifying the version in the gemspec would force people to upgrade puppet while it's still usable for git
+            # So we do some more clever checking
+            #
+            # Executing older versions or via puppet-module tool gives an exit status = 0 .
+            #
+            check_puppet_module_options
+
+            path.mkpath
+
+            target = vendored?(name, version) ? vendored_path(name, version) : name
+
+
+            command = "puppet module install --version #{version} --target-dir '#{path}' --modulepath '#{path}' --ignore-dependencies '#{target}'"
+            output = `#{command}`
+
+            # Check for bad exit code
+            unless $? == 0
+              # Rollback the directory if the puppet module had an error
+              path.unlink
+              raise Error, "Error executing puppet module install:\n#{command}\nError:\n#{output}"
+            end
+
+          end
+
+          def check_puppet_module_options
+            min_version    = Gem::Version.create('2.7.13')
+            puppet_version = Gem::Version.create(`puppet --version`.strip)
+
+            if puppet_version < min_version
+              raise Error, "To get modules from the forge, we use the puppet faces module command. For this you need at least puppet version 2.7.13 and you have #{puppet_version}"
+            end
+          end
+
+          def vendored?(name, version)
+            vendored_path(name, version).exist?
+          end
+
+          def vendored_path(name, version)
+            environment.vendor_cache.join("#{name.sub("/", "-")}-#{version}.tar.gz")
+          end
+
+          def vendor_cache(name, version)
+            File.open(vendored_path(name, version).to_s, 'w') do |f|
+              download(name, version) do |data|
+                f << data
+              end
+            end
+          end
+
+          def download(name, version, &block)
+            data = api_call("api/v1/releases.json?module=#{name}&version=#{version}")
+
+            info = data[name].detect {|h| h['version'] == version.to_s }
+
+            stream(info['file'], &block)
+          end
+
+          def stream(file, &block)
+            Net::HTTP.get_response(URI.parse("#{source}#{file}")) do |res|
+              res.code
+
+              res.read_body(&block)
+            end
+          end
+
+        private
+
+          def api_call(path)
+            base_url = source.to_s
+            resp = Net::HTTP.get_response(URI.parse("#{base_url}/#{path}"))
+            if resp.code.to_i != 200
+              nil
+            else
+              data = resp.body
+              JSON.parse(data)
+            end
+          end
+        end
+
+        class << self
+          LOCK_NAME = 'FORGE'
+
+          def lock_name
+            LOCK_NAME
+          end
+
+          def from_lock_options(environment, options)
+            new(environment, options[:remote], options.reject { |k, v| k == :remote })
+          end
+
+          def from_spec_args(environment, uri, options)
+            recognised_options = []
+            unrecognised_options = options.keys - recognised_options
+            unless unrecognised_options.empty?
+              raise Error, "unrecognised options: #{unrecognised_options.join(", ")}"
+            end
+
+            new(environment, uri, options)
+          end
+        end
+
+        attr_accessor :environment
+        private :environment=
+        attr_reader :uri
+
+        def initialize(environment, uri, options = {})
+          self.environment = environment
+          @uri = uri
+          @cache_path = nil
+        end
+
+        def to_s
+          uri
+        end
+
+        def ==(other)
+          other &&
+          self.class == other.class &&
+          self.uri == other.uri
+        end
+
+        def to_spec_args
+          [uri, {}]
+        end
+
+        def to_lock_options
+          {:remote => uri}
+        end
+
+        def pinned?
+          false
+        end
+
+        def unpin!
+        end
+
+        def install!(manifest)
+          manifest.source == self or raise ArgumentError
+
+          name = manifest.name
+          version = manifest.version
+          install_path = install_path(name)
+          repo = repo(name)
+
+          repo.install_version! version, install_path
+        end
+
+        def manifest(name, version, dependencies)
+          manifest = Manifest.new(self, name)
+          manifest.version = version
+          manifest.dependencies = dependencies
+          manifest
+        end
+
+        def cache_path
+          @cache_path ||= begin
+            dir = Digest::MD5.hexdigest(uri)
+            environment.cache_path.join("source/puppet/forge/#{dir}")
+          end
+        end
+
+        def install_path(name)
+          environment.install_path.join(name.split('/').last)
+        end
+
+        def fetch_version(name, version_uri)
+          versions = repo(name).versions
+          if versions.include? version_uri
+            version_uri
+          else
+            versions.first
+          end
+        end
+
+        def fetch_dependencies(name, version, version_uri)
+          repo(name).dependencies(version).map do |k, v|
+            Dependency.new(k, v, nil)
+          end
+        end
+
+        def manifests(name)
+          repo(name).manifests
+        end
+
+      private
+
+        def repo(name)
+          @repo ||= {}
+          @repo[name] ||= Repo.new(self, name)
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/git.rb

@@ -0,0 +1,114 @@
+require 'librarian/source/git'
+require 'librarian/puppet/source/local'
+
+module Librarian
+  module Source
+    class Git
+      class Repository
+        def hash_from(remote, reference)
+          branch_names = remote_branch_names[remote]
+          if branch_names.include?(reference)
+            reference = "#{remote}/#{reference}"
+          end
+
+          command = %W(rev-parse #{reference}^{commit} --quiet)
+          run!(command, :chdir => true).strip
+        end
+
+        # Naming this method 'version' causes an exception to be raised.
+        def module_version
+          return '0.0.1' unless modulefile?
+
+          metadata  = ::Puppet::ModuleTool::Metadata.new
+          ::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
+
+          metadata.version
+        end
+
+        def dependencies
+          return {} unless modulefile?
+
+          metadata = ::Puppet::ModuleTool::Metadata.new
+
+          ::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
+
+          metadata.dependencies.inject({}) do |h, dependency|
+            name = dependency.instance_variable_get(:@full_module_name)
+            version = dependency.instance_variable_get(:@version_requirement)
+            h.update(name => version)
+          end
+        end
+
+        def modulefile
+          File.join(path, 'Modulefile')
+        end
+
+        def modulefile?
+          File.exists?(modulefile)
+        end
+      end
+    end
+  end
+
+  module Puppet
+    module Source
+      class Git < Librarian::Source::Git
+        include Local
+
+        def cache!
+          return vendor_checkout! if vendor_cached?
+
+          if environment.local?
+            raise Error, "Could not find a local copy of #{uri} at #{sha}."
+          end
+
+          super
+
+          cache_in_vendor(repository.path) if environment.vendor?
+        end
+
+        def vendor_tgz
+          environment.vendor_source + "#{sha}.tar.gz"
+        end
+
+        def vendor_cached?
+          vendor_tgz.exist?
+        end
+
+        def vendor_checkout!
+          repository.path.rmtree if repository.path.exist?
+          repository.path.mkpath
+
+          Dir.chdir(repository.path.to_s) do
+            %x{tar xzf #{vendor_tgz}}
+          end
+
+          repository_cached!
+        end
+
+        def cache_in_vendor(tmp_path)
+          Dir.chdir(tmp_path.to_s) do
+            %x{git archive #{sha} | gzip > #{vendor_tgz}}
+          end
+        end
+
+        def fetch_version(name, extra)
+          cache!
+          found_path = found_path(name)
+          repository.module_version
+        end
+
+        def fetch_dependencies(name, version, extra)
+          repository.dependencies.map do |k, v|
+            Dependency.new(k, v, forge_source)
+          end
+        end
+
+        def forge_source
+          Forge.from_lock_options(environment, :remote=>"http://forge.puppetlabs.com")
+        end
+
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/githubtarball.rb

@@ -0,0 +1,234 @@
+require 'uri'
+require 'net/https'
+require 'json'
+
+module Librarian
+  module Puppet
+    module Source
+      class GitHubTarball
+        class Repo
+
+          attr_accessor :source, :name
+          private :source=, :name=
+
+          def initialize(source, name)
+            self.source = source
+            self.name = name
+          end
+
+          def versions
+            data = api_call("/repos/#{source.uri}/tags")
+            if data.nil?
+              raise Error, "Unable to find module '#{source.uri}' on https://github.com"
+            end
+
+            all_versions = data.map { |r| r['name'] }.sort.reverse
+            all_versions.reject { |version| version =~ /^v/ }.compact
+          end
+
+          def manifests
+            versions.map do |version|
+              Manifest.new(source, name, version)
+            end
+          end
+
+          def install_version!(version, install_path)
+            if environment.local? && !vendored?(source.uri, version)
+              raise Error, "Could not find a local copy of #{source.uri} at #{version}."
+            end
+
+            vendor_cache(source.uri, version) unless vendored?(source.uri, version)
+
+            cache_version_unpacked! version
+
+            if install_path.exist?
+              install_path.rmtree
+            end
+
+            unpacked_path = version_unpacked_cache_path(version).children.first
+            FileUtils.cp_r(unpacked_path, install_path)
+          end
+
+          def environment
+            source.environment
+          end
+
+          def cache_path
+            @cache_path ||= source.cache_path.join(name)
+          end
+
+          def version_unpacked_cache_path(version)
+            cache_path.join('version').join(hexdigest(version.to_s))
+          end
+
+          def hexdigest(value)
+            Digest::MD5.hexdigest(value)
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            path.mkpath
+
+            target = vendored?(source.uri, version) ? vendored_path(source.uri, version) : name
+
+            `tar xzf #{target} -C #{path}`
+          end
+
+          def vendored?(name, version)
+            vendored_path(name, version).exist?
+          end
+
+          def vendored_path(name, version)
+            environment.vendor_cache.mkpath
+            environment.vendor_cache.join("#{name.sub("/", "-")}-#{version}.tar.gz")
+          end
+
+          def vendor_cache(name, version)
+            clean_up_old_cached_versions(name)
+
+            url = "https://api.github.com/repos/#{name}/tarball/#{version}"
+            url << "?access_token=#{ENV['GITHUB_API_TOKEN']}" if ENV['GITHUB_API_TOKEN']
+            `curl #{url} -o #{vendored_path(name, version).to_s} -L 2>&1`
+          end
+
+          def clean_up_old_cached_versions(name)
+            Dir["#{environment.vendor_cache}/#{name.sub('/', '-')}*.tar.gz"].each do |old_version|
+              FileUtils.rm old_version
+            end
+          end
+
+        private
+
+          def api_call(path)
+            url = "https://api.github.com#{path}"
+            url << "?access_token=#{ENV['GITHUB_API_TOKEN']}" if ENV['GITHUB_API_TOKEN']
+            uri = URI.parse(url)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = true
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            request = Net::HTTP::Get.new(uri.request_uri)
+            resp = http.request(request)
+            if resp.code.to_i != 200
+              nil
+            else
+              data = resp.body
+              JSON.parse(data)
+            end
+          end
+        end
+
+        class << self
+          LOCK_NAME = 'GITHUBTARBALL'
+
+          def lock_name
+            LOCK_NAME
+          end
+
+          def from_lock_options(environment, options)
+            new(environment, options[:remote], options.reject { |k, v| k == :remote })
+          end
+
+          def from_spec_args(environment, uri, options)
+            recognised_options = []
+            unrecognised_options = options.keys - recognised_options
+            unless unrecognised_options.empty?
+              raise Error, "unrecognised options: #{unrecognised_options.join(", ")}"
+            end
+
+            new(environment, uri, options)
+          end
+        end
+
+        attr_accessor :environment
+        private :environment=
+        attr_reader :uri
+
+        def initialize(environment, uri, options = {})
+          self.environment = environment
+          @uri = uri
+          @cache_path = nil
+        end
+
+        def to_s
+          uri
+        end
+
+        def ==(other)
+          other &&
+          self.class == other.class &&
+          self.uri == other.uri
+        end
+
+        def to_spec_args
+          [uri, {}]
+        end
+
+        def to_lock_options
+          {:remote => uri}
+        end
+
+        def pinned?
+          false
+        end
+
+        def unpin!
+        end
+
+        def install!(manifest)
+          manifest.source == self or raise ArgumentError
+
+          name = manifest.name
+          version = manifest.version
+          install_path = install_path(name)
+          repo = repo(name)
+
+          repo.install_version! version, install_path
+        end
+
+        def manifest(name, version, dependencies)
+          manifest = Manifest.new(self, name)
+          manifest.version = version
+          manifest.dependencies = dependencies
+          manifest
+        end
+
+        def cache_path
+          @cache_path ||= begin
+            dir = Digest::MD5.hexdigest(uri)
+            environment.cache_path.join("source/puppet/githubtarball/#{dir}")
+          end
+        end
+
+        def install_path(name)
+          environment.install_path.join(name.split('/').last)
+        end
+
+        def fetch_version(name, version_uri)
+          versions = repo(name).versions
+          if versions.include? version_uri
+            version_uri
+          else
+            versions.first
+          end
+        end
+
+        def fetch_dependencies(name, version, version_uri)
+          {}
+        end
+
+        def manifests(name)
+          repo(name).manifests
+        end
+
+      private
+
+        def repo(name)
+          @repo ||= {}
+          @repo[name] ||= Repo.new(self, name)
+        end
+      end
+    end
+  end
+end