librarian-puppet-rethinc 3.0.1.1

data/lib/librarian/puppet/dsl.rb

@@ -0,0 +1,123 @@
+require 'librarian/dsl'
+require 'librarian/dsl/target'
+require 'librarian/puppet/source'
+require 'librarian/puppet/dependency'
+
+module Librarian
+  module Puppet
+    class Dsl < Librarian::Dsl
+
+      FORGE_URL = "https://forgeapi.puppetlabs.com"
+
+      dependency :mod
+
+      source :forge => Source::Forge
+      source :git => Source::Git
+      source :path => Source::Path
+      source :github_tarball => Source::GitHubTarball
+
+      def default_specfile
+        Proc.new do
+          forge FORGE_URL
+          metadata
+        end
+      end
+
+      def self.dependency_type
+        Librarian::Puppet::Dependency
+      end
+
+      def post_process_target(target)
+        # save the default forge defined
+        default_forge = target.sources.select {|s| s.is_a? Librarian::Puppet::Source::Forge}.first
+        Librarian::Puppet::Source::Forge.default = default_forge || Librarian::Puppet::Source::Forge.from_lock_options(environment, :remote => FORGE_URL)
+      end
+
+      def receiver(target)
+        Receiver.new(target)
+      end
+
+      def run(specfile = nil, sources = [])
+        specfile, sources = nil, specfile if specfile.kind_of?(Array) && sources.empty?
+
+        Target.new(self).tap do |target|
+          target.precache_sources(sources)
+          debug_named_source_cache("Pre-Cached Sources", target)
+
+          specfile ||= Proc.new if block_given?
+
+          if specfile.kind_of?(Pathname) and !File.exists?(specfile)
+            debug { "Specfile #{specfile} not found, using defaults" } unless specfile.nil?
+            receiver(target).run(specfile, &default_specfile)
+          else
+            receiver(target).run(specfile)
+          end
+
+          post_process_target(target)
+
+          debug_named_source_cache("Post-Cached Sources", target)
+        end.to_spec
+      end
+
+      class Target < Librarian::Dsl::Target
+        def dependency(name, *args)
+          options = args.last.is_a?(Hash) ? args.pop : {}
+          source = source_from_options(options) || @source
+          dep = dependency_type.new(name, args, source, 'Puppetfile')
+          @dependencies << dep
+        end
+      end
+
+      class Receiver < Librarian::Dsl::Receiver
+        attr_reader :specfile, :working_path
+
+        # save the specfile and call librarian
+        def run(specfile = nil)
+          @working_path = specfile.kind_of?(Pathname) ? specfile.parent : Pathname.new(Dir.pwd)
+          @specfile = specfile
+          super
+        end
+
+        # implement the 'modulefile' syntax for Puppetfile
+        def modulefile
+          f = modulefile_path
+          raise Error, "Modulefile file does not exist: #{f}" unless File.exists?(f)
+          File.read(f).lines.each do |line|
+            regexp = /\s*dependency\s+('|")([^'"]+)\1\s*(?:,\s*('|")([^'"]+)\3)?/
+            regexp =~ line && mod($2, $4)
+          end
+        end
+
+        # implement the 'metadata' syntax for Puppetfile
+        def metadata
+          f = working_path.join('metadata.json')
+          unless File.exists?(f)
+            msg = "Metadata file does not exist: #{f}"
+            # try modulefile, in case we don't have a Puppetfile and we are using the default template
+            if File.exists?(modulefile_path)
+              modulefile
+              return
+            else
+              raise Error, msg
+            end
+          end
+          begin
+            json = JSON.parse(File.read(f))
+          rescue JSON::ParserError => e
+            raise Error, "Unable to parse json file #{f}: #{e}"
+          end
+          dependencyList = json['dependencies']
+          dependencyList.each do |d|
+            mod(d['name'], d['version_requirement'])
+          end
+        end
+
+        private
+
+        def modulefile_path
+          working_path.join('Modulefile')
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/environment.rb

@@ -0,0 +1,66 @@
+require "librarian/environment"
+require "librarian/puppet/dsl"
+require "librarian/puppet/source"
+require "librarian/puppet/lockfile"
+
+module Librarian
+  module Puppet
+    class Environment < Librarian::Environment
+
+      def adapter_name
+        "puppet"
+      end
+
+      def lockfile
+        Lockfile.new(self, lockfile_path)
+      end
+
+      def ephemeral_lockfile
+        Lockfile.new(self, nil)
+      end
+
+      def tmp_path
+        part = config_db["tmp"] || ".tmp"
+        project_path.join(part)
+      end
+
+      def install_path
+        part = config_db["path"] || "modules"
+        project_path.join(part)
+      end
+
+      def vendor_path
+        project_path.join('vendor/puppet')
+      end
+
+      def vendor_cache
+        vendor_path.join('cache')
+      end
+
+      def vendor_source
+        vendor_path.join('source')
+      end
+
+      def vendor!
+        vendor_cache.mkpath  unless vendor_cache.exist?
+        vendor_source.mkpath unless vendor_source.exist?
+      end
+
+      def vendor?
+        vendor_path.exist?
+      end
+
+      def local?
+        config_db['mode'] == 'local'
+      end
+
+      def use_v1_api
+        config_db['use-v1-api']
+      end
+
+      def use_short_cache_path
+        config_db['use-short-cache-path']
+      end
+    end
+  end
+end

data/lib/librarian/puppet/extension.rb

@@ -0,0 +1,9 @@
+require 'librarian/puppet/environment'
+require 'librarian/action/base'
+
+module Librarian
+  module Puppet
+    extend self
+    extend Librarian
+  end
+end

data/lib/librarian/puppet/lockfile.rb

@@ -0,0 +1,78 @@
+# Extend Lockfile to normalize module names from acme/mod to acme-mod
+module Librarian
+  module Puppet
+    class Lockfile < Librarian::Lockfile
+
+      # Extend the parser to normalize module names in old .lock files, converting / to -
+      class Parser < Librarian::Lockfile::Parser
+
+        include Librarian::Puppet::Util
+
+        def extract_and_parse_sources(lines)
+          sources = super
+          sources.each do |source|
+            source[:manifests] = Hash[source[:manifests].map do |name,manifest|
+              [normalize_name(name), manifest]
+            end]
+          end
+          sources
+        end
+
+        def extract_and_parse_dependencies(lines, manifests_index)
+          # when looking up in manifests_index normalize the name beforehand
+          class << manifests_index
+            include Librarian::Puppet::Util
+            alias_method :old_lookup, :[]
+            define_method(:[]) { |k| self.old_lookup(normalize_name(k)) }
+          end
+          dependencies = []
+          while lines.first =~ /^ {2}([\w\-\/]+)(?: \((.*)\))?$/
+            lines.shift
+            name, requirement = $1, $2.split(/,\s*/)
+            dependencies << environment.dsl_class.dependency_type.new(name, requirement, manifests_index[name].source, 'lockfile')
+          end
+          dependencies
+        end
+
+        def compile_placeholder_manifests(sources_ast)
+          manifests = {}
+          sources_ast.each do |source_ast|
+            source_type = source_ast[:type]
+            source = source_type.from_lock_options(environment, source_ast[:options])
+            source_ast[:manifests].each do |manifest_name, manifest_ast|
+              manifests[manifest_name] = ManifestPlaceholder.new(
+                  source,
+                  manifest_name,
+                  manifest_ast[:version],
+                  manifest_ast[:dependencies].map do |k, v|
+                    environment.dsl_class.dependency_type.new(k, v, nil, manifest_name)
+                  end
+              )
+            end
+          end
+          manifests
+        end
+
+        def compile(sources_ast)
+          manifests = compile_placeholder_manifests(sources_ast)
+          manifests = manifests.map do |name, manifest|
+            dependencies = manifest.dependencies.map do |d|
+              environment.dsl_class.dependency_type.new(d.name, d.requirement, manifests[d.name].source, name)
+            end
+            real = Manifest.new(manifest.source, manifest.name)
+            real.version = manifest.version
+            real.dependencies = manifest.dependencies
+            real
+          end
+          ManifestSet.sort(manifests)
+        end
+
+      end
+
+      def load(string)
+        Parser.new(environment).parse(string)
+      end
+
+    end
+  end
+end

data/lib/librarian/puppet/resolver.rb

@@ -0,0 +1,21 @@
+require 'librarian/resolver'
+
+module Librarian
+  module Puppet
+    class Resolver < Librarian::Resolver
+
+      class Implementation < Librarian::Resolver::Implementation
+        def sourced_dependency_for(dependency)
+          return dependency if dependency.source
+
+          source = dependency_source_map[dependency.name] || default_source
+          dependency.class.new(dependency.name, dependency.requirement, source, dependency.parent)
+        end
+      end
+
+      def implementation(spec)
+        Implementation.new(self, spec, :cyclic => cyclic)
+      end
+    end
+  end
+end

data/lib/librarian/puppet/rethinc.rb

@@ -0,0 +1 @@
+require 'librarian/puppet'

data/lib/librarian/puppet/source/forge/repo.rb

@@ -0,0 +1,158 @@
+require 'json'
+require 'open-uri'
+require 'librarian/puppet/util'
+require 'librarian/puppet/source/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class Repo < Librarian::Puppet::Source::Repo
+          include Librarian::Puppet::Util
+
+          def versions
+            return @versions if @versions
+            @versions = get_versions
+            if @versions.empty?
+              info { "No versions found for module #{name}" }
+            else
+              debug { "  Module #{name} found versions: #{@versions.join(", ")}" }
+            end
+            @versions
+          end
+
+          # fetch list of versions ordered for newer to older
+          def get_versions
+            # implement in subclasses
+          end
+
+          # return map with dependencies in the form {module_name => version,...}
+          # version: Librarian::Manifest::Version
+          def dependencies(version)
+            # implement in subclasses
+          end
+
+          # return the url for a specific version tarball
+          # version: Librarian::Manifest::Version
+          def url(name, version)
+            # implement in subclasses
+          end
+
+          def manifests
+            versions.map do |version|
+              Manifest.new(source, name, version)
+            end
+          end
+
+          def install_version!(version, install_path)
+            if environment.local? && !vendored?(name, version)
+              raise Error, "Could not find a local copy of #{name} at #{version}."
+            end
+
+            if environment.vendor?
+              vendor_cache(name, version) unless vendored?(name, version)
+            end
+
+            cache_version_unpacked! version
+
+            if install_path.exist? && rsync? != true
+              install_path.rmtree
+            end
+
+            unpacked_path = version_unpacked_cache_path(version).join(module_name(name))
+
+            unless unpacked_path.exist?
+              raise Error, "#{unpacked_path} does not exist, something went wrong. Try removing it manually"
+            else
+              cp_r(unpacked_path, install_path)
+            end
+
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            # The puppet module command is only available from puppet versions >= 2.7.13
+            #
+            # Specifying the version in the gemspec would force people to upgrade puppet while it's still usable for git
+            # So we do some more clever checking
+            #
+            # Executing older versions or via puppet-module tool gives an exit status = 0 .
+            #
+            check_puppet_module_options
+
+            path.mkpath
+
+            target = vendored?(name, version) ? vendored_path(name, version).to_s : name
+
+            # can't pass the default v3 forge url (http://forgeapi.puppetlabs.com)
+            # to clients that use the v1 API (https://forge.puppet.com)
+            # nor the other way around
+            module_repository = source.uri.to_s
+
+            if Forge.client_api_version() > 1 and module_repository =~ %r{^http(s)?://forge\.puppetlabs\.com}
+              module_repository = "https://forgeapi.puppetlabs.com"
+              warn { "Replacing Puppet Forge API URL to use v3 #{module_repository} as required by your client version #{Librarian::Puppet.puppet_version}" }
+            end
+
+            m = module_repository.match(%r{^http(s)?://forge(api)?\.puppetlabs\.com})
+            if Forge.client_api_version() == 1 and m
+              ssl = m[1]
+              # Puppet 2.7 can't handle the 302 returned by the https url, so stick to http
+              if ssl and Librarian::Puppet::puppet_gem_version < Gem::Version.create('3.0.0')
+                warn { "Using plain http as your version of Puppet #{Librarian::Puppet::puppet_gem_version} can't download from forge.puppetlabs.com using https" }
+                ssl = nil
+              end
+              module_repository = "http#{ssl}://forge.puppetlabs.com"
+            end
+
+            command = %W{puppet module install --version #{version} --target-dir}
+            command.push(*[path.to_s, "--module_repository", module_repository, "--modulepath", path.to_s, "--module_working_dir", path.to_s, "--ignore-dependencies", target])
+            debug { "Executing puppet module install for #{name} #{version}: #{command.join(" ").gsub(module_repository, source.to_s)}" }
+
+            begin
+              Librarian::Posix.run!(command)
+            rescue Posix::CommandFailure => e
+              # Rollback the directory if the puppet module had an error
+              begin
+                path.unlink
+              rescue => u
+                debug("Unable to rollback path #{path}: #{u}")
+              end
+              tar = Dir[File.join(path.to_s, "**/*.tar.gz")]
+              msg = ""
+              if e.message =~ /Unexpected EOF in archive/ and !tar.empty?
+                file = tar.first
+                msg = " (looks like an incomplete download of #{file})"
+              end
+              raise Error, "Error executing puppet module install#{msg}. Check that this command succeeds:\n#{command.join(" ")}\nError:\n#{e.message}"
+            end
+
+          end
+
+          def check_puppet_module_options
+            min_version    = Gem::Version.create('2.7.13')
+
+            if Librarian::Puppet.puppet_gem_version < min_version
+              raise Error, "To get modules from the forge, we use the puppet faces module command. For this you need at least puppet version 2.7.13 and you have #{Librarian::Puppet.puppet_version}"
+            end
+          end
+
+          def vendor_cache(name, version)
+            url = url(name, version)
+            path = vendored_path(name, version).to_s
+            debug { "Downloading #{url} into #{path}"}
+            environment.vendor!
+            File.open(path, 'wb') do |f|
+              URI.open(url, 'rb') do |input|
+                f.write(input.read)
+              end
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/forge/repo_v1.rb

@@ -0,0 +1,92 @@
+require 'json'
+require 'open-uri'
+require 'librarian/puppet/source/forge/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class RepoV1 < Librarian::Puppet::Source::Forge::Repo
+
+          def initialize(source, name)
+            super(source, name)
+            # API returned data for this module including all versions and dependencies, indexed by module name
+            # from https://forge.puppetlabs.com/api/v1/releases.json?module=#{name}
+            @api_data = nil
+            # API returned data for this module and a specific version, indexed by version
+            # from https://forge.puppetlabs.com/api/v1/releases.json?module=#{name}&version=#{version}
+            @api_version_data = {}
+          end
+
+          def get_versions
+            api_data(name).map { |r| r['version'] }.reverse
+          end
+
+          def dependencies(version)
+            api_version_data(name, version)['dependencies']
+          end
+
+          def url(name, version)
+            info = api_version_data(name, version)
+            "#{source}#{info[name].first['file']}"
+          end
+
+        private
+
+          # convert organization/modulename to organization-modulename
+          def normalize_dependencies(data)
+            return nil if data.nil?
+            # convert organization/modulename to organization-modulename
+            data.keys.each do |m|
+              if m =~ %r{.*/.*}
+                data[normalize_name(m)] = data[m]
+                data.delete(m)
+              end
+            end
+            data
+          end
+
+          # get and cache the API data for a specific module with all its versions and dependencies
+          def api_data(module_name)
+            return @api_data[module_name] if @api_data
+            # call API and cache data
+            @api_data = normalize_dependencies(api_call(module_name))
+            if @api_data.nil?
+              raise Error, "Unable to find module '#{name}' on #{source}"
+            end
+            @api_data[module_name]
+          end
+
+          # get and cache the API data for a specific module and version
+          def api_version_data(module_name, version)
+            # if we already got all the versions, find in cached data
+            return @api_data[module_name].detect{|x| x['version'] == version.to_s} if @api_data
+            # otherwise call the api for this version if not cached already
+            @api_version_data[version] = normalize_dependencies(api_call(name, version)) if @api_version_data[version].nil?
+            @api_version_data[version]
+          end
+
+          def api_call(module_name, version=nil)
+            url = source.uri.clone
+            url.path += "#{'/' if url.path.empty? or url.path[-1] != '/'}api/v1/releases.json"
+            url.query = "module=#{module_name.sub('-','/')}" # v1 API expects "organization/module"
+            url.query += "&version=#{version}" unless version.nil?
+            debug { "Querying Forge API for module #{name}#{" and version #{version}" unless version.nil?}: #{url}" }
+
+            begin
+              data = URI.open(url) {|f| f.read}
+              JSON.parse(data)
+            rescue OpenURI::HTTPError => e
+              case e.io.status[0].to_i
+              when 404,410
+                nil
+              else
+                raise e, "Error requesting #{url}: #{e.to_s}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/forge/repo_v3.rb

@@ -0,0 +1,100 @@
+require 'librarian/puppet/source/forge/repo'
+require 'puppet_forge'
+require 'librarian/puppet/version'
+
+module Librarian
+  module Puppet
+    module Source
+      class Forge
+        class RepoV3 < Librarian::Puppet::Source::Forge::Repo
+
+          PuppetForge.user_agent = "librarian-puppet/#{Librarian::Puppet::VERSION}"
+
+          def initialize(source, name)
+            PuppetForge.host = source.uri.clone
+            super(source, name)
+          end
+
+          def get_versions
+            get_module.releases.select{|r| r.deleted_at.nil?}.map{|r| r.version}
+          end
+
+          def dependencies(version)
+            array = get_release(version).metadata[:dependencies].map{|d| [d[:name], d[:version_requirement]]}
+            Hash[*array.flatten(1)]
+          end
+
+          def url(name, version)
+            if name == "#{get_module().owner.username}/#{get_module().name}"
+              release = get_release(version)
+            else
+              # should never get here as we use one repo object for each module (to be changed in the future)
+              debug { "Looking up url for #{name}@#{version}" }
+              release = PuppetForge::V3::Release.find("#{name}-#{version}")
+            end
+            "#{source}#{release.file_uri}"
+          end
+
+        private
+
+          def get_module
+            begin
+              @module ||= PuppetForge::V3::Module.find(name)
+            rescue Faraday::ResourceNotFound => e
+              raise(Error, "Unable to find module '#{name}' on #{source}")
+            end
+            @module
+          end
+
+          def get_release(version)
+            release = get_module.releases.find{|r| r.version == version.to_s}
+            if release.nil?
+              versions = get_module.releases.map{|r| r.version}
+              raise Error, "Unable to find version '#{version}' for module '#{name}' on #{source} amongst #{versions}"
+            end
+            release
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            path.mkpath
+
+            target = vendored?(name, version) ? vendored_path(name, version).to_s : name
+
+            # can't pass the default v3 forge url (http://forgeapi.puppetlabs.com)
+            # to clients that use the v1 API (https://forge.puppet.com)
+            # nor the other way around
+            module_repository = source.uri.to_s
+
+            if Forge.client_api_version() > 1 and module_repository =~ %r{^http(s)?://forge\.puppetlabs\.com}
+              module_repository = "https://forgeapi.puppetlabs.com"
+              warn { "Replacing Puppet Forge API URL to use v3 #{module_repository} as required by your client version #{Librarian::Puppet.puppet_version}" }
+            end
+
+            tar_dst = environment.tmp_path.join("#{target}-#{version}.tar.gz")
+            dest_dir = path.join(module_name(name)) 
+            tmp_dir = environment.tmp_path.join("forge").to_s
+
+            begin
+              release = get_release(version)
+              release.download(tar_dst)
+              release.verify(tar_dst)
+              PuppetForge::Unpacker.unpack(tar_dst, dest_dir, tmp_dir)
+            rescue => e
+              # Rollback the directory if the puppet module had an error
+              begin
+                path.unlink
+              rescue => u
+                debug("Unable to rollback path #{path}: #{u}")
+              end
+              raise Error, "Error downloading module from forge. \nError:\n#{e.message}"
+            end
+          end
+
+        end
+      end
+    end
+  end
+end