anthill-librarian-puppet 3.0.0

data/lib/librarian/puppet/source/githubtarball/repo.rb

@@ -0,0 +1,172 @@
+require 'uri'
+require 'net/https'
+require 'open-uri'
+require 'json'
+
+require 'librarian/puppet/version'
+require 'librarian/puppet/source/repo'
+
+module Librarian
+  module Puppet
+    module Source
+      class GitHubTarball
+        class Repo < Librarian::Puppet::Source::Repo
+          include Librarian::Puppet::Util
+
+          TOKEN_KEY = 'GITHUB_API_TOKEN'
+
+          def versions
+            return @versions if @versions
+            data = api_call("/repos/#{source.uri}/tags")
+            if data.nil?
+              raise Error, "Unable to find module '#{source.uri}' on https://github.com"
+            end
+
+            all_versions = data.map { |r| r['name'].gsub(/^v/, '') }.sort.reverse
+
+            all_versions.delete_if do |version|
+              version !~ /\A\d+\.\d+(\.\d+.*)?\z/
+            end
+
+            @versions = all_versions.compact
+            debug { "  Module #{name} found versions: #{@versions.join(", ")}" }
+            @versions
+          end
+
+          def manifests
+            versions.map do |version|
+              Manifest.new(source, name, version)
+            end
+          end
+
+          def install_version!(version, install_path)
+            if environment.local? && !vendored?(vendored_name, version)
+              raise Error, "Could not find a local copy of #{source.uri} at #{version}."
+            end
+
+            vendor_cache(source.uri.to_s, version) unless vendored?(vendored_name, version)
+
+            cache_version_unpacked! version
+
+            if install_path.exist? && rsync? != true
+              install_path.rmtree
+            end
+
+            unpacked_path = version_unpacked_cache_path(version).children.first
+            cp_r(unpacked_path, install_path)
+          end
+
+          def cache_version_unpacked!(version)
+            path = version_unpacked_cache_path(version)
+            return if path.directory?
+
+            path.mkpath
+
+            target = vendored?(vendored_name, version) ? vendored_path(vendored_name, version) : name
+
+            Librarian::Posix.run!(%W{tar xzf #{target} -C #{path}})
+          end
+
+          def vendor_cache(name, version)
+            clean_up_old_cached_versions(vendored_name(name))
+
+            url = "https://api.github.com/repos/#{name}/tarball/#{version}"
+            add_api_token_to_url(url)
+
+            environment.vendor!
+            File.open(vendored_path(vendored_name(name), version).to_s, 'wb') do |f|
+              begin
+                debug { "Downloading <#{url}> to <#{f.path}>" }
+                open(url,
+                  "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}") do |res|
+                  while buffer = res.read(8192)
+                    f.write(buffer)
+                  end
+                end
+              rescue OpenURI::HTTPError => e
+                raise e, "Error requesting <#{url}>: #{e.to_s}"
+              end
+            end
+          end
+
+          def clean_up_old_cached_versions(name)
+            Dir["#{environment.vendor_cache}/#{name}*.tar.gz"].each do |old_version|
+              FileUtils.rm old_version
+            end
+          end
+
+          def token_key_value
+            ENV[TOKEN_KEY]
+          end
+
+          def token_key_nil?
+            token_key_value.nil? || token_key_value.empty?
+          end
+
+          def add_api_token_to_url url
+            if token_key_nil?
+              debug { "#{TOKEN_KEY} environment value is empty or missing" }
+            elsif url.include? "?"
+              url << "&access_token=#{ENV[TOKEN_KEY]}"
+            else
+              url << "?access_token=#{ENV[TOKEN_KEY]}"
+            end
+            url
+          end
+
+        private
+
+          def api_call(path)
+            tags = []
+            url = "https://api.github.com#{path}?page=1&per_page=100"
+            while true do
+              debug { "  Module #{name} getting tags at: #{url}" }
+              add_api_token_to_url(url)
+              response = http_get(url, :headers => {
+                "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}"
+              })
+
+              code, data = response.code.to_i, response.body
+
+              if code == 200
+                tags.concat JSON.parse(data)
+              else
+                begin
+                  message = JSON.parse(data)['message']
+                  if code == 403 && message && message.include?('API rate limit exceeded')
+                    raise Error, message + " -- increase limit by authenticating via #{TOKEN_KEY}=your-token"
+                  elsif message
+                    raise Error, "Error fetching #{url}: [#{code}] #{message}"
+                  end
+                rescue JSON::ParserError
+                  # response does not return json
+                end
+                raise Error, "Error fetching #{url}: [#{code}] #{response.body}"
+              end
+
+              # next page
+              break if response["link"].nil?
+              next_link = response["link"].split(",").select{|l| l.match /rel=.*next.*/}
+              break if next_link.empty?
+              url = next_link.first.match(/<(.*)>/)[1]
+            end
+            return tags
+          end
+
+          def http_get(url, options)
+            uri = URI.parse(url)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = true
+            request = Net::HTTP::Get.new(uri.request_uri)
+            options[:headers].each { |k, v| request.add_field k, v }
+            http.request(request)
+          end
+
+          def vendored_name(name = source.uri.to_s)
+            name.sub('/','-')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/local.rb

@@ -0,0 +1,176 @@
+require 'librarian/puppet/util'
+
+module Librarian
+  module Puppet
+    module Source
+      module Local
+        include Librarian::Puppet::Util
+
+        def install!(manifest)
+          manifest.source == self or raise ArgumentError
+
+          debug { "Installing #{manifest}" }
+
+          name, version = manifest.name, manifest.version
+          found_path = found_path(name)
+          raise Error, "Path for #{name} doesn't contain a puppet module" if found_path.nil?
+
+          unless name.include? '/' or name.include? '-'
+            warn { "Invalid module name '#{name}', you should qualify it with 'ORGANIZATION-#{name}' for resolution to work correctly" }
+          end
+
+          install_path = environment.install_path.join(module_name(name))
+          if install_path.exist? && rsync? != true
+            debug { "Deleting #{relative_path_to(install_path)}" }
+            install_path.rmtree
+          end
+
+          install_perform_step_copy!(found_path, install_path)
+        end
+
+        def fetch_version(name, extra)
+          cache!
+          found_path = found_path(name)
+          module_version
+        end
+
+        def fetch_dependencies(name, version, extra)
+          dependencies = Set.new
+
+          if specfile?
+            spec = environment.dsl(Pathname(specfile))
+            dependencies.merge spec.dependencies
+          end
+
+          parsed_metadata['dependencies'].each do |d|
+            gem_requirement = Librarian::Dependency::Requirement.new(d['version_requirement']).to_gem_requirement
+            new_dependency = Dependency.new(d['name'], gem_requirement, forge_source)
+            dependencies << new_dependency
+          end
+
+          dependencies
+        end
+
+        def forge_source
+          Forge.default
+        end
+
+        private
+
+        # Naming this method 'version' causes an exception to be raised.
+        def module_version
+          if parsed_metadata['version']
+            parsed_metadata['version']
+          else
+            warn { "Module #{to_s} does not have version, defaulting to 0.0.1" }
+            '0.0.1'
+          end
+        end
+
+        def require_puppet
+          begin
+            require 'puppet'
+            require 'puppet/module_tool'
+          rescue LoadError
+            $stderr.puts <<-EOF
+          Unable to load puppet, the puppet gem is required for :git and :path source.
+          Install it with: gem install puppet
+            EOF
+            exit 1
+          end
+          true
+        end
+
+        def evaluate_modulefile(modulefile)
+          @@require_puppet ||= require_puppet
+
+          metadata = ::Puppet::ModuleTool::Metadata.new
+
+          # Puppet 4 does not have the class
+          unless defined? ::Puppet::ModuleTool::ModulefileReader
+            warn { "Can't parse Modulefile in Puppet >= 4.0 and you are using #{Librarian::Puppet::puppet_version}. Ignoring dependencies in #{modulefile}" }
+            return metadata
+          end
+
+          begin
+            ::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
+            raise SyntaxError, "Missing version" unless metadata.version
+          rescue ArgumentError, SyntaxError => error
+            warn { "Unable to parse #{modulefile}, ignoring: #{error}" }
+            if metadata.respond_to? :version=
+              metadata.version = '0.0.1' # puppet < 3.6
+            else
+              metadata.update({'version' => '0.0.1'}) # puppet >= 3.6
+            end
+          end
+          metadata
+        end
+
+        def parsed_metadata
+          if @metadata.nil?
+            @metadata = if metadata?
+              begin
+                JSON.parse(File.read(metadata))
+              rescue JSON::ParserError => e
+                raise Error, "Unable to parse json file #{metadata}: #{e}"
+              end
+            elsif modulefile?
+              # translate Modulefile to metadata.json
+              evaluated = evaluate_modulefile(modulefile)
+              {
+                'version' => evaluated.version,
+                'dependencies' => evaluated.dependencies.map do |dependency|
+                  {
+                    'name' => dependency.instance_variable_get(:@full_module_name),
+                    'version_requirement' => dependency.instance_variable_get(:@version_requirement)
+                  }
+                end
+              }
+            else
+              {}
+            end
+            @metadata['dependencies'] ||= []
+          end
+          @metadata
+        end
+
+        def modulefile
+          File.join(filesystem_path, 'Modulefile')
+        end
+
+        def modulefile?
+          File.exists?(modulefile)
+        end
+
+        def metadata
+          File.join(filesystem_path, 'metadata.json')
+        end
+
+        def metadata?
+          File.exists?(metadata)
+        end
+
+        def specfile
+          File.join(filesystem_path, environment.specfile_name)
+        end
+
+        def specfile?
+          File.exists?(specfile)
+        end
+
+        def install_perform_step_copy!(found_path, install_path)
+          debug { "Copying #{relative_path_to(found_path)} to #{relative_path_to(install_path)}" }
+          cp_r(found_path, install_path)
+        end
+
+        def manifest?(name, path)
+          return true if path.join('manifests').exist?
+          return true if path.join('lib').join('puppet').exist?
+          return true if path.join('lib').join('facter').exist?
+          debug { "Could not find manifests, lib/puppet or lib/facter under #{path}, maybe it is not a puppet module" }
+          true
+        end
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/path.rb

@@ -0,0 +1,12 @@
+require 'librarian/source/path'
+require 'librarian/puppet/source/local'
+
+module Librarian
+  module Puppet
+    module Source
+      class Path < Librarian::Source::Path
+        include Local
+      end
+    end
+  end
+end

data/lib/librarian/puppet/source/repo.rb

@@ -0,0 +1,38 @@
+# parent class for githubtarball and forge source Repos
+module Librarian
+  module Puppet
+    module Source
+      class Repo
+
+        attr_accessor :source, :name
+        private :source=, :name=
+
+        def initialize(source, name)
+          self.source = source
+          self.name = name
+        end
+
+        def environment
+          source.environment
+        end
+
+        def cache_path
+          @cache_path ||= source.cache_path.join(name)
+        end
+
+        def version_unpacked_cache_path(version)
+          cache_path.join(version.to_s)
+        end
+
+        def vendored?(name, version)
+          vendored_path(name, version).exist?
+        end
+
+        def vendored_path(name, version)
+          environment.vendor_cache.join("#{name}-#{version}.tar.gz")
+        end
+
+      end
+    end
+  end
+end

data/lib/librarian/puppet/templates/Puppetfile

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+#^syntax detection
+
+forge "https://forgeapi.puppetlabs.com"
+
+# use dependencies defined in metadata.json
+metadata
+
+# use dependencies defined in Modulefile
+# modulefile
+
+# A module from the Puppet Forge
+# mod 'puppetlabs-stdlib'
+
+# A module from git
+# mod 'puppetlabs-ntp',
+#   :git => 'git://github.com/puppetlabs/puppetlabs-ntp.git'
+
+# A module from a git branch/tag
+# mod 'puppetlabs-apt',
+#   :git => 'https://github.com/puppetlabs/puppetlabs-apt.git',
+#   :ref => '1.4.x'
+
+# A module from Github pre-packaged tarball
+# mod 'puppetlabs-apache', '0.6.0', :github_tarball => 'puppetlabs/puppetlabs-apache'

data/lib/librarian/puppet/util.rb

@@ -0,0 +1,78 @@
+require 'rsync'
+
+module Librarian
+  module Puppet
+
+    module Util
+
+      def debug(*args, &block)
+        environment.logger.debug(*args, &block)
+      end
+      def info(*args, &block)
+        environment.logger.info(*args, &block)
+      end
+      def warn(*args, &block)
+        environment.logger.warn(*args, &block)
+      end
+
+      def rsync?
+          environment.config_db.local['rsync'] == 'true'
+      end
+
+      # workaround Issue #173 FileUtils.cp_r will fail if there is a symlink that points to a missing file
+      # or when the symlink is copied before the target file when preserve is true
+      # see also https://tickets.opscode.com/browse/CHEF-833
+      #
+      # If the rsync configuration parameter is set, use rsync instead of FileUtils
+      def cp_r(src, dest)
+        if rsync?
+          if Gem.win_platform?
+            src_clean = "#{src}".gsub(/^([a-z])\:/i,'/cygdrive/\1')
+            dest_clean = "#{dest}".gsub(/^([a-z])\:/i,'/cygdrive/\1')
+          else
+            src_clean = src
+            dest_clean = dest
+          end
+          debug { "Copying #{src_clean}/ to #{dest_clean}/ with rsync -avz --delete" }
+          result = Rsync.run(File.join(src_clean, "/"), File.join(dest_clean, "/"), ['-avz', '--delete'])
+          if result.success?
+            debug { "Rsync from #{src_clean}/ to #{dest_clean}/ successfull" }
+          else
+            msg = "Failed to rsync from #{src_clean}/ to #{dest_clean}/: " + result.error
+            raise Error, msg
+          end
+        else
+          begin
+            FileUtils.cp_r(src, dest, :preserve => true)
+          rescue Errno::ENOENT, Errno::EACCES
+            debug { "Failed to copy from #{src} to #{dest} preserving file types, trying again without preserving them" }
+            FileUtils.rm_rf(dest)
+            FileUtils.cp_r(src, dest)
+          end
+        end
+      end
+
+      # Remove user and password from a URI object
+      def clean_uri(uri)
+        new_uri = uri.clone
+        new_uri.user = nil
+        new_uri.password = nil
+        new_uri
+      end
+
+      # normalize module name to use organization-module instead of organization/module
+      def normalize_name(name)
+        name.sub('/','-')
+      end
+
+      # get the module name from organization-module
+      def module_name(name)
+        # module name can't have dashes, so let's assume it is everything after the last dash
+        name.rpartition('-').last
+      end
+
+      # deprecated
+      alias :organization_name :module_name
+    end
+  end
+end