puppet_fixtures 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0b6246620641ff6d3570c494428f46def8a857befdda5bcca94ada409f363e76
+  data.tar.gz: bafb926914441b2118e2a72cfd3ce2108073576c76f6e7e16dc6edacd1585414
+SHA512:
+  metadata.gz: a8fff928c60d024ffff15ef613e2eafbea0c00c51d64057c820b3b2500793c0b24bd72da3ca6e169e4c45868db34b2cc60db4cca47eec8dc77edc3d0cfc2e46b
+  data.tar.gz: e8a69aee563e805b5f17f24b567b40492cb4760b80b7ee988e466c87fb0d013cba74b167f3319f124b33798414ed13717cc96364188412a59a896f9e89499c80

data/bin/puppet-fixtures

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+require 'optparse'
+
+require 'puppet_fixtures'
+
+COMMANDS = ['clean', 'install', 'show']
+
+command = ARGV[0]
+unless COMMANDS.include?(command)
+  $stderr.puts "Usage: #{$0} #{COMMANDS.join('|')}"
+  exit 1
+end
+
+fixtures = PuppetFixtures::Fixtures.new
+
+case command
+when 'clean'
+  fixtures.clean
+when 'install'
+  fixtures.download
+when 'show'
+  if (path = fixtures.fixture_path)
+    puts "Parsing #{fixtures.fixture_path}"
+
+    if fixtures.symlinks.any?
+      puts
+      puts "Symlinks"
+      fixtures.symlinks.each do |_target, symlink|
+        puts "  #{symlink}"
+      end
+    end
+
+    if fixtures.forge_modules.any?
+      puts
+      puts "Forge modules"
+      fixtures.forge_modules.each do |mod, opts|
+        dir = Pathname.new(opts[:target]).relative_path_from(fixtures.module_target_dir)
+        description = mod
+        description += " #{opts[:ref]}"
+        puts "  #{dir} => #{description}"
+      end
+    end
+
+    if fixtures.repositories.any?
+      puts
+      puts "Repositories"
+      fixtures.repositories.each do |repository, opts|
+        dir = Pathname.new(opts[:target]).relative_path_from(fixtures.module_target_dir)
+        description = ["#{opts[:scm]}+#{repository}", opts[:ref]]
+        description << "branch #{opts[:branch]}" if opts[:branch]
+        puts "  #{dir} => #{description.compact.join(' ')}"
+      end
+    end
+  else
+    $stderr.puts "No fixture file found"
+  end
+end

data/lib/puppet_fixtures/tasks.rb

@@ -0,0 +1,15 @@
+require_relative '../puppet_fixtures'
+
+require 'rake'
+
+namespace :fixtures do
+  desc 'Create the fixtures directory'
+  task :prep do
+    PuppetFixtures.new.download(max_thread_limit: ENV.fetch('MAX_FIXTURE_THREAD_COUNT', 10).to_i)
+  end
+
+  desc 'Clean up the fixtures directory'
+  task :clean do
+    PuppetFixtures.new.clean
+  end
+end

data/lib/puppet_fixtures.rb

@@ -0,0 +1,576 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+require 'yaml'
+
+# PuppetFixtures is a mechanism to download Puppet fixtures.
+#
+# These fixtures can be symlinks, repositories (git or Mercurial) or forge
+# modules.
+module PuppetFixtures
+  # @return [Boolean]
+  #   true if the os is a windows system
+  def self.windows?
+    # Ruby only sets File::ALT_SEPARATOR on Windows and Rubys standard library
+    # uses this to check for Windows
+    !!File::ALT_SEPARATOR
+  end
+
+  class Fixtures
+    attr_reader :source_dir
+
+    def initialize(source_dir: Dir.pwd, max_thread_limit: 10)
+      @source_dir = source_dir
+      @max_thread_limit = max_thread_limit
+    end
+
+    # @return [Hash]
+    #   A hash of all the fixture repositories
+    # @example
+    #   {
+    #     "puppetlabs-stdlib"=>{
+    #       "target"=>"https://gitlab.com/puppetlabs/puppet-stdlib.git",
+    #       "ref"=>nil,
+    #       "branch"=>"main",
+    #       "scm"=>nil,
+    #     }
+    #   }
+    def repositories
+      @repositories ||= fixtures['repositories']
+    end
+
+    # @return [Hash]
+    #    A hash of all the fixture forge modules
+    # @example
+    #   {
+    #     "puppetlabs-stdlib"=>{
+    #       "target"=>"spec/fixtures/modules/stdlib",
+    #       "ref"=>nil,
+    #       "branch"=>nil,
+    #       "scm"=>nil,
+    #       "flags"=>"--module_repository=https://myforge.example.com/",
+    #       "subdir"=>nil,
+    #     }
+    #   }
+    def forge_modules
+      @forge_modules ||= fixtures['forge_modules']
+    end
+
+    # @return [Hash[String, Symlink]]
+    #   A hash of symlinks specified in the fixtures file
+    def symlinks
+      @symlinks ||= fixtures['symlinks']
+    end
+
+    def fixtures
+      @fixtures ||= begin
+                      categories = read_fixtures_file['fixtures']
+
+                      categories['symlinks'] ||= begin
+                                                   metadata = PuppetFixtures::Metadata.new(File.join(source_dir, 'metadata.json'))
+                                                   { metadata.name.split('-').last => source_dir }
+                                                 rescue ArgumentError
+                                                   {}
+                                                 end
+                      categories['forge_modules'] ||= {}
+                      categories['repositories'] ||= {}
+
+                      defaults = { 'target' => module_target_dir }
+
+                      ['symlinks', 'forge_modules', 'repositories'].to_h do |category|
+                        # load defaults from the `.fixtures.yml` `defaults` section
+                        # for the requested category and merge them into my defaults
+                        if (category_defaults = categories.dig('defaults', category))
+                          category_defaults = defaults.merge(category_defaults)
+                        else
+                          category_defaults = defaults
+                        end
+
+                        entries = categories[category].to_h do |fixture, opts|
+                          # convert a simple string fixture to a hash, by
+                          # using the string fixture as the `repo` option of the hash.
+                          if opts.instance_of?(String)
+                            opts = { 'repo' => opts }
+                          end
+                          # there should be a warning or something if it's not a hash...
+                          next unless opts.instance_of?(Hash)
+
+                          # merge our options into the defaults to get the
+                          # final option list
+                          opts = category_defaults.merge(opts)
+
+                          next unless include_repo?(opts['puppet_version'])
+
+                          entry = validate_fixture_hash!(
+                            target: File.join(opts['target'], fixture),
+                            ref: opts['ref'] || opts['tag'],
+                            branch: opts['branch'],
+                            scm: opts.fetch('scm', 'git'),
+                            flags: opts['flags'],
+                            subdir: opts['subdir'],
+                          )
+
+                          case category
+                          when 'forge_modules'
+                            entry.delete(:scm)
+                            entry.delete(:branch)
+                            entry.delete(:subdir)
+                          when 'symlinks'
+                            entry = PuppetFixtures::Symlink.new(link: entry[:target], target: opts['repo'])
+                          end
+
+                          [opts['repo'], entry]
+                        end
+
+                        [category, entries]
+                      end
+                    end
+    end
+
+    # @param [String] remote
+    #   The remote url or namespace/name of the module to download
+    # @param [String] scm
+    #   The SCM to use
+    # @return [Boolean]
+    #   Returns true if the module was downloaded successfully, false otherwise
+    def download_repository(remote, target:, scm:, subdir:, ref:, branch:, flags:)
+      repository = PuppetFixtures::Repository.factory(scm: scm, remote: remote, target: target, branch: branch, ref: ref)
+      repository.download(flags, subdir)
+    end
+
+    # @return [String]
+    #   the spec/fixtures/modules directory in the module root folder
+    def module_target_dir
+      # TODO: relative to source_dir?
+      @module_target_dir ||= File.expand_path(File.join('spec', 'fixtures', 'modules'))
+    end
+
+    # @param [String] remote
+    #   the remote url or namespace/name of the module to download
+    # @return [Boolean]
+    #   returns true if the module was downloaded, false otherwise
+    def download_module(remote, ref:, target:, flags:)
+      if File.directory?(target)
+        if !ref || ref.empty?
+          logger.debug("Module #{target} already up to date")
+          return false
+        end
+
+        begin
+          version = PuppetFixtures::Metadata.new(File.join(target, 'metadata.json')).version
+        rescue ArgumentError
+          logger.warn "Unable to detect module version for #{target}; updating"
+        else
+          if ref == version
+            logger.debug("Module #{target} already up to date (#{ref})")
+            return false
+          else
+            logger.debug("Module #{target} version #{version} != #{ref}; updating")
+          end
+        end
+      end
+
+      command = ['puppet', 'module', 'install']
+      command << '--version' << ref if ref
+      command += flags if flags
+      command += ['--ignore-dependencies', '--force', '--target-dir', module_target_dir, remote]
+
+      unless run_command(command)
+        raise "Failed to install module #{remote} to #{module_target_dir}"
+      end
+
+      true
+    end
+
+    def download
+      logger.debug("Downloading to #{module_target_dir}")
+      FileUtils.mkdir_p(module_target_dir)
+
+      if symlinks.empty?
+        logger.debug('No symlinks to create')
+      else
+        symlinks.each_value do |symlink|
+          logger.info("Creating symlink #{symlink}")
+          symlink.create
+        end
+      end
+
+      queue = Queue.new
+
+      repositories.each do |remote, opts|
+        queue << [:repository, remote, opts]
+      end
+      forge_modules.each do |remote, opts|
+        queue << [:forge, remote, opts]
+      end
+
+      if queue.empty?
+        logger.debug('Nothing to download')
+        return
+      end
+
+      instance = self
+
+      thread_count = [@max_thread_limit, queue.size].min
+      logger.debug("Download queue size: #{queue.size}; using #{thread_count} threads")
+
+      threads = thread_count.times.map do |i|
+        Thread.new do
+          type, remote, opts = queue.pop(true)
+          case type
+          when :repository
+            instance.download_repository(remote, **opts)
+          when :forge
+            instance.download_module(remote, **opts)
+          end
+        end
+      end
+
+      begin
+        threads.map(&:join)
+      rescue Interrupt
+        # pass
+      end
+    end
+
+    def clean
+      repositories.each_value do |opts|
+        target = opts[:target]
+        logger.debug("Removing repository #{target}")
+        FileUtils.rm_rf(target)
+      end
+
+      forge_modules.each_value do |opts|
+        target = opts[:target]
+        logger.debug("Removing forge module #{target}")
+        FileUtils.rm_rf(target)
+      end
+
+      symlinks.each_value do |symlink|
+        logger.debug("Removing symlink #{symlink}")
+        symlink.remove
+      end
+    end
+
+    def fixture_path
+      if ENV['FIXTURES_YML']
+        ENV['FIXTURES_YML']
+      elsif File.exist?('.fixtures.yml')
+        '.fixtures.yml'
+      elsif File.exist?('.fixtures.yaml')
+        '.fixtures.yaml'
+      else
+        nil
+      end
+    end
+
+    private
+
+    def include_repo?(version_range)
+      return true unless version_range
+
+      require 'semantic_puppet'
+
+      puppet_spec = Gem::Specification.find_by_name('puppet')
+      puppet_version = SemanticPuppet::Version.parse(puppet_spec.version.to_s)
+
+      constraint = SemanticPuppet::VersionRange.parse(version_range)
+      constraint.include?(puppet_version)
+    end
+
+    def git_remote_url(target)
+      output, status = Open3.capture2e('git', '--git-dir', File.join(target, '.git'), 'ls-remote', '--get-url', 'origin')
+      status.success? ? output.strip : nil
+    end
+
+    # creates a logger so we can log events with certain levels
+    def logger
+      @logger ||= begin
+                    require 'logger'
+                    Logger.new($stderr, level: ENV['ENABLE_LOGGER'] ? Logger::DEBUG : Logger::INFO)
+                  end
+    end
+
+    def read_fixtures_file
+      fixtures_yaml = fixture_path
+
+      fixtures = nil
+      if fixtures_yaml
+        begin
+          fixtures = YAML.load_file(fixtures_yaml)
+        rescue Errno::ENOENT
+          raise "Fixtures file not found: '#{fixtures_yaml}'"
+        rescue Psych::SyntaxError => e
+          raise "Found malformed YAML in '#{fixtures_yaml}' on line #{e.line} column #{e.column}: #{e.problem}"
+        end
+      end
+      fixtures ||= { 'fixtures' => {} }
+
+      unless fixtures.include?('fixtures')
+        # File is non-empty, but does not specify fixtures
+        raise("No 'fixtures' entries found in '#{fixtures_yaml}'; required")
+      end
+
+      fixtures
+    end
+
+    def validate_fixture_hash!(**hash)
+      if hash[:flags].is_a?(String)
+        require 'shellwords'
+        hash[:flags] = Shellwords.split(hash[:flags])
+      end
+
+      if hash['scm'] == 'git' && hash['ref'].include?('/')
+        # Forward slashes in the ref aren't allowed. And is probably a branch name.
+        raise ArgumentError, "The ref for #{hash['target']} is invalid (Contains a forward slash). If this is a branch name, please use the 'branch' setting instead."
+      end
+
+      hash
+    end
+
+    # @param [Array[String]] command
+    def run_command(command, chdir: nil)
+      logger.debug do
+        require 'shellwords'
+        if chdir
+          "Calling command #{Shellwords.join(command)} in #{chdir}"
+        else
+          "Calling command #{Shellwords.join(command)}"
+        end
+      end
+      if chdir
+        system(*command, chdir: chdir)
+      else
+        system(*command)
+      end
+    end
+  end
+
+  class Metadata
+    def initialize(path)
+      raise ArgumentError unless File.file?(path) && File.readable?(path)
+
+      @metadata = JSON.parse(File.read(path))
+    rescue JSON::ParserError => e
+      raise ArgumentError, "Failed to read module metadata at #{path}: #{e}"
+    end
+
+    # @return [String[1]] The module name
+    def name
+      n = @metadata['name']
+      raise ArgumentError "No module name found" if !n || n.empty?
+
+      n
+    end
+
+    # @return [String[1]] The module version
+    def version
+      v = @metadata['version']
+      raise ArgumentError "No module name found" if !v || v.empty?
+
+      v
+    end
+  end
+
+  class Symlink
+    # @param target [String]
+    #   the target directory
+    # @param link [String]
+    #   the name of the link you wish to create
+    def initialize(target:, link:)
+      @target = target
+      @link = link
+    end
+
+    # Create a junction on Windows or otherwise a symlink
+    # works on windows and linux
+    def create
+      return if File.symlink?(@link)
+
+      if PuppetFixtures.windows?
+        begin
+          require 'win32/dir'
+        rescue LoadError
+        end
+        target = File.join(File.dirname(@link), @target) unless Pathname.new(@target).absolute?
+        if Dir.respond_to?(:create_junction)
+          Dir.create_junction(@link, target)
+        else
+          warn 'win32-dir gem not installed, falling back to executing mklink directly'
+          # TODO: use run_command
+          system("call mklink /J \"#{@link.tr('/', '\\')}\" \"#{target.tr('/', '\\')}\"")
+        end
+      else
+        FileUtils.ln_sf(@target, @link)
+      end
+    end
+
+    def remove
+      FileUtils.rm_f(@link)
+    end
+
+    def to_s
+      # TODO: relative?
+      "#{@link} => #{@target}"
+    end
+  end
+
+  module Repository
+    def self.factory(scm:, remote:, target:, branch:, ref:)
+      cls = case scm
+            when 'git'
+              Repository::Git
+            when 'hg'
+              Repository::Mercurial
+            else
+              raise ArgumentError, "Unfortunately #{scm} is not supported yet"
+            end
+      cls.new(remote: remote, target: target, branch: branch, ref: ref)
+    end
+
+    class Base
+      def initialize(remote:, target:, branch:, ref:)
+        @remote = remote
+        @target = target
+        @ref = ref
+        @branch = branch
+      end
+
+      def download(flags = nil, subdir = nil)
+        can_update = false
+        if File.directory?(@target)
+          if remote_url_changed?
+            warn "Remote for #{@target} has changed, recloning repository"
+            FileUtils.rm_rf(@target)
+          else
+            can_update = true
+          end
+        end
+
+        if can_update
+          update
+        else
+          clone(flags)
+          unless File.exist?(@target)
+            raise "Failed to clone repository #{@remote} into #{@target}"
+          end
+        end
+
+        revision
+        remove_subdirectory(subdir) if subdir
+      end
+
+      protected
+
+      def remove_subdirectory(subdir)
+        Dir.mktmpdir do |tmpdir|
+          FileUtils.mv(Dir.glob(File.join(@target, subdir, "{.[^\.]*,*}")), tmpdir)
+          FileUtils.rm_rf(File.join(@target, subdir))
+          FileUtils.mv(Dir.glob(File.join(tmpdir, "{.[^\.]*,*}")), @target.to_s)
+        end
+      end
+
+      def run_command(command, chdir: nil)
+        # TODO: duplicated
+        logger.debug do
+          require 'shellwords'
+          if chdir
+            "Calling command #{Shellwords.join(command)} in #{chdir}"
+          else
+            "Calling command #{Shellwords.join(command)}"
+          end
+        end
+        if chdir
+          system(*command, chdir: chdir)
+        else
+          system(*command)
+        end
+      end
+
+      def logger
+        # TODO: duplicated
+        @logger ||= begin
+                      require 'logger'
+                      # TODO: progname?
+                      Logger.new($stderr, level: ENV['ENABLE_LOGGER'] ? Logger::DEBUG : Logger::INFO)
+                    end
+      end
+    end
+
+    class Git < Base
+      def clone(flags = nil)
+        command = ['git', 'clone']
+        command.push('--depth', '1') unless @ref
+        command.push('-b', @branch) if @branch
+        command.push(flags) if flags
+        command.push(@remote, @target)
+
+        run_command(command)
+      end
+
+      def update
+        # TODO: should this pull?
+        command = ['git', 'fetch']
+        command.push('--unshallow') if shallow_git_repo?
+
+        run_command(command, chdir: @target)
+      end
+
+      def revision
+        return true unless @ref
+
+        command = ['git', 'reset', '--hard', @ref]
+        result = run_command(command, chdir: @target)
+        raise "Invalid ref #{ref} for #{@target}" unless result
+
+        result
+      end
+
+      def remote_url_changed?(remote = 'origin')
+        remote_url(remote) != @remote
+      end
+
+      private
+
+      def remote_url(remote = 'origin')
+        output, status = Open3.capture2e('git', '--git-dir', File.join(@target, '.git'), 'ls-remote', '--get-url', remote)
+        status.success? ? output.strip : nil
+      end
+
+      def shallow_git_repo?
+        File.file?(File.join(@target, '.git', 'shallow'))
+      end
+    end
+
+    class Mercurial < Base
+      def clone(flags = nil)
+        command = ['hg', 'clone']
+        command.push('-b', @branch) if @branch
+        command.push(flags) if flags
+        command.push(@remote, @target)
+
+        run_command(command)
+      end
+
+      def update
+        run_command(['hg', 'pull'])
+      end
+
+      def revision
+        return true unless @ref
+
+        command = ['hg', 'update', '--clean', '-r', @ref]
+        result = run_command(command, chdir: @target)
+        raise "Invalid ref #{ref} for #{@target}" unless result
+
+        result
+      end
+
+      def remote_url_changed?
+        # Not implemented
+        false
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification
+name: puppet_fixtures
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ewoud Kohl van Wijngaarden
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: |
+  Originally part of puppetlabs_spec_helper, but with a significant
+  refactoring to make it available standalone.
+executables:
+- puppet-fixtures
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/puppet-fixtures
+- lib/puppet_fixtures.rb
+- lib/puppet_fixtures/tasks.rb
+homepage: https://github.com/voxpupuli/puppet_fixtures
+licenses:
+- GPL-2.0-only
+metadata:
+  source_code_uri: https://github.com/voxpupuli/puppet_fixtures
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Set up fixtures for Puppet testing
+test_files: []