pdqtest 1.4.1 → 1.9.9beta2

data/lib/pdqtest/skeleton.rb

@@ -4,19 +4,41 @@ require 'pdqtest/puppet'
 require 'pdqtest/version'
 require 'pdqtest/util'
 require 'pdqtest/upgrade'
+require 'pdqtest/pdqtest1x'
+require 'pdqtest/pdk'
+require 'erb'
 
 module PDQTest
   module Skeleton
-    FIXTURES        = '.fixtures.yml'
-    SPEC_DIR        = 'spec'
-    ACCEPTANCE_DIR  = File.join(SPEC_DIR, 'acceptance')
-    CLASSES_DIR     = File.join(SPEC_DIR, 'classes')
-    SKELETON_DIR    = 'skeleton'
-    EXAMPLES_DIR    = 'examples'
-    GEMFILE         = 'Gemfile'
-    HIERA_DIR       =  File.join(SPEC_DIR, 'fixtures', 'hieradata')
-    HIERA_YAML      = 'hiera.yaml'
-    HIERA_TEST      = 'test.yaml'
+
+    TEMP_PDK_MODULE  = "x"
+    FIXTURES         = '.fixtures.yml'
+    SPEC_DIR         = 'spec'
+    ACCEPTANCE_DIR   = File.join(SPEC_DIR, 'acceptance')
+    SKELETON_DIR     = 'skeleton'
+    EXAMPLES_DIR     = 'examples'
+    HIERA_DIR        =  File.join(SPEC_DIR, 'fixtures', 'hieradata')
+    HIERA_YAML       = 'hiera.yaml'
+    HIERA_TEST       = 'test.yaml'
+    PDK_FILES        = [
+        "spec/spec_helper.rb",
+        "spec/default_facts.yml",
+        ".pdkignore",
+        "Gemfile",
+        "Rakefile",
+        ".gitignore",
+    ]
+
+    # PDK adds custom metadata fields which we can ONLY get by creating a new
+    # module. We already do this so stash the details here when we have them
+    @@pdk_metadata = {}
+
+    # Every time we `pdqtest upgrade`, update .sync.yml (merges)
+    SYNC_YML_CONTENT = {
+        ".travis.yml" => {
+            "unmanaged": true
+        }
+    }
 
 
     def self.should_replace_file(target, skeleton)
@@ -26,11 +48,16 @@ module PDQTest
       target_hash != skeleton_hash
     end
 
+    def self.install_skeletons
+      FileUtils.cp_r(Util.resource_path(File.join(SKELETON_DIR) + "/."), ".")
+    end
+
+
     def self.install_skeleton(target_file, skeleton, replace=true)
-      skeleton_file = Util::resource_path(File.join(SKELETON_DIR, skeleton))
+      skeleton_file = Util.resource_path(File.join(SKELETON_DIR, skeleton))
       install = false
       if File.exists?(target_file)
-        if replace and should_replace_file(target_file, skeleton_file)
+        if replace && should_replace_file(target_file, skeleton_file)
           install = true
         end
       else
@@ -51,55 +78,51 @@ module PDQTest
       end
     end
 
-    def self.install_gemfile
+    def self.install_gemfile_project
       install_skeleton(GEMFILE, GEMFILE)
 
       # upgrade the gemfile to *this* version of pdqtest + puppet-strings
       Upgrade.upgrade()
     end
 
-    def self.init
-
-      # move .fixtures.yml out of the way
-      if File.exists?(FIXTURES)
-        File.delete(FIXTURES)
-      end
-
-      # make directory structure for testcases
+    def self.directory_structure
       FileUtils.mkdir_p(ACCEPTANCE_DIR)
-      FileUtils.mkdir_p(CLASSES_DIR)
       FileUtils.mkdir_p(EXAMPLES_DIR)
       FileUtils.mkdir_p(HIERA_DIR)
+    end
 
+    def self.init
+      directory_structure
 
-      # skeleton files if required
-      install_skeleton('Rakefile', 'Rakefile')
-      install_skeleton(File.join('spec', 'spec_helper.rb'), 'spec_helper.rb')
-      install_skeleton('.gitignore', 'dot_gitignore')
-      install_skeleton('.rspec', 'dot_rspec')
-      install_skeleton(File.join(SPEC_DIR, 'fixtures', HIERA_YAML), HIERA_YAML)
-      install_skeleton(File.join(HIERA_DIR, HIERA_TEST), HIERA_TEST)
-
-      install_acceptance()
-      install_gemfile()
-      install_integrations()
+      install_pdk_skeletons
+      install_skeletons
+      install_acceptance
+      Upgrade.upgrade()
 
-      # Make sure there is a Gemfile and we are in it
+      # the very _last_ thing we do is enable PDK in metadata. Once this switch
+      # is set, we never touch the files in PDK_FILES again
+      PDQTest::Pdk.enable_pdk(@@pdk_metadata)
     end
 
-    def self.install_integrations()
+    # on upgrade, do a more limited skeleton copy - just our own integration
+    # points
+    def self.upgrade
       install_skeleton('Makefile', 'Makefile')
+      install_skeleton('make.ps1', 'make.ps1')
       install_skeleton('bitbucket-pipelines.yml', 'bitbucket-pipelines.yml')
-      install_skeleton('.travis.yml', 'dot_travis.yml')
+      install_skeleton('.travis.yml', '.travis.yml')
+      Pdk.amend_sync_yml(SYNC_YML_CONTENT)
     end
 
     def self.install_acceptance(example_file ="init.pp")
+      directory_structure
+
       example_name = File.basename(example_file).gsub(/\.pp$/, '')
       install_template("#{EXAMPLES_DIR}/#{File.basename(example_file)}",'examples_init.pp.erb', {})
 
-      install_skeleton(File.join('spec', 'acceptance', "#{example_name}.bats"), 'init.bats', false)
-      install_skeleton(File.join('spec', 'acceptance', "#{example_name}__before.bats"), 'init__before.bats', false)
-      install_skeleton(File.join('spec', 'acceptance', "#{example_name}__setup.sh"), 'init__setup.sh', false)
+      install_skeleton(File.join('spec', 'acceptance', "#{example_name}.bats"), '../acceptance/init.bats', false)
+      install_skeleton(File.join('spec', 'acceptance', "#{example_name}__before.bats"), '../acceptance/init__before.bats', false)
+      install_skeleton(File.join('spec', 'acceptance', "#{example_name}__setup.sh"), '../acceptance/init__setup.sh', false)
     end
 
     # Scan the examples directory and create a set of acceptance tests. If a
@@ -122,5 +145,62 @@ module PDQTest
         install_acceptance(e)
       }
     end
+
+
+    def self.install_pdk_skeletons
+
+      if ! PDQTest::Pdk.is_pdk_enabled
+        $logger.info "Doing one-time upgrade to PDK - Generating fresh set of files..."
+        project_dir = File.expand_path Dir.pwd
+        Dir.mktmpdir do |tmpdir|
+          Dir.chdir(tmpdir) do
+            status = PDQTest::Pdk.run("new module #{TEMP_PDK_MODULE} --skip-interview")
+
+            if status
+              # snag generated metadata now we are in the temporary module dir
+              Dir.chdir TEMP_PDK_MODULE do
+                @@pdk_metadata = PDQTest::Puppet.module_metadata
+              end
+
+              PDK_FILES.each do |pdk_file|
+                upstream_file = File.join(tmpdir, TEMP_PDK_MODULE, pdk_file)
+
+                # check if we are trying to install a file from PDQTest or have
+                # some random/customised file in place
+                Dir.chdir project_dir do
+                  if PDQTest1x.was_pdqtest_file(pdk_file)
+                    if ! File.exists?(pdk_file) || PDQTest1x.is_pdqtest_file(pdk_file)
+                      # overwrite missing or PDQTest 1x files
+                      install = true
+                    else
+                      raise(<<~END)
+                        Detected an unknown/customised file at
+                          #{pdk_file}
+                        Please see the PDQTest 1x->2x upgrade guide at
+                        https://github.com/declarativesystems/pdqtest/blob/master/doc/upgrading.md
+    
+                        If your sure you don't want this file any more, move it out
+                        of the way and re-run the previous command
+                      END
+                    end
+                  else
+                    install = true
+                  end
+
+                  if install
+                    $logger.info("Detected PDQTest 1.x file at #{pdk_file} (will upgrade to PDK)")
+                    FileUtils.cp(upstream_file, pdk_file)
+                  end
+                end
+              end
+            else
+              raise("error running PDK - unable to init")
+            end
+          end
+        end
+      else
+        $logger.debug "PDK already enabled, no skeletons needed"
+      end
+    end
   end
 end

data/lib/pdqtest/upgrade.rb

@@ -2,7 +2,7 @@ require 'pdqtest/skeleton'
 
 module PDQTest
   module Upgrade
-    GEMFILE             = 'Gemfile'
+    GEMFILE             = 'Gemfile.project'
     GEM_REGEXP          = /gem ('|")([-\w]+)('|").*$/
     GEM_ATTRIB_REGEXP   = /^\s*:\w+/
 
@@ -15,10 +15,6 @@ module PDQTest
         'line'  => "gem 'puppet-strings', :git => 'https://github.com/puppetlabs/puppet-strings'",
         'added' => false,
       },
-      'puppet'          => {
-        'line'  => "gem 'puppet', '#{PDQTest::PUPPET_VERSION}'",
-        'added' => false,
-      }
     }
 
 
@@ -27,50 +23,54 @@ module PDQTest
       t_file = File.open("#{GEMFILE}.tmp","w")
       updating_gem = false
 
-      if File.exists?(GEMFILE)
-        File.open(GEMFILE, 'r') do |f|
-          f.each_line{ |line|
-            if line =~ GEM_REGEXP
-              # a gem stanza
-              processing_gem = $2
-              if GEMS.keys.include?(processing_gem)
-                # fixup one of our monitored gems as needed, mark
-                # this as being a gem that is being updated so
-                # that we can kill any multi-line attributes
-                t_file.puts GEMS[processing_gem]['line']
-                updating_gem = true
-                GEMS[processing_gem]['added'] = true
-              else
-                # a gem we don't care about - write it out as-is
-                t_file.puts line
-                updating_gem = false
-              end
-            elsif updating_gem and line =~ GEM_ATTRIB_REGEXP
-              # do nothing - remove the multi-line attributes
+      # Step 1 - enable gem
+
+      if ! File.exists?(GEMFILE)
+        FileUtils.touch(GEMFILE)
+      end
+      File.open(GEMFILE, 'r') do |f|
+        f.each_line{ |line|
+          if line =~ GEM_REGEXP
+            # a gem stanza
+            processing_gem = $2
+            if GEMS.keys.include?(processing_gem)
+              # fixup one of our monitored gems as needed, mark
+              # this as being a gem that is being updated so
+              # that we can kill any multi-line attributes
+              t_file.puts GEMS[processing_gem]['line']
+              updating_gem = true
+              GEMS[processing_gem]['added'] = true
             else
-              # anything else... (esp comments)
+              # a gem we don't care about - write it out as-is
               t_file.puts line
+              updating_gem = false
             end
-          }
-        end
-
-        # the code above will only UPGRADE existing gem lines, but if this is our
-        # first run, there will be nothing to upgrade, so loop through the GEMS
-        # for any that are not already added and append them
-        GEMS.each { |name, opts|
-          if ! opts['added']
-            t_file.puts opts['line']
+          elsif updating_gem and line =~ GEM_ATTRIB_REGEXP
+            # do nothing - remove the multi-line attributes
+          else
+            # anything else... (esp comments)
+            t_file.puts line
           end
         }
+      end
+
+      # the code above will only UPGRADE existing gem lines, but if this is our
+      # first run, there will be nothing to upgrade, so loop through the GEMS
+      # for any that are not already added and append them
+      GEMS.each { |name, opts|
+        if ! opts['added']
+          t_file.puts opts['line']
+        end
+      }
 
-        t_file.close
-        FileUtils.mv(t_file.path, GEMFILE)
+      t_file.close
 
-        # miscellanious file updates
-        PDQTest::Skeleton::install_integrations()
-      else
-        Escort::Logger.error.error "Gemfile not found in #{Dir.pwd}"
-      end
+      # Must do copy->delete on windows or we get permanent file not found
+      # error...
+      FileUtils.cp(t_file.path, GEMFILE)
+      FileUtils.rm(t_file.path)
+
+      PDQTest::Skeleton.upgrade
     end
 
   end

data/lib/pdqtest/util.rb

@@ -1,8 +1,9 @@
 module PDQTest
   module Util
+    ENV='export TERM=xterm LC_ALL=C PATH=/usr/local/bats/bin:/opt/puppetlabs/puppet/bin:$PATH;'
 
     def self.resource_path(resource)
-      File.join(File.dirname(File.expand_path(__FILE__)), "../../res/#{resource}")
+      joinp(File.dirname(File.expand_path(__FILE__)), "../../res/#{resource}")
     end
 
     def self.app_dir
@@ -10,7 +11,86 @@ module PDQTest
     end
 
     def self.app_dir_expanded
-      File.join(Dir.home, app_dir)
+      joinp(Dir.home, app_dir)
+    end
+
+    def self.host_platform
+      Gem.win_platform? ? :windows : :linux
+    end
+
+    def self.is_windows
+      host_platform == :windows
+    end
+
+    def self.shell
+      # cmd.exe is basically broken under docker, use powershell
+      is_windows ? "powershell" : "bash"
+    end
+
+    # need to wrap commands with shell to gain access to
+    # shell functions like `cd` etc
+    def self.wrap_cmd(cmd)
+      if cmd =~ /^\s*$/ || cmd == "bash" || cmd == "powershell"
+        raise "Missing command to wrap!"
+      end
+
+      if is_windows
+        wrapped = [shell, "-command", "#{cmd} ; exit $LastExitCode"]
+      else
+        wrapped = [shell, "-c", "#{ENV} #{cmd}"]
+      end
+
+      wrapped
+    end
+
+    # File.join joins paths with `/` _always_ so we must create our own
+    # function to join paths correctly for windows since using `/` in docker
+    # is not gonna work
+    def self.joinp(*args)
+      File.join(args).gsub(File::SEPARATOR,
+                                   File::ALT_SEPARATOR || File::SEPARATOR)
+    end
+
+    # 3x" --> 1x" seems only way to escape quotes to keep cmd.exe happy. also
+    # need to use double quotes for all args or they get eaten
+    def self.rm(f)
+      is_windows ? "if (test-path '#{f}'){ Remove-Item '#{f}' -Recurse -Force}" : "rm -rf #{f}"
+    end
+
+    # @param link the symlink file
+    # @param target the real file
+    def self.mk_link(link, target)
+      $logger.debug "symlink: #{link} <==> #{target}"
+      if Util.is_windows
+        Emoji.emoji_message(
+            :shame,
+            "symlinks not supported by ruby/puppet on windows doing COPY instead")
+        # broken until ruby/puppet fixed cmd = "#{rm(link)} ; cmd /C mklink /D '#{link}' '#{target}'"
+
+        # for regular files use `copy-time`, for trees use `robocopy` - since
+        # files are in the container we have no idea what is a file or not so
+        # just guess based on presence of filename extension
+        if link =~ /\.[\w]+$/
+          cmd = "copy-item '#{target}' -destination '#{link}'"
+        else
+          cmd = "robocopy '#{target}' '#{link}' /NFL /NDL /NJH /NJS /nc /ns /np /MIR /XD .git fixtures"
+        end
+      else
+        cmd = "#{rm(link)} && mkdir -p #{File.dirname(link)} && ln -s #{target} #{link}"
+      end
+
+      cmd
+    end
+
+    def self.volumes2binds(volumes)
+      # {test_dir => {pwd => 'rw'} + VOLUMES
+      # ...to...
+      # "pwd:test_dir:rw",
+      volumes.map { |container_dir, host_mapping|
+        host_mapping.map { |dir, mode|
+          "#{dir}:#{container_dir}:#{mode}"
+        }.first
+      }
     end
   end
 end

data/lib/pdqtest/version.rb

@@ -1,4 +1,3 @@
 module PDQTest
-  VERSION = "1.4.1"
-  PUPPET_VERSION = "5.5.6"
+  VERSION = "1.9.9beta2"
 end

data/pdqtest.gemspec

@@ -7,10 +7,10 @@ Gem::Specification.new do |spec|
   spec.name          = "pdqtest"
   spec.version       = PDQTest::VERSION
   spec.authors       = ["Geoff Williams"]
-  spec.email         = ["geoff.williams@puppetlabs.com"]
+  spec.email         = ["geoff@declarativesystems.com"]
 
   spec.summary       = %q{Quick and simple integration tests run inside of a docker container}
-  spec.homepage      = "https://github.com/GeoffWilliams/pdqtest"
+  spec.homepage      = "https://github.com/DeclarativeSystems/pdqtest"
   spec.licenses      = 'Apache-2.0'
 
   # file MUST be in git to be fucking readable!!!!!
@@ -21,22 +21,17 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "2.0.1"
+  spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "coveralls", "0.8.21"
-  spec.add_development_dependency "fakefs", "0.14.2"
-  spec.add_development_dependency "puppet", "5.3.5"
+  spec.add_development_dependency "rspec", "3.7.0"
 
   spec.add_runtime_dependency "rake", "12.3.1"
-  spec.add_runtime_dependency "rspec", "3.7.0"
   spec.add_runtime_dependency "thor", "~> 0.19"
   spec.add_runtime_dependency "minitar", "0.6.1"
-  spec.add_runtime_dependency "hiera", "3.4.2"
   spec.add_runtime_dependency "escort", "0.4.0"
   spec.add_runtime_dependency "docker-api", "1.34.2"
-  spec.add_runtime_dependency "puppet-lint", "2.3.5"
-  spec.add_runtime_dependency "puppet-syntax", "2.4.1"
-  spec.add_runtime_dependency "puppetlabs_spec_helper", "2.7.0"
-  spec.add_runtime_dependency "r10k", "2.6.2"
-  spec.add_runtime_dependency "git_refresh", "0.1.1"
-  spec.add_runtime_dependency "puppet_factset", "0.5.0"
+  spec.add_runtime_dependency "r10k", "2.6.4"
+  spec.add_runtime_dependency "logging", "~> 2.2"
+  spec.add_runtime_dependency "deep_merge", "~> 1.2"
+  spec.add_runtime_dependency "json", "<= 2.0.4"
 end

data/res/skeleton/.puppet-lint.rc

@@ -0,0 +1,5 @@
+--fail-on-warnings
+--no-double_quoted_strings-check
+--no-80chars-check
+--no-variable_scope-check
+--no-quoted_booleans-check

data/res/skeleton/Makefile

@@ -1,11 +1,26 @@
-all:
+all: Gemfile.local
 	bundle exec pdqtest all
-	bundle exec puppet strings generate --format=markdown
 
-shell:
+fast: Gemfile.local
+	bundle exec pdqtest fast
+
+shell: Gemfile.local
 	bundle exec pdqtest --keep-container acceptance
 
-logical:
+shellnopuppet: Gemfile.local
+	bundle exec pdqtest shell
+
+logical: Gemfile.local
 	bundle exec pdqtest syntax
-	bundle exec pdqtest lint
 	bundle exec pdqtest rspec
+
+bundle:
+	pdk bundle install
+
+Gemfile.local:
+	echo "[🐌] Creating symlink and running pdk bundle..."
+	ln -s Gemfile.project Gemfile.local
+	make bundle
+
+
+

data/res/skeleton/make.ps1

@@ -0,0 +1,66 @@
+<#
+.SYNOPSIS
+  Run PDQTest targets
+.DESCRIPTION
+  See the instructions at https://github.com/declarativesystems/pdqtest/blob/master/doc/running_tests.md
+.EXAMPLE
+  .\make.ps1 - Run the default testing target
+.EXAMPLE
+  .\make.ps1 XXX - run the XXX target
+.PARAMETER target
+  Test suite to run
+#>
+param(
+    $target = "all"
+)
+
+$gfl = "Gemfile.local"
+$gfp = "Gemfile.project"
+
+# Relink Gemfile.local
+# https://github.com/declarativesystems/pdqtest/blob/master/doc/pdk.md#why-are-the-launch-scripts-essentialhow-does-the-pdqtest-gem-load-itself
+function Install-GemfileLocal {
+  # on windows, symlinks dont work on vagrant fileshares, so just copy the 
+  # file if needed
+  if (Test-Path $gfl) {
+    $gflMd5 = (Get-FileHash -Path $gfl -Algorithm MD5).Hash
+    $gfpMd5 = (Get-FileHash -Path $gfp -Algorithm MD5).Hash
+    if ($gflMd5 -eq $gfpMd5) {
+      # OK - ready to launch
+    } else {
+      write-error "$($gfl) different content to $($gfp)! Move it out the way or move the content to $($gfp)"
+    }
+  } else {
+    write-host "[(-_-)zzz] Copying $($gfp) to $($gfl) and running pdk bundle..."
+    copy $gfp $gfl
+    pdk bundle install
+  }
+}
+
+
+switch ($target) {
+    "all" {
+      Install-GemfileLocal
+	    bundle exec pdqtest all
+    }
+    "fast" {
+      Install-GemfileLocal
+	    bundle exec pdqtest fast
+    }
+    "shell" {
+      Install-GemfileLocal
+	    bundle exec pdqtest --keep-container acceptance
+    }
+    "shellnopuppet" {
+      Install-GemfileLocal
+	    bundle exec pdqtest shell
+    }
+    "logical" {
+      Install-GemfileLocal
+	    bundle exec pdqtest syntax
+	    bundle exec pdqtest rspec
+    }
+    default {
+        Write-Error "No such target: $($target)"
+    }
+}

data/res/skeleton/spec/fixtures/hiera.yaml

@@ -0,0 +1,13 @@
+# Hiera 5 Global configuration file *****test configuration*****
+#
+# Hiera data expected from spec/fixtures/hieradata to give consistent path for
+# testing both rspec and acceptance
+---
+
+version: 5
+
+hierarchy:
+- name: 'Yaml backend'
+  data_hash: yaml_data
+  paths:
+  - '/spec/fixtures/hieradata/test.yaml'

data/res/skeleton/spec/fixtures/hieradata/test.yaml

@@ -0,0 +1,11 @@
+# test.yaml *****for testing*****
+#
+# This allows you to acceptance test the _global_ hierarchy. Usually this is not
+# what you want- for in-module data, just look at `/hiera.yaml` and `/data`
+#
+# There may be edge cases where this is useful - don't forget double check that
+# * `/spec/fixtures/hiera.yaml`
+# * `/spec/fixtures/hieradata/*`
+# Have really made it to git. They are often excluded via `.gitignore`
+---
+message: "this is a test message from pdqtest/hiera"

data/res/templates/examples_init.pp.erb

@@ -1,2 +1,2 @@
-#<%= PDQTest::Puppet::MAGIC_MARKER %>
+#<%= PDQTest::Puppet::setting(:magic_marker) %>
 include <%= PDQTest::Puppet.module_name %>