onceover 3.3.1 → 3.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16b8e87dd7f42d97a987d5ba6ea7df4882f792ac
-  data.tar.gz: 9654efc7739d62409e488b7f2651914b17e2831c
+  metadata.gz: 8516d34019e0c6c9d8f09fcd8e6f7249dec8b8dd
+  data.tar.gz: b569f4341e7681fe099d090f0f9b245346110145
 SHA512:
-  metadata.gz: ccd6c5e3cd4f9749c6ef58ca8766cef9acfd93597283ab9bb2c67388ab43cd60d22c4bcd02ba45f2add5ba44153e3ffaad8dc8c94266521d3e6d02089bc151d9
-  data.tar.gz: 01d040074b62b0fcd2971be9cfa7d84720fdc1f3028bf68b630e3bbd48cbd2fa5fc045e9d534cb013a8ffb366878b005db632a15b12b1cf7e85a8a025885e804
+  metadata.gz: b77f2b94c07a87304af205b19bb4cf6ad4c7d7c89b67067c47275b5e20dfcb064cb4bf4d1edca03dd337ffaada10dcecd434c086d90a0104e855e64a14c1175f
+  data.tar.gz: 171dcee3922127e988696645a68fc48286ed2a54316d0dbbb86249650b9cda947ab01a01fe12f18a71dab0c8784bcf3c9ce9f3b3530d29f7643fbc953da66ccc

data/.travis.yml

@@ -17,3 +17,4 @@ rvm:
   - 2.3.4
   - 2.4.0
   - 2.4.1
+  - 2.4.2

data/features/cache.feature

@@ -0,0 +1,33 @@
+@cache
+Feature: Create and maintain a .onceover cache
+  Onceover should be able to cache things in the .onceover directory for speed
+  increases and debugging of external modules. This cache should remain
+  up-to-date and should exactly mirror what would be created on the Puppet
+  master.
+
+  Background:
+    Given onceover executable
+
+  Scenario: Creating a cache
+    Given control repo "caching"
+    When I run onceover command "run spec"
+    Then the cache should exist
+    And the cache should contain all controlrepo files
+
+  Scenario: Creating a new file
+    Given control repo "caching"
+    When I create a file "example.txt"
+    And I run onceover command "run spec"
+    Then "example.txt" should be cached correctly
+
+  Scenario: Deleting a file
+    Given control repo "caching"
+    When I delete a file "deleteme.txt"
+    And I run onceover command "run spec"
+    Then "deleteme.txt" should be deleted from the cache
+
+  Scenario: Caching hidden files
+    Given control repo "caching"
+    When I create a file ".hidden/.hiddenfile"
+    And I run onceover command "run spec"
+    Then ".hidden/.hiddenfile" should be cached correctly

data/features/step_definitions/cache.rb

@@ -0,0 +1,38 @@
+Then(/^the cache should exist$/) do
+  cache_dir = File.join(@repo.root_folder,'.onceover')
+  expect(File.directory?(cache_dir)).to be true
+end
+
+Then(/^the cache should contain all controlrepo files/) do
+  # Get all root files
+  puts "Calculating MD5 hashes in repo"
+  repo_digest = Cache_Helper.digest(@repo.root_folder)
+  puts "#{repo_digest.count} MD5 hashes calculated"
+  puts "Calculating MD5 hashes in cache"
+  cache_digest = Cache_Helper.digest(File.join(@repo.root_folder,'.onceover/etc/puppetlabs/code/environments/production/'))
+  puts "#{cache_digest.count} MD5 hashes calculated"
+  expect(cache_digest).to include(repo_digest)
+end
+
+When(/^I (\w+) a file "(.*)"$/) do |action,file|
+  require 'securerandom'
+  actual_file = Pathname.new(File.join(@repo.root_folder,file))
+  case action
+  when "create"
+    FileUtils.mkdir_p(actual_file.dirname)
+    File.write(actual_file,SecureRandom.hex)
+  when "delete"
+    FileUtils.rm(actual_file)
+  end
+end
+
+Then(/^"(.*)" should be cached correctly$/) do |file|
+  original_digest = Cache_Helper.digest(File.join(@repo.root_folder,file))
+  cache_digest    = Cache_Helper.digest(File.join(@repo.root_folder,'.onceover/etc/puppetlabs/code/environments/production/',file))
+  expect(original_digest).to include(cache_digest)
+end
+
+Then(/^"([^"]*)" should be deleted from the cache$/) do |file|
+  deleted_file = Pathname.new(File.join(@repo.root_folder,'.onceover/etc/puppetlabs/code/environments/production/',file))
+  expect(deleted_file.exist?).to be false
+end

data/features/step_definitions/common.rb

@@ -1,4 +1,4 @@
-Given /^onceover executable$/ do
+Given(/^onceover executable$/) do
   @cmd = Command_Helper.new
 end
 
@@ -20,13 +20,13 @@ Given(/^control repo "([^"]*)" without "([^"]*)"$/) do |controlrepo_name, filena
   FileUtils.rm_rf "#{@repo.root_folder}/#{filename}"
 end
 
-When /^I run onceover command "([^"]*)"$/  do |command|
+When(/^I run onceover command "([^"]*)"$/)  do |command|
   @cmd.command = command
   puts @cmd
   @cmd.run
 end
 
-Then /^I see help for commands: "([^"]*)"$/ do |commands|
+Then(/^I see help for commands: "([^"]*)"$/) do |commands|
   # Get chunk of output between COMMANDS and OPTION, there should be help section
   commands_help = @cmd.output[/COMMANDS(.*)OPTIONS/m, 1]
   commands.split(',').each do |command|

data/features/support/cache_helper.rb

@@ -0,0 +1,55 @@
+require 'pathname'
+
+class Cache_Helper
+
+  def cache_exists?
+    File.directory?(dir + '/.onceover')
+  end
+
+  def self.digest(path, opts = {
+    exceptions: ['.','..','.onceover']
+    })
+    if File.directory?(path)
+      # Get the list of files
+      children = Cache_Helper.get_children(path, opts[:exceptions])
+    else
+      children = [File.expand_path(path)]
+    end
+
+    # Calculate hashes
+    hashes = children.map do |child_path|
+      if File.directory? child_path
+        :directory
+      else
+        Digest::MD5.file(child_path)
+      end
+    end
+
+    root = Pathname.new(File.expand_path(path))
+    # Move pathnames back to relative
+    children.map! do |child_path|
+      Pathname.new(child_path).relative_path_from root
+    end
+    Hash[children.zip(hashes)]
+  end
+
+  def self.get_children(dir, exclusions)
+    root_files = []
+    files      = []
+    Dir.chdir(dir) do
+      # Get all root files
+      root_files = Dir.glob('*',File::FNM_DOTMATCH)
+      root_files = root_files - exclusions
+      root_files.each do |file|
+        files << file
+        files << Dir.glob("#{file}/**/*",File::FNM_DOTMATCH)
+      end
+      files.flatten!
+      # Calculate absolue paths
+      files.map! do |file|
+        File.expand_path(file)
+      end
+    end
+    files
+  end
+end

data/lib/onceover/testconfig.rb

@@ -181,27 +181,33 @@ class Onceover
       #
       # If there are more situations like this we can add them to this array as
       # full paths
-      excluded_files = []
-      if ENV['GEM_HOME']
-        logger.debug "Excluding #{ENV['GEM_HOME']} from controlrepo copy"
-        excluded_files << Dir.glob("#{ENV['GEM_HOME']}/**/*")
-        excluded_files.flatten!
+      excluded_dirs = []
+      excluded_dirs << Pathname.new("#{repo.root}/.onceover")
+      excluded_dirs << Pathname.new(ENV['GEM_HOME']) if ENV['GEM_HOME']
+
+      controlrepo_files = get_children_recursive(Pathname.new(repo.root))
+
+      # Exclude the files that should be skipped
+      controlrepo_files.delete_if do |path|
+        parents = [path]
+        path.ascend do |parent|
+          parents << parent
+        end
+        parents.any? { |x| excluded_dirs.include?(x) }
       end
 
-      # Exclude the files we need to
-      controlrepo_files = Dir.glob("#{repo.root}/**/*")
-      files_to_copy     = (controlrepo_files - excluded_files).delete_if { |path| Pathname(path).directory? }
-      folders_to_copy   = (controlrepo_files - excluded_files).keep_if { |path| Pathname(path).directory? }
+      folders_to_copy = controlrepo_files.select { |x| x.directory? }
+      files_to_copy   = controlrepo_files.select { |x| x.file? }
 
       logger.debug "Creating temp dir as a staging directory for copying the controlrepo to #{repo.tempdir}"
       temp_controlrepo = Dir.mktmpdir('controlrepo')
 
       logger.debug "Creating directories under #{temp_controlrepo}"
-      FileUtils.mkdir_p(folders_to_copy.map { |folder| "#{temp_controlrepo}/#{(Pathname(folder).relative_path_from(Pathname(repo.root))).to_s}"})
+      FileUtils.mkdir_p(folders_to_copy.map { |folder| "#{temp_controlrepo}/#{(folder.relative_path_from(Pathname(repo.root))).to_s}"})
 
       logger.debug "Copying files to #{temp_controlrepo}"
       files_to_copy.each do |file|
-        FileUtils.cp(file,"#{temp_controlrepo}/#{(Pathname(file).relative_path_from(Pathname(repo.root))).to_s}")
+        FileUtils.cp(file,"#{temp_controlrepo}/#{(file.relative_path_from(Pathname(repo.root))).to_s}")
       end
 
       # When using puppetfile vs deploy with r10k, we want to respect the :control_branch
@@ -219,10 +225,10 @@ class Onceover
       File.write("#{temp_controlrepo}/Puppetfile", new_puppetfile_contents)
 
 
-      FileUtils.mkdir_p("#{repo.tempdir}/#{repo.environmentpath}/production")
+      FileUtils.mkdir_p("#{repo.tempdir}/#{repo.environmentpath}")
 
       logger.debug "Copying #{temp_controlrepo} to #{repo.tempdir}/#{repo.environmentpath}/production"
-      FileUtils.cp_r(Dir["#{temp_controlrepo}/*"], "#{repo.tempdir}/#{repo.environmentpath}/production")
+      FileUtils.cp_r(temp_controlrepo, "#{repo.tempdir}/#{repo.environmentpath}/production")
       FileUtils.rm_rf(temp_controlrepo)
 
       # Pull the trigger! If it's not already been pulled
@@ -331,5 +337,18 @@ class Onceover
       tests
     end
 
+    private
+
+    def get_children_recursive(pathname)
+      results = []
+      results << pathname
+      pathname.each_child do |child|
+        results << child
+        if child.directory?
+          results << get_children_recursive(child)
+        end
+      end
+      results.flatten
+    end
   end
 end

data/onceover.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "onceover"
-  s.version     = "3.3.1"
+  s.version     = "3.3.2"
   s.authors     = ["Dylan Ratcliffe"]
   s.email       = ["dylan.ratcliffe@puppet.com"]
   s.homepage    = "https://github.com/dylanratcliffe/onceover"

data/spec/fixtures/controlrepos/caching/.gitignore

@@ -0,0 +1 @@
+.onceover

data/spec/fixtures/controlrepos/caching/.travis.yml

@@ -0,0 +1,2 @@
+---
+some: stuff

data/spec/fixtures/controlrepos/caching/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'onceover'
+gem 'beaker', '~> 2.0'
+

data/spec/fixtures/controlrepos/caching/Puppetfile

@@ -0,0 +1,17 @@
+forge "http://forge.puppetlabs.com"
+#
+# I want to download some modules to check if r10k feature in Onceover works correctly.
+#
+
+# Versions should be updated to be the latest at the time you start
+mod "puppetlabs/stdlib",      '4.11.0'
+
+# Modules from Git
+# Examples: https://github.com/puppetlabs/r10k/blob/master/doc/puppetfile.mkd#examples
+mod 'apache',
+ :git    => 'https://github.com/puppetlabs/puppetlabs-apache',
+ :commit => '83401079053dca11d61945bd9beef9ecf7576cbf'
+
+#mod 'apache',
+#  :git    => 'https://github.com/puppetlabs/puppetlabs-apache',
+#  :branch => 'docs_experiment'

data/spec/fixtures/controlrepos/caching/Rakefile

@@ -0,0 +1 @@
+require 'onceover/rake_tasks'

data/spec/fixtures/controlrepos/caching/environment.conf

@@ -0,0 +1,2 @@
+modulepath          = site:modules:$basemodulepath
+config_version      = 'scripts/config_version.sh $environmentpath $environment'

data/spec/fixtures/controlrepos/caching/hieradata/common.yaml

@@ -0,0 +1,2 @@
+---
+message: "This node is using common data"

data/spec/fixtures/controlrepos/caching/hieradata/nodes/example-node.yaml

@@ -0,0 +1 @@
+---

data/spec/fixtures/controlrepos/caching/manifests/site.pp

@@ -0,0 +1,32 @@
+## site.pp ##
+
+# This file (/etc/puppetlabs/puppet/manifests/site.pp) is the main entry point
+# used when an agent connects to a master and asks for an updated configuration.
+#
+# Global objects like filebuckets and resource defaults should go in this file,
+# as should the default node definition. (The default node can be omitted
+# if you use the console and don't define any other nodes in site.pp. See
+# http://docs.puppetlabs.com/guides/language_guide.html#nodes for more on
+# node definitions.)
+
+## Active Configurations ##
+
+# Disable filebucket by default for all File resources:
+#https://docs.puppet.com/pe/2015.3/release_notes.html#filebucket-resource-no-longer-created-by-default
+File { backup => false }
+
+# DEFAULT NODE
+# Node definitions in this file are merged with node data from the console. See
+# http://docs.puppetlabs.com/guides/language_guide.html#nodes for more on
+# node definitions.
+
+# The default node definition matches any node lacking a more specific node
+# definition. If there are no other nodes in this file, classes declared here
+# will be included in every node's catalog, *in addition* to any classes
+# specified in the console for that node.
+
+node default {
+  # This is where you can declare classes for all nodes.
+  # Example:
+  #   class { 'my_class': }
+}

data/spec/fixtures/controlrepos/caching/site/profile/manifests/base.pp

@@ -0,0 +1,5 @@
+class profile::base {
+
+  #the base profile should include component modules that will be on all nodes
+
+}

data/spec/fixtures/controlrepos/caching/site/profile/manifests/example.pp

@@ -0,0 +1,3 @@
+class profile::example {
+
+}

data/spec/fixtures/controlrepos/caching/site/role/manifests/database_server.pp

@@ -0,0 +1,7 @@
+class role::database_server {
+
+  #This role would be made of all the profiles that need to be included to make a database server work
+  #All roles should include the base profile
+  include profile::base
+
+}

data/spec/fixtures/controlrepos/caching/site/role/manifests/example.pp

@@ -0,0 +1,3 @@
+class role::example {
+
+}

data/spec/fixtures/controlrepos/caching/site/role/manifests/webserver.pp

@@ -0,0 +1,7 @@
+class role::webserver {
+
+  #This role would be made of all the profiles that need to be included to make a webserver work
+  #All roles should include the base profile
+  include profile::base
+
+}

data/spec/fixtures/controlrepos/caching/spec/factsets/README.md

@@ -0,0 +1,7 @@
+# Factsets
+
+This directory is where we put any custom factsets that we want to use. They can be generated by running `puppet facts` on the target system.
+
+**Hot tip:** If you already have factsets in here when you run `onceover init` they will be picked up and added to the config file Automatically
+
+More info: https://github.com/dylanratcliffe/onceover#factsets

data/spec/fixtures/controlrepos/caching/spec/onceover.yaml

@@ -0,0 +1,53 @@
+# Classes to be tested
+classes:
+  - role::database_server
+  - role::webserver
+  - role::example
+
+# Nodes to tests classes on, this refers to a 'factset' or 'nodeset'
+# depending on weather you are running 'spec' or 'acceptance' tests
+nodes:
+  - AIX-7.1-powerpc
+  - SLES-12.1-64
+  - Debian-6.0.10-32
+  - CentOS-6.6-64
+  - Ubuntu-12.04-32
+  - Ubuntu-12.04-64
+  - CentOS-6.6-32
+  - Debian-6.0.10-64
+  - AIX-6.1-powerpc
+  - Windows_Server-2012r2-64
+  - Debian-7.8-32
+  - Windows_Server-2008r2-64
+  - SLES-11.3-64
+  - Debian-7.8-64
+  - solaris-10_u9-sparc-64
+  - solaris-11.2-sparc-64
+  - Ubuntu-14.04-32
+  - CentOS-5.11-64
+  - CentOS-5.11-32
+  - CentOS-7.0-64
+  - Ubuntu-14.04-64
+
+# You can group classes here to save typing
+class_groups:
+
+# You can group nodes here to save typing
+# We have created a 'non_windows_nodes' group because we can't
+# give you Windows vagrant boxes to test with because licensing,
+# we can give you fact sets though so go crazy with spec testing!
+node_groups:
+  windows_nodes:
+    - Windows_Server-2012r2-64
+    - Windows_Server-2008r2-64
+  non_windows_nodes:
+    include: 'all_nodes'
+    exclude: 'windows_nodes'
+
+test_matrix:
+  - all_nodes:
+      classes: 'all_classes'
+      tests: 'spec'
+  - non_windows_nodes: 
+      classes: 'all_classes'
+      tests: 'acceptance'

data/spec/fixtures/controlrepos/caching/spec/pre_conditions/README.md

@@ -0,0 +1,24 @@
+# Pre Conditions
+
+This folder should contain any \*.pp files that you want to be included in every test.
+
+A common use of this is defining resources that may not exist in the catalog when you are running tests. For example, if we are using a resource that tries to restart the `pe-puppetserver` service, unless it is compiled on a Puppet Maser the `pe-puppetserver` service will not exist and the catalog will fail to compile. To get around this we can create a .pp file and define the resource like so:
+
+``` puppet
+# We are not going to actually have this service anywhere on our servers but
+# our code needs to refresh it. This is to trick puppet into doing nothing
+service { 'pe-puppetserver':
+  ensure     => 'running',
+  enable     => false,
+  hasrestart => false, # Force Puppet to use start and stop to restart
+  start      => 'echo "Start"', # This will always exit 0
+  stop       => 'echo "Stop"', # This will also always exit 0
+  hasstatus  => false, # Force puppet to use our command for status
+  status     => 'echo "Status"', # This will always exit 0 and therefore Puppet will think the service is running
+  provider   => 'base',
+}
+```
+
+This will mean that the `pe-puppetserver` service is in the catalog for spec testing and will even allow you to try to restart it during acceptance tests without the service actually being present.
+
+More info: https://github.com/dylanratcliffe/onceover#using-workarounds

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: onceover
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.3.2
 platform: ruby
 authors:
 - Dylan Ratcliffe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-05 00:00:00.000000000 Z
+date: 2018-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -341,13 +341,16 @@ files:
 - factsets/Windows_Server-2012r2-64.json
 - factsets/solaris-10_u9-sparc-64.json
 - factsets/solaris-11.2-sparc-64.json
+- features/cache.feature
 - features/help.feature
 - features/init.feature
 - features/run.feature
 - features/show.feature
+- features/step_definitions/cache.rb
 - features/step_definitions/common.rb
 - features/step_definitions/init.rb
 - features/step_definitions/run.rb
+- features/support/cache_helper.rb
 - features/support/command_helper.rb
 - features/support/controlrepo_helper.rb
 - features/support/env.rb
@@ -379,6 +382,24 @@ files:
 - spec/fixtures/controlrepos/basic/site/role/manifests/database_server.pp
 - spec/fixtures/controlrepos/basic/site/role/manifests/example.pp
 - spec/fixtures/controlrepos/basic/site/role/manifests/webserver.pp
+- spec/fixtures/controlrepos/caching/.gitignore
+- spec/fixtures/controlrepos/caching/.travis.yml
+- spec/fixtures/controlrepos/caching/Gemfile
+- spec/fixtures/controlrepos/caching/Puppetfile
+- spec/fixtures/controlrepos/caching/Rakefile
+- spec/fixtures/controlrepos/caching/deleteme.txt
+- spec/fixtures/controlrepos/caching/environment.conf
+- spec/fixtures/controlrepos/caching/hieradata/common.yaml
+- spec/fixtures/controlrepos/caching/hieradata/nodes/example-node.yaml
+- spec/fixtures/controlrepos/caching/manifests/site.pp
+- spec/fixtures/controlrepos/caching/site/profile/manifests/base.pp
+- spec/fixtures/controlrepos/caching/site/profile/manifests/example.pp
+- spec/fixtures/controlrepos/caching/site/role/manifests/database_server.pp
+- spec/fixtures/controlrepos/caching/site/role/manifests/example.pp
+- spec/fixtures/controlrepos/caching/site/role/manifests/webserver.pp
+- spec/fixtures/controlrepos/caching/spec/factsets/README.md
+- spec/fixtures/controlrepos/caching/spec/onceover.yaml
+- spec/fixtures/controlrepos/caching/spec/pre_conditions/README.md
 - spec/fixtures/controlrepos/control_branch/Puppetfile
 - spec/fixtures/controlrepos/control_branch/environment.conf
 - spec/fixtures/controlrepos/minimal/environment.conf
@@ -416,7 +437,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Testing tools for Puppet controlrepos