puppet 3.4.3 → 3.5.0.rc1

data/spec/unit/util/resource_template_spec.rb

@@ -6,20 +6,20 @@ require 'puppet/util/resource_template'
 describe Puppet::Util::ResourceTemplate do
   describe "when initializing" do
     it "should fail if the template does not exist" do
-      Puppet::FileSystem::File.expects(:exist?).with("/my/template").returns false
+      Puppet::FileSystem.expects(:exist?).with("/my/template").returns false
       lambda { Puppet::Util::ResourceTemplate.new("/my/template", mock('resource')) }.should raise_error(ArgumentError)
     end
 
     it "should not create the ERB template" do
       ERB.expects(:new).never
-      Puppet::FileSystem::File.expects(:exist?).with("/my/template").returns true
+      Puppet::FileSystem.expects(:exist?).with("/my/template").returns true
       Puppet::Util::ResourceTemplate.new("/my/template", mock('resource'))
     end
   end
 
   describe "when evaluating" do
     before do
-      Puppet::FileSystem::File.stubs(:exist?).returns true
+      Puppet::FileSystem.stubs(:exist?).returns true
       File.stubs(:read).returns "eh"
 
       @template = stub 'template', :result => nil

data/spec/unit/util/selinux_spec.rb

@@ -69,7 +69,7 @@ describe Puppet::Util::SELinux do
       find_fs('/mnt/nfs/testfile/foobar').should == "nfs"
     end
 
-    it "should reture true for a capable filesystem" do
+    it "should return true for a capable filesystem" do
       selinux_label_support?('/etc/puppet/testfile').should be_true
     end
 
@@ -123,10 +123,10 @@ describe Puppet::Util::SELinux do
     it "should return a context if a default context exists" do
       self.expects(:selinux_support?).returns true
       fstat = stub 'File::Stat', :mode => 0
-      stub_file = stub('/foo', :lstat => fstat)
-      Puppet::FileSystem::File.expects(:new).with('/foo').returns stub_file
+      Puppet::FileSystem.expects(:lstat).with('/foo').returns(fstat)
       self.expects(:find_fs).with("/foo").returns "ext3"
       Selinux.expects(:matchpathcon).with("/foo", 0).returns [0, "user_u:role_r:type_t:s0"]
+
       get_selinux_default_context("/foo").should == "user_u:role_r:type_t:s0"
     end
 
@@ -151,10 +151,10 @@ describe Puppet::Util::SELinux do
     it "should return nil if matchpathcon returns failure" do
       self.expects(:selinux_support?).returns true
       fstat = stub 'File::Stat', :mode => 0
-      stub_file = stub('/foo', :lstat => fstat)
-      Puppet::FileSystem::File.expects(:new).with('/foo').returns stub_file
+      Puppet::FileSystem.expects(:lstat).with('/foo').returns(fstat)
       self.expects(:find_fs).with("/foo").returns "ext3"
       Selinux.expects(:matchpathcon).with("/foo", 0).returns -1
+
       get_selinux_default_context("/foo").should be_nil
     end
 

data/spec/unit/util/storage_spec.rb

@@ -77,7 +77,7 @@ describe Puppet::Util::Storage do
       end
 
       it "should not fail to load" do
-        Puppet::FileSystem::File.exist?(@path).should be_false
+        Puppet::FileSystem.exist?(@path).should be_false
         Puppet[:statedir] = @path
         Puppet::Util::Storage.load
         Puppet[:statefile] = @path
@@ -85,7 +85,7 @@ describe Puppet::Util::Storage do
       end
 
       it "should not lose its internal state when load() is called" do
-        Puppet::FileSystem::File.exist?(@path).should be_false
+        Puppet::FileSystem.exist?(@path).should be_false
 
         Puppet::Util::Storage.cache(:yayness)
         Puppet::Util::Storage.state.should == {:yayness=>{}}
@@ -176,12 +176,12 @@ describe Puppet::Util::Storage do
     end
 
     it "should create the state file if it does not exist" do
-      Puppet::FileSystem::File.exist?(Puppet[:statefile]).should be_false
+      Puppet::FileSystem.exist?(Puppet[:statefile]).should be_false
       Puppet::Util::Storage.cache(:yayness)
 
       Puppet::Util::Storage.store
 
-      Puppet::FileSystem::File.exist?(Puppet[:statefile]).should be_true
+      Puppet::FileSystem.exist?(Puppet[:statefile]).should be_true
     end
 
     it "should raise an exception if the state file is not a regular file" do

data/spec/unit/util/tag_set_spec.rb

@@ -33,7 +33,7 @@ describe Puppet::Util::TagSet do
     array = ['a', 'b', 1, 5.4]
     set.merge(array)
 
-    tes = Puppet::Util::TagSet.from_pson(PSON.parse(set.to_pson))
+    tes = Puppet::Util::TagSet.from_data_hash(PSON.parse(set.to_pson))
     tes.should == set
   end
 

data/spec/unit/util/watcher_spec.rb

@@ -14,10 +14,7 @@ describe Puppet::Util::Watcher do
     let(:filename) { "fake" }
 
     def after_reading_the_sequence(initial, *results)
-      mock_file = mock(filename)
-      Puppet::FileSystem::File.expects(:new).with(filename).at_least(1).returns mock_file
-
-      expectation = mock_file.stubs(:stat)
+      expectation = Puppet::FileSystem.expects(:stat).with(filename).at_least(1)
       ([initial] + results).each do |result|
         expectation = if result.is_a? Class
                         expectation.raises(result)

data/spec/unit/util/yaml_spec.rb

@@ -35,13 +35,13 @@ describe Puppet::Util::Yaml do
 
   context "when the file is empty" do
     it "returns false" do
-      Puppet::FileSystem::File.new(filename).touch
+      Puppet::FileSystem.touch(filename)
 
       expect(Puppet::Util::Yaml.load_file(filename)).to be_false
     end
 
     it "allows return value to be overridden" do
-      Puppet::FileSystem::File.new(filename).touch
+      Puppet::FileSystem.touch(filename)
 
       expect(Puppet::Util::Yaml.load_file(filename, {})).to eq({})
     end

data/spec/unit/util_spec.rb

@@ -19,7 +19,7 @@ describe Puppet::Util do
     end
 
     def get_mode(file)
-      Puppet::FileSystem::File.new(file).lstat.mode & 07777
+      Puppet::FileSystem.lstat(file).mode & 07777
     end
   end
 
@@ -449,7 +449,7 @@ describe Puppet::Util do
 
     it "should copy the permissions of the source file before yielding on Unix", :if => !Puppet.features.microsoft_windows? do
       set_mode(0555, target.path)
-      inode = Puppet::FileSystem::File.new(target.path).stat.ino
+      inode = Puppet::FileSystem.stat(target.path).ino
 
       yielded = false
       subject.replace_file(target.path, 0600) do |fh|
@@ -458,19 +458,19 @@ describe Puppet::Util do
       end
       yielded.should be_true
 
-      Puppet::FileSystem::File.new(target.path).stat.ino.should_not == inode
+      Puppet::FileSystem.stat(target.path).ino.should_not == inode
       get_mode(target.path).should == 0555
     end
 
     it "should use the default permissions if the source file doesn't exist" do
       new_target = target.path + '.foo'
-      Puppet::FileSystem::File.exist?(new_target).should be_false
+      Puppet::FileSystem.exist?(new_target).should be_false
 
       begin
         subject.replace_file(new_target, 0555) {|fh| fh.puts "foo" }
         get_mode(new_target).should == 0555
       ensure
-        Puppet::FileSystem::File.unlink(new_target) if Puppet::FileSystem::File.exist?(new_target)
+        Puppet::FileSystem.unlink(new_target) if Puppet::FileSystem.exist?(new_target)
       end
     end
 
@@ -502,14 +502,14 @@ describe Puppet::Util do
     {:string => '664', :number => 0664, :symbolic => "ug=rw-,o=r--" }.each do |label,mode|
       it "should support #{label} format permissions" do
         new_target = target.path + "#{mode}.foo"
-        Puppet::FileSystem::File.exist?(new_target).should be_false
+        Puppet::FileSystem.exist?(new_target).should be_false
 
         begin
           subject.replace_file(new_target, mode) {|fh| fh.puts "this is an interesting content" }
 
           get_mode(new_target).should == 0664
         ensure
-          Puppet::FileSystem::File.unlink(new_target) if Puppet::FileSystem::File.exist?(new_target)
+          Puppet::FileSystem.unlink(new_target) if Puppet::FileSystem.exist?(new_target)
         end
       end
     end

data/tasks/benchmark.rake

@@ -21,7 +21,6 @@ namespace :benchmark do
         @benchmark = Benchmarker.new(ENV['TARGET'], ENV['SIZE'].to_i)
       end
 
-      desc "Generate the #{name} scenario."
       task :generate => :setup do
         @benchmark.generate
         @benchmark.setup

data/tasks/parallel.rake

@@ -0,0 +1,408 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'thread'
+begin
+  require 'rspec'
+  require 'rspec/core/formatters/helpers'
+  require 'facter'
+rescue LoadError
+  # Don't define the task if we don't have rspec or facter present
+else
+  module Parallel
+    module RSpec
+      #
+      # Responsible for buffering the output of RSpec's progress formatter.
+      #
+      class ProgressFormatBuffer
+        attr_reader :pending_lines
+        attr_reader :failure_lines
+        attr_reader :examples
+        attr_reader :failures
+        attr_reader :pending
+        attr_reader :failed_example_lines
+        attr_reader :state
+
+        module OutputState
+          HEADER = 1
+          PROGRESS = 2
+          SUMMARY = 3
+          PENDING = 4
+          FAILURES = 5
+          DURATION = 6
+          COUNTS = 7
+          FAILED_EXAMPLES = 8
+        end
+
+        def initialize(io, color)
+          @io = io
+          @color = color
+          @state = OutputState::HEADER
+          @pending_lines = []
+          @failure_lines = []
+          @examples = 0
+          @failures = 0
+          @pending = 0
+          @failed_example_lines = []
+        end
+
+        def color?
+          @color
+        end
+
+        def read
+          # Parse and ignore the one line header
+          if @state == OutputState::HEADER
+            begin
+              @io.readline
+            rescue EOFError
+              return nil
+            end
+            @state = OutputState::PROGRESS
+            return ''
+          end
+
+          # If the progress has been read, parse the summary
+          if @state == OutputState::SUMMARY
+            parse_summary
+            return nil
+          end
+
+          # Read the progress output up to 128 bytes at a time
+          # 128 is a small enough number to show some progress, but not too small that
+          # we're constantly writing synchronized output
+          data = @io.read(128)
+          return nil unless data
+
+          data = @remainder + data if @remainder
+
+          # Check for the end of the progress line
+          if (index = data.index "\n")
+            @state = OutputState::SUMMARY
+            @remainder = data[(index+1)..-1]
+            data = data[0...index]
+          # Check for partial ANSI escape codes in colorized output
+          elsif @color && !data.end_with?("\e[0m") && (index = data.rindex("\e[", -6))
+            @remainder = data[index..-1]
+            data = data[0...index]
+          else
+            @remainder = nil
+          end
+
+          data
+        end
+
+        private
+
+        def parse_summary
+          # If there is a remainder, concat it with the next line and handle each line
+          unless @remainder.empty?
+            lines = @remainder
+            eof = false
+            begin
+              lines += @io.readline
+            rescue EOFError
+              eof = true
+            end
+            lines.each_line do |line|
+              parse_summary_line line
+            end
+            return if eof
+          end
+
+          # Process the rest of the lines
+          begin
+            @io.each_line do |line|
+              parse_summary_line line
+            end
+          rescue EOFError
+          end
+        end
+
+        def parse_summary_line(line)
+          line.chomp!
+          return if line.empty?
+
+          if line == 'Pending:'
+            @status = OutputState::PENDING
+            return
+          elsif line == 'Failures:'
+            @status = OutputState::FAILURES
+            return
+          elsif line == 'Failed examples:'
+            @status = OutputState::FAILED_EXAMPLES
+            return
+          elsif (line.match /^Finished in ((\d+\.?\d*) minutes?)? ?(\d+\.?\d*) seconds?$/)
+            @status = OutputState::DURATION
+            return
+          elsif (match = line.gsub(/\e\[\d+m/, '').match /^(\d+) examples?, (\d+) failures?(, (\d+) pending)?$/)
+            @status = OutputState::COUNTS
+            @examples = match[1].to_i
+            @failures = match[2].to_i
+            @pending = (match[4] || 0).to_i
+            return
+          end
+
+          case @status
+            when OutputState::PENDING
+              @pending_lines << line
+            when OutputState::FAILURES
+              @failure_lines << line
+            when OutputState::FAILED_EXAMPLES
+              @failed_example_lines << line
+          end
+        end
+      end
+
+      #
+      # Responsible for parallelizing spec testing.
+      #
+      class Parallelizer
+        include ::RSpec::Core::Formatters::Helpers
+
+        # Number of processes to use
+        attr_reader :process_count
+        # Approximate size of each group of tests
+        attr_reader :group_size
+
+        def initialize(process_count, group_size, color)
+          @process_count = process_count
+          @group_size = group_size
+          @color = color
+        end
+
+        def color?
+          @color
+        end
+
+        def run
+          @start_time = Time.now
+
+          groups = group_specs
+          fail red('error: no specs were found') if groups.length == 0
+
+          begin
+            run_specs groups
+          ensure
+            groups.each do |file|
+              File.unlink(file)
+            end
+          end
+        end
+
+        private
+
+        def group_specs
+          # Spawn the rspec_grouper utility to perform the test grouping
+          # We do this in a separate process to limit this processes' long-running footprint
+          io = IO.popen("ruby util/rspec_grouper #{@group_size}")
+
+          header = true
+          spec_group_files = []
+          io.each_line do |line|
+            line.chomp!
+            header = false if line.empty?
+            next if header || line.empty?
+            spec_group_files << line
+          end
+
+          _, status = Process.waitpid2(io.pid)
+          io.close
+
+          fail red('error: no specs were found.') unless status.success?
+          spec_group_files
+        end
+
+        def run_specs(groups)
+          puts "Processing #{groups.length} spec group(s) with #{@process_count} worker(s)"
+
+          interrupted = false
+          success = true
+          worker_threads = []
+          group_index = -1
+          pids = Array.new(@process_count)
+          mutex = Mutex.new
+
+          # Handle SIGINT by killing child processes
+          original_handler = Signal.trap :SIGINT do
+            break if interrupted
+            interrupted = true
+
+            # Can't synchronize in a trap context, so read dirty
+            pids.each do |pid|
+              begin
+                Process.kill(:SIGKILL, pid) if pid
+              rescue Errno::ESRCH
+              end
+            end
+            puts yellow("\nshutting down...")
+          end
+
+          buffers = []
+
+          process_count.times do |thread_id|
+            worker_threads << Thread.new do
+              while !interrupted do
+                # Get the spec file for this rspec run
+                group = mutex.synchronize { if group_index < groups.length then groups[group_index += 1] else nil end }
+                break unless group && !interrupted
+
+                # Spawn the worker process with redirected output
+                io = IO.popen("ruby util/rspec_runner #{group}")
+                pids[thread_id] = io.pid
+
+                # TODO: make the buffer pluggable to handle other output formats like documentation
+                buffer = ProgressFormatBuffer.new(io, @color)
+
+                # Process the output
+                while !interrupted
+                  output = buffer.read
+                  break unless output && !interrupted
+                  next if output.empty?
+                  mutex.synchronize { print output }
+                end
+
+                # Kill the process if we were interrupted, just to be sure
+                if interrupted
+                  begin
+                    Process.kill(:SIGKILL, pids[thread_id])
+                  rescue Errno::ESRCH
+                  end
+                end
+
+                # Reap the process
+                result = Process.waitpid2(pids[thread_id])[1].success?
+                io.close
+                pids[thread_id] = nil
+                mutex.synchronize do
+                  buffers << buffer
+                  success &= result
+                end
+              end
+            end
+          end
+
+          # Join all worker threads
+          worker_threads.each do |thread|
+            thread.join
+          end
+
+          Signal.trap :SIGINT, original_handler
+          fail yellow('execution was interrupted') if interrupted
+
+          dump_summary buffers
+          success
+        end
+
+        def colorize(text, color_code)
+          if @color
+            "#{color_code}#{text}\e[0m"
+          else
+            text
+          end
+        end
+
+        def red(text)
+          colorize(text, "\e[31m")
+        end
+
+        def green(text)
+          colorize(text, "\e[32m")
+        end
+
+        def yellow(text)
+          colorize(text, "\e[33m")
+        end
+
+        def dump_summary(buffers)
+          puts
+
+          # Print out the pending tests
+          print_header = true
+          buffers.each do |buffer|
+            next if buffer.pending_lines.empty?
+            if print_header
+              puts "\nPending:"
+              print_header = false
+            end
+            puts buffer.pending_lines
+          end
+
+          # Print out the failures
+          print_header = true
+          buffers.each do |buffer|
+            next if buffer.failure_lines.empty?
+            if print_header
+              puts "\nFailures:"
+              print_header = false
+            end
+            puts
+            puts buffer.failure_lines
+          end
+
+          # Print out the run time
+          puts "\nFinished in #{format_duration(Time.now - @start_time)}"
+
+          # Count all of the examples
+          examples = 0
+          failures = 0
+          pending = 0
+          buffers.each do |buffer|
+            examples += buffer.examples
+            failures += buffer.failures
+            pending += buffer.pending
+          end
+          if failures > 0
+            puts red(summary_count_line(examples, failures, pending))
+          elsif pending > 0
+            puts yellow(summary_count_line(examples, failures, pending))
+          else
+            puts green(summary_count_line(examples, failures, pending))
+          end
+
+          # Print out the failed examples
+          print_header = true
+          buffers.each do |buffer|
+            next if buffer.failed_example_lines.empty?
+            if print_header
+              puts "\nFailed examples:"
+              print_header = false
+            end
+            puts buffer.failed_example_lines
+          end
+        end
+
+        def summary_count_line(examples, failures, pending)
+          summary = pluralize(examples, "example")
+          summary << ", " << pluralize(failures, "failure")
+          summary << ", #{pending} pending" if pending > 0
+          summary
+        end
+      end
+    end
+  end
+
+  namespace 'parallel' do
+    def color_output?
+      # Check with RSpec to see if color is enabled
+      config = ::RSpec::Core::Configuration.new
+      config.error_stream = $stderr
+      config.output_stream = $stdout
+      options = ::RSpec::Core::ConfigurationOptions.new []
+      options.parse_options
+      options.configure config
+      config.color
+    end
+
+    desc 'Runs specs in parallel.'
+    task 'spec', :process_count, :group_size do |_, args|
+      # Default group size in rspec examples
+      DEFAULT_GROUP_SIZE = 1000
+
+      process_count = [(args[:process_count] || Facter.processorcount).to_i, 1].max
+      group_size = [(args[:group_size] || DEFAULT_GROUP_SIZE).to_i, 1].max
+
+      abort unless Parallel::RSpec::Parallelizer.new(process_count, group_size, color_output?).run
+    end
+  end
+end