puppet 2.6.4 → 2.6.5

data/lib/puppet/configurer.rb

@@ -72,17 +72,13 @@ class Puppet::Configurer
     @splayed = false
   end
 
-  def initialize_report
-    Puppet::Transaction::Report.new
-  end
-
   # Prepare for catalog retrieval.  Downloads everything necessary, etc.
-  def prepare
+  def prepare(options)
     dostorage
 
-    download_plugins
+    download_plugins unless options[:skip_plugin_download]
 
-    download_fact_plugins
+    download_fact_plugins unless options[:skip_plugin_download]
 
     execute_prerun_command
   end
@@ -126,7 +122,7 @@ class Puppet::Configurer
   # which accepts :tags and :ignoreschedules.
   def run(options = {})
     begin
-      prepare
+      prepare(options)
     rescue SystemExit,NoMemoryError
       raise
     rescue Exception => detail
@@ -134,7 +130,7 @@ class Puppet::Configurer
       Puppet.err "Failed to prepare catalog: #{detail}"
     end
 
-    options[:report] ||= initialize_report
+    options[:report] ||= Puppet::Transaction::Report.new("apply")
     report = options[:report]
     Puppet::Util::Log.newdestination(report)
 
@@ -145,6 +141,8 @@ class Puppet::Configurer
       return
     end
 
+    report.configuration_version = catalog.version
+
     transaction = nil
 
     begin
@@ -172,8 +170,8 @@ class Puppet::Configurer
     send_report(report, transaction)
   end
 
-  def send_report(report, trans = nil)
-    trans.generate_report if trans
+  def send_report(report, trans)
+    report.finalize_report if trans
     puts report.summary if Puppet[:summarize]
     report.save if Puppet[:report]
   rescue => detail

data/lib/puppet/configurer/plugin_handler.rb

@@ -19,8 +19,6 @@ module Puppet::Configurer::PluginHandler
     begin
       Puppet.info "Loading downloaded plugin #{file}"
       load file
-    rescue SystemExit,NoMemoryError
-      raise
     rescue Exception => detail
       Puppet.err "Could not load downloaded file #{file}: #{detail}"
     end

data/lib/puppet/defaults.rb

@@ -2,8 +2,8 @@
 module Puppet
   setdefaults(:main,
     :confdir => [Puppet.run_mode.conf_dir, "The main Puppet configuration directory.  The default for this parameter is calculated based on the user.  If the process
-    is running as root or the user that `puppet master` is supposed to run as, it defaults to a system directory, but if it's running as any other user,
-    it defaults to being in `~`."],
+    is running as root or the user that Puppet is supposed to run as, it defaults to a system directory, but if it's running as any other user,
+    it defaults to being in the user's home directory."],
     :vardir => [Puppet.run_mode.var_dir, "Where Puppet stores dynamic and growing data.  The default for this parameter is calculated specially, like `confdir`_."],
     :name => [Puppet.application_name.to_s, "The name of the application, if we are running as one.  The
       default is essentially $0 without the path or `.rb`."],
@@ -14,7 +14,11 @@ module Puppet
 
   setdefaults(:main,
     :trace => [false, "Whether to print stack traces on some errors"],
-    :autoflush => [false, "Whether log files should always flush to disk."],
+    :autoflush => {
+      :default => false,
+      :desc    => "Whether log files should always flush to disk.",
+      :hook    => proc { |value| Log.autoflush = value }
+    },
     :syslogfacility => ["daemon", "What syslog facility to use when logging to
       syslog.  Syslog has a fixed list of valid facilities, and you must
       choose one of those; you cannot just make one up."],
@@ -598,6 +602,11 @@ module Puppet
       compression, but if it supports it, this setting might reduce performance on high-speed LANs."]
   )
 
+  setdefaults(:inspect,
+      :archive_files => [false, "During an inspect run, whether to archive files whose contents are audited to a file bucket."],
+      :archive_file_server => ["$server", "During an inspect run, the file bucket server to archive files to if archive_files is set."]
+  )
+
   # Plugin information.
 
     setdefaults(

data/lib/puppet/external/pson/pure/generator.rb

@@ -44,34 +44,13 @@ module PSON
       string << '' # XXX workaround: avoid buffer sharing
       string.force_encoding(Encoding::ASCII_8BIT)
       string.gsub!(/["\\\x0-\x1f]/) { MAP[$MATCH] }
-      string.gsub!(/(
-        (?:
-          [\xc2-\xdf][\x80-\xbf]    |
-          [\xe0-\xef][\x80-\xbf]{2} |
-          [\xf0-\xf4][\x80-\xbf]{3}
-            )+ |
-            [\x80-\xc1\xf5-\xff]       # invalid
-              )/nx) { |c|
-                c.size == 1 and raise GeneratorError, "invalid utf8 byte: '#{c}'"
-                s = PSON::UTF8toUTF16.iconv(c).unpack('H*')[0]
-                s.gsub!(/.{4}/n, '\\\\u\&')
-              }
-      string.force_encoding(Encoding::UTF_8)
       string
     rescue Iconv::Failure => e
       raise GeneratorError, "Caught #{e.class}: #{e}"
     end
   else
     def utf8_to_pson(string) # :nodoc:
-      string.
-        gsub(/["\\\x0-\x1f]/n) { MAP[$MATCH] }.
-        gsub(/((?:
-          [\xc2-\xdf][\x80-\xbf]    |
-          [\xe0-\xef][\x80-\xbf]{2} |
-          [\xf0-\xf4][\x80-\xbf]{3}
-            )+)/nx) { |c|
-        PSON::UTF8toUTF16.iconv(c).unpack('H*')[0].gsub(/.{4}/n, '\\\\u\&')
-      }
+      string.gsub(/["\\\x0-\x1f]/n) { MAP[$MATCH] }
     end
   end
   module_function :utf8_to_pson

data/lib/puppet/file_bucket/dipper.rb

@@ -33,10 +33,16 @@ class Puppet::FileBucket::Dipper
     raise(ArgumentError, "File #{file} does not exist") unless ::File.exist?(file)
     contents = ::File.read(file)
     begin
-      file_bucket_file = Puppet::FileBucket::File.new(contents, :bucket_path => @local_path, :path => absolutize_path(file) )
-      dest_path = "#{@rest_path}#{file_bucket_file.name}"
+      file_bucket_file = Puppet::FileBucket::File.new(contents, :bucket_path => @local_path)
+      files_original_path = absolutize_path(file)
+      dest_path = "#{@rest_path}#{file_bucket_file.name}#{files_original_path}"
+
+      # Make a HEAD request for the file so that we don't waste time
+      # uploading it if it already exists in the bucket.
+      unless Puppet::FileBucket::File.head("#{@rest_path}#{file_bucket_file.checksum_type}/#{file_bucket_file.checksum_data}#{files_original_path}")
+        file_bucket_file.save(dest_path)
+      end
 
-      file_bucket_file.save(dest_path)
       return file_bucket_file.checksum_data
     rescue => detail
       puts detail.backtrace if Puppet[:trace]

data/lib/puppet/file_bucket/file.rb

@@ -1,10 +1,9 @@
 require 'puppet/file_bucket'
 require 'puppet/indirector'
 require 'puppet/util/checksums'
+require 'digest/md5'
 
 class Puppet::FileBucket::File
-  include Puppet::Util::Checksums
-
   # This class handles the abstract notion of a file in a filebucket.
   # There are mechanisms to save and load this file locally and remotely in puppet/indirector/filebucketfile/*
   # There is a compatibility class that emulates pre-indirector filebuckets in Puppet::FileBucket::Dipper
@@ -12,71 +11,27 @@ class Puppet::FileBucket::File
   require 'puppet/file_bucket/file/indirection_hooks'
   indirects :file_bucket_file, :terminus_class => :file, :extend => Puppet::FileBucket::File::IndirectionHooks
 
-  attr :path, true
-  attr :paths, true
-  attr :contents, true
-  attr :checksum_type
-  attr :bucket_path, true
-
-  def self.default_checksum_type
-    "md5"
-  end
+  attr :contents
+  attr :bucket_path
 
   def initialize( contents, options = {} )
-    @bucket_path   = options[:bucket_path]
-    @path          = options[:path]
-    @paths         = options[:paths] || []
-
-    @checksum      = options[:checksum]
-    @checksum_type = options[:checksum_type]
-
-    self.contents  = contents
-
-    yield(self) if block_given?
-
-    validate!
-  end
+    raise ArgumentError if !contents.is_a?(String)
+    @contents  = contents
 
-  def validate!
-    validate_checksum_type!(checksum_type)
-    validate_checksum!(checksum) if checksum
+    @bucket_path = options.delete(:bucket_path)
+    raise ArgumentError if options != {}
   end
 
-  def contents=(str)
-    raise "You may not change the contents of a FileBucket File" if @contents
-    validate_content!(str)
-    @contents = str
+  def checksum_type
+    'md5'
   end
 
   def checksum
-    return @checksum if @checksum
-    @checksum = calculate_checksum if contents
-    @checksum
-  end
-
-  def checksum=(checksum)
-    validate_checksum!(checksum)
-    @checksum = checksum
-  end
-
-  def checksum_type=( new_checksum_type )
-    @checksum = nil
-    @checksum_type = new_checksum_type
-  end
-
-  def checksum_type
-    unless @checksum_type
-      if @checksum
-        @checksum_type = sumtype(checksum)
-      else
-        @checksum_type = self.class.default_checksum_type
-      end
-    end
-    @checksum_type
+    "{#{checksum_type}}#{checksum_data}"
   end
 
   def checksum_data
-    sumdata(checksum)
+    @checksum_data ||= Digest::MD5.hexdigest(contents)
   end
 
   def to_s
@@ -84,18 +39,7 @@ class Puppet::FileBucket::File
   end
 
   def name
-    [checksum_type, checksum_data, path].compact.join('/')
-  end
-
-  def name=(name)
-    data = name.split('/',3)
-    self.path = data.pop
-    @checksum_type = nil
-    self.checksum = "{#{data[0]}}#{data[1]}"
-  end
-
-  def conflict_check?
-    true
+    "#{checksum_type}/#{checksum_data}"
   end
 
   def self.from_s( contents )
@@ -103,34 +47,10 @@ class Puppet::FileBucket::File
   end
 
   def to_pson
-    hash = { "contents" => contents }
-    hash["path"] = @path if @path
-    hash.to_pson
+    { "contents" => contents }.to_pson
   end
 
   def self.from_pson( pson )
-    self.new( pson["contents"], :path => pson["path"] )
-  end
-
-  private
-
-  def calculate_checksum
-    "{#{checksum_type}}" + send(checksum_type, contents)
-  end
-
-  def validate_content!(content)
-    raise ArgumentError, "Contents must be a string" if content and ! content.is_a?(String)
-  end
-
-  def validate_checksum!(new_checksum)
-    newtype = sumtype(new_checksum)
-
-    unless sumdata(new_checksum) == (calc_sum = send(newtype, contents))
-      raise Puppet::Error, "Checksum #{new_checksum} does not match contents #{calc_sum}"
-    end
-  end
-
-  def validate_checksum_type!(type)
-    raise ArgumentError, "Invalid checksum type #{type}" unless respond_to?(type)
+    self.new( pson["contents"] )
   end
 end

data/lib/puppet/indirector.rb

@@ -50,6 +50,10 @@ module Puppet::Indirector
       indirection.find(*args)
     end
 
+    def head(*args)
+      indirection.head(*args)
+    end
+
     def destroy(*args)
       indirection.destroy(*args)
     end

data/lib/puppet/indirector/catalog/active_record.rb

@@ -32,7 +32,7 @@ class Puppet::Resource::Catalog::ActiveRecord < Puppet::Indirector::ActiveRecord
 
       if node = Puppet::Node.find(catalog.name)
         host.ip = node.parameters["ipaddress"]
-        host.environment = node.environment
+        host.environment = node.environment.to_s
       end
 
       host.save

data/lib/puppet/indirector/file_bucket_file/file.rb

@@ -14,83 +14,102 @@ module Puppet::FileBucketFile
     end
 
     def find( request )
-      checksum, path = request_to_checksum_and_path( request )
-      find_by_checksum( checksum, request.options )
+      checksum, files_original_path = request_to_checksum_and_path( request )
+      dir_path = path_for(request.options[:bucket_path], checksum)
+      file_path = ::File.join(dir_path, 'contents')
+
+      return nil unless ::File.exists?(file_path)
+      return nil unless path_match(dir_path, files_original_path)
+
+      if request.options[:diff_with]
+        hash_protocol = sumtype(checksum)
+        file2_path = path_for(request.options[:bucket_path], request.options[:diff_with], 'contents')
+        raise "could not find diff_with #{request.options[:diff_with]}" unless ::File.exists?(file2_path)
+        return `diff #{file_path.inspect} #{file2_path.inspect}`
+      else
+        contents = ::File.read file_path
+        Puppet.info "FileBucket read #{checksum}"
+        model.new(contents)
+      end
     end
 
-    def save( request )
-      checksum, path = request_to_checksum_and_path( request )
+    def head(request)
+      checksum, files_original_path = request_to_checksum_and_path(request)
+      dir_path = path_for(request.options[:bucket_path], checksum)
+
+      ::File.exists?(::File.join(dir_path, 'contents')) and path_match(dir_path, files_original_path)
+    end
 
+    def save( request )
       instance = request.instance
-      instance.checksum = checksum if checksum
-      instance.path = path if path
+      checksum, files_original_path = request_to_checksum_and_path(request)
 
-      save_to_disk(instance)
+      save_to_disk(instance, files_original_path)
       instance.to_s
     end
 
     private
 
-    def find_by_checksum( checksum, options )
-      model.new( nil, :checksum => checksum ) do |bucket_file|
-        bucket_file.bucket_path = options[:bucket_path]
-        filename = contents_path_for( bucket_file )
-
-        return nil if ! ::File.exist? filename
-
-        begin
-          contents = ::File.read filename
-          Puppet.info "FileBucket read #{bucket_file.checksum}"
-        rescue RuntimeError => e
-          raise Puppet::Error, "file could not be read: #{e.message}"
+    def path_match(dir_path, files_original_path)
+      return true unless files_original_path # if no path was provided, it's a match
+      paths_path = ::File.join(dir_path, 'paths')
+      return false unless ::File.exists?(paths_path)
+      ::File.open(paths_path) do |f|
+        f.each do |line|
+          return true if line.chomp == files_original_path
         end
-
-        if ::File.exist?(paths_path_for( bucket_file) )
-          ::File.open(paths_path_for( bucket_file) ) do |f|
-            bucket_file.paths = f.readlines.map { |l| l.chomp }
-          end
-        end
-
-        bucket_file.contents = contents
       end
+      return false
     end
 
-    def save_to_disk( bucket_file )
-      # If the file already exists, just return the md5 sum.
-      if ::File.exist?(contents_path_for( bucket_file) )
+    def save_to_disk( bucket_file, files_original_path )
+      filename = path_for(bucket_file.bucket_path, bucket_file.checksum_data, 'contents')
+      dir_path = path_for(bucket_file.bucket_path, bucket_file.checksum_data)
+      paths_path = ::File.join(dir_path, 'paths')
+
+      # If the file already exists, do nothing.
+      if ::File.exist?(filename)
         verify_identical_file!(bucket_file)
       else
         # Make the directories if necessary.
-        unless ::File.directory?( path_for( bucket_file) )
+        unless ::File.directory?(dir_path)
           Puppet::Util.withumask(0007) do
-            ::FileUtils.mkdir_p( path_for( bucket_file) )
+            ::FileUtils.mkdir_p(dir_path)
           end
         end
 
-        Puppet.info "FileBucket adding #{bucket_file.path} as #{bucket_file.checksum}"
+        Puppet.info "FileBucket adding #{bucket_file.checksum}"
 
         # Write the file to disk.
         Puppet::Util.withumask(0007) do
-          ::File.open(contents_path_for(bucket_file), ::File::WRONLY|::File::CREAT, 0440) do |of|
+          ::File.open(filename, ::File::WRONLY|::File::CREAT, 0440) do |of|
             of.print bucket_file.contents
           end
+          ::File.open(paths_path, ::File::WRONLY|::File::CREAT, 0640) do |of|
+            # path will be written below
+          end
         end
       end
 
-      save_path_to_paths_file(bucket_file)
-      bucket_file.checksum_data
+      unless path_match(dir_path, files_original_path)
+        ::File.open(paths_path, 'a') do |f|
+          f.puts(files_original_path)
+        end
+      end
     end
 
     def request_to_checksum_and_path( request )
-      return [request.key, nil] if checksum?(request.key)
-
       checksum_type, checksum, path = request.key.split(/\//, 3)
-      return(checksum_type.to_s == "" ? nil : [ "{#{checksum_type}}#{checksum}", path ])
+      if path == '' # Treat "md5/<checksum>/" like "md5/<checksum>"
+        path = nil
+      end
+      raise "Unsupported checksum type #{checksum_type.inspect}" if checksum_type != 'md5'
+      raise "Invalid checksum #{checksum.inspect}" if checksum !~ /^[0-9a-f]{32}$/
+      [checksum, path]
     end
 
-    def path_for(bucket_file, subfile = nil)
-      bucket_path = bucket_file.bucket_path || Puppet[:bucketdir]
-      digest      = bucket_file.checksum_data
+    def path_for(bucket_path, digest, subfile = nil)
+      bucket_path ||= Puppet[:bucketdir]
 
       dir     = ::File.join(digest[0..7].split(""))
       basedir = ::File.join(bucket_path, dir, digest)
@@ -99,48 +118,18 @@ module Puppet::FileBucketFile
       ::File.join(basedir, subfile)
     end
 
-    def contents_path_for(bucket_file)
-      path_for(bucket_file, "contents")
-    end
-
-    def paths_path_for(bucket_file)
-      path_for(bucket_file, "paths")
-    end
-
-    def content_check?
-      true
-    end
-
     # If conflict_check is enabled, verify that the passed text is
     # the same as the text in our file.
     def verify_identical_file!(bucket_file)
-      return unless content_check?
-      disk_contents = ::File.read(contents_path_for(bucket_file))
+      disk_contents = ::File.read(path_for(bucket_file.bucket_path, bucket_file.checksum_data, 'contents'))
 
       # If the contents don't match, then we've found a conflict.
       # Unlikely, but quite bad.
       if disk_contents != bucket_file.contents
-        raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}", caller
+        raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}"
       else
-        Puppet.info "FileBucket got a duplicate file #{bucket_file.path} (#{bucket_file.checksum})"
+        Puppet.info "FileBucket got a duplicate file #{bucket_file.checksum}"
       end
     end
-
-    def save_path_to_paths_file(bucket_file)
-      return unless bucket_file.path
-
-      # check for dupes
-      if ::File.exist?(paths_path_for( bucket_file) )
-        ::File.open(paths_path_for( bucket_file) ) do |f|
-          return if f.readlines.collect { |l| l.chomp }.include?(bucket_file.path)
-        end
-      end
-
-      # if it's a new file, or if our path isn't in the file yet, add it
-      ::File.open(paths_path_for(bucket_file), ::File::WRONLY|::File::CREAT|::File::APPEND) do |of|
-        of.puts bucket_file.path
-      end
-    end
-
   end
 end