puppet 4.10.1-x86-mingw32 → 4.10.4-x86-mingw32

data/lib/puppet/pops/semantic_error.rb

@@ -14,4 +14,16 @@ class Puppet::Pops::SemanticError < RuntimeError
     @semantic = semantic
     @options = options
   end
+
+  def file
+    @options[:file]
+  end
+
+  def line
+    @options[:line]
+  end
+
+  def pos
+    @options[:pos]
+  end
 end

data/lib/puppet/pops/serialization/deserializer.rb

@@ -47,10 +47,13 @@ module Serialization
         result = type.read(val.attribute_count, self)
         if result.is_a?(Types::PObjectType)
           existing_type = loader.load(:type, result.name)
-
-          # Add result to the loader unless it is the exact same instance as the existing_type. The add
-          # will succeed when the existing_type is nil.
-          loader.add_entry(:type, result.name, result, nil) unless result.equal?(existing_type)
+          if result.eql?(existing_type)
+            result = existing_type
+          else
+            # Add result to the loader unless it is equal to the existing_type. The add
+            # will only succeed when the existing_type is nil.
+            loader.add_entry(:type, result.name, result, nil)
+          end
         end
         result
       when Extension::ObjectStart

data/lib/puppet/pops/types/p_type_set_type.rb

@@ -279,7 +279,7 @@ class PTypeSetType < PMetaType
     if types.is_a?(Hash)
       types.each do |type_name, value|
         full_name = "#{@name}::#{type_name}".freeze
-        typed_name = Loader::TypedName.new(:type, full_name.downcase, name_auth)
+        typed_name = Loader::TypedName.new(:type, full_name, name_auth)
         type = Loader::TypeDefinitionInstantiator.create_type(full_name, value, name_auth)
         loader.set_entry(typed_name, type, Adapters::SourcePosAdapter.adapt(value).to_uri)
         types[type_name] = type
@@ -300,7 +300,7 @@ class PTypeSetType < PMetaType
     if types.is_a?(Hash)
       types.each do |type_name, value|
         full_name = "#{@name}::#{type_name}".freeze
-        typed_name = Loader::TypedName.new(:type, full_name.downcase, name_auth)
+        typed_name = Loader::TypedName.new(:type, full_name, name_auth)
         meta_name = value.is_a?(Hash) ? 'Object' : 'TypeAlias'
         type = Loader::TypeDefinitionInstantiator.create_named_type(full_name, meta_name, value, name_auth)
         loader.set_entry(typed_name, type)

data/lib/puppet/pops/types/string_converter.rb

@@ -544,25 +544,12 @@ class StringConverter
     when :s
       val.to_s
     when :q
-      val.to_s.inspect
-    when :puppet
-      puppet_safe(val.to_s)
-    when :i, :d, :x, :o, :f, :puppet
-      converted = convert(o, PNumericType) # rest is default
-      "%#{f}" % converted
+      val.inspect
     else
-      raise FormatError.new('Runtime', f.format, 'sqidxof')
+      raise FormatError.new('Runtime', f.format, 'sq')
     end
   end
 
-  # Given an unsafe string make it safe for puppet
-  def puppet_safe(str)
-    str = str.inspect # all specials are now quoted
-    # all $ variables must be quoted
-    str.gsub!("\$", "\\\$")
-    str
-  end
-
   # Basically string_PAnyType converts the value to a String and then formats it according
   # to the resulting type
   #
@@ -839,8 +826,9 @@ class StringConverter
     f = get_format(val_type, format_map)
     case f.format
     when :p
-      str_regexp = '/'
-      str_regexp << (val.options == 0 ? val.source : val.to_s) << '/'
+      rx_s = val.options == 0 ? val.source : val.to_s
+      rx_s = rx_s.gsub(/\//, '\/') unless Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0')
+      str_regexp = "/#{rx_s}/"
       f.orig_fmt == '%p' ? str_regexp : Kernel.format(f.orig_fmt.gsub('p', 's'), str_regexp)
     when :s
       str_regexp = val.options == 0 ? val.source : val.to_s

data/lib/puppet/pops/types/type_set_reference.rb

@@ -37,7 +37,7 @@ class TypeSetReference
   end
 
   def resolve(type_parser, loader)
-    typed_name = Loader::TypedName.new(:type, @name.downcase, @name_authority)
+    typed_name = Loader::TypedName.new(:type, @name, @name_authority)
     loaded_entry = loader.load_typed(typed_name)
     type_set = loaded_entry.nil? ? nil : loaded_entry.value
 

data/lib/puppet/pops/validation/checker4_0.rb

@@ -626,6 +626,10 @@ class Checker4_0 < Evaluator::LiteralEvaluator
     # to enable better error message of the result of the expression rather than the static instruction.
     # (This can be revised as there are static constructs that are illegal, but require updating many
     # tests that expect the detailed reporting).
+    type_name_expr = o.type_name
+    if o.form && o.form != :regular && type_name_expr.is_a?(Model::QualifiedName) && type_name_expr.value == 'class'
+      acceptor.accept(Issues::CLASS_NOT_VIRTUALIZABLE, o)
+    end
   end
 
   def check_ResourceBody(o)

data/lib/puppet/pops/validation/validator_factory_4_0.rb

@@ -31,6 +31,7 @@ class ValidatorFactory_4_0 < Factory
     p[Issues::DUPLICATE_DEFAULT]             = Puppet[:strict] == :off ? :ignore : Puppet[:strict]
     p[Issues::NAME_WITH_HYPHEN]              = :error
     p[Issues::EMPTY_RESOURCE_SPECIALIZATION] = :ignore
+    p[Issues::CLASS_NOT_VIRTUALIZABLE]      = Puppet[:strict] == :off ? :warning : Puppet[:strict]
     p
   end
 end

data/lib/puppet/provider/nameservice.rb

@@ -24,10 +24,12 @@ class Puppet::Provider::NameService < Puppet::Provider
 
     def instances
       objects = []
-      listbyname do |name|
-        objects << new(:name => name, :ensure => :present)
+      Puppet::Etc.send("set#{section}ent")
+      begin
+        while ent = Puppet::Etc.send("get#{section}ent")
+          objects << new(:name => ent.name, :canonical_name => ent.canonical_name, :ensure => :present)
+        end
       end
-
       objects
     end
 
@@ -51,6 +53,7 @@ class Puppet::Provider::NameService < Puppet::Provider
     # List everything out by name.  Abstracted a bit so that it works
     # for both users and groups.
     def listbyname
+      Puppet.deprecation_warning(_("listbyname is deprecated and will be removed in a future release of Puppet. Please use `self.instances` to obtain a list of users."))
       names = []
       Puppet::Etc.send("set#{section()}ent")
       begin
@@ -226,7 +229,7 @@ class Puppet::Provider::NameService < Puppet::Provider
     if @objectinfo.nil? or refresh == true
       @etcmethod ||= ("get" + self.class.section.to_s + "nam").intern
       begin
-        @objectinfo = Puppet::Etc.send(@etcmethod, @resource[:name])
+        @objectinfo = Puppet::Etc.send(@etcmethod, @canonical_name)
       rescue ArgumentError
         @objectinfo = nil
       end
@@ -276,6 +279,11 @@ class Puppet::Provider::NameService < Puppet::Provider
     super
     @custom_environment = {}
     @objectinfo = nil
+    if resource.is_a?(Hash) && !resource[:canonical_name].nil?
+      @canonical_name = resource[:canonical_name]
+    else
+      @canonical_name = resource[:name]
+    end
   end
 
   def set(param, value)

data/lib/puppet/provider/package/yum.rb

@@ -39,17 +39,17 @@ Puppet::Type.type(:package).provide :yum, :parent => :rpm, :source => :rpm do
   #
   # @api private
   # @param package [String] The name of the package to query
-  # @param enablerepo [Array<String>] A list of repositories to enable for this query
   # @param disablerepo [Array<String>] A list of repositories to disable for this query
+  # @param enablerepo [Array<String>] A list of repositories to enable for this query
   # @param disableexcludes [Array<String>] A list of repository excludes to disable for this query
   # @return [Hash<Symbol, String>]
-  def self.latest_package_version(package, enablerepo, disablerepo, disableexcludes)
+  def self.latest_package_version(package, disablerepo, enablerepo, disableexcludes)
 
-    key = [enablerepo, disablerepo, disableexcludes]
+    key = [disablerepo, enablerepo, disableexcludes]
 
     @latest_versions ||= {}
     if @latest_versions[key].nil?
-      @latest_versions[key] = check_updates(enablerepo, disablerepo, disableexcludes)
+      @latest_versions[key] = check_updates(disablerepo, enablerepo, disableexcludes)
     end
 
     if @latest_versions[key][package]
@@ -61,15 +61,15 @@ Puppet::Type.type(:package).provide :yum, :parent => :rpm, :source => :rpm do
   # combination of repositories to enable and disable.
   #
   # @api private
-  # @param enablerepo [Array<String>] A list of repositories to enable for this query
   # @param disablerepo [Array<String>] A list of repositories to disable for this query
+  # @param enablerepo [Array<String>] A list of repositories to enable for this query
   # @param disableexcludes [Array<String>] A list of repository excludes to disable for this query
   # @return [Hash<String, Array<Hash<String, String>>>] All packages that were
   #   found with a list of found versions for each package.
-  def self.check_updates(enablerepo, disablerepo, disableexcludes)
+  def self.check_updates(disablerepo, enablerepo, disableexcludes)
     args = [command(:cmd), 'check-update']
-    args.concat(enablerepo.map { |repo| ["--enablerepo=#{repo}"] }.flatten)
     args.concat(disablerepo.map { |repo| ["--disablerepo=#{repo}"] }.flatten)
+    args.concat(enablerepo.map { |repo| ["--enablerepo=#{repo}"] }.flatten)
     args.concat(disableexcludes.map { |repo| ["--disableexcludes=#{repo}"] }.flatten)
 
     output = Puppet::Util::Execution.execute(args, :failonfail => false, :combine => false)
@@ -213,7 +213,7 @@ Puppet::Type.type(:package).provide :yum, :parent => :rpm, :source => :rpm do
 
   # What's the latest package version available?
   def latest
-    upd = self.class.latest_package_version(@resource[:name], enablerepo, disablerepo, disableexcludes)
+    upd = self.class.latest_package_version(@resource[:name], disablerepo, enablerepo, disableexcludes)
     unless upd.nil?
       # FIXME: there could be more than one update for a package
       # because of multiarch

data/lib/puppet/provider/user/useradd.rb

@@ -203,7 +203,7 @@ Puppet::Type.type(:user).provide :useradd, :parent => Puppet::Provider::NameServ
   [:expiry, :password_min_age, :password_max_age, :password].each do |shadow_property|
     define_method(shadow_property) do
       if Puppet.features.libshadow?
-        if ent = Shadow::Passwd.getspnam(@resource.name)
+        if ent = Shadow::Passwd.getspnam(@canonical_name)
           method = self.class.option(shadow_property, :method)
           return unmunge(shadow_property, ent.send(method))
         end

data/lib/puppet/reference/configuration.rb

@@ -32,7 +32,7 @@ config = Puppet::Util::Reference.newreference(:configuration, :depth => 1, :doc
       val = 'Unix/Linux: /var/run/puppetlabs -- Windows: C:\ProgramData\PuppetLabs\puppet\var\run -- Non-root user: ~/.puppetlabs/var/run'
     elsif name.to_s == 'logdir'
       val = 'Unix/Linux: /var/log/puppetlabs/puppet -- Windows: C:\ProgramData\PuppetLabs\puppet\var\log -- Non-root user: ~/.puppetlabs/var/log'
-    elsif name.to_s == 'hiera.yaml'
+    elsif name.to_s == 'hiera_config'
       val = '$confdir/hiera.yaml. However, if a file exists at $codedir/hiera.yaml, Puppet uses that instead.'
     end
 

data/lib/puppet/resource.rb

@@ -168,25 +168,23 @@ class Puppet::Resource
   end
 
   def self.value_to_pson_data(value)
-    if value.is_a? Array
+    if value.is_a?(Array)
       value.map{|v| value_to_pson_data(v) }
-    elsif value.is_a? Puppet::Resource
+    elsif value.is_a?(Hash)
+      result = {}
+      value.each_pair { |k, v| result[value_to_pson_data(k)] = value_to_pson_data(v) }
+      result
+    elsif value.is_a?(Puppet::Resource)
       value.to_s
+    elsif value.is_a?(Symbol) && value == :undef
+      nil
     else
       value
     end
   end
 
   def yaml_property_munge(x)
-    case x
-    when Hash
-      x.inject({}) { |h,kv|
-        k,v = kv
-        h[k] = self.class.value_to_pson_data(v)
-        h
-      }
-    else self.class.value_to_pson_data(x)
-    end
+    self.value.to_pson_data(x)
   end
 
   YAML_ATTRIBUTES = [:@file, :@line, :@exported, :@type, :@title, :@tags, :@parameters]

data/lib/puppet/resource/type_collection.rb

@@ -231,6 +231,7 @@ class Puppet::Resource::TypeCollection
           debug_once "Not attempting to load #{type} #{fqname} as this object was missing during a prior compilation"
         end
       else
+        fqname = munge_name(fqname)
         result = loader.try_load_fqname(type, fqname)
         @notfound[ fqname ] = result.nil?
       end

data/lib/puppet/type/exec.rb

@@ -241,10 +241,14 @@ module Puppet
     end
 
     newparam(:refresh) do
-      desc "How to refresh this command.  By default, the exec is just
-        called again when it receives an event from another resource,
-        but this parameter allows you to define a different command
-        for refreshing."
+      desc "An alternate command to run when the `exec` receives a refresh event
+        from another resource. By default, Puppet runs the main command again.
+        For more details, see the notes about refresh behavior above, in the
+        description for this resource type.
+
+        Note that this alternate command runs with the same `provider`, `path`,
+        `user`, and `group` as the main command. If the `path` isn't set, you
+        must fully qualify the command's name."
 
       validate do |command|
         provider.validatecmd(command)
@@ -406,8 +410,10 @@ module Puppet
 
     newcheck(:unless) do
       desc <<-'EOT'
-        If this parameter is set, then this `exec` will run unless
-        the command has an exit code of 0.  For example:
+        A test command that checks the state of the target system and restricts
+        when the `exec` can run. If present, Puppet runs this test command
+        first, then runs the main command unless the test has an exit code of 0
+        (success). For example:
 
             exec { '/bin/echo root >> /usr/lib/cron/cron.allow':
               path   => '/usr/bin:/usr/sbin:/bin',
@@ -417,17 +423,16 @@ module Puppet
         This would add `root` to the cron.allow file (on Solaris) unless
         `grep` determines it's already there.
 
-        Note that this command follows the same rules as the main command,
-        such as which user and group it's run as.
-        This also means it must be fully qualified if the path is not set.
-        It also uses the same provider as the main command, so any behavior
-        that differs by provider will match.
+        Note that this test command runs with the same `provider`, `path`,
+        `user`, and `group` as the main command. If the `path` isn't set, you
+        must fully qualify the command's name.
 
-        Also note that unless can take an array as its value, e.g.:
+        This parameter can also take an array of commands. For example:
 
             unless => ['test -f /tmp/file1', 'test -f /tmp/file2'],
 
-        This will only run the exec if _all_ conditions in the array return false.
+        This `exec` would only run if every command in the array has a
+        non-zero exit code.
       EOT
 
       validate do |cmds|
@@ -457,28 +462,29 @@ module Puppet
 
     newcheck(:onlyif) do
       desc <<-'EOT'
-        If this parameter is set, then this `exec` will only run if
-        the command has an exit code of 0.  For example:
+        A test command that checks the state of the target system and restricts
+        when the `exec` can run. If present, Puppet runs this test command
+        first, and only runs the main command if the test has an exit code of 0
+        (success). For example:
 
             exec { 'logrotate':
-              path   => '/usr/bin:/usr/sbin:/bin',
-              onlyif => 'test `du /var/log/messages | cut -f1` -gt 100000',
+              path     => '/usr/bin:/usr/sbin:/bin',
+              provider => shell,
+              onlyif   => 'test `du /var/log/messages | cut -f1` -gt 100000',
             }
 
-        This would run `logrotate` only if that test returned true.
-
-        Note that this command follows the same rules as the main command,
-        such as which user and group it's run as.
-        This also means it must be fully qualified if the path is not set.
+        This would run `logrotate` only if that test returns true.
 
-        It also uses the same provider as the main command, so any behavior
-        that differs by provider will match.
+        Note that this test command runs with the same `provider`, `path`,
+        `user`, and `group` as the main command. If the `path` isn't set, you
+        must fully qualify the command's name.
 
-        Also note that onlyif can take an array as its value, e.g.:
+        This parameter can also take an array of commands. For example:
 
             onlyif => ['test -f /tmp/file1', 'test -f /tmp/file2'],
 
-        This will only run the exec if _all_ conditions in the array return true.
+        This `exec` would only run if every command in the array has an
+        exit code of 0 (success).
       EOT
 
       validate do |cmds|

data/lib/puppet/type/file/mode.rb

@@ -13,6 +13,10 @@ module Puppet
       notation. This value **must** be specified as a string; do not use
       un-quoted numbers to represent file modes.
 
+      If the mode is omitted (or explicitly set to `undef`), Puppet does not
+      enforce permissions on existing files and creates new files with
+      permissions of `0644`.
+
       The `file` type uses traditional Unix permission schemes and translates
       them to equivalent permissions for systems which represent permissions
       differently, including Windows. For detailed ACL controls on Windows,

data/lib/puppet/util/character_encoding.rb

@@ -2,94 +2,97 @@
 
 module Puppet::Util::CharacterEncoding
   class << self
-    # Warning! This is a destructive method - the string supplied is modified!
+    # Given a string, attempts to convert a copy of the string to UTF-8. Conversion uses
+    # encode - the string's internal byte representation is modifed to UTF-8.
+    #
+    # This method is intended for situations where we generally trust that the
+    # string's bytes are a faithful representation of the current encoding
+    # associated with it, and can use it as a starting point for transcoding
+    # (conversion) to UTF-8.
+    #
     # @api public
-    # @param [String] string a string to transcode / force_encode to utf-8
-    # @return [String] string if already utf-8, OR
-    #   the same string with external encoding set to utf-8 if bytes are valid utf-8 OR
-    #   the same string transcoded to utf-8 OR
-    #   nil upon a failure to legitimately set external encoding or transcode string
-    def convert_to_utf_8!(string)
-      currently_valid = string.valid_encoding?
-
+    # @param [String] string a string to transcode
+    # @return [String] copy of the original string, in UTF-8 if transcodable
+    def convert_to_utf_8(string)
+      original_encoding = string.encoding
+      string_copy = string.dup
       begin
-        if string.encoding == Encoding::UTF_8
-          if currently_valid
-            return string
-          else
-            # If a string is currently believed to be UTF-8, but is also not
-            # valid_encoding?, we have no recourse but to fail because we have no
-            # idea what encoding this string originally came from where it *was*
-            # valid - all we know is it's not currently valid UTF-8.
-            raise EncodingError
+        if original_encoding == Encoding::UTF_8
+          if !string_copy.valid_encoding?
+            Puppet.debug(_("%{value} is already labeled as UTF-8 but this encoding is invalid. It cannot be transcoded by Puppet.") %
+              { value: string.dump })
           end
-        elsif valid_utf_8_bytes?(string)
-          # Before we try to transcode the string, check if it is valid UTF-8 as
-          # currently constitued (in its non-UTF-8 encoding), and if it is, limit
-          # ourselves to setting the external encoding of the string to UTF-8
-          # rather than actually transcoding it. We do this to handle
-          # a couple scenarios:
-
-          # The first scenario is that the string was originally valid UTF-8 but
-          # the current puppet run is not in a UTF-8 environment. In this case,
-          # the string will likely have invalid byte sequences (i.e.,
-          # string.valid_encoding? == false), and attempting to transcode will
-          # fail with Encoding::InvalidByteSequenceError, referencing the
-          # invalid byte sequence in the original, pre-transcode, string. We
-          # might have gotten here, for example, if puppet is run first in a
-          # user context with UTF-8 encoding (setting the "is" value to UTF-8)
-          # and then later run via cron without UTF-8 specified, resulting in in
-          # EN_US (ISO-8859-1) on many systems. In this scenario we're
-          # effectively best-guessing this string originated as UTF-8 and only
-          # set external encoding to UTF-8 - transcoding would have failed
-          # anyway.
-
-          # The second scenario (more rare, I expect) is that this string does
-          # NOT have invalid byte sequences (string.valid_encoding? == true),
-          # but is *ALSO valid unicode*.
-          # Our example case is "\u16A0" - "RUNIC LETTER FEHU FEOH FE"
-          # http://www.fileformat.info/info/unicode/char/16A0/index.htm
-          # 0xE1 0x9A 0xA0 / 225 154 160
-          # These bytes are valid in ISO-8859-1 but the character they represent
-          # transcodes cleanly in ruby to *different* characters in UTF-8.
-          # That's not what we want if the user intended the original string as
-          # UTF-8. We can only guess, so if the string is valid UTF-8 as
-          # currently constituted, we default to assuming the string originated
-          # in UTF-8 and do not transcode it - we only set external encoding.
-          return string.force_encoding(Encoding::UTF_8)
-        elsif currently_valid
-          # If the string is not currently valid UTF-8 but it can be transcoded
-          # (it is valid in its current encoding), we can guess this string was
-          # not originally unicode. Transcode it to UTF-8. For strings with
-          # original encodings like SHIFT_JIS, this should be the final result.
-          return string.encode!(Encoding::UTF_8)
+          # String is aleady valid UTF-8 - noop
+          return string_copy
         else
-          # If the string is neither valid UTF-8 as-is nor valid in its current
-          # encoding, fail. It requires user remediation.
-          raise EncodingError
+          # If the string comes to us as BINARY encoded, we don't know what it
+          # started as. However, to encode! we need a starting place, and our
+          # best guess is whatever the system currently is (default_external).
+          # So set external_encoding to default_external before we try to
+          # transcode to UTF-8.
+          string_copy.force_encoding(Encoding.default_external) if original_encoding == Encoding::BINARY
+          return string_copy.encode(Encoding::UTF_8)
         end
       rescue EncodingError => detail
+        # Set the encoding on our copy back to its original if we modified it
+        string_copy.force_encoding(original_encoding) if original_encoding == Encoding::BINARY
+
         # Catch both our own self-determined failure to transcode as well as any
         # error on ruby's part, ie Encoding::UndefinedConversionError on a
         # failure to encode!.
-        Puppet.debug(_("%{error}: %{value} is not valid UTF-8 and cannot be transcoded by Puppet.") %
+        Puppet.debug(_("%{error}: %{value} cannot be transcoded by Puppet.") %
           { error: detail.inspect, value: string.dump })
-        return nil
+        return string_copy
       end
     end
 
-    private
+    # Given a string, tests if that string's bytes represent valid UTF-8, and if
+    # so return a copy of the string with external enocding set to UTF-8. Does
+    # not modify the byte representation of the string. If the string does not
+    # represent valid UTF-8, does not set the external encoding.
+    #
+    # This method is intended for situations where we do not believe that the
+    # encoding associated with a string is an accurate reflection of its actual
+    # bytes, i.e., effectively when we believe Ruby is incorrect in its
+    # assertion of the encoding of the string.
+    #
+    # @api public
+    # @param [String] string to set external encoding (re-label) to utf-8
+    # @return [String] a copy of string with external encoding set to utf-8, or
+    # a copy of the original string if override would result in invalid encoding.
+    def override_encoding_to_utf_8(string)
+      string_copy = string.dup
+      original_encoding = string_copy.encoding
+      return string_copy if original_encoding == Encoding::UTF_8
+      if string_copy.force_encoding(Encoding::UTF_8).valid_encoding?
+        return string_copy
+      else
+        Puppet.debug(_("%{value} is not valid UTF-8 and result of overriding encoding would be invalid.") % { value: string.dump })
+        # Set copy back to its original encoding before returning
+        return string_copy.force_encoding(original_encoding)
+      end
+    end
 
-    # Do our best to determine if a string is valid UTF-8 via String#valid_encoding? without permanently
-    # modifying or duplicating the string due to performance concerns
-    # @api private
-    # @param [String] string a string to test
-    # @return [Boolean] whether we think the string is UTF-8 or not
-    def valid_utf_8_bytes?(string)
-      original_encoding = string.encoding
-      valid = string.force_encoding(Encoding::UTF_8).valid_encoding?
-      string.force_encoding(original_encoding)
-      valid
+    REPLACEMENT_CHAR_MAP = {
+      Encoding::UTF_8 => "\uFFFD",
+      Encoding::UTF_16LE => "\xFD\xFF".force_encoding(Encoding::UTF_16LE),
+    }
+
+    # Given a string, return a copy of that string with any invalid byte
+    # sequences in its current encoding replaced with the replacement character
+    # "\uFFFD" (UTF-8) if the string is UTF-8 or UTF-16LE, or "?" otherwise.
+    # @param string a string to remove invalid byte sequences from
+    # @return a copy of string invalid byte sequences replaced by the unicode
+    #   replacement character or "?" character
+    # @note does not modify encoding, but new string will have different bytes
+    #   from original. Only needed for ruby 1.9.3 support.
+    def scrub(string)
+      if string.respond_to?(:scrub)
+        string.scrub
+      else
+        replacement_character = REPLACEMENT_CHAR_MAP[string.encoding] || '?'
+        string.chars.map { |c| c.valid_encoding? ? c : replacement_character }.join
+      end
     end
   end
 end