ndr_support 5.9.6 → 5.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7447cd235fe9715cc6e6c868b0f0048c7ec516b70ea88e323dcdc1fd01e24f11
-  data.tar.gz: 16b4237428fdf1258327ebc86928c4f3cc8db292289ff8151db364f77c505439
+  metadata.gz: 452da481cc47dceb02b62e7eb832db07ba61985fb2cdc8267a40b6194d0ef108
+  data.tar.gz: e06b9a9845389b639c2fbe55aab1f4008d6dad69843a24113bf55b2b88fd7d56
 SHA512:
-  metadata.gz: 186d4b1f8f053e9fa2cba311d7f9760e496ea8597dbea5c7e3e1d34c62fe6237df8c51692ff40a18f98085b58dc9a98942a54b94854b0e32ea30e4033e9d9aa3
-  data.tar.gz: 68ada778e3d6d7b9187f5b95398f408ff08940e0c77f5019a99611279ff4e1bbb9a5bfacd32ed3c0082bfb607ecc93df158ef2399b414d53a0e0bc7c53c0727a
+  metadata.gz: 7d06a0fb2e4adae1cbc71b7e89fd4fc45b527a419ad313d6440dd199ddb845df637b4dd505dce4220dae16bc6f059e96a2faddc455c1901a54bd8acba86b4b3e
+  data.tar.gz: d5abcaa621cf18823123cd1241a519c2c2a9dd856ac6659a9f4d4497adcd11ec4b57beeade25c7e799b9924de02736344b91ef079b2bef5a65785a41a6d20333

data/CHANGELOG.md

@@ -1,6 +1,17 @@
 ## [Unreleased]
 * No unreleased changes
 
+## 5.10.0 / 2023-11-17
+## Changed
+* Generate UTF-8 encoded YAML by default. Disable with `utf8_storage = false`
+* Use `YAML.safe_load` by default. Override with
+  `self.yaml_safe_classes = yaml_safe_classes + [Klass1, Klass2]` and revert to
+  unsafe loading with `yaml_safe_classes = :unsafe` and `gem 'psych', '< 4'`
+
+## 5.9.7 / 2023-11-16
+## Fixed
+* YAMLSupport should preserve escaped backslashes in YAML text
+
 ## 5.9.6 / 2023-11-14
 ## Fixed
 * YAMLSupport should preserve escape sequences in JSON text

data/code_safety.yml

@@ -23,7 +23,7 @@ file safety:
   CHANGELOG.md:
     comments:
     reviewed_by: brian.shand
-    safe_revision: '09ad2e0c210d3e4377de68dac3d54932df6c1994'
+    safe_revision: 646eaebdf824490150e991225f9e15abb67dd4c1
   CODE_OF_CONDUCT.md:
     comments:
     reviewed_by: timgentry
@@ -171,7 +171,7 @@ file safety:
   lib/ndr_support/version.rb:
     comments:
     reviewed_by: brian.shand
-    safe_revision: 26236a310377036e1a27eb9534b4167430da6f3c
+    safe_revision: 765520ebaf3652bed7105995c815afe681dd5363
   lib/ndr_support/working_days.rb:
     comments:
     reviewed_by: josh.pencheon
@@ -179,7 +179,7 @@ file safety:
   lib/ndr_support/yaml/serialization_migration.rb:
     comments:
     reviewed_by: brian.shand
-    safe_revision: '09ad2e0c210d3e4377de68dac3d54932df6c1994'
+    safe_revision: 646eaebdf824490150e991225f9e15abb67dd4c1
   ndr_support.gemspec:
     comments:
     reviewed_by: brian.shand
@@ -283,4 +283,4 @@ file safety:
   test/yaml/serialization_test.rb:
     comments:
     reviewed_by: brian.shand
-    safe_revision: '09ad2e0c210d3e4377de68dac3d54932df6c1994'
+    safe_revision: 646eaebdf824490150e991225f9e15abb67dd4c1

data/lib/ndr_support/version.rb

@@ -3,5 +3,5 @@
 # This defines the NdrSupport version. If you change it, rebuild and commit the gem.
 # Use "rake build" to build the gem, see rake -T for all bundler rake tasks.
 module NdrSupport
-  VERSION = '5.9.6'
+  VERSION = '5.10.0'
 end

data/lib/ndr_support/yaml/serialization_migration.rb

@@ -12,8 +12,29 @@ module NdrSupport
       # accepted by load_yaml
       YAML_SAFE_CLASSES = [Date, DateTime, Time, Symbol].freeze
 
+      # Set list of YAML safe classes, or :unsafe to use unsafe load
+      def yaml_safe_classes=(yaml_safe_classes)
+        @yaml_safe_classes = yaml_safe_classes
+      end
+
+      def yaml_safe_classes
+        @yaml_safe_classes || YAML_SAFE_CLASSES
+      end
+
+      # Allow emitted YAML to contain UTF-8 characters
+      # Defaults to true. (Defaulted to false in ndr_support versions < 6)
+      def utf8_storage=(utf8_storage)
+        @utf8_storage = utf8_storage
+      end
+
+      def utf8_storage
+        return @utf8_storage if @utf8_storage == false
+
+        true # New ndr_support default for versions >= 6, previously false
+      end
+
       # Wrapper around: YAML.load(string)
-      def load_yaml(string, coerce_invalid_chars = false)
+      def load_yaml(string, coerce_invalid_chars = false) # rubocop:disable Style/OptionalBooleanParameter
         fix_encoding!(string, coerce_invalid_chars)
 
         # Achieve same behaviour using `syck` and `psych`:
@@ -21,10 +42,14 @@ module NdrSupport
         fix_encoding!(string, coerce_invalid_chars)
 
         # TODO: Bump NdrSupport major version, and switch to safe_load by default
-        object = if Psych::VERSION.start_with?('3.')
+        object = if yaml_safe_classes == :unsafe
+                   unless Psych::VERSION.start_with?('3.')
+                     raise(SecurityError, 'Unsafe YAML no longer supported')
+                   end
+
                    Psych.load(string)
                  else
-                   Psych.safe_load(string, permitted_classes: YAML_SAFE_CLASSES)
+                   Psych.safe_load(string, permitted_classes: yaml_safe_classes)
                  end
 
         # Ensure that any string related to the object
@@ -37,8 +62,10 @@ module NdrSupport
 
       # Wrapper around: YAML.dump(object)
       def dump_yaml(object)
-        # Psych produces UTF-8 encoded output; we'd rather
-        # have YAML that can be safely stored in stores with
+        return Psych.dump(object) if utf8_storage
+
+        # Psych produces UTF-8 encoded output; historically we
+        # preferred YAML that can be safely stored in stores with
         # other encodings. If #load_yaml is used, the binary
         # encoding of the object will be reversed on load.
         Psych.dump binary_encode_any_high_ascii(object)
@@ -56,14 +83,17 @@ module NdrSupport
       # Within double quotes, YAML allows special characters.
       # While `psych` emits UTF-8 YAML, `syck` double escapes
       # higher characters. We need to unescape any we find:
+      # Both `psych` and `syck` escape lower control characters.
       def handle_special_characters!(string, coerce_invalid_chars)
-        # TODO: Change to only handle syck control characters
         return unless string.start_with?('---') # Only handle YAML that is not JSON
 
         # Replace any encoded hex chars with their actual value:
-        string.gsub!(/((?:\\x[0-9A-F]{2})+)/) do
-          byte_sequence = $1.scan(/[0-9A-F]{2}/)
-          byte_sequence.pack('H2' * byte_sequence.length).tap do |sequence|
+        string.gsub!(/(?<!\\)((?:\\\\)*)((?:\\x[0-9A-F]{2})+)/) do
+          # We use negative lookbehind and the first capturing group to skip over
+          # properly escaped backslashes
+          prefix = ::Regexp.last_match(1) # Prefix is an even number of backslashes
+          byte_sequence = ::Regexp.last_match(2).scan(/[0-9A-F]{2}/)
+          prefix + byte_sequence.pack('H2' * byte_sequence.length).tap do |sequence|
             fix_encoding!(sequence, coerce_invalid_chars)
           end
         end

data/test/yaml/serialization_test.rb

@@ -29,8 +29,14 @@ class SerializationTest < Minitest::Test
     assert_equal "control 0x01 char \n whoops!", load_yaml(chr_1_yaml)
   end
 
+  test 'should handle non-binary yaml with escaped things that look like control chars' do
+    # irb> Psych.dump(['\\x01 \\xAF \\\\xAF', "\x01 \\\x01 \x01\\\x01"])
+    escaped_yaml = "---\n- \"\\\\x01 \\\\xAF \\\\\\\\xAF\"\n- \"\\x01 \\\\\\x01 \\x01\\\\\\x01\"\n"
+    assert_equal ['\\x01 \\xAF \\\\xAF', '0x01 \\0x01 0x01\\0x01'], load_yaml(escaped_yaml)
+  end
+
   test 'should leave non-binary JSON with things that look like control chars unchanged' do
-    hash = { 'report' => ' \x09 ' }
+    hash = { 'report' => ' \x01 ' }
     assert_equal hash, load_yaml(hash.to_json)
   end
 
@@ -38,26 +44,60 @@ class SerializationTest < Minitest::Test
     assert_yaml_coercion_behaviour
   end
 
-  test 'dump_yaml should produce encoding-portable YAML' do
-    original_object = { :basic => 'manana', :complex => 'mañana' }
+  test 'dump_yaml with utf8_storage = false should produce encoding-portable YAML' do
+    self.utf8_storage = false
+    original_object = { basic: 'manana', complex: 'mañana' }
     yaml_produced   = dump_yaml(original_object)
     reloaded_object = load_yaml(yaml_produced)
 
-    assert yaml_produced =~ /basic: manana/, 'binary-encoded more than was necessary'
+    assert_match(/basic: manana/, yaml_produced, 'binary-encoded more than was necessary')
 
     refute yaml_produced.bytes.detect { |byte| byte > 127 }, 'yaml has high-ascii'
     assert reloaded_object.inspect.bytes.detect { |byte| byte > 127 }
     assert_equal original_object, reloaded_object
   end
 
-  test 'encoding-portable YAML should be loadable' do
-    original_object = { :basic => 'manana', :complex => 'mañana' }
+  test 'encoding-portable YAML with utf8_storage = false should be loadable' do
+    self.utf8_storage = false
+    original_object = { basic: 'manana', complex: 'mañana' }
+    yaml_produced   = dump_yaml(original_object)
+
+    assert_equal("---\n:basic: manana\n:complex: !binary |-\n  bWHDsWFuYQ==\n", yaml_produced)
+
+    reloaded_object = load_yaml(yaml_produced)
+    assert_equal original_object, reloaded_object
+  end
+
+  test 'non-encoding-portable YAML with utf8_storage = true should be loadable' do
+    self.utf8_storage = true
+    original_object = { basic: 'manana', complex: 'mañana' }
     yaml_produced   = dump_yaml(original_object)
+    assert_equal("---\n:basic: manana\n:complex: mañana\n", yaml_produced)
 
     reloaded_object = load_yaml(yaml_produced)
     assert_equal original_object, reloaded_object
   end
 
+  test 'yaml_safe_classes should filter which classes can be loaded' do
+    original_object = { basic: 'manana', complex: 'mañana' }
+    yaml_produced   = dump_yaml(original_object)
+    self.yaml_safe_classes = []
+    assert_raises Psych::DisallowedClass, 'Load should fail without Symbol in yaml_safe_classes' do
+      load_yaml(yaml_produced)
+    end
+
+    self.yaml_safe_classes = [Symbol]
+    reloaded_object = load_yaml(yaml_produced)
+    assert_equal original_object, reloaded_object, 'Safe reload with Symbol class specified'
+
+    if Psych::VERSION.start_with?('3.')
+      # Not supported with Ruby >= 3.1 unless you force psych version < 4
+      self.yaml_safe_classes = :unsafe
+      reloaded_object = load_yaml(yaml_produced)
+      assert_equal original_object, reloaded_object, 'Unsafe reload with Symbol class'
+    end
+  end
+
   test 'time-like objects should serialise correctly with psych' do
     assert_timey_wimey_stuff
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_support
 version: !ruby/object:Gem::Version
-  version: 5.9.6
+  version: 5.10.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-14 00:00:00.000000000 Z
+date: 2023-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord