ndr_support 3.1.1

data/test/test_helper.rb

@@ -0,0 +1,41 @@
+require 'simplecov'
+SimpleCov.start
+
+require 'minitest/autorun'
+require 'minitest/unit'
+require 'mocha/mini_test'
+
+require 'active_record'
+require 'active_support/time'
+require 'ndr_support'
+require 'tmpdir'
+
+NdrSupport.apply_era_date_formats!
+
+# We do not use Rails' preferred time zone support, as this would
+# require all dates to be stored in UTC in the database.
+# Thus a birth date of 1975-06-01 would be stored as 1975-05-31 23.00.00.
+# Instead, we want to store all times in local time.
+ActiveRecord::Base.default_timezone = :local
+ActiveRecord::Base.time_zone_aware_attributes = false
+
+SafePath.configure! File.dirname(__FILE__) + '/resources/filesystem_paths.yml'
+
+# Borrowed from ActiveSupport::TestCase
+module Minitest
+  class Test
+    # Allow declarive test syntax:
+    def self.test(name, &block)
+      test_name = "test_#{name.gsub(/\s+/, '_')}".to_sym
+      defined = method_defined? test_name
+      fail "#{test_name} is already defined in #{self}" if defined
+      if block_given?
+        define_method(test_name, &block)
+      else
+        define_method(test_name) do
+          flunk "No implementation provided for #{name}"
+        end
+      end
+    end
+  end
+end

data/test/time/conversions_test.rb

@@ -0,0 +1,15 @@
+require 'test_helper'
+
+class Time::ConversionsTest < Minitest::Test
+  test 'to_time should return same object' do
+    yaml  = '2015-08-06 00:00:00 Z'
+
+    time  = yaml.to_time
+    time2 = time.to_time
+
+    assert Time === time, 'time was not a Time'
+    assert Time === time2, 'time2 was not a Time'
+
+    assert_equal time.object_id, time2.object_id
+  end
+end

data/test/utf8_encoding/control_characters_test.rb

@@ -0,0 +1,84 @@
+require 'test_helper'
+
+# Tests Utf8Encoding::ControlCharacters module.
+class ControlCharactersTest < Minitest::Test
+  include UTF8Encoding
+
+  test 'control char identification' do
+    (0..255).each do |code|
+      expected = code == 127 || (code < 32 && [9, 10, 13].exclude?(code)) ? 4 : 1
+      actual   = escape_control_chars(code.chr).length
+
+      assert_equal expected, actual, "unexpected escaping for value: #{code} (#{code.chr})"
+    end
+  end
+
+  test 'escape_control_chars with harmless string' do
+    string   = 'null \x00 characters suck'
+    expected = 'null \x00 characters suck'
+    actual   = escape_control_chars(string)
+
+    assert_equal expected, actual
+    refute actual.object_id == string.object_id, 'should not have modified in place'
+  end
+
+  test 'escape_control_chars! with harmless string' do
+    string   = 'null \x00 characters suck'
+    expected = 'null \x00 characters suck'
+    actual   = escape_control_chars!(string)
+
+    assert_equal expected, actual
+    assert_equal actual.object_id, string.object_id
+  end
+
+  test 'escape_control_chars with unprintable control characters' do
+    string   = "null \x00 \x7F characters suck"
+    expected = 'null 0x00 0x7f characters suck'
+    actual   = escape_control_chars(string)
+
+    assert_equal expected, actual
+    refute actual.object_id == string.object_id, 'should not have modified in place'
+  end
+
+  test 'escape_control_chars! with unprintable control characters' do
+    string   = "null \x00 characters suck"
+    expected = 'null 0x00 characters suck'
+    actual   = escape_control_chars!(string)
+
+    assert_equal expected, actual
+    assert_equal string.object_id, actual.object_id
+  end
+
+  test 'escape_control_chars! with printable control characters' do
+    string   = "null \x00 characters \r\n really \t suck \x07\x07\x07"
+    expected = "null 0x00 characters \r\n really \t suck 0x070x070x07" # ring ring ring
+
+    assert_equal expected, escape_control_chars!(string)
+  end
+
+  test 'escape_control_chars_in_object! with array' do
+    array    = %W( hello\tcruel \x00 world!\n \x07 )
+    expected = %W( hello\tcruel 0x00 world!\n 0x07 )
+    actual   = escape_control_chars_in_object!(array)
+
+    assert_equal expected, actual
+    assert_equal array.object_id, actual.object_id
+  end
+
+  test 'escape_control_chars_in_object! with hash' do
+    hash     = { :a => "hello\tcruel", :b => "\x00", :c => "world!\n", :d => "\x07" }
+    expected = { :a => "hello\tcruel", :b => '0x00', :c => "world!\n", :d => '0x07' }
+    actual   = escape_control_chars_in_object!(hash)
+
+    assert_equal expected, actual
+    assert_equal hash.object_id, actual.object_id
+  end
+
+  test 'escape_control_chars_in_object! with PORO' do
+    object  = Object.new
+    escaped = escape_control_chars_in_object!(object)
+
+    assert_equal object, escaped
+    assert_equal object.object_id, escaped.object_id
+  end
+end

data/test/utf8_encoding/force_binary_test.rb

@@ -0,0 +1,64 @@
+# encoding: UTF-8
+require 'test_helper'
+
+# Tests Utf8Encoding::ForceBinary module.
+class ForceBinaryTest < Minitest::Test
+  include UTF8Encoding
+
+  test 'binary_encode_any_high_ascii with low-ascii string' do
+    input = 'manana manana'
+
+    assert_equal 'UTF-8', input.encoding.name
+    assert input.valid_encoding?
+
+    output = binary_encode_any_high_ascii(input)
+
+    refute_equal input.object_id, output.object_id
+
+    assert_equal input.bytes.to_a, output.bytes.to_a
+    assert_equal 'UTF-8', output.encoding.name
+    assert output.valid_encoding?
+  end
+
+  test 'binary_encode_any_high_ascii with high-ascii string' do
+    input = 'mañana mañana'
+
+    assert_equal 'UTF-8', input.encoding.name
+    assert input.valid_encoding?
+
+    output = binary_encode_any_high_ascii(input)
+
+    refute_equal input.object_id, output.object_id
+
+    assert_equal input.bytes.to_a, output.bytes.to_a
+    assert_equal 'ASCII-8BIT', output.encoding.name
+    assert output.valid_encoding?
+  end
+
+  test 'binary_encode_any_high_ascii with array' do
+    input  = %w(mañana manana)
+    output = binary_encode_any_high_ascii(input)
+
+    refute_equal input.object_id, output.object_id
+
+    assert_equal %w(UTF-8 UTF-8), input.map { |str| str.encoding.name }
+    assert_equal %w(ASCII-8BIT UTF-8), output.map { |str| str.encoding.name }
+  end
+
+  test 'binary_encode_any_high_ascii with hash' do
+    input  = { :with => 'mañana', :without => 'manana' }
+    output = binary_encode_any_high_ascii(input)
+
+    refute_equal input.object_id, output.object_id
+
+    assert_equal 'ASCII-8BIT', output[:with].encoding.name
+    assert_equal 'UTF-8', output[:without].encoding.name
+  end
+
+  test 'binary_encode_any_high_ascii with other object' do
+    input  = /mañana mañana/
+    output = binary_encode_any_high_ascii(input)
+
+    assert_equal input.object_id, output.object_id, 'should have returned same object'
+  end
+end

data/test/utf8_encoding_test.rb

@@ -0,0 +1,170 @@
+# encoding: UTF-8
+
+require 'test_helper'
+
+class Utf8EncodingTest < Minitest::Test
+  extend UTF8Encoding
+  include UTF8Encoding
+
+  test 'ensure_utf8 should return a new string' do
+    string1 = 'hello'
+    string2 = ensure_utf8(string1)
+
+    refute string1.object_id == string2.object_id
+  end
+
+  test 'ensure_utf8! should return the same string' do
+    string1 = 'hello'
+    string2 = ensure_utf8!(string1)
+
+    assert string1.object_id == string2.object_id
+  end
+
+  test 'ensure_utf8_object! should work with arrays' do
+    array = []
+    expects(:ensure_utf8_array!).with(array).returns(array)
+    assert_equal array, ensure_utf8_object!(array)
+  end
+
+  test 'ensure_utf8_array! should work on elements' do
+    element1 = 'hello'
+    element2 = :world
+    array = [element1, element2]
+
+    expects(:ensure_utf8_object!).with(element1)
+    expects(:ensure_utf8_object!).with(element2)
+
+    assert_equal array, ensure_utf8_array!(array)
+  end
+
+  test 'ensure_utf8_object! should work with hashes' do
+    hash = {}
+    expects(:ensure_utf8_hash!).with(hash).returns(hash)
+    assert_equal hash, ensure_utf8_object!(hash)
+  end
+
+  test 'ensure_utf8_hash! should work on values' do
+    element1 = 'hello'
+    element2 = :world
+    hash = { element1 => element2 }
+
+    expects(:ensure_utf8_object!).with(element1).never
+    expects(:ensure_utf8_object!).with(element2)
+
+    assert_equal hash, ensure_utf8_hash!(hash)
+  end
+
+  test 'ensure_utf8_object! should work with strings' do
+    string = ''
+    expects(:ensure_utf8!).with(string).returns(string)
+    assert_equal string, ensure_utf8_object!(string)
+  end
+
+  test 'coerce_utf8 should return a new string' do
+    string1 = 'hello'
+    string2 = coerce_utf8(string1)
+
+    refute string1.object_id == string2.object_id
+  end
+
+  test 'coerce_utf8! should return the same string' do
+    string1 = 'hello'
+    string2 = coerce_utf8!(string1)
+
+    assert string1.object_id == string2.object_id
+  end
+
+  test 'ensure_utf8 should convert low bytes to UTF-8 if possible' do
+    string1 = 'hello'.force_encoding('Windows-1252')
+    string2 = ensure_utf8!(string1)
+
+    assert_equal string1, string2
+    assert_equal 'UTF-8', string2.encoding.name
+  end
+
+  test 'ensure_utf8 should convert high bytes to UTF-8 if possible' do
+    string1 = "dash \x96 dash".force_encoding('Windows-1252')
+    assert_equal 11, string1.bytes.to_a.length
+    assert_equal 11, string1.chars.to_a.length
+
+    assert string1.valid_encoding?
+
+    string2 = ensure_utf8(string1)
+    assert_equal 13, string2.bytes.to_a.length
+    assert_equal 11, string2.chars.to_a.length
+
+    assert_equal 'UTF-8', string2.encoding.name
+    assert string2.valid_encoding?
+  end
+
+  test 'ensure_utf8 should prefer a given encoding' do
+    string1 = "japan \x8e\xa6 ese"
+    assert_equal 12, string1.bytes.to_a.length
+    assert_equal 12, string1.chars.to_a.length
+
+    string2 = ensure_utf8(string1, 'EUC-JP')
+    assert_equal 13, string2.bytes.to_a.length
+    assert_equal 11, string2.chars.to_a.length
+
+    # "halfwidth katakana letter wo":
+    assert_equal [239, 189, 166], string2.bytes.to_a[6...9]
+
+    assert_equal 'UTF-8', string2.encoding.name
+    assert string2.valid_encoding?
+  end
+
+  test 'ensure_utf8 should handle UTF-16 strings (using bom)' do
+    alpha_beta_le = ensure_utf8("\xff\xfe\xb1\x03\xb2\x03") # Little endian
+    alpha_beta_be = ensure_utf8("\xfe\xff\x03\xb1\x03\xb2") # Big endian
+
+    assert_equal Encoding.find('UTF-8'), alpha_beta_le.encoding
+    assert_equal Encoding.find('UTF-8'), alpha_beta_be.encoding
+
+    assert_equal 2, alpha_beta_le.chars.to_a.length
+    assert_equal 2, alpha_beta_be.chars.to_a.length
+
+    assert_equal alpha_beta_be, alpha_beta_le
+  end
+
+  test 'EncodingError defined' do
+    assert defined?(EncodingError)
+  end
+
+  test 'UTF8CoercionError defined' do
+    assert defined?(UTF8Encoding::UTF8CoercionError)
+  end
+
+  test 'ensure_utf8 should fail if unable to derive encoding' do
+    assert_raises(UTF8Encoding::UTF8CoercionError) do
+      # Not going to work with UTF-8 or Windows-1252:
+      ensure_utf8("rubbish \x90 rubbish")
+    end
+  end
+
+  test 'coerce_utf8 should escape unmappable values' do
+    expected = 'rubbish 0x90 rubbish'
+    actual   = coerce_utf8("rubbish \x90 rubbish")
+
+    assert_equal expected, actual
+    assert actual.valid_encoding?
+    assert_equal Encoding.find('UTF-8'), actual.encoding
+  end
+
+  test 'coerce_utf8 should use given source encoding' do
+    input        = "maybe \xc0 rubbish"
+    win_expected = 'maybe À rubbish'
+    utf_expected = 'maybe 0xc0 rubbish'
+
+    win_actual = coerce_utf8(input, 'Windows-1252')
+
+    assert_equal win_expected, win_actual
+    assert win_actual.valid_encoding?
+    assert_equal Encoding.find('UTF-8'), win_actual.encoding
+
+    utf_actual = coerce_utf8(input, 'UTF-8')
+
+    assert_equal utf_expected, utf_actual
+    assert utf_actual.valid_encoding?
+    assert_equal Encoding.find('UTF-8'), utf_actual.encoding
+  end
+end

data/test/yaml/serialization_test.rb

@@ -0,0 +1,145 @@
+# encoding: UTF-8
+
+require 'test_helper'
+
+class SerializationTest < Minitest::Test
+  include NdrSupport::YAML::SerializationMigration
+
+  test 'should serialize then deserialize an object correctly' do
+    hash = { :a => 1 }
+    assert_equal hash, load_yaml(dump_yaml(hash))
+  end
+
+  test 'should handle syck-encoded characters' do
+    assert_syck_1_8_yaml_loads_correctly
+  end
+
+  test 'should handle binary yaml with control chars' do
+    # irb> "\xC2\xA1null \x00 characters \r\n suck!".to_yaml
+    yaml = "--- !binary |-\n  wqFudWxsIAAgY2hhcmFjdGVycyANCiBzdWNrIQ==\n"
+    assert_equal "¡null 0x00 characters \r\n suck!", load_yaml(yaml)
+
+    # irb> {fulltext: "\xC2\xA1null \x00 characters \r\n suck!"}.to_yaml
+    yamled_hash = "---\n:fulltext: !binary |-\n  wqFudWxsIAAgY2hhcmFjdGVycyANCiBzdWNrIQ==\n"
+    assert_equal({ :fulltext => "¡null 0x00 characters \r\n suck!" }, load_yaml(yamled_hash))
+  end
+
+  # Psych doesn't always base64-encode control characters:
+  test 'should handle non-binary yaml with control chars' do
+    #irb> Psych.dump("control \x01 char \n whoops!")
+    chr_1_yaml = "--- ! \"control \\x01 char \\n whoops!\"\n"
+    assert_equal "control 0x01 char \n whoops!", load_yaml(chr_1_yaml)
+  end
+
+  test 'load_yaml should not coerce to UTF-8 by default' do
+    assert_yaml_coercion_behaviour
+  end
+
+  test 'dump_yaml should produce encoding-portable YAML' do
+    original_object = { :basic => 'manana', :complex => 'mañana' }
+    yaml_produced   = dump_yaml(original_object)
+    reloaded_object = load_yaml(yaml_produced)
+
+    assert yaml_produced =~ /basic: manana/, 'binary-encoded more than was necessary'
+
+    refute yaml_produced.bytes.detect { |byte| byte > 127 }, 'yaml has high-ascii'
+    assert reloaded_object.inspect.bytes.detect { |byte| byte > 127 }
+    assert_equal original_object, reloaded_object
+  end
+
+  test 'encoding-portable YAML should be loadable' do
+    original_object = { :basic => 'manana', :complex => 'mañana' }
+    yaml_produced   = dump_yaml(original_object)
+
+    reloaded_object = load_yaml(yaml_produced)
+    assert_equal original_object, reloaded_object
+  end
+
+  test 'time-like objects should serialise correctly with psych' do
+    assert_timey_wimey_stuff
+  end
+
+  private
+
+  def assert_timey_wimey_stuff
+    assert_times
+    assert_dates
+    assert_datetimes
+    assert_datetimes_with_zones
+  end
+
+  def assert_times
+    # Dumped by 1.9.3 syck, within era.
+    loaded = YAML.load("--- !timestamp 2014-03-01\n")
+    assert [Date, Time].include?(loaded.class), '1.9.3 era timestamp class'
+    assert_equal 2014, loaded.year,  '1.9.3 era timestamp year'
+    assert_equal 3,    loaded.month, '1.9.3 era timestamp month'
+    assert_equal 1,    loaded.day,   '1.9.3 era timestamp day'
+  end
+
+  def assert_dates
+    date = Date.new(2014, 3, 1)
+
+    # Dumped by 1.8.7 syck, within era.
+    loaded = YAML.load("--- 2014-03-01\n")
+    assert_equal date, loaded, '1.8.7 era date'
+  end
+
+  def assert_datetimes
+    datetime = DateTime.new(2014, 3, 1, 12, 45, 15)
+    loaded   = YAML.load(datetime.to_yaml)
+
+    assert [DateTime, Time].include?(loaded.class), 'datetime class'
+    assert_equal datetime, loaded.to_datetime
+    assert_equal datetime.to_time, loaded.to_time
+  end
+
+  def assert_datetimes_with_zones
+    bst_datetime = DateTime.new(2014, 4, 1, 0, 0, 0, '+1')
+    bst_loaded   = load_yaml(bst_datetime.to_yaml)
+
+    assert [DateTime, Time].include?(bst_loaded.class), 'bst datetime class'
+    assert_equal bst_datetime, bst_loaded.to_datetime
+    assert_equal bst_datetime.to_time, bst_loaded.to_time
+
+    gmt_datetime = DateTime.new(2014, 3, 1, 0, 0, 0, '+0')
+    gmt_loaded   = load_yaml(gmt_datetime.to_yaml)
+
+    assert [DateTime, Time].include?(gmt_loaded.class), 'gmt datetime class'
+    assert_equal gmt_datetime, gmt_loaded.to_datetime
+    assert_equal gmt_datetime.to_time, gmt_loaded.to_time
+  end
+
+  def assert_syck_1_8_yaml_loads_correctly
+    yaml = "--- \nname: Dr. Doctor\000\000\000 \ndiagnosis: \"CIN 1 \\xE2\\x80\\x93 CIN 2\"\n"
+    hash = load_yaml(yaml)
+
+    # The null chars should be escaped:
+    assert_equal 'Dr. Doctor0x000x000x00', hash['name']
+
+    # The dash should be 3 bytes, but recognised as one char:
+    assert_equal 15, hash['diagnosis'].bytes.to_a.length
+
+    assert_syck_1_8_handles_encoding(hash)
+  end
+
+  def assert_syck_1_8_handles_encoding(hash)
+    assert_equal 13, hash['diagnosis'].chars.to_a.length
+
+    assert_equal 'UTF-8', hash['diagnosis'].encoding.name
+    assert hash['diagnosis'].valid_encoding?
+  end
+
+  def assert_yaml_coercion_behaviour
+    # UTF-8, with an unmappable byte too:
+    yaml = "---\nfulltextreport: \"Here is \\xE2\\x80\\x93 a weird \\x9D char\"\n"
+
+    # By default, we'd expect the (serialised) \x9D
+    assert_raises(UTF8Encoding::UTF8CoercionError) { load_yaml(yaml) }
+
+    # With the optional second argument, we can force an escape:
+    hash = load_yaml(yaml, true)
+    assert_equal 'Here is – a weird 0x9d char', hash['fulltextreport']
+    assert_equal 'UTF-8', hash['fulltextreport'].encoding.name
+  end
+end