ndr_support 3.1.1

data/lib/ndr_support/string/cleaning.rb

@@ -0,0 +1,136 @@
+class String
+  INVALID_CONTROL_CHARS = /[\x00-\x08\x0b-\x0c\x0e-\x1f]/
+  ROMAN_ONE_TO_FIVE_MAPPING = { 'I' => '1', 'II' => '2', 'III' => '3', 'IIII' => '4', 'IV' => '4', 'V' => '5' }
+
+  # Used for comparing addresses
+  def squash
+    upcase.delete('^A-Z0-9')
+  end
+
+  # Show postcode in various formats.
+  # Parameter "option" can be :user, :compact, :db
+  def postcodeize(option = :user)
+    nspce = delete(' ').upcase
+    unless nspce.blank? || /([A-Z][0-9]|[A-Z][0-9][0-9]|[A-Z][0-9][A-Z]|[A-Z][A-Z][0-9]|[A-Z][A-Z][0-9][0-9]|[A-Z][A-Z][0-9][A-Z])[0-9][A-Z][A-Z]$/ =~ nspce
+      return self  # Don't change old-style or malformed postcodes
+    end
+    case option
+    when :compact
+      nspce
+    when :db
+      case nspce.length
+      when 5 then nspce.insert(-4, '  ')
+      when 6 then nspce.insert(-4, ' ')
+      else nspce
+      end
+    else # anything else, including :user --> friendly format
+      nspce.length < 5 ? nspce : nspce.insert(-4, ' ')
+    end
+  end
+
+  def clean(what)
+    case what
+    when :nhsnumber
+      self.delete('^0-9')[0..9]
+    when :postcode, :get_postcode
+      self.postcodeize(:db)
+    when :lpi
+      self.upcase.delete('^0-9A-Z')
+    when :sex
+      # SECURE: BNS 2012-10-09: But may behave oddly for multi-line input
+      if self =~ /^M|1/i
+        '1'
+      elsif self =~ /^F|2/i
+        '2'
+      else
+        '0'
+      end
+    when :sex_c
+      if self =~ /^M|1/i
+        'M'
+      elsif self =~ /^F|2/i
+        'F'
+      else
+        ''
+      end
+    when :name
+      substitutions = {
+        '.'      => '',
+        /,|;/    => ' ',
+        /\s{2,}/ => ' ',
+        '`'      => '\''
+      }
+      substitutions.inject(self.upcase) { |str, scheme| str.gsub(*scheme) }.strip
+    when :ethniccategory
+      replace_ethniccategory = {
+        '0' => '0',
+        '1' => 'M',
+        '2' => 'N',
+        '3' => 'H',
+        '4' => 'J',
+        '5' => 'K',
+        '6' => 'R',
+        '7' => '8',
+        '&' => 'X',
+        ' ' => 'X',
+        '99' => 'X'
+      }
+      replace_ethniccategory[self] || self.upcase
+    when :code
+      self.split(/ |,|;/).map do |code|
+        code.blank? ? next : code.gsub('.', '')
+      end.compact.join(' ')
+    when :code_icd
+      # regexp = /[A-Z][0-9]{2}(\.(X|[0-9]{1,2})|[0-9]?)( *(D|A)( |,|;|$))/
+      codes = self.upcase.split(/ |,|;/).delete_if { |x| x.squash.blank? }
+      cleaned_codes = []
+      codes.each do |code|
+        if code == 'D' || code == 'A'
+          cleaned_codes[-1] += code
+        else
+          cleaned_codes << code
+        end
+      end
+      cleaned_codes.join(' ')
+    when :code_opcs
+      clean_code_opcs
+    when :hospitalnumber
+      self[-1..-1] =~ /\d/ ? self : self[0..-2]
+    when :xmlsafe, :make_xml_safe
+      self.strip_xml_unsafe_characters
+    when :roman5
+      # This deromanises roman numerals between 1 and 5
+      self.gsub(/[IV]+/i) { |match| ROMAN_ONE_TO_FIVE_MAPPING[match.upcase] }
+    when :tnmcategory
+      self.sub!(/\A[tnm]/i, '')
+      if self =~ /\Ax\z/i
+        self.upcase
+      else
+        self.downcase
+      end
+    else
+      self.gsub(' ?', ' ')
+    end
+  end
+
+  def strip_xml_unsafe_characters
+    self.gsub(String::INVALID_CONTROL_CHARS, '')
+  end
+
+  def xml_unsafe?
+    self =~ String::INVALID_CONTROL_CHARS
+  end
+
+  private
+
+  def clean_code_opcs
+    split(/ |,|;/).map do |code|
+      db_code = code.squash
+      if 4 == db_code.length || db_code =~ /CZ00[12]/
+        db_code
+      else
+        next
+      end
+    end.compact.join(' ')
+  end
+end

data/lib/ndr_support/string/conversions.rb

@@ -0,0 +1,137 @@
+# encoding: utf-8
+require 'active_support/core_ext/string/conversions'
+require 'ndr_support/daterange'
+require 'ndr_support/ourdate'
+require 'ndr_support/ourtime'
+
+# Forward-port ParseDate to Ruby 1.9.x and beyond.
+# We only use this in String#to_date, but keep the logic
+# encapsulated for testing purposes - the behaviour of
+# Date._parse has been known to change.
+unless defined?(::ParseDate)
+  class ParseDate
+    def self.parsedate(str, comp = false)
+      Date._parse(str, comp).
+        values_at(:year, :mon, :mday, :hour, :min, :sec, :zone, :wday)
+    end
+  end
+end
+
+class String
+  SOUNDEX_CHARS = 'BPFVCSKGJQXZDTLMNR'
+  SOUNDEX_NUMS  = '111122222222334556'
+  SOUNDEX_CHARS_EX = '^' + SOUNDEX_CHARS
+  SOUNDEX_CHARS_DEL = '^A-Z'
+
+  # desc: http://en.wikipedia.org/wiki/Soundex
+  def soundex(census = true)
+    str = upcase.delete(SOUNDEX_CHARS_DEL).squeeze
+
+    str[0..0] + str[1..-1].
+      delete(SOUNDEX_CHARS_EX).
+      tr(SOUNDEX_CHARS, SOUNDEX_NUMS)[0..(census ? 2 : -1)].
+      squeeze[0..(census ? 2 : -1)].
+      ljust(3, '0') rescue ''
+  end
+
+  def sounds_like(other)
+    soundex == other.soundex
+  end
+
+  def date1
+    Daterange.new(self).date1
+  end
+
+  def date2
+    Daterange.new(self).date2
+  end
+
+  def thedate
+    Ourdate.new(self).thedate
+  end
+
+  def thetime
+    Ourtime.new(self).thetime
+  end
+
+  # Convert "SMITH JD" into "Smith JD"
+  def surname_and_initials
+    a = split
+    initials = a.pop
+    a.collect(&:capitalize).join(' ') + ' ' + initials
+  end
+
+  # Like titleize but copes with Scottish and Irish names.
+  def surnameize
+    s = slice(0, 2).upcase
+    if s == 'MC' || s == "O'"
+      s.titleize + slice(2..-1).titleize
+    else
+      titleize
+    end
+  end
+
+  # Show NHS numbers with spaces
+  def nhs_numberize
+    return self unless length == 10
+    self[0..2] + ' ' + self[3..5] + ' ' + self[6..9]
+  end
+
+  # truncate a string, with a HTML … at the end
+  def truncate_hellip(n)
+    length > n ? slice(0, n - 1) + '…' : self
+  end
+
+  # Try to convert the string value into a date.
+  # If given a pattern, use it to parse date, otherwise use default setting to parse it
+  def to_date(pattern = nil)
+    return ''  if empty? # TODO: check if this is used... :/
+    return nil if blank?
+
+    pattern = '%d%m%Y' if 'ddmmyyyy' == pattern
+
+    if pattern == 'yyyymmdd' || pattern == 'yyyymmdd_ons'
+      # Workaround for ONS dates (with missing day / month): revert to old
+      # parsing behaviour. (Instead, EDeathRecord should substitute a Daterange)
+      # TODO: Move all death parsing to format 'yyyymmdd_ons'
+      return nil if self =~ /\A([0-9]{4}00[0-9]{2}|[0-9]{6}00)\Z/
+      pattern = '%Y%m%d'
+    end
+
+    if self =~ /\A([0-9][0-9]?)[.]([0-9][0-9]?)[.]([0-9][0-9][0-9][0-9])\Z/ # dd.mm.yyyy
+      return date1 # Uses Daterange to consistently parse our displayed date format
+    end
+
+    if pattern.to_s.include?('%')
+      # Use Date.strptime if the pattern contains a percent sign
+      parsed_date = DateTime.strptime(self, pattern)
+      Ourdate.build_datetime(parsed_date.year, parsed_date.month, parsed_date.day)
+    else
+      # Use '.' rather than '/' as a separator for more consistent parsing:
+      year, month, day, *_ = ParseDate.parsedate(gsub('/', '.'))
+
+      if ['yyyy/dd/mm', 'mm/dd/yyyy'].include?(pattern)
+        month, day = day, month
+      elsif 8 == length && self !~ /\d{8}/
+        # dd/mm/yy, rather than yyyymmdd
+        year, day = day, year
+        year += 100 if year <= Ourdate.today.year % 100
+        year += 1900
+      elsif 9 == length
+        # dd/mmm/yy, rare case.
+        year += 100 if year <= Ourdate.today.year % 100
+        year += 1900
+      end
+
+      Ourdate.build_datetime(year, month, day)
+    end
+  end
+
+  # Try to convert the string value into boolean
+  def to_boolean
+    # SECURE: BNS 2012-10-09: But may behave oddly for multi-line input
+    return true if self == true || self =~ (/^(true|t|yes|y|1)$/i)
+    return false if self == false || self.nil? || self =~ (/^(false|f|no|n|0)$/i)
+    fail ArgumentError, "invalid value for Boolean: \"#{self}\""
+  end
+end

data/lib/ndr_support/tasks.rb

@@ -0,0 +1 @@
+load 'tasks/audit_code.rake'

data/lib/ndr_support/time/conversions.rb

@@ -0,0 +1,13 @@
+class Time
+  # Ruby 1.9 defines Time#to_time natively (as part of the
+  # stdlib Time, rather than core Time), but it returns
+  # the time in the local timezone. ActiveSupport contains
+  # the following definition, but it is only actually used
+  # by Ruby 1.8.7. We wish to continue with that behaviour,
+  # as local time zones have caused problems with our
+  # Time#to_s format (which either formats as '%d.%m.%Y %H:%M'
+  # or '%d.%m.%Y').
+  def to_time
+    self
+  end
+end

data/lib/ndr_support/utf8_encoding.rb

@@ -0,0 +1,72 @@
+require 'ndr_support/utf8_encoding/control_characters'
+require 'ndr_support/utf8_encoding/force_binary'
+require 'ndr_support/utf8_encoding/object_support'
+
+# Provides encoding support to be used for file / rawtext handling.
+module UTF8Encoding
+  include ControlCharacters
+  include ForceBinary
+  include ObjectSupport
+
+  # Raised when we cannot ensure a string is valid UTF-8
+  class UTF8CoercionError < EncodingError; end
+
+  # Our known source encodings, in order of preference:
+  AUTO_ENCODINGS = %w( UTF-8 UTF-16 Windows-1252 )
+  # How should unmappable characters be escaped, when forcing encoding?
+  REPLACEMENT_SCHEME = lambda { |char| '0x' + char.ord.to_s(16).rjust(2, '0') }
+
+  # Returns a new string with valid UTF-8 encoding,
+  # or raises an exception if encoding fails.
+  def ensure_utf8(string, source_encoding = nil)
+    ensure_utf8!(string.dup, source_encoding)
+  end
+
+  # Attempts to encode `string` to UTF-8, in place.
+  # Returns `string`, or raises an exception.
+  def ensure_utf8!(string, source_encoding = nil)
+    # A list of encodings we should try from:
+    candidates = source_encoding ? Array.wrap(source_encoding) : AUTO_ENCODINGS
+
+    # Attempt to coerce the string to UTF-8, from one of the source
+    # candidates (in order of preference):
+    apply_candidates!(string, candidates)
+
+    unless string.valid_encoding?
+      # None of our candidate source encodings worked, so fail:
+      fail(UTF8CoercionError, "Attempted to use: #{candidates}")
+    end
+
+    string
+  end
+
+  # Returns a UTF-8 version of `string`, escaping any unmappable characters.
+  def coerce_utf8(string, source_encoding = nil)
+    coerce_utf8!(string.dup, source_encoding)
+  end
+
+  # Coerces `string` to UTF-8, in place, escaping any unmappable characters.
+  def coerce_utf8!(string, source_encoding = nil)
+    # Try normally first...
+    ensure_utf8!(string, source_encoding)
+  rescue UTF8CoercionError
+    # ...before going back-to-basics, and replacing things that don't map:
+    string.encode!('UTF-8', 'BINARY', :fallback => REPLACEMENT_SCHEME)
+  end
+
+  private
+
+  def apply_candidates!(string, candidates)
+    candidates.detect do |encoding|
+      begin
+        # Attempt to encode as UTF-8 from source `encoding`:
+        string.encode!('UTF-8', encoding)
+        # If that worked, we're done; otherwise, move on.
+        string.valid_encoding?
+      rescue EncodingError
+        # If that failed really badly, move on:
+        false
+      end
+    end
+  end
+end

data/lib/ndr_support/utf8_encoding/control_characters.rb

@@ -0,0 +1,53 @@
+require 'ndr_support/utf8_encoding'
+
+module UTF8Encoding
+  # Allows any supported object to have control characters
+  # escaped, using standard replacement scheme.
+  module ControlCharacters
+    # The range of characters we consider:
+    CONTROL_CHARACTERS = /[\x00-\x1f]|\x7f/
+    # Exceptions that are allowed:
+    ALLOWED_CONTROL_CHARACTERS = %W( \x09 \x0a \x0d )
+
+    # Recursively escape any control characters in `object`.
+    def escape_control_chars_in_object!(object)
+      case object
+      when String
+        escape_control_chars!(object)
+      when Hash
+        escape_control_chars_in_hash!(object)
+      when Array
+        escape_control_chars_in_array!(object)
+      else
+        object
+      end
+    end
+
+    # Returns a copy of `string`, with any control characters escaped.
+    def escape_control_chars(string)
+      escape_control_chars!(string.dup)
+    end
+
+    # Escapes in-place any control characters in `string`, before returning it.
+    def escape_control_chars!(string)
+      string.gsub!(CONTROL_CHARACTERS) do |character|
+        if ALLOWED_CONTROL_CHARACTERS.include?(character)
+          character
+        else
+          UTF8Encoding::REPLACEMENT_SCHEME[character]
+        end
+      end
+      string
+    end
+
+    # Escape control characters in values of the given `hash`.
+    def escape_control_chars_in_hash!(hash)
+      hash.each_value { |value| escape_control_chars_in_object!(value) }
+    end
+
+    # Escape control characters in elements of the given `array`.
+    def escape_control_chars_in_array!(array)
+      array.each { |element| escape_control_chars_in_object!(element) }
+    end
+  end
+end

data/lib/ndr_support/utf8_encoding/force_binary.rb

@@ -0,0 +1,44 @@
+require 'ndr_support/utf8_encoding'
+
+module UTF8Encoding
+  # Allows any supported object to have any high-ascii string
+  # content to be force-encoded from UTF-8 to BINARY (/ASCII-8BIT).
+  # This ensures that any serialisation to YAML, using Psych,
+  # can be stored in other encodings. (Psych by default emits
+  # UTF-8 YAML, which might not survive being stored in a Windows-1252
+  # database, for example.)
+  module ForceBinary
+    # Recursively ensure the correct encoding is being used:
+    def binary_encode_any_high_ascii(object)
+      case object
+      when String
+        binary_encode_if_any_high_ascii(object)
+      when Hash
+        binary_encode_any_high_ascii_in_hash(object)
+      when Array
+        binary_encode_any_high_ascii_in_array(object)
+      else
+        object
+      end
+    end
+
+    private
+
+    # Returns a BINARY-encoded version of `string`, if is cannot be represented as 7bit ASCII.
+    def binary_encode_if_any_high_ascii(string)
+      string = ensure_utf8(string)
+      string.force_encoding('BINARY') if string.bytes.detect { |byte| byte > 127 }
+      string
+    end
+
+    # Ensures all values of the given `hash` are BINARY-encoded, if necessary.
+    def binary_encode_any_high_ascii_in_hash(hash)
+      Hash[hash.map { |key, value| [key, binary_encode_any_high_ascii(value)] }]
+    end
+
+    # Ensures all values of the given `array` are BINARY-encoded, if necessary.
+    def binary_encode_any_high_ascii_in_array(array)
+      array.map { |element| binary_encode_any_high_ascii(element) }
+    end
+  end
+end