htmlentities 3.0.1 → 4.0.0

data/{CHANGES → History.txt}

@@ -1,3 +1,18 @@
+== 4.0.0 (2007-03-15)
+* New instantiation-based interface (but legacy interface is preserved for
+  compatibility.
+* Handles HTML4 as well as XHTML1 (the former lacks the ' entity).
+* Encodes basic entities numerically when :basic isn't specified and :decimal
+  or :hexadecimal is.
+* Performs a maximum of two gsub passes instead of three when encoding, which
+  should be more efficient on long strings.
+
+== 3.1.0 (2007-01-19)
+* Now understands all the entities referred to in the XHTML 1.0 DTD (253 
+  entities compared with 131 in version 3.0.1).
+* Calls to_s on parameters to play nicely with Rails 1.2.1.
+* Entity mapping data is now lazily loaded.
+
 == 3.0.1 (2005-04-08)
 * Improved documentation.
 

data/README.txt

@@ -0,0 +1,44 @@
+== HTMLEntities
+
+HTML entity encoding and decoding for Ruby
+
+The HTMLEntities module facilitates encoding and decoding of
+(X)HTML entities from/to their corresponding UTF-8 codepoints.
+
+To install (requires root/admin privileges):
+
+ ruby setup.rb
+
+Alternatively, you can just use the gem.
+
+== Licence
+
+This code is free to use under the terms of the MIT licence:
+
+  Copyright (c) 2007 Paul Battley
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to
+  deal in the Software without restriction, including without limitation the
+  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+  sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+  IN THE SOFTWARE.
+
+If you'd like to negotiate a different licence for a specific use, just
+contact me -- I'll almost certainly permit it.
+
+== Contact
+
+Comments are welcome. Send an email to pbattley@gmail.com.
+

data/lib/htmlentities.rb

@@ -1,102 +1,47 @@
+require 'htmlentities/legacy'
+
 #
 # HTML entity encoding and decoding for Ruby
 #
 
-module HTMLEntities
-  
-  class InstructionError < RuntimeError
-  end
-
-  module Data #:nodoc:
-
-    #
-    # MAP is a hash of all the HTML entities I could discover, as taken
-    # from the w3schools page on the subject:
-    # http://www.w3schools.com/html/html_entitiesref.asp
-    # The format is 'entity name' => codepoint where entity name is given
-    # without the surrounding ampersand and semicolon.
-    #
-    MAP = {
-      'quot'      => 34,        'apos'      => 39,        'amp'       => 38,
-      'lt'        => 60,        'gt'        => 62,        'nbsp'      => 160,
-      'iexcl'     => 161,       'curren'    => 164,       'cent'      => 162,
-      'pound'     => 163,       'yen'       => 165,       'brvbar'    => 166,
-      'sect'      => 167,       'uml'       => 168,       'copy'      => 169,
-      'ordf'      => 170,       'laquo'     => 171,       'not'       => 172,
-      'shy'       => 173,       'reg'       => 174,       'trade'     => 8482,
-      'macr'      => 175,       'deg'       => 176,       'plusmn'    => 177,
-      'sup2'      => 178,       'sup3'      => 179,       'acute'     => 180,
-      'micro'     => 181,       'para'      => 182,       'middot'    => 183,
-      'cedil'     => 184,       'sup1'      => 185,       'ordm'      => 186,
-      'raquo'     => 187,       'frac14'    => 188,       'frac12'    => 189,
-      'frac34'    => 190,       'iquest'    => 191,       'times'     => 215,
-      'divide'    => 247,       'Agrave'    => 192,       'Aacute'    => 193,
-      'Acirc'     => 194,       'Atilde'    => 195,       'Auml'      => 196,
-      'Aring'     => 197,       'AElig'     => 198,       'Ccedil'    => 199,
-      'Egrave'    => 200,       'Eacute'    => 201,       'Ecirc'     => 202,
-      'Euml'      => 203,       'Igrave'    => 204,       'Iacute'    => 205,
-      'Icirc'     => 206,       'Iuml'      => 207,       'ETH'       => 208,
-      'Ntilde'    => 209,       'Ograve'    => 210,       'Oacute'    => 211,
-      'Ocirc'     => 212,       'Otilde'    => 213,       'Ouml'      => 214,
-      'Oslash'    => 216,       'Ugrave'    => 217,       'Uacute'    => 218,
-      'Ucirc'     => 219,       'Uuml'      => 220,       'Yacute'    => 221,
-      'THORN'     => 222,       'szlig'     => 223,       'agrave'    => 224,
-      'aacute'    => 225,       'acirc'     => 226,       'atilde'    => 227,
-      'auml'      => 228,       'aring'     => 229,       'aelig'     => 230,
-      'ccedil'    => 231,       'egrave'    => 232,       'eacute'    => 233,
-      'ecirc'     => 234,       'euml'      => 235,       'igrave'    => 236,
-      'iacute'    => 237,       'icirc'     => 238,       'iuml'      => 239,
-      'eth'       => 240,       'ntilde'    => 241,       'ograve'    => 242,
-      'oacute'    => 243,       'ocirc'     => 244,       'otilde'    => 245,
-      'ouml'      => 246,       'oslash'    => 248,       'ugrave'    => 249,
-      'uacute'    => 250,       'ucirc'     => 251,       'uuml'      => 252,
-      'yacute'    => 253,       'thorn'     => 254,       'yuml'      => 255,
-      'OElig'     => 338,       'oelig'     => 339,       'Scaron'    => 352,
-      'scaron'    => 353,       'Yuml'      => 376,       'circ'      => 710,
-      'tilde'     => 732,       'ensp'      => 8194,      'emsp'      => 8195,
-      'thinsp'    => 8201,      'zwnj'      => 8204,      'zwj'       => 8205,
-      'lrm'       => 8206,      'rlm'       => 8207,      'ndash'     => 8211,
-      'mdash'     => 8212,      'lsquo'     => 8216,      'rsquo'     => 8217,
-      'sbquo'     => 8218,      'ldquo'     => 8220,      'rdquo'     => 8221,
-      'bdquo'     => 8222,      'dagger'    => 8224,      'Dagger'    => 8225,
-      'hellip'    => 8230,      'permil'    => 8240,      'lsaquo'    => 8249,
-      'rsaquo'    => 8250,      'euro'      => 8364 
-    }
+class HTMLEntities
 
-    MIN_LENGTH = MAP.keys.map{ |a| a.length }.min
-    MAX_LENGTH = MAP.keys.map{ |a| a.length }.max
-    NAMED_ENTITY_REGEXP = /&([a-z]{#{MIN_LENGTH},#{MAX_LENGTH}});/i
-    REVERSE_MAP = MAP.invert
+  VERSION = '4.0.0'
+  FLAVORS = %w[html4 xhtml1]
+  INSTRUCTIONS = [:basic, :named, :decimal, :hexadecimal]
 
-    BASIC_ENTITY_REGEXP = /[<>'"&]/
-
-    UTF8_NON_ASCII_REGEXP = /[\x00-\x1f]|[\xc0-\xfd][\x80-\xbf]+/
+  class InstructionError < RuntimeError
+  end
+  class UnknownFlavor < RuntimeError
+  end
 
-    ENCODE_ENTITIES_COMMAND_ORDER = { 
-      :basic => 0,
-      :named => 1,
-      :decimal => 2,
-      :hexadecimal => 3
-    }
-  
+  #
+  # Create a new HTMLEntities coder for the specified flavor.
+  # Available flavors are 'html4' and 'xhtml1' (the default).
+  # The only difference in functionality between the two is in the handling of the apos
+  # (apostrophe) named entity, which is not defined in HTML4.
+  #
+  def initialize(flavor='xhtml1')
+    @flavor = flavor.to_s.downcase
+    raise UnknownFlavor, "Unknown flavor #{flavor}" unless FLAVORS.include?(@flavor)
   end
-  
+
   #
-  # Decode XML and HTML 4.01 entities in a string into their UTF-8
+  # Decode entities in a string into their UTF-8
   # equivalents.  Obviously, if your string is not already in UTF-8, you'd
   # better convert it before using this method, or the output will be mixed
   # up.
   #
-  # Unknown named entities are not converted
+  # Unknown named entities will not be converted
   #
-  def decode_entities(string)
-    return string.gsub(Data::NAMED_ENTITY_REGEXP) { 
-      (cp = Data::MAP[$1]) ? [cp].pack('U') : $& 
-    }.gsub(/&#([0-9]{1,7});|&#x([0-9a-f]{1,6});/i) { 
-      $1 ? [$1.to_i].pack('U') : [$2.to_i(16)].pack('U') 
+  def decode(source)
+    return source.to_s.gsub(named_entity_regexp) {
+      (cp = map[$1]) ? [cp].pack('U') : $&
+    }.gsub(/&#([0-9]{1,7});|&#x([0-9a-f]{1,6});/i) {
+      $1 ? [$1.to_i].pack('U') : [$2.to_i(16)].pack('U')
     }
   end
-  
+
   #
   # Encode codepoints into their corresponding entities.  Various operations
   # are possible, and may be specified in order:
@@ -119,48 +64,102 @@ module HTMLEntities
   #   non-ASCII characters replaced with their named entity where possible, and
   #   decimal equivalents otherwise.
   #
-  # Note: It is the program's responsibility to ensure that the string
+  # Note: It is the program's responsibility to ensure that the source
   # contains valid UTF-8 before calling this method.
   #
-  def encode_entities(string, *instructions)
-    output = nil
+  def encode(source, *instructions)
+    string = source.to_s.dup
     if (instructions.empty?)
-      instructions = [:basic] 
-    else
-      instructions = instructions.sort_by { |instruction| 
-        Data::ENCODE_ENTITIES_COMMAND_ORDER[instruction] || 
-        (raise InstructionError, "unknown encode_entities command `#{instruction.inspect}'")
-      }
+      instructions = [:basic]
+    elsif (unknown_instructions = instructions - INSTRUCTIONS) != []
+      raise InstructionError,
+      "unknown encode_entities command(s): #{unknown_instructions.inspect}"
+    end
+    
+    basic_entity_encoder =
+    if instructions.include?(:basic) || instructions.include?(:named)
+      :encode_named
+    elsif instructions.include?(:decimal)
+      :encode_decimal
+    else instructions.include?(:hexadecimal)
+      :encode_hexadecimal
     end
-    instructions.each do |instruction|
-      case instruction
-      when :basic
-        # Handled as basic ASCII
-        output = (output || string).gsub(Data::BASIC_ENTITY_REGEXP) {
-          # It's safe to use the simpler [0] here because we know
-          # that the basic entities are ASCII.
-          '&' << Data::REVERSE_MAP[$&[0]] << ';'
-        }
-      when :named
-        # Test everything except printable ASCII 
-        output = (output || string).gsub(Data::UTF8_NON_ASCII_REGEXP) {
-          cp = $&.unpack('U')[0]
-          (e = Data::REVERSE_MAP[cp]) ?  "&#{e};" : $&
-        }
-      when :decimal
-        output = (output || string).gsub(Data::UTF8_NON_ASCII_REGEXP) {
-          "&##{$&.unpack('U')[0]};"
-        }
-      when :hexadecimal
-        output = (output || string).gsub(Data::UTF8_NON_ASCII_REGEXP) {
-          "&#x#{$&.unpack('U')[0].to_s(16)};"
-        }
-      end 
+    string.gsub!(basic_entity_regexp){ __send__(basic_entity_encoder, $&) }
+    
+    extended_entity_encoders = []
+    if instructions.include?(:named)
+      extended_entity_encoders << :encode_named
     end
-    return output
+    if instructions.include?(:decimal)
+      extended_entity_encoders << :encode_decimal
+    elsif instructions.include?(:hexadecimal)
+      extended_entity_encoders << :encode_hexadecimal
+    end
+    unless extended_entity_encoders.empty?
+      string.gsub!(extended_entity_regexp){
+        encode_extended(extended_entity_encoders, $&)
+      }
+    end
+    
+    return string
+  end
+
+private
+
+  def map
+    @map ||= (require "htmlentities/#{@flavor}"; HTMLEntities::MAPPINGS[@flavor])
+  end
+
+  def basic_entity_regexp
+    @basic_entity_regexp ||= (
+      case @flavor
+      when /^html/
+        /[<>"&]/
+      else
+        /[<>'"&]/
+      end
+    )
   end
   
-  extend self
+  def extended_entity_regexp
+    @extended_entity_regexp ||= (
+      regexp = '[\x00-\x1f]|[\xc0-\xfd][\x80-\xbf]+'
+      regexp += "|'" if @flavor == 'html4'
+      Regexp.new(regexp)
+    )
+  end
+
+  def named_entity_regexp
+    @named_entity_regexp ||= (
+      min_length = map.keys.map{ |a| a.length }.min
+      max_length = map.keys.map{ |a| a.length }.max
+      /&([a-z][a-z0-9]{#{min_length-1},#{max_length-1}});/i
+    )
+  end
+
+  def reverse_map
+    @reverse_map ||= map.invert
+  end
+
+  def encode_named(char)
+    cp = char.unpack('U')[0]
+    (e = reverse_map[cp]) && "&#{e};"
+  end
+
+  def encode_decimal(char)
+    "&##{char.unpack('U')[0]};"
+  end
+
+  def encode_hexadecimal(char)
+    "&#x#{char.unpack('U')[0].to_s(16)};"
+  end
   
-end
+  def encode_extended(encoders, char)
+    encoders.each do |encoder|
+      encoded = __send__(encoder, char)
+      return encoded if encoded
+    end
+    return char
+  end
 
+end

data/lib/htmlentities/html4.rb

@@ -0,0 +1,257 @@
+class HTMLEntities
+  MAPPINGS = {} unless defined? MAPPINGS
+  MAPPINGS['html4'] = {
+    'Aacute' => 193,
+    'aacute' => 225,
+    'Acirc' => 194,
+    'acirc' => 226,
+    'acute' => 180,
+    'AElig' => 198,
+    'aelig' => 230,
+    'Agrave' => 192,
+    'agrave' => 224,
+    'alefsym' => 8501,
+    'Alpha' => 913,
+    'alpha' => 945,
+    'amp' => 38,
+    'and' => 8743,
+    'ang' => 8736,
+    'Aring' => 197,
+    'aring' => 229,
+    'asymp' => 8776,
+    'Atilde' => 195,
+    'atilde' => 227,
+    'Auml' => 196,
+    'auml' => 228,
+    'bdquo' => 8222,
+    'Beta' => 914,
+    'beta' => 946,
+    'brvbar' => 166,
+    'bull' => 8226,
+    'cap' => 8745,
+    'Ccedil' => 199,
+    'ccedil' => 231,
+    'cedil' => 184,
+    'cent' => 162,
+    'Chi' => 935,
+    'chi' => 967,
+    'circ' => 710,
+    'clubs' => 9827,
+    'cong' => 8773,
+    'copy' => 169,
+    'crarr' => 8629,
+    'cup' => 8746,
+    'curren' => 164,
+    'Dagger' => 8225,
+    'dagger' => 8224,
+    'dArr' => 8659,
+    'darr' => 8595,
+    'deg' => 176,
+    'Delta' => 916,
+    'delta' => 948,
+    'diams' => 9830,
+    'divide' => 247,
+    'Eacute' => 201,
+    'eacute' => 233,
+    'Ecirc' => 202,
+    'ecirc' => 234,
+    'Egrave' => 200,
+    'egrave' => 232,
+    'empty' => 8709,
+    'emsp' => 8195,
+    'ensp' => 8194,
+    'Epsilon' => 917,
+    'epsilon' => 949,
+    'equiv' => 8801,
+    'Eta' => 919,
+    'eta' => 951,
+    'ETH' => 208,
+    'eth' => 240,
+    'Euml' => 203,
+    'euml' => 235,
+    'euro' => 8364,
+    'exist' => 8707,
+    'fnof' => 402,
+    'forall' => 8704,
+    'frac12' => 189,
+    'frac14' => 188,
+    'frac34' => 190,
+    'frasl' => 8260,
+    'Gamma' => 915,
+    'gamma' => 947,
+    'ge' => 8805,
+    'gt' => 62,
+    'hArr' => 8660,
+    'harr' => 8596,
+    'hearts' => 9829,
+    'hellip' => 8230,
+    'Iacute' => 205,
+    'iacute' => 237,
+    'Icirc' => 206,
+    'icirc' => 238,
+    'iexcl' => 161,
+    'Igrave' => 204,
+    'igrave' => 236,
+    'image' => 8465,
+    'infin' => 8734,
+    'int' => 8747,
+    'Iota' => 921,
+    'iota' => 953,
+    'iquest' => 191,
+    'isin' => 8712,
+    'Iuml' => 207,
+    'iuml' => 239,
+    'Kappa' => 922,
+    'kappa' => 954,
+    'Lambda' => 923,
+    'lambda' => 955,
+    'lang' => 9001,
+    'laquo' => 171,
+    'lArr' => 8656,
+    'larr' => 8592,
+    'lceil' => 8968,
+    'ldquo' => 8220,
+    'le' => 8804,
+    'lfloor' => 8970,
+    'lowast' => 8727,
+    'loz' => 9674,
+    'lrm' => 8206,
+    'lsaquo' => 8249,
+    'lsquo' => 8216,
+    'lt' => 60,
+    'macr' => 175,
+    'mdash' => 8212,
+    'micro' => 181,
+    'middot' => 183,
+    'minus' => 8722,
+    'Mu' => 924,
+    'mu' => 956,
+    'nabla' => 8711,
+    'nbsp' => 160,
+    'ndash' => 8211,
+    'ne' => 8800,
+    'ni' => 8715,
+    'not' => 172,
+    'notin' => 8713,
+    'nsub' => 8836,
+    'Ntilde' => 209,
+    'ntilde' => 241,
+    'Nu' => 925,
+    'nu' => 957,
+    'Oacute' => 211,
+    'oacute' => 243,
+    'Ocirc' => 212,
+    'ocirc' => 244,
+    'OElig' => 338,
+    'oelig' => 339,
+    'Ograve' => 210,
+    'ograve' => 242,
+    'oline' => 8254,
+    'Omega' => 937,
+    'omega' => 969,
+    'Omicron' => 927,
+    'omicron' => 959,
+    'oplus' => 8853,
+    'or' => 8744,
+    'ordf' => 170,
+    'ordm' => 186,
+    'Oslash' => 216,
+    'oslash' => 248,
+    'Otilde' => 213,
+    'otilde' => 245,
+    'otimes' => 8855,
+    'Ouml' => 214,
+    'ouml' => 246,
+    'para' => 182,
+    'part' => 8706,
+    'permil' => 8240,
+    'perp' => 8869,
+    'Phi' => 934,
+    'phi' => 966,
+    'Pi' => 928,
+    'pi' => 960,
+    'piv' => 982,
+    'plusmn' => 177,
+    'pound' => 163,
+    'Prime' => 8243,
+    'prime' => 8242,
+    'prod' => 8719,
+    'prop' => 8733,
+    'Psi' => 936,
+    'psi' => 968,
+    'quot' => 34,
+    'radic' => 8730,
+    'rang' => 9002,
+    'raquo' => 187,
+    'rArr' => 8658,
+    'rarr' => 8594,
+    'rceil' => 8969,
+    'rdquo' => 8221,
+    'real' => 8476,
+    'reg' => 174,
+    'rfloor' => 8971,
+    'Rho' => 929,
+    'rho' => 961,
+    'rlm' => 8207,
+    'rsaquo' => 8250,
+    'rsquo' => 8217,
+    'sbquo' => 8218,
+    'Scaron' => 352,
+    'scaron' => 353,
+    'sdot' => 8901,
+    'sect' => 167,
+    'shy' => 173,
+    'Sigma' => 931,
+    'sigma' => 963,
+    'sigmaf' => 962,
+    'sim' => 8764,
+    'spades' => 9824,
+    'sub' => 8834,
+    'sube' => 8838,
+    'sum' => 8721,
+    'sup' => 8835,
+    'sup1' => 185,
+    'sup2' => 178,
+    'sup3' => 179,
+    'supe' => 8839,
+    'szlig' => 223,
+    'Tau' => 932,
+    'tau' => 964,
+    'there4' => 8756,
+    'Theta' => 920,
+    'theta' => 952,
+    'thetasym' => 977,
+    'thinsp' => 8201,
+    'THORN' => 222,
+    'thorn' => 254,
+    'tilde' => 732,
+    'times' => 215,
+    'trade' => 8482,
+    'Uacute' => 218,
+    'uacute' => 250,
+    'uArr' => 8657,
+    'uarr' => 8593,
+    'Ucirc' => 219,
+    'ucirc' => 251,
+    'Ugrave' => 217,
+    'ugrave' => 249,
+    'uml' => 168,
+    'upsih' => 978,
+    'Upsilon' => 933,
+    'upsilon' => 965,
+    'Uuml' => 220,
+    'uuml' => 252,
+    'weierp' => 8472,
+    'Xi' => 926,
+    'xi' => 958,
+    'Yacute' => 221,
+    'yacute' => 253,
+    'yen' => 165,
+    'Yuml' => 376,
+    'yuml' => 255,
+    'Zeta' => 918,
+    'zeta' => 950,
+    'zwj' => 8205,
+    'zwnj' => 8204
+  }
+end