dommy 0.16.0 → 0.17.0

data/lib/dommy/encodings/multibyte_overrides.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# Generated by script/build_encoding_tables.rb from the Encoding Standard's
+# index-{jis0208,jis0212,big5,euc-kr,gb18030}.txt, against the Ruby
+# encodings LegacyIndexes.ruby_lookup reads. Do not edit by hand.
+#
+# Spec: https://encoding.spec.whatwg.org/#indexes
+
+module Dommy
+  module Encodings
+    # Pointer => code point, where the spec's index and Ruby's table differ.
+    MULTIBYTE_INDEX_OVERRIDES = {
+      jis0208: {
+      }.freeze,
+      jis0212: {
+        116 => 0xFF5E,
+      }.freeze,
+      big5: {
+        1000 => 0x3875, 1001 => 0x21D53, 1002 => 0x2369E, 1003 => 0x26021, 1004 => 0x3EEC, 1005 => 0x258DE,
+        1006 => 0x3AF5, 1007 => 0x7AFC, 1008 => 0x9F97, 1009 => 0x24161, 1010 => 0x2890D, 1011 => 0x231EA,
+        1012 => 0x20A8A, 1013 => 0x2325E, 1014 => 0x430A, 1015 => 0x8484, 1016 => 0x9F96, 1017 => 0x942F,
+        1018 => 0x4930, 1019 => 0x8613, 1020 => 0x5896, 1021 => 0x974A, 1022 => 0x9218, 1023 => 0x79D0,
+        1024 => 0x7A32, 1025 => 0x6660, 1026 => 0x6A29, 1027 => 0x889D, 1028 => 0x744C, 1029 => 0x7BC5,
+        1030 => 0x6782, 1031 => 0x7A2C, 1032 => 0x524F, 1033 => 0x9046, 1034 => 0x34E6, 1035 => 0x73C4,
+        1036 => 0x25DB9, 1037 => 0x74C6, 1038 => 0x9FC7, 1039 => 0x57B3, 1040 => 0x492F, 1041 => 0x544C,
+        1042 => 0x4131, 1043 => 0x2368E, 1044 => 0x5818, 1045 => 0x7A72, 1046 => 0x27B65, 1047 => 0x8B8F,
+        1048 => 0x46AE, 1049 => 0x26E88, 1050 => 0x4181, 1051 => 0x25D99, 1052 => 0x7BAE, 1053 => 0x224BC,
+        1054 => 0x9FC8, 1055 => 0x224C1, 1056 => 0x224C9, 1057 => 0x224CC, 1058 => 0x9FC9, 1059 => 0x8504,
+        1060 => 0x235BB, 1061 => 0x40B4, 1062 => 0x9FCA, 1063 => 0x44E1, 1064 => 0x2ADFF, 1065 => 0x62C1,
+        1066 => 0x706E, 1067 => 0x9FCB, 2082 => 0x7BB8, 2088 => 0x7C06, 2103 => 0x7CCE, 2114 => 0x7DD2,
+        2123 => 0x7E1D, 2148 => 0x8005, 2151 => 0x8028, 2221 => 0x83C1, 2239 => 0x84A8, 2244 => 0x840F,
+        2303 => 0x89A6, 2304 => 0x89A9, 2354 => 0x8D77, 2400 => 0x90FD, 2413 => 0x92B9, 2477 => 0x975C,
+        2498 => 0x97FF, 2605 => 0x9F16, 2673 => 0x8503, 2746 => 0x5159, 2747 => 0x515B, 2748 => 0x515D,
+        2749 => 0x515E, 2771 => 0x936E, 2780 => 0x7479, 2990 => 0x6D67, 3087 => 0x799B, 3259 => 0x9097,
+        3301 => 0x975D, 3436 => 0x701E, 3451 => 0x5B28, 4136 => 0x7201, 4138 => 0x77D7, 4141 => 0x7E87,
+        4182 => 0x99D6, 4206 => 0x91D4, 4220 => 0x60DE, 4230 => 0x6FB6, 4241 => 0x8F36, 4258 => 0x4FBB,
+        4273 => 0x71DF, 4279 => 0x9104, 4282 => 0x9DF0, 4294 => 0x83CF, 4329 => 0x5C10, 4330 => 0x79E3,
+        4349 => 0x5A67, 4419 => 0x8F0B, 4422 => 0x7B51, 4494 => 0x62D0, 4624 => 0x6062, 4694 => 0x75F9,
+        4708 => 0x6C4A, 4742 => 0x9B2E, 4748 => 0x9F17, 4815 => 0x50ED, 4828 => 0x5F0C, 4902 => 0x880F,
+        4922 => 0x62CE, 4982 => 0x7468, 4992 => 0x7162, 4997 => 0x7250, 5029 => 0x2027, 5038 => 0xFE51,
+        5050 => 0x2574, 5120 => 0x00AF, 5121 => 0xFFE3, 5123 => 0x02CD, 5153 => 0xFF5E, 5168 => 0x2295,
+        5169 => 0x2299, 5180 => 0xFF0F, 5181 => 0xFF3C, 5182 => 0x2215, 5183 => 0xFE68, 5185 => 0xFFE5,
+        5187 => 0xFFE0, 5188 => 0xFFE1, 5287 => 0x5341, 5289 => 0x5345, 5432 => 0x2400, 5433 => 0x2401,
+        5434 => 0x2402, 5435 => 0x2403, 5436 => 0x2404, 5437 => 0x2405, 5438 => 0x2406, 5439 => 0x2407,
+        5440 => 0x2408, 5441 => 0x2409, 5442 => 0x240A, 5443 => 0x240B, 5444 => 0x240C, 5445 => 0x240D,
+        5446 => 0x240E, 5447 => 0x240F, 5448 => 0x2410, 5449 => 0x2411, 5450 => 0x2412, 5451 => 0x2413,
+        5452 => 0x2414, 5453 => 0x2415, 5454 => 0x2416, 5455 => 0x2417, 5456 => 0x2418, 5457 => 0x2419,
+        5458 => 0x241A, 5459 => 0x241B, 5460 => 0x241C, 5461 => 0x241D, 5462 => 0x241E, 5463 => 0x241F,
+        5464 => 0x2421, 5465 => 0x20AC, 10942 => 0x5EF4, 10946 => 0x65E0, 10948 => 0x7676, 10950 => 0x96B6,
+        10957 => 0x3003, 10958 => 0x4EDD, 18975 => 0x255E, 18976 => 0x256A, 18977 => 0x2561, 18991 => 0x2550,
+        18992 => 0x256D, 18993 => 0x256E, 18994 => 0x2570, 18995 => 0x256F, 19028 => 0x5029, 19035 => 0x507D,
+        19088 => 0x5305, 19096 => 0x5344, 19112 => 0x537F, 19162 => 0x5605, 19240 => 0x5A77, 19299 => 0x5E75,
+        19305 => 0x5ED0, 19326 => 0x5F58, 19355 => 0x60A4, 19398 => 0x6490, 19439 => 0x6674, 19454 => 0x675E,
+        19553 => 0x6C9C, 19554 => 0x6E1D, 19557 => 0x6E2F, 19611 => 0x716E, 19643 => 0x732A, 19672 => 0x745C,
+        19697 => 0x74E9, 19748 => 0x7809,
+      }.freeze,
+      euc_kr: {
+      }.freeze,
+      gb18030: {
+        6555 => 0x3000, 7182 => 0xFE10, 7183 => 0xFE12, 7184 => 0xFE11, 7185 => 0xFE13, 7186 => 0xFE14,
+        7187 => 0xFE15, 7188 => 0xFE16, 7201 => 0xFE17, 7202 => 0xFE18, 7208 => 0xFE19, 7533 => 0x1E3F,
+        23775 => 0x9FB4, 23783 => 0x9FB5, 23788 => 0x9FB6, 23789 => 0x9FB7, 23795 => 0x9FB8, 23812 => 0x9FB9,
+        23829 => 0x9FBA, 23845 => 0x9FBB,
+      }.freeze,
+    }.freeze
+
+    # Pointers Ruby's table maps that the spec's index leaves empty.
+    MULTIBYTE_INDEX_NULLS = {
+      jis0208: [8836..10715].freeze,
+      jis0212: [].freeze,
+      big5: [].freeze,
+      euc_kr: [].freeze,
+      gb18030: [].freeze,
+    }.freeze
+  end
+end

data/lib/dommy/encodings/prescan.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+module Dommy
+  module Encodings
+    # HTML's "prescan a byte stream to determine its encoding" over the first
+    # 1024 bytes: a walk over the bytes that skips comments and other tags as
+    # the tokenizer would, and reads each `<meta>`'s attributes (quoted values
+    # included) for a `charset`, or an `http-equiv="content-type"` with a
+    # `content` naming one. Running out of bytes ends it with what a leading
+    # `<?xml … encoding="…"?>` declares, if anything.
+    #
+    # Spec: https://html.spec.whatwg.org/#prescan-a-byte-stream-to-determine-its-encoding
+    class Prescan
+      LIMIT = 1024
+      # ASCII whitespace, as bytes: TAB, LF, FF, CR, SPACE.
+      SPACE = [0x09, 0x0A, 0x0C, 0x0D, 0x20].freeze
+      SPACE_OR_SLASH = (SPACE + [0x2F]).freeze
+      SPACE_OR_GT = (SPACE + [0x3E]).freeze
+      private_constant :LIMIT, :SPACE, :SPACE_OR_SLASH, :SPACE_OR_GT
+
+      def self.call(source) = new(source.to_s.b.byteslice(0, LIMIT)).run
+
+      def initialize(bytes)
+        @bytes = bytes
+        @position = 0
+      end
+
+      def run
+        utf16 = utf16_xml_declaration
+        return utf16 if utf16
+
+        catch(:end) do
+          # Only a `<` starts anything; every other byte is passed over.
+          while (@position = @bytes.index("<", @position))
+            found =
+              if starts_with?("<!--") then skip_comment
+              elsif meta_tag? then meta
+              elsif tag? then skip_tag
+              elsif starts_with?("<!") || starts_with?("</") || starts_with?("<?") then skip_to(0x3E)
+              end
+            return found if found.is_a?(String)
+
+            @position += 1
+          end
+        end
+        xml_encoding
+      end
+
+      private
+
+      # "Prescan for UTF-16 XML declarations": `<?x` in UTF-16 is UTF-16.
+      def utf16_xml_declaration
+        return "UTF-16LE" if @bytes.start_with?("<\0?\0x\0".b)
+
+        "UTF-16BE" if @bytes.start_with?("\0<\0?\0x".b)
+      end
+
+      def byte
+        @bytes.getbyte(@position) || throw(:end)
+      end
+
+      def starts_with?(text) = @bytes.byteslice(@position, text.bytesize) == text
+
+      def letter?(value) = value && ((0x41..0x5A).cover?(value) || (0x61..0x7A).cover?(value))
+
+      # `<meta` (ASCII case-insensitive) followed by whitespace or `/`.
+      def meta_tag?
+        @bytes.byteslice(@position, 5)&.casecmp?("<meta") && SPACE_OR_SLASH.include?(@bytes.getbyte(@position + 5))
+      end
+
+      # `<` or `</` followed by an ASCII letter.
+      def tag?
+        return false unless @bytes.getbyte(@position) == 0x3C
+
+        letter?(@bytes.getbyte(@position + 1)) || (@bytes.getbyte(@position + 1) == 0x2F && letter?(@bytes.getbyte(@position + 2)))
+      end
+
+      # To the first `>` preceded by two `-` after the `<`; those may be the
+      # `<!--`'s own, so `<!-->` and `<!--->` are closed comments.
+      def skip_comment
+        index = @bytes.index("-->".b, @position + 2) || throw(:end)
+        @position = index + 2
+      end
+
+      def skip_to(value)
+        index = @bytes.index(value.chr, @position + 1) || throw(:end)
+        @position = index
+      end
+
+      # A tag that is not a `<meta>`: past its name, then its attributes, so a
+      # `>` inside a quoted value does not end it.
+      def skip_tag
+        @position += 1
+        @position += 1 until SPACE_OR_GT.include?(byte)
+        nil while attribute
+      end
+
+      # The `<meta>` steps: its encoding when it declares one, else nil.
+      def meta
+        @position += 5
+        names = []
+        got_pragma = false
+        need_pragma = nil
+        charset = nil
+        while (name, value = attribute)
+          next if names.include?(name)
+
+          names << name
+          case name
+          when "http-equiv"
+            got_pragma = true if value == "content-type"
+          when "content"
+            encoding = encoding_from_content(value)
+            if encoding && charset.nil?
+              charset = encoding
+              need_pragma = true
+            end
+          when "charset"
+            charset = Encodings.get(value) || false
+            need_pragma = false
+          end
+        end
+        return nil if need_pragma.nil? || (need_pragma && !got_pragma) || !charset
+        return "UTF-8" if charset.start_with?("UTF-16")
+
+        charset == "x-user-defined" ? "windows-1252" : charset
+      end
+
+      # "Get an attribute": the next attribute's name and value (ASCII
+      # lowercased, as bytes), or nil at the tag's `>`.
+      def attribute
+        @position += 1 while SPACE_OR_SLASH.include?(byte)
+        return nil if byte == 0x3E
+
+        name = +"".b
+        value = +"".b
+        loop do
+          current = byte
+          if current == 0x3D && !name.empty?
+            @position += 1
+            break
+          elsif SPACE.include?(current)
+            @position += 1 while SPACE.include?(byte)
+            return [name, value] unless byte == 0x3D
+
+            @position += 1
+            break
+          elsif current == 0x2F || current == 0x3E
+            return [name, value]
+          end
+          name << lower(current)
+          @position += 1
+        end
+
+        @position += 1 while SPACE.include?(byte)
+        current = byte
+        if current == 0x22 || current == 0x27
+          loop do
+            @position += 1
+            if byte == current
+              @position += 1
+              return [name, value]
+            end
+            value << lower(byte)
+          end
+        end
+        return [name, value] if current == 0x3E
+
+        loop do
+          current = byte
+          return [name, value] if SPACE.include?(current) || current == 0x3E
+
+          value << lower(current)
+          @position += 1
+        end
+      end
+
+      def lower(value) = (0x41..0x5A).cover?(value) ? value + 0x20 : value
+
+      # HTML's "extracting a character encoding from a meta element", for a
+      # `content` attribute's value: the encoding its `charset=` names, or nil.
+      #
+      # Spec: https://html.spec.whatwg.org/#algorithm-for-extracting-a-character-encoding-from-a-meta-element
+      def encoding_from_content(content)
+        position = 0
+        loop do
+          index = content.index(/charset/i, position) or return nil
+          position = index + 7
+          position += 1 while SPACE.include?(content.getbyte(position))
+          next unless content.getbyte(position) == 0x3D
+
+          position += 1
+          position += 1 while SPACE.include?(content.getbyte(position))
+          first = content.getbyte(position) or return nil
+          if first == 0x22 || first == 0x27
+            close = content.index(first.chr, position + 1) or return nil
+            return Encodings.get(content.byteslice(position + 1...close))
+          end
+          return Encodings.get(content.byteslice(position..)[/\A[^\t\n\f\r ;]*/n])
+        end
+      end
+
+      # "Get an XML encoding": the encoding a leading `<?xml` declaration's
+      # `encoding="…"` (before its `>`, quoted, with no byte at or below 0x20)
+      # names, with UTF-16 read as UTF-8, or nil.
+      def xml_encoding
+        return nil unless @bytes.start_with?("<?xml")
+
+        finish = @bytes.index(">") or return nil
+        position = @bytes.index("encoding")
+        return nil if position.nil? || position >= finish
+
+        position = skip_controls(position + 8)
+        return nil unless @bytes.getbyte(position) == 0x3D
+
+        position = skip_controls(position + 1)
+        quote = @bytes.getbyte(position)
+        return nil unless quote == 0x22 || quote == 0x27
+
+        close = @bytes.index(quote.chr, position + 1) or return nil
+        label = @bytes.byteslice(position + 1...close)
+        return nil if label.each_byte.any? { |value| value <= 0x20 }
+
+        name = Encodings.get(label)
+        name&.start_with?("UTF-16") ? "UTF-8" : name
+      end
+
+      def skip_controls(position)
+        position += 1 while (value = @bytes.getbyte(position)) && value <= 0x20
+        position
+      end
+    end
+  end
+end

data/lib/dommy/encodings.rb

@@ -8,27 +8,14 @@ module Dommy
   # each encoding that turns bytes into code points, streaming, with the
   # replacement-or-throw error mode TextDecoder exposes as `fatal`.
   #
-  # UTF-8, UTF-16 and the legacy single-byte encodings are decoded here, from
-  # the spec's own algorithms and index tables. The legacy multi-byte
-  # encodings (Big5, EUC-JP, ISO-2022-JP, Shift_JIS, EUC-KR, GBK, gb18030) go
-  # through Ruby's converters for the encoding closest to the spec's index,
-  # which differ from it in a few code points and in how many U+FFFD a broken
-  # sequence yields.
+  # Every decoder is the spec's own algorithm: UTF-8, UTF-16 and the legacy
+  # single-byte encodings here, with their index tables, and the legacy
+  # multi-byte ones (Big5, EUC-JP, ISO-2022-JP, Shift_JIS, EUC-KR, GBK,
+  # gb18030) in encodings/legacy_multibyte, which read their indexes through
+  # the Ruby encoding that matches each.
   #
   # Spec: https://encoding.spec.whatwg.org/
   module Encodings
-    # The Ruby encoding whose mapping is closest to the spec's index for each
-    # legacy multi-byte encoding.
-    RUBY_ENCODINGS = {
-      "Big5" => "Big5-HKSCS",
-      "EUC-JP" => "EUC-JP",
-      "ISO-2022-JP" => "ISO-2022-JP",
-      "Shift_JIS" => "Windows-31J",
-      "EUC-KR" => "CP949",
-      "GBK" => "GBK",
-      "gb18030" => "GB18030"
-    }.freeze
-
     REPLACEMENT_CHARACTER = 0xFFFD
 
     # The name of the encoding `label` stands for, or nil. Only ASCII
@@ -36,7 +23,24 @@ module Dommy
     #
     # Spec: https://encoding.spec.whatwg.org/#concept-encoding-get
     def self.get(label)
-      LABELS[label.gsub(/\A[\t\n\f\r ]+|[\t\n\f\r ]+\z/, "").downcase]
+      LABELS[label.gsub(/\A[\t\n\f\r ]+|[\t\n\f\r ]+\z/, "").downcase(:ascii)]
+    end
+
+    # The encoding `encoding` names, given as an Encoding Standard name or
+    # label; an ArgumentError for anything else.
+    def self.name_for(encoding)
+      text = encoding.to_s
+      return text if LABELS.value?(text)
+
+      get(text) or raise ArgumentError, "unknown encoding: #{encoding.inspect}"
+    end
+
+    # The encoding text is encoded in when `name` is the one asked for: UTF-8
+    # in place of replacement, UTF-16BE and UTF-16LE, which have no encoder.
+    #
+    # Spec: https://encoding.spec.whatwg.org/#get-an-output-encoding
+    def self.output_encoding(name)
+      name.start_with?("UTF-16") || name == "replacement" ? "UTF-8" : name
     end
 
     # The byte-order marks that name their own encoding, longest first.
@@ -67,16 +71,98 @@ module Dommy
       valid_utf8(bytes, name) || decoder_for(name).decode(bytes, flush: true).pack("U*")
     end
 
-    # A document's bytes as text, in the encoding its response declares: a
-    # byte-order mark first, then the Content-Type's `charset`, else UTF-8.
-    # (HTML's prescan of a `<meta charset>` is not done.)
+    # A document's bytes as text, and the encoding they were decoded with
+    # (::document_encoding), which is the document's: a caller hands both to
+    # the document, so that its URLs encode a query as its bytes were.
+    # `html: false` (an XML or a text document) skips the `<meta>` prescan.
+    def self.decode_document(bytes, content_type, html: true)
+      encoding = document_encoding(bytes, content_type, html: html)
+      [decode(bytes.to_s, encoding), encoding]
+    end
+
+    # The encoding an HTML document's bytes are in: a byte-order mark's, else
+    # the Content-Type's `charset`, else what a `<meta>` in the first 1024
+    # bytes declares (the prescan), else UTF-8.
     #
     # Spec: https://html.spec.whatwg.org/#determining-the-character-encoding
-    def self.decode_document(bytes, content_type)
+    def self.document_encoding(bytes, content_type, html: true)
+      bytes = bytes.to_s.b
+      BOMS.each { |mark, name| return name if bytes.start_with?(mark) }
       label = Internal::MimeType.charset_of(content_type.to_s)
-      decode(bytes.to_s, (label && get(label)) || "UTF-8")
+      (label && get(label)) || (html && prescan(bytes)) || "UTF-8"
     end
 
+    # HTML's "prescan a byte stream to determine its encoding" (Prescan): what
+    # the first 1024 bytes' `<meta charset>` or `<meta http-equiv=content-type
+    # content="…; charset=…">` declares, else what a leading `<?xml … encoding=…?>`
+    # does, or nil. A declared UTF-16 is UTF-8 (the bytes are ASCII-compatible
+    # if the tag was read), and x-user-defined windows-1252; `<?x` in UTF-16
+    # itself is that UTF-16.
+    def self.prescan(source) = Prescan.call(source)
+
+    # The bytes `string` encodes to in the encoding named `name` (as `get`
+    # returns it); a code point the encoding cannot represent is the bytes the
+    # block returns for it. The legacy multi-byte encoders and the single-byte
+    # ones (an index read backwards) are the spec's own; Ruby encodes the rest,
+    # and UTF-8 is the string's own bytes.
+    #
+    # Spec: https://encoding.spec.whatwg.org/#encode
+    def self.encode(name, string, &unmappable)
+      return LegacyEncoders.encode(name, string, &unmappable).pack("C*") if LegacyEncoders.handles?(name)
+      return single_byte_encode(single_byte_pointers(name), string, &unmappable) if SINGLE_BYTE_INDEXES.key?(name)
+      return x_user_defined_encode(string, &unmappable) if name == "x-user-defined"
+
+      ruby = begin
+        ::Encoding.find(name.to_s)
+      rescue ArgumentError
+        nil
+      end
+      return string.b if ruby.nil? || ruby == ::Encoding::UTF_8
+
+      begin
+        string.encode(ruby).b
+      rescue ::Encoding::UndefinedConversionError, ::Encoding::InvalidByteSequenceError
+        string.each_char.each_with_object(+"".b) do |char, out|
+          out << char.encode(ruby).b
+        rescue ::Encoding::UndefinedConversionError, ::Encoding::InvalidByteSequenceError
+          out << unmappable.call(char.ord).pack("C*")
+        end
+      end
+    end
+
+    # A single-byte index read backwards, code point => byte, made once per
+    # encoding.
+    @single_byte_pointers = {}
+    def self.single_byte_pointers(name)
+      @single_byte_pointers[name] ||= SINGLE_BYTE_INDEXES[name].each_with_index.each_with_object({}) do |(code_point, pointer), map|
+        map[code_point] ||= pointer + 0x80 if code_point
+      end.freeze
+    end
+    private_class_method :single_byte_pointers
+
+    # Spec: https://encoding.spec.whatwg.org/#single-byte-encoder
+    def self.single_byte_encode(pointers, string, &unmappable)
+      string.each_codepoint.each_with_object(+"".b) do |code_point, out|
+        byte = code_point <= 0x7F ? code_point : pointers[code_point]
+        byte ? out << byte : out << unmappable.call(code_point).pack("C*")
+      end
+    end
+    private_class_method :single_byte_encode
+
+    # Spec: https://encoding.spec.whatwg.org/#x-user-defined-encoder
+    def self.x_user_defined_encode(string, &unmappable)
+      string.each_codepoint.each_with_object(+"".b) do |code_point, out|
+        if code_point <= 0x7F
+          out << code_point
+        elsif code_point.between?(0xF780, 0xF7FF)
+          out << (code_point - 0xF780 + 0x80)
+        else
+          out << unmappable.call(code_point).pack("C*")
+        end
+      end
+    end
+    private_class_method :x_user_defined_encode
+
     # Already-valid UTF-8 needs no decoding: the spec's decoder would walk it
     # byte by byte and rebuild it code point by code point, which measures ~40x
     # the cost of asking Ruby, on the whole-buffer path an XHR response takes.
@@ -101,17 +187,21 @@ module Dommy
       when "UTF-16BE" then Utf16Decoder.new(fatal, big_endian: true)
       when "x-user-defined" then XUserDefinedDecoder.new(fatal)
       when "replacement" then ReplacementDecoder.new(fatal)
+      when "ISO-2022-JP" then Iso2022JpDecoder.new(fatal)
+      when "Shift_JIS" then ShiftJisDecoder.new(fatal)
+      when "EUC-JP" then EucJpDecoder.new(fatal)
+      when "Big5" then Big5Decoder.new(fatal)
+      when "EUC-KR" then EucKrDecoder.new(fatal)
+      # GBK's decoder is gb18030's.
+      when "GBK", "gb18030" then Gb18030Decoder.new(fatal)
       else
         index = SINGLE_BYTE_INDEXES[name]
         return SingleByteDecoder.new(fatal, index) if index
 
-        ruby_encoding = RUBY_ENCODINGS[name]
         # `name` comes from ::get, so every spec encoding is covered above. A
         # name from anywhere else is the caller's mistake, and says so rather
         # than surfacing a Hash's KeyError.
-        raise ArgumentError, "not an encoding the standard names: #{name.inspect}" unless ruby_encoding
-
-        ConverterDecoder.new(fatal, ruby_encoding)
+        raise ArgumentError, "not an encoding the standard names: #{name.inspect}"
       end
     end
 
@@ -307,34 +397,8 @@ module Dommy
         [replacement!]
       end
     end
-
-    # A legacy multi-byte encoding, through Ruby's converter. Partial input is
-    # held between calls; at a flush what is left over is an error.
-    class ConverterDecoder < Decoder
-      def initialize(fatal, ruby_encoding)
-        super(fatal)
-        options = fatal ? {} : {invalid: :replace, undef: :replace, replace: "\uFFFD"}
-        @converter = ::Encoding::Converter.new(ruby_encoding, "UTF-8", **options)
-      end
-
-      def decode(bytes, flush:)
-        output = +""
-        result = @converter.primitive_convert(bytes.dup, output, nil, nil, partial_input: !flush)
-        code_points = output.codepoints
-        case result
-        when :finished, :source_buffer_empty
-          code_points
-        when :incomplete_input, :invalid_byte_sequence, :undefined_conversion
-          # Only reached in fatal mode, or by an incomplete sequence at a
-          # flush; either way the stream is over.
-          code_points << replacement!
-        else
-          # :destination_buffer_full and :after_output need a byte limit or an
-          # output filter to occur, and this call passes neither. Reaching here
-          # would mean the converter contract changed under us.
-          raise "unexpected Encoding::Converter result: #{result.inspect}"
-        end
-      end
-    end
   end
 end
+
+require_relative "encodings/legacy_multibyte"
+require_relative "encodings/prescan"