rchardet 1.9.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28403d2d9a7f0a681c74f8f9e71a7cee248f5b369731c64a29f9e5a4cd221de8
-  data.tar.gz: a1b76fc7660bdc0e15797d036af4f90116225985baa37b5cc14d1cb974d64469
+  metadata.gz: ae69d0c1bf33f290595011946f42312ccc7d3c9d4691096a2f26faf3125b398e
+  data.tar.gz: 51fcc82f0d020c9a9815bcec0cb10f12004bc8dbf699c55483e6e7a773da2a95
 SHA512:
-  metadata.gz: 4381fd73d549f71bab23af5677801e0a4b130f406fdd2e0ac352c5319ae719db675791a3f1c205de874f50aace623ca0cd70fc23546e8cfe150ff5a325435fd7
-  data.tar.gz: 65f6e93b9a127e247781a6273d7be44c0f1dd21374f5592d8938799da698b33443416b2806fe47f3d73c530085fd8af0e841637435adde1487fd4a09a0f35597
+  metadata.gz: 88e2bee99000fdb57c6c24f3e1d07ab543166979fb25050b9775964f83a838ab0e55d3f6abd158676250355a9b47e30333c574cb2090052f55dd763042798fcc
+  data.tar.gz: d7d5c7316e1004617b8767f0eab6cc8658f44519f08641888fb19c5c1cfc53e928a153cc4c85ddc8131fdef0fd51ac72508c9fd2263b8fc943466fdca7f7a792

data/lib/rchardet/universaldetector.rb

@@ -42,6 +42,7 @@ module CharDet
       @highBitDetector = /[\x80-\xFF]/n
       @escDetector = /(\033|\~\{)/n
       @escCharSetProber = nil
+      @utf1632prober = nil
       @charSetProbers = []
       reset()
     end
@@ -56,6 +57,9 @@ module CharDet
       if @escCharSetProber
         @escCharSetProber.reset()
       end
+      if @utf1632prober
+        @utf1632prober.reset()
+      end
       for prober in @charSetProbers
         prober.reset()
       end
@@ -117,6 +121,22 @@ module CharDet
       end
 
       @lastChar = aBuf[-1, 1]
+
+      if !@utf1632prober
+        @utf1632prober = UTF1632Prober.new()
+      end
+
+      if @utf1632prober.get_state == EDetecting
+        if @utf1632prober.feed(aBuf) == EFoundIt
+          @result = {
+            "encoding" => @utf1632prober.get_charset_name(),
+            "confidence" => @utf1632prober.get_confidence()
+          }
+        @done = true
+        return
+        end
+      end
+
       if @inputState == EEscAscii
         if !@escCharSetProber
           @escCharSetProber = EscCharSetProber.new()

data/lib/rchardet/utf1632prober.rb

@@ -0,0 +1,232 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  MIN_CHARS_FOR_DETECTION = 20
+  EXPECTED_RATIO = 0.94
+
+  class UTF1632Prober < CharSetProber
+    def initialize
+      super()
+      @position = 0
+      @zeros_at_mod = [0, 0, 0, 0]
+      @nonzeros_at_mod = [0, 0, 0, 0]
+      @state = EDetecting
+      @quad = [0, 0, 0, 0]
+      @invalid_utf16be = false
+      @invalid_utf16le = false
+      @invalid_utf32be = false
+      @invalid_utf32le = false
+      @first_half_surrogate_pair_detected_16be = false
+      @first_half_surrogate_pair_detected_16le = false
+      reset()
+    end
+
+    def reset
+      super()
+      @position = 0
+      @zeros_at_mod = [0, 0, 0, 0]
+      @nonzeros_at_mod = [0, 0, 0, 0]
+      @state = EDetecting
+      @invalid_utf16be = false
+      @invalid_utf16le = false
+      @invalid_utf32be = false
+      @invalid_utf32le = false
+      @first_half_surrogate_pair_detected_16be = false
+      @first_half_surrogate_pair_detected_16le = false
+      @quad = [0, 0, 0, 0]
+    end
+
+    def get_charset_name
+      if is_likely_utf32be
+        return "UTF-32BE"
+      end
+      if is_likely_utf32le
+        return "UTF-32LE"
+      end
+      if is_likely_utf16be
+        return "UTF-16BE"
+      end
+      if is_likely_utf16le
+        return "UTF-16LE"
+      end
+      # default to something valid
+      return "UTF-16"
+    end
+
+    def feed(aBuf)
+      aBuf.each_byte do |b|
+        mod4 = @position % 4
+        @quad[mod4] = b
+        if mod4 == 3
+          validate_utf32_characters(@quad)
+          validate_utf16_characters(@quad[0..2])
+          validate_utf16_characters(@quad[2..4])
+        end
+        if b == 0
+          @zeros_at_mod[mod4] += 1
+        else
+          @nonzeros_at_mod[mod4] += 1
+        end
+        @position += 1
+      end
+
+      return get_state()
+    end
+
+    def get_state
+      if [ENotMe, EFoundIt].include? @state
+        # terminal, decided states
+        return @state
+      end
+      if get_confidence > 0.80
+        @state = EFoundIt
+      elsif @position > 4 * 1024
+        # if we get to 4kb into the file, and we can't conclude it's UTF,
+        # let's give up
+        @state = ENotMe
+      end
+      return @state
+    end
+
+    def get_confidence
+      if is_likely_utf16le || is_likely_utf16be || is_likely_utf32le || is_likely_utf32be
+        0.85
+      else
+        0.00
+      end
+    end
+
+    private
+
+    def approx_32bit_chars
+      return [1.0, @position / 4.0].max
+    end
+
+    def approx_16bit_chars
+      return [1.0, @position / 2.0].max
+    end
+
+    def is_likely_utf32be
+      approx_chars = approx_32bit_chars
+      return approx_chars >= MIN_CHARS_FOR_DETECTION &&
+             @zeros_at_mod[0] / approx_chars > EXPECTED_RATIO &&
+             @zeros_at_mod[1] / approx_chars > EXPECTED_RATIO &&
+             @zeros_at_mod[2] / approx_chars > EXPECTED_RATIO &&
+             @nonzeros_at_mod[3] / approx_chars > EXPECTED_RATIO &&
+             !@invalid_utf32be
+
+    end
+
+    def is_likely_utf32le
+      approx_chars = approx_32bit_chars
+      return approx_chars >= MIN_CHARS_FOR_DETECTION &&
+             @nonzeros_at_mod[0] / approx_chars > EXPECTED_RATIO &&
+             @zeros_at_mod[1] / approx_chars > EXPECTED_RATIO &&
+             @zeros_at_mod[2] / approx_chars > EXPECTED_RATIO &&
+             @zeros_at_mod[3] / approx_chars > EXPECTED_RATIO &&
+             !@invalid_utf32le
+    end
+
+    def is_likely_utf16be
+      approx_chars = approx_16bit_chars
+      return approx_chars >= MIN_CHARS_FOR_DETECTION &&
+             (@nonzeros_at_mod[1] + @nonzeros_at_mod[3]) / approx_chars > EXPECTED_RATIO &&
+             (@zeros_at_mod[0] + @zeros_at_mod[2]) / approx_chars > EXPECTED_RATIO &&
+             !@invalid_utf16be
+    end
+
+    def is_likely_utf16le
+      approx_chars = approx_16bit_chars
+      return approx_chars >= MIN_CHARS_FOR_DETECTION &&
+             (@nonzeros_at_mod[0] + @nonzeros_at_mod[2]) / approx_chars > EXPECTED_RATIO &&
+             (@zeros_at_mod[1] + @zeros_at_mod[3]) / approx_chars > EXPECTED_RATIO &&
+             !@invalid_utf16le
+    end
+
+    # @param [Array<Integer>] quad four consecutive bytes
+    # @return [void]
+    def validate_utf32_characters(quad)
+      "" "
+        Validate if the quad of bytes is valid UTF-32.
+
+        UTF-32 is valid in the range 0x00000000 - 0x0010FFFF
+        excluding 0x0000D800 - 0x0000DFFF
+
+        https://en.wikipedia.org/wiki/UTF-32
+        " ""
+      if quad[0] != 0 or quad[1] > 0x10 or quad[0] == 0 and quad[1] == 0 and (0xD8..0xDF).include?(quad[2])
+        @invalid_utf32be = true
+      end
+      if quad[3] != 0 or quad[2] > 0x10 or quad[3] == 0 and quad[2] == 0 and (0xD8..0xDF).include?(quad[1])
+        @invalid_utf32le = true
+      end
+    end
+
+    # @param [Array<Integer>] pair two consecutive bytes
+    # @return [void]
+    def validate_utf16_characters(pair)
+      "" "
+        Validate if the pair of bytes is  valid UTF-16.
+
+        UTF-16 is valid in the range 0x0000 - 0xFFFF excluding 0xD800 - 0xFFFF
+        with an exception for surrogate pairs, which must be in the range
+        0xD800-0xDBFF followed by 0xDC00-0xDFFF
+
+        https://en.wikipedia.org/wiki/UTF-16
+        " ""
+      if !@first_half_surrogate_pair_detected_16be
+        if (0xD8..0xDB).include? pair[0]
+          @first_half_surrogate_pair_detected_16be = true
+        elsif (0xDC..0xDF).include? pair[0]
+          @invalid_utf16be = true
+        end
+      else
+        if (0xDC..0xDF).include? pair[0]
+          @first_half_surrogate_pair_detected_16be = false
+        else
+          @invalid_utf16be = true
+        end
+      end
+
+      if not @first_half_surrogate_pair_detected_16le
+        if (0xD8..0xDB).include? pair[1]
+          @first_half_surrogate_pair_detected_16le = true
+        elsif (0xDC..0xDF).include? pair[1]
+          @invalid_utf16le = true
+        end
+      else
+        if (0xDC..0xDF).include? pair[1]
+          @first_half_surrogate_pair_detected_16le = false
+        else
+          @invalid_utf16le = true
+        end
+      end
+    end
+  end
+end

data/lib/rchardet/version.rb

@@ -1,3 +1,3 @@
 module CharDet
-  VERSION = "1.9.0"
+  VERSION = "1.10.0"
 end

data/lib/rchardet.rb

@@ -53,6 +53,7 @@ require 'rchardet/sbcsgroupprober'
 require 'rchardet/sjisprober'
 require 'rchardet/universaldetector'
 require 'rchardet/utf8prober'
+require 'rchardet/utf1632prober'
 
 module CharDet
   def CharDet.detect(aBuf)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rchardet
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Michael Grosser
 - Jeff Hodges
 bindir: bin
 cert_chain: []
-date: 2025-01-05 00:00:00.000000000 Z
+date: 2025-09-10 00:00:00.000000000 Z
 dependencies: []
 email:
 - michael@grosser.it
@@ -52,6 +52,7 @@ files:
 - lib/rchardet/sbcsgroupprober.rb
 - lib/rchardet/sjisprober.rb
 - lib/rchardet/universaldetector.rb
+- lib/rchardet/utf1632prober.rb
 - lib/rchardet/utf8prober.rb
 - lib/rchardet/version.rb
 homepage: https://github.com/jmhodges/rchardet