unicoder 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9064718b14baf32790bb6f9a705aab4d07231771f1c663546dea90413b0b68d9
-  data.tar.gz: 40f6145a58220620accb91bda68b06c2e2ceeea12b3cee4dc6a1c021e8fc9194
+  metadata.gz: c698f0042604828d6acf19123dba21935d65387c030df78774885e0e0084c6ef
+  data.tar.gz: f8c1b180273b758079232066ecd4729adbfb41901464414ad56bab6df3c83ee5
 SHA512:
-  metadata.gz: f69ddbfcef5269be204fed89d00018312a623330c28382531922fb5fe0093c9dfee7aa7e7bc5bff069ba1e57341fbb62e915d34bb27f37cec10283b7e8bbc678
-  data.tar.gz: f4499a5ce299023e3752ed69df04e148d9937ac355ae7e415525f46c402761992540de6ae5fc8d1b22ab4a2dc154220d722ad940d4878fca3c9e3f6a306d00a1
+  metadata.gz: 486f62f96ed10a3dac0703f62da4c85fb1b6a594ff922e7bae668bdf62878f3c2b5704bb89f491164d0e416d66e316eb8f7fbb23c5eb971f9231edef26bb5162
+  data.tar.gz: cc7bdff24d99a31021d0b1321447da769b0e25d5209b79e75d6d04c5d1590b858f9f9195c68dd7e35a5817f7dbd00dc7f5f3dae3eaaa268542e4608d7095b035

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## CHANGELOG
 
+### 1.4.0
+
+- Update Unicode and Emoji to 17.0
+  - Some files now have a new location in UCD
+- Update CLDR to v46
+- Update IVD to 2025-07-14
+
 ### 1.3.0
 
 - confusable: Add ignorables

data/lib/unicoder/builders/display_width.rb

@@ -4,15 +4,11 @@ module Unicoder
       include Builder
       include MultiDimensionalArrayBuilder
 
-      IGNORE_CATEGORIES     = %w[Cs Co Cn].freeze
-      ZERO_WIDTH_CATEGORIES = %w[Mn Me Cf].freeze
+      ZERO_WIDTH_CATEGORIES = %w[Mn Me Zl Zp Cf].freeze
 
-      ZERO_WIDTH_RANGES = [
+      ZERO_WIDTH_HANGUL = [
         *0x1160..0x11FF, # HANGUL JUNGSEONG
         *0xD7B0..0xD7FF, # HANGUL JUNGSEONG
-        *0x2060..0x206F, # Ignorables
-        *0xFFF0..0xFFF8, # Ignorables
-        *0xE0000..0xE0FFF, # Ignorables
       ].freeze
 
       WIDE_RANGES = [
@@ -34,19 +30,36 @@ module Unicoder
         0xD    =>  0, # \r CARRIAGE RETURN
         0xE    =>  0, #    SHIFT OUT
         0xF    =>  0, #    SHIFT IN
-        0x00AD =>  nil, #    SOFT HYPHEN
+        # 0x85   =>  0, #    NEXT LINE
+        0xAD   =>  nil, #  SOFT HYPHEN, nil = 1 (default)
         0x2E3A =>  2, #    TWO-EM DASH
         0x2E3B =>  3, #    THREE-EM DASH
       }.freeze
 
       def initialize_index
-        @index = []
+        @index = {
+          WIDTH_ONE: [],
+          WIDTH_TWO: [],
+        }
+        @ignorable = []
       end
 
       def parse!
-        parse_file :east_asian_width, :line, regex: /^(?<codepoints>\S+?)\s*;\s*(?<width>\S+)\s+#\s(?<category>\S+).*$/ do |line|
-          next if IGNORE_CATEGORIES.include?(line["category"])
+        # Find Ignorables
+        parse_file :core_properties, :line, begin: /^# Derived Property: Default_Ignorable_Code_Point$/, end: /^# ================================================$/, regex: /^(?<codepoints>\S+)\s+; Default_Ignorable_Code_Point.*$/ do |line|
+          if line["codepoints"]['..']
+            single_or_multiple_codepoints = Range.new(*line["codepoints"].split('..').map{ |codepoint|
+              codepoint.to_i(16)
+            })
+          else
+            single_or_multiple_codepoints = line["codepoints"].to_i(16)
+          end
 
+          @ignorable += [*single_or_multiple_codepoints]
+        end
+
+        # Assign based on East Asian Width
+        parse_file :east_asian_width, :line, regex: /^(?<codepoints>\S+?)\s*;\s*(?<width>\S+)\s+#\s(?<category>\S+).*$/ do |line|
           if line["codepoints"]['..']
             codepoints = Range.new(*line["codepoints"].split('..').map{ |codepoint|
               codepoint.to_i(16)
@@ -56,33 +69,45 @@ module Unicoder
           end
 
           codepoints.each{ |codepoint|
-            assign_codepoint codepoint, determine_width(codepoint, line["category"], line["width"])
+            assign :WIDTH_ONE, codepoint, determine_width(codepoint, line["category"], line["width"], 1)
+            assign :WIDTH_TWO, codepoint, determine_width(codepoint, line["category"], line["width"], 2)
           }
         end
 
-        ZERO_WIDTH_RANGES.each{ |codepoint|
-          assign_codepoint codepoint, 0
+        # Assign Ranges
+        ## Zero-width
+        (ZERO_WIDTH_HANGUL | @ignorable).each{ |codepoint|
+          assign :WIDTH_ONE, codepoint, 0
+          assign :WIDTH_TWO, codepoint, 0
         }
 
+        ## Full-width
         WIDE_RANGES.each{ |codepoint|
-          assign_codepoint codepoint, 2
+          assign :WIDTH_ONE, codepoint, 2
+          assign :WIDTH_TWO, codepoint, 2
         }
 
+        ## Table
         SPECIAL_WIDTHS.each{ |codepoint, value|
-          assign_codepoint codepoint, value
+          assign :WIDTH_ONE, codepoint, value
+          assign :WIDTH_TWO, codepoint, value
         }
 
-        4.times{ compress! }
+        # Compres Index
+        4.times{ compress! @index[:WIDTH_ONE] }
+        4.times{ compress! @index[:WIDTH_TWO] }
+        remove_trailing_nils! @index[:WIDTH_ONE]
+        remove_trailing_nils! @index[:WIDTH_TWO]
       end
 
-      def determine_width(codepoint, category, east_asian_width)
+      def determine_width(codepoint, category, east_asian_width, ambiguous)
         if  ( ZERO_WIDTH_CATEGORIES.include?(category) &&
               [codepoint].pack('U') !~ /\p{Cf}(?<=\p{Arabic})/ )
           0
         elsif east_asian_width == "F" || east_asian_width == "W"
           2
         elsif east_asian_width == "A"
-          :A
+          ambiguous == 1 ? nil : ambiguous
         else
           nil
         end

data/lib/unicoder/constants.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 module Unicoder
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 
   UNICODE_VERSIONS = %w[
+    17.0.0
     16.0.0
     15.1.0
     15.0.0
@@ -22,6 +23,7 @@ module Unicoder
   CURRENT_UNICODE_VERSION = UNICODE_VERSIONS.first
 
   EMOJI_VERSIONS = %w[
+   17.0
    16.0
    15.1
    15.0
@@ -38,6 +40,7 @@ module Unicoder
   ].freeze
 
   EMOJI_RELATED_UNICODE_VERSIONS = {
+   "17.0" => "17.0.0",
    "16.0" => "16.0.0",
    "15.1" => "15.1.0",
    "15.0" => "15.0.0",
@@ -55,11 +58,11 @@ module Unicoder
 
   CURRENT_EMOJI_VERSION = EMOJI_VERSIONS.first
 
-  IVD_VERSION = "2022-09-13"
+  IVD_VERSION = "2025-07-14"
 
-  CLDR_VERSION = "46"
+  CLDR_VERSION = "47"
 
-  UNICODE_DATA_ENDPOINT = "ftp://ftp.unicode.org/Public"
+  UNICODE_DATA_ENDPOINT = "http://ftp.unicode.org/Public"
 
   LOCAL_DATA_DIRECTORY = File.expand_path(File.dirname(__FILE__) + "/../../data/unicode").freeze
 
@@ -67,7 +70,8 @@ module Unicoder
     east_asian_width:          "/UNICODE_VERSION/ucd/EastAsianWidth.txt",
     unicode_data:              "/UNICODE_VERSION/ucd/UnicodeData.txt",
     name_aliases:              "/UNICODE_VERSION/ucd/NameAliases.txt",
-    confusables:               "/security/UNICODE_VERSION/confusables.txt",
+    confusables:               "/UNICODE_VERSION/security/confusables.txt",
+    confusables_before_17:     "/security/UNICODE_VERSION/confusables.txt",
     blocks:                    "/UNICODE_VERSION/ucd/Blocks.txt",
     core_properties:           "/UNICODE_VERSION/ucd/DerivedCoreProperties.txt",
     scripts:                   "/UNICODE_VERSION/ucd/Scripts.txt",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unicoder
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-04 00:00:00.000000000 Z
+date: 2025-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rationalist