unicoder 1.1.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f531e7ea0b5d27ea2bcb05bba6ddd93e0b7325d3d097abaf9ed815cc11d9a197
-  data.tar.gz: 8c3d133fb02f3b3d516c9a07390f768509eb3b22f2ae2850ad134819b0390462
+  metadata.gz: 9064718b14baf32790bb6f9a705aab4d07231771f1c663546dea90413b0b68d9
+  data.tar.gz: 40f6145a58220620accb91bda68b06c2e2ceeea12b3cee4dc6a1c021e8fc9194
 SHA512:
-  metadata.gz: 05bd755d93de557ca8786c740675e88847da99c31c07a1517040f51fc0e9846537eeb9317087d955f4cc65d4c6d2434cd83a5216ad21dba8f583edd1166eb10a
-  data.tar.gz: 1644c4b6dee2db05f6d7ec91a5250798045e358e3580e141327f39d8c150cce5ad7d687cce8ffca796ca1de728151bb0a29b9d8084ae7fe6b87ec4e080fb95cb
+  metadata.gz: f69ddbfcef5269be204fed89d00018312a623330c28382531922fb5fe0093c9dfee7aa7e7bc5bff069ba1e57341fbb62e915d34bb27f37cec10283b7e8bbc678
+  data.tar.gz: f4499a5ce299023e3752ed69df04e148d9937ac355ae7e415525f46c402761992540de6ae5fc8d1b22ab4a2dc154220d722ad940d4878fca3c9e3f6a306d00a1

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 ## CHANGELOG
 
+### 1.3.0
+
+- confusable: Add ignorables
+- confusable: Nest index and make ESM/charkeys version, fix ";"
+
+### 1.2.1
+
+- name: Fix some CJK Compatibility Ideographs not declared in CP_RANGES
+
+### 1.2.0
+
+- Change format for sequence_name's sub-index for unqalified Emoji sequences
+
+### 1.1.2
+
+- Update CLDR to v46
+
 ### 1.1.1
 
 - Fix bug related to unsafe characters

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    unicoder (1.1.1)
+    unicoder (1.3.0)
       oga (~> 2.9)
       rationalist (~> 2.0)
       rubyzip (~> 1.2)

data/README.md

@@ -39,7 +39,9 @@ Index Name    | Module
 --------------|----
 name, sequence\_name, type | [unicode-name.js](https://github.com/janlelis/unicode-name.js)
 numeric\_value| [unicode-number.js](https://github.com/janlelis/unicode-number.js)
-scripts | [unicode-scripts.js](https://github.com/janlelis/unicode-scripts.js)
+scripts | [unicode-script.js](https://github.com/janlelis/unicode-script.js)
+blocks | [unicode-block.js](https://github.com/janlelis/unicode-block.js)
+categories | [unicode-category.js](https://github.com/janlelis/unicode-category.js)
 
 ## MIT License
 

data/lib/unicoder/builder.rb

@@ -73,8 +73,18 @@ module Unicoder
       file = File.read(LOCAL_DATA_DIRECTORY + filename)
 
       if parse_mode == :line
+        active = !parse_options[:begin]
+        
         file.each_line{ |line|
-          yield Hash[ $~.names.zip( $~.captures ) ] if line =~ parse_options[:regex]
+          if !active && parse_options[:begin] && line.match?(parse_options[:begin])
+            active = true
+          elsif active && parse_options[:end] && line.match?(parse_options[:end])
+            active = false
+          end
+
+          if active
+            yield Hash[ $~.names.zip( $~.captures ) ] if line =~ parse_options[:regex]
+          end
         }
       elsif parse_mode == :xml
         require "oga"

data/lib/unicoder/builders/categories.rb

@@ -9,6 +9,12 @@ module Unicoder
         @index = {
           CATEGORIES: [],
           CATEGORY_NAMES: {},
+          OFFSETS: [
+            0x10000,
+            0x1000,
+            0x100,
+            0x10
+          ],
         }
         @range_start = nil
       end

data/lib/unicoder/builders/confusable.rb

@@ -3,17 +3,38 @@ module Unicoder
     class Confusable
       include Builder
 
+      def initialize_index
+        @index = {
+          CONFUSABLE: {},
+          IGNORABLE: [],
+        }
+      end
+
       def parse!
-        parse_file :confusables, :line, regex: /^(?<from>\S+)\s+;\s+(?<to>.+)\s+;.*$/ do |line|
+        parse_file :confusables, :line, regex: /^(?<from>\S+)\s+;\s+(?<to>.+?)\s+;.*$/ do |line|
           source = line["from"].to_i(16)
           if line["to"].include?(" ")
             replace_with = line["to"].split(" ").map{ |codepoint|
+              cp = codepoint.to_i(16)
+              option =~ /charvalues/ ? [cp].pack("U") : cp
+            }
+          else
+            cp = line["to"].to_i(16)
+            replace_with = option =~ /charvalues/ ? [cp].pack("U") : cp
+          end
+          assign :CONFUSABLE, source, replace_with
+        end
+
+        parse_file :core_properties, :line, begin: /^# Derived Property: Default_Ignorable_Code_Point$/, end: /^# ================================================$/, regex: /^(?<codepoints>\S+)\s+; Default_Ignorable_Code_Point.*$/ do |line|
+          if line["codepoints"]['..']
+            single_or_multiple_codepoints = line["codepoints"].split('..').map{ |codepoint|
               codepoint.to_i(16)
             }
           else
-            replace_with = line["to"].to_i(16)
+            single_or_multiple_codepoints = line["codepoints"].to_i(16)
           end
-          @index[source] = replace_with
+
+          @index[:IGNORABLE] << single_or_multiple_codepoints
         end
       end
     end

data/lib/unicoder/builders/name.rb

@@ -27,7 +27,7 @@ module Unicoder
             "EGYPTIAN HIEROGLYPH-" => [[0x13460, 0x143FA]],
             "KHITAN SMALL SCRIPT CHARACTER-" => [[0x18B00, 0x18CFF]],
             "NUSHU CHARACTER-" => [[0x1B170, 0x1B2FB]],
-            "CJK COMPATIBILITY IDEOGRAPH-" => [[0x2F800, 0x2FA1D]],
+            "CJK COMPATIBILITY IDEOGRAPH-" => [[0xF900, 0xFAFF], [0x2F800, 0x2FA1D]],
           },
           # see https://en.wikipedia.org/wiki/Korean_language_and_computers#Hangul_Syllables_Area
           JAMO: {

data/lib/unicoder/builders/sequence_name.rb

@@ -11,7 +11,7 @@ module Unicoder
       def initialize_index
         @index = {
           SEQUENCES: {},
-          SEQUENCES_NOT_QUALIFIED: {},
+          EMOJI_NOT_QUALIFIED: {},
         }
         @words = []
       end
@@ -74,8 +74,12 @@ module Unicoder
           name = line["name"].gsub(/\\x{(\h+)}/){ [$1.to_i(16)].pack("U") }.upcase
           codepoints = line["codepoints"].split.map{|cp| cp.to_i(16) }
           assign_codepoint codepoints, name
+
+
+          # Build all combinations of VS16 present and missing and add to second index
           if codepoints.include?(0xFE0F)
-            # Build all combinations of VS16 present and missing
+            sequence = codepoints.pack("U*")
+
             codepoints.slice_after(0xFE0F).reduce([[]]){|acc,cur|
               if cur.include? 0xFE0F
                 acc.flat_map{|prev| [prev + (cur - [0xFE0F]), prev + cur] }
@@ -85,13 +89,13 @@ module Unicoder
             }.
             select {|sub_codepoints| sub_codepoints != codepoints }.
             each { |sub_codepoints|
-              assign_codepoint (sub_codepoints), name, @index[:SEQUENCES_NOT_QUALIFIED]
+              sub_sequence = sub_codepoints.pack("U*")
+              @index[:EMOJI_NOT_QUALIFIED][sub_sequence] = sequence
             }
           end
         end
 
         replace_common_words! :SEQUENCES, @words, REPLACE_COUNT, REPLACE_BASE, REPLACE_MIN_WORD_LENGTH
-        replace_common_words! :SEQUENCES_NOT_QUALIFIED, @words, REPLACE_COUNT, REPLACE_BASE, REPLACE_MIN_WORD_LENGTH
       end
     end
   end

data/lib/unicoder/constants.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Unicoder
-  VERSION = "1.1.1"
+  VERSION = "1.3.0"
 
   UNICODE_VERSIONS = %w[
     16.0.0
@@ -57,7 +57,7 @@ module Unicoder
 
   IVD_VERSION = "2022-09-13"
 
-  CLDR_VERSION = "45"
+  CLDR_VERSION = "46"
 
   UNICODE_DATA_ENDPOINT = "ftp://ftp.unicode.org/Public"
 
@@ -69,6 +69,7 @@ module Unicoder
     name_aliases:              "/UNICODE_VERSION/ucd/NameAliases.txt",
     confusables:               "/security/UNICODE_VERSION/confusables.txt",
     blocks:                    "/UNICODE_VERSION/ucd/Blocks.txt",
+    core_properties:           "/UNICODE_VERSION/ucd/DerivedCoreProperties.txt",
     scripts:                   "/UNICODE_VERSION/ucd/Scripts.txt",
     script_extensions:         "/UNICODE_VERSION/ucd/ScriptExtensions.txt",
     property_value_aliases:    "/UNICODE_VERSION/ucd/PropertyValueAliases.txt",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unicoder
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-14 00:00:00.000000000 Z
+date: 2024-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rationalist
@@ -61,7 +61,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile

data/.travis.yml

@@ -1,20 +0,0 @@
-sudo: false
-language: ruby
-
-rvm:
-- 2.7
-- 2.6
-- 2.5
-- 2.4
-- 2.3
-- ruby-head
-- jruby-9.2.9.0
-- truffleruby
-
-matrix:
-  allow_failures:
-    - rvm: 2.3
-    - rvm: ruby-head
-    - rvm: jruby-2.9.2.0
-    - rvm: truffleruby
-#   fast_finish: true