unicode-scripts 1.9.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3af7d40f02bdf4a276addf67e1f3dc0bdb76b4ba7658b68fb7de9a433ddba2a3
-  data.tar.gz: 19af4744dc3fee1e6f61e07134fdcca25dd6883bcc71b60707f6e3e22a03e2d1
+  metadata.gz: feaabd20c3a3869a96e62e34d7c39b83739365549904ecdb129e83d9f73540d4
+  data.tar.gz: 40af16102c2aa63b35051f09b65cd8e2d14c32fbce21a7c802ea260121ade5b5
 SHA512:
-  metadata.gz: 27ce63fcb077fff302de67202d8418327c097c9bd1bfd9b2cfe34bf452148fefda1d3caf07a6ae3d3d106c4603220769d277aaa9a6774a5a979873ef7e929e53
-  data.tar.gz: ca20a49dc38d5cb9ca76747bcbd89f2b0d6256b39da6aa994fd9e69862ca159b8738f5a9c984e67ef23c8cdb6739d097b73c69adb6cd1d61bd499b1b5c47407f
+  metadata.gz: 8d5f215ed6b03d5192eef673d22f0705cac149e3701427570ab52b4e3c538ac1537b7ca5a0768a66e6d5ffdd4d66b9363b4fdafbdf74112df1e7e59ab639cf2c
+  data.tar.gz: 735b9611f0bfee72dd074a8873c3c269d23a150b6d7a9da64ca33b7d02c5a65316a45eaa47bb60fc44a554b70d3efbd3bcd8cf0c94185e01d7fdd5f767766839

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## CHANGELOG
 
+### 1.11.0
+
+- Add augmented scripts and mixed-script detection (as described in UTS39)
+
+### 1.10.0
+
+- Unicode 16.0
+
 ### 1.9.0
 
 - Unicode 15.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    unicode-scripts (1.9.0)
+    unicode-scripts (1.11.0)
 
 GEM
   remote: https://rubygems.org/

data/MIT-LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2016-2023 Jan Lelis, https://janlelis.com
+Copyright (c) 2016-2024 Jan Lelis, https://janlelis.com
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,12 +1,12 @@
 # Unicode::Scripts [![[version]](https://badge.fury.io/rb/unicode-scripts.svg)](https://badge.fury.io/rb/unicode-scripts)  [![[ci]](https://github.com/janlelis/unicode-scripts/workflows/Test/badge.svg)](https://github.com/janlelis/unicode-scripts/actions?query=workflow%3ATest)
 
-Retrieve the [Unicode script(s)](https://en.wikipedia.org/wiki/Script_%28Unicode%29) a string belongs to. Can also return the *Script_Extension* property which is defined as characters which are "commonly used with more than one script, but with a limited number of scripts".
+Retrieve all [Unicode script(s)](https://en.wikipedia.org/wiki/Script_%28Unicode%29) a string belongs to. Can also return the *Script_Extension* property (scx) which is defined as characters which are "commonly used with more than one script, but with a limited number of scripts".
 
-Unicode version: **15.1.0** (September 2023)
+Based on the *Script_Extension*, this library can also return the [augmented script set](https://www.unicode.org/reports/tr39/#def-augmented-script-set) to figure out if a string is **mixed-script** or **single-script**. Mixed scripts can be an indicator of suspicious user inputs.
 
-Supported Rubies: **3.2**, **3.1**, **3.0**
+Unicode version: **16.0.0** (September 2024)
 
-Old Rubies that might still work: **2.X**
+Supported Rubies: **3.x** (might work: **2.x**)
 
 ## Gemfile
 
@@ -14,7 +14,7 @@ Old Rubies that might still work: **2.X**
 gem "unicode-scripts"
 ```
 
-## Usage
+## Usage - Scripts and Script Extensions
 
 ```ruby
 require "unicode/scripts"
@@ -29,381 +29,104 @@ Unicode::Scripts.script("ᴦ") # => "Greek"
 
 # Script_Extension property
 Unicode::Scripts.script_extensions("॥")
-# => ["Bengali", "Devanagari", "Dogra", "Grantha", "Gujarati","Gunjala_Gondi", "Gurmukhi", "Kannada",
-"Khudawadi", "Limbu", "Mahajani", "Malayalam", "Masaram_Gondi", "Nandinagari", "Oriya", "Sinhala",
-"Syloti_Nagri", "Takri", "Tamil", "Telugu", "Tirhuta"]
+# => ["Bengali", "Devanagari", "Dogra", "Grantha", "Gujarati", "Gunjala_Gondi", "Gurmukhi","Gurung_Khema",
+      "Kannada","Khudawadi",  "Limbu",  "Mahajani",  "Malayalam", "Masaram_Gondi", "Nandinagari", "Ol_Onal",
+      "Oriya", "Sinhala", "Syloti_Nagri", "Takri", "Tamil", "Telugu", "Tirhuta"]
 ```
 
-## Hints
-### Regex Matching
+## Usage - Augmented Scripts
 
-If you have a string and want to match a substring/character from a specific Unicode script, you actually won't need this gem. Instead, you can use the [Regexp Unicode Property Syntax `\p{}`](https://ruby-doc.org/core/Regexp.html#class-Regexp-label-Character+Properties):
+Like script extensions, but adds meta scripts for Asian languages and treats _Common_/_Inherited_ values as ALL scripts.
 
 ```ruby
-"Coptic letter: ⲁ".scan(/\p{Coptic}/) # => ["ⲁ"]
+require "unicode/scripts"
+
+Unicode::Scripts.augmented_scripts("ねガ") # => ['Hira', 'Kana', 'Jpan']
+Unicode::Scripts.augmented_scripts("1") # => ["Adlm", "Aghb", "Ahom", … ]
 ```
 
-See [Idiosyncratic Ruby: Proper Unicoding](https://idiosyncratic-ruby.com/41-proper-unicoding.html) for more info.
+## Usage - Resolved Script
+
+Intersection of all augmented scripts per character.
+
+```ruby
+require "unicode/scripts"
+
+Unicode::Scripts.resolved_scripts("СігсӀе") # =>  [ 'Cyrl' ]
+Unicode::Scripts.resolved_scripts("Сirсlе") # =>  []
+Unicode::Scripts.resolved_scripts("𝖢𝗂𝗋𝖼𝗅𝖾") # => ['Adlm', 'Aghb', 'Ahom', … ]
+Unicode::Scripts.resolved_scripts("1") # => ['Adlm','Aghb', 'Ahom', … ]
+Unicode::Scripts.resolved_scripts("ねガ") # =>  ['Hira', 'Kana', 'Jpan']
+```
+
+Please note that the **resolved script** can contain multiple scripts, as per standard.
+
+## Usage - Mixed-Script Detection
 
-### Script Names
+Mixed-script if resolved script set is empty, single-script otherwise.
+
+```ruby
+require "unicode/scripts"
+
+Unicode::Scripts.mixed?("СігсӀе"); # => false
+Unicode::Scripts.mixed?("Сirсlе"); # => true
+Unicode::Scripts.mixed?("𝖢𝗂𝗋𝖼𝗅𝖾"); # => false
+Unicode::Scripts.mixed?("1"); # => false
+Unicode::Scripts.mixed?("ねガ"); # => false
+
+Unicode::Scripts.single?("СігсӀе"); # => true
+Unicode::Scripts.single?("Сirсlе"); # => false
+Unicode::Scripts.single?("𝖢𝗂𝗋𝖼𝗅𝖾"); # => true
+Unicode::Scripts.single?("1"); # => true
+Unicode::Scripts.single?("ねガ"); # => true
+```
+
+Please note that a **single-script** string might actually contain multiple scripts, as per standard (e.g. for Asian languages)
+
+### List of All Scripts
 
 You can extract all script names from the gem like this:
 
 ```ruby
 require "unicode/scripts"
-puts Unicode::Scripts.names
-
-# # # Output # # #
-
-Adlam
-Ahom
-Anatolian_Hieroglyphs
-Arabic
-Armenian
-Avestan
-Balinese
-Bamum
-Bassa_Vah
-Batak
-Bengali
-Bhaiksuki
-Bopomofo
-Brahmi
-Braille
-Buginese
-Buhid
-Canadian_Aboriginal
-Carian
-Caucasian_Albanian
-Chakma
-Cham
-Cherokee
-Chorasmian
-Common
-Coptic
-Cuneiform
-Cypriot
-Cypro_Minoan
-Cyrillic
-Deseret
-Devanagari
-Dives_Akuru
-Dogra
-Duployan
-Egyptian_Hieroglyphs
-Elbasan
-Elymaic
-Ethiopic
-Georgian
-Glagolitic
-Gothic
-Grantha
-Greek
-Gujarati
-Gunjala_Gondi
-Gurmukhi
-Han
-Hangul
-Hanifi_Rohingya
-Hanunoo
-Hatran
-Hebrew
-Hiragana
-Imperial_Aramaic
-Inherited
-Inscriptional_Pahlavi
-Inscriptional_Parthian
-Javanese
-Kaithi
-Kannada
-Katakana
-Katakana_Or_Hiragana
-Kawi
-Kayah_Li
-Kharoshthi
-Khitan_Small_Script
-Khmer
-Khojki
-Khudawadi
-Lao
-Latin
-Lepcha
-Limbu
-Linear_A
-Linear_B
-Lisu
-Lycian
-Lydian
-Mahajani
-Makasar
-Malayalam
-Mandaic
-Manichaean
-Marchen
-Masaram_Gondi
-Medefaidrin
-Meetei_Mayek
-Mende_Kikakui
-Meroitic_Cursive
-Meroitic_Hieroglyphs
-Miao
-Modi
-Mongolian
-Mro
-Multani
-Myanmar
-Nabataean
-Nag_Mundari
-Nandinagari
-New_Tai_Lue
-Newa
-Nko
-Nushu
-Nyiakeng_Puachue_Hmong
-Ogham
-Ol_Chiki
-Old_Hungarian
-Old_Italic
-Old_North_Arabian
-Old_Permic
-Old_Persian
-Old_Sogdian
-Old_South_Arabian
-Old_Turkic
-Old_Uyghur
-Oriya
-Osage
-Osmanya
-Pahawh_Hmong
-Palmyrene
-Pau_Cin_Hau
-Phags_Pa
-Phoenician
-Psalter_Pahlavi
-Rejang
-Runic
-Samaritan
-Saurashtra
-Sharada
-Shavian
-Siddham
-SignWriting
-Sinhala
-Sogdian
-Sora_Sompeng
-Soyombo
-Sundanese
-Syloti_Nagri
-Syriac
-Tagalog
-Tagbanwa
-Tai_Le
-Tai_Tham
-Tai_Viet
-Takri
-Tamil
-Tangsa
-Tangut
-Telugu
-Thaana
-Thai
-Tibetan
-Tifinagh
-Tirhuta
-Toto
-Ugaritic
-Unknown
-Vai
-Vithkuqi
-Wancho
-Warang_Citi
-Yezidi
-Yi
-Zanabazar_Square
+puts Unicode::Scripts.names # list of scripts
 ```
 
-### Short Script Names
+To get all 4 letter script codes (ISO 15924):
+
+```ruby
+require "unicode/scripts"
+puts Unicode::Scripts.names(format: :short) # list of scripts
+```
 
-You can extract all 4 letter script names from the gem like this:
+Augmented scripts: 
 
 ```ruby
 require "unicode/scripts"
-puts Unicode::Scripts.names(format: :short)
-
-# # # Output # # #
-
-Adlm
-Aghb
-Ahom
-Arab
-Armi
-Armn
-Avst
-Bali
-Bamu
-Bass
-Batk
-Beng
-Bhks
-Bopo
-Brah
-Brai
-Bugi
-Buhd
-Cakm
-Cans
-Cari
-Cham
-Cher
-Chrs
-Copt
-Cpmn
-Cprt
-Cyrl
-Deva
-Diak
-Dogr
-Dsrt
-Dupl
-Egyp
-Elba
-Elym
-Ethi
-Geor
-Glag
-Gong
-Gonm
-Goth
-Gran
-Grek
-Gujr
-Guru
-Hang
-Hani
-Hano
-Hatr
-Hebr
-Hira
-Hluw
-Hmng
-Hmnp
-Hrkt
-Hung
-Ital
-Java
-Kali
-Kana
-Kawi
-Khar
-Khmr
-Khoj
-Kits
-Knda
-Kthi
-Lana
-Laoo
-Latn
-Lepc
-Limb
-Lina
-Linb
-Lisu
-Lyci
-Lydi
-Mahj
-Maka
-Mand
-Mani
-Marc
-Medf
-Mend
-Merc
-Mero
-Mlym
-Modi
-Mong
-Mroo
-Mtei
-Mult
-Mymr
-Nagm
-Nand
-Narb
-Nbat
-Newa
-Nkoo
-Nshu
-Ogam
-Olck
-Orkh
-Orya
-Osge
-Osma
-Ougr
-Palm
-Pauc
-Perm
-Phag
-Phli
-Phlp
-Phnx
-Plrd
-Prti
-Qaac
-Qaai
-Rjng
-Rohg
-Runr
-Samr
-Sarb
-Saur
-Sgnw
-Shaw
-Shrd
-Sidd
-Sind
-Sinh
-Sogd
-Sogo
-Sora
-Soyo
-Sund
-Sylo
-Syrc
-Tagb
-Takr
-Tale
-Talu
-Taml
-Tang
-Tavt
-Telu
-Tfng
-Tglg
-Thaa
-Thai
-Tibt
-Tirh
-Tnsa
-Toto
-Ugar
-Vaii
-Vith
-Wara
-Wcho
-Xpeo
-Xsux
-Yezi
-Yiii
-Zanb
-Zinh
-Zyyy
-Zzzz
+puts Unicode::Scripts.names(format: :short, augmented: :only)
 ```
 
-See [unicode-x](https://github.com/janlelis/unicode-x) for more Unicode related micro libraries.
+You can find a list of all scripts in Unicode, with links to Wikipedia on [character.construction/scripts](https://character.construction/scripts)
+
+## Hints
+### Regex Matching
+
+If you have a string and want to match a substring/character from a specific Unicode script, you actually won't need this gem. Instead, you can use the [Regexp Unicode Property Syntax `\p{}`](https://ruby-doc.org/core/Regexp.html#class-Regexp-label-Character+Properties):
+
+```ruby
+"Coptic letter: ⲁ".scan(/\p{Coptic}/) # => ["ⲁ"]
+```
+
+See [Idiosyncratic Ruby: Proper Unicoding](https://idiosyncratic-ruby.com/41-proper-unicoding.html) for more info.
+
+## Also See
+
+- JavaScript implementation (same data & algorithms): [unicode-script.js](https://github.com/janlelis/unicode-script.js)
+- Index created with: [unicoder](https://github.com/janlelis/unicoder)
+- Get the Unicode blocks of a string: [unicode-blocks gem](https://github.com/janlelis/unicode-blocks)
+- See [unicode-x](https://github.com/janlelis/unicode-x) for more Unicode related micro libraries for Ruby.
 
 ## MIT License
 
-- Copyright (C) 2016-2023 Jan Lelis <https://janlelis.com>. Released under the MIT license.
+- Copyright (C) 2016-2024 Jan Lelis <https://janlelis.com>. Released under the MIT license.
 - Unicode data: https://www.unicode.org/copyright.html#Exhibit1

data/lib/unicode/scripts/constants.rb

@@ -2,9 +2,11 @@
 
 module Unicode
   module Scripts
-    VERSION = "1.9.0"
-    UNICODE_VERSION = "15.1.0"
+    VERSION = "1.11.0"
+    UNICODE_VERSION = "16.0.0"
     DATA_DIRECTORY = File.expand_path(File.dirname(__FILE__) + "/../../../data/").freeze
     INDEX_FILENAME = (DATA_DIRECTORY + "/scripts.marshal.gz").freeze
+
+    AUGMENTED_SCRIPT_CODES = ["Hanb", "Jpan", "Kore"]
   end
 end

data/lib/unicode/scripts.rb

@@ -46,11 +46,77 @@ module Unicode
       }.sort
     end
 
-    def self.names(format: :long)
+    def self.augmented_scripts(string)
       require_relative 'scripts/index' unless defined? ::Unicode::Scripts::INDEX
-      format == :long ?
-          INDEX[:SCRIPT_NAMES].sort :
-          INDEX[:SCRIPT_ALIASES].keys.sort
+
+      augmented = string.each_codepoint.inject([]){ |res, codepoint|
+        if new_scripts = INDEX[:SCRIPT_EXTENSIONS][codepoint]
+          script_extension_names = new_scripts.map{ |new_script|
+            INDEX[:SCRIPT_ALIASES].key(new_script)
+          }
+        else
+          script_extension_names = scripts([codepoint].pack("U"), format: :short)
+        end
+
+        res | script_extension_names
+      }
+
+      if augmented.include? "Hani"
+        augmented |= ["Hanb", "Jpan", "Kore"]
+      end
+      if augmented.include?("Hira") || augmented.include?("Kana")
+        augmented |= ["Jpan"]
+      end
+      if augmented.include? "Hang"
+        augmented |= ["Kore"]
+      end
+      if augmented.include? "Bopo"
+        augmented |= ["Hanb"]
+      end
+      if augmented.include?("Zyyy") || augmented.include?("Zinh")
+        augmented |= names(format: :short, augmented: :include )
+      end
+    
+      augmented.sort
+    end
+
+    def self.resolved_scripts(string)
+      string.chars.reduce(
+        Unicode::Scripts.names(format: :short, augmented: :include)
+      ){ |acc, char|
+        acc & augmented_scripts(char)
+      }
+    end
+
+    def self.mixed?(string)
+      resolved_scripts(string).empty?
+    end
+
+    def self.single?(string)
+      !resolved_scripts(string).empty?
+    end
+
+    # Lists scripts. Options:
+    # - format - :long, :short
+    # - augmented - :include, :exclude, :only
+    def self.names(format: :long, augmented: :exclude)
+      if format == :long && augmented != :exclude
+        raise ArgumentError, "only short four-letter script codes (ISO 15924) supported when listing augmented scripts"
+      end
+
+      if augmented == :only
+        return AUGMENTED_SCRIPT_CODES
+      end
+
+      require_relative 'scripts/index' unless defined? ::Unicode::Scripts::INDEX
+
+      if format == :long
+        INDEX[:SCRIPT_NAMES].sort
+      elsif augmented == :exclude
+        INDEX[:SCRIPT_ALIASES].keys.sort
+      else
+        (INDEX[:SCRIPT_ALIASES].keys + AUGMENTED_SCRIPT_CODES).sort
+      end
     end
   end
 end

data/spec/unicode_scripts_spec.rb

@@ -63,6 +63,7 @@ describe Unicode::Scripts do
         "Gujarati",
         "Gunjala_Gondi",
         "Gurmukhi",
+        "Gurung_Khema",
         "Kannada",
         "Khudawadi",
         "Limbu",
@@ -70,6 +71,7 @@ describe Unicode::Scripts do
         "Malayalam",
         "Masaram_Gondi",
         "Nandinagari",
+        "Ol_Onal",
         "Oriya",
         "Sinhala",
         "Syloti_Nagri",
@@ -89,12 +91,14 @@ describe Unicode::Scripts do
         "Gonm",
         "Gran",
         "Gujr",
+        "Gukh",
         "Guru",
         "Knda",
         "Limb",
         "Mahj",
         "Mlym",
         "Nand",
+        "Onao",
         "Orya",
         "Sind",
         "Sinh",
@@ -126,11 +130,61 @@ describe Unicode::Scripts do
     end
   end
 
+  describe ".augmented_scripts" do
+    it "will always return an Array" do
+      assert_equal [], Unicode::Scripts.augmented_scripts("")
+    end
+
+    it "will return all extended scripts that characters in the string belong to + augmented" do
+      assert_equal ["Hira", "Jpan", "Kana"], Unicode::Scripts.augmented_scripts("ねガ")
+    end
+    
+    it "will replace Common with all scripts" do
+      assert_equal \
+        Unicode::Scripts.names(format: :short, augmented: :include),
+        Unicode::Scripts.augmented_scripts("1")
+    end
+  end
+
+  describe ".resolved_scripts" do
+    it "return intersection of augmented scripts per character" do
+      assert_equal ["Cyrl"], Unicode::Scripts.resolved_scripts("СігсӀе")
+      assert_equal [], Unicode::Scripts.resolved_scripts("Сirсlе")
+      assert_equal \
+        Unicode::Scripts.names(format: :short, augmented: :include),
+        Unicode::Scripts.resolved_scripts("𝖢𝗂𝗋𝖼𝗅𝖾")
+    end
+  end
+
+  describe "mixed?" do
+    it "will return true if .resolved_scripts(string) is empty" do
+      assert_equal false, Unicode::Scripts.mixed?("СігсӀе")
+      assert Unicode::Scripts.mixed?("Сirсlе")
+      assert_equal false, Unicode::Scripts.mixed?("𝖢𝗂𝗋𝖼𝗅𝖾")
+      assert_equal false, Unicode::Scripts.mixed?("1")
+      assert_equal false, Unicode::Scripts.mixed?("ねガ")
+    end
+  end
+  
+  describe "single?" do
+    it "will return true if .resolved_scripts(string) is not empty" do
+      assert Unicode::Scripts.single?("СігсӀе")
+      assert_equal false, Unicode::Scripts.single?("Сirсlе")
+      assert Unicode::Scripts.single?("𝖢𝗂𝗋𝖼𝗅𝖾")
+      assert Unicode::Scripts.single?("1")
+      assert Unicode::Scripts.single?("ねガ")
+    end
+  end
+
   describe ".names" do
     it "will return a list of all script names" do
       assert_kind_of Array, Unicode::Scripts.names
       assert_includes Unicode::Scripts.names, "Inscriptional_Parthian"
     end
+
+    it "will return a list of all augmented script codes" do
+      assert_equal Unicode::Scripts.names(format: :short, augmented: :only), ["Hanb", "Jpan", "Kore"]
+    end
   end
 end
 

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: unicode-scripts
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.11.0
 platform: ruby
 authors:
 - Jan Lelis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-01 00:00:00.000000000 Z
+date: 2024-11-03 00:00:00.000000000 Z
 dependencies: []
-description: "[Unicode 15.1.0] Retrieve the Unicode script(s) a string belongs to.
+description: "[Unicode 16.0.0] Retrieve the Unicode script(s) a string belongs to.
   Can also return the Script_Extension property which is defined as characters which
   are 'commonly used with more than one script, but with a limited number of scripts'. "
 email:
@@ -39,7 +39,7 @@ licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -54,8 +54,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.4
-signing_key: 
+rubygems_version: 3.5.21
+signing_key:
 specification_version: 4
 summary: Which script(s) does a Unicode string belong to?
 test_files: