unicode-emoji 3.7.0 → 4.0.0

data/spec/emoji_test_txt_spec.rb

@@ -0,0 +1,181 @@
+require_relative "../lib/unicode/emoji"
+require "minitest/autorun"
+require "open-uri"
+
+def iterate_emoji
+  EMOJI_TEST_FILE.scan(/^(?:# (?<sub>sub)?group: (?<group_name>.*)$)|(?:(?<codepoints>.+?)\s*; (?<qual_status>.+?)-?qualified )/) do
+    if $~[:codepoints]
+      yield $~[:codepoints].split.map{|e| e.to_i(16)}.pack("U*"), $~[:qual_status]
+    end
+  end
+end
+
+describe "emoji-test.txt" do
+  EMOJI_TEST_FILE = begin
+    emoji_test_path = File.join(__dir__, "data/emoji-test.txt")
+    if File.exist? emoji_test_path
+      file = File.read(emoji_test_path)
+    else
+      puts "Downloading emoji-test.txt from the consortium"
+      URI.open "https://www.unicode.org/Public/emoji/#{Unicode::Emoji::EMOJI_VERSION}/emoji-test.txt" do |f|
+        file = f.read
+        File.write(File.join(__dir__, "data/emoji-test.txt"), @file)
+      end
+    end
+
+    file
+  end
+
+  # qual_status:
+  # - fully - fully-qualified emoji sequences
+  # - minimally - minimallyq-ualified emoji sequences (some VS16 missing, but not first one)
+  # - un - unqualified emoji sequences (some VS16 missing)
+
+  describe "REGEX" do
+    describe "detects fully-qualified emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully"
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_INCLUDE_TEXT" do
+    describe "detects fully-qualified emoji and (unqualified) singleton text emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" || qual_status == "un" && emoji.size <= 2
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_TEXT]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_TEXT]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_INCLUDE_MQE" do
+    describe "detects fully-qualified emoji and minimally-qualified emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" || qual_status == "minimally"
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_MQE]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_MQE]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_INCLUDE_MQE_UQE" do
+    describe "detects all emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          assert_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_MQE_UQE]
+        end
+      end
+    end
+  end
+
+  describe "REGEX_VALID" do
+    describe "detects fully-qualified, minimally-qualified emoji, and unqualified emoji with ZWJ" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" || qual_status == "minimally" || qual_status == "un" && emoji.size >= 3
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_VALID]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_VALID]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_VALID_INCLUDE_TEXT" do
+    describe "detects all emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          assert_equal emoji, emoji[Unicode::Emoji::REGEX_VALID_INCLUDE_TEXT]
+        end
+      end
+    end
+  end
+
+  describe "REGEX_WELL_FORMED" do
+    describe "detects fully-qualified, minimally-qualified emoji, and unqualified emoji with ZWJ" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" || qual_status == "minimally" || qual_status == "un" && emoji.size >= 3
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_WELL_FORMED]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_WELL_FORMED]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_WELL_FORMED_INCLUDE_TEXT" do
+    describe "detects all emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          assert_equal emoji, emoji[Unicode::Emoji::REGEX_WELL_FORMED_INCLUDE_TEXT]
+        end
+      end
+    end
+  end
+
+  describe "REGEX_POSSIBLE" do
+    describe "detects all emoji, except unqualified keycap sequences" do
+      # fixing test not regex, since implementation of this regex should match the one in the standard
+      unqualified_keycaps = Unicode::Emoji::EMOJI_KEYCAPS.map{|keycap|
+        [keycap, Unicode::Emoji::EMOJI_KEYCAP_SUFFIX].pack("U*")
+      }
+
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if !unqualified_keycaps.include?(emoji)
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_POSSIBLE]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_POSSIBLE]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_TEXT" do
+    describe "detects (unqualified) singleton text emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          # if qual_status == "un" && emoji =~ /^.[\u{FE0E 20E3}]?$/
+          if qual_status == "un" && emoji.size <= 2
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_TEXT]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_TEXT]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_BASIC" do
+    describe "detects (fully-qualified) singleton emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" && emoji =~ /^.\u{FE0F}?$/
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_BASIC]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_BASIC]
+          end
+        end
+      end
+    end
+  end
+end

data/spec/unicode_emoji_spec.rb

@@ -3,12 +3,12 @@ require "minitest/autorun"
 
 describe Unicode::Emoji do
   describe ".properties" do
-    it "returns an Array for Emoji properties if has codepoints" do
+    it "returns an Array of Emoji properties for given codepoint" do
       assert_equal ["Emoji", "Emoji_Presentation", "Extended_Pictographic"], Unicode::Emoji.properties("😴")
       assert_equal ["Emoji", "Extended_Pictographic"], Unicode::Emoji.properties("♠")
     end
 
-    it "returns nil for Emoji properties if has no codepoints" do
+    it "returns nil if codepoint has no Emoji prop" do
       assert_nil Unicode::Emoji.properties("A")
     end
   end
@@ -39,8 +39,19 @@ describe Unicode::Emoji do
       assert_equal "▶\u{FE0F}", $&
     end
 
-    it "does not match singleton 'component' emoji codepoints" do
+    it "matches singleton skin tone modifiers and hair components" do
       "🏻 light skin tone" =~ Unicode::Emoji::REGEX
+      assert_equal "🏻", $&
+
+      "🦰 emoji component red hair" =~ Unicode::Emoji::REGEX
+      assert_equal "🦰", $&
+    end
+
+    it "does not match singleton components that are not skin tone modifiers or hair components" do
+      "1 digit one" =~ Unicode::Emoji::REGEX
+      assert_nil $&
+
+      "🇦 regional indicator symbol letter a" =~ Unicode::Emoji::REGEX
       assert_nil $&
     end
 
@@ -92,6 +103,16 @@ describe Unicode::Emoji do
       assert_equal "🤾🏽‍♀️", $&
     end
 
+    it "does not match MQE zwj sequences" do
+      "🤾🏽‍♀ woman playing handball: medium skin tone, missing VS16" =~ Unicode::Emoji::REGEX
+      refute_equal "🤾🏽‍♀", $&
+    end
+
+    it "does not match UQE emoji" do
+      "🏌‍♂️ man golfing, missing VS16" =~ Unicode::Emoji::REGEX
+      refute_equal "🏌‍♂️", $&
+    end
+
     it "does not match valid zwj sequences that are not recommended" do
       "🤠‍🤢 vomiting cowboy" =~ Unicode::Emoji::REGEX
       assert_equal "🤠", $&
@@ -132,8 +153,6 @@ describe Unicode::Emoji do
       assert_equal "🪾", $&
     end
 
-    
-
     # See gh#12 and https://github.com/matt17r/nw5k/commit/05a34d3c9211a23e5ae6853bb19fd2f224779ef4#diff-afb6f8bc3bae71b75743e00882a060863e2430cbe858ec9014e5956504dfc61cR2
     it "matches family emoji correctly" do
       ["👨‍👩‍👧‍👦", "👨‍👩‍👦‍👦", "👨‍👩‍👧‍👧", "👨‍👨‍👧‍👦", "👨‍👨‍👦‍👦", "👨‍👨‍👧‍👧", "👩‍👩‍👧‍👦", "👩‍👩‍👦‍👦", "👩‍👩‍👧‍👧", "👨‍👦‍👦", "👨‍👧‍👦", "👨‍👧‍👧", "👩‍👦‍👦", "👩‍👧‍👦", "👩‍👧‍👧"].each { |family|
@@ -142,6 +161,30 @@ describe Unicode::Emoji do
     end
   end
 
+  describe "REGEX_INCLUDE_MQE" do
+    it "matches MQE emoji" do
+      "🤾🏽‍♀ woman playing handball: medium skin tone, missing VS16" =~ Unicode::Emoji::REGEX_INCLUDE_MQE
+      assert_equal "🤾🏽‍♀", $&
+    end
+
+    it "does not match UQE emoji" do
+      "🏌‍♂️ man golfing, missing VS16" =~ Unicode::Emoji::REGEX_INCLUDE_MQE
+      refute_equal "🏌‍♂️", $&
+    end
+  end
+
+  describe "REGEX_INCLUDE_MQE_UQE" do
+    it "matches MQE emoji" do
+      "🤾🏽‍♀ woman playing handball: medium skin tone, missing VS16" =~ Unicode::Emoji::REGEX_INCLUDE_MQE_UQE
+      assert_equal "🤾🏽‍♀", $&
+    end
+
+    it "matches UQE emoji" do
+      "🏌‍♂️ man golfing, missing VS16" =~ Unicode::Emoji::REGEX_INCLUDE_MQE_UQE
+      assert_equal "🏌‍♂️", $&
+    end
+  end
+
   describe "REGEX_VALID" do
     it "matches most singleton emoji codepoints" do
       "😴 sleeping face" =~ Unicode::Emoji::REGEX_VALID
@@ -168,8 +211,19 @@ describe Unicode::Emoji do
       assert_equal "▶\u{FE0F}", $&
     end
 
-    it "does not match singleton 'component' emoji codepoints" do
+    it "matches singleton skin tone modifiers and hair components" do
       "🏻 light skin tone" =~ Unicode::Emoji::REGEX_VALID
+      assert_equal "🏻", $&
+
+      "🦰 emoji component red hair" =~ Unicode::Emoji::REGEX_VALID
+      assert_equal "🦰", $&
+    end
+
+    it "does not match singleton components that are not skin tone modifiers or hair components" do
+      "1 digit one" =~ Unicode::Emoji::REGEX_VALID
+      assert_nil $&
+
+      "🇦 regional indicator symbol letter a" =~ Unicode::Emoji::REGEX_VALID
       assert_nil $&
     end
 
@@ -263,8 +317,19 @@ describe Unicode::Emoji do
       assert_equal "▶\u{FE0F}", $&
     end
 
-    it "does not match singleton 'component' emoji codepoints" do
+    it "matches singleton skin tone modifiers and hair components" do
       "🏻 light skin tone" =~ Unicode::Emoji::REGEX_WELL_FORMED
+      assert_equal "🏻", $&
+
+      "🦰 emoji component red hair" =~ Unicode::Emoji::REGEX_WELL_FORMED
+      assert_equal "🦰", $&
+    end
+
+    it "does not match singleton components that are not skin tone modifiers or hair components" do
+      "1 digit one" =~ Unicode::Emoji::REGEX_WELL_FORMED
+      assert_nil $&
+
+      "🇦 regional indicator symbol letter a" =~ Unicode::Emoji::REGEX_WELL_FORMED
       assert_nil $&
     end
 
@@ -363,9 +428,20 @@ describe Unicode::Emoji do
       assert_equal "▶\u{FE0F}", $&
     end
 
-    it "matches singleton 'component' emoji codepoints" do
+    it "matches singleton skin tone modifiers and hair components" do
       "🏻 light skin tone" =~ Unicode::Emoji::REGEX_POSSIBLE
       assert_equal "🏻", $&
+
+      "🦰 emoji component red hair" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🦰", $&
+    end
+
+    it "matches singleton components that are not skin tone modifiers or hair components" do
+      "1 digit one" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "1", $&
+
+      "🇦 regional indicator symbol letter a" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🇦", $&
     end
 
     it "matches modified emoji if modifier base emoji is used" do
@@ -454,17 +530,28 @@ describe Unicode::Emoji do
     end
 
     it "does not match textual singleton emoji" do
-      "▶ play button" =~ Unicode::Emoji::REGEX
+      "▶ play button" =~ Unicode::Emoji::REGEX_BASIC
       assert_nil $&
     end
 
     it "matches textual singleton emoji in combination with emoji variation selector" do
-      "▶\u{FE0F} play button" =~ Unicode::Emoji::REGEX
+      "▶\u{FE0F} play button" =~ Unicode::Emoji::REGEX_BASIC
       assert_equal "▶\u{FE0F}", $&
     end
 
-    it "does not match singleton 'component' emoji codepoints" do
+    it "matches singleton skin tone modifiers and hair components" do
       "🏻 light skin tone" =~ Unicode::Emoji::REGEX_BASIC
+      assert_equal "🏻", $&
+
+      "🦰 emoji component red hair" =~ Unicode::Emoji::REGEX_BASIC
+      assert_equal "🦰", $&
+    end
+
+    it "does not match singleton components that are not skin tone modifiers or hair components" do
+      "1 digit one" =~ Unicode::Emoji::REGEX_BASIC
+      assert_nil $&
+
+      "🇦 regional indicator symbol letter a" =~ Unicode::Emoji::REGEX_BASIC
       assert_nil $&
     end
 
@@ -525,9 +612,20 @@ describe Unicode::Emoji do
       assert_nil $&
     end
 
-    it "does not match singleton 'component' emoji codepoints" do
+    it "does not match singleton skin tone modifiers and hair components" do
       "🏻 light skin tone" =~ Unicode::Emoji::REGEX_TEXT
       assert_nil $&
+
+      "🦰 emoji component red hair" =~ Unicode::Emoji::REGEX_TEXT
+      assert_nil $&
+    end
+
+    it "does not match singleton components that are not skin tone modifiers or hair components" do
+      "1 digit one" =~ Unicode::Emoji::REGEX_TEXT
+      assert_nil $&
+
+      "🇦 regional indicator symbol letter a" =~ Unicode::Emoji::REGEX_TEXT
+      assert_nil $&
     end
 
     it "does not match modified emoji" do
@@ -564,13 +662,28 @@ describe Unicode::Emoji do
     end
   end
 
-  describe "REGEX_ANY" do
+  describe "REGEX_PROP_EMOJI" do
     it "returns any emoji-related codepoint (but no variation selectors or tags)" do
-      matches = "1 string 😴\u{FE0F} sleeping face with 🇵 and modifier 🏾, also 🏴󠁧󠁢󠁳󠁣󠁴󠁿 Scotland".scan(Unicode::Emoji::REGEX_ANY)
+      matches = "1 string 😴\u{FE0F} sleeping face with 🇵 and modifier 🏾, also 🏴󠁧󠁢󠁳󠁣󠁴󠁿 Scotland".scan(Unicode::Emoji::REGEX_PROP_EMOJI)
       assert_equal ["1", "😴", "🇵", "🏾", "🏴"], matches
     end
   end
 
+  describe "REGEX_EMOJI_KEYCAP" do
+    it "matches emoji keycap sequences" do
+      "2️⃣ keycap: 2" =~ Unicode::Emoji::REGEX_EMOJI_KEYCAP
+      assert_equal "2️⃣", $&
+    end
+
+    it "does not match non-emoji keycap sequences" do
+      "8⃣ text keycap: 8" =~ Unicode::Emoji::REGEX_EMOJI_KEYCAP
+      assert_nil $&
+
+      "#⃣ text keycap: #" =~ Unicode::Emoji::REGEX_EMOJI_KEYCAP
+      assert_nil $&
+    end
+  end
+
   describe "REGEX_PICTO" do
     it "matches codepoints with Extended_Pictograph property (almost all emoji are, but also others)" do
       matches = "U+1F32D 🌭 HOT DOG, U+203C ‼ DOUBLE EXCLAMATION MARK, U+26E8 ⛨ BLACK CROSS ON SHIELD".scan(Unicode::Emoji::REGEX_PICTO)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unicode-emoji
 version: !ruby/object:Gem::Version
-  version: 3.7.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-18 00:00:00.000000000 Z
+date: 2024-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode-version
@@ -47,24 +47,38 @@ files:
 - lib/unicode/emoji.rb
 - lib/unicode/emoji/constants.rb
 - lib/unicode/emoji/generated/regex.rb
-- lib/unicode/emoji/generated/regex_any.rb
 - lib/unicode/emoji/generated/regex_basic.rb
+- lib/unicode/emoji/generated/regex_emoji_keycap.rb
+- lib/unicode/emoji/generated/regex_include_mqe.rb
+- lib/unicode/emoji/generated/regex_include_mqe_uqe.rb
 - lib/unicode/emoji/generated/regex_include_text.rb
 - lib/unicode/emoji/generated/regex_picto.rb
 - lib/unicode/emoji/generated/regex_picto_no_emoji.rb
 - lib/unicode/emoji/generated/regex_possible.rb
+- lib/unicode/emoji/generated/regex_prop_component.rb
+- lib/unicode/emoji/generated/regex_prop_emoji.rb
+- lib/unicode/emoji/generated/regex_prop_modifier.rb
+- lib/unicode/emoji/generated/regex_prop_modifier_base.rb
+- lib/unicode/emoji/generated/regex_prop_presentation.rb
 - lib/unicode/emoji/generated/regex_text.rb
 - lib/unicode/emoji/generated/regex_valid.rb
 - lib/unicode/emoji/generated/regex_valid_include_text.rb
 - lib/unicode/emoji/generated/regex_well_formed.rb
 - lib/unicode/emoji/generated/regex_well_formed_include_text.rb
 - lib/unicode/emoji/generated_native/regex.rb
-- lib/unicode/emoji/generated_native/regex_any.rb
 - lib/unicode/emoji/generated_native/regex_basic.rb
+- lib/unicode/emoji/generated_native/regex_emoji_keycap.rb
+- lib/unicode/emoji/generated_native/regex_include_mqe.rb
+- lib/unicode/emoji/generated_native/regex_include_mqe_uqe.rb
 - lib/unicode/emoji/generated_native/regex_include_text.rb
 - lib/unicode/emoji/generated_native/regex_picto.rb
 - lib/unicode/emoji/generated_native/regex_picto_no_emoji.rb
 - lib/unicode/emoji/generated_native/regex_possible.rb
+- lib/unicode/emoji/generated_native/regex_prop_component.rb
+- lib/unicode/emoji/generated_native/regex_prop_emoji.rb
+- lib/unicode/emoji/generated_native/regex_prop_modifier.rb
+- lib/unicode/emoji/generated_native/regex_prop_modifier_base.rb
+- lib/unicode/emoji/generated_native/regex_prop_presentation.rb
 - lib/unicode/emoji/generated_native/regex_text.rb
 - lib/unicode/emoji/generated_native/regex_valid.rb
 - lib/unicode/emoji/generated_native/regex_valid_include_text.rb
@@ -73,6 +87,9 @@ files:
 - lib/unicode/emoji/index.rb
 - lib/unicode/emoji/lazy_constants.rb
 - lib/unicode/emoji/list.rb
+- spec/data/.keep
+- spec/data/emoji-test.txt
+- spec/emoji_test_txt_spec.rb
 - spec/unicode_emoji_spec.rb
 - unicode-emoji.gemspec
 homepage: https://github.com/janlelis/unicode-emoji
@@ -103,4 +120,7 @@ signing_key:
 specification_version: 4
 summary: Emoji data and regex
 test_files:
+- spec/data/.keep
+- spec/data/emoji-test.txt
+- spec/emoji_test_txt_spec.rb
 - spec/unicode_emoji_spec.rb

data/lib/unicode/emoji/generated/regex_any.rb

@@ -1,8 +0,0 @@
-# This file was generated by a script, please do not edit it by hand.
-# See `$ rake generate_constants` and data/generate_constants.rb for more info.
-
-module Unicode
-  module Emoji
-    REGEX_ANY = /[\#\*0-9©®‼⁉™ℹ↔-↙↩↪⌚⌛⌨⏏⏩-⏳⏸-⏺Ⓜ▪▫▶◀◻-◾☀-☄☎☑☔☕☘☝☠☢☣☦☪☮☯☸-☺♀♂♈-♓♟♠♣♥♦♨♻♾♿⚒-⚗⚙⚛⚜⚠⚡⚧⚪⚫⚰⚱⚽⚾⛄⛅⛈⛎⛏⛑⛓⛔⛩⛪⛰-⛵⛷-⛺⛽✂✅✈-✍✏✒✔✖✝✡✨✳✴❄❇❌❎❓-❕❗❣❤➕-➗➡➰➿⤴⤵⬅-⬇⬛⬜⭐⭕〰〽㊗㊙🀄🃏🅰🅱🅾🅿🆎🆑-🆚🇦-🇿🈁🈂🈚🈯🈲-🈺🉐🉑🌀-🌡🌤-🎓🎖🎗🎙-🎛🎞-🏰🏳-🏵🏷-📽📿-🔽🕉-🕎🕐-🕧🕯🕰🕳-🕺🖇🖊-🖍🖐🖕🖖🖤🖥🖨🖱🖲🖼🗂-🗄🗑-🗓🗜-🗞🗡🗣🗨🗯🗳🗺-🙏🚀-🛅🛋-🛒🛕-🛗🛜-🛥🛩🛫🛬🛰🛳-🛼🟠-🟫🟰🤌-🤺🤼-🥅🥇-🧿🩰-🩼🪀-🪉🪏-🫆🫎-🫜🫟-🫩🫰-🫸]/
-  end
-end