unicode-emoji 3.6.0 → 3.8.0

data/spec/emoji_test_txt_spec.rb

@@ -0,0 +1,181 @@
+require_relative "../lib/unicode/emoji"
+require "minitest/autorun"
+require "open-uri"
+
+def iterate_emoji
+  EMOJI_TEST_FILE.scan(/^(?:# (?<sub>sub)?group: (?<group_name>.*)$)|(?:(?<codepoints>.+?)\s*; (?<qual_status>.+?)-?qualified )/) do
+    if $~[:codepoints]
+      yield $~[:codepoints].split.map{|e| e.to_i(16)}.pack("U*"), $~[:qual_status]
+    end
+  end
+end
+
+describe "emoji-test.txt" do
+  EMOJI_TEST_FILE = begin
+    emoji_test_path = File.join(__dir__, "data/emoji-test.txt")
+    if File.exist? emoji_test_path
+      file = File.read(emoji_test_path)
+    else
+      puts "Downloading emoji-test.txt from the consortium"
+      URI.open "https://www.unicode.org/Public/emoji/#{Unicode::Emoji::EMOJI_VERSION}/emoji-test.txt" do |f|
+        file = f.read
+        File.write(File.join(__dir__, "data/emoji-test.txt"), @file)
+      end
+    end
+
+    file
+  end
+
+  # qual_status:
+  # - fully - fully-qualified emoji sequences
+  # - minimally - minimallyq-ualified emoji sequences (some VS16 missing, but not first one)
+  # - un - unqualified emoji sequences (some VS16 missing)
+
+  describe "REGEX" do
+    describe "detects fully-qualified emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully"
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_INCLUDE_TEXT" do
+    describe "detects fully-qualified emoji and (unqualified) singleton text emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" || qual_status == "un" && emoji.size <= 2
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_TEXT]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_TEXT]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_INCLUDE_MQE" do
+    describe "detects fully-qualified emoji and minimally-qualified emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" || qual_status == "minimally"
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_MQE]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_MQE]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_INCLUDE_MQE_UQE" do
+    describe "detects all emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          assert_equal emoji, emoji[Unicode::Emoji::REGEX_INCLUDE_MQE_UQE]
+        end
+      end
+    end
+  end
+
+  describe "REGEX_VALID" do
+    describe "detects fully-qualified, minimally-qualified emoji, and unqualified emoji with ZWJ" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" || qual_status == "minimally" || qual_status == "un" && emoji.size >= 3
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_VALID]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_VALID]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_VALID_INCLUDE_TEXT" do
+    describe "detects all emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          assert_equal emoji, emoji[Unicode::Emoji::REGEX_VALID_INCLUDE_TEXT]
+        end
+      end
+    end
+  end
+
+  describe "REGEX_WELL_FORMED" do
+    describe "detects fully-qualified, minimally-qualified emoji, and unqualified emoji with ZWJ" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" || qual_status == "minimally" || qual_status == "un" && emoji.size >= 3
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_WELL_FORMED]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_WELL_FORMED]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_WELL_FORMED_INCLUDE_TEXT" do
+    describe "detects all emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          assert_equal emoji, emoji[Unicode::Emoji::REGEX_WELL_FORMED_INCLUDE_TEXT]
+        end
+      end
+    end
+  end
+
+  describe "REGEX_POSSIBLE" do
+    describe "detects all emoji, except unqualified keycap sequences" do
+      # fixing test not regex, since implementation of this regex should match the one in the standard
+      unqualified_keycaps = Unicode::Emoji::EMOJI_KEYCAPS.map{|keycap|
+        [keycap, Unicode::Emoji::EMOJI_KEYCAP_SUFFIX].pack("U*")
+      }
+
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if !unqualified_keycaps.include?(emoji)
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_POSSIBLE]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_POSSIBLE]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_TEXT" do
+    describe "detects (unqualified) singleton text emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          # if qual_status == "un" && emoji =~ /^.[\u{FE0E 20E3}]?$/
+          if qual_status == "un" && emoji.size <= 2
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_TEXT]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_TEXT]
+          end
+        end
+      end
+    end
+  end
+
+  describe "REGEX_BASIC" do
+    describe "detects (fully-qualified) singleton emoji" do
+      iterate_emoji do |emoji, qual_status|
+        it(emoji) do
+          if qual_status == "fully" && emoji =~ /^.\u{FE0F}?$/
+            assert_equal emoji, emoji[Unicode::Emoji::REGEX_BASIC]
+          else
+            refute_equal emoji, emoji[Unicode::Emoji::REGEX_BASIC]
+          end
+        end
+      end
+    end
+  end
+end

data/spec/unicode_emoji_spec.rb

@@ -3,12 +3,12 @@ require "minitest/autorun"
 
 describe Unicode::Emoji do
   describe ".properties" do
-    it "returns an Array for Emoji properties if has codepoints" do
+    it "returns an Array of Emoji properties for given codepoint" do
       assert_equal ["Emoji", "Emoji_Presentation", "Extended_Pictographic"], Unicode::Emoji.properties("😴")
       assert_equal ["Emoji", "Extended_Pictographic"], Unicode::Emoji.properties("♠")
     end
 
-    it "returns nil for Emoji properties if has no codepoints" do
+    it "returns nil if codepoint has no Emoji prop" do
       assert_nil Unicode::Emoji.properties("A")
     end
   end
@@ -92,6 +92,16 @@ describe Unicode::Emoji do
       assert_equal "🤾🏽‍♀️", $&
     end
 
+    it "does not match MQE zwj sequences" do
+      "🤾🏽‍♀ woman playing handball: medium skin tone, missing VS16" =~ Unicode::Emoji::REGEX
+      refute_equal `🤾🏽‍♀`, $&
+    end
+
+    it "does not match UQE emoji" do
+      "🏌‍♂️ man golfing, missing VS16" =~ Unicode::Emoji::REGEX
+      refute_equal `🏌‍♂️`, $&
+    end
+
     it "does not match valid zwj sequences that are not recommended" do
       "🤠‍🤢 vomiting cowboy" =~ Unicode::Emoji::REGEX
       assert_equal "🤠", $&
@@ -132,8 +142,6 @@ describe Unicode::Emoji do
       assert_equal "🪾", $&
     end
 
-    
-
     # See gh#12 and https://github.com/matt17r/nw5k/commit/05a34d3c9211a23e5ae6853bb19fd2f224779ef4#diff-afb6f8bc3bae71b75743e00882a060863e2430cbe858ec9014e5956504dfc61cR2
     it "matches family emoji correctly" do
       ["👨‍👩‍👧‍👦", "👨‍👩‍👦‍👦", "👨‍👩‍👧‍👧", "👨‍👨‍👧‍👦", "👨‍👨‍👦‍👦", "👨‍👨‍👧‍👧", "👩‍👩‍👧‍👦", "👩‍👩‍👦‍👦", "👩‍👩‍👧‍👧", "👨‍👦‍👦", "👨‍👧‍👦", "👨‍👧‍👧", "👩‍👦‍👦", "👩‍👧‍👦", "👩‍👧‍👧"].each { |family|
@@ -142,6 +150,30 @@ describe Unicode::Emoji do
     end
   end
 
+  describe "REGEX_INCLUDE_MQE" do
+    it "matches MQE emoji" do
+      "🤾🏽‍♀ woman playing handball: medium skin tone, missing VS16" =~ Unicode::Emoji::REGEX_INCLUDE_MQE
+      assert_equal `🤾🏽‍♀`, $&
+    end
+
+    it "does not match UQE emoji" do
+      "🏌‍♂️ man golfing, missing VS16" =~ Unicode::Emoji::REGEX_INCLUDE_MQE
+      refute_equal `🏌‍♂️`, $&
+    end
+  end
+
+  describe "REGEX_INCLUDE_MQE_UQE" do
+    it "matches MQE emoji" do
+      "🤾🏽‍♀ woman playing handball: medium skin tone, missing VS16" =~ Unicode::Emoji::REGEX_INCLUDE_MQE_UQE
+      assert_equal `🤾🏽‍♀`, $&
+    end
+
+    it "matches UQE emoji" do
+      "🏌‍♂️ man golfing, missing VS16" =~ Unicode::Emoji::REGEX_INCLUDE_MQE_UQE
+      assert_equal `🏌‍♂️`, $&
+    end
+  end
+
   describe "REGEX_VALID" do
     it "matches most singleton emoji codepoints" do
       "😴 sleeping face" =~ Unicode::Emoji::REGEX_VALID
@@ -216,6 +248,11 @@ describe Unicode::Emoji do
       assert_equal "🏴󠁧󠁢󠁡󠁧󠁢󠁿", $&
     end
 
+    it "matches valid tag sequences (compressed one)" do
+      "🏴󠁬󠁶󠀰󠀴󠀲󠁿 lv042" =~ Unicode::Emoji::REGEX_VALID
+      assert_equal "🏴󠁬󠁶󠀰󠀴󠀲󠁿", $&
+    end
+
     it "does not match invalid tag sequences" do
       "🏴󠁧󠁢󠁡󠁡󠁡󠁿 GB AAA" =~ Unicode::Emoji::REGEX_VALID
       assert_equal "🏴", $& # only base flag is matched
@@ -306,11 +343,21 @@ describe Unicode::Emoji do
       assert_equal "🏴󠁧󠁢󠁡󠁧󠁢󠁿", $&
     end
 
-    it "does match invalid tag sequences" do
+    it "matches invalid base tag sequences" do
       "😴󠁧󠁢󠁡󠁡󠁡󠁿 GB AAA" =~ Unicode::Emoji::REGEX_WELL_FORMED
       assert_equal "😴󠁧󠁢󠁡󠁡󠁡󠁿", $&
     end
 
+    it "does not match too long tag sequences (only black flag is matched)" do
+      "🏴󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁿 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" =~ Unicode::Emoji::REGEX_WELL_FORMED
+      assert_equal "🏴", $&
+    end
+
+    it "does not match invalid tag sequences (only black flag is matched)" do
+      "🏴󠀤󠁿 $" =~ Unicode::Emoji::REGEX_WELL_FORMED
+      assert_equal "🏴", $&
+    end
+
     it "matches recommended zwj sequences" do
       "🤾🏽‍♀️ woman playing handball: medium skin tone" =~ Unicode::Emoji::REGEX_WELL_FORMED
       assert_equal "🤾🏽‍♀️", $&
@@ -322,6 +369,106 @@ describe Unicode::Emoji do
     end
   end
 
+  describe "REGEX_POSSIBLE" do
+    it "matches most singleton emoji codepoints" do
+      "😴 sleeping face" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "😴", $&
+    end
+
+    it "matches singleton emoji in combination with emoji variation selector" do
+      "😴\u{FE0F} sleeping face" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "😴\u{FE0F}", $&
+    end
+
+    it "matches singleton emoji (without VS) when in combination with text variation selector" do
+      "😴\u{FE0E} sleeping face" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "😴", $&
+    end
+
+    it "matches textual singleton emoji" do
+      "▶ play button" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "▶", $&
+    end
+
+    it "matches textual singleton emoji in combination with emoji variation selector" do
+      "▶\u{FE0F} play button" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "▶\u{FE0F}", $&
+    end
+
+    it "matches singleton 'component' emoji codepoints" do
+      "🏻 light skin tone" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🏻", $&
+    end
+
+    it "matches modified emoji if modifier base emoji is used" do
+      "🛌🏽 person in bed: medium skin tone" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🛌🏽", $&
+    end
+
+    it "matches modified emoji even if no modifier base emoji is used" do
+      "🌵🏽 cactus" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🌵🏽", $&
+    end
+
+    it "matches valid region flags" do
+      "🇵🇹 Portugal" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🇵🇹", $&
+    end
+
+    it "does match invalid region flags" do
+      "🇵🇵 PP Land" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🇵🇵", $&
+    end
+
+    it "matches emoji keycap sequences" do
+      "2️⃣ keycap: 2" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "2️⃣", $&
+    end
+
+    it "matches only digit of non-emoji keycap sequences" do
+      "8⃣ text keycap: 8" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "8", $&
+
+      "#⃣ text keycap: #" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "#", $&
+    end
+
+    it "matches recommended tag sequences" do
+      "🏴󠁧󠁢󠁳󠁣󠁴󠁿 Scotland" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🏴󠁧󠁢󠁳󠁣󠁴󠁿", $&
+    end
+
+    it "matches valid tag sequences, even though they are not recommended" do
+      "🏴󠁧󠁢󠁡󠁧󠁢󠁿 GB AGB" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🏴󠁧󠁢󠁡󠁧󠁢󠁿", $&
+    end
+
+    it "matches invalid base tag sequences" do
+      "😴󠁧󠁢󠁡󠁡󠁡󠁿 GB AAA" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "😴󠁧󠁢󠁡󠁡󠁡󠁿", $&
+    end
+
+    it "matches too long tag sequences" do
+      "🏴󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁿 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🏴󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁁󠁿", $&
+    end
+
+    it "machtes invalid tag sequences (only black flag is matched)" do
+      "🏴󠀤󠁿 $" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🏴󠀤󠁿", $&
+    end
+
+    it "matches recommended zwj sequences" do
+      "🤾🏽‍♀️ woman playing handball: medium skin tone" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🤾🏽‍♀️", $&
+    end
+
+    it "matches valid zwj sequences, even though they are not recommended" do
+      "🤠‍🤢 vomiting cowboy" =~ Unicode::Emoji::REGEX_POSSIBLE
+      assert_equal "🤠‍🤢", $&
+    end
+  end
+
   describe "REGEX_BASIC" do
     it "matches most singleton emoji codepoints" do
       "😴 sleeping face" =~ Unicode::Emoji::REGEX_BASIC

data/unicode-emoji.gemspec

@@ -5,8 +5,8 @@ require File.dirname(__FILE__) + "/lib/unicode/emoji/constants"
 Gem::Specification.new do |gem|
   gem.name          = "unicode-emoji"
   gem.version       = Unicode::Emoji::VERSION
-  gem.summary       = "Retrieve Emoji data about Unicode codepoints."
-  gem.description   = "[Emoji #{Unicode::Emoji::EMOJI_VERSION}] A small Ruby library which provides Unicode Emoji data and regexes, incorporating the latest Unicode and Emoji standards. Includes a categorized list of recommended Emoji."
+  gem.summary       = "Emoji data and regex"
+  gem.description   = "[Emoji #{Unicode::Emoji::EMOJI_VERSION}] Provides Unicode Emoji data and regexes, incorporating the latest Unicode and Emoji standards. Includes a categorized list of recommended Emoji."
   gem.authors       = ["Jan Lelis"]
   gem.email         = ["hi@ruby.consulting"]
   gem.homepage      = "https://github.com/janlelis/unicode-emoji"
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.metadata      = { "rubygems_mfa_required" => "true" }
 
-  gem.required_ruby_version = ">= 2.3", "< 4.0"
+  gem.required_ruby_version = ">= 2.5", "< 4.0"
 
   gem.add_dependency "unicode-version", "~> 1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unicode-emoji
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.8.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-17 00:00:00.000000000 Z
+date: 2024-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode-version
@@ -24,9 +24,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-description: "[Emoji 16.0] A small Ruby library which provides Unicode Emoji data
-  and regexes, incorporating the latest Unicode and Emoji standards. Includes a categorized
-  list of recommended Emoji."
+description: "[Emoji 16.0] Provides Unicode Emoji data and regexes, incorporating
+  the latest Unicode and Emoji standards. Includes a categorized list of recommended
+  Emoji."
 email:
 - hi@ruby.consulting
 executables: []
@@ -49,9 +49,12 @@ files:
 - lib/unicode/emoji/generated/regex.rb
 - lib/unicode/emoji/generated/regex_any.rb
 - lib/unicode/emoji/generated/regex_basic.rb
+- lib/unicode/emoji/generated/regex_include_mqe.rb
+- lib/unicode/emoji/generated/regex_include_mqe_uqe.rb
 - lib/unicode/emoji/generated/regex_include_text.rb
 - lib/unicode/emoji/generated/regex_picto.rb
 - lib/unicode/emoji/generated/regex_picto_no_emoji.rb
+- lib/unicode/emoji/generated/regex_possible.rb
 - lib/unicode/emoji/generated/regex_text.rb
 - lib/unicode/emoji/generated/regex_valid.rb
 - lib/unicode/emoji/generated/regex_valid_include_text.rb
@@ -60,9 +63,12 @@ files:
 - lib/unicode/emoji/generated_native/regex.rb
 - lib/unicode/emoji/generated_native/regex_any.rb
 - lib/unicode/emoji/generated_native/regex_basic.rb
+- lib/unicode/emoji/generated_native/regex_include_mqe.rb
+- lib/unicode/emoji/generated_native/regex_include_mqe_uqe.rb
 - lib/unicode/emoji/generated_native/regex_include_text.rb
 - lib/unicode/emoji/generated_native/regex_picto.rb
 - lib/unicode/emoji/generated_native/regex_picto_no_emoji.rb
+- lib/unicode/emoji/generated_native/regex_possible.rb
 - lib/unicode/emoji/generated_native/regex_text.rb
 - lib/unicode/emoji/generated_native/regex_valid.rb
 - lib/unicode/emoji/generated_native/regex_valid_include_text.rb
@@ -70,6 +76,10 @@ files:
 - lib/unicode/emoji/generated_native/regex_well_formed_include_text.rb
 - lib/unicode/emoji/index.rb
 - lib/unicode/emoji/lazy_constants.rb
+- lib/unicode/emoji/list.rb
+- spec/data/.keep
+- spec/data/emoji-test.txt
+- spec/emoji_test_txt_spec.rb
 - spec/unicode_emoji_spec.rb
 - unicode-emoji.gemspec
 homepage: https://github.com/janlelis/unicode-emoji
@@ -85,7 +95,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.5'
   - - "<"
     - !ruby/object:Gem::Version
       version: '4.0'
@@ -98,6 +108,9 @@ requirements: []
 rubygems_version: 3.5.21
 signing_key:
 specification_version: 4
-summary: Retrieve Emoji data about Unicode codepoints.
+summary: Emoji data and regex
 test_files:
+- spec/data/.keep
+- spec/data/emoji-test.txt
+- spec/emoji_test_txt_spec.rb
 - spec/unicode_emoji_spec.rb