tanuki_emoji 0.9.0 → 0.13.0

data/lib/tanuki_emoji/db/emoji_test_parser.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require 'strscan'
+require 'date'
+require 'pathname'
+require 'i18n'
+require_relative 'emoji_data'
+
+module TanukiEmoji
+  module Db
+    # Reads and extract content from emoji-test.txt
+    class EmojiTestParser
+      DATA_FILE = "#{::TanukiEmoji::Db::UNICODE_DATA_DIR}/emoji-test.txt".freeze
+
+      # https://www.unicode.org/reports/tr51/#Versioning
+      EMOJI_UNICODE_VERSION = {
+        '0.6' => '6.0',
+        '0.7' => '7.0',
+        '1.0' => '8.0',
+        '2.0' => '8.0',
+        '3.0' => '9.0',
+        '4.0' => '9.0',
+        '5.0' => '10.0'
+      }.freeze
+
+      # Return the path to the default data file (emoji-data.txt)
+      #
+      # @return [Pathname] path to the default data file
+      def self.data_file
+        ::Pathname.new(File.expand_path(File.join(__dir__, '../../../', DATA_FILE)))
+      end
+
+      attr_reader :data_file
+      attr_accessor :group_category
+
+      def initialize(index:, data_file: self.class.data_file)
+        @data_file = data_file
+        @index = index
+        @group_category = nil
+
+        return if I18n.available_locales.include?(:en)
+
+        I18n.available_locales = I18n.available_locales + [:en]
+      end
+
+      # Return the parsed data from the data file
+      #
+      # @return [Array<EmojiData>] collection of EmojiData
+      def data
+        parsed = []
+
+        load do |line|
+          parse_line(line).tap do |result|
+            parsed << result unless result.nil?
+          end
+        end
+
+        parsed
+      end
+
+      def raw_data
+        lines = []
+        load do |line|
+          lines << line
+        end
+
+        lines
+      end
+
+      def load!
+        alpha_code = nil
+
+        data.each do |emoji_data|
+          I18n.with_locale(:en) do
+            alpha_code = I18n.transliterate(emoji_data.description)
+              .gsub(/[^a-zA-Z#*\d]+/, '_')
+              .downcase
+              .chomp('_')
+          end
+
+          alpha_code = 'keycap_asterisk' if alpha_code == 'keycap_*'
+          alpha_code = 'keycap_hash' if alpha_code == 'keycap_#'
+
+          # This might be a different qualified version, basically same emoji but slightly different
+          # code point. Search on the alpha code and pull that. If found, add as alternate code point.
+          # "smiling face" is one example.
+          emoji = @index.find_by_alpha_code(alpha_code)
+
+          if emoji
+            emoji.add_codepoints(emoji_data.codepoints)
+
+            @index.update(emoji)
+          else
+            # not found, add a new emoji
+            emoji = Character.new(alpha_code,
+              codepoints: emoji_data.codepoints,
+              alpha_code: alpha_code,
+              description: emoji_data.description,
+              category: emoji_data.group_category)
+
+            emoji.unicode_version = emoji_data.version
+
+            @index.add(emoji)
+          end
+        end
+      end
+
+      private
+
+      # Parse a line extracting data and ignoring comments and metadata
+      #
+      # @param [String] line a line from the data file
+      # @return [EmojiData] emoji data parsed from the line
+      def parse_line(line)
+        @scanner = StringScanner.new(line)
+        data = {}
+
+        skip_line_breaks
+
+        return if empty_line?
+
+        if comment_token?
+          @scanner.scan(/# group: (?<group_category>[^\n]+)/)
+          @group_category = fetch_first_capture.strip if @scanner.matched?
+
+          return
+        end
+
+        @scanner.scan(/(?<codepoints>.+) ;/)
+        data[:codepoints] = unicode_hex_to_codepoint(fetch_first_capture)
+
+        skip_spaces
+
+        # match `<qualification>`
+        @scanner.scan(/(?<qualification>[a-zA-Z\- ]+) #/)
+        data[:qualification] = fetch_first_capture.strip
+
+        skip_spaces
+
+        # emoji character
+        @scanner.scan(/(?<emoji>.+?)\sE/)
+        data[:emoji] = fetch_first_capture.strip
+
+        skip_spaces
+
+        # Version in which codepoint was introduced
+        @scanner.scan(/(?<version>\d+\.\d+)/)
+        data[:version] = map_emoji_to_unicode_version(fetch_first_capture)
+
+        skip_spaces
+
+        # description text
+        @scanner.scan(/(?<description>[^\n]+)/)
+        data[:description] = fetch_first_capture
+
+        # Group
+        data[:group_category] = group_category
+
+        EmojiTestData.new(*data.values)
+      end
+
+      def skip_line_breaks
+        @scanner.skip(/\n/)
+      end
+
+      def skip_spaces
+        @scanner.skip(/\s+/)
+      end
+
+      def comment_token?
+        @scanner.peek(1) == '#'
+      end
+
+      def empty_line?
+        @scanner.peek(1) == ''
+      end
+
+      def fetch_first_capture
+        @scanner.captures&.dig(0)
+      end
+
+      def load(&block)
+        File.open(data_file).each(&block)
+      end
+
+      def unicode_hex_to_codepoint(unicode)
+        unicode.strip.split.map(&:hex).pack("U*")
+      end
+
+      # anything after E5.0 maps to same unicode version
+      def map_emoji_to_unicode_version(emoji_version)
+        EMOJI_UNICODE_VERSION[emoji_version] || emoji_version
+      end
+    end
+  end
+end

data/lib/tanuki_emoji/db/gemojione.rb

@@ -5,9 +5,26 @@ require 'json'
 module TanukiEmoji
   module Db
     # Gemojione Emoji database
+    # In order to maintain compatibility with alpha_codes that have been
+    # stored in a DB originally using the gemojione codes, we change the original
+    # gemojione code to be the primary and make the unicode version to be an alias.
+    # So instead of the alpha code being `thumbs_up` based on the unicode naming,
+    # it's `thumbsup`, with an alias of `thumbs_up`
     class Gemojione
       DATA_FILE = 'vendor/gemojione/index-3.3.0.json'
 
+      # rubocop:disable Style/AsciiComments
+      # These are specific gemojione whos alpha codes map slightly differently.
+      # For example, :cow: in gemojione is 🐮, while in Unicode it is 🐄,
+      # which is :cow2: in gemojione. Now :cow_face: will give 🐮.
+      # See https://gitlab.com/gitlab-org/ruby/gems/tanuki_emoji/-/merge_requests/65#note_2113986561
+      EMOJI_DIFFERENCES =
+        {
+          unicode: %w[📅 🐪 🐈 🐄 🐕 🐎 🐁 ✏️ 🐖 🐇 🛰️ ☃️ 🐅 🚆 ☂️ 🐋],
+          gemojione: %w[📆 🐫 🐱 🐮 🐶 🐴 🐭 📝 🐷 🐰 📡 ⛄ 🐯 🚋 ☔ 🐳]
+        }.freeze
+      # rubocop:enable Style/AsciiComments
+
       def self.data_file
         File.expand_path(File.join(__dir__, '../../../', DATA_FILE))
       end
@@ -24,28 +41,31 @@ module TanukiEmoji
           JSON.parse(file.read, symbolize_names: true)
         end
 
-        db.each do |emoji_name, emoji_data|
-          emoji = Character.new(emoji_name.to_s,
-                                codepoints: emoji_data[:moji],
-                                alpha_code: emoji_data[:shortname],
-                                description: emoji_data[:name],
-                                category: emoji_data[:category])
+        db.each_value do |emoji_data|
+          emoji = @index.find_by_codepoints(emoji_data[:moji])
 
-          emoji_data[:unicode_alternates].each do |unicode_alternates|
-            codepoints = unicode_hex_to_codepoint(unicode_alternates)
+          # if it's not found, don't try to add something that isn't in the
+          # Unicode set.
+          next unless emoji
 
-            emoji.add_codepoints(codepoints)
-          end
+          if emoji.alpha_code != emoji_data[:shortname]
+            org_alpha_code = emoji.alpha_code
+            org_alpha_code_sym = TanukiEmoji::Character.format_name(org_alpha_code).to_sym
+            emoji.replace_alpha_code(emoji_data[:shortname])
 
-          emoji_data[:aliases].each do |alpha_code|
-            emoji.add_alias(alpha_code)
+            # rubocop:disable Style/AsciiComments
+            # Ensure that we're not adding an alias that is part of the gemonione data.
+            # For example, Unicode uses `sunglasses` for 🕶️, which is `dark_sunglasses` in gemojione.
+            # `sunglasses` is 😎 which is `smiling_face_with_sunglasses` in Unicode.
+            # We don't want `sunglasses` to be added as an alias of `dark_sunglasses`, because that
+            # would interfere with `sunglasses` being the primary code for `smiling_face_with_sunglasses`
+            # rubocop:enable Style/AsciiComments
+            emoji.add_alias(org_alpha_code) unless db.key?(org_alpha_code_sym) || EMOJI_DIFFERENCES[:unicode].include?(emoji.codepoints)
           end
 
-          emoji_data[:aliases_ascii].each do |ascii_string|
-            emoji.add_ascii_alias(ascii_string)
-          end
+          add_emoji_data(emoji, emoji_data)
 
-          @index.add(emoji)
+          @index.update(emoji)
         end
       end
 
@@ -54,6 +74,22 @@ module TanukiEmoji
       def unicode_hex_to_codepoint(unicode)
         unicode.split('-').map { |i| i.to_i(16) }.pack('U*')
       end
+
+      def add_emoji_data(emoji, emoji_data)
+        emoji_data[:unicode_alternates].each do |unicode_alternates|
+          codepoints = unicode_hex_to_codepoint(unicode_alternates)
+
+          emoji.add_codepoints(codepoints)
+        end
+
+        emoji_data[:aliases].each do |alpha_code|
+          emoji.add_alias(alpha_code)
+        end
+
+        emoji_data[:aliases_ascii].each do |ascii_string|
+          emoji.add_ascii_alias(ascii_string)
+        end
+      end
     end
   end
 end

data/lib/tanuki_emoji/db/unicode_ordering.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+require_relative 'emoji_data'
+
 module TanukiEmoji
   module Db
     # Emoji Unicode Ordering database
     class UnicodeOrdering
-      DATA_FILE = 'vendor/unicode/emoji-ordering.txt'
+      DATA_FILE = "#{::TanukiEmoji::Db::UNICODE_DATA_DIR}/emoji-ordering.txt".freeze
 
       def self.data_file
         File.expand_path(File.join(__dir__, '../../../', DATA_FILE))

data/lib/tanuki_emoji/db.rb

@@ -7,5 +7,7 @@ module TanukiEmoji
     autoload :UnicodeVersion, 'tanuki_emoji/db/unicode_version'
     autoload :EmojiData, 'tanuki_emoji/db/emoji_data'
     autoload :EmojiDataParser, 'tanuki_emoji/db/emoji_data_parser'
+    autoload :EmojiTestParser, 'tanuki_emoji/db/emoji_test_parser'
+    autoload :AdditionalAliases, 'tanuki_emoji/db/additional_aliases'
   end
 end

data/lib/tanuki_emoji/errors.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module TanukiEmoji
-  class Error < StandardError; end
+  Error = Class.new(StandardError)
 
   # Error: An Emoji with the same alpha code has been previously indexed
   class AlphaCodeAlreadyIndexedError < Error
@@ -14,7 +14,7 @@ module TanukiEmoji
       @alpha_code = alpha_code
 
       message = "Cannot index Emoji '#{name}' with alpha code '#{alpha_code}'. " \
-                "An Emoji with that alpha code has already been indexed."
+        "An Emoji with that alpha code has already been indexed."
 
       super(message)
     end
@@ -31,7 +31,7 @@ module TanukiEmoji
       @codepoint = codepoint
 
       message = "Cannot index '#{name}' Emoji with codepoint: '#{codepoint}'. " \
-                "An Emoji with that codepoint has already been indexed."
+        "An Emoji with that codepoint has already been indexed."
 
       super(message)
     end

data/lib/tanuki_emoji/index.rb

@@ -29,18 +29,7 @@ module TanukiEmoji
           raise ::TanukiEmoji::CodepointAlreadyIndexedError.new emoji.name, codepoints if @codepoints_index.key? codepoints
         end
 
-        @name_index[emoji.name] = emoji
-        @alpha_code_index[emoji.alpha_code] = emoji
-        @codepoints_index[emoji.codepoints] = emoji
-
-        emoji.codepoints_alternates.each do |codepoints|
-          @codepoints_index[codepoints] = emoji
-        end
-
-        emoji.aliases.each do |alpha_code|
-          @name_index[::TanukiEmoji::Character.format_name(alpha_code)] = emoji
-          @alpha_code_index[::TanukiEmoji::Character.format_alpha_code(alpha_code)] = emoji
-        end
+        add_to_index(emoji)
       end
 
       all << emoji
@@ -48,11 +37,19 @@ module TanukiEmoji
       emoji
     end
 
+    def update(emoji)
+      insertion_mutex.synchronize do
+        add_to_index(emoji)
+      end
+    end
+
     # Find an Emoji by its :alpha_code:
     #
     # @param [String] alpha_code
     # @return [TanukiEmoji::Character]
     def find_by_alpha_code(alpha_code)
+      return unless @alpha_code_index
+
       @alpha_code_index[Character.format_alpha_code(alpha_code)]
     end
 
@@ -61,6 +58,8 @@ module TanukiEmoji
     # @param [String] unicode_codepoints
     # @return [TanukiEmoji::Character]
     def find_by_codepoints(unicode_codepoints)
+      return unless @codepoints_index
+
       @codepoints_index[unicode_codepoints]
     end
 
@@ -83,7 +82,7 @@ module TanukiEmoji
     # @return [Regexp] regular expression that matches indexed `:alpha_code:`
     def alpha_code_pattern
       /(?<=[^[:alnum:]:]|\n|^)
-       :(#{@name_index.keys.map { |name| Regexp.escape(name) }.join('|')}):
+       :(#{@name_index.keys.map { |name| Regexp.escape(name) }.join("|")}):
        (?=[^[:alnum:]:]|$)/x
     end
 
@@ -94,33 +93,51 @@ module TanukiEmoji
     def codepoints_pattern(exclude_text_presentation: false)
       possible_codepoints = sorted_codepoints.map { |moji, _| Regexp.escape(moji) }.join('|')
       variation_selector = ""
-      variation_selector = /(?!#{TanukiEmoji::Character::PLAIN_VARIATION_SELECTOR_STRING})/ if exclude_text_presentation
+      variation_selector = /(?!#{TanukiEmoji::Character::PLAIN_VARIATION_SELECTOR_STRING})/o if exclude_text_presentation
 
       /(#{possible_codepoints})#{variation_selector}/
     end
 
     private
 
+    # rubocop:disable Layout/ClassStructure
     def initialize
       @all = []
 
       load_data_files
     end
+    # rubocop:enable Layout/ClassStructure
 
     def insertion_mutex
       @insertion_mutex ||= Mutex.new
     end
 
     def load_data_files
+      Db::EmojiTestParser.new(index: self).load!
       Db::Gemojione.new(index: self).load!
-      Db::UnicodeVersion.new(index: self).load!
       Db::UnicodeOrdering.new(index: self).load!
+      Db::AdditionalAliases.new(index: self).load!
     end
 
     # Order the codepoints to match the most specific (longest) sequences first,
     # so #gsub doesn't unintentionally split an emoji from its modifier(s).
     def sorted_codepoints
-      @sorted_codepoints ||= @codepoints_index.dup.sort_by { |_, v| -v.hex.length }
+      @sorted_codepoints ||= @codepoints_index.dup.sort_by { |k, v| -v.hex(k).length }
+    end
+
+    def add_to_index(emoji)
+      @name_index[emoji.name] = emoji
+      @alpha_code_index[emoji.alpha_code] = emoji
+      @codepoints_index[emoji.codepoints] = emoji
+
+      emoji.codepoints_alternates.each do |codepoints|
+        @codepoints_index[codepoints] = emoji
+      end
+
+      emoji.aliases.each do |alpha_code|
+        @name_index[::TanukiEmoji::Character.format_name(alpha_code)] = emoji
+        @alpha_code_index[::TanukiEmoji::Character.format_alpha_code(alpha_code)] = emoji
+      end
     end
   end
 end

data/lib/tanuki_emoji/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TanukiEmoji
-  VERSION = "0.9.0"
+  VERSION = '0.13.0'
 end

data/lib/tanuki_emoji.rb

@@ -43,10 +43,10 @@ module TanukiEmoji
   # @return [TanukiEmoji::Character]
   def self.add(name, codepoints:, alpha_code:, description:, category:)
     emoji = Character.new(name,
-                          codepoints: codepoints,
-                          alpha_code: alpha_code,
-                          description: description,
-                          category: category)
+      codepoints: codepoints,
+      alpha_code: alpha_code,
+      description: description,
+      category: category)
 
     index.add(emoji)
   end

data/tanuki_emoji.gemspec

@@ -22,12 +22,22 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    git_files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features|bin)/}) }
+    git_files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features|bin|pages)/}) }
     asset_files = Dir.glob('app/assets/**/*').reject { |f| f.match(%r{\.DS_Store}) }
 
     git_files + asset_files
   end
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'i18n', '~> 1.14'
+
   spec.add_development_dependency 'gitlab-dangerfiles', '~> 2.11.0'
+  spec.add_development_dependency 'gitlab-styles', '~> 12.0'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec_junit_formatter', '~> 0.4.0'
+  spec.add_development_dependency 'rubocop', '~> 1.62'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'simplecov-cobertura', '~> 1.4.2'
 end

data/vendor/additional_aliases.json

@@ -0,0 +1,25 @@
+{
+  "flag_united_nations": {
+    "emoji": "🇺🇳",
+    "aliases": ["un"],
+    "comment": "Needed for noto-image flag support"
+  },
+  "flag_england": {
+    "emoji": "🏴󠁧󠁢󠁥󠁮󠁧󠁿",
+    "aliases": ["england"],
+    "noto_image": "gb-eng",
+    "comment": "Needed for noto-image flag support"
+  },
+  "flag_scotland": {
+    "emoji": "🏴󠁧󠁢󠁳󠁣󠁴󠁿",
+    "aliases": ["scotland"],
+    "noto_image": "gb-sct",
+    "comment": "Needed for noto-image flag support"
+},
+  "flag_wales": {
+    "emoji": "🏴󠁧󠁢󠁷󠁬󠁳󠁿",
+    "aliases": ["wales"],
+    "noto_image": "gb-wls",
+    "comment": "Needed for noto-image flag support"
+  }
+}

data/vendor/unicode/13.0/ReadMe.txt

@@ -0,0 +1,21 @@
+# Unicode Emoji
+# © 2020 Unicode®, Inc.
+# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+
+This directory contains data files for Unicode Emoji, Version 13.0
+
+Public/emoji/13.0/
+
+  emoji-sequences.txt
+  emoji-zwj-sequences.txt
+  emoji-test.txt
+
+The following related files are found in the UCD for Version 13.0
+
+Public/13.0.0/ucd/emoji/
+
+  emoji-data.txt
+  emoji-variation-sequences.txt
+
+For documentation, see UTS #51 Unicode Emoji, Version 13.0