negarmoji 0.1.6 → 0.1.8

data/lib/negarmoji/character.rb

@@ -87,7 +87,8 @@ module Emoji # :nodoc:
       if custom?
         "#{name}.#{extension}"
       else
-        "#{hex_inspect}.#{extension}"
+        hex_name = hex_inspect.gsub(%r{-(fe0f|200d)\b}, "")
+        "#{hex_name}.#{extension}"
       end
     end
   end

data/lib/negarmoji/emoji.rb

@@ -134,5 +134,5 @@ module Emoji # :nodoc:
   end
 end
 
-# Preload negarmoji into memory
+# Preload negarmoji into memory.
 Emoji.all

data/lib/negarmoji/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Emoji
-  VERSION = "0.1.6"
+  VERSION = "0.1.8"
 end

data/script/downloader.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# path variables.
+VENDORDIR="${PWD%/*}"/vendor
+TEMPDIR="temp"
+
+# create temporary directory for files.
+mkdir -p $TEMPDIR
+
+# download gemoji and openmoji json files.
+curl -L https://raw.githubusercontent.com/hfg-gmuend/openmoji/master/data/openmoji.json --output $TEMPDIR/openmoji.json
+curl -L https://raw.githubusercontent.com/github/gemoji/master/db/emoji.json --output $TEMPDIR/gemoji.json
+
+# download latest unicode emoji test from gemoji.
+curl -L https://raw.githubusercontent.com/github/gemoji/master/vendor/unicode-emoji-test.txt --output "$VENDORDIR"/unicode-emoji-test.txt

data/script/filename_generator.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "negarmoji"
+require "json"
+
+file_names = Emoji.all.map(&:image_filename).flatten
+
+File.open(ARGV[0], "w") do |file|
+  file.write(JSON.pretty_generate(file_names))
+end

data/script/json_generator.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+
+# Python Standard Library
+import json
+import codecs
+import copy
+
+def handle_alias_special_case(alias):
+    # special aliases causing duplication problems.
+    special_cases = [
+                     "pretzel",
+                     "sunglasses",
+                     "pouting_face",
+                     "kiss",
+                     "horse",
+                     "camel",
+                     "post_office",
+                     "satellite",
+                     "umbrella",
+                     "snowman",
+                     "calendar"
+                    ]
+
+    if alias in special_cases:
+        alias = list(alias)
+        alias.append("2")
+        alias = "".join(alias)
+        return alias
+    elif alias == "flag_åland_islands":
+        return "flag_aland_islands"
+    else:
+        return alias
+
+def create_alias(annotation, category):
+    """Create alias from annotation."""
+    # clean annotation from bad characters.
+    alias = annotation.strip()
+    alias = alias.replace("&", "")
+    alias = alias.replace("!", "")
+    alias = alias.replace("(", "")
+    alias = alias.replace(")", "")
+    alias = alias.replace("“", "")
+    alias = alias.replace("”", "")
+    alias = alias.replace("’", "")
+    alias = alias.replace("#", "hashtag")
+    alias = alias.replace("*", "astrix")
+    alias = alias.replace("ä", "a")
+    alias = alias.replace("ã", "a")
+    # doesn't remove flag_åland_islands :|
+    alias = alias.replace("å", "a")
+    alias = alias.replace("ç", "c")
+    alias = alias.replace("é", "e")
+    alias = alias.replace("ô", "o")
+    alias = alias.replace("í", "i")
+    alias = alias.replace("ñ", "n")
+    alias = alias.replace("ü", "u")
+    alias = alias.replace(" ", "_")
+    alias = alias.replace("-", "_")
+    alias = alias.replace(".", "_")
+    alias = alias.replace(":", "_")
+    alias = alias.replace(",", "_")
+    alias = alias.replace("__", "_")
+    # handle openmoji extra emojis.
+    if category == "Extras & Openmoji":
+        alias = list(alias)
+        alias.append("-extra")
+        alias = "".join(alias)
+    # lowercase characters.
+    alias = alias.lower()
+    return handle_alias_special_case(alias)
+
+def create_new_emoji(emoji, description, category, aliases, tags, unicode_version):
+    """base structure for a emoji dictionary."""
+    new_dict = {
+      "emoji": emoji
+    , "description": description
+    , "category": category
+    , "aliases": [
+      aliases
+    ]
+    , "tags": [
+      tags
+    ]
+    , "unicode_version": f"{unicode_version}.0" if unicode_version is not None else ""
+    , "ios_version": f"{unicode_version}.0" if unicode_version is not None else ""
+    }
+    return new_dict
+
+def generate_negarmoji_json(gemoji_json_file, openmoji_json_file, output_file_path):
+    """Generate negarmoji json from combining openmoji json and gemoji json."""
+
+    # read in gemoji and openmoji json files.
+    with open(gemoji_json_file, "rb") as read_file:
+        gemoji = json.load(read_file)
+    with open(openmoji_json_file, "rb") as read_file:
+        openmoji = json.load(read_file)
+
+    # collect emojis from emoji dictionaries.
+    gemoji_emojis = set([emoji_dict["emoji"] for emoji_dict in gemoji])
+    openmoji_emojis = set([emoji_dict["emoji"] for emoji_dict in openmoji])
+
+    # some emojis are same in both gemoji and openmoji emojis but they differ in
+    # the end sequence, openmoji usually have an extra \ufe0f character, so in
+    # this part of code I'm trying to normalize gemoji dictionary by appending
+    # \ufe0f to the end of some emojis which results in a same emoji in
+    # the openmoji dictionary.
+
+    # find emojis in gemoji which will be same with an emoji in openmoji if we
+    # add \ufe0f at the end of it.
+    difference = [
+                  (emoji.replace("\ufe0f", ""), emoji)
+                  for emoji in openmoji_emojis
+                  if emoji.replace("\ufe0f", "") in gemoji_emojis
+                 ]
+
+    # normalize gemoji dictionary by replacing emojis.
+    for emoji_dict in gemoji:
+        for diff in difference:
+            if emoji_dict["emoji"] == diff[0]:
+                emoji_dict["emoji"] = diff[1]
+                break
+
+    # re-create gemoji emojis from normalized gemoji dictionary.
+    gemoji_emojis = set([emoji_dict["emoji"] for emoji_dict in gemoji])
+
+    # find difference between openmoji and gemoji. since we have normalized
+    # gemoji, in this stage only unique new emojis from openmoji will show up
+    # and not the same emojis with and extra \ufe0f at the end of it.
+    difference = openmoji_emojis - gemoji_emojis
+
+    # create base negarmoji dictionary from gemoji.
+    negarmoji = copy.deepcopy(gemoji)
+
+    # convert new unique emojis from openmoji dictionary type to negarmoji dictionary
+    for emoji_dict in openmoji:
+        if emoji_dict["emoji"] in difference:
+            # filter properties.
+            emoji = emoji_dict["emoji"]
+            description = emoji_dict["annotation"]
+            category = emoji_dict["group"].replace("-", " & ").title()
+            aliases = create_alias(emoji_dict["annotation"], category)
+            tags = emoji_dict["tags"]
+            unicode_version = emoji_dict["unicode"]
+            # create emoji dictionary.
+            new_emoji_dict = create_new_emoji(emoji, description, category, aliases, tags, unicode_version)
+            # add new dictionary to negarmoji dictionary.
+            negarmoji.append(new_emoji_dict)
+
+    # add extra aliases for emojis from openmoji annotations.
+    for emoji_dict in negarmoji:
+        for emoji_dict_2 in openmoji:
+            if emoji_dict["emoji"] == emoji_dict_2["emoji"]:
+                new_alias = create_alias(emoji_dict_2["annotation"], emoji_dict_2["group"].replace("-", " & ").title())
+                if new_alias not in emoji_dict["aliases"]:
+                    check_duplicate = list(new_alias)
+                    check_duplicate.append("2")
+                    check_duplicate = "".join(check_duplicate)
+                    if check_duplicate not in emoji_dict["aliases"]:
+                        emoji_dict["aliases"].append(new_alias)
+                break
+
+    with codecs.open(output_file_path, "w", encoding="utf8") as write_file:
+        json.dump(negarmoji, write_file, ensure_ascii=False, indent=4)

data/script/test.sh

@@ -8,5 +8,5 @@
 # -x  Print commands and their arguments as they are executed.
 set -ex
 
-script/test_style.sh
-script/test_module.sh
+script/test_style.sh || ./test_style.sh
+script/test_module.sh || ./test_module.sh

data/script/updater.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+# Python Standard Library
+import os
+import glob
+import subprocess
+
+# Custom modules
+from json_generator import generate_negarmoji_json
+
+# configure path variables.
+file_path = os.path.abspath(os.path.dirname(__file__))
+base_path = os.path.abspath(os.path.dirname(file_path))
+temp_path = os.path.join(file_path, "temp")
+gemoji_json_file = os.path.join(temp_path, "gemoji.json")
+openmoji_json_file = os.path.join(temp_path, "openmoji.json")
+negarmoji_json_file = os.path.join(base_path, "db", "negarmoji.json")
+filenames_json_file = os.path.join(base_path, "db", "filenames.json")
+
+# download latest files from gemoji and openmoji repositories.
+process = subprocess.run(["./downloader.sh"], stdout=subprocess.PIPE)
+
+# generate negarmoji json.
+generate_negarmoji_json(gemoji_json_file, openmoji_json_file, negarmoji_json_file)
+
+# generate emoji filenames json.
+process = subprocess.run(["./filename_generator.rb", f"{filenames_json_file}"], stdout=subprocess.PIPE)
+
+# delete temp folder.
+process = subprocess.run(["rm", "-rf", f"{temp_path}"], stdout=subprocess.PIPE)

data/test/emoji_test.rb

@@ -1,8 +1,8 @@
-require "test_helper"
-require_relative "../db/emoji-test-parser"
+require 'test_helper'
+require_relative '../db/emoji-test-parser'
 
 class EmojiTest < TestCase
-  test "fetching all negarmoji" do
+  test "fetching all emoji" do
     count = Emoji.all.size
     assert count > 845, "there were too few emojis: #{count}"
   end
@@ -13,26 +13,26 @@ class EmojiTest < TestCase
     assert count > min_size, "there were too few unicode mappings: #{count}"
   end
 
-  test "finding negarmoji by alias" do
-    assert_equal "smile", Emoji.find_by_alias("smile").name
+  test "finding emoji by alias" do
+    assert_equal 'smile', Emoji.find_by_alias('smile').name
   end
 
-  test "finding nonexistent negarmoji by alias returns nil" do
-    assert_nil Emoji.find_by_alias("$$$")
+  test "finding nonexistent emoji by alias returns nil" do
+    assert_nil Emoji.find_by_alias('$$$')
   end
 
-  test "finding negarmoji by unicode" do
+  test "finding emoji by unicode" do
     emoji = Emoji.find_by_unicode("\u{1f604}") # grinning face with smiling eyes
     assert_equal "\u{1f604}", emoji.raw
   end
 
-  test "finding nonexistent negarmoji by unicode returns nil" do
+  test "finding nonexistent emoji by unicode returns nil" do
     assert_nil Emoji.find_by_unicode("\u{1234}")
   end
 
   test "unicode_aliases" do
     emoji = Emoji.find_by_unicode("\u{2728}") # sparkles
-    assert_equal %w[2728 2728-fe0f], emoji.unicode_aliases.map { |u| Emoji::Character.hex_inspect(u) }
+    assert_equal ["2728", "2728-fe0f"], emoji.unicode_aliases.map { |u| Emoji::Character.hex_inspect(u) }
   end
 
   test "unicode_aliases doesn't necessarily include form without VARIATION_SELECTOR_16" do
@@ -41,22 +41,21 @@ class EmojiTest < TestCase
   end
 
   test "emojis have tags" do
-    emoji = Emoji.find_by_alias("smile")
-    assert emoji.tags.include?("happy")
-    assert emoji.tags.include?("joy")
-    assert emoji.tags.include?("pleased")
+    emoji = Emoji.find_by_alias('smile')
+    assert emoji.tags.include?('happy')
+    assert emoji.tags.include?('joy')
+    assert emoji.tags.include?('pleased')
   end
 
   GENDER_EXCEPTIONS = [
-      "man_with_gua_pi_mao",
-      "woman_with_headscarf",
-      "man_in_tuxedo",
-      "pregnant_woman",
-      "isle_of_man",
-      "blonde_woman",
-      %r{^couple(kiss)?_},
-      %r{^family_}
-  ].freeze
+    "man_with_gua_pi_mao",
+    "woman_with_headscarf",
+    "pregnant_woman",
+    "isle_of_man",
+    "blonde_woman",
+    /^couple(kiss)?_/,
+    /^family_/,
+  ]
 
   test "emojis have valid names" do
     aliases = Emoji.all.flat_map(&:aliases)
@@ -65,58 +64,81 @@ class EmojiTest < TestCase
     alias_count = Hash.new(0)
     aliases.each do |name|
       alias_count[name] += 1
-      invalid << name if name !~ %r{\A[\w+-]+\Z}
+      invalid << name if name !~ /\A[\w+-]+\Z/
     end
 
     duplicates = alias_count.select { |_, count| count > 1 }.keys
 
-    assert_equal [], invalid, "some negarmoji have invalid names"
-    assert_equal [], duplicates, "some negarmoji aliases have duplicates"
+    assert_equal [], invalid, "some emoji have invalid names"
+    assert_equal [], duplicates, "some emoji aliases have duplicates"
+  end
+
+  test "missing or incorrect unicodes" do
+    emoji_map, _ = EmojiTestParser.parse(File.expand_path("../../vendor/unicode-emoji-test.txt", __FILE__))
+    source_unicode_emoji = emoji_map.values
+    supported_sequences = Emoji.all.flat_map(&:unicode_aliases)
+    text_glyphs = Emoji.const_get(:TEXT_GLYPHS)
+
+    missing = 0
+    message = "Missing or incorrect unicodes:\n"
+    source_unicode_emoji.each do |emoji|
+      emoji[:sequences].each do |raw|
+        next if text_glyphs.include?(raw) || Emoji.find_by_unicode(raw)
+        message << "%s (%s)" % [Emoji::Character.hex_inspect(raw), emoji[:description]]
+        if found = Emoji.find_by_unicode(raw.gsub("\u{fe0f}", ""))
+          message << " - could be %s (:%s:)" % [found.hex_inspect, found.name]
+        end
+        message << "\n"
+        missing += 1
+      end
+    end
+
+    assert_equal 0, missing, message
   end
 
-  test "negarmoji have category" do
+  test "emoji have category" do
     missing = Emoji.all.select { |e| e.category.to_s.empty? }
-    assert_equal [], missing.map(&:name), "some negarmoji don't have a category"
+    assert_equal [], missing.map(&:name), "some emoji don't have a category"
 
-    emoji = Emoji.find_by_alias("family_man_woman_girl")
-    assert_equal "People & Body", emoji.category
+    emoji = Emoji.find_by_alias('family_man_woman_girl')
+    assert_equal 'People & Body', emoji.category
 
     categories = Emoji.all.map(&:category).uniq.compact
     assert_equal [
-                     "Smileys & Emotion",
-                     "People & Body",
-                     "Animals & Nature",
-                     "Food & Drink",
-                     "Travel & Places",
-                     "Activities",
-                     "Objects",
-                     "Symbols",
-                     "Flags",
-                     "Component",
-                     "Extras & Openmoji",
-                     "Extras & Unicode"
+      "Smileys & Emotion",
+      "People & Body",
+      "Animals & Nature",
+      "Food & Drink",
+      "Travel & Places",
+      "Activities",
+      "Objects",
+      "Symbols",
+      "Flags",
+      "Component",
+      "Extras & Openmoji",
+      "Extras & Unicode"
     ], categories
   end
 
-  test "negarmoji have description" do
+  test "emoji have description" do
     missing = Emoji.all.select { |e| e.description.to_s.empty? }
-    assert_equal [], missing.map(&:name), "some negarmoji don't have a description"
+    assert_equal [], missing.map(&:name), "some emoji don't have a description"
 
-    emoji = Emoji.find_by_alias("family_man_woman_girl")
-    assert_equal "family: man, woman, girl", emoji.description
+    emoji = Emoji.find_by_alias('family_man_woman_girl')
+    assert_equal 'family: man, woman, girl', emoji.description
   end
 
-  test "negarmoji have Unicode version" do
-    emoji = Emoji.find_by_alias("family_man_woman_girl")
-    assert_equal "6.0", emoji.unicode_version
+  test "emoji have Unicode version" do
+    emoji = Emoji.find_by_alias('family_man_woman_girl')
+    assert_equal '6.0', emoji.unicode_version
   end
 
-  test "negarmoji have iOS version" do
+  test "emoji have iOS version" do
     missing = Emoji.all.select { |e| e.ios_version.to_s.empty? }
-    assert_equal [], missing.map(&:name), "some negarmoji don't have an iOS version"
+    assert_equal [], missing.map(&:name), "some emoji don't have an iOS version"
 
-    emoji = Emoji.find_by_alias("family_man_woman_girl")
-    assert_equal "8.3", emoji.ios_version
+    emoji = Emoji.find_by_alias('family_man_woman_girl')
+    assert_equal '8.3', emoji.ios_version
   end
 
   test "no custom emojis" do
@@ -183,11 +205,12 @@ class EmojiTest < TestCase
 
     begin
       assert_equal emoji, Emoji.find_by_alias("weary")
+      assert_equal emoji, Emoji.find_by_alias("weary_face")
       assert_equal emoji, Emoji.find_by_alias("whining")
       assert_equal emoji, Emoji.find_by_unicode("\u{1f629}")
       assert_equal emoji, Emoji.find_by_unicode("\u{1f629}\u{266a}")
 
-      assert_equal %w[weary whining], emoji.aliases
+      assert_equal %w[weary weary_face whining], emoji.aliases
       assert_includes emoji.tags, "complaining"
     ensure
       emoji.aliases.pop