script_detector_2 0.1.0 → 0.4.0

data/lib/script_detector_2/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScriptDetector2
-  VERSION = '0.1.0'
+  VERSION = '0.4.0'
 end

data/lib/script_detector_2.rb

@@ -8,10 +8,10 @@ require_relative 'script_detector_2/string'
 module ScriptDetector2
   class << self
     # @param string [String]
-    # @return [Boolean]
+    # @return [Boolean] true if +string+ appears to be Japanese
     def japanese?(string)
-      return true if string =~ /[\p{Hiragana}\p{Katakana}]/
-      return false if string =~ /\p{Hangul}/
+      return true if kana?(string)
+      return false if hangul?(string)
 
       kanji = string.scan(/\p{Han}/)
       return false unless kanji.any?
@@ -20,15 +20,22 @@ module ScriptDetector2
     end
 
     # @param string [String]
-    # @return [Boolean]
+    # @return [Boolean] true if +string+ contains Hiragana or Katakana
+    def kana?(string)
+      /[\p{Hiragana}\p{Katakana}]/.match?(string)
+    end
+
+    # @param string [String]
+    # @return [Boolean] true if +string+ appears to be Chinese (either
+    #   Simplified or Traditional)
     def chinese?(string)
       return false if string =~ /[\p{Hiragana}\p{Katakana}\p{Hangul}]/
 
-      string =~ /\p{Han}/
+      /\p{Han}/.match?(string)
     end
 
     # @param string [String]
-    # @return [Boolean]
+    # @return [Boolean] true if +string+ appears to be Simplified Chinese
     def simplified_chinese?(string)
       return false if string =~ /[\p{Hiragana}\p{Katakana}\p{Hangul}]/
 
@@ -39,7 +46,7 @@ module ScriptDetector2
     end
 
     # @param string [String]
-    # @return [Boolean]
+    # @return [Boolean] true if +string+ appears to be Traditional Chinese
     def traditional_chinese?(string)
       return false if string =~ /[\p{Hiragana}\p{Katakana}\p{Hangul}]/
 
@@ -50,10 +57,10 @@ module ScriptDetector2
     end
 
     # @param string [String]
-    # @return [Boolean]
+    # @return [Boolean] true if +string+ appears to be Korean
     def korean?(string)
-      return true if string =~ /\p{Hangul}/
-      return false if string =~ /[\p{Hiragana}\p{Katakana}]/
+      return true if hangul?(string)
+      return false if kana?(string)
 
       hanja = string.scan(/\p{Han}/)
       return false unless hanja.any?
@@ -61,16 +68,77 @@ module ScriptDetector2
       hanja.all?(KOREAN_PATTERN)
     end
 
+    # @param string [String]
+    # @return [Boolean] true if +string+ contains Hangul
+    def hangul?(string)
+      /\p{Hangul}/.match?(string)
+    end
+
+    # Make a best-effort attempt to guess the singular script of +string+.
+    # Result is a symbol representing one of the scripts defined by ISO 15924,
+    # namely one of:
+    # - Hans (Simplified Chinese)
+    # - Hant (Traditional Chinese)
+    # - Hani (Unspecified Han)
+    # - Jpan (Japanese: Han, Hiragana, Katakana)
+    # - Kore (Korean: Hangul, Han)
+    # - Zyyy (Undetermined)
+    #
+    # Note that this is likely to give poor results for very short strings,
+    # which are often inherently ambiguous.
+    #
     # @param string [String]
     # @return [Symbol]
     def identify_script(string)
-      if japanese?(string) then :Jpan
+      return :Jpan if kana?(string)
+      return :Kore if hangul?(string)
+
+      is_hant = traditional_chinese?(string)
+      is_hans = simplified_chinese?(string)
+      return :Hani if is_hant && is_hans
+
+      is_japanese = japanese?(string)
+      return :Hani if is_japanese && (is_hant || is_hans)
+
+      # At this point we have determined that the string does not contain
+      # Hangul; for such a string to be Korean would be unusual. Allowing Korean
+      # to dilute the result to Hani is going to be a loss on average, so we
+      # don't handle it like Japanese above.
+
+      if is_hans then :Hans
+      elsif is_hant then :Hant
+      elsif is_japanese then :Jpan
       elsif korean?(string) then :Kore
-      elsif traditional_chinese?(string) then :Hant
-      elsif simplified_chinese?(string) then :Hans
       elsif chinese?(string) then :Hani
-      else :Zyyy
+      else
+        :Zyyy
       end
     end
+
+    # Identify all CJK scripts represented in +string+. Result is a list of symbols
+    # representing scripts defined by ISO 15924, namely one or more of:
+    # - Hans (Simplified Chinese)
+    # - Hant (Traditional Chinese)
+    # - Hani (Unspecified Chinese)
+    # - Jpan (Japanese: Han, Hiragana, Katakana)
+    # - Kore (Korean: Hangul, Han)
+    # - Zyyy (Undetermined)
+    #
+    # This method does not attempt to identify other scripts such as Latn.
+    #
+    # @param string [String]
+    # @return [Array<Symbol>]
+    def identify_scripts(string)
+      result = []
+
+      result << :Hans if simplified_chinese?(string)
+      result << :Hant if traditional_chinese?(string)
+      result << :Jpan if japanese?(string)
+      result << :Kore if korean?(string)
+      result << :Hani if chinese?(string) && result.empty?
+      result << :Zyyy if result.empty?
+
+      result
+    end
   end
 end

data/script_detector_2.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/amake/script_detector_2.git'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

data/tasks/gen_src.rake

@@ -2,7 +2,7 @@
 
 TMP_DIR = 'tmp'
 UNIHAN_ZIP = File.join(TMP_DIR, 'Unihan.zip')
-UNIHAN_URL = 'https://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip'
+UNIHAN_URL = 'https://www.unicode.org/Public/14.0.0/ucd/Unihan.zip'
 
 directory TMP_DIR
 

data/tasks/unihan.rb

@@ -5,15 +5,20 @@ module Unihan
   CODEPOINT_PATTERN = /U\+(?<hex>[A-F0-9]+)/.freeze
 
   class << self
-    # @param readings_data [Hash<Integer,Hash{String => String}>]
+    # @param dict_data [Hash<Integer,Hash{String => String}>]
     # @param tags [Array<String>]
     # @return [Regexp]
     def gen_unihan_core_pattern(dict_data, *tags)
-      codepoints = dict_data.select do |_, data|
+      gen_pattern(codepoints_for_tags(dict_data, tags))
+    end
+
+    # @param dict_data [Hash<Integer,Hash{String => String}>]
+    # @param tags [Array<String>]
+    # @return [Array<Integer>]
+    def codepoints_for_tags(dict_data, tags)
+      dict_data.select do |_, data|
         tags.all? { |t| data['kUnihanCore2020']&.include?(t) }
       end.keys
-
-      gen_pattern(codepoints)
     end
 
     # @param codepoints [Array<Integer>]
@@ -22,11 +27,13 @@ module Unihan
       alts = group(codepoints).map do |first, last|
         if first == last
           format('\u{%x}', first)
+        elsif first.succ == last
+          format('\u{%<first>x}\u{%<last>x}', first: first, last: last)
         else
           format('\u{%<first>x}-\u{%<last>x}', first: first, last: last)
         end
       end
-      /[#{alts.join}]/
+      /\A[#{alts.join}]+\Z/
     end
 
     # @param codepoints [Array<Integer>]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: script_detector_2
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Aaron Madlon-Kay
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-22 00:00:00.000000000 Z
+date: 2021-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -76,6 +76,7 @@ files:
 - ".dir-locals.el"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
+- ".ruby-version"
 - ".solargraph.yml"
 - CHANGELOG.md
 - Gemfile
@@ -99,6 +100,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/amake/script_detector_2
   source_code_uri: https://github.com/amake/script_detector_2.git
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths: