neologdish-normalizer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2aa7f75228455a5870d56e6f8b74dd74dcb9f8c0619bb86eda89e02518478afd
-  data.tar.gz: b92a3042b6f7c3ad7b2653bb7f04643e2769733104ee78e1c689170776729b53
+  metadata.gz: 57bda9b5b4b5de8dc0b6582e3a5eb5d2103d3207a405bb7b977b52e0eff02322
+  data.tar.gz: a35a668f517d31229e370dd4839c911f2e1b16f6c868e8537d9e61f2e44c7eee
 SHA512:
-  metadata.gz: 4479aa95d7ba3d2aca68feecf5a4674dc5af9f71c795ac748bcb6e38558b7d9e73c4d068b071441c020b96a4765b58224ad0328fe4856aafcf934bf6eb4d6e99
-  data.tar.gz: fb794e40955c4460bf592fac2f9baf415bb9c486ff07a288fc9b3a083bb8256a98d949291b497fd36b2e846ea083dee3925afd8001ae6a7ce90e4536badf6836
+  metadata.gz: bf2788e135ccecdfdea6a4be0e52e4e1b151e4fc6c8847844299d2bbc1578c23553f92d6d47bea0e955dff2ddb941f9944e94cbfe420bbe51b733c2094497f6e
+  data.tar.gz: 8da7c4aa8848bef56fd5d612f22e08e0cf33cde2ccc0af829d2394da3f8bd9cb5c2ae38223adcd7d3b871df6b3909f1c550a36beb86f0bdcf6a9ab14c0d65345

data/.rubocop.yml

@@ -1,10 +1,9 @@
 require:
-  - rubocop-rake
   - rubocop-minitest
-
+plugins:
+  - rubocop-rake
 AllCops:
   NewCops: enable
-
 Metrics:
   Enabled: false
 Layout/LineLength:

data/.ruby-version

@@ -1 +1 @@
-3.3.5
+3.4.4

data/README.md

@@ -47,5 +47,5 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/moznion/neologdish-normalizer.
+Bug reports and pull requests are welcome on GitHub at https://github.com/moznion/neologdish-normalizer-ruby.
 

data/lib/neologdish/normalizer/version.rb

@@ -2,6 +2,6 @@
 
 module Neologdish
   module Normalizer
-    VERSION = '0.1.0' #: String
+    VERSION = '0.2.0'
   end
 end

data/lib/neologdish/normalizer.rb

@@ -5,6 +5,9 @@ require_relative 'normalizer/version'
 module Neologdish
   # A Japanese text normalizer module according to the neologd convention.
   module Normalizer
+    NORMALIZED_HYPHEN = "\u002d" # -
+    NORMALIZED_VOWEL = "\u30fc" # ー
+
     CONVERSION_MAP = {
       # Normalize [0-9a-zA-Z] to half-width
       '０' => '0', '１' => '1', '２' => '2', '３' => '3', '４' => '4', '５' => '5', '６' => '6', '７' => '7', '８' => '8', '９' => '9',
@@ -15,11 +18,31 @@ module Neologdish
       'ｋ' => 'k', 'ｌ' => 'l', 'ｍ' => 'm', 'ｎ' => 'n', 'ｏ' => 'o', 'ｐ' => 'p', 'ｑ' => 'q', 'ｒ' => 'r', 'ｓ' => 's', 'ｔ' => 't',
       'ｕ' => 'u', 'ｖ' => 'v', 'ｗ' => 'w', 'ｘ' => 'x', 'ｙ' => 'y', 'ｚ' => 'z',
       # normalize the hyphen/minus-ish characters to '-'
-      '˗' => '-', '֊' => '-', '‐' => '-', '‑' => '-', '‒' => '-', '–' => '-', '⁃' => '-', '⁻' => '-', '₋' => '-', '−' => '-',
+      "\u02d7" => NORMALIZED_HYPHEN, # ˗
+      "\u058a" => NORMALIZED_HYPHEN, # ֊
+      "\u2010" => NORMALIZED_HYPHEN, # ‐
+      "\u2011" => NORMALIZED_HYPHEN, # ‑
+      "\u2012" => NORMALIZED_HYPHEN, # ‒
+      "\u2013" => NORMALIZED_HYPHEN, # –
+      "\u2043" => NORMALIZED_HYPHEN, # ⁃
+      "\u207b" => NORMALIZED_HYPHEN, # ⁻
+      "\u208b" => NORMALIZED_HYPHEN, # ₋
+      "\u2212" => NORMALIZED_HYPHEN, # −
       # normalize the long-vowel mark-ish characters to 'ー'
-      '﹣' => 'ー', '－' => 'ー', 'ｰ' => 'ー', '—' => 'ー', '―' => 'ー', '─' => 'ー', '━' => 'ー',
+      "\u2014" => NORMALIZED_VOWEL, # —
+      "\u2015" => NORMALIZED_VOWEL, # ―
+      "\u2500" => NORMALIZED_VOWEL, # ─
+      "\u2501" => NORMALIZED_VOWEL, # ━
+      "\ufe63" => NORMALIZED_VOWEL, # ﹣
+      "\uff0d" => NORMALIZED_VOWEL, # －
+      "\uff70" => NORMALIZED_VOWEL, # ｰ
       # remove the tilde-ish characters
-      '~' => '', '∼' => '', '∾' => '', '〜' => '', '〰' => '', '～' => '',
+      "\u007e" => '', # ~
+      "\u223c" => '', # ∼
+      "\u223e" => '', # ∾
+      "\u301c" => '', # 〜
+      "\u3030" => '', # 〰
+      "\uff5e" => '', # ～
       # normalize the full-width special symbol characters (/！”＃＄％＆’（）＊＋，−．／：；＜＞？＠［￥］＾＿｀｛｜｝) and space characters to half-width
       '　' => ' ', '！' => '!', '”' => '"', '＃' => '#', '＄' => '$', '％' => '%', '＆' => '&', '’' => "'", '（' => '(', '）' => ')',
       '＊' => '*', '＋' => '+', '，' => ',', '．' => '.', '／' => '/', '：' => ':', '；' => ';', '＜' => '<', '＞' => '>', '？' => '?',
@@ -53,18 +76,39 @@ module Neologdish
       'ｯ' => 'ッ', 'ｬ' => 'ヤ', 'ｭ' => 'ユ', 'ｮ' => 'ヨ'
     }.freeze #: Hash[String, String]
 
+    DAKUON_HANDAKUON_POSSIBLES = {
+      'ウ' => true,
+      'カ' => true, 'キ' => true, 'ク' => true, 'ケ' => true, 'コ' => true,
+      'サ' => true, 'シ' => true, 'ス' => true, 'セ' => true, 'ソ' => true,
+      'タ' => true, 'チ' => true, 'ツ' => true, 'テ' => true, 'ト' => true,
+      'ハ' => true, 'ヒ' => true, 'フ' => true, 'ヘ' => true, 'ホ' => true,
+      'う' => true,
+      'か' => true, 'き' => true, 'く' => true, 'け' => true, 'こ' => true,
+      'さ' => true, 'し' => true, 'す' => true, 'せ' => true, 'そ' => true,
+      'た' => true, 'ち' => true, 'つ' => true, 'て' => true, 'と' => true,
+      'は' => true, 'ひ' => true, 'ふ' => true, 'へ' => true, 'ほ' => true
+    }.freeze #: Hash[String, bool]
+
     DAKUON_KANA_MAP = {
+      'ウ' => 'ヴ',
       'カ' => 'ガ', 'キ' => 'ギ', 'ク' => 'グ', 'ケ' => 'ゲ', 'コ' => 'ゴ',
       'サ' => 'ザ', 'シ' => 'ジ', 'ス' => 'ズ', 'セ' => 'ゼ', 'ソ' => 'ゾ',
       'タ' => 'ダ', 'チ' => 'ヂ', 'ツ' => 'ヅ', 'テ' => 'デ', 'ト' => 'ド',
-      'ハ' => 'バ', 'ヒ' => 'ビ', 'フ' => 'ブ', 'ヘ' => 'ベ', 'ホ' => 'ボ'
+      'ハ' => 'バ', 'ヒ' => 'ビ', 'フ' => 'ブ', 'ヘ' => 'ベ', 'ホ' => 'ボ',
+      'う' => 'ゔ',
+      'か' => 'が', 'き' => 'ぎ', 'く' => 'ぐ', 'け' => 'げ', 'こ' => 'ご',
+      'さ' => 'ざ', 'し' => 'じ', 'す' => 'ず', 'せ' => 'ぜ', 'そ' => 'ぞ',
+      'た' => 'だ', 'ち' => 'ぢ', 'つ' => 'づ', 'て' => 'で', 'と' => 'ど',
+      'は' => 'ば', 'ひ' => 'び', 'ふ' => 'ぶ', 'へ' => 'べ', 'ほ' => 'ぼ'
     }.freeze #: Hash[String, String]
 
     HANDAKUON_KANA_MAP = {
-      'ハ' => 'パ', 'ヒ' => 'ピ', 'フ' => 'プ', 'ヘ' => 'ペ', 'ホ' => 'ポ'
+      'ハ' => 'パ', 'ヒ' => 'ピ', 'フ' => 'プ', 'ヘ' => 'ペ', 'ホ' => 'ポ',
+      'は' => 'ぱ', 'ひ' => 'ぴ', 'ふ' => 'ぷ', 'へ' => 'ぺ', 'ほ' => 'ぽ'
     }.freeze #: Hash[String, String]
 
-    private_constant :CONVERSION_MAP, :LATIN_MAP, :HALF_WIDTH_KANA_MAP, :DAKUON_KANA_MAP, :HANDAKUON_KANA_MAP
+    private_constant :CONVERSION_MAP, :LATIN_MAP, :HALF_WIDTH_KANA_MAP, :DAKUON_KANA_MAP, :HANDAKUON_KANA_MAP, :DAKUON_HANDAKUON_POSSIBLES,
+                     :NORMALIZED_HYPHEN, :NORMALIZED_VOWEL
 
     # Normalize the given text.
     #
@@ -77,23 +121,29 @@ module Neologdish
       squeezee = ''
       prev_latin = false
       whitespace_encountered = false
-      encountered_half_width_kana = nil
+      dakuon_handakuon_possible = nil
       normalized = str.chars.map do |c|
         prefix = ''
         c = conversion_map[c] || c
 
         # normalize the Half-width kana to full-width
-        if encountered_half_width_kana
-          if (c == 'ﾞ' && (k = DAKUON_KANA_MAP[encountered_half_width_kana])) ||
-             (c == 'ﾟ' && (k = HANDAKUON_KANA_MAP[encountered_half_width_kana]))
+        if dakuon_handakuon_possible
+          if (["\u309b", "\u3099", "\uff9e"].include?(c) && (k = DAKUON_KANA_MAP[dakuon_handakuon_possible])) ||
+             (["\u309c", "\u309a", "\uff9f"].include?(c) && (k = HANDAKUON_KANA_MAP[dakuon_handakuon_possible]))
             c = ''
             prefix = k
           else
-            prefix = encountered_half_width_kana
+            prefix = dakuon_handakuon_possible
           end
         end
 
         if (encountered_half_width_kana = HALF_WIDTH_KANA_MAP[c])
+          c = encountered_half_width_kana
+        end
+
+        dakuon_handakuon_possible = nil
+        if DAKUON_HANDAKUON_POSSIBLES[c]
+          dakuon_handakuon_possible = c
           c = ''
         end
 
@@ -120,7 +170,7 @@ module Neologdish
         prev_latin = is_latin
 
         prefix + c
-      end.join + (encountered_half_width_kana || '')
+      end.join + (dakuon_handakuon_possible || '')
 
       normalized.strip
     end

data/sig/generated/neologdish/normalizer/version.rbs

@@ -2,6 +2,6 @@
 
 module Neologdish
   module Normalizer
-    VERSION: String
+    VERSION: ::String
   end
 end

data/sig/generated/neologdish/normalizer.rbs

@@ -3,12 +3,18 @@
 module Neologdish
   # A Japanese text normalizer module according to the neologd convention.
   module Normalizer
+    NORMALIZED_HYPHEN: ::String
+
+    NORMALIZED_VOWEL: ::String
+
     CONVERSION_MAP: Hash[String, String]
 
     LATIN_MAP: Hash[String, bool]
 
     HALF_WIDTH_KANA_MAP: Hash[String, String]
 
+    DAKUON_HANDAKUON_POSSIBLES: Hash[String, bool]
+
     DAKUON_KANA_MAP: Hash[String, String]
 
     HANDAKUON_KANA_MAP: Hash[String, String]
@@ -18,6 +24,6 @@ module Neologdish
     # @rbs str: String
     # @rbs override_conversion_map: Hash[String, String]
     # @rbs return: String
-    def normalize: (String str, ?Hash[String, String] override_conversion_map) -> String
+    def self?.normalize: (String str, ?Hash[String, String] override_conversion_map) -> String
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: neologdish-normalizer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - moznion
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: A Japanese text normalization library follows the conventions of neologd
   with some performance optimizations. It is designed to preprocess Japanese text
@@ -29,14 +28,13 @@ files:
 - renovate.json
 - sig/generated/neologdish/normalizer.rbs
 - sig/generated/neologdish/normalizer/version.rbs
-homepage: https://github.com/moznion/neologdish-normalizer
+homepage: https://github.com/moznion/neologdish-normalizer-ruby
 licenses: []
 metadata:
-  homepage_uri: https://github.com/moznion/neologdish-normalizer
-  source_code_uri: https://github.com/moznion/neologdish-normalizer
-  changelog_uri: https://github.com/moznion/neologdish-normalizer/releases
+  homepage_uri: https://github.com/moznion/neologdish-normalizer-ruby
+  source_code_uri: https://github.com/moznion/neologdish-normalizer-ruby
+  changelog_uri: https://github.com/moznion/neologdish-normalizer-ruby/releases
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -51,8 +49,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: A Japanese text normalization library follows the conventions of neologd
 test_files: []