kanji-translator 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d693a0727ff440e9630d4ac66f28ff42da8c707a3d222deedfaa2a394304e55
-  data.tar.gz: 5c26cc604c58942286badd65577a7b8094f9f304d34ae5e8474489ce18757676
+  metadata.gz: 56e5d89e3ee4e402ab0100d1c86b5bd156481671d4dbd6633f7b9577e4155304
+  data.tar.gz: bc7eff78a596f65fdc387ac9fa0f3245f4cc72781a785a8f4298fd18b696003a
 SHA512:
-  metadata.gz: b3e115cb0d6057dde1642b4fdf8c61acc59b4a8275e471df8a04e4ba01311dfe119677ef4a5fd7d39348687532d5e5d7711c9da2b598529960461eddff3fc140
-  data.tar.gz: 279143b96cef0e5ecccb6cd8e35b80409e6ca734ba9ec72ef61b0080aea94c76fb4d93f958155e43d4562a1a4143121f0cee4140d6f36b59fb6918f68823ab5e
+  metadata.gz: 4a36777ecf342e246a9009c788016d54a74d5a6fa439d6b0ad0e0d3fdd4d1bd86440142c06e67dea29ddcf3be927d08a872db132cca4c0237c698410bcce49d7
+  data.tar.gz: 7dba7989316c7c2c29cf5eaf2ed7799f3b92cdb2d133a126a1bc7541d9a157b5003f1ce2f3f1d34e13c1c0eb0279e6786007cde9776daf32ac366a316b76f71b

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [1.1.0] - 2025-09-08
+
+- Fixed: `to_hira` の結果を必ずひらがなに正規化（返却HTMLにカタカナが混ざる場合の不整合を解消）。
+- Improved: スラッグ生成の分割アルゴリズムを整理し、ASCII連結・空白（全角含む）境界を厳密化。混在テキストでの精度向上。
+- Changed: `to_slug(text, separator: "-", **opts)` に整理（`separator` は直接キーワード、他は `**opts`）。互換性は維持。
+- Refactor: 正規表現・正規化処理の定数化/関数抽出、内部メソッドを `private_class_method` 化。
+
 ## [1.0.0] - 2025-09-08
 
 - Breaking: `to_slug` のデフォルト挙動を `segmenter: :tiny` に変更（語境界ごとにハイフン区切り）。

data/README.md

@@ -49,20 +49,16 @@ require "kanji/translator/core_ext/string"
   - ひらがな読みをカタカナに変換して返します。
 - `Kanji::Translator.to_roma(text, **opts)`
   - 簡易ヘボン式のローマ字（ASCII、小文字）で返します。拗音/促音（ゃゅょ/っ）に対応。長音記号「ー」は無視します（例: おう→ou）。
-- `Kanji::Translator.to_slug(text, separator: "-", downcase: true, collapse: true, segmenter: :tiny, **opts)`
+- `Kanji::Translator.to_slug(text, separator: "-", downcase: true, collapse: true, **opts)`
   - `to_roma` の結果をスラッグ化します。
     - 非英数字を `separator` に置換、連続区切りを圧縮、前後の区切りをトリムします。
-   - `segmenter: :tiny`（デフォルト）: TinySegmenter で分かち、語ごとにハイフン区切り（例: "学校案内" → "gakkou-annai"）。
-   - `segmenter: :space`: 空白でのみ分割（例: "学校 案内" → "gakkou-annai"）。
-   - `segmenter: nil`: 分かちなし（語の自動区切りなし）。
+   - 内部で TinySegmenter による分かち書きを行い、語境界ごとにハイフン区切りします（例: "学校案内" → "gakkou-annai"）。
 
 例（オプション）:
 
 ```ruby
 Kanji::Translator.to_hira("漢字", timeout: 3, retries: 1)
 Kanji::Translator.to_slug("東京タワー 2010") #=> "toukyou-tawa-2010"
-Kanji::Translator.to_slug("学校 案内", segmenter: :space) #=> "gakkou-annai"
-Kanji::Translator.to_slug("学校案内", segmenter: nil) #=> "gakkouannai"
 Kanji::Translator.to_slug("Foo Bar", separator: "_") #=> "foo_bar"
 ```
 

data/lib/kanji/translator/version.rb

@@ -2,6 +2,6 @@
 
 module Kanji
   module Translator
-    VERSION = "1.0.0"
+    VERSION = "1.1.0"
   end
 end

data/lib/kanji/translator.rb

@@ -13,12 +13,26 @@ module Kanji
 
     USER_AGENT = "kanji-translator/#{VERSION}".freeze
     HOST = "yomikatawa.com"
+    ASCII_RE = /[A-Za-z0-9]/
+    SPACE_RE = /[\s\u3000]/ # ASCII whitespace or IDEOGRAPHIC SPACE
+    NON_ALNUM_RE = /[^a-z0-9]+/
+    JAPANESE_RE = /[一-龯々〆ヵヶぁ-ゖゝゞァ-ヴー]/
+    BOUNDARY = :__BOUNDARY__
 
     def self.to_hira(text, timeout: 5, retries: 2, backoff: 0.5, user_agent: USER_AGENT)
       raise ArgumentError, "text must be a String" unless text.is_a?(String)
 
+      # Fast-path for kana inputs: avoid network and normalize locally
+      if text.match?(/\A[ぁ-ゖーゝゞ]+\z/)
+        return text
+      elsif text.match?(/\A[ァ-ヴーヽヾヵヶ]+\z/)
+        return katakana_to_hiragana(text)
+      end
+
       body = fetch_page(text, timeout: timeout, retries: retries, backoff: backoff, user_agent: user_agent)
-      parse_hiragana(body)
+      hira = parse_hiragana(body)
+      # Ensure result is normalized to hiragana only (remote may mix katakana like 固有名詞)
+      katakana_to_hiragana(hira)
     end
 
     def self.to_kata(text, **)
@@ -31,35 +45,18 @@ module Kanji
       hiragana_to_romaji(hira)
     end
 
-    def self.to_slug(text, **opts)
-      sep       = opts.fetch(:separator, "-")
+    def self.to_slug(text, separator: "-", **opts)
+      sep       = separator
       downcase  = opts.fetch(:downcase, true)
       collapse  = opts.fetch(:collapse, true)
-      segmenter = opts.fetch(:segmenter, :tiny)
       net_opts  = slice_opts(opts, :timeout, :retries, :backoff, :user_agent)
 
-      s = case segmenter
-          when :tiny
-            tokens = segment_with_tiny(text)
-            parts = tokens.filter_map { |tok| normalize_slug_part(tok, net_opts) }
-            parts.join(sep)
-          when :space
-            tokens = segment_with_space(text)
-            parts = tokens.filter_map { |tok| normalize_slug_part(tok, net_opts) }
-            parts.join(sep)
-          else
-            roma = to_roma(text, **net_opts)
-            roma.dup
-          end
+      tokens = segment_with_tiny(text)
+      raw_parts = tokens.filter_map { |tok| normalize_slug_part(tok, net_opts) }
+      parts = merge_ascii_parts(raw_parts)
+      s = parts.join(sep)
 
-      s = s.downcase if downcase
-      # Replace non-alphanumeric with separator
-      s = s.gsub(/[^a-z0-9]+/, sep)
-      # Collapse duplicate separators
-      s = s.gsub(/#{Regexp.escape(sep)}{2,}/, sep) if collapse && !sep.empty?
-      # Trim leading/trailing separators
-      s = s.gsub(/^#{Regexp.escape(sep)}|#{Regexp.escape(sep)}$/, "") unless sep.empty?
-      s
+      normalize_slug_string(s, sep: sep, downcase: downcase, collapse: collapse)
     end
 
     def self.fetch_page(text, timeout:, retries:, backoff:, user_agent: USER_AGENT)
@@ -124,6 +121,10 @@ module Kanji
       hira.tr("ぁ-ゔゝゞー", "ァ-ヴヽヾー")
     end
 
+    def self.katakana_to_hiragana(kata)
+      kata.tr("ァ-ヴヽヾヵヶー", "ぁ-ゔゝゞかけー")
+    end
+
     DIGRAPHS = {
       "きゃ" => "kya", "きゅ" => "kyu", "きぇ" => "kye", "きょ" => "kyo",
       "ぎゃ" => "gya", "ぎゅ" => "gyu", "ぎぇ" => "gye", "ぎょ" => "gyo",
@@ -214,30 +215,81 @@ module Kanji
 
     def self.segment_with_tiny(text)
       require "tiny_segmenter"
-      TinySegmenter.new.segment(text)
+      seg = TinySegmenter.new
+      tokens = []
+      i = 0
+      while i < text.length
+        ch = text[i]
+        if ch =~ ASCII_RE
+          j = i + 1
+          j += 1 while j < text.length && text[j] =~ ASCII_RE
+          tokens << text[i...j]
+          i = j
+        elsif ch =~ SPACE_RE
+          # treat whitespace (incl. IDEOGRAPHIC SPACE) as a hard boundary
+          tokens << BOUNDARY unless tokens.last == BOUNDARY
+          i += 1
+        else
+          # collect contiguous non-ASCII-non-space and segment via TinySegmenter
+          j = i + 1
+          j += 1 while j < text.length && text[j] !~ /[A-Za-z0-9\s\u3000]/
+          chunk = text[i...j]
+          tokens.concat(seg.segment(chunk))
+          i = j
+        end
+      end
+      tokens
     rescue LoadError
       raise Error, "tiny_segmenter gem is not installed. Add `tiny_segmenter` or omit segmenter option."
     end
 
     def self.japanese_token?(tok)
       # Kanji, Kana, prolonged sound mark, iteration marks, small kana
-      !!(tok =~ /[一-龯々〆ヵヶぁ-ゖゝゞァ-ヴー]/)
+      !!(tok =~ JAPANESE_RE)
     end
 
-    def self.segment_with_space(text)
-      text.split(/\s+/)
+    def self.normalize_slug_part(tok, net_opts)
+      if tok == BOUNDARY
+        { type: :boundary, text: nil }
+      elsif japanese_token?(tok)
+        { type: :j, text: to_roma(tok, **net_opts) }
+      elsif tok =~ ASCII_RE
+        { type: :ascii, text: tok }
+      end
     end
 
-    def self.normalize_slug_part(tok, net_opts)
-      if japanese_token?(tok)
-        to_roma(tok, **net_opts)
-      elsif tok =~ /[A-Za-z0-9]+/
-        tok
+    def self.merge_ascii_parts(parts)
+      merged = []
+      parts.each do |p|
+        if p[:type] == :boundary
+          merged << p
+        elsif !merged.empty? && merged.last[:type] == :ascii && p[:type] == :ascii
+          merged.last[:text] << p[:text]
+        else
+          merged << { type: p[:type], text: p[:text].dup }
+        end
       end
+      merged.reject { |p| p[:type] == :boundary }.map { |p| p[:text] }
+    end
+
+    def self.normalize_slug_string(str, sep:, downcase:, collapse:)
+      s = str
+      s = s.downcase if downcase
+      # Replace non-alphanumeric with separator
+      s = s.gsub(NON_ALNUM_RE, sep)
+      # Collapse duplicate separators
+      s = s.gsub(/#{Regexp.escape(sep)}{2,}/, sep) if collapse && !sep.empty?
+      # Trim leading/trailing separators
+      s = s.gsub(/^#{Regexp.escape(sep)}|#{Regexp.escape(sep)}$/, "") unless sep.empty?
+      s
     end
 
     def self.slice_opts(hash, *keys)
       hash.slice(*keys)
     end
+
+    private_class_method :segment_with_tiny, :japanese_token?, :normalize_slug_part, :merge_ascii_parts,
+                         :normalize_slug_string, :slice_opts, :backoff_for, :katakana_to_hiragana,
+                         :hiragana_to_katakana
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kanji-translator
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Hiromu Kodani