kanji-translator 0.1.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4f92f7987bc8ed9600c1b5849f56c0538f6af423afd53249ae70bc83e53a383
-  data.tar.gz: 226e287702852d0d05563b4a38aba069caef02e52c56c76d4011ef22e928c43c
+  metadata.gz: 56e5d89e3ee4e402ab0100d1c86b5bd156481671d4dbd6633f7b9577e4155304
+  data.tar.gz: bc7eff78a596f65fdc387ac9fa0f3245f4cc72781a785a8f4298fd18b696003a
 SHA512:
-  metadata.gz: 7e09f7183a129996b4fbfeff7e56b86f51e7b6b53ef7db428bb3c45b4e1bea4acb644251e996dbb68fe05c39785b202d76d7d3aca1d50b9c6299ce15ec1c74a7
-  data.tar.gz: a99f5a503d9b9a1a715d3d9405a86306c110ebc13b0c6bb5c17c494453460f04c05130176a4662138c8d2e24dfa119fd0080f61943c0364e5dd2cab963d24d54
+  metadata.gz: 4a36777ecf342e246a9009c788016d54a74d5a6fa439d6b0ad0e0d3fdd4d1bd86440142c06e67dea29ddcf3be927d08a872db132cca4c0237c698410bcce49d7
+  data.tar.gz: 7dba7989316c7c2c29cf5eaf2ed7799f3b92cdb2d133a126a1bc7541d9a157b5003f1ce2f3f1d34e13c1c0eb0279e6786007cde9776daf32ac366a316b76f71b

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 ## [Unreleased]
 
+## [1.1.0] - 2025-09-08
+
+- Fixed: `to_hira` の結果を必ずひらがなに正規化（返却HTMLにカタカナが混ざる場合の不整合を解消）。
+- Improved: スラッグ生成の分割アルゴリズムを整理し、ASCII連結・空白（全角含む）境界を厳密化。混在テキストでの精度向上。
+- Changed: `to_slug(text, separator: "-", **opts)` に整理（`separator` は直接キーワード、他は `**opts`）。互換性は維持。
+- Refactor: 正規表現・正規化処理の定数化/関数抽出、内部メソッドを `private_class_method` 化。
+
+## [1.0.0] - 2025-09-08
+
+- Breaking: `to_slug` のデフォルト挙動を `segmenter: :tiny` に変更（語境界ごとにハイフン区切り）。
+- Added: `segmenter: :space` オプションを追加。
+- Added: 依存に `tiny_segmenter (~> 0.0.6)` を追加。
+- Docs/Tests: READMEとRSpecを更新し新仕様を反映。
+
 ## [0.1.0] - 2025-09-08
 
 - Initial release

data/README.md

@@ -8,13 +8,11 @@
 
 Ruby 3.2以上が必要です。
 
-Rubygems公開後:
-
 ```bash
 bundle add kanji-translator
 ```
 
-未公開期間にGitHubから使う場合:
+GitHubから使う場合（任意）:
 
 ```ruby
 # Gemfile
@@ -54,6 +52,7 @@ require "kanji/translator/core_ext/string"
 - `Kanji::Translator.to_slug(text, separator: "-", downcase: true, collapse: true, **opts)`
   - `to_roma` の結果をスラッグ化します。
     - 非英数字を `separator` に置換、連続区切りを圧縮、前後の区切りをトリムします。
+   - 内部で TinySegmenter による分かち書きを行い、語境界ごとにハイフン区切りします（例: "学校案内" → "gakkou-annai"）。
 
 例（オプション）:
 

data/lib/kanji/translator/version.rb

@@ -2,6 +2,6 @@
 
 module Kanji
   module Translator
-    VERSION = "0.1.0"
+    VERSION = "1.1.0"
   end
 end

data/lib/kanji/translator.rb

@@ -13,12 +13,26 @@ module Kanji
 
     USER_AGENT = "kanji-translator/#{VERSION}".freeze
     HOST = "yomikatawa.com"
+    ASCII_RE = /[A-Za-z0-9]/
+    SPACE_RE = /[\s\u3000]/ # ASCII whitespace or IDEOGRAPHIC SPACE
+    NON_ALNUM_RE = /[^a-z0-9]+/
+    JAPANESE_RE = /[一-龯々〆ヵヶぁ-ゖゝゞァ-ヴー]/
+    BOUNDARY = :__BOUNDARY__
 
     def self.to_hira(text, timeout: 5, retries: 2, backoff: 0.5, user_agent: USER_AGENT)
       raise ArgumentError, "text must be a String" unless text.is_a?(String)
 
+      # Fast-path for kana inputs: avoid network and normalize locally
+      if text.match?(/\A[ぁ-ゖーゝゞ]+\z/)
+        return text
+      elsif text.match?(/\A[ァ-ヴーヽヾヵヶ]+\z/)
+        return katakana_to_hiragana(text)
+      end
+
       body = fetch_page(text, timeout: timeout, retries: retries, backoff: backoff, user_agent: user_agent)
-      parse_hiragana(body)
+      hira = parse_hiragana(body)
+      # Ensure result is normalized to hiragana only (remote may mix katakana like 固有名詞)
+      katakana_to_hiragana(hira)
     end
 
     def self.to_kata(text, **)
@@ -31,17 +45,18 @@ module Kanji
       hiragana_to_romaji(hira)
     end
 
-    def self.to_slug(text, separator: "-", downcase: true, collapse: true, **)
-      roma = to_roma(text, **)
-      s = downcase ? roma.downcase : roma.dup
-      sep = separator
-      # Replace non-alphanumeric with separator
-      s = s.gsub(/[^a-z0-9]+/, sep)
-      # Collapse duplicate separators
-      s = s.gsub(/#{Regexp.escape(sep)}{2,}/, sep) if collapse && !sep.empty?
-      # Trim leading/trailing separators
-      s = s.gsub(/^#{Regexp.escape(sep)}|#{Regexp.escape(sep)}$/, "") unless sep.empty?
-      s
+    def self.to_slug(text, separator: "-", **opts)
+      sep       = separator
+      downcase  = opts.fetch(:downcase, true)
+      collapse  = opts.fetch(:collapse, true)
+      net_opts  = slice_opts(opts, :timeout, :retries, :backoff, :user_agent)
+
+      tokens = segment_with_tiny(text)
+      raw_parts = tokens.filter_map { |tok| normalize_slug_part(tok, net_opts) }
+      parts = merge_ascii_parts(raw_parts)
+      s = parts.join(sep)
+
+      normalize_slug_string(s, sep: sep, downcase: downcase, collapse: collapse)
     end
 
     def self.fetch_page(text, timeout:, retries:, backoff:, user_agent: USER_AGENT)
@@ -106,6 +121,10 @@ module Kanji
       hira.tr("ぁ-ゔゝゞー", "ァ-ヴヽヾー")
     end
 
+    def self.katakana_to_hiragana(kata)
+      kata.tr("ァ-ヴヽヾヵヶー", "ぁ-ゔゝゞかけー")
+    end
+
     DIGRAPHS = {
       "きゃ" => "kya", "きゅ" => "kyu", "きぇ" => "kye", "きょ" => "kyo",
       "ぎゃ" => "gya", "ぎゅ" => "gyu", "ぎぇ" => "gye", "ぎょ" => "gyo",
@@ -193,5 +212,84 @@ module Kanji
       jitter = rand * 0.05
       sleep_s + jitter
     end
+
+    def self.segment_with_tiny(text)
+      require "tiny_segmenter"
+      seg = TinySegmenter.new
+      tokens = []
+      i = 0
+      while i < text.length
+        ch = text[i]
+        if ch =~ ASCII_RE
+          j = i + 1
+          j += 1 while j < text.length && text[j] =~ ASCII_RE
+          tokens << text[i...j]
+          i = j
+        elsif ch =~ SPACE_RE
+          # treat whitespace (incl. IDEOGRAPHIC SPACE) as a hard boundary
+          tokens << BOUNDARY unless tokens.last == BOUNDARY
+          i += 1
+        else
+          # collect contiguous non-ASCII-non-space and segment via TinySegmenter
+          j = i + 1
+          j += 1 while j < text.length && text[j] !~ /[A-Za-z0-9\s\u3000]/
+          chunk = text[i...j]
+          tokens.concat(seg.segment(chunk))
+          i = j
+        end
+      end
+      tokens
+    rescue LoadError
+      raise Error, "tiny_segmenter gem is not installed. Add `tiny_segmenter` or omit segmenter option."
+    end
+
+    def self.japanese_token?(tok)
+      # Kanji, Kana, prolonged sound mark, iteration marks, small kana
+      !!(tok =~ JAPANESE_RE)
+    end
+
+    def self.normalize_slug_part(tok, net_opts)
+      if tok == BOUNDARY
+        { type: :boundary, text: nil }
+      elsif japanese_token?(tok)
+        { type: :j, text: to_roma(tok, **net_opts) }
+      elsif tok =~ ASCII_RE
+        { type: :ascii, text: tok }
+      end
+    end
+
+    def self.merge_ascii_parts(parts)
+      merged = []
+      parts.each do |p|
+        if p[:type] == :boundary
+          merged << p
+        elsif !merged.empty? && merged.last[:type] == :ascii && p[:type] == :ascii
+          merged.last[:text] << p[:text]
+        else
+          merged << { type: p[:type], text: p[:text].dup }
+        end
+      end
+      merged.reject { |p| p[:type] == :boundary }.map { |p| p[:text] }
+    end
+
+    def self.normalize_slug_string(str, sep:, downcase:, collapse:)
+      s = str
+      s = s.downcase if downcase
+      # Replace non-alphanumeric with separator
+      s = s.gsub(NON_ALNUM_RE, sep)
+      # Collapse duplicate separators
+      s = s.gsub(/#{Regexp.escape(sep)}{2,}/, sep) if collapse && !sep.empty?
+      # Trim leading/trailing separators
+      s = s.gsub(/^#{Regexp.escape(sep)}|#{Regexp.escape(sep)}$/, "") unless sep.empty?
+      s
+    end
+
+    def self.slice_opts(hash, *keys)
+      hash.slice(*keys)
+    end
+
+    private_class_method :segment_with_tiny, :japanese_token?, :normalize_slug_part, :merge_ascii_parts,
+                         :normalize_slug_string, :slice_opts, :backoff_for, :katakana_to_hiragana,
+                         :hiragana_to_katakana
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kanji-translator
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Hiromu Kodani
@@ -23,6 +23,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: tiny_segmenter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.6
 description: |-
   Fetches readings for Japanese Kanji from yomikatawa.com and converts them
   to hiragana, katakana, or Hepburn-style romaji. Includes timeout/retry