zabon 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d0f3961a7b9e53f882a480036cfb49a80eb7340f62d4a9c8d1b1e3e3bad0c82c
+  data.tar.gz: 779a1fac27148c57b58f79c03920d920b44c07751f9e64c527735b2748fa5bde
+SHA512:
+  metadata.gz: ec94a2d9f1fee896f2fb3c5deb27913e33971063c7397224c9a5a6d4394ce8a0d6a8d43bc96d1d954969d5d0f5105d3385eabc86b7f6e20e23a1ba33c0e26d8c
+  data.tar.gz: 40f2a3fc5c52b24b7346e312c26092435e87fd8f3405ce88440c540742e18270c5c75865e92bcbe339865492369dbf18001053f7cea2fe95642ff0b056cfec30

data/.bundler-audit.yml

@@ -0,0 +1,6 @@
+ignore:
+  # nokogiri GHSA-wx95-c6cv-8532: does not check return value from xmlC14NExecute.
+  # Patched in nokogiri >= 1.19.1, which requires Ruby >= 3.2.
+  # zabon supports Ruby 3.1+ and does not use nokogiri's C14N functionality directly
+  # (nokogiri is a transitive dependency via actionview). Revisit when Ruby 3.1 is EOL.
+  - GHSA-wx95-c6cv-8532

data/.rubocop.yml

@@ -0,0 +1,41 @@
+require:
+  - rubocop-minitest
+  - rubocop-performance
+  - rubocop-rake
+
+AllCops:
+  TargetRubyVersion: 3.1
+
+Layout/LineLength:
+  Max: 180
+  Exclude:
+    - test/zabon_test.rb
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/RedundantRegexpEscape:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false

data/.ruby-gemset

@@ -0,0 +1 @@
+zabon

data/.ruby-version

@@ -0,0 +1 @@
+3.3

data/AGENTS.md

@@ -0,0 +1,51 @@
+# Agent Guidelines
+
+## Repository overview
+
+zabon is a Ruby gem for Japanese text segmentation (Kinsoku Shori / 禁則処理).
+It ports the mikan.js algorithm to Ruby and integrates with Rails via `ActionView::Helper`.
+
+## Running checks
+
+```bash
+bundle exec rake          # full suite: bundler-audit, rubocop, tests
+bundle exec rake test     # tests only
+bundle exec rubocop       # linter only
+```
+
+All three must be green before committing.
+
+## Code conventions
+
+- Ruby 3.1+ required; use Ruby 3 idioms: endless methods, `Hash#except`, `Struct keyword_init:`.
+- `frozen_string_literal: true` on every file.
+- Double-quoted strings (`EnforcedStyle: double_quotes` in `.rubocop.yml`).
+- Predicate methods return a plain boolean — use `Regexp#match?`, never `match`.
+
+## constants.rb encoding
+
+`lib/zabon/constants.rb` stores some voiced hiragana particles in NFD form (e.g. `で` = `\u3066\u3099`).
+This is **load-bearing**: those particles intentionally do not match NFC strings.
+Never normalise this file to NFC. When editing it, use byte-level operations and verify
+`test_sentence6` and `test_sentence8` still pass.
+
+## Commits
+
+Follow conventional commits (`fix:`, `feat:`, `ref:`, `test:`, `chore:`, `ci:`, `license:`, `meta:`).
+One logical change per commit. Include `Co-Authored-By:` when AI-generated.
+
+## CI matrix
+
+Ruby 3.1, 3.2, 3.3, 3.4 on ubuntu-latest. Always verify fixes on **all** matrix versions locally
+before pushing — incompatibilities between Ruby versions are not always obvious.
+
+## Tests
+
+Tests in `test/zabon_test.rb` must satisfy `expected.join == source` — the segmentation must be
+lossless. When adding new test sentences, run `Zabon.split(source)` first to get the real output,
+then copy it into `expected`; do not guess segment boundaries.
+
+## Attribution
+
+zabon is a port of [mikan.js](https://github.com/trkbt10/mikan.js) by trkbt10 (MIT).
+The upstream copyright notice is preserved in `LICENSE.txt` — do not remove it.

data/CHANGELOG.md

@@ -0,0 +1,34 @@
+# Changelog
+
+## 0.2.0 (2026-03-26)
+
+### Breaking changes
+
+- Ruby >= 3.1 is now required (was >= 2.5)
+
+### Features
+
+- Add `ので` as a standalone JOSHI particle, correcting an accidental concatenation in the particle list
+- Add optional Sentry context (`Sentry.set_context`) before segmentation — zero hard dependency, guarded by `defined?(Sentry)`
+- Add `reset_config!` to reset configuration to defaults
+- Add `examples/` — a minimal single-file Rails app demonstrating `zabon_translate` with locale switching
+
+### Fixes
+
+- `Segment#hiragana?` was returning `MatchData` instead of a boolean
+- `Analyzer#segments` had implicit operator precedence on a compound condition; added explicit parentheses
+- `Helper#zabon_translate` was detecting missing translations via a brittle string sniff (`include?("translation_missing")`); replaced with `I18n.exists?`
+- `Helper#zabon_translate` called `strip_tags` via a module method that shadowed `ActionView::Helpers::SanitizeHelper#strip_tags` in the ancestor chain; inlined the `ActionView::Base.full_sanitizer` call directly
+- `require "uri"` added before `require "action_view"` to fix a `NameError` on Ruby 3.1 where `URI` is not pre-loaded
+- `Zabon::Helper` now works correctly when included in a plain controller or any non-`ActionView::Base` context
+
+### Changes
+
+- Ruby 3 idioms adopted throughout: endless methods (`Segment`), `Hash#except` (`Helper`), `Struct keyword_init:` (`Configuration`)
+- CI matrix updated to Ruby 3.1, 3.2, 3.3, 3.4
+- Upstream copyright notice for mikan.js (trkbt10) added to `LICENSE.txt`
+- `examples/` excluded from the gem package
+
+## 0.1.0
+
+Initial release.

data/Gemfile

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+group :code_quality do
+  gem "bundler-audit"
+  gem "rubocop"
+  gem "rubocop-minitest"
+  gem "rubocop-performance"
+  gem "rubocop-rake"
+  gem "simplecov"
+end
+
+group :development do
+  gem "rake"
+end
+
+group :test do
+  gem "minitest"
+end

data/LICENSE.txt

@@ -0,0 +1,25 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Joesi
+
+Portions of this software are derived from mikan.js
+(https://github.com/trkbt10/mikan.js), Copyright (c) trkbt10,
+used under the MIT License.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,178 @@
+# zabon.ruby 🍊
+
+A Ruby gem / Rails helper for dealing with Japanese line-breaking logic. It is basically a port of [mikan.js](https://github.com/trkbt10/mikan.js), which implements a regular expression based algorithm to segment text into semantic chunks. No machine learning needed 🤖☺️. In addition the resulting text segments can be wrapped in a configurable HTML tag. All praise 👏👏👏 for the algorithm goes to [trkbt10](https://github.com/trkbt10).
+
+## Usage
+``` ruby
+# split this sentence
+Zabon.split('この文を分割する')
+ => ["この", "文を", "分割する"]
+
+```
+
+## Configuration
+
+Configuration is used for tag that the results can be wrapped in. It's making heavy use of Rails tag helpers.
+E.g. put this in an initializer in your Rails app.
+
+``` ruby
+Zabon.configure do |config|
+  config.tag = :div # default: :span
+  config.tag_options = { class: 'zabon_trara', style: 'font_size: 5em' } # default:  { class: 'zabon', style: 'display: inline-block' }
+  config.strip_tags = false # default true
+end
+```
+
+### Rails
+
+The gem ships a Railtie that automatically includes `Zabon::Helper` into `ActionView::Base`, so `zabon_translate` is available in all views without any further setup.
+
+Call `zabon_translate` directly in views for strings that need segmentation:
+
+```erb
+<%= zabon_translate("page.title") %>
+```
+
+To replace the standard `t()` globally, add the following to an initializer. Note that this affects **all** ActionView translation calls — prefer explicit `zabon_translate` for finer control.
+
+```ruby
+# config/initializers/zabon.rb
+module ActionView
+  module Helpers
+    module TranslationHelper
+      alias_method :translate_without_zabon, :translate
+
+      def translate(key, **options)
+        zabon_translate(key, orig_translate: :translate_without_zabon, **options)
+      end
+
+      alias t translate
+    end
+  end
+end
+```
+
+## Japanese grammar 🇯🇵
+
+Just enough Japanese to understand the algorithm :)
+
+### Writing system ✍️
+
+The Japanese writing system uses for different components:
+
+* [Hiragana (ひらがな)](https://en.wikipedia.org/wiki/Hiragana), a syllabary alphabet used for Japanese words not covered by kanji and mostly for grammatical inflections
+* [Katakana (カタカナ)](https://en.wikipedia.org/wiki/Katakana), a syllabary alphabet used for transcription of foreign-language words into Japanese; for emphasis; [onomatopoeia](https://en.wikipedia.org/wiki/Onomatopoeia); for scientific terms and often Japanese companies.
+* [Kanji (漢字)](https://en.wikipedia.org/wiki/Kanji), a set of Chinese characters directly incorporated into the written Japanese language with often Japanese pronunciation, which can be
+* [Romaji](https://en.wikipedia.org/wiki/Romanization_of_Japanese), use of Latin script in Japanese language
+
+### Particles
+
+[Joshi (助詞)](https://en.wikipedia.org/wiki/Japanese_particles), Japanese particles written in Hiragana, are suffixes or short words that follow a modified noun, verb, adjective, or sentence. Their grammatical range can indicate various meanings and functions:
+
+* case markers
+* parallel markers
+* sentence ending particles
+* interjectory particles
+* adverbial particles
+* binding particles
+* conjunctive particles
+* phrasal particles
+
+### Line breaking
+
+Certain characters in Japanese should not come at the end of a line, certain characters should not come at the start of a line, and some characters should never be split up across two lines. These rules are called [Kinsoku Shori 禁則処理](https://en.wikipedia.org/wiki/Line_breaking_rules_in_East_Asian_languages#Line_breaking_rules_in_Japanese_text_(Kinsoku_Shori)):
+
+simplified:
+
+| Class | Can't begin a line | Can't finish a line |
+|-------|--------------------|---------------------|
+| small _kana_ | ぁぃぅぇぉっ... |                  |
+| parentheses  | ）〉》】...     | （〈《【...        |
+| quotations   | 」』”...       | 「『“...           |
+| punctuation  | 、。・！？...   |               |
+
+### Text segmentation
+
+Written Japanese uses no spaces and little punctuation to delimit words. Readers instead depend on grammatical cues (e.g. Japanese, particles and verb endings), the relative frequency of character combinations, and semantic context, in order to determine what words have been written. This is a non trivial problem which is often solved by applying machine learning algorithms. Without a careful approach, breaks can occur randomly and usually in the middle of a word. This is an issue with typography on the web and results in a degradation of readability.
+
+### Zabon ???
+
+I made a couple of assumptions when choosing the name:
+1. 🍊 The original algorithm name **Mikan** might be transscription of 蜜柑, a Japanese citrus fruit (Mandarin, Satsuma)
+2. There already is a gem called [mikan](https://rubygems.org/gems/mikan), didn't want to go for **mikan_ruby** or similar b/c of autoloading
+3. 🍇 My guess is the original author chose this name, b/c he was searching for something simpler then Google's **Budou** (葡萄)
+4. 🔪 Both fruits have in common, that they can be easily split apart in segments
+5. So I was searching for another fruit that can be easily split apart, what can be split better apart than a Pomelo (文旦, ぶんたん) -  **Zabon** (derived from Portoguese: zamboa)
+
+Who knows if that's how it was 🤷🏻‍♂️😂.
+
+## The Algorithm
+
+This algorithm does NOT find the most minimal segmentation of unbreakable text segments and probably will have problems if a text is solely written in one alphabet. It also does not support Furigana (yet). It does basic text segmentation and stitches the segments back together in segments which can be made unbreakable. The unbreakability we achieve by wrapping them in a <span> tag with certain CSS rules.
+
+### Splitting
+
+1. Split text across different alphabets used: split text into parts that are written in Kanjis, Hiragana, Katakana, Latin (incl. double width characters). The assumption here is that parts written in the same script should belong together.
+
+2. Then split up each element further by splitting up particles are sequences that might be used as particles. The original author of the algorithm has identified the following list (でなければ, について, かしら, くらい, けれど, なのか, ばかり, ながら, ことよ, こそ, こと, さえ, しか, した, たり, だけ, だに, だの, つつ, ても, てよ, でも, とも, から, など, なりので, のに, ほど, まで, もの, やら, より, って, で, と, な, に, ね, の, も, は, ば, へ, や, わ, を, か, が, さ, し, ぞ, て). To me that looks about right, but maybe there are missing some.
+
+3. Split along further by splitting up brackets and quotations: ([,〈,《,「,『,｢,【,〔,〚,〖,〘,❮,❬,❪,❨,(,<,{,❲,❰,｛,❴,] + the matching end brackets and quotations.
+
+### Stitching                                                                                                               
+                                                                                                               
+1. Now we have a list of minimal segments and try to stitch them back together in a result set, so that they will fulfil Japanese line breaking rules. We are gonna look at tuples from left to right, looking at the current segment and the previous segment.
+
+2. If the current segment is a beginning bracket or quotation; we look at the next segment, we have a definitiv start of an unbreakable segment.
+
+3. If the current segment is an ending bracket or quotation; we append to the last entry of the result set and don't look back anymore; we've reached the end of a segment and start a new one with the next iteration.
+
+4. If the previous segment is a beginning bracket; we stitch it together with the current segment to become a new segment. In the next iteration we don’t need to look at the previous segment anymore and continue.
+
+5. If he current segment is a particle or a punctuation mark and we are not looking back (see step 7.); we append the current segment to the last entry of the result set.
+
+6. If he current segment is a particle or a punctuation mark or if the previous segment is not a bracket, quotation or punctuation mark or a conjunctive particle (と, の,に) and the current segment is in Hiragana; we append to the last entry of the result set.
+
+7. If no condition from stiching steps 1-2 are matching we can safely add the current segment to the result set.
+
+## Other solutions
+### [Google Budou](https://github.com/google/budou)
+
+Budou is a python library, which uses word segmenters to analyze input sentences. It can  concatenate proper into meaningful chunks utilizing part-of-speech tagging and other syntactic information. Processed chunks are wrapped in a SPAN tag. Depending on the text segmentation algorithm used, it also has support for Chinese & Korean. Since this library is written in Python, it cannot be used simply used in Ruby, PHP, or Node.js.
+
+#### Text segmenter backends
+You can choose different segmenter backends depending on the needs of your environment. Currently, the segmenters below are supported.
+
+* [Google Cloud Natural Language API](https://cloud.google.com/natural-language/): external API calls, can be costly
+* [MeCab](https://taku910.github.io/mecab/): Japanese POS tagger & morphological analyzer with lots of language bindings, e.g. also used in Google Japanese Input and Japanese Input on Mac OS X
+* [TinySegmenter](http://chasen.org/~taku/software/TinySegmenter/): extremely compact word separation algorithm in Javascript which produces MeCab compatible word separation without depending on external APIs, no dictionaires, classifies input
+
+[TinySegmenter](http://chasen.org/~taku/software/TinySegmenter/) is an extremely compact word separation algorithm in Javascript which produces MeCab compatible word separation without depending on external APIs. It classifies the input by using entities like characters, N-Grams, Hiragana, Katakana (Japanese phonetic lettering system / syllabaries) and their combinations as features to determine whether a character is preceded by a word boundary. A [Naive Bayes]((https://towardsdatascience.com/naive-bayes-explained-9d2b96f4a9c0) model was trained using the [RWCP corpus](http://research.nii.ac.jp/src/en/list.html) and to make that model even more compact Boosting was used for [L1 norm regularization](https://blog.mlreview.com/l1-norm-regularization-and-sparsity-explained-for-dummies-5b0e4be3938a). Basically it compresess the model and get rid off redundant features as much as possible.
+
+### CSS `line-break: strict`
+
+Worth knowing: CSS has had native support for some Kinsoku Shori rules for a while now.
+
+```css
+p {
+  line-break: strict;
+  overflow-wrap: break-word;
+}
+```
+
+`line-break: strict` applies character-level Unicode line-breaking rules, which covers small kana, prolonged sound marks, and common punctuation cases. Browser implementations are not perfectly consistent and the spec intentionally leaves the precise rule set up to the user agent, so you may see subtle differences across browsers.
+
+zabon takes a fundamentally different approach. Instead of telling the browser where not to break, it wraps each segment in a `display: inline-block` element that the browser cannot split internally. This gives you precise semantic grouping that works the same way in every browser, and also opens up per-segment styling like hover effects, search highlighting, or animations. The trade-off is server-side processing and extra markup.
+
+If basic punctuation and small-kana rules are all you need, `line-break: strict` might be enough and has zero runtime cost. If you need guaranteed atomic grouping or per-segment control, zabon is the better fit.
+
+## Resources
+
+* [Regular Expressions for Japanese characters](https://gist.github.com/terrancesnyder/1345094)
+* [Word breaking in Japanese is Hard](https://docs.microsoft.com/en-us/archive/blogs/jonasbar/word-breaking-japanese-is-hard)
+* [mikan.sharp](https://github.com/YoungjaeKim/mikan.sharp)
+* [mikan.php](https://github.com/sters/mikan.php)
+* [Kinsoku - Japanese line breaking rules for LaTeX](https://github.com/jamesohortle/kinsoku)
+* [Kuromoji - Japanese morphological analyzer written in Java](https://www.atilika.org/)
+* [WrapText CJK - line breaking rules in Lua](https://github.com/subsoap/wraptext)
+* [TinySegmenter - Ruby Port](https://github.com/6/tiny_segmenter)
+* [How to Pomelo](https://github.com/dingsdax/zabon/wiki/How-to-Pomelo!)

data/Rakefile

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "bundler/audit/task"
+require "rake/testtask"
+require "rubocop/rake_task"
+
+Bundler::Audit::Task.new
+RuboCop::RakeTask.new
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.warning = false
+  t.verbose = true
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+desc "Run code quality checks"
+task code_quality: %i[bundle:audit rubocop]
+
+task default: %i[code_quality test]

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "zabon"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/zabon/analyzer.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Zabon
+  class Analyzer
+    class << self
+      def split(text)
+        text.split(KEYWORDS)
+            .flat_map { |segment| segment.split(JOSHI) }
+            .flat_map { |segment| segment.split(BRACKETS_BEGIN) }
+            .flat_map { |segment| segment.split(BRACKETS_END) }
+            .flatten.reject(&:empty?)
+      end
+
+      def segments(text)
+        result = [""] # we do this, so we can += to append to the last result item, without checking for nil
+        previous_segment = nil
+
+        split(text).each do |segment|
+          current_segment = Segment.new(segment)
+
+          # if the current segment is a beginning bracket => we look further
+          if current_segment.bracket_begin?
+            previous_segment = current_segment
+            next
+          end
+
+          # if the current segment is an ending bracket =>
+          # we append to the last entry of the result set and don't look back anymore,
+          # we've reached the end of a segment and start a new one with the next iteration
+          if current_segment.bracket_end?
+            result[-1] += current_segment
+            previous_segment = nil
+            next
+          end
+
+          # if the previous segment is a beginning bracket =>
+          # we stitch together previous segment & current segment to become a new segment
+          # we don't look back anymore
+          if previous_segment&.bracket_begin?
+            current_segment = Segment.new(previous_segment + current_segment)
+            previous_segment = nil
+          end
+
+          # if we are not at the start, the current segment is a particle or a period and
+          # we are not looking back, we append to the last entry of the result set
+          if result.size > 1 && current_segment.joshi_or_period? && previous_segment.nil?
+            result[-1] += current_segment
+            previous_segment = current_segment
+            next
+          end
+
+          # if we are not at the start, the current segment is a particle or a period or
+          # the previous segment is not a bracket or period or a conjunctive particle and the current segment is hiragana
+          # we append to the last entry of the result set
+          if (result.size > 2 && current_segment.joshi_or_period?) || (previous_segment&.keyword? && current_segment.hiragana? && !/^[とのに]$/.match?(previous_segment))
+            result[-1] += current_segment
+            # if the current segment is not a particle, we are no looking back anymore, we start a new segment
+            previous_segment = current_segment.joshi? ? current_segment : nil
+            next
+          end
+
+          # no stitching left, append the current segment to the result set
+          result << current_segment
+          previous_segment = current_segment
+        end
+
+        result.reject(&:empty?) # we clear out any possible blank strings in the result set
+      end
+    end
+  end
+end

data/lib/zabon/configuration.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Zabon
+  Configuration = Struct.new(:tag, :tag_options, :strip_tags, keyword_init: true) do
+    def initialize(tag: :span, tag_options: { class: "zabon", style: "display: inline-block" }, strip_tags: true)
+      super
+    end
+  end
+end

data/lib/zabon/constants.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Zabon
+  # Joshi (助詞), Japanese particles written in Hiragana, are suffixes or short words that follow a modified noun, verb, adjective, or sentence.
+  # Some particals can appear in two types. They give pretty reliable cues depending on the following character, whether a line break is allowed or not.
+  JOSHI = /
+    (でなければ|について|かしら|くらい|けれど|なのか|ばかり|ながら|ことよ|こそ|こと|さえ|しか|した|たり|だけ|だに|だの|つつ|ても|てよ|でも|
+    とも|から|など|なりので|ので|のに|ほど|まで|もの|やら|より|って|で|と|な|に|ね|の|も|は|ば|へ|や|わ|を|か|が|さ|し|ぞ|て)
+  /x
+
+  # A simple way to find word segementations in Japanese is
+  # to tokenise by grouping characters continuously by script (Hiragana, Katakana, Kanji, Romaji)
+  #
+  # The following regular expression matches in this order:
+  # * non breaking space
+  # * domains
+  # * any Japanese Kanji or Chinese character
+  # * Hirgana (+ chisai kana)
+  # * Katakana (+ chisai kana)
+  # * Latin
+  # * Latin (double width)
+  KEYWORDS = /
+    (\ |
+    [a-zA-Z0-9]+\.[a-z]{2,}|
+    [一-龠々〆ヵヶゝ]+|
+    [ぁ-んゝ]+|
+    [ァ-ヴー]+|
+    [a-zA-Z0-9]+|
+    [ａ-ｚＡ-Ｚ０-９]+)
+  /x
+
+  # Brackets & Quotations
+  BRACKETS_BEGIN = /([〈《「『｢（(\[【〔〚〖〘❮❬❪❨(<{❲❰｛❴])/
+  BRACKETS_END   = /([〉》」』｣)）\]】〕〗〙〛}>\)❩❫❭❯❱❳❵｝])/
+
+  PERIODS = /([\.\,。、！\!？\?]+)$/
+
+  HIRAGANA = /[ぁ-んゝ]+/
+end

data/lib/zabon/helper.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "uri"
+require "action_view"
+
+module Zabon
+  module Helper
+    include ActionView::Helpers::TagHelper
+
+    # can be used as a replacement for ActionView::Helpers::TranslationHelper.translate
+    # will use original translate method if locale is not :ja or translation is missing
+    # if not will split translation into semantic chunks wrap them into a configurable HTML tag
+    # and join again
+    def zabon_translate(key, **options)
+      orig_translate = options[:orig_translate] || :translate
+      translate_options = options.except(:orig_translate)
+
+      locale = (options[:locale] || I18n.locale || :en).to_sym
+
+      return public_send(orig_translate, key, **translate_options) if locale != :ja # if locale is not Japanese we use original method
+
+      return key.map { |k| zabon_translate(k, **options) } if key.is_a?(Array)
+
+      return public_send(orig_translate, key, **translate_options) unless I18n.exists?(key, locale: :ja)
+
+      orig_translation = public_send(orig_translate, key, **translate_options)
+
+      orig_translation = ActionView::Base.full_sanitizer.sanitize(orig_translation, tags: []) if Zabon.config.strip_tags
+
+      Sentry.set_context("zabon", { key: key, locale: locale }) if defined?(Sentry)
+
+      translation = Zabon.split(orig_translation).map do |segment|
+        content_tag(Zabon.config.tag, segment, Zabon.config.tag_options)
+      end.join.html_safe
+
+      block_given? ? yield(translation, key) : translation
+    end
+  end
+end

data/lib/zabon/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+require "action_view"
+
+module Zabon
+  class Railtie < Rails::Railtie
+    initializer "zabon.helper" do
+      ActionView::Base.include Zabon::Helper
+    end
+  end
+end

data/lib/zabon/segment.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Zabon
+  class Segment < String
+    def hiragana? = @hiragana ||= HIRAGANA.match?(self)
+    def keyword? = @keyword ||= KEYWORDS.match?(self)
+    def bracket_begin? = @bracket_begin ||= BRACKETS_BEGIN.match?(self)
+    def bracket_end? = @bracket_end ||= BRACKETS_END.match?(self)
+    def joshi? = @joshi ||= JOSHI.match?(self)
+    def period? = @period ||= PERIODS.match?(self)
+    def joshi_or_period? = @joshi_or_period ||= joshi? || period?
+  end
+end

data/lib/zabon/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Zabon
+  VERSION = "0.2.0"
+end

data/lib/zabon.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "zabon/analyzer"
+require "zabon/configuration"
+require "zabon/constants"
+require "zabon/helper"
+require "zabon/segment"
+require "zabon/version"
+require "zabon/railtie" if defined?(Rails::Railtie)
+
+module Zabon
+  class << self
+    def split(text)
+      Analyzer.segments(text)
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
+    def configure
+      yield config
+    end
+
+    def reset_config!
+      @config = Configuration.new
+    end
+  end
+end

data/zabon.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "lib/zabon/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "zabon"
+  spec.version       = Zabon::VERSION
+  spec.authors       = ["Johannes D."]
+  spec.email         = ["dingsdax@fastmail.fm"]
+
+  spec.summary       = "Japanese line breaking algorithm: Ruby port of mikan.js"
+  spec.description   = "Splits up a (Japanese) string into semantic segment; wrap result in a HTML tag"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 3.1"
+
+  spec.metadata["source_code_uri"] = "https://github.com/dingsdax/zabon.git"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:examples|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "actionview"
+  spec.add_dependency "railties"
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: zabon
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Johannes D.
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: actionview
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Splits up a (Japanese) string into semantic segment; wrap result in a
+  HTML tag
+email:
+- dingsdax@fastmail.fm
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".bundler-audit.yml"
+- ".rubocop.yml"
+- ".ruby-gemset"
+- ".ruby-version"
+- AGENTS.md
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/zabon.rb
+- lib/zabon/analyzer.rb
+- lib/zabon/configuration.rb
+- lib/zabon/constants.rb
+- lib/zabon/helper.rb
+- lib/zabon/railtie.rb
+- lib/zabon/segment.rb
+- lib/zabon/version.rb
+- zabon.gemspec
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/dingsdax/zabon.git
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.7.1
+specification_version: 4
+summary: 'Japanese line breaking algorithm: Ruby port of mikan.js'
+test_files: []