yanagi 0.1.0

data/lib/yanagi/cyrillic.rb

@@ -0,0 +1,310 @@
+# frozen_string_literal: true
+
+require_relative "normalize"
+require_relative "rules"
+require_relative "mora"
+
+module Yanagi
+  Result = Struct.new(:text, :source, :confidence, :notes, keyword_init: true) do
+    def to_s
+      text.to_s
+    end
+
+    def to_str
+      text.to_s
+    end
+
+    def ==(other)
+      if other.is_a?(String)
+        text == other
+      else
+        super
+      end
+    end
+
+    def inspect
+      "#<Yanagi::Result text=#{text.inspect} source=#{source.inspect} confidence=#{confidence.inspect}>"
+    end
+  end
+
+  module Cyrillic
+    O_COLUMN = %w[
+      お こ そ と の ほ も よ ろ を
+      ご ぞ ど ぼ ぽ
+      きょ しょ ちょ にょ ひょ みょ りょ ぎょ じょ びょ ぴょ
+      ふぉ ゔぉ
+    ].freeze
+
+    U_COLUMN = %w[
+      う く す つ ぬ ふ む ゆ る
+      ぐ ず づ ぶ ぷ
+      しゅ ちゅ じゅ きゅ ぎゅ にゅ ひゅ みゅ りゅ びゅ ぴゅ
+      ゔ どぅ とぅ
+    ].freeze
+
+    A_COLUMN = %w[
+      あ か さ た な は ま や ら わ
+      が ざ だ ば ぱ
+      きゃ しゃ ちゃ にゃ ひゃ みゃ りゃ ぎゃ じゃ びゃ ぴゃ
+      ふぁ ゔぁ
+    ].freeze
+
+    E_COLUMN = %w[
+      え け せ て ね へ め れ ゑ
+      げ ぜ で べ ぺ
+      しぇ ちぇ つぇ ふぇ うぇ ゔぇ
+    ].freeze
+
+    I_COLUMN = %w[
+      い き し ち に ひ み り ゐ
+      ぎ じ ぢ び ぴ
+      ふぃ ゔぃ てぃ でぃ
+    ].freeze
+
+    def self.call(input = nil, kanji: nil, reading: nil)
+      raw_str = (reading || input).to_s.strip
+      kanji_str = kanji.to_s.strip
+
+      # 1. Whole-word exonym check
+      exonyms = Rules.exonyms
+      norm_key = Normalize.to_hiragana(Normalize.nfkc(raw_str))
+      if exonyms.key?(norm_key.to_sym)
+        entry = exonyms[norm_key.to_sym]
+        return Result.new(
+          text: entry[:cyrillic],
+          source: :exonym,
+          confidence: 1.0,
+          notes: entry[:note] || "Exonym"
+        )
+      end
+
+      # 2. Check exceptions table
+      exceptions = Rules.exceptions
+      if exceptions.is_a?(Array)
+        match = exceptions.find do |e|
+          (!kanji_str.empty? && e[:kanji] == kanji_str) ||
+            e[:term] == raw_str ||
+            e[:expected] == raw_str
+        end
+        if match
+          return Result.new(
+            text: match[:canonical] || match[:expected],
+            source: :exception_table,
+            confidence: 1.0,
+            notes: match[:reason]
+          )
+        end
+      end
+
+      # 3. Check lexicon (if present)
+      lexicon = Rules.lexicon
+      if lexicon.is_a?(Hash) && !kanji_str.empty? && lexicon.key?(kanji_str.to_sym)
+        entry = lexicon[kanji_str.to_sym]
+        return Result.new(
+          text: entry[:cyrillic] || entry[:reading_cyrillic],
+          source: :lexicon,
+          confidence: 1.0,
+          notes: entry[:gloss]
+        )
+      end
+
+      # 4. If input is romaji (only ASCII letters and apostrophes/hyphens/spaces)
+      if raw_str.match?(/\A[a-zA-Z'\- ]+\z/)
+        converted_kana = romaji_to_hiragana(raw_str)
+        return render_kana(converted_kana)
+      end
+
+      # 5. Derived Cyrillic transliteration from kana
+      render_kana(raw_str)
+    end
+
+    def self.render_kana(kana_input)
+      moras = Tokenizer.tokenize(kana_input)
+      mora_map = Rules.mora_map
+
+      # Step 1: Render each mora into candidate text (handling long vowel collapse and i diphthongs)
+      rendered_segments = []
+
+      moras.each_with_index do |mora, idx|
+        prev_mora = idx > 0 ? moras[idx - 1] : nil
+
+        case mora.kind
+        when :passthrough
+          rendered_segments << { mora: mora, text: mora.kana.to_s, kind: :passthrough }
+
+        when :chouonpu
+          # Long vowel mark collapses under policy
+          rendered_segments << { mora: mora, text: "", kind: :chouonpu }
+
+        when :sokuon
+          # Sokuon is resolved in Step 2 with lookahead
+          rendered_segments << { mora: mora, text: nil, kind: :sokuon }
+
+        when :moraic_n
+          # Moraic n is resolved in Step 2 with lookahead
+          rendered_segments << { mora: mora, text: nil, kind: :moraic_n }
+
+        when :syllable
+          kana = mora.kana
+          base_cyr = mora_map.dig(kana.to_sym, :cyrillic)&.to_s || kana
+
+          # Long vowel collapse rules:
+          # - う after O-column or U-column collapses
+          if kana == "う" && prev_mora && prev_mora.syllable?
+            if O_COLUMN.include?(prev_mora.kana) || U_COLUMN.include?(prev_mora.kana)
+              rendered_segments << { mora: mora, text: "", kind: :collapsed_long_vowel }
+              next
+            end
+          end
+
+          # - お after O-column collapses (e.g. おおたけ -> Отаке)
+          if kana == "お" && prev_mora && prev_mora.syllable? && O_COLUMN.include?(prev_mora.kana)
+            rendered_segments << { mora: mora, text: "", kind: :collapsed_long_vowel }
+            next
+          end
+
+          # - あ after A-column collapses
+          if kana == "あ" && prev_mora && prev_mora.syllable? && A_COLUMN.include?(prev_mora.kana)
+            rendered_segments << { mora: mora, text: "", kind: :collapsed_long_vowel }
+            next
+          end
+
+          # - え after E-column collapses
+          if kana == "え" && prev_mora && prev_mora.syllable? && E_COLUMN.include?(prev_mora.kana)
+            rendered_segments << { mora: mora, text: "", kind: :collapsed_long_vowel }
+            next
+          end
+
+          # - い diphthong vs vowel:
+          #   い after E-column -> 'й' (e.g. めい -> мей, せんせい -> сенсей)
+          #   い after A-column -> 'й' (e.g. しゅうさい -> шюсай, たい -> тай)
+          #   い after U-column -> 'і' (e.g. かるいざわ -> каруідзава, ぬいもん -> нуімон)
+          #   い after O-column -> 'і' (e.g. ごい -> ґоі, どい -> доі)
+          #   い after I-column -> 'і' (e.g. きいん -> кіін, いいだ -> ііда, りいち -> ріічі)
+          if kana == "い" && prev_mora && prev_mora.syllable?
+            if E_COLUMN.include?(prev_mora.kana) || A_COLUMN.include?(prev_mora.kana)
+              rendered_segments << { mora: mora, text: "й", kind: :syllable }
+              next
+            end
+          end
+
+          rendered_segments << { mora: mora, text: base_cyr, kind: :syllable }
+        end
+      end
+
+      # Step 2: Resolve sokuon (っ) and moraic n (ん) based on next rendered segment
+      final_parts = []
+      rendered_segments.each_with_index do |seg, idx|
+        if seg[:kind] == :sokuon
+          # Find next non-empty rendered segment
+          next_seg = rendered_segments[(idx + 1)..].find { |s| s[:text] && !s[:text].empty? }
+          if next_seg
+            next_text = next_seg[:text]
+            geminated = geminate_consonant(next_text)
+            final_parts << geminated if geminated
+          end
+        elsif seg[:kind] == :moraic_n
+          # Find next non-empty rendered segment
+          next_seg = rendered_segments[(idx + 1)..].find { |s| s[:text] && !s[:text].empty? }
+          if next_seg && starts_with_labial?(next_seg[:text])
+            final_parts << "м"
+          else
+            final_parts << "н"
+          end
+        else
+          final_parts << seg[:text]
+        end
+      end
+
+      Result.new(
+        text: final_parts.join,
+        source: :derived,
+        confidence: 1.0,
+        notes: nil
+      )
+    end
+
+    # Sokuon doubles only before the plosives «п» and «к», which Ukrainian
+    # carries comfortably (Іппекі, кеппекі, Ніккай, Хоккекьо). Before the
+    # sibilants and affricates it is not rendered: шш, чч and ддж read as
+    # foreign, so шічяку, Тешю, доджін-дзаші.
+    GEMINATING_CONSONANTS = %w[п к].freeze
+
+    def self.geminate_consonant(next_rendered_text)
+      return "" if next_rendered_text.nil? || next_rendered_text.empty?
+
+      first_char = next_rendered_text[0]
+      GEMINATING_CONSONANTS.include?(first_char) ? first_char : ""
+    end
+
+    def self.starts_with_labial?(text)
+      return false if text.nil? || text.empty?
+      # Labials: 'п', 'б'
+      text.start_with?("п", "б")
+    end
+
+    # Simple romaji to hiragana conversion for Mode A romaji input
+    def self.romaji_to_hiragana(str)
+      norm = str.downcase.gsub("'", "'")
+      mora_map = Rules.mora_map
+
+      # Build inverted map from romaji -> hiragana kana
+      @romaji_to_hira_table ||= begin
+        table = {}
+        mora_map.each do |kana_sym, data|
+          rom = data[:romaji]&.to_s
+          table[rom] = kana_sym.to_s if rom
+        end
+        # Sort by romaji length descending to match longest substrings first
+        table.sort_by { |k, _v| -k.length }.to_h
+      end
+
+      res = []
+      i = 0
+      len = norm.length
+      while i < len
+        matched = false
+
+        # Check sokuon in romaji (e.g. kk, pp, tt, ss, tch, ssh, etc.)
+        c = norm[i]
+        c_next = norm[i + 1]
+        c3 = norm[i, 3]
+
+        if c3 == "tch"
+          res << "っ"
+          i += 1
+          next
+        elsif c == c_next && c =~ /[bcdfghjklmpqrstvwxyz]/ && c != "n"
+          res << "っ"
+          i += 1
+          next
+        end
+
+        # Match table
+        @romaji_to_hira_table.each do |rom, kana|
+          if norm[i..].start_with?(rom)
+            res << kana
+            i += rom.length
+            matched = true
+            break
+          end
+        end
+
+        unless matched
+          res << norm[i]
+          i += 1
+        end
+      end
+
+      res.join
+    end
+  end
+
+  def self.cyrillic(input = nil, kanji: nil, reading: nil)
+    Cyrillic.call(input, kanji: kanji, reading: reading)
+  end
+
+  def self.from_romaji(romaji_str)
+    Cyrillic.call(romaji_str)
+  end
+end

data/lib/yanagi/doc_sync.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require_relative "rules"
+
+module Yanagi
+  class DocSync
+    attr_reader :path
+
+    def self.default_path
+      candidates = [
+        File.expand_path("~/projects/meijin/shared/transliteration.md"),
+        File.expand_path("shared/transliteration.md", Dir.pwd),
+        File.expand_path("../../shared/transliteration.md", __dir__)
+      ]
+      candidates.find { |p| File.exist?(p) } || candidates.first
+    end
+
+    def initialize(path: nil)
+      @path = path || self.class.default_path
+    end
+
+    def diff(rules = nil)
+      rules ||= Rules.mora_map
+      return [{ kind: :file_not_found, path: @path }] unless File.exist?(@path)
+
+      content = File.read(@path, encoding: "UTF-8")
+      differences = []
+
+      # 1. Parse §1 table
+      table_1_entries = parse_section_1_table(content)
+      table_1_entries.each do |entry|
+        entry[:kanas].each do |kana|
+          rule_data = rules[kana.to_sym]
+          next unless rule_data
+
+          expected_cyr = rule_data[:cyrillic]
+          expected_forb = rule_data[:forbidden] || []
+
+          # Check if doc Cyrillic matches expected (either exact mora, base consonant prefix, or included in listed items)
+          matched_cyr = entry[:cyrillic_items].include?(expected_cyr) ||
+                        (entry[:base_cyrillic] && expected_cyr.start_with?(entry[:base_cyrillic]))
+
+          unless matched_cyr
+            differences << {
+              kind: :mora_mismatch,
+              section: 1,
+              kana: kana,
+              doc: entry[:base_cyrillic] || entry[:cyrillic_items].join(", "),
+              rules: expected_cyr
+            }
+          end
+
+          # Check forbidden variants
+          if entry[:forbidden_items].any?
+            # At least one forbidden form or base prefix in doc should match rule's forbidden
+            matched_forb = entry[:forbidden_items].any? do |df|
+              expected_forb.any? { |rf| rf == df || rf.start_with?(df) }
+            end || (entry[:base_forbidden] && expected_forb.any? { |rf| rf.start_with?(entry[:base_forbidden]) })
+
+            unless matched_forb
+              differences << {
+                kind: :forbidden_mismatch,
+                section: 1,
+                kana: kana,
+                doc_forbidden: entry[:forbidden_items].join(", "),
+                rules_forbidden: expected_forb
+              }
+            end
+          end
+        end
+      end
+
+      # 2. Parse §2 bullet points (Yōon)
+      yoon_entries = parse_section_2_yoon(content)
+      yoon_entries.each do |entry|
+        entry[:kanas].each_with_index do |kana, idx|
+          expected_cyr = rules.dig(kana.to_sym, :cyrillic)
+          next unless expected_cyr
+
+          doc_cyr = entry[:cyrillics][idx] || entry[:cyrillics].first
+          if doc_cyr && doc_cyr != expected_cyr
+            differences << {
+              kind: :mora_mismatch,
+              section: 2,
+              kana: kana,
+              doc: doc_cyr,
+              rules: expected_cyr
+            }
+          end
+        end
+      end
+
+      differences
+    end
+
+    def synced?(rules = nil)
+      diff(rules).empty?
+    end
+
+    private
+
+    def clean_markdown_markup(str)
+      str.to_s.gsub(/\*\*/, "").gsub(/__/, "").gsub(/`/, "").strip
+    end
+
+    def parse_section_1_table(content)
+      entries = []
+      in_sec_1 = false
+
+      content.each_line do |line|
+        if line =~ /^##\s+1\.\s+/
+          in_sec_1 = true
+          next
+        elsif line =~ /^##\s+2\.\s+/
+          break
+        end
+
+        next unless in_sec_1
+        next unless line.start_with?("|")
+        parts = line.split("|").map(&:strip).reject(&:empty?)
+        next if parts.empty? || parts[0].start_with?("-") || parts[0] =~ /Японська/i
+
+        kana_col = parts[0]
+        cyr_col = parts[1]
+        forb_col = parts[2]
+
+        kanas = extract_kanas(kana_col)
+
+        clean_cyr = clean_markdown_markup(cyr_col)
+        clean_forb = clean_markdown_markup(forb_col)
+
+        base_cyr = clean_cyr.split(/[\s(]/).first
+        base_forb = clean_forb.split(/[\s(]/).first
+
+        cyr_items = if clean_cyr =~ /\((.+?)\)/
+                      $1.split(",").flat_map { |s| s.split("/") }.map(&:strip).reject(&:empty?)
+                    else
+                      [base_cyr]
+                    end
+
+        forb_items = if clean_forb =~ /\((.+?)\)/
+                       $1.split(",").flat_map { |s| s.split("/") }.map(&:strip).reject(&:empty?)
+                     else
+                       clean_forb.split(",").flat_map { |s| s.split("/") }.map(&:strip).reject(&:empty?)
+                     end
+
+        entries << {
+          kanas: kanas,
+          base_cyrillic: base_cyr,
+          cyrillic_items: cyr_items,
+          base_forbidden: base_forb,
+          forbidden_items: forb_items
+        }
+      end
+
+      entries
+    end
+
+    def parse_section_2_yoon(content)
+      entries = []
+      in_sec_2 = false
+
+      content.each_line do |line|
+        if line =~ /^##\s+2\.\s+/
+          in_sec_2 = true
+          next
+        elsif line =~ /^##\s+3\.\s+/
+          break
+        end
+
+        next unless in_sec_2
+        if line =~ /^\s*-\s+(.+?)\s*→\s*(.+?)(?:\(|$)/
+          left = $1.strip
+          right = $2.strip
+
+          kanas = extract_kanas(left)
+          cyrillics = clean_markdown_markup(right).split(/[\s\/]+/).map(&:strip).reject(&:empty?)
+
+          entries << { kanas: kanas, cyrillics: cyrillics }
+        end
+      end
+
+      entries
+    end
+
+    def extract_kanas(str)
+      str.scan(/[\u3040-\u309F]+/)
+    end
+  end
+
+  def self.doc_sync(path: nil)
+    DocSync.new(path: path)
+  end
+end

data/lib/yanagi/lexicon.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "yaml"
+require_relative "rules"
+require_relative "normalize"
+require_relative "cyrillic"
+
+module Yanagi
+  module Lexicon
+    UKRAINIAN_SUFFIXES = %w[
+      ами ями ах ях ів їв ою ею єю
+      ом ем єм
+      а е и і о у ю я ї й
+    ].freeze
+
+    MIN_STEM_LENGTH = 4
+
+    def self.load_entries
+      Rules.lexicon || {}
+    end
+
+    # Terms of non-Japanese origin (Chinese works, place names) are transliterated
+    # by their own language's conventions, so Japanese mora rules must not apply.
+    # Detected from the glossary note column, which states the provenance.
+    NON_JP_NOTE_MARKERS = [
+      "Китай", "китайськ", "конфуціанськ", "Даоськ", "даоськ", "Янцзи"
+    ].freeze
+
+    def self.detect_origin(note)
+      text = note.to_s
+      return "zh" if NON_JP_NOTE_MARKERS.any? { |m| text.include?(m) }
+
+      "ja"
+    end
+
+    def self.build_from_glossary(glossary_path, out_path: nil)
+      content = File.read(glossary_path, encoding: "UTF-8")
+      lexicon = {}
+
+      content.each_line do |line|
+        next unless line.start_with?("|")
+        parts = line.split("|").map(&:strip).reject(&:empty?)
+        next if parts.empty? || parts[0].start_with?("-") || parts[0] =~ /Японське/i
+
+        col0 = parts[0]
+        col1 = parts[1]
+        col2 = parts[2]
+
+        kanji = nil
+        reading = nil
+        if col0 =~ /^(.+?)\s*\((.+?)\)$/
+          kanji = Regexp.last_match(1).strip
+          reading = Regexp.last_match(2).strip
+        else
+          kanji = col0.strip
+          reading = nil
+        end
+
+        uk_clean = col1.gsub(/^[«"'\`]|["'\`»]$/, "").strip
+        key = kanji.gsub(/^[«"'\`]|["'\`»]$/, "").strip
+        next if key.empty?
+
+        cyr = reading || uk_clean
+
+        lexicon[key] = {
+          "kanji" => kanji,
+          "reading" => reading,
+          "cyrillic" => cyr,
+          "ukrainian" => col1,
+          "note" => col2,
+          "origin" => detect_origin(col2)
+        }
+      end
+
+      out_file = out_path || Rules.path_for("lexicon.yml")
+      File.write(out_file, YAML.dump(lexicon))
+      reload!
+      Rules.reload!
+      lexicon
+    end
+
+    def self.merge_proposal(proposal_path, out_path: nil)
+      proposals = YAML.safe_load_file(proposal_path, permitted_classes: [Symbol, Date], symbolize_names: true) || []
+      proposals = [proposals] unless proposals.is_a?(Array)
+
+      current_lexicon = Rules.lexicon ? Rules.lexicon.transform_keys(&:to_s) : {}
+      new_lexicon = current_lexicon.dup
+
+      errors = []
+
+      proposals.each do |prop|
+        status = prop[:status]&.to_s
+        unless status == "accepted"
+          errors << "Proposal #{prop[:term] || prop[:kanji]} status is '#{status}', must be 'accepted' to merge"
+          next
+        end
+
+        kanji = prop[:kanji]&.to_s
+        reading = prop[:reading]&.to_s
+        cyrillic = prop[:cyrillic]&.to_s
+
+        if reading && !reading.empty? && cyrillic && !cyrillic.empty?
+          derived_cyr = Yanagi.cyrillic(reading).text
+          if derived_cyr != cyrillic && prop[:override] != true
+            errors << "Proposal #{kanji || reading} Cyrillic '#{cyrillic}' does not match engine derived '#{derived_cyr}'"
+            next
+          end
+        end
+
+        key = (kanji && !kanji.empty?) ? kanji : cyrillic
+        new_lexicon[key] = {
+          "kanji" => kanji,
+          "reading" => reading,
+          "cyrillic" => cyrillic,
+          "ukrainian" => prop[:ukrainian] || cyrillic,
+          "note" => prop[:note]
+        }
+      end
+
+      unless errors.empty?
+        raise Error, "Cannot merge proposals:\n- #{errors.join("\n- ")}"
+      end
+
+      out_file = out_path || Rules.path_for("lexicon.yml")
+      File.write(out_file, YAML.dump(new_lexicon))
+      reload!
+      Rules.reload!
+      new_lexicon
+    end
+
+    def self.strip_inflection(token)
+      w = token.downcase.gsub(/^[«"'\`\(\[\{]|["'\`\)\]\}\.,;:!?—–]+$/, "")
+      return [w, ""] if w.length <= MIN_STEM_LENGTH
+
+      UKRAINIAN_SUFFIXES.each do |suffix|
+        if w.end_with?(suffix) && (w.length - suffix.length) >= MIN_STEM_LENGTH
+          stem = w[0...(w.length - suffix.length)]
+          return [stem, suffix]
+        end
+      end
+
+      [w, ""]
+    end
+
+    def self.reload!
+      @exact_index = nil
+      @stem_index = nil
+    end
+
+    def self.exact_index
+      @exact_index ||= begin
+        idx = {}
+        load_entries.each do |k, entry|
+          cyr = (entry[:cyrillic] || entry["cyrillic"])&.to_s&.downcase
+          next unless cyr
+
+          # Index full term
+          idx[cyr] = { key: k.to_s, entry: entry, stem: cyr, suffix: "", exact: true }
+
+          # Also index individual words of multi-word phrases (e.g. 'хонінбо', 'шюсай')
+          words = cyr.split(/[\s-]+/).map(&:strip).reject(&:empty?)
+          if words.length > 1
+            words.each do |w|
+              idx[w] ||= { key: k.to_s, entry: entry, stem: w, suffix: "", exact: true } if w.length >= MIN_STEM_LENGTH
+            end
+          end
+        end
+        idx
+      end
+    end
+
+    def self.stem_index
+      @stem_index ||= begin
+        idx = {}
+        load_entries.each do |k, entry|
+          cyr = (entry[:cyrillic] || entry["cyrillic"])&.to_s&.downcase
+          next unless cyr
+
+          words = cyr.split(/[\s-]+/).map(&:strip).reject(&:empty?)
+          words.each do |w|
+            stem, _ = strip_inflection(w)
+            idx[stem] ||= { key: k.to_s, entry: entry, stem: stem } if stem.length >= MIN_STEM_LENGTH
+            idx[w] ||= { key: k.to_s, entry: entry, stem: w } if w.length >= MIN_STEM_LENGTH
+          end
+        end
+        idx
+      end
+    end
+
+    def self.find_by_stem(token)
+      clean = token.downcase.gsub(/^[«"'\`\(\[\{]|["'\`\)\]\}\.,;:!?—–]+$/, "")
+      return exact_index[clean] if exact_index.key?(clean)
+
+      stem, suffix = strip_inflection(clean)
+      return nil if stem.length < MIN_STEM_LENGTH
+
+      if stem_index.key?(stem)
+        info = stem_index[stem]
+        return {
+          key: info[:key],
+          entry: info[:entry],
+          stem: stem,
+          suffix: suffix,
+          exact: false
+        }
+      end
+
+      nil
+    end
+  end
+end