yanagi 0.1.0

data/lib/yanagi/mora.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative "normalize"
+require_relative "rules"
+
+module Yanagi
+  Mora = Struct.new(:kana, :kind, :onset, :nucleus, keyword_init: true) do
+    def sokuon?
+      kind == :sokuon
+    end
+
+    def chouonpu?
+      kind == :chouonpu
+    end
+
+    def moraic_n?
+      kind == :moraic_n
+    end
+
+    def syllable?
+      kind == :syllable
+    end
+
+    def passthrough?
+      kind == :passthrough
+    end
+  end
+
+  module Tokenizer
+    def self.tokenize(input)
+      hira = Normalize.to_hiragana(Normalize.nfkc(input.to_s))
+      moras = []
+      mora_map = Rules.mora_map
+
+      i = 0
+      len = hira.length
+      while i < len
+        c1 = hira[i]
+        c2 = hira[i, 2]
+
+        if c1 == "っ"
+          moras << Mora.new(kana: "っ", kind: :sokuon, onset: nil, nucleus: nil)
+          i += 1
+        elsif c1 == "ー"
+          moras << Mora.new(kana: "ー", kind: :chouonpu, onset: nil, nucleus: nil)
+          i += 1
+        elsif c1 == "ん"
+          moras << Mora.new(kana: "ん", kind: :moraic_n, onset: "n", nucleus: "n")
+          i += 1
+        elsif c2 && mora_map.key?(c2.to_sym)
+          info = mora_map[c2.to_sym]
+          moras << Mora.new(
+            kana: c2,
+            kind: :syllable,
+            onset: info[:onset]&.to_s || "",
+            nucleus: info[:nucleus]&.to_s || ""
+          )
+          i += 2
+        elsif mora_map.key?(c1.to_sym)
+          info = mora_map[c1.to_sym]
+          moras << Mora.new(
+            kana: c1,
+            kind: :syllable,
+            onset: info[:onset]&.to_s || "",
+            nucleus: info[:nucleus]&.to_s || ""
+          )
+          i += 1
+        else
+          moras << Mora.new(kana: c1, kind: :passthrough, onset: nil, nucleus: nil)
+          i += 1
+        end
+      end
+
+      moras
+    end
+  end
+
+  def self.tokenize(input)
+    Tokenizer.tokenize(input)
+  end
+end

data/lib/yanagi/normalize.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Yanagi
+  module Normalize
+    KATA_START = 0x30A1
+    KATA_END   = 0x30F6
+    KATA_SHIFT = 0x60
+
+    # NFKC-normalise a string (compatibility decomposition then canonical composition).
+    # Input may arrive tagged ASCII-8BIT (e.g. from ARGV), which unicode_normalize
+    # rejects, so coerce to UTF-8 first.
+    def self.nfkc(str)
+      s = str.to_s
+      s = s.dup.force_encoding(Encoding::UTF_8) unless s.encoding == Encoding::UTF_8
+      s.unicode_normalize(:nfkc)
+    end
+
+    # Convert a katakana string to hiragana (passthrough for everything else).
+    def self.to_hiragana(str)
+      str.to_s.chars.map do |ch|
+        cp = ch.ord
+        (cp >= KATA_START && cp <= KATA_END) ? (cp - KATA_SHIFT).chr(Encoding::UTF_8) : ch
+      end.join
+    end
+  end
+end

data/lib/yanagi/romaji.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "mora"
+require_relative "rules"
+
+module Yanagi
+  module Romaji
+    def self.call(input)
+      moras = input.is_a?(Array) ? input : Tokenizer.tokenize(input)
+      mora_map = Rules.mora_map
+      res = []
+
+      moras.each_with_index do |mora, idx|
+        case mora.kind
+        when :sokuon
+          # Find next non-sokuon/chouonpu mora
+          next_mora = moras[(idx + 1)..].find { |m| m.syllable? || m.moraic_n? }
+          if next_mora
+            next_rom = mora_map.dig(next_mora.kana.to_sym, :romaji)&.to_s || next_mora.kana.to_s
+            consonant = next_rom.start_with?("ch") ? "t" : next_rom[0]
+            res << consonant if consonant
+          end
+        when :chouonpu
+          prev_char = res.last&.chars&.last
+          res << prev_char if prev_char
+        when :moraic_n
+          res << "n"
+        when :syllable
+          rom = mora_map.dig(mora.kana.to_sym, :romaji)&.to_s || mora.kana.to_s
+          res << rom
+        else
+          res << mora.kana.to_s
+        end
+      end
+
+      res.join
+    end
+    def self.matches?(reading, romaji)
+      expected = call(reading)
+      return true if romaji == expected
+
+      norm_actual = romaji.to_s.gsub(/\d+$/, "").gsub(/([aeiou])-/, '\1\1').gsub("oo", "ou").gsub("dewa", "deha")
+      norm_exp = expected.gsub("oo", "ou")
+      norm_actual == norm_exp
+    end
+  end
+
+  def self.romaji(input)
+    Romaji.call(input)
+  end
+
+  def self.romaji_matches?(reading, romaji)
+    Romaji.matches?(reading, romaji)
+  end
+end

data/lib/yanagi/rules.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Yanagi
+  module Rules
+    DATA_DIR = File.expand_path("../../data", __dir__).freeze
+
+    @mutex = Mutex.new
+
+    # Policy rules ship with the gem. Corpus-derived data (the lexicon and the
+    # native-Ukrainian allowlist) is generated from a specific translation
+    # project and is NOT distributed: point YANAGI_DATA_DIR at a directory
+    # holding those files, or build them with `yanagi lexicon build`.
+    CORPUS_FILES = %w[lexicon.yml native_ua_allowlist.yml].freeze
+
+    def self.data_dir
+      DATA_DIR
+    end
+
+    # Where corpus-derived files are read from. Defaults to the gem's data
+    # directory so a local checkout keeps working without configuration.
+    def self.corpus_dir
+      ENV.fetch("YANAGI_DATA_DIR", DATA_DIR)
+    end
+
+    def self.path_for(filename)
+      dir = CORPUS_FILES.include?(filename) ? corpus_dir : DATA_DIR
+      File.join(dir, filename)
+    end
+
+    def self.load_yaml(filename)
+      path = path_for(filename)
+      return {}.freeze unless File.exist?(path)
+
+      data = YAML.safe_load_file(path, permitted_classes: [Symbol, Date], symbolize_names: true) || {}
+      deep_freeze(data)
+    end
+
+    def self.mora_map
+      @mora_map ||= load_yaml("mora.yml")
+    end
+
+    def self.combinatorial
+      @combinatorial ||= load_yaml("combinatorial.yml")
+    end
+
+    def self.exonyms
+      @exonyms ||= load_yaml("exonyms.yml")
+    end
+
+    def self.lexicon
+      @lexicon ||= load_yaml("lexicon.yml")
+    end
+
+    def self.native_ua_allowlist
+      @native_ua_allowlist ||= load_yaml("native_ua_allowlist.yml")
+    end
+
+    def self.exceptions
+      @exceptions ||= load_yaml("exceptions.yml")
+    end
+
+    def self.reload!
+      @mutex.synchronize do
+        @mora_map = nil
+        @combinatorial = nil
+        @exonyms = nil
+        @lexicon = nil
+        @native_ua_allowlist = nil
+        @exceptions = nil
+      end
+    end
+
+    def self.deep_freeze(obj)
+      case obj
+      when Hash
+        obj.transform_keys(&:freeze).transform_values { |v| deep_freeze(v) }.freeze
+      when Array
+        obj.map { |v| deep_freeze(v) }.freeze
+      when String
+        obj.dup.freeze
+      else
+        obj.freeze
+      end
+    end
+    private_class_method :deep_freeze
+  end
+end

data/lib/yanagi/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Yanagi
+  VERSION = "0.1.0"
+end

data/lib/yanagi.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative "yanagi/version"
+require_relative "yanagi/normalize"
+require_relative "yanagi/rules"
+require_relative "yanagi/mora"
+require_relative "yanagi/romaji"
+require_relative "yanagi/cyrillic"
+require_relative "yanagi/doc_sync"
+require_relative "yanagi/lexicon"
+require_relative "yanagi/audit"
+
+module Yanagi
+  class Error < StandardError; end
+end

data/yanagi.gemspec

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative "lib/yanagi/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "yanagi"
+  spec.version       = Yanagi::VERSION
+  spec.authors       = ["shogi-dojo"]
+  spec.email         = ["play@shogi-dojo.com"]
+
+  spec.summary       = "Deterministic Japanese to Ukrainian transliteration engine"
+  spec.description   = "Zero-dependency Japanese to Ukrainian transliteration and policy enforcement gem."
+  spec.homepage      = "https://github.com/shogi-dojo/yanagi"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 3.0.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/shogi-dojo/yanagi/tree/main"
+  spec.metadata["bug_tracker_uri"] = "https://github.com/shogi-dojo/yanagi/issues"
+  spec.metadata["changelog_uri"] = "https://github.com/shogi-dojo/yanagi/blob/main/CHANGELOG.md"
+
+  spec.files = Dir.chdir(__dir__) do
+    # Corpus-derived data (lexicon, native-Ukrainian allowlist) is generated
+    # from a specific translation project and is not distributed.
+    Dir["{lib,data,exe}/**/*", "LICENSE*", "README*", "CHANGELOG*", "yanagi.gemspec"] -
+      Dir["data/{lexicon,native_ua_allowlist}.yml"]
+  end
+  spec.bindir        = "exe"
+  spec.executables   = ["yanagi"]
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "minitest", ">= 5.16"
+  spec.add_development_dependency "rake", "~> 13.0"
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: yanagi
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- shogi-dojo
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: Zero-dependency Japanese to Ukrainian transliteration and policy enforcement
+  gem.
+email:
+- play@shogi-dojo.com
+executables:
+- yanagi
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- data/combinatorial.yml
+- data/exceptions.yml
+- data/exonyms.yml
+- data/mora.yml
+- exe/yanagi
+- lib/yanagi.rb
+- lib/yanagi/audit.rb
+- lib/yanagi/cli.rb
+- lib/yanagi/cyrillic.rb
+- lib/yanagi/doc_sync.rb
+- lib/yanagi/lexicon.rb
+- lib/yanagi/mora.rb
+- lib/yanagi/normalize.rb
+- lib/yanagi/romaji.rb
+- lib/yanagi/rules.rb
+- lib/yanagi/version.rb
+- yanagi.gemspec
+homepage: https://github.com/shogi-dojo/yanagi
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/shogi-dojo/yanagi
+  source_code_uri: https://github.com/shogi-dojo/yanagi/tree/main
+  bug_tracker_uri: https://github.com/shogi-dojo/yanagi/issues
+  changelog_uri: https://github.com/shogi-dojo/yanagi/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: Deterministic Japanese to Ukrainian transliteration engine
+test_files: []