yosina 0.1.0

data/codegen/dataset.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'json'
+
+# Dataset source definitions
+DatasetSourceDefs = Struct.new(
+  :spaces,
+  :radicals,
+  :mathematical_alphanumerics,
+  :ideographic_annotations,
+  :hyphens,
+  :ivs_svs_base,
+  :kanji_old_new,
+  :combined,
+  :circled_or_squared,
+  keyword_init: true
+)
+
+# Dataset container
+Dataset = Struct.new(
+  :spaces,
+  :radicals,
+  :mathematical_alphanumerics,
+  :ideographic_annotations,
+  :hyphens,
+  :ivs_svs_base,
+  :kanji_old_new,
+  :combined,
+  :circled_or_squared,
+  keyword_init: true
+)
+
+# Convert Unicode codepoint string to character
+def unicode_to_char(unicode_str)
+  return '' if unicode_str.nil?
+
+  # Remove U+ prefix and convert to integer
+  codepoint = unicode_str.sub(/^U\+/, '').to_i(16)
+  [codepoint].pack('U*')
+end
+
+# Load simple mapping data (key -> value or key -> null for removal)
+def load_simple_data(filepath)
+  data = JSON.parse(File.read(filepath))
+  mappings = {}
+
+  data.each do |key, value|
+    char_key = unicode_to_char(key)
+    char_value = value.nil? ? '' : unicode_to_char(value)
+    mappings[char_key] = char_value
+  end
+
+  mappings
+end
+
+# Load kanji old-new form data (array of pairs)
+def load_kanji_old_new_data(filepath)
+  data = JSON.parse(File.read(filepath))
+  mappings = {}
+
+  data.each do |pair|
+    next unless pair.is_a?(Array) && pair.length == 2
+
+    old_form = pair[0]
+    new_form = pair[1]
+
+    # Convert IVS sequences to characters
+    old_chars = []
+    new_chars = []
+
+    if old_form['ivs']
+      old_chars = old_form['ivs'].map { |cp| unicode_to_char(cp) }
+    elsif old_form['svs']
+      old_chars = old_form['svs'].map { |cp| unicode_to_char(cp) }
+    end
+
+    if new_form['ivs']
+      new_chars = new_form['ivs'].map { |cp| unicode_to_char(cp) }
+    elsif new_form['svs']
+      new_chars = new_form['svs'].map { |cp| unicode_to_char(cp) }
+    end
+
+    next if old_chars.empty? || new_chars.empty?
+
+    old_key = old_chars.join
+    new_value = new_chars.join
+
+    mappings[old_key] = new_value
+  end
+
+  mappings
+end
+
+# Load hyphens data (array of hyphen records)
+def load_hyphens_data(filepath)
+  data = JSON.parse(File.read(filepath))
+  records = []
+
+  data.each do |record|
+    hyphen_char = unicode_to_char(record['code'])
+
+    # Extract mappings from the record
+    hyphens_record = {
+      hyphen: hyphen_char,
+      ascii: nil,
+      jisx0201: nil,
+      jisx0208_90: nil,
+      jisx0208_90_windows: nil,
+      jisx0208_verbatim: nil
+    }
+
+    # Map ASCII field
+    if record['ascii'] && !record['ascii'].empty?
+      hyphens_record[:ascii] = record['ascii'].map { |cp| unicode_to_char(cp) }.join
+    end
+
+    # Map JIS X 0201 field
+    if record['jisx0201'] && !record['jisx0201'].empty?
+      hyphens_record[:jisx0201] = record['jisx0201'].map { |cp| unicode_to_char(cp) }.join
+    end
+
+    # Map JIS X 0208-1978 field (use as jisx0208_90)
+    if record['jisx0208-1978'] && !record['jisx0208-1978'].empty?
+      hyphens_record[:jisx0208_90] = record['jisx0208-1978'].map { |cp| unicode_to_char(cp) }.join
+    end
+
+    # Map JIS X 0208-1978 Windows field
+    if record['jisx0208-1978-windows'] && !record['jisx0208-1978-windows'].empty?
+      hyphens_record[:jisx0208_90_windows] = record['jisx0208-1978-windows'].map { |cp| unicode_to_char(cp) }.join
+    end
+
+    records << hyphens_record
+  end
+
+  records
+end
+
+# Load IVS/SVS base data (array of mapping records)
+def load_ivs_svs_base_data(filepath)
+  data = JSON.parse(File.read(filepath))
+  records = []
+
+  data.each do |record|
+    # Build IVS sequence
+    ivs_chars = []
+    ivs_chars = record['ivs'].map { |cp| unicode_to_char(cp) } if record['ivs'].is_a?(Array)
+
+    # Build SVS sequence
+    svs_chars = []
+    svs_chars = record['svs'].map { |cp| unicode_to_char(cp) } if record['svs'].is_a?(Array)
+
+    # Get base characters
+    base90_char = record['base90'] ? unicode_to_char(record['base90']) : nil
+    base2004_char = record['base2004'] ? unicode_to_char(record['base2004']) : nil
+
+    # Only add record if we have IVS data
+    next if ivs_chars.empty?
+
+    records << {
+      ivs: ivs_chars.join,
+      svs: svs_chars.empty? ? nil : svs_chars.join,
+      base90: base90_char,
+      base2004: base2004_char
+    }
+  end
+
+  records
+end
+
+# Load combined characters data (key -> string to be split into characters)
+def load_combined_data(filepath)
+  data = JSON.parse(File.read(filepath))
+  mappings = {}
+
+  data.each do |key, value|
+    char_key = unicode_to_char(key)
+    char_list = value.chars
+    mappings[char_key] = char_list
+  end
+
+  mappings
+end
+
+# Load circled or squared characters data
+def load_circled_or_squared_data(filepath)
+  data = JSON.parse(File.read(filepath))
+  mappings = {}
+
+  data.each do |key, record|
+    char_key = unicode_to_char(key)
+    record_data = {
+      rendering: record['rendering'],
+      type: record['type'],
+      emoji: record['emoji']
+    }
+    mappings[char_key] = record_data
+  end
+
+  mappings
+end
+
+# Build dataset from data root directory
+def build_dataset_from_data_root(data_root, defs)
+  Dataset.new(
+    spaces: load_simple_data(data_root / defs.spaces),
+    radicals: load_simple_data(data_root / defs.radicals),
+    mathematical_alphanumerics: load_simple_data(data_root / defs.mathematical_alphanumerics),
+    ideographic_annotations: load_simple_data(data_root / defs.ideographic_annotations),
+    hyphens: load_hyphens_data(data_root / defs.hyphens),
+    ivs_svs_base: load_ivs_svs_base_data(data_root / defs.ivs_svs_base),
+    kanji_old_new: load_kanji_old_new_data(data_root / defs.kanji_old_new),
+    combined: load_combined_data(data_root / defs.combined),
+    circled_or_squared: load_circled_or_squared_data(data_root / defs.circled_or_squared)
+  )
+end

data/codegen/emitters/circled_or_squared_transliterator_data.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'utils'
+
+# Render a circled or squared character transliterator
+def render_circled_or_squared_transliterator_data(mappings)
+  # Generate mapping entries for circled/squared characters
+  mapping_entries = mappings.map do |key, record|
+    type_abbrev = record[:type] == 'circle' ? 'c' : 's'
+    record_repr = "{ rendering: #{to_string_literal(record[:rendering])}" \
+                  ", type: #{to_string_literal(type_abbrev)}, emoji: #{record[:emoji]} }"
+    "            #{to_string_literal(key)} => #{record_repr}"
+  end.join(",\n")
+
+  dedent 4, <<~RUBY
+        # frozen_string_literal: true
+
+        module Yosina
+          module Transliterators
+            # Replace circled or squared characters with their corresponding templates
+            module CircledOrSquared
+              # Generated mapping data from circled-or-squared.json
+              CIRCLED_OR_SQUARED_MAPPINGS = {
+    #{mapping_entries}
+              }.freeze
+            end
+          end
+        end
+  RUBY
+end

data/codegen/emitters/combined_transliterator_data.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative 'utils'
+
+# Render a combined character transliterator
+def render_combined_transliterator_data(mappings)
+  # Generate mapping entries for combined characters -> character arrays
+  mapping_entries = mappings.map do |key, value_array|
+    value_repr = "[#{value_array.map { |c| to_string_literal(c) }.join(', ')}]"
+    "            #{to_string_literal(key)} => #{value_repr}"
+  end.join(",\n")
+
+  dedent 4, <<~RUBY
+        # frozen_string_literal: true
+
+        module Yosina
+          module Transliterators
+            # Replace each combined character with its corresponding individual characters
+            module Combined
+              # Generated mapping data from combined-chars.json
+              COMBINED_MAPPINGS = {
+    #{mapping_entries}
+              }.freeze
+            end
+          end
+        end
+  RUBY
+end

data/codegen/emitters/hyphens_transliterator_data.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative 'utils'
+
+# Render hyphens transliterator data
+def render_hyphens_transliterator_data(records)
+  # Generate records array - now using proper hyphens data structure
+  records_data = records.map do |record|
+    # Build HyphensRecord fields
+    fields = []
+    fields << "ascii: #{to_string_literal(record[:ascii])}" if record[:ascii]
+    fields << "jisx0201: #{to_string_literal(record[:jisx0201])}" if record[:jisx0201]
+    fields << "jisx0208_90: #{to_string_literal(record[:jisx0208_90])}" if record[:jisx0208_90]
+    fields << "jisx0208_90_windows: #{to_string_literal(record[:jisx0208_90_windows])}" if record[:jisx0208_90_windows]
+    fields << "jisx0208_verbatim: #{to_string_literal(record[:jisx0208_verbatim])}" if record[:jisx0208_verbatim]
+
+    record_repr = if fields.empty?
+                    'HyphensRecord.new'
+                  else
+                    "HyphensRecord.new(#{fields.join(', ')})"
+                  end
+
+    "        #{to_string_literal(record[:hyphen])} => #{record_repr}"
+  end.join(",\n")
+
+  <<~RUBY
+    # frozen_string_literal: true
+
+    module Yosina
+      module Transliterators
+        # Generated hyphens data
+        module HyphensData
+          # Record for hyphen transliteration data
+          HyphensRecord = Struct.new(:ascii, :jisx0201, :jisx0208_90, :jisx0208_90_windows, :jisx0208_verbatim, keyword_init: true) do
+            def initialize(ascii: nil, jisx0201: nil, jisx0208_90: nil, jisx0208_90_windows: nil, jisx0208_verbatim: nil)
+              super
+            end
+          end
+
+          # Generated mapping data
+          HYPHENS_MAPPINGS = {
+    #{records_data}
+          }.freeze
+        end
+      end
+    end
+  RUBY
+end

data/codegen/emitters/ivs_svs_base_transliterator_data.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require_relative 'utils'
+
+# Render IVS/SVS base transliterator data
+def render_ivs_svs_base_transliterator_data(records)
+  # Build compressed data similar to Python implementation
+  compressed_parts = []
+  records.each do |record|
+    compressed_parts << (record[:ivs] || '')
+    compressed_parts << (record[:svs] || '')
+    compressed_parts << (record[:base90] || '')
+    compressed_parts << (record[:base2004] || '')
+  end
+  compressed_data = compressed_parts.join("\0")
+  compressed_data_escaped = to_string_literal(compressed_data)
+
+  dedent(4, <<~RUBY
+    # frozen_string_literal: true
+
+    module Yosina
+      module Transliterators
+        # Generated IVS/SVS base data
+        module IvsSvsBaseData
+          # Record for IVS/SVS base transliteration data
+          IvsSvsBaseRecord = Struct.new(:ivs, :svs, :base90, :base2004, keyword_init: true) do
+            def initialize(ivs:, svs: nil, base90: nil, base2004: nil)
+              super
+            end
+          end
+
+          # Compressed data table - 4 strings per record: [ivs, svs, base90, base2004, ...]
+          COMPRESSED_DATA = #{compressed_data_escaped}.freeze
+          RECORDS_COUNT = #{records.length}
+
+          # Expand compressed data into a mapping dictionary
+          def self.expand_compressed_data
+            mappings = {}
+            # Split by null bytes to get all fields
+            fields = COMPRESSED_DATA.split("\\0")
+            # Process 4 fields at a time (ivs, svs, base90, base2004)
+            (0...fields.length).step(4) do |i|
+              next unless i + 3 < fields.length
+              ivs = fields[i]
+              svs = fields[i + 1].empty? ? nil : fields[i + 1]
+              base90 = fields[i + 2].empty? ? nil : fields[i + 2]
+              base2004 = fields[i + 3].empty? ? nil : fields[i + 3]
+              # Only add if ivs is not empty
+              if !ivs.empty?
+                mappings[ivs] = IvsSvsBaseRecord.new(
+                  ivs: ivs,
+                  svs: svs,
+                  base90: base90,
+                  base2004: base2004
+                )
+              end
+            end
+            mappings
+          end
+
+          # Lazy-loaded mappings cache
+          @mappings_cache = nil
+          @base_to_variants_cache = nil
+          @variants_to_base_cache = nil
+
+          # Get the IVS/SVS mappings dictionary, loading it if necessary
+          def self.get_ivs_svs_mappings
+            @mappings_cache ||= expand_compressed_data
+          end
+
+          # Build optimized lookup tables for base-to-variants and variants-to-base mappings
+          def self.populate_lookup_tables
+            return if @base_to_variants_cache && @variants_to_base_cache
+
+            mappings = get_ivs_svs_mappings
+
+            # For base->IVS/SVS lookup (used in "ivs-or-svs" mode)
+            base_to_variants_2004 = {}
+            base_to_variants_90 = {}
+
+            # For IVS/SVS->base lookup (used in "base" mode)
+            variants_to_base = {}
+
+            mappings.each do |variant_seq, record|
+              # Map base characters to their IVS/SVS variants
+              if record.base2004 && !base_to_variants_2004.key?(record.base2004)
+                base_to_variants_2004[record.base2004] = record
+              end
+
+              if record.base90 && !base_to_variants_90.key?(record.base90)
+                base_to_variants_90[record.base90] = record
+              end
+
+              # Map IVS/SVS variants back to base characters
+              variants_to_base[variant_seq] = record
+            end
+
+            @base_to_variants_cache = {
+              'unijis_2004' => base_to_variants_2004,
+              'unijis_90' => base_to_variants_90
+            }
+            @variants_to_base_cache = variants_to_base
+          end
+
+          # Get base character to variants mapping for the specified charset
+          def self.get_base_to_variants_mappings(charset = 'unijis_2004')
+            populate_lookup_tables
+            @base_to_variants_cache[charset]
+          end
+
+          # Get variants to base character mapping
+          def self.get_variants_to_base_mappings
+            populate_lookup_tables
+            @variants_to_base_cache
+          end
+        end
+      end
+    end
+  RUBY
+  )
+end

data/codegen/emitters/simple_transliterator.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative 'utils'
+
+# Render a simple transliterator with a mapping table
+def render_simple_transliterator(name, description, mappings)
+  class_name = name.split('_').map(&:capitalize).join
+  constant_name = "#{name.upcase}_MAPPINGS"
+
+  # Generate mapping entries
+  mapping_entries = mappings.map do |key, value|
+    "            #{to_string_literal(key)} => #{to_string_literal(value)}"
+  end.join(",\n")
+
+  dedent 4, <<~RUBY
+        # frozen_string_literal: true
+
+        module Yosina
+          module Transliterators
+            # #{description}
+            module #{class_name}
+              # Generated mapping data from #{name}.json
+              #{constant_name} = {
+    #{mapping_entries}
+              }.freeze
+
+              # Transliterator for #{name}
+              class Transliterator < Yosina::BaseTransliterator
+                # Initialize the transliterator with options
+                #
+                # @param _options [Hash] Configuration options (currently unused)
+                def initialize(_options = {})
+                  # Options currently unused for #{name} transliterator
+                  super()
+                end
+
+                # #{description}
+                #
+                # @param input_chars [Enumerable<Char>] The characters to transliterate
+                # @return [Enumerable<Char>] The transliterated characters
+                def call(input_chars)
+                  offset = 0
+
+                  result = input_chars.filter_map do |char|
+                    replacement = #{constant_name}[char.c]
+                    c = if replacement
+                      # Skip empty replacements (character removal)
+                      next if replacement.empty?
+                      Char.new(c: replacement, offset: offset, source: char)
+                    else
+                      char.with_offset(offset)
+                    end
+                    offset += c.c.length
+                    c
+                  end
+
+                  class << result
+                    include Yosina::Chars
+                  end
+
+                  result
+                end
+              end
+
+              # Factory method to create a #{name} transliterator
+              #
+              # @param options [Hash] Configuration options
+              # @return [Transliterator] A new #{name} transliterator instance
+              def self.call(options = {})
+                Transliterator.new(options)
+              end
+            end
+          end
+        end
+  RUBY
+end

data/codegen/emitters/utils.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+def _to_unicode_escapes_inner(str)
+  special_char_exists = false
+  [
+    str.codepoints.map do |codepoint|
+      if codepoint > 127
+        special_char_exists = true
+        # rubocop:disable Style/FormatString
+        '\\u{%x}' % codepoint
+        # rubocop:enable Style/FormatString
+      elsif [34, 39, 92].include?(codepoint)
+        special_char_exists = true
+        "\\#{codepoint.chr}"
+      else
+        codepoint.chr
+      end
+    end.join,
+    special_char_exists
+  ]
+end
+
+# Convert a string to use Unicode escape sequences for non-ASCII characters
+def to_unicode_escapes(str)
+  _to_unicode_escapes_inner(str)
+end
+
+def to_string_literal(str)
+  result, special_char_exists = _to_unicode_escapes_inner(str)
+  if special_char_exists
+    "\"#{result}\""
+  else
+    "'#{result}'"
+  end
+end
+
+def dedent(shift, lines)
+  lines.lines.map do |l|
+    if l[0, shift].each_char.all? { |c| c == ' ' }
+      l[shift..]
+    else
+      l
+    end
+  end.join
+end

data/codegen/emitters.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Main emitters file that requires all individual emitter modules
+require_relative 'emitters/simple_transliterator'
+require_relative 'emitters/hyphens_transliterator_data'
+require_relative 'emitters/ivs_svs_base_transliterator_data'
+require_relative 'emitters/combined_transliterator_data'
+require_relative 'emitters/circled_or_squared_transliterator_data'

data/codegen/main.rb

@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'json'
+require 'pathname'
+
+require_relative 'dataset'
+require_relative 'emitters'
+
+# Main code generation entry point
+def main
+  # Determine project paths
+  current_dir = Pathname(__FILE__).parent
+  project_root = current_dir.parent
+  data_root = project_root.parent / 'data'
+  dest_root = project_root / 'lib' / 'yosina' / 'transliterators'
+
+  puts "Loading dataset from: #{data_root}"
+  puts "Writing output to: #{dest_root}"
+
+  # Ensure destination directory exists
+  dest_root.mkpath
+
+  # Define dataset source definitions
+  defs = DatasetSourceDefs.new(
+    spaces: 'spaces.json',
+    radicals: 'radicals.json',
+    mathematical_alphanumerics: 'mathematical-alphanumerics.json',
+    ideographic_annotations: 'ideographic-annotation-marks.json',
+    hyphens: 'hyphens.json',
+    ivs_svs_base: 'ivs-svs-base-mappings.json',
+    kanji_old_new: 'kanji-old-new-form.json',
+    combined: 'combined-chars.json',
+    circled_or_squared: 'circled-or-squared.json'
+  )
+
+  # Load the dataset
+  dataset = build_dataset_from_data_root(data_root, defs)
+
+  # Generate simple transliterators
+  simple_transliterators = [
+    [
+      'spaces',
+      'Replace various space characters with plain whitespace',
+      dataset.spaces
+    ],
+    [
+      'radicals',
+      'Replace Kangxi radicals with equivalent CJK ideographs',
+      dataset.radicals
+    ],
+    [
+      'mathematical_alphanumerics',
+      'Replace mathematical alphanumeric symbols with plain characters',
+      dataset.mathematical_alphanumerics
+    ],
+    [
+      'ideographic_annotations',
+      'Replace ideographic annotation marks used in traditional translation',
+      dataset.ideographic_annotations
+    ],
+    [
+      'kanji_old_new',
+      'Replace old-style kanji with modern equivalents',
+      dataset.kanji_old_new
+    ]
+  ]
+
+  simple_transliterators.each do |name, description, data|
+    output = render_simple_transliterator(name, description, data)
+    filename = "#{snake_case(name)}.rb"
+    filepath = dest_root / filename
+    puts "Generating: #{filename}"
+    filepath.write(output)
+  end
+
+  # Generate hyphens data
+  output = render_hyphens_transliterator_data(dataset.hyphens)
+  filepath = dest_root / 'hyphens_data.rb'
+  puts 'Generating: hyphens_data.rb'
+  filepath.write(output)
+
+  # Generate IVS/SVS base data
+  output = render_ivs_svs_base_transliterator_data(dataset.ivs_svs_base)
+  filepath = dest_root / 'ivs_svs_base_data.rb'
+  puts 'Generating: ivs_svs_base_data.rb'
+  filepath.write(output)
+
+  # Generate combined transliterator
+  output = render_combined_transliterator_data(dataset.combined)
+  filepath = dest_root / 'combined_data.rb'
+  puts 'Generating: combined_data.rb'
+  filepath.write(output)
+
+  # Generate circled or squared transliterator
+  output = render_circled_or_squared_transliterator_data(dataset.circled_or_squared)
+  filepath = dest_root / 'circled_or_squared_data.rb'
+  puts 'Generating: circled_or_squared_data.rb'
+  filepath.write(output)
+
+  puts 'Code generation complete!'
+end
+
+# Convert camelCase to snake_case
+def snake_case(str)
+  str.gsub(/([A-Z])/, '_\1').downcase.sub(/^_/, '')
+end
+
+main if $PROGRAM_NAME == __FILE__