ucode 0.1.0

data/lib/ucode/parsers/standardized_variants.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/standardized_variant"
+
+module Ucode
+  module Parsers
+    # Parses `StandardizedVariants.txt` — variation selector sequences.
+    #
+    # Format (UAX #44):
+    #   base_cp VS_cp; description; [contexts]; # trailing comment
+    #
+    # `base_cp` + `variation_selector_id` is the key; `description` is
+    # the visual result; `contexts` (optional) is a space-separated
+    # list of shaping contexts (e.g. `no-break`).
+    class StandardizedVariants < Base
+      class << self
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            sequence_field = fields[0]
+            description = fields[1]
+            next if description.nil? || description.empty?
+
+            sequence = sequence_field.to_s.split(/\s+/).reject(&:empty?)
+            next if sequence.length < 2
+
+            base = parse_hex_cp(sequence[0])
+            vs = parse_hex_cp(sequence[1])
+
+            yield Models::StandardizedVariant.new(
+              base_id: format("U+%04X", base),
+              variation_selector_id: format("U+%04X", vs),
+              description: description,
+              contexts: parse_contexts(fields[2])
+            )
+          end
+
+          nil
+        end
+
+        private
+
+        def parse_contexts(field)
+          return [] if field.nil? || field.empty?
+
+          field.split(/\s*;\s*/).flat_map { |part| part.split(/\s+/) }.reject(&:empty?)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/unicode_data/hangul_name.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Parsers
+    class UnicodeData < Base
+      # Computes the official Unicode name for a Hangul syllable codepoint
+      # per the algorithm in Chapter 3 of the Unicode Standard (TR #15).
+      #
+      # The name is "HANGUL SYLLABLE " followed by the concatenation of the
+      # short names of the L, V, (optional T) Jamo that compose it.
+      #
+      # Constants are the canonical Jamo short names from UnicodeData.txt
+      # (also published separately as Jamo.txt). Indexing into these arrays
+      # by (cp - BASE_L/V/T) gives the short name for that Jamo.
+      module HangulName
+        S_BASE = 0xAC00
+        L_BASE = 0x1100
+        V_BASE = 0x1161
+        T_BASE = 0x11A7
+
+        L_COUNT = 19
+        V_COUNT = 21
+        T_COUNT = 28
+        N_COUNT = V_COUNT * T_COUNT # 588
+        S_COUNT = L_COUNT * N_COUNT # 11_172
+
+        LEAD_SHORT_NAMES = %w[
+          G GG N D DD R M B BB S SS
+          J JJ C K T P H
+        ].freeze
+
+        VOWEL_SHORT_NAMES = %w[
+          A AE YA YAE EO E YEO YE O WA WAE OE YO
+          U WEO WE WI YU EU YI I
+        ].freeze
+
+        TRAIL_SHORT_NAMES = [
+          "",    # 11A7 has no short name; used for LV (no trail)
+          "G", "GG", "GS", "N", "NJ", "NH", "D",
+          "L", "LG", "LM", "LB", "LS", "LT", "LH",
+          "M", "B", "BS", "S", "SS", "NG", "J",
+          "C", "K", "T", "P", "H"
+        ].freeze
+
+        class << self
+          # Returns true if `cp` is in the Hangul syllable block.
+          def hangul_syllable?(cp)
+            cp.is_a?(Integer) &&
+              cp >= S_BASE &&
+              cp < S_BASE + S_COUNT
+          end
+
+          # Returns the synthesized name for a Hangul syllable codepoint,
+          # or nil if `cp` is not in the Hangul syllable block.
+          def call(cp)
+            return nil unless hangul_syllable?(cp)
+
+            s_index = cp - S_BASE
+            l_index = s_index / N_COUNT
+            v_index = (s_index % N_COUNT) / T_COUNT
+            t_index = s_index % T_COUNT
+
+            parts = [LEAD_SHORT_NAMES[l_index], VOWEL_SHORT_NAMES[v_index]]
+            parts << TRAIL_SHORT_NAMES[t_index] if t_index.positive?
+
+            "HANGUL SYLLABLE #{parts.join}"
+          end
+        end
+      end
+      private_constant :HangulName
+    end
+  end
+end

data/lib/ucode/parsers/unicode_data.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/codepoint"
+
+module Ucode
+  module Parsers
+    # Parses `UnicodeData.txt` — the primary per-codepoint record file.
+    #
+    # Field layout (UAX #44, 15 `;`-separated fields):
+    #   0.  codepoint
+    #   1.  name (`<control>` or `<Type, First>` / `<Type, Last>` for ranges)
+    #   2.  general_category
+    #   3.  canonical_combining_class
+    #   4.  bidi_class
+    #   5.  decomposition_type_and_mapping (combined: optional `<tag>` + cps)
+    #   6.  numeric_value_decimal (deprecated duplicate of 8 for Nd)
+    #   7.  numeric_value_digit    (deprecated duplicate of 8 for Nl)
+    #   8.  numeric_value          (canonical)
+    #   9.  bidi_mirrored (Y/N)
+    #   10. Unicode_1_Name         (deprecated, kept as `name1`)
+    #   11. ISO_10646_comment      (deprecated, ignored)
+    #   12. simple_uppercase_mapping
+    #   13. simple_lowercase_mapping
+    #   14. simple_titlecase_mapping
+    #
+    # Hangul syllables and CJK ideographs appear as range markers
+    # (`<..., First>` / `<..., Last>`). The range is expanded to one
+    # CodePoint per codepoint with the appropriate synthesized name.
+    class UnicodeData < Base
+      autoload :HangulName, "ucode/parsers/unicode_data/hangul_name"
+
+      FIRST_MARKER = "First"
+      LAST_MARKER = "Last"
+      private_constant :FIRST_MARKER, :LAST_MARKER
+
+      class << self
+        # Yields one CodePoint per codepoint in `path`. Range markers
+        # (`<..., First>` to `<..., Last>`) are expanded to one CodePoint
+        # per codepoint, with names synthesized per Unicode rules.
+        #
+        # Returns a lazy Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          pending_range = nil
+
+          each_line(path) do |line|
+            begin
+              fields = line.fields
+
+              if pending_range
+                unless fields[1]&.end_with?("#{LAST_MARKER}>")
+                  raise MalformedLineError.new(
+                    "expected <#{pending_range[:template]}, #{LAST_MARKER}>, " \
+                    "got #{fields[1].inspect}",
+                    context: { file: path.to_s, line: line.number }
+                  )
+                end
+
+                last_cp = parse_hex_cp(fields[0])
+                expand_range(pending_range, last_cp).each { |cp| yield cp }
+                pending_range = nil
+                next
+              end
+
+              cp = parse_hex_cp(fields[0])
+              name = fields[1]
+
+              if range_start?(name)
+                pending_range = {
+                  first_cp: cp,
+                  template: extract_template(name),
+                  general_category: fields[2],
+                  combining_class: fields[3].to_i,
+                  bidi_class: fields[4],
+                  bidi_mirrored: fields[9]
+                }
+                next
+              end
+
+              yield build_codepoint(
+                cp: cp,
+                name: synthesize_name(cp, name),
+                general_category: fields[2],
+                combining_class: fields[3].to_i,
+                bidi_class: fields[4],
+                decomposition_field: fields[5],
+                numeric_decimal: fields[6],
+                numeric_digit: fields[7],
+                numeric_value: fields[8],
+                bidi_mirrored: fields[9],
+                unicode_1_name: fields[10],
+                simple_upper_id: fields[12],
+                simple_lower_id: fields[13],
+                simple_title_id: fields[14]
+              )
+            rescue MalformedLineError => e
+              e.context[:file] ||= path.to_s
+              e.context[:line] ||= line.number
+              raise
+            end
+          end
+
+          nil
+        end
+
+        private
+
+        def range_start?(name)
+          name&.end_with?("#{FIRST_MARKER}>")
+        end
+
+        def extract_template(name)
+          name.delete_prefix("<").delete_suffix(", #{FIRST_MARKER}>")
+        end
+
+        # Synthesizes the official name for codepoints whose UnicodeData
+        # name is a placeholder. For `<control>` and other non-range
+        # placeholders the raw name is returned verbatim. For CJK and
+        # Hangul ranges the per-codepoint name is computed algorithmically.
+        def synthesize_name(cp, name)
+          case name
+          when "<control>" then "<control>"
+          when /\A<.*CJK.*>\z/
+            "CJK UNIFIED IDEOGRAPH-#{format("%04X", cp)}"
+          else
+            HangulName.call(cp) || name
+          end
+        end
+
+        # Expands a (first, last, template) range into one CodePoint per
+        # codepoint with the synthesized per-codepoint name.
+        def expand_range(range, last_cp)
+          first_cp = range[:first_cp]
+          Enumerator.new do |yielder|
+            first_cp.upto(last_cp) do |cp|
+              yielder << build_codepoint(
+                cp: cp,
+                name: synthesize_name(cp, "<#{range[:template]}, #{FIRST_MARKER}>"),
+                general_category: range[:general_category],
+                combining_class: range[:combining_class] || 0,
+                bidi_class: range[:bidi_class],
+                bidi_mirrored: range[:bidi_mirrored]
+              )
+            end
+          end
+        end
+
+        def build_codepoint(cp:, name:, general_category:, combining_class:,
+                            bidi_class:, decomposition_field: nil,
+                            numeric_decimal: nil, numeric_digit: nil, numeric_value: nil,
+                            bidi_mirrored: nil, unicode_1_name: nil,
+                            simple_upper_id: nil, simple_lower_id: nil, simple_title_id: nil)
+          Models::CodePoint.new(
+            cp: cp,
+            id: format("U+%04X", cp),
+            name: name,
+            name1: cp_or_nil(unicode_1_name),
+            general_category: general_category,
+            combining_class: combining_class.to_i,
+            bidi: build_bidi(bidi_class, bidi_mirrored),
+            decomposition: build_decomposition(decomposition_field),
+            numeric: build_numeric(general_category, numeric_decimal, numeric_digit, numeric_value),
+            casing: build_casing(simple_upper_id, simple_lower_id, simple_title_id)
+          )
+        end
+
+        def build_bidi(bidi_class, mirrored)
+          return nil if (bidi_class.nil? || bidi_class.empty?) &&
+                        (mirrored.nil? || mirrored.empty?)
+
+          Models::CodePoint::Bidi.new(
+            bidi_class: cp_or_nil(bidi_class),
+            is_mirrored: mirrored == "Y"
+          )
+        end
+
+        # Field 5 is a single combined field: optional `<tag>` prefix
+        # followed by space-separated codepoint hexes. No prefix means
+        # canonical decomposition (`can`).
+        def build_decomposition(combined)
+          return nil if combined.nil? || combined.empty?
+
+          type = "can"
+          mapping = combined
+
+          if combined.start_with?("<")
+            close = combined.index(">")
+            type = combined[1...close]
+            mapping = combined[(close + 1)..]
+          end
+
+          ids = mapping.split(/\s+/).reject(&:empty?).map do |hex|
+            format("U+%04X", parse_hex_cp(hex))
+          end
+
+          Models::CodePoint::Decomposition.new(
+            type: type,
+            codepoint_ids: ids
+          )
+        end
+
+        # Derives Numeric_Type from general_category (Nd/Nl/No) and uses
+        # field 8 as the canonical value. Fields 6 and 7 are deprecated
+        # duplicates of 8 for Nd and Nl respectively; they are consulted
+        # only as a fallback when field 8 is unexpectedly blank.
+        def build_numeric(gc, decimal_field, digit_field, numeric_field)
+          type = numeric_type_for_gc(gc)
+          return nil unless type
+
+          raw = [numeric_field, digit_field, decimal_field].find { |v| !v.nil? && !v.empty? }
+          return nil if raw.nil?
+
+          numerator, denominator = parse_numeric_value(raw)
+          Models::CodePoint::NumericValue.new(
+            type: type,
+            numerator: numerator,
+            denominator: denominator
+          )
+        end
+
+        def numeric_type_for_gc(gc)
+          case gc&.to_s
+          when /\ANd/ then "de"
+          when /\ANl/ then "di"
+          when /\ANo/ then "nu"
+          end
+        end
+
+        def parse_numeric_value(value)
+          if value.include?("/")
+            num, denom = value.split("/", 2)
+            [num.to_i, denom.to_i]
+          else
+            [value.to_i, 1]
+          end
+        end
+
+        def build_casing(upper_id, lower_id, title_id)
+          return nil if blank?(upper_id) && blank?(lower_id) && blank?(title_id)
+
+          Models::CodePoint::Casing.new(
+            simple_upper_id: cp_id(upper_id),
+            simple_lower_id: cp_id(lower_id),
+            simple_title_id: cp_id(title_id)
+          )
+        end
+
+        def cp_id(field)
+          return nil if blank?(field)
+
+          format("U+%04X", parse_hex_cp(field))
+        end
+
+        def cp_or_nil(field)
+          return nil if blank?(field)
+
+          field
+        end
+
+        def blank?(field)
+          field.nil? || field.empty?
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/unihan.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "ucode/parsers/base"
+require "ucode/error"
+
+module Ucode
+  module Parsers
+    # Parses all eight Unihan files (`Unihan_IRGSources.txt`,
+    # `Unihan_NumericValues.txt`, `Unihan_RadicalStrokeCounts.txt`,
+    # `Unihan_Readings.txt`, `Unihan_DictionaryIndices.txt`,
+    # `Unihan_DictionaryLikeData.txt`, `Unihan_Variants.txt`,
+    # `Unihan_OtherMappings.txt`).
+    #
+    # File format is uniform across all eight (Unihan documentation):
+    #
+    #   U+XXXX<TAB>kField<TAB>value
+    #
+    # The value may be a space-separated list (`kRSUnicode`, `kDefinition`
+    # for prose, `kCangjieInput` for multiple codes). `.split` (whitespace)
+    # produces the values array uniformly. Coordinator groups records by
+    # `cp` and writes into `CodePoint.unihan.fields[field]`.
+    #
+    # One parser, not eight: the format is uniform. The filename carries
+    # no parse-time information — every line is self-describing via its
+    # field name. Adding a new Unihan file is a one-line change to
+    # `FILES`; no parser modification (OCP).
+    class Unihan < Base
+      FILES = %w[
+        Unihan_DictionaryIndices.txt
+        Unihan_DictionaryLikeData.txt
+        Unihan_IRGSources.txt
+        Unihan_NumericValues.txt
+        Unihan_RadicalStrokeCounts.txt
+        Unihan_Readings.txt
+        Unihan_Variants.txt
+        Unihan_OtherMappings.txt
+      ].freeze
+
+      # Stream record: one Unihan line. Internal pipeline data — a Struct
+      # avoids lutaml-model ceremony for transient values. The final
+      # `UnihanEntry` model carries the merged, persisted shape. The
+      # member is `field_values` (not `values`) to avoid overriding
+      # `Struct#values` (the array of all member values).
+      Record = Struct.new(:cp, :field, :field_values, keyword_init: true) do
+        def cp_id
+          format("U+%04X", cp)
+        end
+      end
+
+      class << self
+        # Yields one Record per non-comment line in a single Unihan file.
+        # Returns a lazy Enumerator when no block is given.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          path_str = path.to_s
+          lineno = 0
+
+          File.foreach(path_str) do |raw|
+            lineno += 1
+            line = raw.chomp
+            next if line.empty? || line.start_with?("#")
+
+            begin
+              yield parse_line(line)
+            rescue MalformedLineError => e
+              e.context[:file] ||= path_str
+              e.context[:line] ||= lineno
+              raise
+            end
+          end
+
+          nil
+        end
+
+        # Iterates every known Unihan file in `dir`, yielding one Record
+        # per data line across all files. Missing files are silently
+        # skipped (incremental runs, partial downloads).
+        def each_in_dir(dir)
+          return enum_for(:each_in_dir, dir) unless block_given?
+
+          dir_path = Pathname.new(dir)
+          FILES.each do |filename|
+            path = dir_path.join(filename)
+            next unless path.exist?
+
+            each_record(path) { |record| yield record }
+          end
+
+          nil
+        end
+
+        private
+
+        # Parses one TAB-separated Unihan data line into a Record. The
+        # `split("\t", 3)` limit preserves any tabs inside the value
+        # (defensive — real Unihan data does not contain them).
+        def parse_line(line)
+          cp_str, field, value = line.split("\t", 3)
+          unless cp_str && field && value && !value.empty?
+            raise MalformedLineError.new(
+              "invalid Unihan line: #{line.inspect}",
+              context: { line: line }
+            )
+          end
+
+          cp_str = cp_str.strip
+          unless cp_str.start_with?("U+") && cp_str.length > 2
+            raise MalformedLineError.new(
+              "invalid Unihan codepoint: #{cp_str.inspect}",
+              context: { cp: cp_str }
+            )
+          end
+
+          Record.new(
+            cp: parse_hex_cp(cp_str[2..]),
+            field: field.strip,
+            field_values: value.strip.split
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Ucode
+  # Parsers — one class per UCD text file.
+  #
+  # All parsers stream: they read line by line via `File.foreach`, never
+  # accumulate the whole file in memory, and yield one record at a time.
+  # When called without a block, they return a lazy Enumerator so the
+  # Coordinator can compose them.
+  module Parsers
+    autoload :Base, "ucode/parsers/base"
+    autoload :UnicodeData, "ucode/parsers/unicode_data"
+    autoload :Blocks, "ucode/parsers/blocks"
+    autoload :Scripts, "ucode/parsers/scripts"
+    autoload :ScriptExtensions, "ucode/parsers/script_extensions"
+    autoload :PropertyAliases, "ucode/parsers/property_aliases"
+    autoload :PropertyValueAliases, "ucode/parsers/property_value_aliases"
+    autoload :NameAliases, "ucode/parsers/name_aliases"
+    autoload :NamedSequences, "ucode/parsers/named_sequences"
+    autoload :SpecialCasing, "ucode/parsers/special_casing"
+    autoload :CaseFolding, "ucode/parsers/case_folding"
+    autoload :BidiMirroring, "ucode/parsers/bidi_mirroring"
+    autoload :BidiBrackets, "ucode/parsers/bidi_brackets"
+    autoload :CjkRadicals, "ucode/parsers/cjk_radicals"
+    autoload :StandardizedVariants, "ucode/parsers/standardized_variants"
+    autoload :NamesList, "ucode/parsers/names_list"
+    autoload :DerivedAge, "ucode/parsers/derived_age"
+    autoload :DerivedCoreProperties, "ucode/parsers/derived_core_properties"
+    autoload :ExtractedProperties, "ucode/parsers/extracted_properties"
+    autoload :Auxiliary, "ucode/parsers/auxiliary"
+    autoload :Unihan, "ucode/parsers/unihan"
+  end
+end

data/lib/ucode/range_entry.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Ucode
+  # Value object representing one row in a run-length-encoded UCD index.
+  #
+  # Sorted by `first_cp`. Entries within a single Index are disjoint (no
+  # overlapping ranges). This is a leaf value object — not a
+  # `Lutaml::Model::Serializable` model — because it has no wire shape,
+  # no nested types, and is consumed only by the YAML-backed Index. The
+  # `to_h` / `from_h` pair below is the deliberate serialization contract
+  # for the YAML file format and is exempt from the no-to_h rule by
+  # design (that rule covers model classes only).
+  class RangeEntry
+    include Comparable
+
+    attr_reader :first_cp, :last_cp, :name
+
+    def initialize(first_cp, last_cp, name)
+      @first_cp = first_cp
+      @last_cp = last_cp
+      @name = name
+    end
+
+    def covers?(codepoint)
+      codepoint >= @first_cp && codepoint <= @last_cp
+    end
+
+    def size
+      @last_cp - @first_cp + 1
+    end
+
+    def <=>(other)
+      [@first_cp, @last_cp] <=> [other.first_cp, other.last_cp]
+    end
+
+    def ==(other)
+      other.is_a?(RangeEntry) &&
+        @first_cp == other.first_cp &&
+        @last_cp == other.last_cp &&
+        @name == other.name
+    end
+    alias eql? ==
+
+    def hash
+      [@first_cp, @last_cp, @name].hash
+    end
+
+    def to_h
+      { first_cp: @first_cp, last_cp: @last_cp, name: @name }
+    end
+
+    def self.from_h(hash)
+      new(hash[:first_cp] || hash["first_cp"],
+          hash[:last_cp] || hash["last_cp"],
+          hash[:name] || hash["name"])
+    end
+  end
+end