ucode 0.1.0

data/lib/ucode/models/unihan_entry.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # Unihan dictionary data for CJK codepoints. Flat-hash design: every
+    # `kFoo` field is a key in `fields`, with array values (Unihan fields
+    # are space-separated lists; uniform arrays simplify the shape).
+    #
+    # The semantic grouping (readings / radicals / variants / sources / etc.)
+    # is a presentation concern, derived client-side by prefix. The data
+    # model stays open — Unihan adds fields across versions, and the hash
+    # absorbs additions without model changes.
+    class UnihanEntry < Lutaml::Model::Serializable
+      attribute :fields, :hash, default: -> { {} }
+
+      key_value do
+        map "fields", to: :fields
+      end
+    end
+  end
+end

data/lib/ucode/models.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Ucode
+  # Models — typed class representations of every UCD aggregate.
+  #
+  # Conventions (apply to every class in this namespace):
+  #
+  # - Inheritance, not include:
+  #
+  #     class Foo < Lutaml::Model::Serializable
+  #
+  # - Wire shape declared via `key_value do … end` (covers JSON + YAML).
+  #   NEVER `mapping do`, NEVER `json do`.
+  #
+  # - Codepoint references are "U+XXXX" strings — never nested CodePoint
+  #   objects. Keeps the data normalized: each codepoint's full data lives
+  #   only in its own folder.
+  #
+  # - Polymorphism: `polymorphic_class: true` + `polymorphic_map:` on the
+  #   base discriminator; `polymorphic: [...]` on the consumer attribute
+  #    + `polymorphic: { attribute:, class_map: }` on its mapping.
+  #
+  # - NEVER define `to_h` / `from_h` / `to_json` / `from_json`. All
+  #   (de)serialization goes through lutaml-model.
+  #
+  module Models
+    autoload :PropertyAlias, "ucode/models/property_alias"
+    autoload :PropertyValueAlias, "ucode/models/property_value_alias"
+    autoload :Plane, "ucode/models/plane"
+    autoload :Block, "ucode/models/block"
+    autoload :Script, "ucode/models/script"
+    autoload :CodePoint, "ucode/models/codepoint"
+    autoload :UnihanEntry, "ucode/models/unihan_entry"
+    autoload :NamesListEntry, "ucode/models/names_list_entry"
+    autoload :NameAlias, "ucode/models/name_alias"
+    autoload :NamedSequence, "ucode/models/named_sequence"
+    autoload :SpecialCasingRule, "ucode/models/special_casing_rule"
+    autoload :CaseFoldingRule, "ucode/models/case_folding_rule"
+    autoload :BidiMirroring, "ucode/models/bidi_mirroring"
+    autoload :BidiBracketPair, "ucode/models/bidi_bracket_pair"
+    autoload :CjkRadical, "ucode/models/cjk_radical"
+    autoload :StandardizedVariant, "ucode/models/standardized_variant"
+    autoload :BinaryPropertyAssignment, "ucode/models/binary_property_assignment"
+    autoload :Relationship, "ucode/models/relationship"
+    autoload :Audit, "ucode/models/audit"
+  end
+end

data/lib/ucode/parsers/auxiliary.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/extracted_properties"
+
+module Ucode
+  module Parsers
+    # Generic range/value parser for the auxiliary segmentation files
+    # under `auxiliary/` (GraphemeBreakProperty, WordBreakProperty,
+    # SentenceBreakProperty, VerticalOrientation, IndicPositionalCategory,
+    # IndicSyllabicCategory, IdentifierStatus, IdentifierType) plus the
+    # top-level `LineBreak.txt` and `EastAsianWidth.txt`.
+    #
+    # File format is identical to ExtractedProperties (UAX #44 range/value):
+    #
+    #   XXXX..YYYY; value
+    #   XXXX; value
+    #
+    # Coordinator dispatches by file name to the right CodePoint
+    # attribute. This class exists as a distinct name so call sites read
+    # "auxiliary" instead of "extracted" — the parsing logic is shared
+    # via inheritance. Adding auxiliary-specific behavior later does not
+    # require touching ExtractedProperties (OCP).
+    class Auxiliary < ExtractedProperties
+    end
+  end
+end

data/lib/ucode/parsers/base.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "ucode/error"
+
+module Ucode
+  module Parsers
+    # Shared infrastructure for every UCD text-file parser. Subclasses
+    # implement `.each_record(path) { |record| ... }` returning an
+    # Enumerator when called without a block.
+    #
+    # All methods are class methods — parsers are stateless.
+    #
+    # UCD text-file format (UAX #44):
+    #   - Fields separated by `;`
+    #   - Lines starting with `#` are comments
+    #   - Blank lines are ignored
+    #   - Some lines carry an inline `# trailing comment` after the data
+    class Base
+      # One physical line from the source file, post-filter (blanks and
+      # comment-only lines are skipped before yielding).
+      Line = Struct.new(:number, :text, :comment, keyword_init: true) do
+        # Returns the data part of the line — everything before the first
+        # `#`, rstripped. For lines with no comment this is just the text.
+        def data
+          idx = text.index("#")
+          idx.nil? ? text : text[0...idx].rstrip
+        end
+
+        # Splits the data part on `;` into stripped fields.
+        def fields
+          data.split(";").map(&:strip)
+        end
+
+        # Returns the n-th field (0-based), or nil if out of range.
+        def field(n)
+          fields[n]
+        end
+      end
+
+      HEX_PATTERN = /\A[0-9A-Fa-f]{1,6}\z/.freeze
+      private_constant :HEX_PATTERN
+
+      RANGE_SEPARATOR = ".."
+      private_constant :RANGE_SEPARATOR
+
+      class << self
+        # Iterates non-blank, non-comment lines from `path`, yielding Line
+        # records. Returns an Enumerator when no block is given so callers
+        # can chain (`.first(n)`, `.lazy.map`, etc.).
+        #
+        # Lines that are entirely whitespace or start with `#` are skipped
+        # silently — comment text is preserved on data lines that carry an
+        # inline `# trailing comment`.
+        def each_line(path)
+          return enum_for(:each_line, path) unless block_given?
+
+          lineno = 0
+          File.foreach(path.to_s) do |raw|
+            lineno += 1
+            stripped = raw.strip
+            next if stripped.empty?
+            next if stripped.start_with?("#")
+
+            yield build_line(lineno, raw)
+          end
+        end
+
+        # Parses an n-th `;`-separated field from a line of text or a Line
+        # struct. Strips surrounding whitespace. Returns nil if the field
+        # is missing or out of range.
+        def parse_field(line, n)
+          fields = line_fields(line)
+          return nil if fields.length <= n
+
+          fields[n]
+        end
+
+        # Parses a codepoint-or-range field per UAX #44. Accepts:
+        #   "0041"           → 0x0041 (Integer)
+        #   "3400..4DBF"     → 0x3400..0x4DBF (Range)
+        #
+        # Returns nil for blank input. Raises Ucode::MalformedLineError
+        # for invalid hex.
+        def parse_codepoint_or_range(field)
+          return nil if field.nil? || field.empty?
+
+          if field.include?(RANGE_SEPARATOR)
+            first_str, last_str = field.split(RANGE_SEPARATOR, 2)
+            first = parse_hex_cp(first_str)
+            last = parse_hex_cp(last_str)
+            Range.new(first, last)
+          else
+            parse_hex_cp(field)
+          end
+        end
+
+        # Parses a single hex codepoint string into an Integer. Raises
+        # Ucode::MalformedLineError with the offending input in context
+        # for invalid input.
+        def parse_hex_cp(input)
+          s = input.to_s.strip
+          unless s.match?(HEX_PATTERN)
+            raise MalformedLineError.new(
+              "invalid codepoint: #{input.inspect}",
+              context: { input: input }
+            )
+          end
+          s.to_i(16)
+        end
+
+        private
+
+        # Builds a Line struct from a raw text line. Splits off any
+        # trailing `# comment` into the Line's `comment` field.
+        def build_line(number, raw)
+          text = raw.chomp
+          hash_idx = text.index("#")
+
+          if hash_idx.nil?
+            Line.new(number: number, text: text, comment: nil)
+          else
+            Line.new(
+              number: number,
+              text: text[0...hash_idx].rstrip,
+              comment: text[(hash_idx + 1)..].strip
+            )
+          end
+        end
+
+        def line_fields(line)
+          data = line.is_a?(Line) ? line.data : line.to_s
+          data.split(";").map(&:strip)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/bidi_brackets.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/bidi_bracket_pair"
+
+module Ucode
+  module Parsers
+    # Parses `BidiBrackets.txt` — paired bracket partners.
+    #
+    # Format (UAX #44):
+    #   cp; paired_cp; type
+    #
+    # `type` is `o` (open) or `c` (close). Coordinator merges each row
+    # into `CodePoint#bidi.paired_bracket_id` and `.paired_bracket_type`.
+    class BidiBrackets < Base
+      class << self
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 3
+
+            cp = parse_hex_cp(fields[0])
+            paired_cp = parse_hex_cp(fields[1])
+            type = fields[2]
+            next if type.nil? || type.empty?
+
+            yield Models::BidiBracketPair.new(
+              codepoint: cp,
+              paired_id: format("U+%04X", paired_cp),
+              type: type
+            )
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/bidi_mirroring.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/bidi_mirroring"
+
+module Ucode
+  module Parsers
+    # Parses `BidiMirroring.txt` — the bidi mirroring glyph partner.
+    #
+    # Format (UAX #44):
+    #   cp; mirrored_cp
+    #
+    # Coordinator merges each row into `CodePoint#bidi.mirroring_glyph_id`.
+    class BidiMirroring < Base
+      class << self
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            cp = parse_hex_cp(fields[0])
+            mirrored_cp = parse_hex_cp(fields[1])
+
+            yield Models::BidiMirroring.new(
+              codepoint: cp,
+              mirrored_id: format("U+%04X", mirrored_cp)
+            )
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/blocks.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/block"
+
+module Ucode
+  module Parsers
+    # Parses `Blocks.txt` — one block range per line.
+    #
+    # Format (UAX #44):
+    #   XXXX..XXXX; Block Name
+    #
+    # The `id` is the block name with runs of whitespace collapsed to a
+    # single underscore. The `name` is preserved verbatim. Per the
+    # project rules (CLAUDE.md), block names are NOT otherwise slugified.
+    #
+    # `plane_number` is derived from the high bits of `range_first`.
+    class Blocks < Base
+      class << self
+        # Yields one Block per non-comment line. Returns a lazy
+        # Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            range_field = fields[0]
+            name = fields[1]
+            next if name.nil? || name.empty?
+
+            range = parse_codepoint_or_range(range_field)
+            yield build_block(range, name)
+          end
+
+          nil
+        end
+
+        private
+
+        def build_block(range, name)
+          first, last = bounds_of(range)
+          Models::Block.new(
+            id: name.gsub(/\s+/, "_"),
+            name: name,
+            range_first: first,
+            range_last: last,
+            plane_number: first >> 16
+          )
+        end
+
+        def bounds_of(range)
+          if range.is_a?(Range)
+            [range.begin, range.end]
+          else
+            [range, range]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/case_folding.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/case_folding_rule"
+
+module Ucode
+  module Parsers
+    # Parses `CaseFolding.txt` — case folding mappings for comparison.
+    #
+    # Format (UAX #44):
+    #   cp; status; mapping; # name
+    #
+    # `status` is one of: C (common), F (full), S (simple), T (turkic).
+    # `mapping` is one or more space-separated hex codepoints.
+    class CaseFolding < Base
+      class << self
+        # Yields one CaseFoldingRule per non-comment line. Returns a lazy
+        # Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 3
+
+            cp = parse_hex_cp(fields[0])
+            status = fields[1]
+            next if status.nil? || status.empty?
+
+            yield Models::CaseFoldingRule.new(
+              codepoint: cp,
+              status: status,
+              mapping_ids: parse_mapping(fields[2]),
+              comment: line.comment
+            )
+          end
+
+          nil
+        end
+
+        private
+
+        def parse_mapping(field)
+          return [] if field.nil? || field.empty?
+
+          field.split(/\s+/).reject(&:empty?).map do |hex|
+            format("U+%04X", parse_hex_cp(hex))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/cjk_radicals.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/cjk_radical"
+
+module Ucode
+  module Parsers
+    # Parses `CJKRadicals.txt` — KangXi radical → CJK radical ideograph
+    # → canonical ideograph mapping.
+    #
+    # Format (UAX #44):
+    #   radical_number; cjk_radical; ideograph
+    #
+    # `cjk_radical` and `ideograph` are either a single hex codepoint
+    # (`2F00`) or a range in the form `XXXX..YYYY`. Range rows are
+    # expanded to one CjkRadical per codepoint.
+    #
+    # Coordinator merges each row into the relevant CodePoint.
+    class CjkRadicals < Base
+      class << self
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 3
+
+            radical_number = parse_radical_number(fields[0])
+            cjk_radical_field = fields[1]
+            ideograph_field = fields[2]
+            next if radical_number.nil?
+
+            yield_models(radical_number, cjk_radical_field, ideograph_field).each do |model|
+              yield model
+            end
+          end
+
+          nil
+        end
+
+        private
+
+        # The radical number is a positive integer; some rows carry a
+        # trailing comment-stripped form. Reject anything non-numeric.
+        def parse_radical_number(field)
+          return nil if field.nil? || field.empty?
+
+          Integer(field, exception: false)
+        end
+
+        def yield_models(radical_number, cjk_radical_field, ideograph_field)
+          cjk_ids = expand_ids(cjk_radical_field)
+          ideograph_ids = expand_ids(ideograph_field)
+
+          if cjk_ids.size == 1 && ideograph_ids.size == 1
+            return [Models::CjkRadical.new(
+              radical_number: radical_number,
+              cjk_radical_id: cjk_ids.first,
+              ideograph_id: ideograph_ids.first
+            )]
+          end
+
+          if cjk_ids.size == 1 && ideograph_ids.size > 1
+            return ideograph_ids.map do |ideograph_id|
+              Models::CjkRadical.new(
+                radical_number: radical_number,
+                cjk_radical_id: cjk_ids.first,
+                ideograph_id: ideograph_id
+              )
+            end
+          end
+
+          if cjk_ids.size > 1 && ideograph_ids.size == 1
+            return cjk_ids.map do |cjk_radical_id|
+              Models::CjkRadical.new(
+                radical_number: radical_number,
+                cjk_radical_id: cjk_radical_id,
+                ideograph_id: ideograph_ids.first
+              )
+            end
+          end
+
+          cjk_ids.zip(ideograph_ids).map do |cjk_id, ideograph_id|
+            Models::CjkRadical.new(
+              radical_number: radical_number,
+              cjk_radical_id: cjk_id,
+              ideograph_id: ideograph_id
+            )
+          end
+        end
+
+        def expand_ids(field)
+          return [] if field.nil? || field.empty?
+
+          range = parse_codepoint_or_range(field)
+          cps = range.is_a?(Range) ? range.to_a : [range]
+          cps.map { |cp| format("U+%04X", cp) }
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/derived_age.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+
+module Ucode
+  module Parsers
+    # Parses `DerivedAge.txt` — the Unicode version in which each
+    # codepoint was first assigned.
+    #
+    # Format (UAX #44):
+    #   XXXX..YYYY; M.N
+    #   XXXX; M.N
+    #
+    # The age is a Unicode version string like "1.1", "5.2", "15.0".
+    # Coordinator merges each row into `CodePoint#age`.
+    #
+    # Ranges are expanded per-codepoint (one Tuple per cp) because the
+    # Coordinator needs per-cp assignment for `CodePoint#age`.
+    class DerivedAge < Base
+      # Lightweight record yielded by `.each_record`. Models are
+      # heavyweight for stream-only data — the Coordinator consumes
+      # these immediately.
+      Tuple = Struct.new(:cp, :age, keyword_init: true) do
+        def cp_id
+          format("U+%04X", cp)
+        end
+      end
+
+      class << self
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            range = parse_codepoint_or_range(fields[0])
+            age = fields[1]
+            next if age.nil? || age.empty?
+
+            each_cp(range) { |cp| yield Tuple.new(cp: cp, age: age) }
+          end
+
+          nil
+        end
+
+        private
+
+        def each_cp(range)
+          if range.is_a?(Range)
+            range.each { |cp| yield cp }
+          else
+            yield range
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/derived_core_properties.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/binary_property_assignment"
+
+module Ucode
+  module Parsers
+    # Parses `DerivedCoreProperties.txt` — derived binary properties
+    # (Alphabetic, Uppercase, White_Space, Bidi_Control, …).
+    #
+    # Format (UAX #44):
+    #   XXXX..YYYY; Property_Name
+    #   XXXX; Property_Name
+    #
+    # The file only lists positive assignments; absence means the
+    # property is false. Each yielded `BinaryPropertyAssignment` has
+    # `enabled: true`.
+    #
+    # Coordinator appends each `property_short` (resolved to the long
+    # form via PropertyAliases if needed) to `CodePoint#binary_properties`.
+    class DerivedCoreProperties < Base
+      class << self
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            range = parse_codepoint_or_range(fields[0])
+            property = fields[1]
+            next if property.nil? || property.empty?
+
+            each_cp(range) { |cp| yield build_assignment(cp, property) }
+          end
+
+          nil
+        end
+
+        private
+
+        def each_cp(range)
+          if range.is_a?(Range)
+            range.each { |cp| yield cp }
+          else
+            yield range
+          end
+        end
+
+        def build_assignment(cp, property)
+          Models::BinaryPropertyAssignment.new(
+            codepoint: cp,
+            property_short: property,
+            enabled: true
+          )
+        end
+      end
+    end
+  end
+end