ucode 0.1.0

data/lib/ucode/parsers/extracted_properties.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+
+module Ucode
+  module Parsers
+    # Generic range/value parser for the files under `extracted/`
+    # (DerivedGeneralCategory, DerivedJoiningGroup, DerivedLineBreak,
+    # DerivedNumericType, …).
+    #
+    # Format is uniform across every file (UAX #44):
+    #   XXXX..YYYY; value
+    #   XXXX; value
+    #
+    # The parser is intentionally dumb: it yields `(first, last, value)`
+    # triples without knowing what the value means. The Coordinator
+    # dispatches by source file name (DerivedGeneralCategory.txt →
+    # CodePoint#general_category, etc.). This decoupling means a new
+    # extracted file adds one line to the Coordinator, not a new parser.
+    #
+    # Ranges are NOT expanded — yielding per-codepoint would explode the
+    # stream for CJK ranges. The Coordinator expands lazily if needed.
+    class ExtractedProperties < Base
+      # Lightweight record yielded by `.each_record`. The Coordinator
+      # consumes these immediately; no need for full lutaml-model
+      # overhead. Members are named `range_first` / `range_last` (not
+      # `first` / `last`) to avoid overriding `Enumerable#first`.
+      Tuple = Struct.new(:range_first, :range_last, :value, keyword_init: true) do
+        # The inclusive Range of codepoints this assignment covers.
+        def range
+          Range.new(range_first, range_last)
+        end
+
+        # Enumerator over every codepoint id in this tuple's range.
+        def cp_ids
+          (range_first..range_last).map { |cp| format("U+%04X", cp) }
+        end
+
+        def single?
+          range_first == range_last
+        end
+      end
+
+      class << self
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            range = parse_codepoint_or_range(fields[0])
+            value = fields[1]
+            next if value.nil? || value.empty?
+
+            yield build_tuple(range, value)
+          end
+
+          nil
+        end
+
+        private
+
+        def build_tuple(range, value)
+          if range.is_a?(Range)
+            Tuple.new(range_first: range.first, range_last: range.last, value: value)
+          else
+            Tuple.new(range_first: range, range_last: range, value: value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/name_aliases.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/name_alias"
+
+module Ucode
+  module Parsers
+    # Parses `NameAliases.txt` — alternate / correction / control names
+    # attached to a codepoint.
+    #
+    # Format (UAX #44):
+    #   cp; alias_text; type
+    #
+    # `type` is one of: correction, control, alternate, figment,
+    # abbreviation.
+    class NameAliases < Base
+      class << self
+        # Yields one NameAlias per non-comment line. Returns a lazy
+        # Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 3
+
+            cp = parse_hex_cp(fields[0])
+            text = fields[1]
+            type = fields[2]
+            next if text.nil? || text.empty? || type.nil? || type.empty?
+
+            yield Models::NameAlias.new(
+              codepoint: cp,
+              text: text,
+              type: type
+            )
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/named_sequences.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/named_sequence"
+
+module Ucode
+  module Parsers
+    # Parses `NamedSequences.txt` — named multi-codepoint sequences.
+    #
+    # Format (UAX #44):
+    #   cp1 cp2 cp3 ...; Name
+    #
+    # The first field is a space-separated list of hex codepoints; the
+    # second is the human-readable name.
+    class NamedSequences < Base
+      class << self
+        # Yields one NamedSequence per non-comment line. Returns a lazy
+        # Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            sequence_field = fields[0]
+            name = fields[1]
+            next if name.nil? || name.empty?
+
+            yield Models::NamedSequence.new(
+              name: name,
+              codepoint_ids: parse_sequence(sequence_field)
+            )
+          end
+
+          nil
+        end
+
+        private
+
+        def parse_sequence(field)
+          return [] if field.nil? || field.empty?
+
+          field.split(/\s+/).reject(&:empty?).map do |hex|
+            format("U+%04X", parse_hex_cp(hex))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/names_list.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/error"
+require "ucode/models/names_list_entry"
+require "ucode/models/relationship"
+
+module Ucode
+  module Parsers
+    # Parses `NamesList.txt` — the human-curated annotated names file
+    # Unicode uses to render the Code Charts' name pages.
+    #
+    # Format (per the file's own header):
+    #
+    #   cp; Name            ← header line at column 0 → new NamesListEntry
+    #     → U+XXXX note    ← indented annotation lines
+    #     × U+XXXX U+YYYY note
+    #     ≡ U+XXXX note
+    #     = alias text
+    #     * footnote text
+    #
+    # Plus dropped lines:
+    #
+    #   `# comment`         ← file-level comment
+    #   `% instruction`     ← dropped (instructional)
+    #   `~ heading`         ← dropped (table-of-contents)
+    #
+    # Annotation scopes attach to the most recent header. Lines that do
+    # not start a new header are silently ignored.
+    #
+    # Implemented as a small state machine: one current NamesListEntry is
+    # held in a local; header lines flush the previous entry, annotation
+    # lines append to the current entry. Regex cannot express this
+    # scoping.
+    class NamesList < Base
+      HEADER_PATTERN = /\A([0-9A-Fa-f]{4,6})\s*;\s*(.+?)\s*\z/.freeze
+      private_constant :HEADER_PATTERN
+
+      CP_REF_PATTERN = /\AU\+([0-9A-Fa-f]{4,6})\b/.freeze
+      private_constant :CP_REF_PATTERN
+
+      RENDERED_PATTERN = /\(rendered:\s*(.+?)\)\z/.freeze
+      private_constant :RENDERED_PATTERN
+
+      MARKER_CROSS_REFERENCE = "→".freeze
+      MARKER_SAMPLE_SEQUENCE = "×".freeze
+      MARKER_COMPAT_EQUIV    = "≡".freeze
+      MARKER_ALIAS           = "=".freeze
+      MARKER_FOOTNOTE        = "*".freeze
+      MARKER_INSTRUCTIONAL   = "%".freeze
+      MARKER_HEADING         = "~".freeze
+
+      SOURCE_TAG = "names_list".freeze
+      private_constant :SOURCE_TAG
+
+      class << self
+        # Yields one NamesListEntry per codepoint header. Returns a lazy
+        # Enumerator when no block is given.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          entry = nil
+          lineno = 0
+          path_str = path.to_s
+
+          File.foreach(path_str) do |raw|
+            lineno += 1
+            line = raw.chomp
+
+            begin
+              if header_line?(line)
+                yield entry if entry
+                entry = build_header(line)
+              elsif indented_line?(line) && entry
+                parsed = parse_annotation(line)
+                attach_annotation(entry, parsed) if parsed
+              end
+              # else: blank, comment, heading, or pre-header — skip
+            rescue MalformedLineError => e
+              e.context[:file] ||= path_str
+              e.context[:line] ||= lineno
+              raise
+            end
+          end
+
+          yield entry if entry
+          nil
+        end
+
+        private
+
+        # Column-0 line whose first non-blank char is a hex digit and
+        # which carries the `;` separator. Excludes `%`, `~`, `#`.
+        def header_line?(line)
+          return false if line.empty?
+          return false if line.start_with?("#", "%", "~", "@")
+
+          !line.match(HEADER_PATTERN).nil?
+        end
+
+        # Indented annotation: column 0 is whitespace and the line is
+        # non-empty.
+        def indented_line?(line)
+          return false if line.empty?
+
+          line[0] == " " || line[0] == "\t"
+        end
+
+        def build_header(line)
+          m = line.match(HEADER_PATTERN)
+          unless m
+            raise MalformedLineError.new(
+              "invalid NamesList.txt header: #{line.inspect}",
+              context: { line: line }
+            )
+          end
+
+          Models::NamesListEntry.new(
+            codepoint: m[1].to_i(16),
+            name: m[2]
+          )
+        end
+
+        # Parses one indented annotation line. Returns a
+        # `[container_attribute, Relationship]` pair, or `nil` if the
+        # marker is dropped (`%`, `~`) or unknown.
+        def parse_annotation(line)
+          stripped = line.lstrip
+          marker = stripped[0]
+          rest = stripped[1..].to_s.lstrip
+
+          case marker
+          when MARKER_CROSS_REFERENCE
+            target_ids, note = split_targets_and_note(rest)
+            [
+              :cross_references,
+              build_cross_reference(target_ids, note),
+            ]
+          when MARKER_SAMPLE_SEQUENCE
+            target_ids, note = split_targets_and_note(rest)
+            [
+              :sample_sequences,
+              build_sample_sequence(target_ids, note),
+            ]
+          when MARKER_COMPAT_EQUIV
+            target_ids, note = split_targets_and_note(rest)
+            [
+              :compatibility_equivalents,
+              build_compat_equiv(target_ids, note),
+            ]
+          when MARKER_ALIAS
+            [:informal_aliases, build_alias(rest)]
+          when MARKER_FOOTNOTE
+            [:footnotes, build_footnote(rest)]
+          when MARKER_INSTRUCTIONAL, MARKER_HEADING
+            nil
+          else
+            nil
+          end
+        end
+
+        def build_cross_reference(target_ids, note)
+          Models::Relationship::CrossReference.new(
+            target_ids: target_ids,
+            description: note.empty? ? nil : note,
+            source: SOURCE_TAG
+          )
+        end
+
+        def build_sample_sequence(target_ids, note)
+          rendered = extract_rendered(note)
+          Models::Relationship::SampleSequence.new(
+            target_ids: target_ids,
+            description: note.empty? ? nil : note,
+            rendered_form: rendered,
+            source: SOURCE_TAG
+          )
+        end
+
+        def build_compat_equiv(target_ids, note)
+          Models::Relationship::CompatEquiv.new(
+            target_ids: target_ids,
+            description: note.empty? ? nil : note,
+            source: SOURCE_TAG
+          )
+        end
+
+        def build_alias(text)
+          Models::Relationship::InformalAlias.new(
+            description: text.empty? ? nil : text,
+            source: SOURCE_TAG
+          )
+        end
+
+        def build_footnote(text)
+          Models::Relationship::Footnote.new(
+            description: text.empty? ? nil : text,
+            category: detect_footnote_category(text),
+            source: SOURCE_TAG
+          )
+        end
+
+        # Splits a `U+XXXX [U+YYYY ...] note` payload into leading target
+        # ids (zero-padded `U+XXXX` form) and the trailing prose note.
+        def split_targets_and_note(rest)
+          targets = []
+          remaining = rest.dup
+
+          while (m = remaining.match(CP_REF_PATTERN))
+            targets << format("U+%04X", m[1].to_i(16))
+            remaining = remaining[m[0].length..].to_s.lstrip
+          end
+
+          [targets, remaining]
+        end
+
+        # Pulls `(rendered: X)` suffix from sample-sequence notes when
+        # present. Returns nil otherwise.
+        def extract_rendered(note)
+          m = note.match(RENDERED_PATTERN)
+          return nil unless m
+
+          m[1].strip
+        end
+
+        # Heuristic footnote category. The Unicode names list does not
+        # tag these explicitly; the categories are useful for UI grouping.
+        def detect_footnote_category(text)
+          first = text.split(/\s+/, 2).first&.downcase
+          case first
+          when "cap", "capital", "small", "lowercase", "uppercase",
+               "letter", "letterform", "glyph", "shape"
+            "letterform"
+          when "see", "compare", "vs", "versus", "distinguished"
+            "comparison"
+          when "history", "origin", "originally", "introduced"
+            "history"
+          else
+            "general"
+          end
+        end
+
+        def attach_annotation(entry, parsed)
+          attr_name, instance = parsed
+          entry.public_send(attr_name) << instance
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/property_aliases.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/property_alias"
+
+module Ucode
+  module Parsers
+    # Parses `PropertyAliases.txt` — Unicode property short ↔ long name.
+    #
+    # Format (UAX #44):
+    #   short; long_name; other_alias; other_alias; ...
+    #
+    # Example: `ccc; Canonical_Combining_Class; ccc`
+    class PropertyAliases < Base
+      class << self
+        # Yields one PropertyAlias per non-comment line. Returns a lazy
+        # Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            short = fields[0]
+            long = fields[1]
+            others = fields[2..].reject { |f| f.nil? || f.empty? }
+
+            yield Models::PropertyAlias.new(
+              short: short,
+              long: long,
+              other_aliases: others
+            )
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/property_value_aliases.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/property_value_alias"
+
+module Ucode
+  module Parsers
+    # Parses `PropertyValueAliases.txt` — per-property value aliases.
+    #
+    # Format (UAX #44):
+    #   property; short_value; long_value; other_alias; ...
+    #
+    # Examples:
+    #   gc; Lu; Uppercase_Letter
+    #   sc; Latn; Latin
+    #   ccc; 0; NR
+    class PropertyValueAliases < Base
+      class << self
+        # Yields one PropertyValueAlias per non-comment line. Returns a
+        # lazy Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 3
+
+            property = fields[0]
+            short = fields[1]
+            long = fields[2]
+            others = fields[3..].reject { |f| f.nil? || f.empty? }
+
+            yield Models::PropertyValueAlias.new(
+              property: property,
+              short: short,
+              long: long,
+              other_aliases: others
+            )
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/script_extensions.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+
+module Ucode
+  module Parsers
+    # Parses `ScriptExtensions.txt` — additional scripts per codepoint.
+    #
+    # Format (UAX #44):
+    #   XXXX..XXXX ; Latn Grek Cyrl  # trailing comment
+    #
+    # A codepoint can be associated with many scripts. The parser yields
+    # one Tuple per (codepoint, script_code) pair; the Coordinator merges
+    # these into CodePoint#script_extensions.
+    #
+    # `script_code` is the ISO 15924 4-letter code already present in the
+    # source file (e.g. `Latn`, `Grek`). No alias resolution is needed.
+    class ScriptExtensions < Base
+      # One (codepoint, ISO 15924 code) pair yielded by `.each_record`.
+      Tuple = Struct.new(:cp, :script_code, keyword_init: true) do
+        def cp_id
+          format("U+%04X", cp)
+        end
+      end
+
+      class << self
+        # Yields one Tuple per (codepoint, script_code) pair. Returns a
+        # lazy Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            codes_field = fields[1]
+            next if codes_field.nil? || codes_field.empty?
+
+            range = parse_codepoint_or_range(fields[0])
+            codes = codes_field.split(/\s+/)
+
+            each_cp(range) do |cp|
+              codes.each do |code|
+                yield Tuple.new(cp: cp, script_code: code)
+              end
+            end
+          end
+
+          nil
+        end
+
+        private
+
+        def each_cp(range)
+          if range.is_a?(Range)
+            range.each { |cp| yield cp }
+          else
+            yield range
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/scripts.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/script"
+
+module Ucode
+  module Parsers
+    # Parses `Scripts.txt` — the primary Script property assignment per
+    # codepoint range.
+    #
+    # Format (UAX #44):
+    #   XXXX..XXXX ; Script_Name # trailing comment
+    #   XXXX       ; Script_Name # trailing comment
+    #
+    # Yields one Script per line, with `range_first` and `range_last`
+    # set. The Coordinator bsearches the resulting sorted array by cp.
+    # The ISO 15924 `code` is resolved later by the Coordinator via
+    # PropertyValueAliases (property=sc).
+    class Scripts < Base
+      class << self
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 2
+
+            name = fields[1]
+            next if name.nil? || name.empty?
+            next if name == "@missing"
+
+            range = parse_codepoint_or_range(fields[0])
+            yield build_script(range, name)
+          end
+
+          nil
+        end
+
+        private
+
+        def build_script(range, name)
+          first, last = bounds_of(range)
+          Models::Script.new(
+            name: name,
+            range_first: first,
+            range_last: last
+          )
+        end
+
+        def bounds_of(range)
+          if range.is_a?(Range)
+            [range.begin, range.end]
+          else
+            [range, range]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/parsers/special_casing.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "ucode/parsers/base"
+require "ucode/models/special_casing_rule"
+
+module Ucode
+  module Parsers
+    # Parses `SpecialCasing.txt` — context-sensitive case mappings.
+    #
+    # Format (UAX #44):
+    #   cp; lower; title; upper; [conditions;] # name
+    #
+    # The `lower`/`title`/`upper` fields are either empty or a
+    # space-separated list of hex codepoints. `conditions` is a
+    # space-separated list of context identifiers (`Final_Sigma`,
+    # `After_I`) and/or locale codes (`tr`, `az`). Filtering by
+    # condition is the consumer's job.
+    class SpecialCasing < Base
+      class << self
+        # Yields one SpecialCasingRule per non-comment line. Returns a
+        # lazy Enumerator when called without a block.
+        def each_record(path)
+          return enum_for(:each_record, path) unless block_given?
+
+          each_line(path) do |line|
+            fields = line.fields
+            next if fields.length < 4
+
+            cp = parse_hex_cp(fields[0])
+
+            yield Models::SpecialCasingRule.new(
+              codepoint: cp,
+              lower_ids: parse_mapping(fields[1]),
+              title_ids: parse_mapping(fields[2]),
+              upper_ids: parse_mapping(fields[3]),
+              conditions: parse_conditions(fields[4]),
+              comment: line.comment
+            )
+          end
+
+          nil
+        end
+
+        private
+
+        def parse_mapping(field)
+          return [] if field.nil? || field.empty?
+
+          field.split(/\s+/).reject(&:empty?).map do |hex|
+            format("U+%04X", parse_hex_cp(hex))
+          end
+        end
+
+        def parse_conditions(field)
+          return [] if field.nil? || field.empty?
+
+          field.split(/\s+/).reject(&:empty?)
+        end
+      end
+    end
+  end
+end