ucode 0.1.0 → 0.2.0

data/lib/ucode/models/glyph_source_map.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+require "ucode/models/glyph_source"
+
+module Ucode
+  module Models
+    # Top-level shape of `config/unicode17_universal_glyph_set.yml`.
+    # Pairs envelope metadata (Unicode + ucode version, generated_at)
+    # with the block→sources map itself.
+    #
+    # Block keys are the verbatim Unicode block name with runs of
+    # whitespace collapsed to a single underscore — the canonical
+    # block id used everywhere else in this codebase (see
+    # {Ucode::Parsers::Blocks}): "Basic_Latin", "Greek_and_Coptic",
+    # "CJK_Unified_Ideographs_Extension_J". Never slugified beyond
+    # whitespace collapsing.
+    #
+    # Wire shape (note: `map:` is a hash keyed by block id, not an
+    # array):
+    #
+    #   unicode_version: "17.0.0"
+    #   ucode_version: "0.2.0"
+    #   generated_at: "2026-06-28T00:00:00Z"
+    #   default_sources:                 # applies when a block's sources are absent/empty
+    #     - kind: fontist
+    #       label: noto-sans
+    #       priority: 1
+    #       license: OFL
+    #   map:
+    #     Basic_Latin:
+    #       sources:
+    #         - kind: fontist
+    #           label: noto-sans
+    #           priority: 1
+    #     Sidetic:
+    #       sources: []
+    #
+    # An entry with `sources: []` (or omitted) is valid: it declares
+    # "no block-specific Tier 1 font; fall back to `default_sources`,
+    # then to Pillars 1-3". The fallback chain is implemented in
+    # {#sources_for}; the raw map is left untouched.
+    #
+    # The hash is stored as a raw `:hash` attribute (lutaml-model
+    # collection semantics don't pair cleanly with a hash-keyed wire
+    # shape); the typed accessors wrap each entry's raw hashes in
+    # {GlyphSource} instances on demand.
+    class GlyphSourceMap < Lutaml::Model::Serializable
+      attribute :unicode_version, :string
+      attribute :ucode_version, :string
+      attribute :generated_at, :string
+      attribute :default_sources_raw, :hash, collection: true, default: -> { [] }
+      attribute :block_sources, :hash, default: -> { {} }
+
+      key_value do
+        map "unicode_version", to: :unicode_version
+        map "ucode_version", to: :ucode_version
+        map "generated_at", to: :generated_at
+        map "default_sources", to: :default_sources_raw
+        map "map", to: :block_sources
+      end
+
+      # @param block_id [String] verbatim block id (underscore form)
+      # @return [Array<GlyphSource>] sources for the block, in
+      #   priority order (ascending). Falls through block-specific →
+      #   `default_sources` → empty.
+      def sources_for(block_id)
+        raw = block_sources[block_id]
+        list = extract_sources_list(raw)
+        list = default_sources_list if list.empty?
+        list.map { |h| GlyphSource.from_hash(h.transform_keys(&:to_s)) }
+          .sort_by(&:priority)
+      end
+
+      # @return [Array<GlyphSource>] the top-level default sources,
+      #   typed and priority-sorted. Empty when not declared.
+      def default_sources
+        default_sources_list
+          .map { |h| GlyphSource.from_hash(h.transform_keys(&:to_s)) }
+          .sort_by(&:priority)
+      end
+
+      # @param block_id [String]
+      # @return [Boolean] true if the block has any entry in the map
+      #   (even with empty sources). Does not consider `default_sources`.
+      def has_block?(block_id)
+        block_sources.key?(block_id)
+      end
+
+      # @return [Array<String>] every block_id that appears in the map
+      #   (regardless of whether it has sources).
+      def block_ids
+        block_sources.keys
+      end
+
+      # @return [Array<String>] block_ids whose own `sources:` list has
+      #   at least one entry. Blocks relying on `default_sources` are
+      #   excluded — they have no block-specific policy.
+      def configured_block_ids
+        block_sources.each_with_object([]) do |(block_id, raw), acc|
+          acc << block_id if any_sources?(raw)
+        end
+      end
+
+      private
+
+      # Each block's value in the YAML is either:
+      #   - `{sources: [...]}` (canonical form), or
+      #   - `[...]` (shorthand: the sources list directly).
+      # Return the sources array in both cases; empty for `nil`.
+      def extract_sources_list(raw)
+        return [] if raw.nil?
+        return raw if raw.is_a?(Array)
+        return Array(raw["sources"]) if raw.is_a?(Hash) && raw.key?("sources")
+        return Array(raw[:sources]) if raw.is_a?(Hash) && raw.key?(:sources)
+
+        []
+      end
+
+      # Each block's value in the YAML is either `{sources: [...]}` or
+      # directly an array (shorthand). Normalize to the array of
+      # source-hashes form.
+      def any_sources?(raw)
+        return false if raw.nil?
+        return raw.any? if raw.is_a?(Array)
+
+        raw.is_a?(Hash) && Array(raw["sources"] || raw[:sources]).any?
+      end
+
+      # `default_sources` on the wire is a list of source hashes. Older
+      # configs may omit it; treat absence as an empty list.
+      def default_sources_list
+        Array(default_sources_raw)
+      end
+    end
+  end
+end

data/lib/ucode/models/specialist_font.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # One entry in `config/specialist_fonts.yml` — a Tier 1 font that
+    # fontist's formula index does not carry (academic sites, SIL
+    # downloads, GitHub releases). The fetcher walks a list of these
+    # and materializes each `path` on disk.
+    #
+    # Wire shape (YAML):
+    #
+    #   label: Lentariso
+    #   version: "1.033"
+    #   license: OFL
+    #   url: "https://github.com/.../Lentariso.otf"
+    #   sha256: "<hex>"          # null until first successful fetch
+    #   path: "data/fonts/Lentariso.otf"
+    #   extract: false
+    #   extract_member: null     # required when extract: true
+    #   provenance: "Imperial Aramaic / Phoenician / Sidetic coverage"
+    #
+    # `url: null` marks a local-only entry: the user supplies the
+    # file at `path` (which may use `~` and shell globs); the fetcher
+    # never attempts a network download for these.
+    class SpecialistFont < Lutaml::Model::Serializable
+      LICENSE_OFL = "OFL"
+      private_constant :LICENSE_OFL
+
+      attribute :label, :string
+      attribute :version, :string
+      attribute :license, :string, default: -> { LICENSE_OFL }
+      attribute :url, :string
+      attribute :sha256, :string
+      attribute :path, :string
+      attribute :extract, :boolean, default: -> { false }
+      attribute :extract_member, :string
+      attribute :provenance, :string
+
+      key_value do
+        map "label", to: :label
+        map "version", to: :version
+        map "license", to: :license
+        map "url", to: :url
+        map "sha256", to: :sha256
+        map "path", to: :path
+        map "extract", to: :extract
+        map "extract_member", to: :extract_member
+        map "provenance", to: :provenance
+      end
+
+      def local_only?
+        url.nil? || url.empty?
+      end
+
+      def ofl?
+        license == LICENSE_OFL
+      end
+
+      def hash_known?
+        !sha256.nil? && !sha256.empty?
+      end
+
+      def extract?
+        extract == true
+      end
+    end
+  end
+end

data/lib/ucode/models/specialist_font_manifest.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+require "ucode/models/specialist_font"
+
+module Ucode
+  module Models
+    # Typed view over `config/specialist_fonts.yml`. Carries the full
+    # list of {SpecialistFont} entries; provides lookup by label so
+    # the fetcher can honor `--label Lentariso` without scanning the
+    # array itself.
+    #
+    # The manifest is pure data — it does not know the path it was
+    # loaded from. Persistence of computed SHA256 hashes back to disk
+    # is the responsibility of {Ucode::Fetch::SpecialistFontFetcher},
+    # which owns the file path and writes atomically after a run.
+    class SpecialistFontManifest < Lutaml::Model::Serializable
+      attribute :fonts, SpecialistFont, collection: true
+
+      key_value do
+        map "fonts", to: :fonts
+      end
+
+      # @param label [String] exact label match
+      # @return [SpecialistFont, nil]
+      def find_by_label(label)
+        fonts.find { |font| font.label == label }
+      end
+
+      # @return [Array<String>] labels of every entry, in declared order
+      def labels
+        fonts.map(&:label)
+      end
+
+      # @param label [String]
+      # @return [SpecialistFontManifest] a new manifest containing only
+      #   the matching font. Returns self unchanged if the label is
+      #   unknown (the fetcher reports it as a separate failure).
+      def only(label)
+        match = find_by_label(label)
+        return self if match.nil?
+
+        self.class.new(fonts: [match])
+      end
+    end
+  end
+end

data/lib/ucode/models/unihan_entry.rb

@@ -2,21 +2,93 @@
 
 require "lutaml/model"
 
+require "ucode/models/unihan_field"
+
 module Ucode
   module Models
-    # Unihan dictionary data for CJK codepoints. Flat-hash design: every
-    # `kFoo` field is a key in `fields`, with array values (Unihan fields
-    # are space-separated lists; uniform arrays simplify the shape).
+    # Unihan dictionary data for CJK codepoints, grouped into the 8
+    # categories defined by the Unihan standard. Each category
+    # corresponds to one Unihan file:
+    #
+    #   Unihan_DictionaryIndices.txt     → dictionary_indices
+    #   Unihan_DictionaryLikeData.txt    → dictionary_like_data
+    #   Unihan_IRGSources.txt            → irg_sources
+    #   Unihan_NumericValues.txt         → numeric_values
+    #   Unihan_RadicalStrokeCounts.txt   → radical_stroke_counts
+    #   Unihan_Readings.txt              → readings
+    #   Unihan_Variants.txt              → variants
+    #   Unihan_OtherMappings.txt         → other_mappings
     #
-    # The semantic grouping (readings / radicals / variants / sources / etc.)
-    # is a presentation concern, derived client-side by prefix. The data
-    # model stays open — Unihan adds fields across versions, and the hash
-    # absorbs additions without model changes.
+    # Each category attribute is a collection of {UnihanField} records.
+    # Category is set at parse time from the source filename (via
+    # `FILE_TO_CATEGORY`) — Unicode does not reorganize files across
+    # versions, so this is stable without per-field hardcoding.
     class UnihanEntry < Lutaml::Model::Serializable
-      attribute :fields, :hash, default: -> { {} }
+      # Symbol → attribute name. Mirrors the 8 Unihan files.
+      CATEGORIES = {
+        dictionary_indices: :dictionary_indices,
+        dictionary_like_data: :dictionary_like_data,
+        irg_sources: :irg_sources,
+        numeric_values: :numeric_values,
+        radical_stroke_counts: :radical_stroke_counts,
+        readings: :readings,
+        variants: :variants,
+        other_mappings: :other_mappings,
+      }.freeze
+
+      # Filename → category symbol. Used by the parser to bucket
+      # records without callers needing to know the mapping.
+      FILE_TO_CATEGORY = {
+        "Unihan_DictionaryIndices.txt" => :dictionary_indices,
+        "Unihan_DictionaryLikeData.txt" => :dictionary_like_data,
+        "Unihan_IRGSources.txt" => :irg_sources,
+        "Unihan_NumericValues.txt" => :numeric_values,
+        "Unihan_RadicalStrokeCounts.txt" => :radical_stroke_counts,
+        "Unihan_Readings.txt" => :readings,
+        "Unihan_Variants.txt" => :variants,
+        "Unihan_OtherMappings.txt" => :other_mappings,
+      }.freeze
+
+      attribute :dictionary_indices, UnihanField, collection: true, default: -> { [] }
+      attribute :dictionary_like_data, UnihanField, collection: true, default: -> { [] }
+      attribute :irg_sources, UnihanField, collection: true, default: -> { [] }
+      attribute :numeric_values, UnihanField, collection: true, default: -> { [] }
+      attribute :radical_stroke_counts, UnihanField, collection: true, default: -> { [] }
+      attribute :readings, UnihanField, collection: true, default: -> { [] }
+      attribute :variants, UnihanField, collection: true, default: -> { [] }
+      attribute :other_mappings, UnihanField, collection: true, default: -> { [] }
+
+      # Pushes a field into the right category bucket. Used by the
+      # Coordinator when streaming records from the parser.
+      #
+      # @param category [Symbol] one of CATEGORIES keys
+      # @param name [String] e.g. "kMandarin"
+      # @param values [Array<String>] space-split values from Unihan
+      def add(category, name, values)
+        attr_name = CATEGORIES.fetch(category) { return }
+        public_send(attr_name) << UnihanField.new(name: name, values: values)
+      end
+
+      # True if any category has data.
+      def any?
+        CATEGORIES.keys.any? { |sym| !public_send(sym).empty? }
+      end
+
+      # All fields across every category, flattened to {name => values}.
+      # Iteration helper for consumers that want a flat view (search
+      # indexing, downstream filtering).
+      #
+      # @return [Hash{String => Array<String>}]
+      def all_fields
+        CATEGORIES.keys.each_with_object({}) do |sym, h|
+          public_send(sym).each { |f| h[f.name] = f.values }
+        end
+      end
 
       key_value do
-        map "fields", to: :fields
+        CATEGORIES.each do |symbol, attr_name|
+          map attr_name, to: symbol
+        end
       end
     end
   end

data/lib/ucode/models/unihan_field.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # One Unihan field assignment: a k-field name plus its space-split
+    # values. e.g. `kMandarin → ["jìng"]`, `kHanyuPinyin → ["64047.030:jìng"]`.
+    # The values list is uniform across all Unihan fields — even single-valued
+    # ones are arrays, which simplifies consumer logic.
+    class UnihanField < Lutaml::Model::Serializable
+      attribute :name, :string
+      attribute :values, :string, collection: true, default: -> { [] }
+
+      key_value do
+        map "name", to: :name
+        map "values", to: :values
+      end
+    end
+  end
+end

data/lib/ucode/models/universal_set_entry.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # One row in a {UniversalSetManifest}. Records the resolved glyph
+    # for a single codepoint: which tier produced it, which source
+    # font, and a stable content hash + size so downstream consumers
+    # can detect changes without re-reading the SVG.
+    #
+    # Wire shape (one entry per assigned codepoint in the manifest's
+    # `entries:` array):
+    #
+    #   {
+    #     "codepoint": 65,
+    #     "id": "U+0041",
+    #     "tier": "tier-1",
+    #     "source": "noto-sans",
+    #     "svg_sha256": "abc...",
+    #     "svg_size_bytes": 412
+    #   }
+    #
+    # `source` is the source identifier extracted from the resolver
+    # {Ucode::Glyphs::Source::Result#provenance} — i.e. the part after
+    # the `tier:` prefix ("noto-sans" for "tier-1:noto-sans"). This is
+    # what audits (TODO 25) group by when answering "how many
+    # codepoints does font X cover in this set?".
+    class UniversalSetEntry < Lutaml::Model::Serializable
+      attribute :codepoint, :integer
+      attribute :id, :string
+      attribute :tier, :string
+      attribute :source, :string
+      attribute :svg_sha256, :string
+      attribute :svg_size_bytes, :integer, default: 0
+
+      key_value do
+        map "codepoint", to: :codepoint
+        map "id", to: :id
+        map "tier", to: :tier
+        map "source", to: :source
+        map "svg_sha256", to: :svg_sha256
+        map "svg_size_bytes", to: :svg_size_bytes
+      end
+    end
+  end
+end

data/lib/ucode/models/universal_set_manifest.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+require "ucode/models/universal_set_entry"
+
+module Ucode
+  module Models
+    # Manifest emitted at the end of a universal glyph set build
+    # (TODO 24). The single index into the set: every codepoint that
+    # was attempted gets one {UniversalSetEntry}, and the totals +
+    # per-tier rollups let consumers (audits, fontist.org) answer
+    # "what does this set cover?" without reading every SVG.
+    #
+    # Wire shape:
+    #
+    #   {
+    #     "unicode_version": "17.0.0",
+    #     "ucode_version": "0.2.0",
+    #     "generated_at": "2026-06-28T00:00:00Z",
+    #     "source_config_sha256": "abc...",
+    #     "totals": {
+    #       "codepoints_assigned": 150012,
+    #       "codepoints_built": 150012,
+    #       "codepoints_skipped": 0,
+    #       "codepoints_failed": 0
+    #     },
+    #     "by_tier": {
+    #       "tier-1": 148512, "pillar-1": 800,
+    #       "pillar-2": 200, "pillar-3": 1500
+    #     },
+    #     "entries": [ { ... UniversalSetEntry ... }, ... ]
+    #   }
+    #
+    # `source_config_sha256` pins which Tier 1 source map produced
+    # this set. Audits use it to detect drift between the reference
+    # set and the config they were validated against.
+    #
+    # This class is passive — accumulation logic lives in
+    # {Ucode::Glyphs::UniversalSet::ManifestAccumulator}; this class
+    # only describes the wire shape and handles (de)serialization via
+    # lutaml-model.
+    class UniversalSetManifest < Lutaml::Model::Serializable
+      # Total counts for one build run.
+      class Totals < Lutaml::Model::Serializable
+        attribute :codepoints_assigned, :integer, default: 0
+        attribute :codepoints_built, :integer, default: 0
+        attribute :codepoints_skipped, :integer, default: 0
+        attribute :codepoints_failed, :integer, default: 0
+
+        key_value do
+          map "codepoints_assigned", to: :codepoints_assigned
+          map "codepoints_built", to: :codepoints_built
+          map "codepoints_skipped", to: :codepoints_skipped
+          map "codepoints_failed", to: :codepoints_failed
+        end
+      end
+
+      attribute :unicode_version, :string
+      attribute :ucode_version, :string
+      attribute :generated_at, :string
+      attribute :source_config_sha256, :string
+      attribute :totals, Totals
+      attribute :by_tier, :hash, default: -> { {} }
+      attribute :entries, UniversalSetEntry, collection: true, default: -> { [] }
+
+      key_value do
+        map "unicode_version", to: :unicode_version
+        map "ucode_version", to: :ucode_version
+        map "generated_at", to: :generated_at
+        map "source_config_sha256", to: :source_config_sha256
+        map "totals", to: :totals
+        map "by_tier", to: :by_tier
+        map "entries", to: :entries
+      end
+    end
+  end
+end

data/lib/ucode/models/validation_report.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # Post-build validation report (TODO 21 §Validation). Emitted as
+    # `output/validation-report.json` by {Ucode::Repo::BuildValidator}
+    # after a canonical build run. Records the outcome of the four
+    # automated validation checks:
+    #
+    #   1. `completeness` — every codepoint folder has both
+    #      `index.json` and `glyph.svg`.
+    #   2. `schema` — every `index.json` deserializes via
+    #      `Ucode::Models::CodePoint.from_hash`.
+    #   3. `provenance_sanity` — every deserialized CodePoint carries
+    #      a non-nil `glyph.source.tier`.
+    #   4. `block_coverage` — per-block built count matches the
+    #      baseline (skipped when no baseline is supplied).
+    #
+    # The fifth TODO 21 check (sample inspection) is manual and out
+    # of scope for the automated validator.
+    #
+    # Like {BuildReport}, this model is passive: the accumulation
+    # logic lives in {Ucode::Repo::BuildValidator}; this class only
+    # describes the wire shape and handles (de)serialization.
+    class ValidationReport < Lutaml::Model::Serializable
+      # Aggregate pass/fail counts for the run.
+      class Totals < Lutaml::Model::Serializable
+        attribute :codepoints_checked, :integer, default: 0
+        attribute :failures, :integer, default: 0
+        attribute :checks_run, :integer, default: 0
+        attribute :checks_passed, :integer, default: 0
+
+        key_value do
+          map "codepoints_checked", to: :codepoints_checked
+          map "failures", to: :failures
+          map "checks_run", to: :checks_run
+          map "checks_passed", to: :checks_passed
+        end
+      end
+
+      # Per-check summary. `status` is one of `passed` / `failed` /
+      # `skipped`. `total` is the number of codepoints the check
+      # evaluated against (0 for `skipped`). `failures` is the count
+      # of recorded failures for this check.
+      class CheckSummary < Lutaml::Model::Serializable
+        STATUS_PASSED = "passed"
+        STATUS_FAILED = "failed"
+        STATUS_SKIPPED = "skipped"
+
+        attribute :name, :string
+        attribute :status, :string
+        attribute :total, :integer, default: 0
+        attribute :failures, :integer, default: 0
+
+        key_value do
+          map "name", to: :name
+          map "status", to: :status
+          map "total", to: :total
+          map "failures", to: :failures
+        end
+      end
+
+      # One failure record. `codepoint` is the integer codepoint (or
+      # nil for structural failures like block_coverage); `block` is
+      # the verbatim block id (folder name); `check` names the check
+      # that produced this failure; `message` is a free-form
+      # human-readable explanation.
+      class Failure < Lutaml::Model::Serializable
+        attribute :codepoint, :integer
+        attribute :block, :string
+        attribute :check, :string
+        attribute :message, :string
+
+        key_value do
+          map "codepoint", to: :codepoint
+          map "block", to: :block
+          map "check", to: :check
+          map "message", to: :message
+        end
+      end
+
+      attribute :unicode_version, :string
+      attribute :generated_at, :string
+      attribute :totals, Totals
+      attribute :checks, CheckSummary, collection: true, default: -> { [] }
+      attribute :failures, Failure, collection: true, default: -> { [] }
+
+      key_value do
+        map "unicode_version", to: :unicode_version
+        map "generated_at", to: :generated_at
+        map "totals", to: :totals
+        map "checks", to: :checks
+        map "failures", to: :failures
+      end
+    end
+  end
+end

data/lib/ucode/models.rb

@@ -31,6 +31,7 @@ module Ucode
     autoload :Script, "ucode/models/script"
     autoload :CodePoint, "ucode/models/codepoint"
     autoload :UnihanEntry, "ucode/models/unihan_entry"
+    autoload :UnihanField, "ucode/models/unihan_field"
     autoload :NamesListEntry, "ucode/models/names_list_entry"
     autoload :NameAlias, "ucode/models/name_alias"
     autoload :NamedSequence, "ucode/models/named_sequence"
@@ -43,5 +44,13 @@ module Ucode
     autoload :BinaryPropertyAssignment, "ucode/models/binary_property_assignment"
     autoload :Relationship, "ucode/models/relationship"
     autoload :Audit, "ucode/models/audit"
+    autoload :BuildReport, "ucode/models/build_report"
+    autoload :ValidationReport, "ucode/models/validation_report"
+    autoload :GlyphSource, "ucode/models/glyph_source"
+    autoload :GlyphSourceMap, "ucode/models/glyph_source_map"
+    autoload :SpecialistFont, "ucode/models/specialist_font"
+    autoload :SpecialistFontManifest, "ucode/models/specialist_font_manifest"
+    autoload :UniversalSetEntry, "ucode/models/universal_set_entry"
+    autoload :UniversalSetManifest, "ucode/models/universal_set_manifest"
   end
 end

data/lib/ucode/parsers/named_sequences.rb

@@ -8,10 +8,10 @@ module Ucode
     # Parses `NamedSequences.txt` — named multi-codepoint sequences.
     #
     # Format (UAX #44):
-    #   cp1 cp2 cp3 ...; Name
+    #   Name; cp1 cp2 cp3 ...
     #
-    # The first field is a space-separated list of hex codepoints; the
-    # second is the human-readable name.
+    # The first field is the human-readable name; the second is a
+    # space-separated list of hex codepoints.
     class NamedSequences < Base
       class << self
         # Yields one NamedSequence per non-comment line. Returns a lazy
@@ -23,8 +23,8 @@ module Ucode
             fields = line.fields
             next if fields.length < 2
 
-            sequence_field = fields[0]
-            name = fields[1]
+            name = fields[0]
+            sequence_field = fields[1]
             next if name.nil? || name.empty?
 
             yield Models::NamedSequence.new(