ucode 0.1.0 → 0.1.1

data/lib/ucode/models/unihan_entry.rb

@@ -2,21 +2,93 @@
 
 require "lutaml/model"
 
+require "ucode/models/unihan_field"
+
 module Ucode
   module Models
-    # Unihan dictionary data for CJK codepoints. Flat-hash design: every
-    # `kFoo` field is a key in `fields`, with array values (Unihan fields
-    # are space-separated lists; uniform arrays simplify the shape).
+    # Unihan dictionary data for CJK codepoints, grouped into the 8
+    # categories defined by the Unihan standard. Each category
+    # corresponds to one Unihan file:
+    #
+    #   Unihan_DictionaryIndices.txt     → dictionary_indices
+    #   Unihan_DictionaryLikeData.txt    → dictionary_like_data
+    #   Unihan_IRGSources.txt            → irg_sources
+    #   Unihan_NumericValues.txt         → numeric_values
+    #   Unihan_RadicalStrokeCounts.txt   → radical_stroke_counts
+    #   Unihan_Readings.txt              → readings
+    #   Unihan_Variants.txt              → variants
+    #   Unihan_OtherMappings.txt         → other_mappings
     #
-    # The semantic grouping (readings / radicals / variants / sources / etc.)
-    # is a presentation concern, derived client-side by prefix. The data
-    # model stays open — Unihan adds fields across versions, and the hash
-    # absorbs additions without model changes.
+    # Each category attribute is a collection of {UnihanField} records.
+    # Category is set at parse time from the source filename (via
+    # `FILE_TO_CATEGORY`) — Unicode does not reorganize files across
+    # versions, so this is stable without per-field hardcoding.
     class UnihanEntry < Lutaml::Model::Serializable
-      attribute :fields, :hash, default: -> { {} }
+      # Symbol → attribute name. Mirrors the 8 Unihan files.
+      CATEGORIES = {
+        dictionary_indices: :dictionary_indices,
+        dictionary_like_data: :dictionary_like_data,
+        irg_sources: :irg_sources,
+        numeric_values: :numeric_values,
+        radical_stroke_counts: :radical_stroke_counts,
+        readings: :readings,
+        variants: :variants,
+        other_mappings: :other_mappings,
+      }.freeze
+
+      # Filename → category symbol. Used by the parser to bucket
+      # records without callers needing to know the mapping.
+      FILE_TO_CATEGORY = {
+        "Unihan_DictionaryIndices.txt" => :dictionary_indices,
+        "Unihan_DictionaryLikeData.txt" => :dictionary_like_data,
+        "Unihan_IRGSources.txt" => :irg_sources,
+        "Unihan_NumericValues.txt" => :numeric_values,
+        "Unihan_RadicalStrokeCounts.txt" => :radical_stroke_counts,
+        "Unihan_Readings.txt" => :readings,
+        "Unihan_Variants.txt" => :variants,
+        "Unihan_OtherMappings.txt" => :other_mappings,
+      }.freeze
+
+      attribute :dictionary_indices, UnihanField, collection: true, default: -> { [] }
+      attribute :dictionary_like_data, UnihanField, collection: true, default: -> { [] }
+      attribute :irg_sources, UnihanField, collection: true, default: -> { [] }
+      attribute :numeric_values, UnihanField, collection: true, default: -> { [] }
+      attribute :radical_stroke_counts, UnihanField, collection: true, default: -> { [] }
+      attribute :readings, UnihanField, collection: true, default: -> { [] }
+      attribute :variants, UnihanField, collection: true, default: -> { [] }
+      attribute :other_mappings, UnihanField, collection: true, default: -> { [] }
+
+      # Pushes a field into the right category bucket. Used by the
+      # Coordinator when streaming records from the parser.
+      #
+      # @param category [Symbol] one of CATEGORIES keys
+      # @param name [String] e.g. "kMandarin"
+      # @param values [Array<String>] space-split values from Unihan
+      def add(category, name, values)
+        attr_name = CATEGORIES.fetch(category) { return }
+        public_send(attr_name) << UnihanField.new(name: name, values: values)
+      end
+
+      # True if any category has data.
+      def any?
+        CATEGORIES.keys.any? { |sym| !public_send(sym).empty? }
+      end
+
+      # All fields across every category, flattened to {name => values}.
+      # Iteration helper for consumers that want a flat view (search
+      # indexing, downstream filtering).
+      #
+      # @return [Hash{String => Array<String>}]
+      def all_fields
+        CATEGORIES.keys.each_with_object({}) do |sym, h|
+          public_send(sym).each { |f| h[f.name] = f.values }
+        end
+      end
 
       key_value do
-        map "fields", to: :fields
+        CATEGORIES.each do |symbol, attr_name|
+          map attr_name, to: symbol
+        end
       end
     end
   end

data/lib/ucode/models/unihan_field.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # One Unihan field assignment: a k-field name plus its space-split
+    # values. e.g. `kMandarin → ["jìng"]`, `kHanyuPinyin → ["64047.030:jìng"]`.
+    # The values list is uniform across all Unihan fields — even single-valued
+    # ones are arrays, which simplifies consumer logic.
+    class UnihanField < Lutaml::Model::Serializable
+      attribute :name, :string
+      attribute :values, :string, collection: true, default: -> { [] }
+
+      key_value do
+        map "name", to: :name
+        map "values", to: :values
+      end
+    end
+  end
+end

data/lib/ucode/models/universal_set_entry.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # One row in a {UniversalSetManifest}. Records the resolved glyph
+    # for a single codepoint: which tier produced it, which source
+    # font, and a stable content hash + size so downstream consumers
+    # can detect changes without re-reading the SVG.
+    #
+    # Wire shape (one entry per assigned codepoint in the manifest's
+    # `entries:` array):
+    #
+    #   {
+    #     "codepoint": 65,
+    #     "id": "U+0041",
+    #     "tier": "tier-1",
+    #     "source": "noto-sans",
+    #     "svg_sha256": "abc...",
+    #     "svg_size_bytes": 412
+    #   }
+    #
+    # `source` is the source identifier extracted from the resolver
+    # {Ucode::Glyphs::Source::Result#provenance} — i.e. the part after
+    # the `tier:` prefix ("noto-sans" for "tier-1:noto-sans"). This is
+    # what audits (TODO 25) group by when answering "how many
+    # codepoints does font X cover in this set?".
+    class UniversalSetEntry < Lutaml::Model::Serializable
+      attribute :codepoint, :integer
+      attribute :id, :string
+      attribute :tier, :string
+      attribute :source, :string
+      attribute :svg_sha256, :string
+      attribute :svg_size_bytes, :integer, default: 0
+
+      key_value do
+        map "codepoint", to: :codepoint
+        map "id", to: :id
+        map "tier", to: :tier
+        map "source", to: :source
+        map "svg_sha256", to: :svg_sha256
+        map "svg_size_bytes", to: :svg_size_bytes
+      end
+    end
+  end
+end

data/lib/ucode/models/universal_set_manifest.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+require "ucode/models/universal_set_entry"
+
+module Ucode
+  module Models
+    # Manifest emitted at the end of a universal glyph set build
+    # (TODO 24). The single index into the set: every codepoint that
+    # was attempted gets one {UniversalSetEntry}, and the totals +
+    # per-tier rollups let consumers (audits, fontist.org) answer
+    # "what does this set cover?" without reading every SVG.
+    #
+    # Wire shape:
+    #
+    #   {
+    #     "unicode_version": "17.0.0",
+    #     "ucode_version": "0.2.0",
+    #     "generated_at": "2026-06-28T00:00:00Z",
+    #     "source_config_sha256": "abc...",
+    #     "totals": {
+    #       "codepoints_assigned": 150012,
+    #       "codepoints_built": 150012,
+    #       "codepoints_skipped": 0,
+    #       "codepoints_failed": 0
+    #     },
+    #     "by_tier": {
+    #       "tier-1": 148512, "pillar-1": 800,
+    #       "pillar-2": 200, "pillar-3": 1500
+    #     },
+    #     "entries": [ { ... UniversalSetEntry ... }, ... ]
+    #   }
+    #
+    # `source_config_sha256` pins which Tier 1 source map produced
+    # this set. Audits use it to detect drift between the reference
+    # set and the config they were validated against.
+    #
+    # This class is passive — accumulation logic lives in
+    # {Ucode::Glyphs::UniversalSet::ManifestAccumulator}; this class
+    # only describes the wire shape and handles (de)serialization via
+    # lutaml-model.
+    class UniversalSetManifest < Lutaml::Model::Serializable
+      # Total counts for one build run.
+      class Totals < Lutaml::Model::Serializable
+        attribute :codepoints_assigned, :integer, default: 0
+        attribute :codepoints_built, :integer, default: 0
+        attribute :codepoints_skipped, :integer, default: 0
+        attribute :codepoints_failed, :integer, default: 0
+
+        key_value do
+          map "codepoints_assigned", to: :codepoints_assigned
+          map "codepoints_built", to: :codepoints_built
+          map "codepoints_skipped", to: :codepoints_skipped
+          map "codepoints_failed", to: :codepoints_failed
+        end
+      end
+
+      attribute :unicode_version, :string
+      attribute :ucode_version, :string
+      attribute :generated_at, :string
+      attribute :source_config_sha256, :string
+      attribute :totals, Totals
+      attribute :by_tier, :hash, default: -> { {} }
+      attribute :entries, UniversalSetEntry, collection: true, default: -> { [] }
+
+      key_value do
+        map "unicode_version", to: :unicode_version
+        map "ucode_version", to: :ucode_version
+        map "generated_at", to: :generated_at
+        map "source_config_sha256", to: :source_config_sha256
+        map "totals", to: :totals
+        map "by_tier", to: :by_tier
+        map "entries", to: :entries
+      end
+    end
+  end
+end

data/lib/ucode/models/validation_report.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # Post-build validation report (TODO 21 §Validation). Emitted as
+    # `output/validation-report.json` by {Ucode::Repo::BuildValidator}
+    # after a canonical build run. Records the outcome of the four
+    # automated validation checks:
+    #
+    #   1. `completeness` — every codepoint folder has both
+    #      `index.json` and `glyph.svg`.
+    #   2. `schema` — every `index.json` deserializes via
+    #      `Ucode::Models::CodePoint.from_hash`.
+    #   3. `provenance_sanity` — every deserialized CodePoint carries
+    #      a non-nil `glyph.source.tier`.
+    #   4. `block_coverage` — per-block built count matches the
+    #      baseline (skipped when no baseline is supplied).
+    #
+    # The fifth TODO 21 check (sample inspection) is manual and out
+    # of scope for the automated validator.
+    #
+    # Like {BuildReport}, this model is passive: the accumulation
+    # logic lives in {Ucode::Repo::BuildValidator}; this class only
+    # describes the wire shape and handles (de)serialization.
+    class ValidationReport < Lutaml::Model::Serializable
+      # Aggregate pass/fail counts for the run.
+      class Totals < Lutaml::Model::Serializable
+        attribute :codepoints_checked, :integer, default: 0
+        attribute :failures, :integer, default: 0
+        attribute :checks_run, :integer, default: 0
+        attribute :checks_passed, :integer, default: 0
+
+        key_value do
+          map "codepoints_checked", to: :codepoints_checked
+          map "failures", to: :failures
+          map "checks_run", to: :checks_run
+          map "checks_passed", to: :checks_passed
+        end
+      end
+
+      # Per-check summary. `status` is one of `passed` / `failed` /
+      # `skipped`. `total` is the number of codepoints the check
+      # evaluated against (0 for `skipped`). `failures` is the count
+      # of recorded failures for this check.
+      class CheckSummary < Lutaml::Model::Serializable
+        STATUS_PASSED = "passed"
+        STATUS_FAILED = "failed"
+        STATUS_SKIPPED = "skipped"
+
+        attribute :name, :string
+        attribute :status, :string
+        attribute :total, :integer, default: 0
+        attribute :failures, :integer, default: 0
+
+        key_value do
+          map "name", to: :name
+          map "status", to: :status
+          map "total", to: :total
+          map "failures", to: :failures
+        end
+      end
+
+      # One failure record. `codepoint` is the integer codepoint (or
+      # nil for structural failures like block_coverage); `block` is
+      # the verbatim block id (folder name); `check` names the check
+      # that produced this failure; `message` is a free-form
+      # human-readable explanation.
+      class Failure < Lutaml::Model::Serializable
+        attribute :codepoint, :integer
+        attribute :block, :string
+        attribute :check, :string
+        attribute :message, :string
+
+        key_value do
+          map "codepoint", to: :codepoint
+          map "block", to: :block
+          map "check", to: :check
+          map "message", to: :message
+        end
+      end
+
+      attribute :unicode_version, :string
+      attribute :generated_at, :string
+      attribute :totals, Totals
+      attribute :checks, CheckSummary, collection: true, default: -> { [] }
+      attribute :failures, Failure, collection: true, default: -> { [] }
+
+      key_value do
+        map "unicode_version", to: :unicode_version
+        map "generated_at", to: :generated_at
+        map "totals", to: :totals
+        map "checks", to: :checks
+        map "failures", to: :failures
+      end
+    end
+  end
+end

data/lib/ucode/models.rb

@@ -31,6 +31,7 @@ module Ucode
     autoload :Script, "ucode/models/script"
     autoload :CodePoint, "ucode/models/codepoint"
     autoload :UnihanEntry, "ucode/models/unihan_entry"
+    autoload :UnihanField, "ucode/models/unihan_field"
     autoload :NamesListEntry, "ucode/models/names_list_entry"
     autoload :NameAlias, "ucode/models/name_alias"
     autoload :NamedSequence, "ucode/models/named_sequence"
@@ -43,5 +44,13 @@ module Ucode
     autoload :BinaryPropertyAssignment, "ucode/models/binary_property_assignment"
     autoload :Relationship, "ucode/models/relationship"
     autoload :Audit, "ucode/models/audit"
+    autoload :BuildReport, "ucode/models/build_report"
+    autoload :ValidationReport, "ucode/models/validation_report"
+    autoload :GlyphSource, "ucode/models/glyph_source"
+    autoload :GlyphSourceMap, "ucode/models/glyph_source_map"
+    autoload :SpecialistFont, "ucode/models/specialist_font"
+    autoload :SpecialistFontManifest, "ucode/models/specialist_font_manifest"
+    autoload :UniversalSetEntry, "ucode/models/universal_set_entry"
+    autoload :UniversalSetManifest, "ucode/models/universal_set_manifest"
   end
 end

data/lib/ucode/parsers/named_sequences.rb

@@ -8,10 +8,10 @@ module Ucode
     # Parses `NamedSequences.txt` — named multi-codepoint sequences.
     #
     # Format (UAX #44):
-    #   cp1 cp2 cp3 ...; Name
+    #   Name; cp1 cp2 cp3 ...
     #
-    # The first field is a space-separated list of hex codepoints; the
-    # second is the human-readable name.
+    # The first field is the human-readable name; the second is a
+    # space-separated list of hex codepoints.
     class NamedSequences < Base
       class << self
         # Yields one NamedSequence per non-comment line. Returns a lazy
@@ -23,8 +23,8 @@ module Ucode
             fields = line.fields
             next if fields.length < 2
 
-            sequence_field = fields[0]
-            name = fields[1]
+            name = fields[0]
+            sequence_field = fields[1]
             next if name.nil? || name.empty?
 
             yield Models::NamedSequence.new(

data/lib/ucode/parsers/unihan.rb

@@ -37,12 +37,32 @@ module Ucode
         Unihan_OtherMappings.txt
       ].freeze
 
+      # Filename → category symbol. The parser tags every Record
+      # with the category derived from its source file, so consumers
+      # (Coordinator → UnihanEntry) don't need to know the mapping.
+      # Unicode does not reorganize files across versions, so this
+      # mapping is stable without per-field hardcoding.
+      FILE_TO_CATEGORY = {
+        "Unihan_DictionaryIndices.txt" => :dictionary_indices,
+        "Unihan_DictionaryLikeData.txt" => :dictionary_like_data,
+        "Unihan_IRGSources.txt" => :irg_sources,
+        "Unihan_NumericValues.txt" => :numeric_values,
+        "Unihan_RadicalStrokeCounts.txt" => :radical_stroke_counts,
+        "Unihan_Readings.txt" => :readings,
+        "Unihan_Variants.txt" => :variants,
+        "Unihan_OtherMappings.txt" => :other_mappings,
+      }.freeze
+
       # Stream record: one Unihan line. Internal pipeline data — a Struct
       # avoids lutaml-model ceremony for transient values. The final
       # `UnihanEntry` model carries the merged, persisted shape. The
       # member is `field_values` (not `values`) to avoid overriding
       # `Struct#values` (the array of all member values).
-      Record = Struct.new(:cp, :field, :field_values, keyword_init: true) do
+      #
+      # `category` is the symbol UnihanEntry uses to bucket the field
+      # into its category attribute (readings / variants / etc.). Set
+      # by `each_in_dir` from the source filename via FILE_TO_CATEGORY.
+      Record = Struct.new(:cp, :field, :field_values, :category, keyword_init: true) do
         def cp_id
           format("U+%04X", cp)
         end
@@ -50,25 +70,16 @@ module Ucode
 
       class << self
         # Yields one Record per non-comment line in a single Unihan file.
-        # Returns a lazy Enumerator when no block is given.
-        def each_record(path)
-          return enum_for(:each_record, path) unless block_given?
+        # The caller must pass the source filename so the Record carries
+        # its category. Returns a lazy Enumerator when no block is given.
+        def each_record(path, filename: nil)
+          return enum_for(:each_record, path, filename: filename) unless block_given?
 
           path_str = path.to_s
-          lineno = 0
+          category = FILE_TO_CATEGORY.fetch(filename || File.basename(path_str), nil)
 
-          File.foreach(path_str) do |raw|
-            lineno += 1
-            line = raw.chomp
-            next if line.empty? || line.start_with?("#")
-
-            begin
-              yield parse_line(line)
-            rescue MalformedLineError => e
-              e.context[:file] ||= path_str
-              e.context[:line] ||= lineno
-              raise
-            end
+          each_line_with_lineno(path_str) do |line, lineno|
+            yield tagged_record(line, category, path_str, lineno)
           end
 
           nil
@@ -76,7 +87,8 @@ module Ucode
 
         # Iterates every known Unihan file in `dir`, yielding one Record
         # per data line across all files. Missing files are silently
-        # skipped (incremental runs, partial downloads).
+        # skipped (incremental runs, partial downloads). Each Record
+        # carries its category so callers don't need to re-derive it.
         def each_in_dir(dir)
           return enum_for(:each_in_dir, dir) unless block_given?
 
@@ -85,7 +97,7 @@ module Ucode
             path = dir_path.join(filename)
             next unless path.exist?
 
-            each_record(path) { |record| yield record }
+            each_record(path, filename: filename) { |record| yield record }
           end
 
           nil
@@ -93,6 +105,25 @@ module Ucode
 
         private
 
+        def each_line_with_lineno(path_str)
+          lineno = 0
+          File.foreach(path_str) do |raw|
+            lineno += 1
+            line = raw.chomp
+            next if line.empty? || line.start_with?("#")
+
+            yield line, lineno
+          end
+        end
+
+        def tagged_record(line, category, path_str, lineno)
+          parse_line(line).tap { |r| r.category = category }
+        rescue MalformedLineError => e
+          e.context[:file] ||= path_str
+          e.context[:line] ||= lineno
+          raise
+        end
+
         # Parses one TAB-separated Unihan data line into a Record. The
         # `split("\t", 3)` limit preserves any tabs inside the value
         # (defensive — real Unihan data does not contain them).

data/lib/ucode/repo/aggregate_writer.rb

@@ -17,7 +17,7 @@ module Ucode
     #   output/blocks/index.json              (block index)
     #   output/scripts/<code>.json
     #   output/index/names.json               (cp_id → name)
-    #   output/index/labels.json              (cp_id → {name, gc, sc})
+    #   output/index/labels.json              (cp_id → {name, gc, sc, cc, bc, mir})
     #   output/index/codepoint_to_block.json  (cp_id → block_id)
     #   output/relationships/*.json           (per-property tables)
     #   output/enums.json                     (property aliases + value aliases)
@@ -79,6 +79,7 @@ module Ucode
       def initialize(output_root)
         @output_root = Pathname.new(output_root)
         @block_codepoint_ids = Hash.new { |h, k| h[k] = [] }
+        @block_ages = Hash.new { |h, k| h[k] = nil }
         @script_codepoint_ids = Hash.new { |h, k| h[k] = [] }
         @names_index = {}
         @labels_index = {}
@@ -94,6 +95,7 @@ module Ucode
         return if cp.block_id.nil?
 
         @block_codepoint_ids[cp.block_id] << cp.id
+        track_block_age(cp)
         if cp.script_code
           @script_codepoint_ids[cp.script_code] << cp.id
         end
@@ -136,10 +138,37 @@ module Ucode
       # ---- Per-codepoint accumulator helpers ---------------------------
 
       def build_label(cp)
-        label = { "name" => cp.name, "gc" => cp.general_category, "sc" => cp.script_code }
+        label = {
+          "name" => cp.name,
+          "gc" => cp.general_category,
+          "sc" => cp.script_code,
+          "cc" => cp.combining_class,
+          "bc" => cp.bidi&.bidi_class,
+          "mir" => cp.bidi&.is_mirrored ? true : nil,
+        }
         label.reject { |_, v| v.nil? }
       end
 
+      # Per-block `age` is the earliest DerivedAge of any codepoint in
+      # the block, compared as a Gem::Version. Stored as the original
+      # string (e.g. "1.1", "17.0.0"). nil when no codepoint in the
+      # block has an age (rare — only happens for entirely-reserved
+      # blocks, which the parser excludes anyway).
+      def track_block_age(cp)
+        return if cp.age.nil? || cp.age.empty?
+
+        current = @block_ages[cp.block_id]
+        @block_ages[cp.block_id] = if current.nil?
+                                     cp.age
+                                   else
+                                     min_age(current, cp.age)
+                                   end
+      end
+
+      def min_age(a, b)
+        Gem::Version.new(a) < Gem::Version.new(b) ? a : b
+      end
+
       # ---- Plane files -------------------------------------------------
 
       def write_planes(blocks)
@@ -176,6 +205,7 @@ module Ucode
 
       def write_blocks(blocks)
         count = blocks.sum do |block|
+          block.age = @block_ages[block.id]
           path = Paths.block_metadata_path(@output_root, block.id)
           write_atomic(path, block_payload(block)) ? 1 : 0
         end
@@ -191,6 +221,7 @@ module Ucode
             "first_cp"     => block.range_first,
             "last_cp"      => block.range_last,
             "plane_number" => block.plane_number,
+            "age"          => @block_ages[block.id],
           }
         end
         write_atomic(path, to_pretty_json(summary)) ? 1 : 0
@@ -203,6 +234,7 @@ module Ucode
           "range_first"    => block.range_first,
           "range_last"     => block.range_last,
           "plane_number"   => block.plane_number,
+          "age"            => @block_ages[block.id],
           "codepoint_ids"  => (@block_codepoint_ids[block.id] || []),
         )
       end