ucode 0.1.0 → 0.2.0

data/lib/ucode/glyphs/universal_set.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Glyphs
+    # Universal glyph set — one SVG per assigned Unicode codepoint,
+    # sourced via the 4-tier resolver using the curated Tier 1 config
+    # from TODO 23. The set is the canonical reference for "what
+    # Unicode 17 looks like" — every codepoint has exactly one glyph,
+    # with documented provenance, in a flat layout designed for fast
+    # lookup by audits (TODO 25) and the fontist.org consumer
+    # (TODO 27).
+    #
+    # Output layout (per TODO 24):
+    #
+    #   output/universal_glyph_set/
+    #   ├── manifest.json             # one entry per codepoint + totals
+    #   ├── glyphs/
+    #   │   ├── U+0000.svg
+    #   │   └── ...
+    #   └── reports/
+    #       ├── by_tier.json          # tier-1: N1, pillar-1: N2, ...
+    #       ├── by_block.json         # per-block tier breakdown
+    #       └── gaps.json             # assigned codepoints with no glyph
+    #
+    # Components:
+    #
+    # - {Builder} drains a codepoint stream through the resolver and
+    #   writes glyphs + manifest atomically.
+    # - {ManifestAccumulator} is the thread-safe tally that produces
+    #   the final {Ucode::Models::UniversalSetManifest}.
+    # - {ManifestWriter} emits the manifest and per-tier / per-block /
+    #   gaps reports under the output root.
+    # - {Idempotency} wraps {Ucode::Repo::AtomicWrites} with the
+    #   "skip if SVG unchanged" semantic documented in TODO 24.
+    module UniversalSet
+      autoload :Builder, "ucode/glyphs/universal_set/builder"
+      autoload :ManifestAccumulator, "ucode/glyphs/universal_set/manifest_accumulator"
+      autoload :ManifestWriter, "ucode/glyphs/universal_set/manifest_writer"
+      autoload :Idempotency, "ucode/glyphs/universal_set/idempotency"
+      autoload :PreBuildCheck, "ucode/glyphs/universal_set/pre_build_check"
+      autoload :Validator, "ucode/glyphs/universal_set/validator"
+      autoload :CoverageReport, "ucode/glyphs/universal_set/coverage_report"
+    end
+  end
+end

data/lib/ucode/glyphs.rb

@@ -20,8 +20,15 @@ module Ucode
     autoload :CellExtractor, "ucode/glyphs/cell_extractor"
     autoload :MonolithPageMap, "ucode/glyphs/monolith_page_map"
     autoload :Writer, "ucode/glyphs/writer"
+    autoload :Pipeline, "ucode/glyphs/pipeline"
     autoload :LastResort, "ucode/glyphs/last_resort"
     autoload :EmbeddedFonts, "ucode/glyphs/embedded_fonts"
     autoload :RealFonts, "ucode/glyphs/real_fonts"
+    autoload :Source, "ucode/glyphs/source"
+    autoload :Resolver, "ucode/glyphs/resolver"
+    autoload :SourceConfig, "ucode/glyphs/source_config"
+    autoload :SourceBuilder, "ucode/glyphs/source_builder"
+    autoload :Sources, "ucode/glyphs/sources"
+    autoload :UniversalSet, "ucode/glyphs/universal_set"
   end
 end

data/lib/ucode/models/audit/baseline.rb

@@ -18,6 +18,11 @@ module Ucode
         attribute :fontisan_version, :string
         attribute :source, :string
         attribute :generated_at, :string
+        # Which CoverageReference produced the per-block counts.
+        # "ucd" for UcdOnlyReference (default), "universal-set" for
+        # UniversalSetReference (TODO 25). nil on legacy reports —
+        # consumers should treat nil as "ucd".
+        attribute :reference_kind, :string
 
         key_value do
           map "unicode_version",  to: :unicode_version
@@ -25,6 +30,7 @@ module Ucode
           map "fontisan_version", to: :fontisan_version
           map "source",           to: :source
           map "generated_at",     to: :generated_at
+          map "reference_kind",   to: :reference_kind
         end
       end
     end

data/lib/ucode/models/audit/block_summary.rb

@@ -30,6 +30,12 @@ module Ucode
         attribute :status, :string
         attribute :missing_codepoints, :integer, collection: true, default: -> { [] }
         attribute :covered_codepoints, :integer, collection: true, default: -> { [] }
+        # Per-codepoint provenance for the missing set. Populated only
+        # when the audit ran against a UniversalSetReference (TODO 25).
+        # Empty for UCD-only audits — the field serializes as [] and
+        # consumers treat that as "no provenance available".
+        attribute :missing_codepoint_provenance, CodepointProvenance,
+                  collection: true, default: -> { [] }
 
         key_value do
           map "name",               to: :name
@@ -44,6 +50,7 @@ module Ucode
           map "status",             to: :status
           map "missing_codepoints", to: :missing_codepoints
           map "covered_codepoints", to: :covered_codepoints
+          map "missing_codepoint_provenance", to: :missing_codepoint_provenance
         end
 
         # Derive the canonical status string for a block given its

data/lib/ucode/models/audit/codepoint_provenance.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Per-codepoint provenance row attached to a {BlockSummary}'s
+      # `missing_codepoint_provenance` collection.
+      #
+      # Populated only when the audit used a
+      # {Ucode::Audit::UniversalSetReference}. UCD-only audits omit
+      # the field entirely — preserving the legacy wire shape.
+      #
+      # Wire shape (one entry per missing codepoint):
+      #
+      #   {
+      #     "codepoint": 10981,
+      #     "tier": "tier-1",
+      #     "source": "lentariso"
+      #   }
+      #
+      # `tier` and `source` mirror the universal-set manifest
+      # ({UniversalSetEntry}) and let downstream renderers (TODO 26)
+      # display the missing glyph + its provenance next to each row.
+      class CodepointProvenance < Lutaml::Model::Serializable
+        attribute :codepoint, :integer
+        attribute :tier, :string
+        attribute :source, :string
+
+        key_value do
+          map "codepoint", to: :codepoint
+          map "tier",      to: :tier
+          map "source",    to: :source
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/audit/release_face.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Per-face entry in a {ReleaseFormulaEntry}.
+      #
+      # Compact card view derived from a per-face {AuditReport}. Carries
+      # identity + coverage totals + relative paths into the release
+      # tree. The fontist.org renderer iterates these cards to build its
+      # font index; clicking a card fetches the per-face audit subtree
+      # at `index_path`.
+      class ReleaseFaceEntry < Lutaml::Model::Serializable
+        attribute :postscript_name,    :string
+        attribute :family_name,        :string
+        attribute :weight_class,       :integer
+        attribute :total_codepoints,   :integer
+        attribute :covered_codepoints, :integer
+        attribute :blocks_complete,    :integer
+        attribute :blocks_partial,     :integer
+        attribute :source_sha256,      :string
+        attribute :index_path,         :string
+        attribute :html_path,          :string
+
+        key_value do
+          map "postscript_name",    to: :postscript_name
+          map "family_name",        to: :family_name
+          map "weight_class",       to: :weight_class
+          map "total_codepoints",   to: :total_codepoints
+          map "covered_codepoints", to: :covered_codepoints
+          map "blocks_complete",    to: :blocks_complete
+          map "blocks_partial",     to: :blocks_partial
+          map "source_sha256",      to: :source_sha256
+          map "index_path",         to: :index_path
+          map "html_path",          to: :html_path
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/audit/release_formula.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Per-formula entry in a {ReleaseManifest}.
+      #
+      # A formula is a single fontist formula (one installable unit).
+      # Each formula contributes one or more face entries to the release
+      # tree. The `slug` is the formula's URL-safe identifier used as
+      # the directory name under `<release_root>/audit/<slug>/`.
+      #
+      # `source_path` records where the original library audit ran so a
+      # consumer reading the manifest can trace the audit back to its
+      # input directory.
+      class ReleaseFormulaEntry < Lutaml::Model::Serializable
+        attribute :slug, :string
+        attribute :source_path, :string
+        attribute :faces_total, :integer
+        attribute :faces, ReleaseFaceEntry, collection: true, default: -> { [] }
+
+        key_value do
+          map "slug",        to: :slug
+          map "source_path", to: :source_path
+          map "faces_total", to: :faces_total
+          map "faces",       to: :faces
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/audit/release_manifest.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Top-level release manifest for the fontist.org-consumable
+      # artifact (TODO 27).
+      #
+      # One manifest per release tree at `<release_root>/manifest.json`.
+      # Records the ucode/unicode versions, optional source-config
+      # sha256 (for Tier 1 curation provenance), aggregate formula/face
+      # counts, the universal-set reference section, and the per-formula
+      # face index.
+      #
+      # fontist.org's `scripts/fetch-data.sh` reads this manifest first
+      # to decide whether to fetch the universal-set zip and which
+      # per-formula audit subtrees to pull.
+      class ReleaseManifest < Lutaml::Model::Serializable
+        attribute :ucode_version,        :string
+        attribute :unicode_version,      :string
+        attribute :generated_at,         :string
+        attribute :source_config_sha256, :string
+        attribute :formulas_total,       :integer
+        attribute :faces_total,          :integer
+        attribute :universal_set, ReleaseUniversalSet
+        attribute :formulas, ReleaseFormulaEntry, collection: true, default: -> { [] }
+
+        key_value do
+          map "ucode_version",        to: :ucode_version
+          map "unicode_version",      to: :unicode_version
+          map "generated_at",         to: :generated_at
+          map "source_config_sha256", to: :source_config_sha256
+          map "formulas_total",       to: :formulas_total
+          map "faces_total",          to: :faces_total
+          map "universal_set",        to: :universal_set
+          map "formulas",             to: :formulas
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/audit/release_universal_set.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Universal-set reference section of a {ReleaseManifest}.
+      #
+      # Records whether the release tree carries a co-located universal
+      # glyph set (TODO 24) and where to find it. fontist.org consumes
+      # this to decide whether to render missing-codepoint thumbnails
+      # from the universal set or fall back to text-only chips.
+      #
+      # When `available` is false, `reason` carries a short diagnostic
+      # string ("universal-set directory not found at <path>"). The
+      # other fields are nil.
+      class ReleaseUniversalSet < Lutaml::Model::Serializable
+        attribute :available, Lutaml::Model::Type::Boolean
+        attribute :manifest_path, :string
+        attribute :glyphs_dir, :string
+        attribute :unicode_version, :string
+        attribute :totals, :hash, default: -> { {} }
+        attribute :reason, :string
+
+        key_value do
+          map "available",       to: :available
+          map "manifest_path",   to: :manifest_path
+          map "glyphs_dir",      to: :glyphs_dir
+          map "unicode_version", to: :unicode_version
+          map "totals",          to: :totals
+          map "reason",          to: :reason
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/audit.rb

@@ -23,6 +23,7 @@ module Ucode
       autoload :PlaneSummary, "ucode/models/audit/plane_summary"
       autoload :Discrepancy, "ucode/models/audit/discrepancy"
       autoload :CodepointDetail, "ucode/models/audit/codepoint_detail"
+      autoload :CodepointProvenance, "ucode/models/audit/codepoint_provenance"
 
       # Ported from fontisan (namespace swap + minor renames)
       autoload :AuditReport, "ucode/models/audit/audit_report"
@@ -45,6 +46,14 @@ module Ucode
       autoload :DuplicateGroup, "ucode/models/audit/duplicate_group"
       autoload :LibrarySummary, "ucode/models/audit/library_summary"
       autoload :AuditDiff, "ucode/models/audit/audit_diff"
+
+      # Release-tree models (TODO 27) — fontist.org-consumable
+      # artifact manifest. ReleaseManifest is the top-level shape;
+      # the others are nested entries.
+      autoload :ReleaseManifest,     "ucode/models/audit/release_manifest"
+      autoload :ReleaseFormulaEntry, "ucode/models/audit/release_formula"
+      autoload :ReleaseFaceEntry,    "ucode/models/audit/release_face"
+      autoload :ReleaseUniversalSet, "ucode/models/audit/release_universal_set"
     end
   end
 end

data/lib/ucode/models/block.rb

@@ -13,6 +13,7 @@ module Ucode
       attribute :range_first, :integer
       attribute :range_last, :integer
       attribute :plane_number, :integer
+      attribute :age, :string
       attribute :codepoint_ids, :string, collection: true, default: -> { [] }
 
       key_value do
@@ -21,6 +22,7 @@ module Ucode
         map "range_first", to: :range_first
         map "range_last", to: :range_last
         map "plane_number", to: :plane_number
+        map "age", to: :age
         map "codepoint_ids", to: :codepoint_ids
       end
 

data/lib/ucode/models/build_report.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # Build report for one canonical Unicode dataset run. The
+    # deliverable spec'd in TODO 21: emitted at the end of a Mode 1
+    # build as `output/build-report.json`, summarizing what got built,
+    # how (per-tier), per-block, and any failures.
+    #
+    # The model is passive — accumulation logic lives in
+    # {Ucode::Repo::BuildReportAccumulator}; this class only describes
+    # the wire shape and handles (de)serialization via lutaml-model.
+    #
+    # Wire format (see TODO 21):
+    #
+    #   {
+    #     "unicode_version": "17.0.0",
+    #     "ucode_version": "0.2.0",
+    #     "generated_at": "2026-07-01T12:00:00Z",
+    #     "totals": { "assigned": 150012, "built": 150012,
+    #                 "skipped": 0, "failed": 0 },
+    #     "by_tier": { "tier-1": 150012, "pillar-1": 3000, ... },
+    #     "by_block": [
+    #       { "name": "Basic Latin", "assigned": 128, "built": 128,
+    #         "tier_breakdown": { "tier-1": 128 } },
+    #       ...
+    #     ],
+    #     "failures": []
+    #   }
+    #
+    # `by_tier` counts overlap across tiers (a codepoint attempted via
+    # Tier 1 but falling through to Pillar 1 is counted in both);
+    # `built` per-codepoint is the tier that actually produced its
+    # glyph.
+    class BuildReport < Lutaml::Model::Serializable
+      # Total counts for the run.
+      class Totals < Lutaml::Model::Serializable
+        attribute :assigned, :integer, default: 0
+        attribute :built, :integer, default: 0
+        attribute :skipped, :integer, default: 0
+        attribute :failed, :integer, default: 0
+
+        key_value do
+          map "assigned", to: :assigned
+          map "built", to: :built
+          map "skipped", to: :skipped
+          map "failed", to: :failed
+        end
+      end
+
+      # Per-block rollup. One entry per Unicode block in the run.
+      class BlockSummary < Lutaml::Model::Serializable
+        attribute :name, :string
+        attribute :assigned, :integer, default: 0
+        attribute :built, :integer, default: 0
+        attribute :tier_breakdown, :hash, default: -> { {} }
+
+        key_value do
+          map "name", to: :name
+          map "assigned", to: :assigned
+          map "built", to: :built
+          map "tier_breakdown", to: :tier_breakdown
+        end
+      end
+
+      # One failure record. `codepoint` is the integer codepoint (or
+      # nil if the failure is structural); `tier` is the resolver tier
+      # that raised (or nil); `error_class` and `message` carry the
+      # exception details; `backtrace` is optional.
+      class Failure < Lutaml::Model::Serializable
+        attribute :codepoint, :integer
+        attribute :block_name, :string
+        attribute :tier, :string
+        attribute :error_class, :string
+        attribute :message, :string
+        attribute :backtrace, :string, collection: true, default: -> { [] }
+
+        key_value do
+          map "codepoint", to: :codepoint
+          map "block_name", to: :block_name
+          map "tier", to: :tier
+          map "error_class", to: :error_class
+          map "message", to: :message
+          map "backtrace", to: :backtrace
+        end
+      end
+
+      attribute :unicode_version, :string
+      attribute :ucode_version, :string
+      attribute :generated_at, :string
+      attribute :totals, Totals
+      attribute :by_tier, :hash, default: -> { {} }
+      attribute :by_block, BlockSummary, collection: true, default: -> { [] }
+      attribute :failures, Failure, collection: true, default: -> { [] }
+
+      key_value do
+        map "unicode_version", to: :unicode_version
+        map "ucode_version", to: :ucode_version
+        map "generated_at", to: :generated_at
+        map "totals", to: :totals
+        map "by_tier", to: :by_tier
+        map "by_block", to: :by_block
+        map "failures", to: :failures
+      end
+    end
+  end
+end

data/lib/ucode/models/codepoint/glyph.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    class CodePoint < Lutaml::Model::Serializable
+      # Glyph bundle for one codepoint. Records where the SVG lives on
+      # disk and which resolver tier produced it.
+      #
+      # The `svg_path` is relative to the codepoint's own directory
+      # (always "glyph.svg" — the layout is fixed in {Ucode::Repo::Paths}).
+      # The `source` bundle carries the resolver tier name and
+      # provenance string, so the dataset is debuggable end-to-end:
+      # every glyph in the build can be traced back to its origin
+      # (real font, embedded ToUnicode, correlator, or Last Resort).
+      class Glyph < Lutaml::Model::Serializable
+        # Provenance bundle for a glyph — which tier of the 4-tier
+        # resolver produced it. The Ruby class name `Source` mirrors
+        # the wire field name; it is unrelated to the
+        # {Ucode::Glyphs::Source} abstract base.
+        class Source < Lutaml::Model::Serializable
+          attribute :tier, :string
+          attribute :provenance, :string
+
+          key_value do
+            map "tier", to: :tier
+            map "provenance", to: :provenance
+          end
+        end
+
+        attribute :svg_path, :string, default: -> { "glyph.svg" }
+        attribute :source, Source
+
+        key_value do
+          map "svg_path", to: :svg_path
+          map "source", to: :source
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/codepoint.rb

@@ -26,6 +26,7 @@ module Ucode
       autoload :Emoji, "ucode/models/codepoint/emoji"
       autoload :Identifier, "ucode/models/codepoint/identifier"
       autoload :Normalization, "ucode/models/codepoint/normalization"
+      autoload :Glyph, "ucode/models/codepoint/glyph"
 
       # Identity + scalar attributes
       attribute :cp, :integer
@@ -74,6 +75,7 @@ module Ucode
                 collection: true, default: -> { [] }
       attribute :unihan, "Ucode::Models::UnihanEntry"
       attribute :names_list, "Ucode::Models::NamesListEntry"
+      attribute :glyph, Glyph
 
       key_value do
         map "codepoint", to: :cp
@@ -116,6 +118,7 @@ module Ucode
         map "standardized_variants", to: :standardized_variants
         map "unihan", to: :unihan
         map "names_list", to: :names_list
+        map "glyph", to: :glyph
       end
     end
   end

data/lib/ucode/models/glyph_source.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    # One Tier 1 font entry inside a {GlyphSourceMap}. Corresponds to
+    # one `- kind: …` item under a block's `sources:` list in
+    # `config/unicode17_universal_glyph_set.yml`.
+    #
+    # This is the typed representation of a font curation choice. The
+    # {Ucode::Glyphs::Sources::Tier1RealFont} source consumes it to
+    # resolve and load the font; the resolver treats each entry as an
+    # independent tier-1 attempt.
+    #
+    # Wire shape (YAML / JSON identical):
+    #
+    #   kind: fontist          # one of: fontist, path, system
+    #   label: noto-sans       # human + provenance key
+    #   priority: 1            # lower wins; resolver tries in order
+    #   license: OFL           # optional; OFL / PROPRIETARY / etc.
+    #   provenance: "Google Noto Sans"   # optional citation
+    #   path: "/abs/font.ttf"  # required when kind == :path
+    #
+    # `kind` is stored as a plain string on the wire (lutaml-model has
+    # no Symbol adapter for key_value); the {#kind_sym} reader casts it
+    # for internal dispatch.
+    class GlyphSource < Lutaml::Model::Serializable
+      KIND_FONTIST = "fontist"
+      KIND_PATH = "path"
+      KIND_SYSTEM = "system"
+      KINDS = [KIND_FONTIST, KIND_PATH, KIND_SYSTEM].freeze
+      private_constant :KIND_FONTIST, :KIND_PATH, :KIND_SYSTEM, :KINDS
+
+      attribute :kind, :string
+      attribute :label, :string
+      attribute :priority, :integer, default: -> { 100 }
+      attribute :license, :string
+      attribute :provenance, :string
+      attribute :path, :string
+
+      key_value do
+        map "kind", to: :kind
+        map "label", to: :label
+        map "priority", to: :priority
+        map "license", to: :license
+        map "provenance", to: :provenance
+        map "path", to: :path
+      end
+
+      # @return [Symbol] :fontist, :path, :system; raises if kind is
+      #   blank — every entry must declare its kind.
+      def kind_sym
+        raise ArgumentError, "GlyphSource#kind is required" if kind.nil? || kind.empty?
+
+        kind.to_sym
+      end
+
+      # @return [Boolean] true when this entry requires a `path` field
+      #   (kind == :path). Used by the loader to validate structure.
+      def requires_path?
+        kind_sym == :path
+      end
+
+      # Renders this source as the legacy font-spec string consumed by
+      # {Ucode::Glyphs::RealFonts::FontLocator}: `label=/path/to/font`
+      # for kind=path, or `label` (the fontist formula name) for
+      # kind=fontist. The locator's `locate` understands both shapes.
+      #
+      # This is the one adapter method that lets the typed model
+      # integrate with the existing locator without rewriting it.
+      #
+      # @return [String]
+      def to_font_spec
+        case kind_sym
+        when :path
+          raise ArgumentError, "GlyphSource#{label} has kind=path but no path" if path.nil? || path.empty?
+
+          "#{label}=#{path}"
+        when :fontist, :system
+          label
+        end
+      end
+    end
+  end
+end