ucode 0.1.0 → 0.1.1

data/lib/ucode/glyphs/universal_set/pre_build_check.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/error"
+require "ucode/glyphs/real_fonts/font_locator"
+require "ucode/glyphs/source_config"
+require "ucode/glyphs/source_config/coverage_assertion"
+require "ucode/glyphs/source_config/gap_report"
+require "ucode/models/glyph_source_map"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Report produced by {PreBuildCheck}. Carries the raw findings
+      # without deciding whether to abort — that decision lives on
+      # the check itself so callers can inspect the report without
+      # rescuing.
+      PreBuildReport = Struct.new(:source_config_path, :unicode_version,
+                                  :config_loaded, :missing_fonts,
+                                  :coverage_gaps, keyword_init: true) do
+        def ok?
+          config_loaded && missing_fonts.empty?
+        end
+      end
+
+      # Pre-flight validation for a universal-set build. Runs the
+      # three checks TODO 31 specifies:
+      #
+      # 1. **Source config loads cleanly.** `SourceConfig.new(path:)`
+      #    returns a map without raising, and the file exists.
+      # 2. **All fonts present.** Every source in the map resolves to
+      #    a file on disk (kind=path) or via fontist's index
+      #    (kind=fontist, install: false). Missing fonts are listed.
+      # 3. **Coverage assertion runs.** TODO 29's CoverageAssertion
+      #    walks every assigned codepoint; gaps are surfaced but do
+      #    not abort (expected for residual curation cases).
+      #
+      # The check raises {Ucode::UniversalSetPreBuildError} when
+      # `missing_fonts` is non-empty or the config fails to load. The
+      # CLI catches this and renders the failing checks; the build
+      # never starts with known-bad inputs.
+      class PreBuildCheck
+        # @param source_config_path [String, Pathname]
+        # @param database [Ucode::Database] open database for the
+        #   target Unicode version. Used by CoverageAssertion.
+        # @param cmaps [#covers?] defaults to RealFonts::CmapCache.
+        #   Injectable for testing (e.g. StaticCmaps).
+        # @param font_locator [#locate] defaults to a fresh
+        #   FontLocator. Injectable for testing.
+        def initialize(source_config_path:, database:, cmaps: nil,
+                       font_locator: RealFonts::FontLocator.new)
+          @source_config_path = Pathname.new(source_config_path)
+          @database = database
+          @cmaps = cmaps || RealFonts::CmapCache.new
+          @font_locator = font_locator
+        end
+
+        # @raise [Ucode::UniversalSetPreBuildError] when missing_fonts
+        #   is non-empty or the source config fails to load.
+        # @return [PreBuildReport]
+        def call
+          report = build_report
+          unless report.ok?
+            raise Ucode::UniversalSetPreBuildError.new(
+              "pre-build validation failed",
+              context: {
+                source_config_path: @source_config_path.to_s,
+                missing_fonts: report.missing_fonts,
+                config_loaded: report.config_loaded,
+              },
+            )
+          end
+
+          report
+        end
+
+        private
+
+        def build_report
+          config, loaded = load_config
+          missing = loaded ? collect_missing_fonts(config.map) : []
+          gaps = loaded ? run_coverage_assertion(config.map) : empty_gap_report
+
+          PreBuildReport.new(
+            source_config_path: @source_config_path.to_s,
+            unicode_version: @database.ucd_version,
+            config_loaded: loaded,
+            missing_fonts: missing,
+            coverage_gaps: gaps,
+          )
+        end
+
+        def load_config
+          config = SourceConfig.new(path: @source_config_path)
+          [config, config.exist?]
+        rescue StandardError => e
+          warn_with(e)
+          [nil, false]
+        end
+
+        def collect_missing_fonts(source_map)
+          unique_sources(source_map).each_with_object([]) do |src, acc|
+            acc.concat(findings_for(src))
+          end
+        end
+
+        # All distinct sources referenced by the map, typed. Block-
+        # specific sources plus the top-level defaults. Deduplicated
+        # by (kind, label, path) so a font referenced by N blocks is
+        # only checked once.
+        def unique_sources(source_map)
+          block_sources = source_map.block_ids.flat_map do |block_id|
+            source_map.sources_for(block_id)
+          end
+          (block_sources + source_map.default_sources).uniq do |src|
+            [src.kind, src.label, src.path]
+          end
+        end
+
+        # Resolve one source against the filesystem / fontist index.
+        # Returns an array of findings (empty when the source is OK).
+        def findings_for(src)
+          kind = safe_kind(src)
+          case kind
+          when :path
+            path_resolves?(src.path) ? [] : [missing_path(src)]
+          when :fontist, :system
+            fontist_resolves?(src.label) ? [] : [missing_fontist(src, kind)]
+          when nil
+            [malformed_entry(src)]
+          end
+        end
+
+        # Returns the source's kind as a symbol, or nil when the
+        # entry is malformed (no `kind` field). A nil kind is itself
+        # a finding — every entry must declare its kind.
+        def safe_kind(src)
+          src.kind.nil? || src.kind.empty? ? nil : src.kind.to_sym
+        end
+
+        def path_resolves?(raw_path)
+          return false if raw_path.nil? || raw_path.empty?
+
+          expanded = File.expand_path(raw_path)
+          Dir.glob(expanded).any? { |p| File.file?(p) }
+        end
+
+        def fontist_resolves?(label)
+          return false if label.nil? || label.empty?
+
+          result = @font_locator.locate(label, install: false)
+          !result.nil? && !result.path.nil?
+        rescue StandardError
+          false
+        end
+
+        def missing_path(src)
+          { kind: "path", label: src.label, spec: src.path,
+            reason: "file not found at #{src.path.inspect}" }
+        end
+
+        def missing_fontist(src, kind)
+          { kind: kind.to_s, label: src.label, spec: src.label,
+            reason: "fontist could not resolve formula #{src.label.inspect}" }
+        end
+
+        def malformed_entry(src)
+          { kind: "(missing)", label: src.label,
+            spec: src.path || src.label,
+            reason: "source entry has no `kind` field — must be fontist, path, or system" }
+        end
+
+        def run_coverage_assertion(source_map)
+          SourceConfig::CoverageAssertion.new(
+            source_map: source_map, database: @database, cmaps: @cmaps,
+          ).call
+        end
+
+        def empty_gap_report
+          SourceConfig::GapReport.new(
+            unicode_version: @database.ucd_version,
+            generated_at: Time.now.utc.iso8601,
+            gaps_by_block: {}.freeze,
+            total_gaps: 0,
+          )
+        end
+
+        def warn_with(error)
+          Ucode.configuration.logger&.warn do
+            "pre-build: source config failed to load: #{error.class}: #{error.message}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/validator.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+require "time"
+
+require "ucode/models"
+require "ucode/glyphs/universal_set/idempotency"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Post-build validator for a universal-set build (TODO 31 §Post-
+      # build validation). Walks the manifest + glyphs directory and
+      # runs the four structural checks:
+      #
+      #   1. `manifest_loadable` — `manifest.json` parses via
+      #      `Ucode::Models::UniversalSetManifest.from_hash`.
+      #   2. `glyph_files_present` — every entry has a corresponding
+      #      `glyphs/<id>.svg` on disk.
+      #   3. `totals_reconcile` — manifest totals match the actual
+      #      entry counts (`built == entries.size`).
+      #   4. `provenance_complete` — every entry has non-empty `tier`
+      #      and `source`.
+      #
+      # Tofu (pillar-3) investigation and per-tier / per-block
+      # breakdowns live in {CoverageReport} — those are coverage
+      # questions, not structural ones. The idempotency check (TODO 31
+      # §5) is exercised by re-running the build, not by reading
+      # on-disk state.
+      #
+      # The validator is stateless from the outside: one call to
+      # {#validate} walks the manifest, builds a
+      # {Ucode::Models::ValidationReport}, and writes it atomically to
+      # `<output_root>/reports/validation.json`. Safe to re-run.
+      class Validator
+        include Idempotency
+
+        CHECK_MANIFEST = "manifest_loadable"
+        CHECK_GLYPHS = "glyph_files_present"
+        CHECK_TOTALS = "totals_reconcile"
+        CHECK_PROVENANCE = "provenance_complete"
+        ALL_CHECKS = [
+          CHECK_MANIFEST, CHECK_GLYPHS, CHECK_TOTALS, CHECK_PROVENANCE
+        ].freeze
+        private_constant :ALL_CHECKS
+
+        # @param output_root [String, Pathname] directory holding
+        #   `manifest.json` + `glyphs/` + `reports/`.
+        # @param unicode_version [String, nil] stamped onto the report;
+        #   nil falls back to the manifest's recorded version.
+        def initialize(output_root, unicode_version: nil)
+          @output_root = Pathname.new(output_root)
+          @unicode_version = unicode_version
+        end
+
+        # Walk the manifest + glyphs dir, run all checks, emit
+        # `reports/validation.json`. Returns the structured outcome.
+        #
+        # @return [Hash] { report:, report_path:, passed:, manifest_loaded: }
+        def validate
+          manifest, manifest_failures = load_manifest
+          entries = manifest ? manifest.entries : []
+
+          findings = manifest_failures.dup
+          if manifest
+            findings.concat(check_glyph_files(entries))
+            findings.concat(check_totals(manifest))
+            findings.concat(check_provenance(entries))
+          end
+
+          report = build_report(entries, findings, manifest)
+          report_path = write_report(report)
+          {
+            report: report,
+            report_path: report_path,
+            passed: report.totals.failures.zero?,
+            manifest_loaded: !manifest.nil?,
+          }
+        end
+
+        private
+
+        def load_manifest
+          path = manifest_path(@output_root)
+          unless path.exist?
+            return [nil, [make_failure(CHECK_MANIFEST,
+                                       "manifest.json not found at #{path}")]]
+          end
+
+          hash = JSON.parse(path.read)
+          model = Ucode::Models::UniversalSetManifest.from_hash(hash)
+          [model, []]
+        rescue JSON::ParserError => e
+          [nil, [make_failure(CHECK_MANIFEST,
+                              "manifest JSON parse failed: #{e.message}")]]
+        rescue StandardError => e
+          [nil, [make_failure(CHECK_MANIFEST,
+                              "manifest deserialization failed: #{e.class}: #{e.message}")]]
+        end
+
+        def check_glyph_files(entries)
+          entries.each_with_object([]) do |entry, acc|
+            path = glyph_path(@output_root, entry.id)
+            next if path.exist?
+
+            acc << make_failure(CHECK_GLYPHS,
+                                "missing glyph file at #{path}",
+                                codepoint: entry.codepoint)
+          end
+        end
+
+        def check_totals(manifest)
+          entries_size = manifest.entries.size
+          built = manifest.totals.codepoints_built
+          return [] if built == entries_size
+
+          [make_failure(CHECK_TOTALS,
+                        "totals.codepoints_built=#{built} but entries.size=#{entries_size}")]
+        end
+
+        def check_provenance(entries)
+          entries.each_with_object([]) do |entry, acc|
+            acc.concat(provenance_findings_for(entry))
+          end
+        end
+
+        def provenance_findings_for(entry)
+          findings = []
+          if entry.tier.nil? || entry.tier.empty?
+            findings << make_failure(CHECK_PROVENANCE, "entry has no tier",
+                                     codepoint: entry.codepoint)
+          end
+          if entry.source.nil? || entry.source.empty?
+            findings << make_failure(CHECK_PROVENANCE, "entry has no source",
+                                     codepoint: entry.codepoint)
+          end
+          findings
+        end
+
+        def build_report(entries, findings, manifest)
+          checks = ALL_CHECKS.map do |name|
+            build_check_summary(name, findings, entries.size, manifest)
+          end
+
+          Ucode::Models::ValidationReport.new(
+            unicode_version: (@unicode_version || manifest&.unicode_version).to_s,
+            generated_at: Time.now.utc.iso8601,
+            totals: Ucode::Models::ValidationReport::Totals.new(
+              codepoints_checked: entries.size,
+              failures: findings.length,
+              checks_run: checks.count { |c| c.status != "skipped" },
+              checks_passed: checks.count { |c| c.status == "passed" },
+            ),
+            checks: checks,
+            failures: findings,
+          )
+        end
+
+        def build_check_summary(name, findings, entries_size, manifest)
+          count = findings.count { |f| f.check == name }
+          status = check_status(name, count, manifest)
+          total = check_total(name, entries_size, manifest)
+
+          Ucode::Models::ValidationReport::CheckSummary.new(
+            name: name,
+            status: status,
+            total: total,
+            failures: count,
+          )
+        end
+
+        def check_status(name, count, manifest)
+          return "skipped" if manifest.nil? && name != CHECK_MANIFEST
+
+          count.zero? ? "passed" : "failed"
+        end
+
+        def check_total(name, entries_size, manifest)
+          return 1 if name == CHECK_MANIFEST
+          return 0 if manifest.nil?
+
+          entries_size
+        end
+
+        def write_report(report)
+          path = @output_root.join(REPORTS_DIR, "validation.json")
+          path.dirname.mkpath
+          write_atomic(path, report.to_json(pretty: true))
+          path
+        end
+
+        def make_failure(check, message, codepoint: nil)
+          Ucode::Models::ValidationReport::Failure.new(
+            codepoint: codepoint,
+            block: nil,
+            check: check,
+            message: message,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Glyphs
+    # Universal glyph set — one SVG per assigned Unicode codepoint,
+    # sourced via the 4-tier resolver using the curated Tier 1 config
+    # from TODO 23. The set is the canonical reference for "what
+    # Unicode 17 looks like" — every codepoint has exactly one glyph,
+    # with documented provenance, in a flat layout designed for fast
+    # lookup by audits (TODO 25) and the fontist.org consumer
+    # (TODO 27).
+    #
+    # Output layout (per TODO 24):
+    #
+    #   output/universal_glyph_set/
+    #   ├── manifest.json             # one entry per codepoint + totals
+    #   ├── glyphs/
+    #   │   ├── U+0000.svg
+    #   │   └── ...
+    #   └── reports/
+    #       ├── by_tier.json          # tier-1: N1, pillar-1: N2, ...
+    #       ├── by_block.json         # per-block tier breakdown
+    #       └── gaps.json             # assigned codepoints with no glyph
+    #
+    # Components:
+    #
+    # - {Builder} drains a codepoint stream through the resolver and
+    #   writes glyphs + manifest atomically.
+    # - {ManifestAccumulator} is the thread-safe tally that produces
+    #   the final {Ucode::Models::UniversalSetManifest}.
+    # - {ManifestWriter} emits the manifest and per-tier / per-block /
+    #   gaps reports under the output root.
+    # - {Idempotency} wraps {Ucode::Repo::AtomicWrites} with the
+    #   "skip if SVG unchanged" semantic documented in TODO 24.
+    module UniversalSet
+      autoload :Builder, "ucode/glyphs/universal_set/builder"
+      autoload :ManifestAccumulator, "ucode/glyphs/universal_set/manifest_accumulator"
+      autoload :ManifestWriter, "ucode/glyphs/universal_set/manifest_writer"
+      autoload :Idempotency, "ucode/glyphs/universal_set/idempotency"
+      autoload :PreBuildCheck, "ucode/glyphs/universal_set/pre_build_check"
+      autoload :Validator, "ucode/glyphs/universal_set/validator"
+      autoload :CoverageReport, "ucode/glyphs/universal_set/coverage_report"
+    end
+  end
+end

data/lib/ucode/glyphs.rb

@@ -23,5 +23,11 @@ module Ucode
     autoload :LastResort, "ucode/glyphs/last_resort"
     autoload :EmbeddedFonts, "ucode/glyphs/embedded_fonts"
     autoload :RealFonts, "ucode/glyphs/real_fonts"
+    autoload :Source, "ucode/glyphs/source"
+    autoload :Resolver, "ucode/glyphs/resolver"
+    autoload :SourceConfig, "ucode/glyphs/source_config"
+    autoload :SourceBuilder, "ucode/glyphs/source_builder"
+    autoload :Sources, "ucode/glyphs/sources"
+    autoload :UniversalSet, "ucode/glyphs/universal_set"
   end
 end

data/lib/ucode/models/audit/baseline.rb

@@ -18,6 +18,11 @@ module Ucode
         attribute :fontisan_version, :string
         attribute :source, :string
         attribute :generated_at, :string
+        # Which CoverageReference produced the per-block counts.
+        # "ucd" for UcdOnlyReference (default), "universal-set" for
+        # UniversalSetReference (TODO 25). nil on legacy reports —
+        # consumers should treat nil as "ucd".
+        attribute :reference_kind, :string
 
         key_value do
           map "unicode_version",  to: :unicode_version
@@ -25,6 +30,7 @@ module Ucode
           map "fontisan_version", to: :fontisan_version
           map "source",           to: :source
           map "generated_at",     to: :generated_at
+          map "reference_kind",   to: :reference_kind
         end
       end
     end

data/lib/ucode/models/audit/block_summary.rb

@@ -30,6 +30,12 @@ module Ucode
         attribute :status, :string
         attribute :missing_codepoints, :integer, collection: true, default: -> { [] }
         attribute :covered_codepoints, :integer, collection: true, default: -> { [] }
+        # Per-codepoint provenance for the missing set. Populated only
+        # when the audit ran against a UniversalSetReference (TODO 25).
+        # Empty for UCD-only audits — the field serializes as [] and
+        # consumers treat that as "no provenance available".
+        attribute :missing_codepoint_provenance, CodepointProvenance,
+                  collection: true, default: -> { [] }
 
         key_value do
           map "name",               to: :name
@@ -44,6 +50,7 @@ module Ucode
           map "status",             to: :status
           map "missing_codepoints", to: :missing_codepoints
           map "covered_codepoints", to: :covered_codepoints
+          map "missing_codepoint_provenance", to: :missing_codepoint_provenance
         end
 
         # Derive the canonical status string for a block given its

data/lib/ucode/models/audit/codepoint_provenance.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Per-codepoint provenance row attached to a {BlockSummary}'s
+      # `missing_codepoint_provenance` collection.
+      #
+      # Populated only when the audit used a
+      # {Ucode::Audit::UniversalSetReference}. UCD-only audits omit
+      # the field entirely — preserving the legacy wire shape.
+      #
+      # Wire shape (one entry per missing codepoint):
+      #
+      #   {
+      #     "codepoint": 10981,
+      #     "tier": "tier-1",
+      #     "source": "lentariso"
+      #   }
+      #
+      # `tier` and `source` mirror the universal-set manifest
+      # ({UniversalSetEntry}) and let downstream renderers (TODO 26)
+      # display the missing glyph + its provenance next to each row.
+      class CodepointProvenance < Lutaml::Model::Serializable
+        attribute :codepoint, :integer
+        attribute :tier, :string
+        attribute :source, :string
+
+        key_value do
+          map "codepoint", to: :codepoint
+          map "tier",      to: :tier
+          map "source",    to: :source
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/audit/release_face.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Per-face entry in a {ReleaseFormulaEntry}.
+      #
+      # Compact card view derived from a per-face {AuditReport}. Carries
+      # identity + coverage totals + relative paths into the release
+      # tree. The fontist.org renderer iterates these cards to build its
+      # font index; clicking a card fetches the per-face audit subtree
+      # at `index_path`.
+      class ReleaseFaceEntry < Lutaml::Model::Serializable
+        attribute :postscript_name,    :string
+        attribute :family_name,        :string
+        attribute :weight_class,       :integer
+        attribute :total_codepoints,   :integer
+        attribute :covered_codepoints, :integer
+        attribute :blocks_complete,    :integer
+        attribute :blocks_partial,     :integer
+        attribute :source_sha256,      :string
+        attribute :index_path,         :string
+        attribute :html_path,          :string
+
+        key_value do
+          map "postscript_name",    to: :postscript_name
+          map "family_name",        to: :family_name
+          map "weight_class",       to: :weight_class
+          map "total_codepoints",   to: :total_codepoints
+          map "covered_codepoints", to: :covered_codepoints
+          map "blocks_complete",    to: :blocks_complete
+          map "blocks_partial",     to: :blocks_partial
+          map "source_sha256",      to: :source_sha256
+          map "index_path",         to: :index_path
+          map "html_path",          to: :html_path
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/audit/release_formula.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Per-formula entry in a {ReleaseManifest}.
+      #
+      # A formula is a single fontist formula (one installable unit).
+      # Each formula contributes one or more face entries to the release
+      # tree. The `slug` is the formula's URL-safe identifier used as
+      # the directory name under `<release_root>/audit/<slug>/`.
+      #
+      # `source_path` records where the original library audit ran so a
+      # consumer reading the manifest can trace the audit back to its
+      # input directory.
+      class ReleaseFormulaEntry < Lutaml::Model::Serializable
+        attribute :slug, :string
+        attribute :source_path, :string
+        attribute :faces_total, :integer
+        attribute :faces, ReleaseFaceEntry, collection: true, default: -> { [] }
+
+        key_value do
+          map "slug",        to: :slug
+          map "source_path", to: :source_path
+          map "faces_total", to: :faces_total
+          map "faces",       to: :faces
+        end
+      end
+    end
+  end
+end

data/lib/ucode/models/audit/release_manifest.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Models
+    module Audit
+      # Top-level release manifest for the fontist.org-consumable
+      # artifact (TODO 27).
+      #
+      # One manifest per release tree at `<release_root>/manifest.json`.
+      # Records the ucode/unicode versions, optional source-config
+      # sha256 (for Tier 1 curation provenance), aggregate formula/face
+      # counts, the universal-set reference section, and the per-formula
+      # face index.
+      #
+      # fontist.org's `scripts/fetch-data.sh` reads this manifest first
+      # to decide whether to fetch the universal-set zip and which
+      # per-formula audit subtrees to pull.
+      class ReleaseManifest < Lutaml::Model::Serializable
+        attribute :ucode_version,        :string
+        attribute :unicode_version,      :string
+        attribute :generated_at,         :string
+        attribute :source_config_sha256, :string
+        attribute :formulas_total,       :integer
+        attribute :faces_total,          :integer
+        attribute :universal_set, ReleaseUniversalSet
+        attribute :formulas, ReleaseFormulaEntry, collection: true, default: -> { [] }
+
+        key_value do
+          map "ucode_version",        to: :ucode_version
+          map "unicode_version",      to: :unicode_version
+          map "generated_at",         to: :generated_at
+          map "source_config_sha256", to: :source_config_sha256
+          map "formulas_total",       to: :formulas_total
+          map "faces_total",          to: :faces_total
+          map "universal_set",        to: :universal_set
+          map "formulas",             to: :formulas
+        end
+      end
+    end
+  end
+end