ucode 0.1.0 → 0.2.0

data/lib/ucode/glyphs/universal_set/manifest_accumulator.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require "digest"
+require "time"
+
+require "ucode/models"
+require "ucode/repo/paths"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Thread-safe accumulator that observes the {Builder}'s per-
+      # codepoint attempts and produces the final
+      # {Ucode::Models::UniversalSetManifest} plus the per-block
+      # breakdown the {ManifestWriter} emits as `reports/by_block.json`.
+      #
+      # Mirrors the {Ucode::Repo::BuildReportAccumulator} pattern from
+      # Mode 1: the orchestrating command passes this instance to the
+      # builder, which calls {#record_build} (or {#record_skip}) from
+      # inside its worker pool. After the drain completes,
+      # {#to_manifest} returns the immutable snapshot.
+      #
+      # == Semantics
+      #
+      # - `codepoints_assigned` counts every codepoint the builder
+      #   attempted (passed the block_filter guard).
+      # - `codepoints_built` counts codepoints whose resolver returned
+      #   a glyph.
+      # - `codepoints_skipped` counts codepoints that resolved to nil
+      #   (no tier produced a glyph) — these are the "gaps" the gaps
+      #   report enumerates.
+      # - `codepoints_failed` counts exceptions recorded via
+      #   {#record_failure}.
+      #
+      # `by_tier` counts the winning tier per codepoint (one increment
+      # per built codepoint). The map uses the wire form ("tier-1",
+      # "pillar-1", ...) so the manifest is stable across Ruby symbol
+      # changes.
+      #
+      # `by_block` is a hash keyed by block_id, with built / skipped /
+      # failed counters per block. Computed from the codepoint stream
+      # the Builder drains — the accumulator reads {CodePoint#block_id}
+      # directly. Block ids follow the canonical underscore form.
+      class ManifestAccumulator
+        TIER_TO_WIRE = {
+          tier1: "tier-1",
+          pillar1: "pillar-1",
+          pillar2: "pillar-2",
+          pillar3: "pillar-3",
+        }.freeze
+        private_constant :TIER_TO_WIRE
+
+        # @param unicode_version [String]
+        # @param ucode_version [String]
+        # @param source_config_sha256 [String]
+        def initialize(unicode_version:, ucode_version:, source_config_sha256:)
+          @unicode_version = unicode_version
+          @ucode_version = ucode_version
+          @source_config_sha256 = source_config_sha256
+          @totals = { codepoints_assigned: 0, codepoints_built: 0,
+                      codepoints_skipped: 0, codepoints_failed: 0 }
+          @by_tier = Hash.new(0)
+          @by_block = Hash.new do |h, block_id|
+            h[block_id] = { built: 0, skipped: 0, failed: 0 }
+          end
+          @entries = []
+          @gaps = []
+          @failures = []
+          @mutex = Mutex.new
+        end
+
+        # Observer entry — the builder calls this for every codepoint
+        # the resolver produced a glyph for. Records the entry and
+        # bumps the built counter + per-tier + per-block rollups.
+        #
+        # @param codepoint [Ucode::Models::CodePoint]
+        # @param result [Ucode::Glyphs::Source::Result] non-nil
+        # @param svg [String] the SVG bytes that were written
+        # @return [void]
+        def record_build(codepoint, result, svg:)
+          entry = build_entry(codepoint.cp, result, svg)
+          tier_wire = wire_tier(result.tier)
+          synchronize do
+            @totals[:codepoints_assigned] += 1
+            @totals[:codepoints_built] += 1
+            @by_tier[tier_wire] += 1
+            @by_block[codepoint.block_id][:built] += 1
+            @entries << entry
+          end
+        end
+
+        # Observer entry — the builder calls this when the resolver
+        # returned nil for a codepoint. Counts the attempt and adds
+        # it to the gaps list for the gaps report.
+        #
+        # @param codepoint [Ucode::Models::CodePoint]
+        # @return [void]
+        def record_skip(codepoint)
+          synchronize do
+            @totals[:codepoints_assigned] += 1
+            @totals[:codepoints_skipped] += 1
+            @by_block[codepoint.block_id][:skipped] += 1
+            @gaps << codepoint.cp
+          end
+        end
+
+        # Record an exception. The builder rescues per-codepoint
+        # errors and routes them here so one bad codepoint doesn't
+        # abort the run.
+        #
+        # @param codepoint [Ucode::Models::CodePoint, nil]
+        # @param error [StandardError]
+        # @return [void]
+        def record_failure(codepoint, error)
+          synchronize do
+            @totals[:codepoints_assigned] += 1 unless codepoint.nil?
+            @totals[:codepoints_failed] += 1
+            @by_block[codepoint&.block_id][:failed] += 1 unless codepoint.nil?
+            @failures << { codepoint: codepoint&.cp,
+                           block_id: codepoint&.block_id,
+                           error_class: error.class.name,
+                           message: error.message }
+          end
+        end
+
+        # @return [Ucode::Models::UniversalSetManifest] immutable snapshot
+        def to_manifest
+          synchronize do
+            Ucode::Models::UniversalSetManifest.new(
+              unicode_version: @unicode_version,
+              ucode_version: @ucode_version,
+              generated_at: Time.now.utc.iso8601,
+              source_config_sha256: @source_config_sha256,
+              totals: Ucode::Models::UniversalSetManifest::Totals.new(@totals),
+              by_tier: @by_tier.dup,
+              entries: @entries.dup,
+            )
+          end
+        end
+
+        # @return [Hash{String=>Hash}] per-block built/skipped/failed
+        #   counts, deep-copied so callers can't mutate accumulator state.
+        def by_block
+          synchronize do
+            @by_block.transform_values(&:dup)
+          end
+        end
+
+        # @return [Array<Integer>] codepoints that resolved to nil, sorted
+        def gaps
+          synchronize { @gaps.sort }
+        end
+
+        # @return [Array<Hash>] recorded failures (each with codepoint,
+        #   block_id, error_class, message)
+        def failures
+          synchronize { @failures.dup }
+        end
+
+        private
+
+        def build_entry(codepoint, result, svg)
+          Ucode::Models::UniversalSetEntry.new(
+            codepoint: codepoint,
+            id: Ucode::Repo::Paths.cp_id(codepoint),
+            tier: wire_tier(result.tier),
+            source: source_label(result.provenance),
+            svg_sha256: sha256(svg),
+            svg_size_bytes: svg.bytesize,
+          )
+        end
+
+        # Extract the source identifier from a dotted provenance
+        # string ("tier-1:noto-sans" -> "noto-sans"). When there's no
+        # `:` separator, returns the input verbatim — defensive
+        # against malformed provenance.
+        def source_label(provenance)
+          provenance.to_s.split(":", 2).last || provenance.to_s
+        end
+
+        def sha256(payload)
+          Digest::SHA256.hexdigest(payload)
+        end
+
+        def wire_tier(symbol)
+          TIER_TO_WIRE.fetch(symbol, symbol.to_s)
+        end
+
+        def synchronize(&)
+          @mutex.synchronize(&)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/manifest_writer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "json"
+
+require "ucode/glyphs/universal_set/idempotency"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Writes the final manifest + reports under the output root.
+      #
+      # One manifest, three reports:
+      #
+      # - `manifest.json` — full {Ucode::Models::UniversalSetManifest}.
+      # - `reports/by_tier.json` — `by_tier` counts alone (small file
+      #   for quick "how much of the set is tier 1?" inspection).
+      # - `reports/by_block.json` — per-block built/skipped totals,
+      #   computed from the manifest's entries + the codepoint's
+      #   block_id (resolved by the Builder).
+      # - `reports/gaps.json` — array of codepoint integers that
+      #   resolved to nil (should be empty for a healthy run).
+      #
+      # All writes are atomic via {Idempotency} (which includes
+      # {Ucode::Repo::AtomicWrites}). Re-running on an unchanged
+      # manifest is a no-op modulo `generated_at`.
+      class ManifestWriter
+        include Idempotency
+
+        # @param output_root [String, Pathname]
+        def initialize(output_root)
+          @output_root = Pathname.new(output_root)
+        end
+
+        # Write the manifest + reports atomically.
+        #
+        # @param manifest [Ucode::Models::UniversalSetManifest]
+        # @param by_block [Hash{String=>Hash}] per-block breakdown:
+        #   `{ "Basic_Latin" => { built: 64, skipped: 0, failed: 0 } }`.
+        #   Computed by the {Builder}; this writer just serializes it.
+        # @param gaps [Array<Integer>] codepoints with no glyph
+        # @param failures [Array<Hash>] per-codepoint failures
+        # @return [Pathname] path to the written manifest
+        def write(manifest, by_block:, gaps:, failures:)
+          write_atomic(manifest_path(@output_root), manifest_to_json(manifest))
+          write_atomic(by_tier_report_path(@output_root), to_pretty_json(manifest.by_tier))
+          write_atomic(by_block_report_path(@output_root), to_pretty_json(by_block))
+          write_atomic(gaps_report_path(@output_root),
+                       to_pretty_json(gaps: gaps, failures: failures))
+          manifest_path(@output_root)
+        end
+
+        private
+
+        def manifest_to_json(manifest)
+          manifest.to_json(pretty: true)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/pre_build_check.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/error"
+require "ucode/glyphs/real_fonts/font_locator"
+require "ucode/glyphs/source_config"
+require "ucode/glyphs/source_config/coverage_assertion"
+require "ucode/glyphs/source_config/gap_report"
+require "ucode/models/glyph_source_map"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Report produced by {PreBuildCheck}. Carries the raw findings
+      # without deciding whether to abort — that decision lives on
+      # the check itself so callers can inspect the report without
+      # rescuing.
+      PreBuildReport = Struct.new(:source_config_path, :unicode_version,
+                                  :config_loaded, :missing_fonts,
+                                  :coverage_gaps, keyword_init: true) do
+        def ok?
+          config_loaded && missing_fonts.empty?
+        end
+      end
+
+      # Pre-flight validation for a universal-set build. Runs the
+      # three checks TODO 31 specifies:
+      #
+      # 1. **Source config loads cleanly.** `SourceConfig.new(path:)`
+      #    returns a map without raising, and the file exists.
+      # 2. **All fonts present.** Every source in the map resolves to
+      #    a file on disk (kind=path) or via fontist's index
+      #    (kind=fontist, install: false). Missing fonts are listed.
+      # 3. **Coverage assertion runs.** TODO 29's CoverageAssertion
+      #    walks every assigned codepoint; gaps are surfaced but do
+      #    not abort (expected for residual curation cases).
+      #
+      # The check raises {Ucode::UniversalSetPreBuildError} when
+      # `missing_fonts` is non-empty or the config fails to load. The
+      # CLI catches this and renders the failing checks; the build
+      # never starts with known-bad inputs.
+      class PreBuildCheck
+        # @param source_config_path [String, Pathname]
+        # @param database [Ucode::Database] open database for the
+        #   target Unicode version. Used by CoverageAssertion.
+        # @param cmaps [#covers?] defaults to RealFonts::CmapCache.
+        #   Injectable for testing (e.g. StaticCmaps).
+        # @param font_locator [#locate] defaults to a fresh
+        #   FontLocator. Injectable for testing.
+        def initialize(source_config_path:, database:, cmaps: nil,
+                       font_locator: RealFonts::FontLocator.new)
+          @source_config_path = Pathname.new(source_config_path)
+          @database = database
+          @cmaps = cmaps || RealFonts::CmapCache.new
+          @font_locator = font_locator
+        end
+
+        # @raise [Ucode::UniversalSetPreBuildError] when missing_fonts
+        #   is non-empty or the source config fails to load.
+        # @return [PreBuildReport]
+        def call
+          report = build_report
+          unless report.ok?
+            raise Ucode::UniversalSetPreBuildError.new(
+              "pre-build validation failed",
+              context: {
+                source_config_path: @source_config_path.to_s,
+                missing_fonts: report.missing_fonts,
+                config_loaded: report.config_loaded,
+              },
+            )
+          end
+
+          report
+        end
+
+        private
+
+        def build_report
+          config, loaded = load_config
+          missing = loaded ? collect_missing_fonts(config.map) : []
+          gaps = loaded ? run_coverage_assertion(config.map) : empty_gap_report
+
+          PreBuildReport.new(
+            source_config_path: @source_config_path.to_s,
+            unicode_version: @database.ucd_version,
+            config_loaded: loaded,
+            missing_fonts: missing,
+            coverage_gaps: gaps,
+          )
+        end
+
+        def load_config
+          config = SourceConfig.new(path: @source_config_path)
+          [config, config.exist?]
+        rescue StandardError => e
+          warn_with(e)
+          [nil, false]
+        end
+
+        def collect_missing_fonts(source_map)
+          unique_sources(source_map).each_with_object([]) do |src, acc|
+            acc.concat(findings_for(src))
+          end
+        end
+
+        # All distinct sources referenced by the map, typed. Block-
+        # specific sources plus the top-level defaults. Deduplicated
+        # by (kind, label, path) so a font referenced by N blocks is
+        # only checked once.
+        def unique_sources(source_map)
+          block_sources = source_map.block_ids.flat_map do |block_id|
+            source_map.sources_for(block_id)
+          end
+          (block_sources + source_map.default_sources).uniq do |src|
+            [src.kind, src.label, src.path]
+          end
+        end
+
+        # Resolve one source against the filesystem / fontist index.
+        # Returns an array of findings (empty when the source is OK).
+        def findings_for(src)
+          kind = safe_kind(src)
+          case kind
+          when :path
+            path_resolves?(src.path) ? [] : [missing_path(src)]
+          when :fontist, :system
+            fontist_resolves?(src.label) ? [] : [missing_fontist(src, kind)]
+          when nil
+            [malformed_entry(src)]
+          end
+        end
+
+        # Returns the source's kind as a symbol, or nil when the
+        # entry is malformed (no `kind` field). A nil kind is itself
+        # a finding — every entry must declare its kind.
+        def safe_kind(src)
+          src.kind.nil? || src.kind.empty? ? nil : src.kind.to_sym
+        end
+
+        def path_resolves?(raw_path)
+          return false if raw_path.nil? || raw_path.empty?
+
+          expanded = File.expand_path(raw_path)
+          Dir.glob(expanded).any? { |p| File.file?(p) }
+        end
+
+        def fontist_resolves?(label)
+          return false if label.nil? || label.empty?
+
+          result = @font_locator.locate(label, install: false)
+          !result.nil? && !result.path.nil?
+        rescue StandardError
+          false
+        end
+
+        def missing_path(src)
+          { kind: "path", label: src.label, spec: src.path,
+            reason: "file not found at #{src.path.inspect}" }
+        end
+
+        def missing_fontist(src, kind)
+          { kind: kind.to_s, label: src.label, spec: src.label,
+            reason: "fontist could not resolve formula #{src.label.inspect}" }
+        end
+
+        def malformed_entry(src)
+          { kind: "(missing)", label: src.label,
+            spec: src.path || src.label,
+            reason: "source entry has no `kind` field — must be fontist, path, or system" }
+        end
+
+        def run_coverage_assertion(source_map)
+          SourceConfig::CoverageAssertion.new(
+            source_map: source_map, database: @database, cmaps: @cmaps,
+          ).call
+        end
+
+        def empty_gap_report
+          SourceConfig::GapReport.new(
+            unicode_version: @database.ucd_version,
+            generated_at: Time.now.utc.iso8601,
+            gaps_by_block: {}.freeze,
+            total_gaps: 0,
+          )
+        end
+
+        def warn_with(error)
+          Ucode.configuration.logger&.warn do
+            "pre-build: source config failed to load: #{error.class}: #{error.message}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/validator.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+require "time"
+
+require "ucode/models"
+require "ucode/glyphs/universal_set/idempotency"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Post-build validator for a universal-set build (TODO 31 §Post-
+      # build validation). Walks the manifest + glyphs directory and
+      # runs the four structural checks:
+      #
+      #   1. `manifest_loadable` — `manifest.json` parses via
+      #      `Ucode::Models::UniversalSetManifest.from_hash`.
+      #   2. `glyph_files_present` — every entry has a corresponding
+      #      `glyphs/<id>.svg` on disk.
+      #   3. `totals_reconcile` — manifest totals match the actual
+      #      entry counts (`built == entries.size`).
+      #   4. `provenance_complete` — every entry has non-empty `tier`
+      #      and `source`.
+      #
+      # Tofu (pillar-3) investigation and per-tier / per-block
+      # breakdowns live in {CoverageReport} — those are coverage
+      # questions, not structural ones. The idempotency check (TODO 31
+      # §5) is exercised by re-running the build, not by reading
+      # on-disk state.
+      #
+      # The validator is stateless from the outside: one call to
+      # {#validate} walks the manifest, builds a
+      # {Ucode::Models::ValidationReport}, and writes it atomically to
+      # `<output_root>/reports/validation.json`. Safe to re-run.
+      class Validator
+        include Idempotency
+
+        CHECK_MANIFEST = "manifest_loadable"
+        CHECK_GLYPHS = "glyph_files_present"
+        CHECK_TOTALS = "totals_reconcile"
+        CHECK_PROVENANCE = "provenance_complete"
+        ALL_CHECKS = [
+          CHECK_MANIFEST, CHECK_GLYPHS, CHECK_TOTALS, CHECK_PROVENANCE
+        ].freeze
+        private_constant :ALL_CHECKS
+
+        # @param output_root [String, Pathname] directory holding
+        #   `manifest.json` + `glyphs/` + `reports/`.
+        # @param unicode_version [String, nil] stamped onto the report;
+        #   nil falls back to the manifest's recorded version.
+        def initialize(output_root, unicode_version: nil)
+          @output_root = Pathname.new(output_root)
+          @unicode_version = unicode_version
+        end
+
+        # Walk the manifest + glyphs dir, run all checks, emit
+        # `reports/validation.json`. Returns the structured outcome.
+        #
+        # @return [Hash] { report:, report_path:, passed:, manifest_loaded: }
+        def validate
+          manifest, manifest_failures = load_manifest
+          entries = manifest ? manifest.entries : []
+
+          findings = manifest_failures.dup
+          if manifest
+            findings.concat(check_glyph_files(entries))
+            findings.concat(check_totals(manifest))
+            findings.concat(check_provenance(entries))
+          end
+
+          report = build_report(entries, findings, manifest)
+          report_path = write_report(report)
+          {
+            report: report,
+            report_path: report_path,
+            passed: report.totals.failures.zero?,
+            manifest_loaded: !manifest.nil?,
+          }
+        end
+
+        private
+
+        def load_manifest
+          path = manifest_path(@output_root)
+          unless path.exist?
+            return [nil, [make_failure(CHECK_MANIFEST,
+                                       "manifest.json not found at #{path}")]]
+          end
+
+          hash = JSON.parse(path.read)
+          model = Ucode::Models::UniversalSetManifest.from_hash(hash)
+          [model, []]
+        rescue JSON::ParserError => e
+          [nil, [make_failure(CHECK_MANIFEST,
+                              "manifest JSON parse failed: #{e.message}")]]
+        rescue StandardError => e
+          [nil, [make_failure(CHECK_MANIFEST,
+                              "manifest deserialization failed: #{e.class}: #{e.message}")]]
+        end
+
+        def check_glyph_files(entries)
+          entries.each_with_object([]) do |entry, acc|
+            path = glyph_path(@output_root, entry.id)
+            next if path.exist?
+
+            acc << make_failure(CHECK_GLYPHS,
+                                "missing glyph file at #{path}",
+                                codepoint: entry.codepoint)
+          end
+        end
+
+        def check_totals(manifest)
+          entries_size = manifest.entries.size
+          built = manifest.totals.codepoints_built
+          return [] if built == entries_size
+
+          [make_failure(CHECK_TOTALS,
+                        "totals.codepoints_built=#{built} but entries.size=#{entries_size}")]
+        end
+
+        def check_provenance(entries)
+          entries.each_with_object([]) do |entry, acc|
+            acc.concat(provenance_findings_for(entry))
+          end
+        end
+
+        def provenance_findings_for(entry)
+          findings = []
+          if entry.tier.nil? || entry.tier.empty?
+            findings << make_failure(CHECK_PROVENANCE, "entry has no tier",
+                                     codepoint: entry.codepoint)
+          end
+          if entry.source.nil? || entry.source.empty?
+            findings << make_failure(CHECK_PROVENANCE, "entry has no source",
+                                     codepoint: entry.codepoint)
+          end
+          findings
+        end
+
+        def build_report(entries, findings, manifest)
+          checks = ALL_CHECKS.map do |name|
+            build_check_summary(name, findings, entries.size, manifest)
+          end
+
+          Ucode::Models::ValidationReport.new(
+            unicode_version: (@unicode_version || manifest&.unicode_version).to_s,
+            generated_at: Time.now.utc.iso8601,
+            totals: Ucode::Models::ValidationReport::Totals.new(
+              codepoints_checked: entries.size,
+              failures: findings.length,
+              checks_run: checks.count { |c| c.status != "skipped" },
+              checks_passed: checks.count { |c| c.status == "passed" },
+            ),
+            checks: checks,
+            failures: findings,
+          )
+        end
+
+        def build_check_summary(name, findings, entries_size, manifest)
+          count = findings.count { |f| f.check == name }
+          status = check_status(name, count, manifest)
+          total = check_total(name, entries_size, manifest)
+
+          Ucode::Models::ValidationReport::CheckSummary.new(
+            name: name,
+            status: status,
+            total: total,
+            failures: count,
+          )
+        end
+
+        def check_status(name, count, manifest)
+          return "skipped" if manifest.nil? && name != CHECK_MANIFEST
+
+          count.zero? ? "passed" : "failed"
+        end
+
+        def check_total(name, entries_size, manifest)
+          return 1 if name == CHECK_MANIFEST
+          return 0 if manifest.nil?
+
+          entries_size
+        end
+
+        def write_report(report)
+          path = @output_root.join(REPORTS_DIR, "validation.json")
+          path.dirname.mkpath
+          write_atomic(path, report.to_json(pretty: true))
+          path
+        end
+
+        def make_failure(check, message, codepoint: nil)
+          Ucode::Models::ValidationReport::Failure.new(
+            codepoint: codepoint,
+            block: nil,
+            check: check,
+            message: message,
+          )
+        end
+      end
+    end
+  end
+end