ucode 0.1.0 → 0.1.1

data/lib/ucode/glyphs/universal_set/builder.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/glyphs/universal_set/idempotency"
+require "ucode/glyphs/universal_set/manifest_accumulator"
+require "ucode/glyphs/universal_set/manifest_writer"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Drains a codepoint stream through the 4-tier {Resolver} and
+      # produces the universal glyph set: one SVG per codepoint +
+      # manifest.json + reports.
+      #
+      # This is the orchestrator described by TODO 24. It owns three
+      # concerns and only three:
+      #
+      #   1. Iterate the codepoint stream (single-threaded or worker
+      #      pool, depending on `parallel_workers:`).
+      #   2. For each codepoint: resolve via the {Resolver}, write
+      #      the SVG via {Idempotency}, route the outcome to the
+      #      {ManifestAccumulator}.
+      #   3. After the drain: hand the manifest + per-block breakdown
+      #      to the {ManifestWriter} for atomic emission.
+      #
+      # The Builder is intentionally agnostic of how the codepoint
+      # stream is produced. The CLI command (TODO 24) constructs a
+      # {Ucode::Coordinator} enumerator; tests construct a small
+      # Array. The Builder doesn't know about UCD text files, fontist,
+      # or PDFs — those live behind the {Resolver}.
+      #
+      # == Idempotency
+      #
+      # SVG writes go through {Idempotency#write_glyph}, which uses
+      # {Ucode::Repo::AtomicWrites#write_atomic} for byte-level
+      # idempotency. Re-running with the same resolver + SVG payloads
+      # produces zero file writes. The manifest is regenerated each
+      # run; its `generated_at` updates but its entries remain stable
+      # when content is unchanged.
+      class Builder
+        include Idempotency
+
+        # @param output_root [String, Pathname] directory that will hold
+        #   `manifest.json`, `glyphs/`, `reports/`.
+        # @param resolver [Ucode::Glyphs::Resolver]
+        # @param unicode_version [String]
+        # @param ucode_version [String]
+        # @param source_config_sha256 [String] hex digest of the YAML
+        #   config that produced this build (recorded in the manifest
+        #   so audits can detect drift).
+        # @param parallel_workers [Integer] size of the worker pool.
+        #   Set to 1 (or less) for inline mode — used in tests.
+        # @param block_filter [String, nil] only build codepoints whose
+        #   `block_id` matches this verbatim (canonical underscore form).
+        def initialize(output_root:, resolver:, unicode_version:,
+                       ucode_version:, source_config_sha256:,
+                       parallel_workers: 1, block_filter: nil)
+          @output_root = Pathname.new(output_root)
+          @resolver = resolver
+          @unicode_version = unicode_version
+          @ucode_version = ucode_version
+          @source_config_sha256 = source_config_sha256
+          @parallel_workers = parallel_workers
+          @block_filter = block_filter
+        end
+
+        # Drain `codepoints` through the resolver and emit the
+        # manifest + reports. Returns the path to the written manifest.
+        #
+        # @param codepoints [Enumerable<Ucode::Models::CodePoint>]
+        # @return [Pathname] path to the written manifest.json
+        def build(codepoints)
+          accumulator = ManifestAccumulator.new(
+            unicode_version: @unicode_version,
+            ucode_version: @ucode_version,
+            source_config_sha256: @source_config_sha256,
+          )
+          drain(codepoints, accumulator)
+          write_outputs(accumulator)
+        end
+
+        private
+
+        def drain(codepoints, accumulator)
+          return drain_inline(codepoints, accumulator) if @parallel_workers <= 1
+
+          drain_threaded(codepoints, accumulator)
+        end
+
+        def drain_inline(codepoints, accumulator)
+          codepoints.each do |cp|
+            build_one(cp, accumulator)
+          end
+        end
+
+        def drain_threaded(codepoints, accumulator)
+          queue = Queue.new
+          workers = Array.new(@parallel_workers) do
+            Thread.new do
+              loop do
+                cp = queue.pop
+                break if cp.nil?
+
+                build_one(cp, accumulator)
+              end
+            end
+          end
+
+          codepoints.each do |cp|
+            queue << cp
+          end
+          @parallel_workers.times { queue << nil }
+          workers.each(&:join)
+        end
+
+        # Resolve one codepoint, write its SVG (if any), and route
+        # the outcome to the accumulator. Exceptions are caught here
+        # so a single bad codepoint doesn't abort the run.
+        #
+        # @param cp [Ucode::Models::CodePoint]
+        # @param accumulator [ManifestAccumulator]
+        def build_one(cp, accumulator)
+          return unless matches_filter?(cp)
+
+          result = @resolver.resolve(cp.cp)
+          if result.nil?
+            accumulator.record_skip(cp)
+            return
+          end
+
+          svg = result.svg
+          write_glyph(@output_root, cp_id(cp), svg)
+          accumulator.record_build(cp, result, svg: svg)
+        rescue StandardError => e
+          accumulator.record_failure(cp, e)
+        end
+
+        def matches_filter?(cp)
+          return true if @block_filter.nil?
+
+          cp.block_id == @block_filter
+        end
+
+        def cp_id(cp)
+          Ucode::Repo::Paths.cp_id(cp.cp)
+        end
+
+        def write_outputs(accumulator)
+          manifest = accumulator.to_manifest
+          ManifestWriter.new(@output_root).write(
+            manifest,
+            by_block: accumulator.by_block,
+            gaps: accumulator.gaps,
+            failures: accumulator.failures,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/coverage_report.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+
+require "ucode/glyphs/universal_set/idempotency"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Standalone emitter for the universal-set coverage reports
+      # described by TODO 31 §Per-tier / §Per-block / §Gap
+      # investigation. Reads a {Ucode::Models::UniversalSetManifest}
+      # and produces three JSON files under `<output_root>/reports/`:
+      #
+      # - `by_tier.json` — manifest.by_tier verbatim (`tier-1` => N,
+      #   `pillar-1` => N, ...). Quick "how much of the set is real
+      #   fonts vs. tofu?" answer.
+      # - `by_block.json` — per-block per-tier breakdown:
+      #
+      #     { "Sidetic": { "assigned": 26, "tier-1": 26,
+      #                     "pillar-1": 0, "pillar-2": 0, "pillar-3": 0 } }
+      #
+      #   `assigned` is the count of manifest entries whose codepoint
+      #   falls in this block (via {Ucode::Database#lookup_block}).
+      #   Each tier key counts the entries that resolved at that tier.
+      # - `gaps.json` — array of `{ codepoint, block, reason }` for
+      #   every manifest entry at `pillar-3`. These are the "tofu
+      #   leaks" TODO 31 calls out as actionable curation follow-ups
+      #   (excluding the documented-residual cases: unassigned, PUA,
+      #   noncharacter — those are correctly Last Resort).
+      #
+      # An optional `failures:` payload (from
+      # {ManifestAccumulator#failures}) writes a fourth file,
+      # `failures.json`, with per-codepoint exception log. Kept
+      # separate from `gaps.json` so the two concepts (tofu vs.
+      # crash) don't collide.
+      #
+      # All writes are atomic via {Idempotency}. Re-running on an
+      # unchanged manifest is a no-op modulo nothing — JSON output is
+      # stable (sorted keys, deterministic ordering).
+      class CoverageReport
+        include Idempotency
+
+        # Reason stamped on every pillar-3 gap entry. The detailed
+        # "why did this fall through?" path is in the manifest entry's
+        # `source` field; this string is the high-level category.
+        TOFU_REASON = "resolved to pillar-3 (Last Resort placeholder)"
+        private_constant :TOFU_REASON
+
+        # @param output_root [String, Pathname] directory holding
+        #   `manifest.json` + `reports/`.
+        # @param database [Ucode::Database] used for codepoint → block
+        #   lookup. The `report` CLI command opens one for the target
+        #   Unicode version; tests pass a small in-memory database.
+        def initialize(output_root, database:)
+          @output_root = Pathname.new(output_root)
+          @database = database
+        end
+
+        # Write the three coverage reports. Returns the structured
+        # payload so callers (CLI) can render a summary without
+        # re-reading the files.
+        #
+        # @param manifest [Ucode::Models::UniversalSetManifest]
+        # @param failures [Array<Hash>] optional per-codepoint
+        #   exception log from {ManifestAccumulator#failures}. When
+        #   non-empty, also writes `reports/failures.json`.
+        # @return [Hash] { by_tier:, by_block:, gaps:, failures:,
+        #   by_tier_path:, by_block_path:, gaps_path:, failures_path: }
+        def emit(manifest, failures: [])
+          by_tier = manifest.by_tier
+          by_block = build_by_block(manifest)
+          gaps = build_gaps(manifest)
+
+          by_tier_path = by_tier_report_path(@output_root)
+          by_block_path = by_block_report_path(@output_root)
+          gaps_path = gaps_report_path(@output_root)
+          write_atomic(by_tier_path, to_pretty_json(by_tier))
+          write_atomic(by_block_path, to_pretty_json(by_block))
+          write_atomic(gaps_path, to_pretty_json(gaps))
+          failures_path = write_failures(failures)
+
+          {
+            by_tier: by_tier,
+            by_block: by_block,
+            gaps: gaps,
+            failures: failures,
+            by_tier_path: by_tier_path,
+            by_block_path: by_block_path,
+            gaps_path: gaps_path,
+            failures_path: failures_path,
+          }
+        end
+
+        private
+
+        def build_by_block(manifest)
+          tally = Hash.new do |h, block|
+            h[block] = { "assigned" => 0, "tier-1" => 0, "pillar-1" => 0,
+                         "pillar-2" => 0, "pillar-3" => 0 }
+          end
+
+          manifest.entries.each do |entry|
+            block = @database.lookup_block(entry.codepoint)
+            next unless block
+
+            tally[block]["assigned"] += 1
+            tally[block][entry.tier] = (tally[block][entry.tier] || 0) + 1
+          end
+
+          # Sort by block name for deterministic output — re-running
+          # on the same manifest produces byte-identical JSON.
+          tally.sort.to_h
+        end
+
+        def build_gaps(manifest)
+          manifest.entries.each_with_object([]) do |entry, acc|
+            next unless entry.tier == "pillar-3"
+
+            acc << {
+              "codepoint" => entry.codepoint,
+              "block" => @database.lookup_block(entry.codepoint),
+              "reason" => TOFU_REASON,
+            }
+          end
+        end
+
+        def write_failures(failures)
+          return nil if failures.empty?
+
+          path = @output_root.join(REPORTS_DIR, "failures.json")
+          write_atomic(path, to_pretty_json(failures))
+          path
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/idempotency.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/repo/atomic_writes"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Idempotency + path helpers for the universal set build.
+      #
+      # TODO 24 specifies "a codepoint whose source font mtime +
+      # content hash are unchanged is skipped." The content-hash half
+      # is exactly {Ucode::Repo::AtomicWrites#write_atomic} —
+      # byte-identical payloads are a no-op. The mtime half is a
+      # future optimization (skip the resolver call entirely when the
+      # font hasn't changed); for now, byte-comparison gives semantic
+      # correctness, which is the load-bearing property.
+      #
+      # This module centralizes the universal-set write semantic so
+      # future mtime-based short-circuitting lands in one place. The
+      # {Builder} and {ManifestWriter} mix this in.
+      module Idempotency
+        include Ucode::Repo::AtomicWrites
+
+        # Directory under the output root that holds the per-codepoint SVGs.
+        GLYPHS_DIR = "glyphs"
+        # Directory under the output root that holds the by-tier / by-block /
+        # gaps reports emitted alongside the manifest.
+        REPORTS_DIR = "reports"
+        # The manifest filename at the output root.
+        MANIFEST_FILENAME = "manifest.json"
+        # Report filenames.
+        BY_TIER_REPORT = "by_tier.json"
+        BY_BLOCK_REPORT = "by_block.json"
+        GAPS_REPORT = "gaps.json"
+
+        private_constant :GLYPHS_DIR, :REPORTS_DIR, :MANIFEST_FILENAME,
+                         :BY_TIER_REPORT, :BY_BLOCK_REPORT, :GAPS_REPORT
+
+        # Write the SVG payload to the canonical `glyphs/<id>.svg`
+        # path if-and-only-if the content changed. Returns true when
+        # the file was written; false when skipped (byte-identical).
+        #
+        # @param output_root [Pathname]
+        # @param cp_id [String] e.g. "U+0041"
+        # @param svg [String]
+        # @return [Boolean]
+        def write_glyph(output_root, cp_id, svg)
+          write_atomic(glyph_path(output_root, cp_id), svg)
+        end
+
+        # @param output_root [Pathname]
+        # @param cp_id [String]
+        # @return [Pathname] <output_root>/glyphs/<cp_id>.svg
+        def glyph_path(output_root, cp_id)
+          Pathname.new(output_root).join(GLYPHS_DIR, "#{cp_id}.svg")
+        end
+
+        # @param output_root [Pathname]
+        # @return [Pathname]
+        def manifest_path(output_root)
+          Pathname.new(output_root).join(MANIFEST_FILENAME)
+        end
+
+        # @param output_root [Pathname]
+        # @return [Pathname]
+        def by_tier_report_path(output_root)
+          Pathname.new(output_root).join(REPORTS_DIR, BY_TIER_REPORT)
+        end
+
+        # @param output_root [Pathname]
+        # @return [Pathname]
+        def by_block_report_path(output_root)
+          Pathname.new(output_root).join(REPORTS_DIR, BY_BLOCK_REPORT)
+        end
+
+        # @param output_root [Pathname]
+        # @return [Pathname]
+        def gaps_report_path(output_root)
+          Pathname.new(output_root).join(REPORTS_DIR, GAPS_REPORT)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/manifest_accumulator.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require "digest"
+require "time"
+
+require "ucode/models"
+require "ucode/repo/paths"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Thread-safe accumulator that observes the {Builder}'s per-
+      # codepoint attempts and produces the final
+      # {Ucode::Models::UniversalSetManifest} plus the per-block
+      # breakdown the {ManifestWriter} emits as `reports/by_block.json`.
+      #
+      # Mirrors the {Ucode::Repo::BuildReportAccumulator} pattern from
+      # Mode 1: the orchestrating command passes this instance to the
+      # builder, which calls {#record_build} (or {#record_skip}) from
+      # inside its worker pool. After the drain completes,
+      # {#to_manifest} returns the immutable snapshot.
+      #
+      # == Semantics
+      #
+      # - `codepoints_assigned` counts every codepoint the builder
+      #   attempted (passed the block_filter guard).
+      # - `codepoints_built` counts codepoints whose resolver returned
+      #   a glyph.
+      # - `codepoints_skipped` counts codepoints that resolved to nil
+      #   (no tier produced a glyph) — these are the "gaps" the gaps
+      #   report enumerates.
+      # - `codepoints_failed` counts exceptions recorded via
+      #   {#record_failure}.
+      #
+      # `by_tier` counts the winning tier per codepoint (one increment
+      # per built codepoint). The map uses the wire form ("tier-1",
+      # "pillar-1", ...) so the manifest is stable across Ruby symbol
+      # changes.
+      #
+      # `by_block` is a hash keyed by block_id, with built / skipped /
+      # failed counters per block. Computed from the codepoint stream
+      # the Builder drains — the accumulator reads {CodePoint#block_id}
+      # directly. Block ids follow the canonical underscore form.
+      class ManifestAccumulator
+        TIER_TO_WIRE = {
+          tier1: "tier-1",
+          pillar1: "pillar-1",
+          pillar2: "pillar-2",
+          pillar3: "pillar-3",
+        }.freeze
+        private_constant :TIER_TO_WIRE
+
+        # @param unicode_version [String]
+        # @param ucode_version [String]
+        # @param source_config_sha256 [String]
+        def initialize(unicode_version:, ucode_version:, source_config_sha256:)
+          @unicode_version = unicode_version
+          @ucode_version = ucode_version
+          @source_config_sha256 = source_config_sha256
+          @totals = { codepoints_assigned: 0, codepoints_built: 0,
+                      codepoints_skipped: 0, codepoints_failed: 0 }
+          @by_tier = Hash.new(0)
+          @by_block = Hash.new do |h, block_id|
+            h[block_id] = { built: 0, skipped: 0, failed: 0 }
+          end
+          @entries = []
+          @gaps = []
+          @failures = []
+          @mutex = Mutex.new
+        end
+
+        # Observer entry — the builder calls this for every codepoint
+        # the resolver produced a glyph for. Records the entry and
+        # bumps the built counter + per-tier + per-block rollups.
+        #
+        # @param codepoint [Ucode::Models::CodePoint]
+        # @param result [Ucode::Glyphs::Source::Result] non-nil
+        # @param svg [String] the SVG bytes that were written
+        # @return [void]
+        def record_build(codepoint, result, svg:)
+          entry = build_entry(codepoint.cp, result, svg)
+          tier_wire = wire_tier(result.tier)
+          synchronize do
+            @totals[:codepoints_assigned] += 1
+            @totals[:codepoints_built] += 1
+            @by_tier[tier_wire] += 1
+            @by_block[codepoint.block_id][:built] += 1
+            @entries << entry
+          end
+        end
+
+        # Observer entry — the builder calls this when the resolver
+        # returned nil for a codepoint. Counts the attempt and adds
+        # it to the gaps list for the gaps report.
+        #
+        # @param codepoint [Ucode::Models::CodePoint]
+        # @return [void]
+        def record_skip(codepoint)
+          synchronize do
+            @totals[:codepoints_assigned] += 1
+            @totals[:codepoints_skipped] += 1
+            @by_block[codepoint.block_id][:skipped] += 1
+            @gaps << codepoint.cp
+          end
+        end
+
+        # Record an exception. The builder rescues per-codepoint
+        # errors and routes them here so one bad codepoint doesn't
+        # abort the run.
+        #
+        # @param codepoint [Ucode::Models::CodePoint, nil]
+        # @param error [StandardError]
+        # @return [void]
+        def record_failure(codepoint, error)
+          synchronize do
+            @totals[:codepoints_assigned] += 1 unless codepoint.nil?
+            @totals[:codepoints_failed] += 1
+            @by_block[codepoint&.block_id][:failed] += 1 unless codepoint.nil?
+            @failures << { codepoint: codepoint&.cp,
+                           block_id: codepoint&.block_id,
+                           error_class: error.class.name,
+                           message: error.message }
+          end
+        end
+
+        # @return [Ucode::Models::UniversalSetManifest] immutable snapshot
+        def to_manifest
+          synchronize do
+            Ucode::Models::UniversalSetManifest.new(
+              unicode_version: @unicode_version,
+              ucode_version: @ucode_version,
+              generated_at: Time.now.utc.iso8601,
+              source_config_sha256: @source_config_sha256,
+              totals: Ucode::Models::UniversalSetManifest::Totals.new(@totals),
+              by_tier: @by_tier.dup,
+              entries: @entries.dup,
+            )
+          end
+        end
+
+        # @return [Hash{String=>Hash}] per-block built/skipped/failed
+        #   counts, deep-copied so callers can't mutate accumulator state.
+        def by_block
+          synchronize do
+            @by_block.transform_values(&:dup)
+          end
+        end
+
+        # @return [Array<Integer>] codepoints that resolved to nil, sorted
+        def gaps
+          synchronize { @gaps.sort }
+        end
+
+        # @return [Array<Hash>] recorded failures (each with codepoint,
+        #   block_id, error_class, message)
+        def failures
+          synchronize { @failures.dup }
+        end
+
+        private
+
+        def build_entry(codepoint, result, svg)
+          Ucode::Models::UniversalSetEntry.new(
+            codepoint: codepoint,
+            id: Ucode::Repo::Paths.cp_id(codepoint),
+            tier: wire_tier(result.tier),
+            source: source_label(result.provenance),
+            svg_sha256: sha256(svg),
+            svg_size_bytes: svg.bytesize,
+          )
+        end
+
+        # Extract the source identifier from a dotted provenance
+        # string ("tier-1:noto-sans" -> "noto-sans"). When there's no
+        # `:` separator, returns the input verbatim — defensive
+        # against malformed provenance.
+        def source_label(provenance)
+          provenance.to_s.split(":", 2).last || provenance.to_s
+        end
+
+        def sha256(payload)
+          Digest::SHA256.hexdigest(payload)
+        end
+
+        def wire_tier(symbol)
+          TIER_TO_WIRE.fetch(symbol, symbol.to_s)
+        end
+
+        def synchronize(&)
+          @mutex.synchronize(&)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/manifest_writer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "json"
+
+require "ucode/glyphs/universal_set/idempotency"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Writes the final manifest + reports under the output root.
+      #
+      # One manifest, three reports:
+      #
+      # - `manifest.json` — full {Ucode::Models::UniversalSetManifest}.
+      # - `reports/by_tier.json` — `by_tier` counts alone (small file
+      #   for quick "how much of the set is tier 1?" inspection).
+      # - `reports/by_block.json` — per-block built/skipped totals,
+      #   computed from the manifest's entries + the codepoint's
+      #   block_id (resolved by the Builder).
+      # - `reports/gaps.json` — array of codepoint integers that
+      #   resolved to nil (should be empty for a healthy run).
+      #
+      # All writes are atomic via {Idempotency} (which includes
+      # {Ucode::Repo::AtomicWrites}). Re-running on an unchanged
+      # manifest is a no-op modulo `generated_at`.
+      class ManifestWriter
+        include Idempotency
+
+        # @param output_root [String, Pathname]
+        def initialize(output_root)
+          @output_root = Pathname.new(output_root)
+        end
+
+        # Write the manifest + reports atomically.
+        #
+        # @param manifest [Ucode::Models::UniversalSetManifest]
+        # @param by_block [Hash{String=>Hash}] per-block breakdown:
+        #   `{ "Basic_Latin" => { built: 64, skipped: 0, failed: 0 } }`.
+        #   Computed by the {Builder}; this writer just serializes it.
+        # @param gaps [Array<Integer>] codepoints with no glyph
+        # @param failures [Array<Hash>] per-codepoint failures
+        # @return [Pathname] path to the written manifest
+        def write(manifest, by_block:, gaps:, failures:)
+          write_atomic(manifest_path(@output_root), manifest_to_json(manifest))
+          write_atomic(by_tier_report_path(@output_root), to_pretty_json(manifest.by_tier))
+          write_atomic(by_block_report_path(@output_root), to_pretty_json(by_block))
+          write_atomic(gaps_report_path(@output_root),
+                       to_pretty_json(gaps: gaps, failures: failures))
+          manifest_path(@output_root)
+        end
+
+        private
+
+        def manifest_to_json(manifest)
+          manifest.to_json(pretty: true)
+        end
+      end
+    end
+  end
+end