ucode 0.1.0 → 0.2.0

data/lib/ucode/glyphs/sources/pillar3_last_resort.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "ucode/glyphs/source"
+require "ucode/glyphs/last_resort/renderer"
+
+module Ucode
+  module Glyphs
+    module Sources
+      # Pillar 3 glyph source: Last Resort UFO placeholder outlines.
+      #
+      # Wraps {LastResort::Renderer}, which chains cmap → contents →
+      # glif → svg for every codepoint the Last Resort Font's Format 13
+      # cmap maps (essentially all of Unicode 0x0..0x10FFFF). This is
+      # the catch-all tier: any codepoint no higher tier produced a
+      # glyph for lands here and gets a placeholder outline.
+      #
+      # The Renderer returns nil only for codepoints outside the cmap
+      # (extremely rare — the Format 13 cmap is exhaustive). For
+      # everything else it returns a {LastResort::Renderer::Result}
+      # with the SVG. We adapt that to {Source::Result}.
+      class Pillar3LastResort < Source
+        # @param renderer [LastResort::Renderer] the renderer to
+        #   delegate to. Callers typically construct it with the
+        #   resolved {LastResort::Source}.
+        def initialize(renderer:)
+          super()
+          @renderer = renderer
+        end
+
+        # @return [Symbol] :pillar3
+        def tier
+          :pillar3
+        end
+
+        # @return [String] "pillar-3:last-resort"
+        def provenance
+          "pillar-3:last-resort"
+        end
+
+        # (see Source#fetch)
+        def fetch(codepoint)
+          result = @renderer.render(codepoint)
+          return nil unless result&.ok?
+
+          Result.new(tier: tier, codepoint: codepoint,
+                     svg: result.svg, provenance: provenance)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/sources/tier1_real_font.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "fontisan"
+
+require "ucode/glyphs/source"
+require "ucode/glyphs/real_fonts/font_locator"
+require "ucode/glyphs/embedded_fonts/svg"
+
+module Ucode
+  module Glyphs
+    module Sources
+      # Tier 1 glyph source: real-font cmap + outline extraction.
+      #
+      # For codepoints inside its assigned block range, looks up the
+      # GID in the font's cmap, extracts the outline via
+      # `Fontisan::OutlineExtractor`, and renders a standalone SVG via
+      # {EmbeddedFonts::Svg} (which y-flips to SVG coordinates and
+      # builds a padded viewBox around the outline bbox).
+      #
+      # Codepoints outside the block range, missing from the cmap, or
+      # producing an empty outline return nil — the {Resolver} then
+      # falls through to lower tiers. This is the preferred source:
+      # highest fidelity, no chart-grid chrome composited in.
+      #
+      # One Tier1RealFont per (block, font) pair. The {SourceBuilder}
+      # expands a {SourceConfig} into a flat array of these, one per
+      # configured block × {Ucode::Models::GlyphSource} entry. When
+      # multiple Tier 1 fonts are configured for the same block, each
+      # becomes a separate source and the resolver tries them in
+      # declared order.
+      class Tier1RealFont < Source
+        # @param block_range [Range<Integer>] codepoints this source
+        #   serves. Codepoints outside the range return nil without
+        #   consulting the font.
+        # @param source [Ucode::Models::GlyphSource] typed curation
+        #   entry. Drives font resolution via
+        #   {RealFonts::FontLocator} through {GlyphSource#to_font_spec}.
+        # @param install [Boolean] passed through to FontLocator. When
+        #   true (default), fontist downloads missing fonts. Tests
+        #   disable this to avoid network calls.
+        def initialize(block_range:, source:, install: true)
+          super()
+          @block_range = block_range
+          @source = source
+          @install = install
+        end
+
+        # @return [Symbol] :tier1
+        def tier
+          :tier1
+        end
+
+        # @return [String] "tier-1:<label>" — the label is the
+        #   {GlyphSource#label} from the curation entry.
+        def provenance
+          "tier-1:#{@source.label}"
+        end
+
+        # (see Source#fetch)
+        def fetch(codepoint)
+          return nil unless @block_range.cover?(codepoint)
+
+          gid = cmap[codepoint]
+          return nil unless gid
+
+          outline = extractor.extract(gid)
+          return nil if outline.nil? || outline.empty?
+
+          svg = EmbeddedFonts::Svg.new(outline, codepoint: codepoint,
+                                                base_font: base_font).to_s
+          Result.new(tier: tier, codepoint: codepoint, svg: svg,
+                     provenance: provenance)
+        rescue StandardError
+          # Font load failures, outline extraction errors, etc. — all
+          # translate to "this source can't help". The resolver will
+          # try the next tier.
+          nil
+        end
+
+        private
+
+        def cmap
+          @cmap ||= font.table("cmap").unicode_mappings
+        end
+
+        def font
+          @font ||= Fontisan::FontLoader.load(path)
+        end
+
+        def path
+          @path ||= RealFonts::FontLocator.new.locate(@source.to_font_spec, install: @install).path
+        end
+
+        def extractor
+          @extractor ||= Fontisan::OutlineExtractor.new(font)
+        end
+
+        def base_font
+          File.basename(path.to_s, ".*")
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/sources.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Glyphs
+    # Concrete {Source} subclasses — one per tier of the 4-tier glyph
+    # sourcing strategy. Each adapts an existing implementation
+    # (RealFonts, EmbeddedFonts::Catalog, LastResort::Renderer) to the
+    # {Source} interface so the {Resolver} can orchestrate them
+    # uniformly.
+    #
+    # Adding a new source is a pure extension (new file + autoload) —
+    # the Resolver and Source interface are closed for modification.
+    module Sources
+      autoload :Tier1RealFont, "ucode/glyphs/sources/tier1_real_font"
+      autoload :Pillar1EmbeddedTounicode,
+               "ucode/glyphs/sources/pillar1_embedded_tounicode"
+      autoload :Pillar3LastResort, "ucode/glyphs/sources/pillar3_last_resort"
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/builder.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/glyphs/universal_set/idempotency"
+require "ucode/glyphs/universal_set/manifest_accumulator"
+require "ucode/glyphs/universal_set/manifest_writer"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Drains a codepoint stream through the 4-tier {Resolver} and
+      # produces the universal glyph set: one SVG per codepoint +
+      # manifest.json + reports.
+      #
+      # This is the orchestrator described by TODO 24. It owns three
+      # concerns and only three:
+      #
+      #   1. Iterate the codepoint stream (single-threaded or worker
+      #      pool, depending on `parallel_workers:`).
+      #   2. For each codepoint: resolve via the {Resolver}, write
+      #      the SVG via {Idempotency}, route the outcome to the
+      #      {ManifestAccumulator}.
+      #   3. After the drain: hand the manifest + per-block breakdown
+      #      to the {ManifestWriter} for atomic emission.
+      #
+      # The Builder is intentionally agnostic of how the codepoint
+      # stream is produced. The CLI command (TODO 24) constructs a
+      # {Ucode::Coordinator} enumerator; tests construct a small
+      # Array. The Builder doesn't know about UCD text files, fontist,
+      # or PDFs — those live behind the {Resolver}.
+      #
+      # == Idempotency
+      #
+      # SVG writes go through {Idempotency#write_glyph}, which uses
+      # {Ucode::Repo::AtomicWrites#write_atomic} for byte-level
+      # idempotency. Re-running with the same resolver + SVG payloads
+      # produces zero file writes. The manifest is regenerated each
+      # run; its `generated_at` updates but its entries remain stable
+      # when content is unchanged.
+      class Builder
+        include Idempotency
+
+        # @param output_root [String, Pathname] directory that will hold
+        #   `manifest.json`, `glyphs/`, `reports/`.
+        # @param resolver [Ucode::Glyphs::Resolver]
+        # @param unicode_version [String]
+        # @param ucode_version [String]
+        # @param source_config_sha256 [String] hex digest of the YAML
+        #   config that produced this build (recorded in the manifest
+        #   so audits can detect drift).
+        # @param parallel_workers [Integer] size of the worker pool.
+        #   Set to 1 (or less) for inline mode — used in tests.
+        # @param block_filter [String, nil] only build codepoints whose
+        #   `block_id` matches this verbatim (canonical underscore form).
+        def initialize(output_root:, resolver:, unicode_version:,
+                       ucode_version:, source_config_sha256:,
+                       parallel_workers: 1, block_filter: nil)
+          @output_root = Pathname.new(output_root)
+          @resolver = resolver
+          @unicode_version = unicode_version
+          @ucode_version = ucode_version
+          @source_config_sha256 = source_config_sha256
+          @parallel_workers = parallel_workers
+          @block_filter = block_filter
+        end
+
+        # Drain `codepoints` through the resolver and emit the
+        # manifest + reports. Returns the path to the written manifest.
+        #
+        # @param codepoints [Enumerable<Ucode::Models::CodePoint>]
+        # @return [Pathname] path to the written manifest.json
+        def build(codepoints)
+          accumulator = ManifestAccumulator.new(
+            unicode_version: @unicode_version,
+            ucode_version: @ucode_version,
+            source_config_sha256: @source_config_sha256,
+          )
+          drain(codepoints, accumulator)
+          write_outputs(accumulator)
+        end
+
+        private
+
+        def drain(codepoints, accumulator)
+          return drain_inline(codepoints, accumulator) if @parallel_workers <= 1
+
+          drain_threaded(codepoints, accumulator)
+        end
+
+        def drain_inline(codepoints, accumulator)
+          codepoints.each do |cp|
+            build_one(cp, accumulator)
+          end
+        end
+
+        def drain_threaded(codepoints, accumulator)
+          queue = Queue.new
+          workers = Array.new(@parallel_workers) do
+            Thread.new do
+              loop do
+                cp = queue.pop
+                break if cp.nil?
+
+                build_one(cp, accumulator)
+              end
+            end
+          end
+
+          codepoints.each do |cp|
+            queue << cp
+          end
+          @parallel_workers.times { queue << nil }
+          workers.each(&:join)
+        end
+
+        # Resolve one codepoint, write its SVG (if any), and route
+        # the outcome to the accumulator. Exceptions are caught here
+        # so a single bad codepoint doesn't abort the run.
+        #
+        # @param cp [Ucode::Models::CodePoint]
+        # @param accumulator [ManifestAccumulator]
+        def build_one(cp, accumulator)
+          return unless matches_filter?(cp)
+
+          result = @resolver.resolve(cp.cp)
+          if result.nil?
+            accumulator.record_skip(cp)
+            return
+          end
+
+          svg = result.svg
+          write_glyph(@output_root, cp_id(cp), svg)
+          accumulator.record_build(cp, result, svg: svg)
+        rescue StandardError => e
+          accumulator.record_failure(cp, e)
+        end
+
+        def matches_filter?(cp)
+          return true if @block_filter.nil?
+
+          cp.block_id == @block_filter
+        end
+
+        def cp_id(cp)
+          Ucode::Repo::Paths.cp_id(cp.cp)
+        end
+
+        def write_outputs(accumulator)
+          manifest = accumulator.to_manifest
+          ManifestWriter.new(@output_root).write(
+            manifest,
+            by_block: accumulator.by_block,
+            gaps: accumulator.gaps,
+            failures: accumulator.failures,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/coverage_report.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+
+require "ucode/glyphs/universal_set/idempotency"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Standalone emitter for the universal-set coverage reports
+      # described by TODO 31 §Per-tier / §Per-block / §Gap
+      # investigation. Reads a {Ucode::Models::UniversalSetManifest}
+      # and produces three JSON files under `<output_root>/reports/`:
+      #
+      # - `by_tier.json` — manifest.by_tier verbatim (`tier-1` => N,
+      #   `pillar-1` => N, ...). Quick "how much of the set is real
+      #   fonts vs. tofu?" answer.
+      # - `by_block.json` — per-block per-tier breakdown:
+      #
+      #     { "Sidetic": { "assigned": 26, "tier-1": 26,
+      #                     "pillar-1": 0, "pillar-2": 0, "pillar-3": 0 } }
+      #
+      #   `assigned` is the count of manifest entries whose codepoint
+      #   falls in this block (via {Ucode::Database#lookup_block}).
+      #   Each tier key counts the entries that resolved at that tier.
+      # - `gaps.json` — array of `{ codepoint, block, reason }` for
+      #   every manifest entry at `pillar-3`. These are the "tofu
+      #   leaks" TODO 31 calls out as actionable curation follow-ups
+      #   (excluding the documented-residual cases: unassigned, PUA,
+      #   noncharacter — those are correctly Last Resort).
+      #
+      # An optional `failures:` payload (from
+      # {ManifestAccumulator#failures}) writes a fourth file,
+      # `failures.json`, with per-codepoint exception log. Kept
+      # separate from `gaps.json` so the two concepts (tofu vs.
+      # crash) don't collide.
+      #
+      # All writes are atomic via {Idempotency}. Re-running on an
+      # unchanged manifest is a no-op modulo nothing — JSON output is
+      # stable (sorted keys, deterministic ordering).
+      class CoverageReport
+        include Idempotency
+
+        # Reason stamped on every pillar-3 gap entry. The detailed
+        # "why did this fall through?" path is in the manifest entry's
+        # `source` field; this string is the high-level category.
+        TOFU_REASON = "resolved to pillar-3 (Last Resort placeholder)"
+        private_constant :TOFU_REASON
+
+        # @param output_root [String, Pathname] directory holding
+        #   `manifest.json` + `reports/`.
+        # @param database [Ucode::Database] used for codepoint → block
+        #   lookup. The `report` CLI command opens one for the target
+        #   Unicode version; tests pass a small in-memory database.
+        def initialize(output_root, database:)
+          @output_root = Pathname.new(output_root)
+          @database = database
+        end
+
+        # Write the three coverage reports. Returns the structured
+        # payload so callers (CLI) can render a summary without
+        # re-reading the files.
+        #
+        # @param manifest [Ucode::Models::UniversalSetManifest]
+        # @param failures [Array<Hash>] optional per-codepoint
+        #   exception log from {ManifestAccumulator#failures}. When
+        #   non-empty, also writes `reports/failures.json`.
+        # @return [Hash] { by_tier:, by_block:, gaps:, failures:,
+        #   by_tier_path:, by_block_path:, gaps_path:, failures_path: }
+        def emit(manifest, failures: [])
+          by_tier = manifest.by_tier
+          by_block = build_by_block(manifest)
+          gaps = build_gaps(manifest)
+
+          by_tier_path = by_tier_report_path(@output_root)
+          by_block_path = by_block_report_path(@output_root)
+          gaps_path = gaps_report_path(@output_root)
+          write_atomic(by_tier_path, to_pretty_json(by_tier))
+          write_atomic(by_block_path, to_pretty_json(by_block))
+          write_atomic(gaps_path, to_pretty_json(gaps))
+          failures_path = write_failures(failures)
+
+          {
+            by_tier: by_tier,
+            by_block: by_block,
+            gaps: gaps,
+            failures: failures,
+            by_tier_path: by_tier_path,
+            by_block_path: by_block_path,
+            gaps_path: gaps_path,
+            failures_path: failures_path,
+          }
+        end
+
+        private
+
+        def build_by_block(manifest)
+          tally = Hash.new do |h, block|
+            h[block] = { "assigned" => 0, "tier-1" => 0, "pillar-1" => 0,
+                         "pillar-2" => 0, "pillar-3" => 0 }
+          end
+
+          manifest.entries.each do |entry|
+            block = @database.lookup_block(entry.codepoint)
+            next unless block
+
+            tally[block]["assigned"] += 1
+            tally[block][entry.tier] = (tally[block][entry.tier] || 0) + 1
+          end
+
+          # Sort by block name for deterministic output — re-running
+          # on the same manifest produces byte-identical JSON.
+          tally.sort.to_h
+        end
+
+        def build_gaps(manifest)
+          manifest.entries.each_with_object([]) do |entry, acc|
+            next unless entry.tier == "pillar-3"
+
+            acc << {
+              "codepoint" => entry.codepoint,
+              "block" => @database.lookup_block(entry.codepoint),
+              "reason" => TOFU_REASON,
+            }
+          end
+        end
+
+        def write_failures(failures)
+          return nil if failures.empty?
+
+          path = @output_root.join(REPORTS_DIR, "failures.json")
+          write_atomic(path, to_pretty_json(failures))
+          path
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/universal_set/idempotency.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/repo/atomic_writes"
+
+module Ucode
+  module Glyphs
+    module UniversalSet
+      # Idempotency + path helpers for the universal set build.
+      #
+      # TODO 24 specifies "a codepoint whose source font mtime +
+      # content hash are unchanged is skipped." The content-hash half
+      # is exactly {Ucode::Repo::AtomicWrites#write_atomic} —
+      # byte-identical payloads are a no-op. The mtime half is a
+      # future optimization (skip the resolver call entirely when the
+      # font hasn't changed); for now, byte-comparison gives semantic
+      # correctness, which is the load-bearing property.
+      #
+      # This module centralizes the universal-set write semantic so
+      # future mtime-based short-circuitting lands in one place. The
+      # {Builder} and {ManifestWriter} mix this in.
+      module Idempotency
+        include Ucode::Repo::AtomicWrites
+
+        # Directory under the output root that holds the per-codepoint SVGs.
+        GLYPHS_DIR = "glyphs"
+        # Directory under the output root that holds the by-tier / by-block /
+        # gaps reports emitted alongside the manifest.
+        REPORTS_DIR = "reports"
+        # The manifest filename at the output root.
+        MANIFEST_FILENAME = "manifest.json"
+        # Report filenames.
+        BY_TIER_REPORT = "by_tier.json"
+        BY_BLOCK_REPORT = "by_block.json"
+        GAPS_REPORT = "gaps.json"
+
+        private_constant :GLYPHS_DIR, :REPORTS_DIR, :MANIFEST_FILENAME,
+                         :BY_TIER_REPORT, :BY_BLOCK_REPORT, :GAPS_REPORT
+
+        # Write the SVG payload to the canonical `glyphs/<id>.svg`
+        # path if-and-only-if the content changed. Returns true when
+        # the file was written; false when skipped (byte-identical).
+        #
+        # @param output_root [Pathname]
+        # @param cp_id [String] e.g. "U+0041"
+        # @param svg [String]
+        # @return [Boolean]
+        def write_glyph(output_root, cp_id, svg)
+          write_atomic(glyph_path(output_root, cp_id), svg)
+        end
+
+        # @param output_root [Pathname]
+        # @param cp_id [String]
+        # @return [Pathname] <output_root>/glyphs/<cp_id>.svg
+        def glyph_path(output_root, cp_id)
+          Pathname.new(output_root).join(GLYPHS_DIR, "#{cp_id}.svg")
+        end
+
+        # @param output_root [Pathname]
+        # @return [Pathname]
+        def manifest_path(output_root)
+          Pathname.new(output_root).join(MANIFEST_FILENAME)
+        end
+
+        # @param output_root [Pathname]
+        # @return [Pathname]
+        def by_tier_report_path(output_root)
+          Pathname.new(output_root).join(REPORTS_DIR, BY_TIER_REPORT)
+        end
+
+        # @param output_root [Pathname]
+        # @return [Pathname]
+        def by_block_report_path(output_root)
+          Pathname.new(output_root).join(REPORTS_DIR, BY_BLOCK_REPORT)
+        end
+
+        # @param output_root [Pathname]
+        # @return [Pathname]
+        def gaps_report_path(output_root)
+          Pathname.new(output_root).join(REPORTS_DIR, GAPS_REPORT)
+        end
+      end
+    end
+  end
+end