ucode 0.1.0 → 0.2.0

data/lib/ucode/glyphs/pipeline.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "set"
+
+require "ucode/cache"
+require "ucode/glyphs/pdf_fetcher"
+require "ucode/glyphs/writer"
+require "ucode/parsers"
+
+module Ucode
+  module Glyphs
+    # Assembles the per-block specs that {Glyphs::Writer#write_all} drains.
+    #
+    # Owns three pieces of orchestration that {Commands::GlyphsCommand}
+    # used to carry inline:
+    #
+    #   - block loading from {Cache.ucd_dir}/Blocks.txt (with an optional
+    #     block filter)
+    #   - PDF fetcher construction (with monolith fallback)
+    #   - the per-block page-map heuristic (per-block PDFs are page 1 =
+    #     title, page 2 = first chart page starting at the block's first
+    #     codepoint; true for most BMP blocks; multi-page blocks need a
+    #     richer resolver — mismatches yield placeholder SVGs only, never
+    #     wrong glyphs)
+    #
+    # The Command stays a thin wrapper that prints the experimental
+    # warning and wires the writer. See Candidate 3 of the 2026-06-29
+    # architecture review.
+    class Pipeline
+      # Path to the monolith fallback file when no per-block PDF is on
+      # disk yet. Overridable for tests.
+      DEFAULT_MONOLITH_PATH = "CodeCharts.pdf"
+      # Cache path for the page-map corpus. Overridable for tests.
+      DEFAULT_PAGE_MAP_CACHE = "data/codecharts_page_map.json"
+
+      Spec = Struct.new(:block, :pdf_path, :page_map, keyword_init: true)
+
+      # @param version [String] resolved UCD version (callers must
+      #   resolve via {VersionResolver.resolve} first)
+      # @param block_filter [Array<String>, nil] block ids to limit to;
+      #   nil = every block
+      # @param monolith_path [String, Pathname, nil] fallback monolith
+      # @param page_map_cache [String, Pathname] cache for the page map
+      def initialize(version:, block_filter: nil,
+                     monolith_path: DEFAULT_MONOLITH_PATH,
+                     page_map_cache: DEFAULT_PAGE_MAP_CACHE)
+        @version = version
+        @block_filter = block_filter
+        @monolith_path = monolith_path
+        @page_map_cache = page_map_cache
+      end
+
+      # Load every block from the cached Blocks.txt (filtered by
+      # `@block_filter` when set) and pair each one with a fetched PDF
+      # path and a page map. Blocks whose PDF cannot be fetched are
+      # silently dropped — the placeholder pass downstream covers them.
+      #
+      # @param force [Boolean] re-fetch PDFs even when cached
+      # @return [Array<Spec>]
+      def build_specs(force: false)
+        blocks = load_blocks
+        fetcher = build_fetcher(blocks)
+        blocks.map { |block| spec_for(block, fetcher, force) }.compact
+      end
+
+      private
+
+      def load_blocks
+        path = Cache.ucd_dir(@version).join("Blocks.txt")
+        return [] unless path.exist?
+
+        all = Parsers::Blocks.each_record(path).to_a
+        return all unless @block_filter && !@block_filter.empty?
+
+        filter_set = @block_filter.to_set
+        all.select { |block| filter_set.include?(block.id) }
+      end
+
+      def build_fetcher(blocks)
+        monolith = @monolith_path ? Pathname.new(@monolith_path) : nil
+        monolith = monolith.exist? ? monolith : nil
+        PdfFetcher.new(
+          @version,
+          monolith_path: monolith,
+          blocks: blocks,
+          page_map_cache: @page_map_cache,
+        )
+      end
+
+      def spec_for(block, fetcher, force)
+        pdf_path = fetcher.fetch(block_first_cp: block.range_first, force: force)
+        return nil unless pdf_path
+
+        Spec.new(block: block, pdf_path: pdf_path, page_map: page_map_for(block))
+      end
+
+      # Per-block PDFs are page 1 = title, page 2 = first chart page
+      # starting at the block's first codepoint. True for most BMP
+      # blocks; multi-page blocks (CJK) need a richer resolver.
+      def page_map_for(block)
+        { 2 => block.range_first }
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/real_fonts/cmap_cache.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "fontisan"
+
+require "ucode/glyphs/real_fonts/font_locator"
+require "ucode/models/glyph_source"
+
+module Ucode
+  module Glyphs
+    module RealFonts
+      # Lazily loads each Tier 1 font's cmap and answers per-codepoint
+      # coverage queries. Used by {SourceConfig::CoverageAssertion}
+      # to walk every assigned codepoint without re-parsing the same
+      # font once per block.
+      #
+      # One font load per unique label. The cache key is the source's
+      # `label` (fontist formula name or `name=path` short name) —
+      # if two blocks reference the same label, the cmap loads once.
+      #
+      # Fonts that cannot be located or parsed produce an empty set;
+      # {CoverageAssertion} records every assigned codepoint in those
+      # blocks as a gap. Missing fonts are themselves curation
+      # findings — the walker surfaces them rather than hiding them
+      # behind an exception.
+      class CmapCache
+        # @param font_locator [FontLocator] injectable for testing.
+        #   Defaults to a fresh instance with `install: false`
+        #   semantics (we never auto-install during a coverage walk;
+        #   that's a separate operation).
+        def initialize(font_locator: FontLocator.new)
+          @font_locator = font_locator
+          @cmaps = {}
+        end
+
+        # @param source [Ucode::Models::GlyphSource]
+        # @param codepoint [Integer]
+        # @return [Boolean] true when the source's cmap includes the
+        #   codepoint. False when the font is missing, fails to load,
+        #   or doesn't have an outline for that codepoint.
+        def covers?(source, codepoint)
+          cmap_for(source).include?(codepoint)
+        end
+
+        private
+
+        def cmap_for(source)
+          @cmaps[source.label] ||= load_cmap(source)
+        end
+
+        def load_cmap(source)
+          path = resolve_path(source)
+          return Set.new unless path
+
+          font = Fontisan::FontLoader.load(path.to_s)
+          cmap = font.table(Fontisan::Constants::CMAP_TAG)
+          return Set.new unless cmap
+
+          cmap.unicode_mappings.keys.to_set
+        rescue StandardError
+          Set.new
+        end
+
+        def resolve_path(source)
+          result = @font_locator.locate(source.to_font_spec, install: false)
+          result&.path
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/real_fonts.rb

@@ -26,6 +26,7 @@ module Ucode
                "ucode/glyphs/real_fonts/font_coverage_report"
       autoload :FontLocator, "ucode/glyphs/real_fonts/font_locator"
       autoload :CoverageAuditor, "ucode/glyphs/real_fonts/coverage_auditor"
+      autoload :CmapCache, "ucode/glyphs/real_fonts/cmap_cache"
       autoload :Writer, "ucode/glyphs/real_fonts/writer"
     end
   end

data/lib/ucode/glyphs/resolver.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Glyphs
+    # Priority-ordered glyph resolver — the heart of the 4-tier sourcing
+    # strategy.
+    #
+    # Holds a flat array of {Source} instances (any tier, any number per
+    # tier) and tries them in `order:` until one returns a {Source::Result}.
+    # Tries are tier-major, source-minor: within a tier, sources are
+    # tried in the order they were passed to the constructor. This lets
+    # callers express "try FSung-1 before FSung-2 before Noto CJK JP" by
+    # simply ordering the Tier 1 sources that way.
+    #
+    # The default order is Tier 1 → Pillar 1 → Pillar 2 → Pillar 3, but
+    # callers can override (e.g. tests may want [:pillar3] only).
+    #
+    # The resolver is a pure orchestrator: it doesn't know about UCD
+    # blocks, fontist formulas, or PDF parsing. Those concerns live in
+    # the individual Source subclasses and in {SourceBuilder}.
+    class Resolver
+      DEFAULT_ORDER = %i[tier1 pillar1 pillar2 pillar3].freeze
+      private_constant :DEFAULT_ORDER
+
+      # @param sources [Array<Source>] flat list; grouped by tier
+      #   internally. Sources with the same tier are tried in the order
+      #   they appear here.
+      # @param order [Array<Symbol>] tier resolution order. Default:
+      #   %i[tier1 pillar1 pillar2 pillar3].
+      def initialize(sources:, order: DEFAULT_ORDER)
+        @sources_by_tier = sources.group_by(&:tier)
+        @order = order
+      end
+
+      # @param codepoint [Integer]
+      # @return [Source::Result, nil] nil only when every source in
+      #   every configured tier returned nil. With a Pillar 3 source
+      #   configured, this should be unreachable for assigned
+      #   codepoints — Pillar 3 catches the tail.
+      def resolve(codepoint)
+        @order.each do |tier|
+          Array(@sources_by_tier[tier]).each do |source|
+            result = source.fetch(codepoint)
+            return result if result
+          end
+        end
+        nil
+      end
+
+      # @return [Array<Source>] every source the resolver holds, flat.
+      def sources
+        @sources_by_tier.values.flatten
+      end
+
+      # @param tier [Symbol]
+      # @return [Array<Source>] sources registered for the given tier
+      def sources_for_tier(tier)
+        Array(@sources_by_tier[tier])
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/source.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Glyphs
+    # Abstract glyph source — one tier of the 4-tier sourcing strategy.
+    #
+    # The canonical {Resolver} holds an ordered array of Source subclasses
+    # and returns the first non-nil Result for a given codepoint. Each
+    # tier is one subclass:
+    #
+    #   * Tier 1   — {Sources::Tier1RealFont}: real-font cmap + outline
+    #                extraction (highest fidelity).
+    #   * Pillar 1 — {Sources::Pillar1EmbeddedTounicode}: PDF-embedded
+    #                CIDFont + /ToUnicode CMap.
+    #   * Pillar 2 — {Sources::Pillar2Correlator}: PDF content-stream
+    #                positional correlation for fonts without /ToUnicode.
+    #   * Pillar 3 — {Sources::Pillar3LastResort}: Last Resort UFO
+    #                placeholder outlines (catches the tail).
+    #
+    # Subclasses must implement {#tier}, {#provenance}, and {#fetch}.
+    # {#fetch} returns nil when the source cannot produce a glyph for
+    # the given codepoint — this is NOT an error, it's the signal for
+    # the resolver to try the next source.
+    class Source
+      # One resolved glyph. Carries the SVG payload and enough
+      # provenance to debug "where did this glyph come from?" without
+      # holding a reference back to the source.
+      Result = Struct.new(:tier, :codepoint, :svg, :provenance, keyword_init: true)
+
+      # @return [Symbol] one of :tier1, :pillar1, :pillar2, :pillar3
+      def tier
+        raise NotImplementedError
+      end
+
+      # @return [String] dotted provenance string, e.g.
+      #   "tier-1:lentariso", "pillar-3:last-resort"
+      def provenance
+        raise NotImplementedError
+      end
+
+      # @param codepoint [Integer]
+      # @return [Result, nil] nil when this source cannot produce a glyph
+      def fetch(codepoint)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/source_builder.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "ucode/glyphs/source_config"
+require "ucode/glyphs/sources"
+
+module Ucode
+  module Glyphs
+    # Builds {Source} instances by joining a {SourceConfig} (block →
+    # font mapping) with a {Ucode::Database} (block name → codepoint
+    # range).
+    #
+    # This is the single place that knows how to turn configuration +
+    # UCD metadata into live Source objects. Keeping that knowledge
+    # out of {SourceConfig} (which is a pure data loader) and out of
+    # {Resolver} (which is a pure orchestrator) keeps each class's
+    # responsibility narrow.
+    #
+    # For each block with at least one Tier 1 source in the config,
+    # the builder resolves the block's codepoint range from the UCD
+    # database and constructs one {Sources::Tier1RealFont} per
+    # configured {Ucode::Models::GlyphSource}. Blocks in the config
+    # that aren't in the UCD database are silently skipped — they may
+    # be future blocks or typos, and either way there's no range to
+    # serve.
+    class SourceBuilder
+      # @param config [SourceConfig]
+      # @param database [Ucode::Database] UCD index used to resolve
+      #   block names to codepoint ranges
+      def initialize(config:, database:)
+        @config = config
+        @database = database
+      end
+
+      # @param install [Boolean] forwarded to {Sources::Tier1RealFont}.
+      #   Tests pass false to suppress fontist downloads.
+      # @return [Array<Source>] one Tier1RealFont per (block, source)
+      #   pair in the config whose block exists in the UCD database
+      def tier1_sources(install: true)
+        @config.configured_block_ids.flat_map do |block_id|
+          range = block_range_for(block_id)
+          next [] unless range
+
+          @config.fonts_for(block_id).map do |source|
+            Sources::Tier1RealFont.new(block_range: range, source: source, install: install)
+          end
+        end
+      end
+
+      private
+
+      def block_range_for(block_id)
+        entries = @database.block_ranges_by_name(block_id)
+        return nil if entries.empty?
+
+        first_cp = entries.map(&:first_cp).min
+        last_cp = entries.map(&:last_cp).max
+        (first_cp..last_cp)
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/source_config/coverage_assertion.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "time"
+
+require "ucode/glyphs/source_config"
+require "ucode/glyphs/source_config/gap_report"
+
+module Ucode
+  module Glyphs
+    class SourceConfig
+      # Development-time walker that asks: "for every assigned
+      # codepoint in this Unicode version, does at least one Tier 1
+      # source's cmap cover it?" Codepoints with no Tier 1 coverage
+      # are recorded in a {GapReport}.
+      #
+      # This is a **curation review tool**, not a build gate. The
+      # universal-set build (TODO 24) still runs and falls through to
+      # pillars 1-3 for any gap; this report just makes the gaps
+      # visible to a human curator.
+      #
+      # Dependencies are injected so the walker stays pure:
+      #
+      # - `source_map` — typed {Ucode::Models::GlyphSourceMap} from
+      #   {SourceConfig#map}.
+      # - `database` — open {Ucode::Database} for the Unicode version
+      #   being audited. Supplies the assigned-codepoint ranges.
+      # - `cmaps` — any object responding to
+      #   `covers?(GlyphSource, Integer) => Boolean`. Default:
+      #   {RealFonts::CmapCache}, which lazily loads each referenced
+      #   font's cmap via fontisan.
+      #
+      # The walker never raises for a missing font or a failed cmap
+      # load — those codepoints are recorded as gaps. A missing font
+      # is itself a curation finding.
+      class CoverageAssertion
+        # @param source_map [Ucode::Models::GlyphSourceMap]
+        # @param database [Ucode::Database]
+        # @param cmaps [#covers?] object responding to
+        #   `covers?(source, codepoint) => Boolean`. Defaults to a
+        #   fresh {RealFonts::CmapCache}.
+        # @param unicode_version [String, nil] recorded on the report.
+        #   Defaults to the database's `ucd_version`.
+        def initialize(source_map:, database:, cmaps:,
+                       unicode_version: nil)
+          @source_map = source_map
+          @database = database
+          @cmaps = cmaps
+          @unicode_version = unicode_version || database.ucd_version
+        end
+
+        # @return [GapReport]
+        def call
+          gaps = Hash.new { |h, k| h[k] = [] }
+          total = 0
+
+          @database.block_entries.each do |range|
+            block_id = range.name
+            sources = @source_map.sources_for(block_id)
+            next if sources.empty? # uncurated block; not a gap, just unconfigured
+
+            (range.first_cp..range.last_cp).each do |cp|
+              next if sources.any? { |src| @cmaps.covers?(src, cp) }
+
+              gaps[block_id] << cp
+              total += 1
+            end
+          end
+
+          GapReport.new(
+            unicode_version: @unicode_version,
+            generated_at: Time.now.utc.iso8601,
+            gaps_by_block: gaps.freeze,
+            total_gaps: total,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/source_config/gap_report.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Glyphs
+    class SourceConfig
+      # Typed result of {CoverageAssertion#call}. Lists every assigned
+      # codepoint that no Tier 1 source's cmap covers, grouped by block.
+      #
+      # Pure value object — never raises, never mutates. Callers decide
+      # what to do with gaps:
+      #
+      # - **CI**: warn, fail the build on regressions vs. baseline.
+      # - **Local curator**: print, decide what to add.
+      # - **Production build**: continue — pillar 1-2-3 catch up.
+      #
+      # The shape round-trips through lutaml-model so it can be emitted
+      # alongside the universal-set build reports (TODO 31).
+      class GapReport < Lutaml::Model::Serializable
+        attribute :unicode_version, :string
+        attribute :generated_at, :string
+        attribute :gaps_by_block, :hash, default: -> { {} }
+        attribute :total_gaps, :integer, default: -> { 0 }
+
+        key_value do
+          map "unicode_version", to: :unicode_version
+          map "generated_at", to: :generated_at
+          map "gaps_by_block", to: :gaps_by_block
+          map "total_gaps", to: :total_gaps
+        end
+
+        # @return [Boolean] true when every assigned codepoint in the
+        #   walked range has at least one Tier 1 covering font.
+        def empty?
+          total_gaps.zero?
+        end
+
+        # @param block_id [String]
+        # @return [Array<Integer>] codepoints with no Tier 1 coverage
+        #   in this block. Empty for blocks with full coverage or
+        #   blocks that weren't walked.
+        def codepoints_for(block_id)
+          Array(gaps_by_block[block_id])
+        end
+
+        # @return [Array<String>] block ids that have at least one gap.
+        def block_ids_with_gaps
+          gaps_by_block.keys
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/source_config.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "yaml"
+
+require "ucode/models/glyph_source_map"
+
+module Ucode
+  module Glyphs
+    # Loads the curated Tier 1 font map from
+    # `config/unicode17_universal_glyph_set.yml` into a typed
+    # {Ucode::Models::GlyphSourceMap}.
+    #
+    # This is the policy half of the 4-tier resolver (TODO 23):
+    # "which font wins for which block, this Unicode version". The
+    # resolver mechanics live in {Resolver} + {Source}; the
+    # per-version curation lives in the YAML.
+    #
+    # Block ids in the YAML use the canonical underscore form
+    # ("Basic_Latin", "CJK_Unified_Ideographs_Extension_J") — same
+    # convention as {Ucode::Parsers::Blocks} and the rest of the
+    # codebase. Never slugified beyond whitespace collapse.
+    #
+    # Loader semantics:
+    # - Missing file → `exist?` returns false; `map` is an empty
+    #   `GlyphSourceMap`; all queries return empty.
+    # - Empty `map:` section → same as missing file.
+    # - Malformed YAML → raises (the curator must fix the file).
+    class SourceConfig
+      # Sub-constants for curation review (TODO 29). The class itself
+      # remains the loader; these are the development-time walkers +
+      # typed results that consume the loaded map.
+      autoload :CoverageAssertion,
+               "ucode/glyphs/source_config/coverage_assertion"
+      autoload :GapReport, "ucode/glyphs/source_config/gap_report"
+
+      # Default location of the curated Tier 1 font map. Public so the
+      # canonical build + universal set commands can reference it when
+      # no override is supplied. Keeping it on the class (not an
+      # instance attr) lets callers use it without constructing a
+      # SourceConfig first.
+      DEFAULT_PATH = Pathname.new("config/unicode17_universal_glyph_set.yml")
+
+      # @param path [String, Pathname] path to the YAML config file.
+      def initialize(path: DEFAULT_PATH)
+        @path = Pathname.new(path)
+      end
+
+      # @return [Pathname] the resolved config file path
+      attr_reader :path
+
+      # @return [Boolean] true if the config file exists on disk
+      def exist?
+        @path.exist?
+      end
+
+      # The loaded typed map. Memoized on first access. An empty
+      # {Ucode::Models::GlyphSourceMap} when the file is missing or
+      # has no `map:` section.
+      #
+      # @return [Ucode::Models::GlyphSourceMap]
+      def map
+        @map ||= load_map
+      end
+
+      # @param block_id [String] verbatim block id (underscore form)
+      # @return [Array<Ucode::Models::GlyphSource>] sources for this
+      #   block in priority order; empty when unconfigured.
+      def fonts_for(block_id)
+        map.sources_for(block_id)
+      end
+
+      # @return [Array<String>] block_ids with at least one Tier 1
+      #   source configured.
+      def configured_block_ids
+        map.configured_block_ids
+      end
+
+      # Class-method shortcut: load and return the typed map. Useful
+      # for one-shot scripts that don't need to query `exist?` first.
+      #
+      # @param yaml_path [String, Pathname]
+      # @return [Ucode::Models::GlyphSourceMap]
+      def self.load(yaml_path = DEFAULT_PATH)
+        new(path: yaml_path).map
+      end
+
+      private
+
+      def load_map
+        return empty_map unless @path.exist?
+
+        parsed = YAML.safe_load(@path.read, aliases: true)
+        return empty_map unless parsed.is_a?(Hash)
+
+        Ucode::Models::GlyphSourceMap.from_hash(parsed)
+      end
+
+      def empty_map
+        Ucode::Models::GlyphSourceMap.new
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/sources/pillar1_embedded_tounicode.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "ucode/glyphs/source"
+require "ucode/glyphs/embedded_fonts/renderer"
+
+module Ucode
+  module Glyphs
+    module Sources
+      # Pillar 1 glyph source: Code Charts PDF-embedded CIDFont
+      # outlines resolved via `/ToUnicode` CMap.
+      #
+      # Delegates to {EmbeddedFonts::Renderer}, which walks the
+      # PDF object graph (Type0 → CIDFont → FontDescriptor →
+      # FontFile2/3), looks up the GID via `/ToUnicode`, and renders
+      # the outline as a standalone SVG via {EmbeddedFonts::Svg}.
+      #
+      # == Pillar 2 unification
+      #
+      # TODO 20 lists a separate +Sources::Pillar2Correlator+ class.
+      # It is intentionally omitted. {ContentStreamCorrelator} alone
+      # returns +Hash{Integer=>Integer}+ (codepoint → GID mappings),
+      # not SVGs; it only produces SVGs when invoked through
+      # {EmbeddedFonts::Catalog} via its +correlator_configs:+
+      # registry. The Catalog already unifies pillars 1 and 2 at
+      # index-build time, so a Source-layer split would either
+      # duplicate the Catalog's index or require tagging each
+      # FontEntry with the sub-mechanism that served it — both
+      # violations of MECE. Pillar 2 fallback is configured by
+      # constructing the wrapped Catalog with +correlator_configs:+.
+      class Pillar1EmbeddedTounicode < Source
+        # @param renderer [EmbeddedFonts::Renderer] the renderer to
+        #   delegate to. Callers typically construct it with the
+        #   {EmbeddedFonts::Catalog} built from the resolved Code
+        #   Charts {EmbeddedFonts::Source}. To enable pillar-2
+        #   fallback, that Catalog must be constructed with
+        #   +correlator_configs:+.
+        def initialize(renderer:)
+          super()
+          @renderer = renderer
+        end
+
+        # @return [Symbol] :pillar1
+        def tier
+          :pillar1
+        end
+
+        # @return [String] "pillar-1:embedded-tounicode"
+        def provenance
+          "pillar-1:embedded-tounicode"
+        end
+
+        # (see Source#fetch)
+        def fetch(codepoint)
+          result = @renderer.render(codepoint)
+          return nil unless result
+
+          Result.new(tier: tier, codepoint: codepoint,
+                     svg: result.svg, provenance: provenance)
+        end
+      end
+    end
+  end
+end