ucode 0.4.0 → 0.5.1

data/lib/ucode/code_chart/coverage_gap_index.rb

@@ -0,0 +1,115 @@
+# frozen_string: true
+
+require "pathname"
+require "yaml"
+
+module Ucode
+  module CodeChart
+    # Lists Unicode blocks that have OFL coverage gaps — assigned
+    # codepoints no Tier 1 donor font covers. REQ R5's
+    # `--coverage-gap-only` flag.
+    #
+    # ## Inputs
+    #
+    # The index takes a {coverage_by_block} map
+    # (`{block_id => [covered_codepoints]}`) and a {blocks} lookup.
+    # For each block, it computes the gap (assigned − covered) and
+    # emits a {GapAnalyzer::BlockGap} when the gap is non-empty.
+    #
+    # ## Reuse
+    #
+    # Internally composes {GapAnalyzer::Analyzer} with a synthetic
+    # manifest (covering exactly the donor-supplied codepoints). No
+    # duplication of the gap-math; the OCP boundary is at
+    # {GapAnalyzer}.
+    #
+    # ## Coverage source
+    #
+    # The actual "is this codepoint covered by any OFL font?" lookup
+    # lives outside this class — callers pass in the coverage map.
+    # Future work: a Tier 1 CoverageCollector that walks fontist's
+    # known OFL sources and emits the coverage_by_block input. Until
+    # then, the caller supplies it (typically from a YAML file).
+    class CoverageGapIndex
+      # Wraps the coverage map as a {GapAnalyzer::Manifest}-shaped
+      # object so the analyzer can be reused unchanged. Pure
+      # adapter — no parsing, no I/O.
+      SyntheticManifest = Struct.new(:coverage, :ucd_version, keyword_init: true) do
+        def coverage_by_block
+          coverage
+        end
+      end
+      private_constant :SyntheticManifest
+
+      # @param coverage_by_block [Hash{String=>Array<Integer>}]
+      # @param blocks [Hash{String=>Ucode::Models::Block}]
+      # @param ucd_version [String]
+      def initialize(coverage_by_block:, blocks:, ucd_version:)
+        @coverage_by_block = coverage_by_block
+        @blocks = blocks
+        @ucd_version = ucd_version
+      end
+
+      # @yieldparam block_gap [Ucode::CodeChart::GapAnalyzer::BlockGap]
+      # @return [Enumerator, void]
+      def each_gap_block(&)
+        return enum_for(:each_gap_block) unless block_given?
+
+        analyzer.each_block_gap(&)
+      end
+
+      # @return [Array<Ucode::CodeChart::GapAnalyzer::BlockGap>]
+      def gap_blocks
+        analyzer.block_gaps
+      end
+
+      # @return [Integer]
+      def total_missing_codepoints
+        analyzer.total_missing_codepoints
+      end
+
+      private
+
+      def analyzer
+        @analyzer ||= GapAnalyzer::Analyzer.new(
+          manifest: synthetic_manifest,
+          blocks: @blocks,
+        )
+      end
+
+      # Keys every block in {blocks} so the analyzer sees
+      # unmentioned blocks as fully uncovered (the analyzer only
+      # iterates manifest keys).
+      def synthetic_manifest
+        full_coverage = {}
+        @blocks.each_key { |id| full_coverage[id] = @coverage_by_block[id] || [] }
+        SyntheticManifest.new(coverage: full_coverage,
+                              ucd_version: @ucd_version)
+      end
+
+      # Build a {CoverageGapIndex} from a coverage YAML file:
+      #
+      #   ucd_version: "17.0.0"
+      #   coverage:
+      #     Sidetic: ["U+10920", "U+10921"]
+      #     Beria_Erfe: ["U+10940"]
+      #
+      # Missing blocks in the YAML are treated as "no donor coverage"
+      # → the entire assigned set is the gap. Useful for the
+      # `--coverage-gap-only` CLI flow.
+      class << self
+        # @param path [Pathname, String]
+        # @param blocks [Hash{String=>Ucode::Models::Block}]
+        # @return [CoverageGapIndex]
+        def from_yaml(path, blocks:)
+          data = YAML.safe_load(Pathname.new(path).read) || {}
+          new(
+            coverage_by_block: data.fetch("coverage", {}),
+            blocks: blocks,
+            ucd_version: data.fetch("ucd_version"),
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/extractor.rb

@@ -30,8 +30,29 @@ module Ucode
     # not checked out.
     class Extractor
       # Result of extracting one codepoint.
-      Result = Struct.new(:codepoint, :svg, :tier, :provenance,
-                          keyword_init: true)
+      #
+      # Carries the SVG payload plus everything downstream concerns
+      # need without re-derivation:
+      #
+      #   * `base_font` — the PDF-embedded BaseFont name (e.g.
+      #     "GPJAHB+WolofGaraySansSerif"). Nil for non-PDF sources.
+      #   * `gid` — the GID inside that font. Nil for non-PDF sources.
+      #   * `source_page` — 1-based PDF page number where the glyph
+      #     appears. Nil when the Catalog didn't compute a location
+      #     (ToUnicode-only path) — populated by TODO 04's
+      #     `Catalog#location_for` integration.
+      #   * `source_cell` — `{x: Float, y: Float}` (PDF user space,
+      #     origin bottom-left) for the specimen. Same nil rule.
+      #   * `extractor_version` — `Ucode::VERSION` at extraction time.
+      #
+      # All optional fields default nil so existing call sites that
+      # only read `codepoint, svg, tier, provenance` keep working.
+      Result = Struct.new(
+        :codepoint, :svg, :tier, :provenance,
+        :base_font, :gid, :source_page, :source_cell,
+        :extractor_version,
+        keyword_init: true,
+      )
 
       # @param block [Ucode::Models::Block] block whose assigned
       #   codepoints will be extracted
@@ -47,13 +68,26 @@ module Ucode
       #   Pillar 3 (Last Resort) source. nil = no Pillar 3 fallback.
       #   Callers that want Last Resort placeholders inject the
       #   pre-built source here.
+      # @param assigned_only [Boolean] when true, iterate only
+      #   assigned codepoints (via {BlockIndex}). Default false:
+      #   iterate the full block range, matching the legacy
+      #   behavior (Pillar 3 fills unassigned slots when injected;
+      #   otherwise the Resolver returns nil for them and they're
+      #   silently skipped).
+      # @param codepoints [Array<Integer>, nil] explicit codepoint
+      #   list — overrides {BlockIndex} iteration. Used by
+      #   {BatchRunner} to extract only the gap set. nil = use
+      #   BlockIndex.
       def initialize(block:, pdf_path:, cache_dir: nil,
-                     tier1_sources: nil, pillar3_source: nil)
+                     tier1_sources: nil, pillar3_source: nil,
+                     assigned_only: false, codepoints: nil)
         @block = block
         @pdf_path = Pathname.new(pdf_path)
         @cache_dir = cache_dir && Pathname.new(cache_dir)
         @tier1_sources = tier1_sources || []
         @pillar3_source = pillar3_source
+        @assigned_only = assigned_only
+        @codepoints = codepoints
       end
 
       # @return [Array<Result>] one Result per codepoint that any
@@ -71,6 +105,11 @@ module Ucode
             svg: resolver_result.svg,
             tier: resolver_result.tier,
             provenance: resolver_result.provenance,
+            base_font: resolver_result.base_font,
+            gid: resolver_result.gid,
+            source_page: resolver_result.source_page,
+            source_cell: resolver_result.source_cell,
+            extractor_version: Ucode::VERSION,
           )
         end
         results
@@ -78,18 +117,27 @@ module Ucode
 
       private
 
-      # Yields every codepoint in the block's range in ascending
-      # order. We yield the whole range because the Resolver's
-      # tiers handle unassigned codepoints — Pillar 3 (when
-      # configured) maps every codepoint via its Format 13 cmap,
-      # so unassigned slots get a placeholder. With no Pillar 3
-      # injected, only assigned codepoints (those the embedded
-      # font actually covers) yield Results; the rest are silently
-      # skipped, satisfying the REQ's "skip unassigned codepoints".
+      # Yields every codepoint the {BlockIndex} exposes. With
+      # `assigned_only: false` (default), iterates the full block
+      # range so Pillar 3 (when configured) can map every codepoint
+      # via its Format 13 cmap, giving unassigned slots a placeholder.
+      # With `assigned_only: true`, iterates only assigned codepoints
+      # — useful for {GapAnalyzer}-driven extraction where unassigned
+      # slots have no chart specimen and no placeholder is desired.
       def each_codepoint(&)
         return enum_for(:each_codepoint) unless block_given?
 
-        (@block.range_first..@block.range_last).each(&)
+        if @codepoints
+          @codepoints.each(&)
+        elsif @assigned_only
+          block_index.each_assigned_codepoint(&)
+        else
+          block_index.each_codepoint_in_range(&)
+        end
+      end
+
+      def block_index
+        @block_index ||= BlockIndex.new(block: @block)
       end
 
       def build_resolver

data/lib/ucode/code_chart/fetcher.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "digest"
+require "pathname"
+
+module Ucode
+  module CodeChart
+    # Feature-facing PDF fetch + cache + integrity check for one
+    # Unicode block's Code Charts PDF. Wraps {Ucode::Fetch::Http} for
+    # the network I/O and adds:
+    #
+    #   * sha256 sidecar — recomputed on every cache hit; mismatch
+    #     raises {Ucode::CodeChartChecksumError} so tampering is
+    #     detected before extraction consumes a corrupt PDF.
+    #   * typed errors — HTTP 4xx raises {Ucode::CodeChartNotFoundError}
+    #     without retry; 5xx still retried by {Fetch::Http}.
+    #   * idempotency — cache hit returns the existing path without a
+    #     network call when both the PDF and the sha256 sidecar exist.
+    #
+    # Single CodeChart-feature-facing API: `Fetcher#fetch(block:)`.
+    # The HTTP layer stays a private collaborator; new transports
+    # (mirror, S3) subclass + register, no caller change.
+    class Fetcher
+      # @param version [String] UCD version, used as the cache namespace.
+      # @param http [Module<Ucode::Fetch::Http>, nil] injectable for
+      #   tests. nil = the real {Ucode::Fetch::Http}.
+      def initialize(version:, http: nil)
+        @version = version
+        @http = http || Fetch::Http
+      end
+
+      # @param block [Ucode::Models::Block]
+      # @return [Pathname] the cached PDF path. Downloads when missing.
+      # @raise [Ucode::CodeChartNotFoundError] HTTP 4xx or non-PDF body.
+      # @raise [Ucode::NetworkError] HTTP 5xx after all retries.
+      # @raise [Ucode::CodeChartChecksumError] cached PDF's sha256
+      #   doesn't match the sidecar.
+      def fetch(block:)
+        fetch_by_first_cp(block_first_cp: block.range_first, block_id: block.id)
+      end
+
+      # Alternative entrypoint when the caller has only the first
+      # codepoint. Same semantics as {#fetch}.
+      #
+      # @param block_first_cp [Integer]
+      # @param block_id [String, nil] for error context only.
+      # @return [Pathname]
+      def fetch_by_first_cp(block_first_cp:, **_kwargs)
+        path = per_block_path(block_first_cp)
+        return path if cache_valid?(path)
+
+        download(block_first_cp)
+        path
+      end
+
+      private
+
+      def per_block_path(block_first_cp)
+        Cache.pdfs_dir(@version).join("U#{hex_slug(block_first_cp)}.pdf")
+      end
+
+      def sidecar_path(pdf_path)
+        Pathname.new("#{pdf_path}.sha256")
+      end
+
+      def cache_valid?(pdf_path)
+        return false unless pdf_path.exist?
+
+        sidecar = sidecar_path(pdf_path)
+        return false unless sidecar.exist?
+
+        verify_sha256!(pdf_path, sidecar.read.strip)
+        true
+      end
+
+      # Raises {CodeChartChecksumError} when the on-disk PDF's hash
+      # doesn't match the recorded sidecar. Otherwise returns void.
+      def verify_sha256!(pdf_path, expected)
+        actual = Digest::SHA256.file(pdf_path).hexdigest
+        return if actual == expected
+
+        raise Ucode::CodeChartChecksumError.new(
+          "Code Charts PDF sha256 mismatch",
+          context: { pdf: pdf_path.to_s, expected: expected, actual: actual },
+        )
+      end
+
+      def download(block_first_cp, **_kwargs)
+        path = per_block_path(block_first_cp)
+        url = "#{Ucode.configuration.charts_base_url}/U#{hex_slug(block_first_cp)}.pdf"
+
+        @http.get(
+          url,
+          dest: path,
+          validate: :pdf,
+          not_found_class: Ucode::CodeChartNotFoundError,
+        )
+
+        write_sidecar(path)
+      end
+
+      def write_sidecar(pdf_path)
+        sha = Digest::SHA256.file(pdf_path).hexdigest
+        sidecar_path(pdf_path).write("#{sha}\n")
+      end
+
+      def hex_slug(codepoint)
+        codepoint.to_s(16).upcase.rjust(4, "0")
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/gap_analyzer/block_gap.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Ucode
+  module CodeChart
+    module GapAnalyzer
+      # Typed per-block result: which block, which codepoints the
+      # manifest's donor sources DON'T cover, and the UCD version
+      # the manifest declared.
+      #
+      # The unit of work {BatchRunner} iterates. Frozen on
+      # construction so callers can pass it around without worry.
+      BlockGap = Struct.new(
+        :block_id,
+        :missing_codepoints,
+        :ucd_version,
+        keyword_init: true,
+      ) do
+        def initialize(*)
+          super
+          self.missing_codepoints = Array(missing_codepoints).sort.freeze
+        end
+
+        # @return [Integer]
+        def size
+          missing_codepoints.size
+        end
+
+        # @return [Boolean]
+        def empty?
+          missing_codepoints.empty?
+        end
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/gap_analyzer/essenfont_manifest.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+require "ucode/code_chart/gap_analyzer/manifest"
+
+module Ucode
+  module CodeChart
+    module GapAnalyzer
+      # Parses the essenfont-style manifest YAML:
+      #
+      #   ucd_version: "17.0.0"
+      #   sources:
+      #     - name: noto-sans-sidetic
+      #       block: Sidetic
+      #       covered_codepoints: ["U+10920", "U+10921"]
+      #     - name: lentariso
+      #       block: Beria_Erfe
+      #       covered_codepoints: ["U+10940"]
+      #
+      # Coverage is unioned per block (a block with multiple donor
+      # sources has all their codepoints merged before the gap is
+      # computed).
+      class EssenfontManifest < Manifest
+        # @return [String]
+        def ucd_version
+          parsed.fetch("ucd_version")
+        end
+
+        # @return [Hash{String=>Array<Integer>}]
+        def coverage_by_block
+          sources = parsed.fetch("sources", [])
+          sources.each_with_object({}) do |src, acc|
+            block = src["block"] || (raise ArgumentError, "source missing 'block': #{src.inspect}")
+            cps = Array(src["covered_codepoints"]).map { |s| parse_cp(s) }
+            acc[block] ||= []
+            acc[block].concat(cps)
+          end
+        end
+
+        private
+
+        def parsed
+          @parsed ||= YAML.safe_load(read_text) || {}
+        end
+
+        # Accepts "U+10920" or "0x10920". Returns Integer. Bare
+        # decimal is rejected — too ambiguous (is "10920" hex or
+        # decimal?) and the manifest convention is to use one of the
+        # explicit prefixes.
+        def parse_cp(s)
+          m = s.match(/\A(?:U\+|0x)([0-9A-Fa-f]+)\z/)
+          return m[1].to_i(16) if m
+
+          raise ArgumentError, "unparseable codepoint (use U+XXXX or 0xXXXX): #{s.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/gap_analyzer/manifest.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Ucode
+  module CodeChart
+    module GapAnalyzer
+      # Abstract manifest parser. Subclasses own ONE manifest schema
+      # and expose a uniform `{block_id => covered_codepoints}` shape
+      # to {GapAnalyzer}.
+      #
+      # Subclasses must implement {#parse}.
+      class Manifest
+        # @param path [Pathname, String] manifest file path
+        def initialize(path)
+          @path = Pathname.new(path)
+        end
+
+        # @return [String] the UCD version declared in the manifest
+        def ucd_version
+          raise NotImplementedError
+        end
+
+        # @return [Hash{String=>Array<Integer>}] block_id → array of
+        #   codepoints the manifest's donor sources cover
+        def coverage_by_block
+          raise NotImplementedError
+        end
+
+        # @raise [ArgumentError] when the manifest file doesn't exist
+        def read_text
+          return @path.read unless @path.exist?
+
+          @path.read
+        end
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/gap_analyzer.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Ucode
+  module CodeChart
+    # Derives per-block "missing codepoint" sets from a donor manifest.
+    # REQ R5/R7. Pure computation — no I/O beyond reading the manifest
+    # file (handled by the {Manifest} subclass).
+    #
+    # ## OCP
+    #
+    # Adding a new manifest format (fontisan JSON, raw codepoint list)
+    # = one Manifest subclass + one constructor entry. GapAnalyzer
+    # core stays shape-agnostic.
+    #
+    # ## BlockGap
+    #
+    # Typed handoff. Callers ({BatchRunner}, CLI) don't grep manifest
+    # internals — they iterate {#each_block_gap} and receive a
+    # {BlockGap} per block.
+    module GapAnalyzer
+      autoload :Manifest, "ucode/code_chart/gap_analyzer/manifest"
+      autoload :EssenfontManifest, "ucode/code_chart/gap_analyzer/essenfont_manifest"
+      autoload :BlockGap, "ucode/code_chart/gap_analyzer/block_gap"
+
+      # Composes a {Manifest} parser with the {BlockIndex} factory
+      # to produce {BlockGap}s.
+      class Analyzer
+        # @param manifest [Manifest] parsed manifest
+        # @param blocks [Hash{String=>Ucode::Models::Block}] block_id →
+        #   Block model lookup. Used to construct BlockIndex per gap
+        #   block.
+        # @param block_index_class [Class] injectable BlockIndex class
+        #   for tests
+        def initialize(manifest:, blocks:, block_index_class: BlockIndex)
+          @manifest = manifest
+          @blocks = blocks
+          @block_index_class = block_index_class
+        end
+
+        # @yieldparam block_gap [BlockGap]
+        # @return [Enumerator, void]
+        def each_block_gap
+          return enum_for(:each_block_gap) unless block_given?
+
+          @manifest.coverage_by_block.each_key do |block_id|
+            gap = build_gap(block_id)
+            yield gap unless gap.empty?
+          end
+        end
+
+        # @return [Array<BlockGap>] one per block with at least one
+        #   missing codepoint. Blocks with full coverage are excluded.
+        def block_gaps
+          each_block_gap.to_a
+        end
+
+        # @return [Integer] sum of missing codepoints across blocks
+        def total_missing_codepoints
+          block_gaps.sum(&:size)
+        end
+
+        private
+
+        def build_gap(block_id)
+          block = @blocks[block_id] ||
+            raise(Ucode::UnknownBlockError.new(
+                    "manifest references unknown block",
+                    context: { block_id: block_id },
+                  ))
+
+          index = @block_index_class.new(block: block)
+          covered = Set.new(@manifest.coverage_by_block.fetch(block_id, []))
+          missing = index.assigned_codepoints.reject { |cp| covered.include?(cp) }
+          BlockGap.new(
+            block_id: block_id,
+            missing_codepoints: missing,
+            ucd_version: @manifest.ucd_version,
+          )
+        end
+      end
+    end
+  end
+end