ucode 0.4.0 → 0.5.1

data/lib/ucode/code_chart/verifier.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "pathname"
+
+module Ucode
+  module CodeChart
+    # Pixel-diff verification for extracted SVG glyphs against the
+    # source PDF cell. REQ R4.
+    #
+    # ## Strategy chain (OCP)
+    #
+    # One renderer strategy per external tool. The strategy exposes
+    # three primitive operations: render an SVG to PNG, render a PDF
+    # page region to PNG, and compute a pixel-diff percentage between
+    # two PNGs. {Verifier} orchestrates these primitives; it has no
+    # knowledge of any specific CLI tool.
+    #
+    # Adding a new renderer (cairo, imagemagick, …) = one Strategy
+    # subclass + one entry in {Verifier::Builder.pick}. Verifier core
+    # never changes.
+    #
+    # ## Result types (see Verifier::Result)
+    #
+    #   * `Pass` — diff < FAIL_THRESHOLD (default 1.0%).
+    #   * `Fail` — diff ≥ threshold; carries the percent + diff
+    #     artifact path for inspection.
+    #   * `Skipped` — extractor produced no `source_page`/`source_cell`
+    #     (e.g. ToUnicode-only path, or Last Resort placeholder), so no
+    #     honest cell-diff is possible. Surfaces as a warning, NOT a
+    #     pass.
+    class Verifier
+      autoload :Strategy, "ucode/code_chart/verifier/strategy"
+      autoload :ResvgStrategy, "ucode/code_chart/verifier/resvg_strategy"
+      autoload :MutoolStrategy, "ucode/code_chart/verifier/mutool_strategy"
+      autoload :Builder, "ucode/code_chart/verifier/builder"
+      autoload :PageRenderCache, "ucode/code_chart/verifier/page_render_cache"
+      autoload :Result, "ucode/code_chart/verifier/result"
+
+      DEFAULT_CELL_SIZE = 40.0
+      private_constant :DEFAULT_CELL_SIZE
+
+      # @param strategy [Strategy, nil] nil = {Builder.pick} auto.
+      # @param diff_dir [Pathname, String] where diff artifacts land
+      # @param threshold [Float, nil] override {Strategy::FAIL_THRESHOLD}
+      def initialize(diff_dir:, strategy: nil, threshold: nil)
+        @strategy = strategy || Builder.pick
+        @diff_dir = Pathname.new(diff_dir)
+        @threshold = threshold || (Strategy::FAIL_THRESHOLD if @strategy)
+        if @strategy
+          @page_cache = PageRenderCache.new(diff_dir: @diff_dir.join(".cache"),
+                                            strategy: @strategy)
+        end
+      end
+
+      # @return [Boolean] true iff a usable strategy was found
+      def available?
+        !@strategy.nil? && @strategy.available?
+      end
+
+      # @param result [Ucode::CodeChart::Extractor::Result]
+      # @param pdf_path [Pathname, String, nil] source PDF
+      # @return [Result::Pass, Result::Fail, Result::Skipped]
+      def verify(result, pdf_path:)
+        unless @strategy
+          return Result::Skipped.new(codepoint: result.codepoint,
+                                     reason: :no_strategy)
+        end
+
+        loc = extract_location(result)
+        unless loc
+          return Result::Skipped.new(codepoint: result.codepoint,
+                                     reason: :no_location)
+        end
+
+        path = Pathname.new(pdf_path)
+        unless path.exist?
+          return Result::Skipped.new(codepoint: result.codepoint,
+                                     reason: :no_pdf)
+        end
+
+        verify_with_location(result, path, loc)
+      end
+
+      private
+
+      def extract_location(result)
+        return nil unless result.source_page && result.source_cell
+
+        {
+          page: result.source_page,
+          x: result.source_cell[:x],
+          y: result.source_cell[:y],
+        }
+      end
+
+      def verify_with_location(result, pdf_path, loc)
+        svg_png = render_svg(result)
+        cell_png = render_cell(pdf_path, loc)
+
+        percent = @strategy.diff(svg_png, cell_png)
+        if percent < @threshold
+          Result::Pass.new(codepoint: result.codepoint, percent: percent)
+        else
+          artifact = write_artifact(result, svg_png, cell_png)
+          Result::Fail.new(codepoint: result.codepoint,
+                           percent: percent,
+                           diff_path: artifact)
+        end
+      ensure
+        [svg_png, cell_png].each { |p| FileUtils.rm_f(p.to_s) if p&.exist? }
+      end
+
+      def render_svg(result)
+        svg_path = @diff_dir.join("#{format_cp(result.codepoint)}.svg")
+        svg_path.dirname.mkpath
+        svg_path.write(result.svg)
+        png_path = Pathname.new("#{svg_path}.png")
+        @strategy.render_svg(svg_path, png_path)
+        png_path
+      ensure
+        FileUtils.rm_f(svg_path.to_s) if svg_path&.exist?
+      end
+
+      def render_cell(pdf_path, loc)
+        rect = cell_rect(loc)
+        png_path = @diff_dir.join(".cache",
+                                  "cell-#{loc[:page]}-" \
+                                  "#{format('%05.2f', loc[:x])}-" \
+                                  "#{format('%05.2f', loc[:y])}.png")
+        png_path.dirname.mkpath
+        @strategy.render_pdf_region(pdf_path, loc[:page], rect, png_path)
+        png_path
+      end
+
+      def cell_rect(loc)
+        half = DEFAULT_CELL_SIZE / 2
+        { x: loc[:x] - half, y: loc[:y] - half,
+          w: DEFAULT_CELL_SIZE, h: DEFAULT_CELL_SIZE }
+      end
+
+      def write_artifact(result, png_a, png_b)
+        dest = @diff_dir.join("#{format_cp(result.codepoint)}.diff.png")
+        @strategy.write_diff_artifact(png_a, png_b, dest) || dest
+      end
+
+      def format_cp(codepoint)
+        "U+#{codepoint.to_s(16).upcase.rjust(4, '0')}"
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/writer.rb

@@ -3,12 +3,6 @@
 require "digest"
 require "pathname"
 
-require "ucode/code_chart/extractor"
-require "ucode/code_chart/provenance"
-require "ucode/code_chart/sidecar"
-require "ucode/error"
-require "ucode/version_resolver"
-
 module Ucode
   module CodeChart
     # Orchestrates extraction + provenance sidecar writing for one
@@ -49,10 +43,17 @@ module Ucode
       # @param cache_dir [Pathname, String, nil] font-stream cache
       #   directory for the EmbeddedFonts::PdfSource.
       # @param now [Time, nil] timestamp override (for tests).
-      # @param pillar3_source, tier1_sources: forwarded to the Extractor.
+      # @param pillar3_source, tier1_sources, assigned_only, codepoints:
+      #   forwarded to the Extractor.
+      # @param extractor [Ucode::CodeChart::Extractor, nil] pre-built
+      #   Extractor instance. When nil (default), the Writer constructs
+      #   one from the other parameters. Injecting lets callers (tests)
+      #   bypass the mutool-dependent default Extractor.
       def initialize(output_root:, pdf_path:, ucd_version: nil,
                      cache_dir: nil, now: nil,
-                     pillar3_source: nil, tier1_sources: nil)
+                     pillar3_source: nil, tier1_sources: nil,
+                     assigned_only: false, codepoints: nil,
+                     extractor: nil)
         @output_root = Pathname.new(output_root)
         @pdf_path = Pathname.new(pdf_path)
         @ucd_version = ucd_version || VersionResolver.resolve(nil)
@@ -60,6 +61,9 @@ module Ucode
         @now = now
         @pillar3_source = pillar3_source
         @tier1_sources = tier1_sources
+        @assigned_only = assigned_only
+        @codepoints = codepoints
+        @extractor = extractor
       end
 
       # Extracts every codepoint in `block` and writes `<block_id>/<cp>.svg`
@@ -72,27 +76,30 @@ module Ucode
         block_dir = @output_root.join(block.id)
         block_dir.mkpath
 
-        pdf_sha = CodeChart.sha256_of(@pdf_path)
+        pdf_sha = Provenance.sha256_of(@pdf_path)
 
         sidecar = Sidecar.new(output_root: block_dir)
-        extractor = Extractor.new(
+        extractor = @extractor || Extractor.new(
           block: block,
           pdf_path: @pdf_path,
           cache_dir: @cache_dir,
           pillar3_source: @pillar3_source,
           tier1_sources: @tier1_sources,
+          assigned_only: @assigned_only,
+          codepoints: @codepoints,
         )
-
         results = extractor.extract
         svgs = 0
         sidecars = 0
         results.each do |result|
           write_svg(block_dir, result)
           svgs += 1
-          provenance = CodeChart.build(
+          provenance = Provenance.build(
             block: block, codepoint: result.codepoint,
             ucd_version: @ucd_version, pdf_path: @pdf_path,
-            now: @now,
+            pdf_sha: pdf_sha, now: @now,
+            base_font: result.base_font, gid: result.gid,
+            source_page: result.source_page, source_cell: result.source_cell,
           )
           sidecar.write(provenance)
           sidecars += 1

data/lib/ucode/code_chart.rb

@@ -31,9 +31,15 @@ module Ucode
   # (e.g. a future pure-Ruby PDF parser per ADR-0001) does not
   # change the public API.
   module CodeChart
+    autoload :BatchRunner, "ucode/code_chart/batch_runner"
+    autoload :BlockIndex, "ucode/code_chart/block_index"
+    autoload :CoverageGapIndex, "ucode/code_chart/coverage_gap_index"
     autoload :Extractor, "ucode/code_chart/extractor"
+    autoload :Fetcher, "ucode/code_chart/fetcher"
+    autoload :GapAnalyzer, "ucode/code_chart/gap_analyzer"
     autoload :Provenance, "ucode/code_chart/provenance"
     autoload :Sidecar, "ucode/code_chart/sidecar"
+    autoload :Verifier, "ucode/code_chart/verifier"
     autoload :Writer, "ucode/code_chart/writer"
   end
 end

data/lib/ucode/error.rb

@@ -115,12 +115,22 @@ module Ucode
   # (corrupt PDF, bad object ref, etc.). The context carries the argv.
   class MutoolError < GlyphError; end
 
+  # CodeChart feature errors — concerns under `Ucode::CodeChart::*`.
+  # Distinct from the generic {GlyphError} hierarchy so callers can
+  # scope rescue clauses to feature failures.
+  class CodeChartError < GlyphError; end
+
   # The Code Charts PDF for a requested block cannot be obtained: the
   # network returned 4xx/5xx, the response wasn't application/pdf, or
   # the body didn't start with the `%PDF` magic. Distinct from
   # {EmbeddedFontsMissingError} (which fires when the file is already
   # on disk and we just can't open it): this fires at fetch time.
-  class CodeChartNotFoundError < GlyphError; end
+  class CodeChartNotFoundError < CodeChartError; end
+
+  # A cached Code Charts PDF failed sha256 verification — the file on
+  # disk doesn't match the sidecar hash from the previous successful
+  # download. Implies external tampering or a filesystem fault.
+  class CodeChartChecksumError < CodeChartError; end
 
   # Pre-build validation failed for a universal-set build. The
   # context carries the failing checks so the CLI can render a

data/lib/ucode/fetch/http.rb

@@ -29,11 +29,18 @@ module Ucode
         #   {Ucode::CodeChartNotFoundError} with the offending header
         #   value in `context:` on failure. nil = no validation (the
         #   default for non-PDF callers like UcdZip and UnihanZip).
+        # @param not_found_class [Class, nil] when set, HTTP 4xx
+        #   responses raise this class (instantiated with message +
+        #   context) instead of being treated as retriable transport
+        #   errors. nil = 4xx is retriable like any other non-success.
         # @return [Pathname] destination path on success.
         # @raise [Ucode::NetworkError] if all retries fail.
         # @raise [Ucode::CodeChartNotFoundError] when `validate: :pdf`
         #   and the response fails content validation.
-        def get(url, dest:, retries: nil, timeout: nil, validate: nil)
+        # @raise [<not_found_class>] when `not_found_class:` is set
+        #   and the server returns 4xx.
+        def get(url, dest:, retries: nil, timeout: nil, validate: nil,
+                not_found_class: nil)
           uri = url.is_a?(URI) ? url : URI(url)
           destination = Pathname.new(dest)
           destination.dirname.mkpath
@@ -44,7 +51,8 @@ module Ucode
 
           last_error = nil
           (attempts + 1).times do |attempt|
-            response = stream_to(uri, destination, read_timeout)
+            response = stream_to(uri, destination, read_timeout,
+                                 not_found_class: not_found_class)
             validate_response!(validate, response, destination) if validate
             return destination
           rescue ValidationFailure => e
@@ -80,13 +88,15 @@ module Ucode
           end
         end
 
-        def stream_to(uri, destination, read_timeout)
+        def stream_to(uri, destination, read_timeout, not_found_class: nil)
           response = nil
           Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
                                               read_timeout: read_timeout) do |http|
             request = Net::HTTP::Get.new(uri)
             http.request(request) do |r|
               unless r.is_a?(Net::HTTPSuccess)
+                raise ValidationFailure.new(not_found_error(not_found_class, uri, r)) if not_found_class && r.is_a?(Net::HTTPClientError)
+
                 raise "HTTP #{r.code} #{r.message}"
               end
 
@@ -97,6 +107,16 @@ module Ucode
           response or raise "no response received"
         end
 
+        # Builds the not-found error (e.g. CodeChartNotFoundError)
+        # for a 4xx response, fed through ValidationFailure so the
+        # retry loop in `get` doesn't re-attempt a permanent miss.
+        def not_found_error(klass, uri, response)
+          klass.new(
+            "HTTP #{response.code} #{response.message}",
+            context: { url: uri.to_s, status: response.code.to_i },
+          )
+        end
+
         def write_body(response, destination)
           partial = destination.sub_ext("#{destination.extname}.part")
           File.open(partial, "wb") do |file|

data/lib/ucode/glyphs/embedded_fonts/catalog.rb

@@ -53,6 +53,38 @@ module Ucode
           index[codepoint]
         end
 
+        # Locate where in the PDF a codepoint's specimen was rendered.
+        # Returns `{page:, x:, y:}` (PDF user space, origin
+        # bottom-left) by joining `#lookup` with the trace cache's
+        # `(font, gid)` search. Nil when:
+        #   * the codepoint isn't in this PDF, OR
+        #   * the embedded font has no traced specimen (ToUnicode-only
+        #     path with no positional correlation data), OR
+        #   * the trace cache wasn't populated (lazy + never asked
+        #     for any positional strategy).
+        #
+        # Memoized per-Catalog; the second call for the same
+        # codepoint is O(1).
+        #
+        # @param codepoint [Integer]
+        # @return [Hash{Symbol=>Integer, Float}, nil]
+        def location_for(codepoint)
+          @locations ||= {}
+          return @locations[codepoint] if @locations.key?(codepoint)
+
+          entry = lookup(codepoint)
+          result = if entry.nil?
+                     nil
+                   else
+                     trace_cache.find_glyph(
+                       base_font: entry.base_font,
+                       gid: entry.gid_for(codepoint),
+                     )
+                   end
+          @locations[codepoint] = result
+          result
+        end
+
         # @return [Array<Integer>] every codepoint this PDF covers
         def codepoints
           index.keys
@@ -114,6 +146,20 @@ module Ucode
             force_positional_for_font_ids: @force_positional_for_font_ids,
             correlator_configs: @correlator_configs,
             indexer: indexer,
+            trace_cache: trace_cache,
+          )
+        end
+
+        # The PageTraceCache shared with the CodepointMapper. Lazily
+        # constructed on first reference; #location_for uses the same
+        # cache, so the per-page `mutool trace` cost is paid exactly
+        # once across the catalog + location lookups for a PDF.
+        #
+        # @return [PageTraceCache]
+        def trace_cache
+          @trace_cache ||= PageTraceCache.new(
+            pdf: @source.pdf_path,
+            page_count: indexer.page_count,
           )
         end
       end

data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb

@@ -68,13 +68,18 @@ module Ucode
         # stubs for tests should construct strategies directly and pass
         # them to +#initialize+.
         #
+        # @param trace_cache [PageTraceCache, nil] when provided, the
+        #   TraceStrategy shares this cache (lets the caller reuse the
+        #   traced pages for downstream concerns like Catalog's
+        #   location lookup). nil = construct internally.
         # @return [CodepointMapper]
         def self.build(source:, correlator_configs:, indexer:,
                        block_range: nil, force_positional_for_font_ids: Set.new,
                        mutool_show: Mutool::Show.new,
                        mutool_draw: Mutool::Draw.new,
-                       mutool_trace: Mutool::Trace.new)
-          trace_cache = PageTraceCache.new(
+                       mutool_trace: Mutool::Trace.new,
+                       trace_cache: nil)
+          trace_cache ||= PageTraceCache.new(
             pdf: source.pdf_path,
             page_count: indexer.page_count,
             mutool: mutool_trace,
@@ -152,7 +157,9 @@ module Ucode
         def intrinsic_out_of_scope?(intrinsic_result)
           return false unless @block_range
 
-          intrinsic_result.keys.none? { |cp| @block_range.include?(cp) }
+          # block_range is a Range, not an Array; Array#intersect? would
+          # force an eager .to_a conversion on potentially huge CJK ranges.
+          intrinsic_result.keys.all? { |cp| !@block_range.include?(cp) }
         end
 
         # Positional chain: union of all positional strategies' results.

data/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb

@@ -72,6 +72,28 @@ module Ucode
           end
         end
 
+        # Locate the first occurrence of a specific (font, gid) pair
+        # across all traced pages. Returns nil when no match. Used by
+        # {Catalog#location_for} to attribute a codepoint's source
+        # page + (x, y) without exposing the cache's internal layout
+        # to callers.
+        #
+        # @param base_font [String] specimen font BaseFont name
+        # @param gid [Integer] glyph id inside that font
+        # @return [Hash{Symbol=>Integer, Float}, nil]
+        #   `{ page: Integer, x: Float, y: Float }` or nil
+        def find_glyph(base_font:, gid:)
+          glyphs_by_page.each_with_index do |page_glyphs, idx|
+            next if idx.zero?
+
+            match = page_glyphs.find do |g|
+              g.font_name == base_font && g.gid == gid
+            end
+            return { page: idx, x: match.x, y: match.y } if match
+          end
+          nil
+        end
+
         private
 
         def fetch_glyphs_by_page

data/lib/ucode/glyphs/embedded_fonts/renderer.rb

@@ -12,7 +12,16 @@ module Ucode
       # renderer.
       class Renderer
         # Result of rendering one codepoint.
-        Result = Struct.new(:codepoint, :base_font, :gid, :svg, keyword_init: true) do
+        #
+        # `source_page` + `source_cell` come from the Catalog's
+        # location index when available. Nil when the catalog hasn't
+        # traced the PDF (ToUnicode-only path) — downstream concerns
+        # treat that as "no positional data; skip cell-based diff".
+        Result = Struct.new(
+          :codepoint, :base_font, :gid, :svg,
+          :source_page, :source_cell,
+          keyword_init: true,
+        ) do
           def ok?
             !svg.nil?
           end
@@ -37,7 +46,15 @@ module Ucode
           return nil if outline.nil? || outline.empty?
 
           svg = Svg.new(outline, codepoint: codepoint, base_font: entry.base_font).to_s
-          Result.new(codepoint: codepoint, base_font: entry.base_font, gid: gid, svg: svg)
+          location = @catalog.location_for(codepoint)
+          Result.new(
+            codepoint: codepoint,
+            base_font: entry.base_font,
+            gid: gid,
+            svg: svg,
+            source_page: location&.fetch(:page),
+            source_cell: location && { x: location[:x], y: location[:y] },
+          )
         end
       end
     end

data/lib/ucode/glyphs/source.rb

@@ -25,7 +25,17 @@ module Ucode
       # One resolved glyph. Carries the SVG payload and enough
       # provenance to debug "where did this glyph come from?" without
       # holding a reference back to the source.
-      Result = Struct.new(:tier, :codepoint, :svg, :provenance, keyword_init: true)
+      #
+      # The optional `base_font`, `gid`, `source_page`, `source_cell`
+      # fields carry the renderer's localization data so downstream
+      # concerns (Verifier pixel-diff against the source PDF cell,
+      # Provenance audit trail) don't need to re-derive it. Sources
+      # that don't know this data (e.g. LastResort) leave them nil.
+      Result = Struct.new(
+        :tier, :codepoint, :svg, :provenance,
+        :base_font, :gid, :source_page, :source_cell,
+        keyword_init: true,
+      )
 
       # @return [Symbol] one of :tier1, :pillar1, :pillar2, :pillar3
       def tier

data/lib/ucode/glyphs/sources/pillar1_embedded_tounicode.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "ucode/glyphs/source"
-require "ucode/glyphs/embedded_fonts/renderer"
-
 module Ucode
   module Glyphs
     module Sources
@@ -54,8 +51,16 @@ module Ucode
           result = @renderer.render(codepoint)
           return nil unless result
 
-          Result.new(tier: tier, codepoint: codepoint,
-                     svg: result.svg, provenance: provenance)
+          Result.new(
+            tier: tier,
+            codepoint: codepoint,
+            svg: result.svg,
+            provenance: provenance,
+            base_font: result.base_font,
+            gid: result.gid,
+            source_page: result.source_page,
+            source_cell: result.source_cell,
+          )
         end
       end
     end

data/lib/ucode/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ucode
-  VERSION = "0.4.0"
+  VERSION = "0.5.1"
 end

data/lib/ucode.rb

@@ -34,7 +34,9 @@ module Ucode
   autoload :GlyphError, "ucode/error"
   autoload :LastResortMissingError, "ucode/error"
   autoload :EmbeddedFontsMissingError, "ucode/error"
+  autoload :CodeChartError, "ucode/error"
   autoload :CodeChartNotFoundError, "ucode/error"
+  autoload :CodeChartChecksumError, "ucode/error"
   autoload :UnknownUnicodeVersionError, "ucode/error"
 
   # Infrastructure

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ucode
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-08 00:00:00.000000000 Z
+date: 2026-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -239,9 +239,24 @@ files:
 - lib/ucode/cache.rb
 - lib/ucode/cli.rb
 - lib/ucode/code_chart.rb
+- lib/ucode/code_chart/batch_runner.rb
+- lib/ucode/code_chart/block_index.rb
+- lib/ucode/code_chart/coverage_gap_index.rb
 - lib/ucode/code_chart/extractor.rb
+- lib/ucode/code_chart/fetcher.rb
+- lib/ucode/code_chart/gap_analyzer.rb
+- lib/ucode/code_chart/gap_analyzer/block_gap.rb
+- lib/ucode/code_chart/gap_analyzer/essenfont_manifest.rb
+- lib/ucode/code_chart/gap_analyzer/manifest.rb
 - lib/ucode/code_chart/provenance.rb
 - lib/ucode/code_chart/sidecar.rb
+- lib/ucode/code_chart/verifier.rb
+- lib/ucode/code_chart/verifier/builder.rb
+- lib/ucode/code_chart/verifier/mutool_strategy.rb
+- lib/ucode/code_chart/verifier/page_render_cache.rb
+- lib/ucode/code_chart/verifier/result.rb
+- lib/ucode/code_chart/verifier/resvg_strategy.rb
+- lib/ucode/code_chart/verifier/strategy.rb
 - lib/ucode/code_chart/writer.rb
 - lib/ucode/commands.rb
 - lib/ucode/commands/audit.rb