ucode 0.4.0 → 0.5.0

data/lib/ucode/code_chart/provenance.rb

@@ -1,18 +1,21 @@
 # frozen_string_literal: true
 
 require "digest"
+require "pathname"
 require "time"
 
-require "ucode/version"
+require "lutaml/model"
 
 module Ucode
   module CodeChart
     # Per-codepoint provenance value object — every field the REQ
-    # (R5) requires in the sidecar JSON next to each extracted SVG.
+    # (R6) requires in the sidecar JSON next to each extracted SVG.
     #
-    # Single source of truth for the provenance schema: the
-    # {Sidecar} writer reads this Struct, the Writer constructs it.
-    # Adding a field is one place to change.
+    # The single source of truth for the sidecar schema: a
+    # lutaml-model class with an explicit `key_value` mapping block.
+    # Adding a field = one `attribute` declaration + one `map` line
+    # in the mapping block. The {Sidecar} writer calls `to_hash`
+    # (framework-provided); no hand-rolled `to_h` anywhere.
     #
     # `extractor_version` reads from `Ucode::VERSION` at construction
     # so the field stays in sync with the gem's version bump — single
@@ -20,63 +23,101 @@ module Ucode
     #
     # `extracted_at` is the extraction event timestamp (UTC ISO8601),
     # not the file-write timestamp.
-    Provenance = Struct.new(
-      :codepoint,
-      :block,
-      :source_pdf_url,
-      :source_pdf_sha256,
-      :ucd_version,
-      :extracted_at,
-      :extractor_version,
-      keyword_init: true,
-    )
+    class Provenance < Lutaml::Model::Serializable
+      attribute :codepoint, :string
+      attribute :block, :string
+      attribute :source_pdf_url, :string
+      attribute :source_pdf_sha256, :string
+      attribute :ucd_version, :string
+      attribute :extracted_at, :string
+      attribute :extractor_version, :string
+      attribute :base_font, :string
+      attribute :gid, :integer
+      attribute :source_page, :integer
+      attribute :source_cell, :hash
 
-    # Computes the source PDF's URL from a block name and first
-    # codepoint. Mirrors the per-block URL convention in
-    # {Ucode::Fetch::CodeCharts}: the hex representation of the
-    # codepoint, zero-padded to a minimum of 4 digits (e.g.
-    # `U0000.pdf` for BMP, `U10920.pdf` for Plane 1,
-    # `U100000.pdf` for Plane 16 SPUA-B).
-    #
-    # @param block_first_cp [Integer]
-    # @return [String]
-    def self.code_chart_url(block_first_cp)
-      slug = block_first_cp.to_s(16).upcase.rjust(4, "0")
-      "#{Ucode.configuration.charts_base_url}/U#{slug}.pdf"
-    end
+      key_value do
+        map "codepoint", to: :codepoint
+        map "block", to: :block
+        map "source_pdf_url", to: :source_pdf_url
+        map "source_pdf_sha256", to: :source_pdf_sha256
+        map "ucd_version", to: :ucd_version
+        map "extracted_at", to: :extracted_at
+        map "extractor_version", to: :extractor_version
+        map "base_font", to: :base_font
+        map "gid", to: :gid
+        map "source_page", to: :source_page
+        map "source_cell", to: :source_cell
+      end
 
-    # Builds a Provenance from the inputs the {Writer} has on hand
-    # (block, codepoint, ucd_version, pdf_path). Computes the PDF
-    # hash + URL once. The `extracted_at` timestamp is fixed at
-    # call time so re-running the same block produces identical
-    # provenance JSON for unchanged codepoints.
-    #
-    # @param block [Ucode::Models::Block]
-    # @param codepoint [Integer]
-    # @param ucd_version [String]
-    # @param pdf_path [Pathname, String]
-    # @param now [Time, nil] override for tests
-    # @return [Provenance]
-    def self.build(block:, codepoint:, ucd_version:, pdf_path:, now: nil)
-      path = Pathname.new(pdf_path)
-      Provenance.new(
-        codepoint: format("U+%04X", codepoint),
-        block: block.id,
-        source_pdf_url: code_chart_url(block.range_first),
-        source_pdf_sha256: sha256_of(path),
-        ucd_version: ucd_version,
-        extracted_at: (now || Time.now.utc).iso8601,
-        extractor_version: Ucode::VERSION,
-      )
-    end
+      # Computes the source PDF's URL from a block name and first
+      # codepoint. Mirrors the per-block URL convention in
+      # {Ucode::Fetch::CodeCharts}: the hex representation of the
+      # codepoint, zero-padded to a minimum of 4 digits (e.g.
+      # `U0000.pdf` for BMP, `U10920.pdf` for Plane 1,
+      # `U100000.pdf` for Plane 16 SPUA-B).
+      #
+      # @param block_first_cp [Integer]
+      # @return [String]
+      def self.code_chart_url(block_first_cp)
+        slug = block_first_cp.to_s(16).upcase.rjust(4, "0")
+        "#{Ucode.configuration.charts_base_url}/U#{slug}.pdf"
+      end
+
+      # Builds a Provenance from the inputs the {Writer} has on hand
+      # (block, codepoint, ucd_version, pdf_path). Computes the PDF
+      # hash + URL once. The `extracted_at` timestamp is fixed at
+      # call time so re-running the same block produces identical
+      # provenance JSON for unchanged codepoints.
+      #
+      # The optional `base_font`, `gid`, `source_page`, `source_cell`
+      # come from the {Extractor::Result} when available. Nil values
+      # are emitted as JSON `null` — preserves the audit trail's
+      # honesty about what the extractor actually knew.
+      #
+      # @param block [Ucode::Models::Block]
+      # @param codepoint [Integer]
+      # @param ucd_version [String]
+      # @param pdf_path [Pathname, String]
+      # @param pdf_sha [String, nil] pre-computed sha256 hex digest.
+      #   When nil, computed from `pdf_path`. The {Writer} passes its
+      #   already-computed summary hash to avoid re-reading the PDF
+      #   per codepoint.
+      # @param now [Time, nil] override for tests
+      # @param base_font [String, nil] PDF BaseFont name
+      # @param gid [Integer, nil] GID inside that font
+      # @param source_page [Integer, nil] 1-based PDF page number
+      # @param source_cell [Hash{Symbol=>Float}, nil] `{x:, y:}` PDF
+      #   user space coordinates of the specimen
+      # @return [Provenance]
+      def self.build(block:, codepoint:, ucd_version:, pdf_path:,
+                     pdf_sha: nil, now: nil,
+                     base_font: nil, gid: nil,
+                     source_page: nil, source_cell: nil)
+        path = Pathname.new(pdf_path)
+        new(
+          codepoint: format("U+%04X", codepoint),
+          block: block.id,
+          source_pdf_url: code_chart_url(block.range_first),
+          source_pdf_sha256: pdf_sha || sha256_of(path),
+          ucd_version: ucd_version,
+          extracted_at: (now || Time.now.utc).iso8601,
+          extractor_version: Ucode::VERSION,
+          base_font: base_font,
+          gid: gid,
+          source_page: source_page,
+          source_cell: source_cell,
+        )
+      end
 
-    # @param path [Pathname]
-    # @return [String] hex digest, "" when the path doesn't exist
-    #   (callers can decide how to handle a missing hash)
-    def self.sha256_of(path)
-      return "" unless path.exist?
+      # @param path [Pathname]
+      # @return [String] hex digest, "" when the path doesn't exist
+      #   (callers can decide how to handle a missing hash)
+      def self.sha256_of(path)
+        return "" unless path.exist?
 
-      Digest::SHA256.file(path).hexdigest
+        Digest::SHA256.file(path).hexdigest
+      end
     end
   end
 end

data/lib/ucode/code_chart/sidecar.rb

@@ -3,8 +3,6 @@
 require "json"
 require "pathname"
 
-require "ucode/repo/atomic_writes"
-
 module Ucode
   module CodeChart
     # Writes a {Provenance} to disk as the sidecar JSON next to its
@@ -17,7 +15,8 @@ module Ucode
     # Idempotent via {Ucode::Repo::AtomicWrites#write_atomic}: a
     # re-write of byte-identical content is a no-op (no temp-file
     # rename). Provenance JSON is canonical (sorted keys via Ruby's
-    # stdlib JSON), so the byte-equality test is sound.
+    # stdlib JSON over the lutaml-model `to_hash` output), so the
+    # byte-equality test is sound.
     class Sidecar
       include Ucode::Repo::AtomicWrites
 
@@ -31,7 +30,7 @@ module Ucode
       # @return [Pathname] the written sidecar path
       def write(provenance)
         path = path_for(provenance)
-        payload = "#{JSON.pretty_generate(provenance.to_h)}\n"
+        payload = "#{JSON.pretty_generate(provenance.to_hash)}\n"
         write_atomic(path, payload)
         path
       end

data/lib/ucode/code_chart/verifier/builder.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Ucode
+  module CodeChart
+    class Verifier
+      # Selects a {Strategy} based on tool availability. Preference
+      # order:
+      #
+      #   1. `UCODE_VERIFIER_STRATEGY` env var override — explicit
+      #      user choice. Value is the strategy class name (e.g.
+      #      `"resvg"`, `"mutool"`).
+      #   2. {ResvgStrategy} — preferred for accuracy and speed.
+      #   3. {MutoolStrategy} — fallback (always installed if the
+      #      trace pipeline works).
+      #
+      # Returns nil when no strategy is available; {Verifier} then
+      # emits `Skipped` for every glyph.
+      module Builder
+        ORDER = %w[resvg mutool].freeze
+        private_constant :ORDER
+
+        class << self
+          # @param strategies [Array<Strategy>, nil] injectable list
+          #   for tests. nil = construct the default chain.
+          # @return [Strategy, nil]
+          def pick(strategies: nil)
+            strategies ||= default_strategies
+            env_choice = ENV["UCODE_VERIFIER_STRATEGY"]
+            if env_choice
+              found = strategies.find { |s| name_of(s) == env_choice }
+              return found if found&.available?
+            end
+            strategies.find(&:available?)
+          end
+
+          private
+
+          def default_strategies
+            [ResvgStrategy.new, MutoolStrategy.new]
+          end
+
+          def name_of(strategy)
+            strategy.class.name.split("::").last.sub("Strategy", "").downcase
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/verifier/mutool_strategy.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "pathname"
+
+module Ucode
+  module CodeChart
+    class Verifier
+      # {Strategy} backed by `mutool draw`. Slower than resvg but
+      # always available on a system that has the trace pipeline
+      # working (mutool is already required elsewhere).
+      #
+      # Pixel diff: byte-wise comparison of the two rendered PNGs.
+      # Crude (real pixel diff needs ChunkyPNG or vips), but
+      # sufficient to catch the regression classes the REQ lists:
+      # clipped glyphs, missing paths, wrong-cell content,
+      # coordinate-flip errors. Returns the percentage of differing
+      # bytes — a useful relative signal even when not a true
+      # pixel-perfect diff.
+      class MutoolStrategy < Strategy
+        # @param runner [Ucode::Glyphs::EmbeddedFonts::Mutool::SystemRunner]
+        #   injectable for tests
+        def initialize(runner: Ucode::Glyphs::EmbeddedFonts::Mutool::SystemRunner.new)
+          super()
+          @runner = runner
+        end
+
+        def available?
+          system("which mutool >/dev/null 2>&1")
+        end
+
+        def render_svg(svg_path, png_path, scale: 2.0)
+          dpi = (72 * scale).round
+          @runner.run("mutool", "draw", "-o", png_path.to_s,
+                      "-r", dpi.to_s, svg_path.to_s)
+          Pathname.new(png_path)
+        end
+
+        def render_pdf_region(pdf_path, page, rect, png_path, scale: 2.0)
+          dpi = (72 * scale).round
+          @runner.run("mutool", "draw", "-o", png_path.to_s,
+                      "-r", dpi.to_s,
+                      "-R", format_rect(rect, scale),
+                      pdf_path.to_s, page.to_s)
+          Pathname.new(png_path)
+        end
+
+        def diff(png_a, png_b)
+          bytes_a = File.binread(png_a)
+          bytes_b = File.binread(png_b)
+          return 100.0 if bytes_a.bytesize != bytes_b.bytesize
+
+          same = bytes_a.each_byte.with_index.count do |byte, idx|
+            byte == bytes_b.getbyte(idx)
+          end
+          (100.0 * (1.0 - same.to_f / bytes_a.bytesize)).round(2)
+        end
+
+        def write_diff_artifact(png_a, png_b, dest)
+          data_a = File.binread(png_a)
+          data_b = File.binread(png_b)
+          Pathname.new(dest).binwrite(data_a + data_b)
+          Pathname.new(dest)
+        end
+
+        private
+
+        # mutool's -R flag takes a bbox in page coordinates
+        # pre-scaled by the DPI factor: `<x0,y0,x1,y1>`.
+        def format_rect(rect, scale)
+          x0 = (rect[:x] * scale).round
+          y0 = (rect[:y] * scale).round
+          x1 = x0 + (rect[:w] * scale).round
+          y1 = y0 + (rect[:h] * scale).round
+          "#{x0},#{y0},#{x1},#{y1}"
+        end
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/verifier/page_render_cache.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Ucode
+  module CodeChart
+    class Verifier
+      # Caches per-page PDF→PNG renders so multiple glyphs on the
+      # same page share one `mutool draw` invocation. The Verifier
+      # asks for the rendered page; the cache either reuses the
+      # existing PNG or renders it on demand.
+      #
+      # PNG path layout: `<diff_dir>/.cache/page-<N>-<sha>.png`
+      # where `<sha>` is a short hash of the PDF path (so two
+      # different PDFs with page 1 don't collide).
+      class PageRenderCache
+        # @param diff_dir [Pathname, String]
+        # @param strategy [Strategy]
+        def initialize(diff_dir:, strategy:)
+          @diff_dir = Pathname.new(diff_dir)
+          @strategy = strategy
+          @cache = {}
+        end
+
+        # @param pdf_path [Pathname, String]
+        # @param page [Integer] 1-based page number
+        # @param scale [Float]
+        # @return [Pathname] the rendered page PNG path
+        def render_page(pdf_path, page, scale: 2.0)
+          key = [pdf_path.to_s, page, scale]
+          return @cache[key] if @cache.key?(key)
+
+          @diff_dir.mkpath
+          png = @diff_dir.join("page-#{page}-#{short_hash(pdf_path)}.png")
+          unless png.exist?
+            @strategy.render_pdf_region(
+              pdf_path, page,
+              full_page_rect, png,
+              scale: scale,
+            )
+          end
+          @cache[key] = png
+          png
+        end
+
+        private
+
+        # Naive "full page" rect. Per-cell cropping is the
+        # strategy's responsibility when the caller passes a
+        # specific rect; the cache only handles "give me the whole
+        # page so I can derive cell rects from it".
+        def full_page_rect
+          { x: 0, y: 0, w: 612, h: 792 }
+        end
+
+        def short_hash(path)
+          require "digest"
+          Digest::SHA256.file(path).hexdigest[0, 8]
+        end
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/verifier/result.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Ucode
+  module CodeChart
+    class Verifier
+      # Typed result of verifying one {Extractor::Result}. One of
+      # three concrete classes; pattern-match on the class to read
+      # type-specific fields.
+      module Result
+        # Diff < {Verifier::Strategy::FAIL_THRESHOLD}. The extracted
+        # SVG matches the source PDF cell within tolerance.
+        Pass = Struct.new(:codepoint, :percent, keyword_init: true)
+
+        # Diff ≥ threshold. Carries the diff artifact path so a human
+        # can inspect the failure.
+        Fail = Struct.new(:codepoint, :percent, :diff_path,
+                          keyword_init: true)
+
+        # No positional data on the {Extractor::Result}; no honest
+        # cell-diff is possible. `reason` carries one of:
+        #   * `:no_location` — source_page/source_cell were nil.
+        #   * `:no_pdf` — the PDF path is missing/unreadable.
+        Skipped = Struct.new(:codepoint, :reason, keyword_init: true)
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/verifier/resvg_strategy.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "pathname"
+
+module Ucode
+  module CodeChart
+    class Verifier
+      # {Strategy} backed by the `resvg` CLI (`resvg --help`). Preferred
+      # over {MutoolStrategy} for accuracy and speed when available —
+      # resvg is purpose-built for SVG rendering and produces
+      # pixel-stable output across platforms.
+      #
+      # Same byte-wise diff fallback as {MutoolStrategy}. PNG output
+      # is normalized via resvg's `--quantize 8` to reduce byte-level
+      # noise across renderer versions.
+      class ResvgStrategy < Strategy
+        # @param runner [Ucode::Glyphs::EmbeddedFonts::Mutool::SystemRunner]
+        #   injectable for tests
+        def initialize(runner: Ucode::Glyphs::EmbeddedFonts::Mutool::SystemRunner.new)
+          super()
+          @runner = runner
+        end
+
+        def available?
+          system("which resvg >/dev/null 2>&1")
+        end
+
+        def render_svg(svg_path, png_path, scale: 2.0)
+          width = (1000 * scale).round
+          @runner.run("resvg", "-w", width.to_s,
+                      "--quantize", "8",
+                      svg_path.to_s, png_path.to_s)
+          Pathname.new(png_path)
+        end
+
+        def render_pdf_region(pdf_path, page, _rect, png_path, scale: 2.0)
+          # resvg doesn't render PDFs; we rely on mutool to first
+          # convert the page region to SVG, then resvg to PNG. Two
+          # subprocess hops, but reuses each tool's strength.
+          intermediate = Pathname.new("#{png_path}.svg")
+          dpi = (72 * scale).round
+          @runner.run("mutool", "draw", "-F", "svg", "-o", intermediate.to_s,
+                      "-r", dpi.to_s, pdf_path.to_s, page.to_s)
+          @runner.run("resvg", "--quantize", "8",
+                      intermediate.to_s, png_path.to_s)
+          FileUtils.rm_f(intermediate.to_s)
+          Pathname.new(png_path)
+        end
+
+        def diff(png_a, png_b)
+          bytes_a = File.binread(png_a)
+          bytes_b = File.binread(png_b)
+          return 100.0 if bytes_a.bytesize != bytes_b.bytesize
+
+          same = bytes_a.each_byte.with_index.count do |byte, idx|
+            byte == bytes_b.getbyte(idx)
+          end
+          (100.0 * (1.0 - same.to_f / bytes_a.bytesize)).round(2)
+        end
+
+        def write_diff_artifact(png_a, png_b, dest)
+          data_a = File.binread(png_a)
+          data_b = File.binread(png_b)
+          Pathname.new(dest).binwrite(data_a + data_b)
+          Pathname.new(dest)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/code_chart/verifier/strategy.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Ucode
+  module CodeChart
+    class Verifier
+      # Abstract base for one verification-renderer strategy. Each
+      # subclass owns ONE external CLI tool (resvg, mutool, etc.)
+      # and implements the three primitives the {Verifier} needs.
+      #
+      # Subclasses must implement:
+      #   * {#available?} — is the underlying tool installed?
+      #   * {#render_svg} — render an SVG to a PNG at the given scale.
+      #   * {#render_pdf_region} — render a rectangular region of one
+      #     PDF page to a PNG at the given scale.
+      #   * {#diff} — pixel-diff percentage between two PNGs.
+      #
+      # Adding a new renderer = one subclass + one entry in {Builder}.
+      # No edit to {Verifier}.
+      class Strategy
+        # Diff percentage at or above which a glyph is considered
+        # visually different from the source. Tuned to absorb
+        # anti-aliasing differences across renderers; callers can
+        # override via the {Verifier} constructor.
+        FAIL_THRESHOLD = 1.0
+
+        # @return [Boolean] true iff the underlying tool is on PATH
+        def available?
+          raise NotImplementedError
+        end
+
+        # @param svg_path [Pathname, String] source SVG
+        # @param png_path [Pathname, String] destination PNG
+        # @param scale [Float] zoom factor (2.0 = 200%)
+        # @return [Pathname] the written PNG path
+        def render_svg(_svg_path, _png_path, scale: 2.0)
+          raise NotImplementedError
+        end
+
+        # @param pdf_path [Pathname, String] source PDF
+        # @param page [Integer] 1-based page number
+        # @param rect [Hash{Symbol=>Float}] `{x:, y:, w:, h:}` in PDF
+        #   user space (origin bottom-left) of the region to render.
+        # @param png_path [Pathname, String] destination PNG
+        # @param scale [Float] zoom factor
+        # @return [Pathname]
+        def render_pdf_region(_pdf_path, _page, _rect, _png_path, scale: 2.0)
+          raise NotImplementedError
+        end
+
+        # @param png_a [Pathname, String]
+        # @param png_b [Pathname, String]
+        # @return [Float] percentage of differing pixels (0.0–100.0)
+        def diff(_png_a, _png_b)
+          raise NotImplementedError
+        end
+
+        # Optional: produce a visual diff artifact (e.g. side-by-side
+        # or red/blue overlay). When unsupported, returns nil and the
+        # {Verifier} skips writing the artifact.
+        #
+        # @param png_a [Pathname, String]
+        # @param png_b [Pathname, String]
+        # @param dest [Pathname, String]
+        # @return [Pathname, nil]
+        def write_diff_artifact(_png_a, _png_b, _dest)
+          nil
+        end
+      end
+    end
+  end
+end