ucode 0.1.0

data/lib/ucode/glyphs/page_renderer.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require "open3"
+require "pathname"
+require "tmpdir"
+
+require "ucode/error"
+
+module Ucode
+  module Glyphs
+    # Strategy interface for PDF-page-to-SVG rendering.
+    #
+    # Subclasses implement `renderer_name`, `binary_name`, and
+    # `build_command`. The base class handles availability check,
+    # command execution, error handling, and the renderer registry.
+    #
+    # **OCP**: a new renderer is a new subclass file + one entry in
+    # `KNOWN_RENDERERS`. The base class and existing renderers are not
+    # modified.
+    #
+    # **Vector-only requirement**: every renderer here must emit SVG
+    # `<path>` elements (vector data) for the Code Charts PDFs, not
+    # raster images. Callers verify this via `path_count` on the output.
+    class PageRenderer
+      OUTPUT_FORMAT = :svg
+
+      # Fixture used by `works?` to smoke-test renderers. Resolved lazily
+      # so missing-fixture environments (installed gem without spec assets)
+      # don't fail at load time.
+      DEFAULT_SMOKE_FIXTURE =
+        File.expand_path("../../../spec/fixtures/pdfs/basic_latin.pdf", __dir__)
+
+      # Ordered list of known concrete renderer class names (as symbols),
+      # most-preferred first. Resolved lazily via `const_get` so that
+      # loading any one renderer does not eagerly load all of them — this
+      # avoids a circular require (each renderer file requires this file
+      # to inherit from PageRenderer).
+      KNOWN_RENDERERS = %i[
+        MutoolRenderer
+        Pdf2svgRenderer
+        DvisvgmRenderer
+        PdftocairoRenderer
+      ].freeze
+      private_constant :KNOWN_RENDERERS
+
+      class << self
+        # @return [Symbol] short identifier (e.g. :mutool)
+        def renderer_name
+          raise NotImplementedError
+        end
+
+        # @return [String, Symbol] the binary looked up on PATH
+        def binary_name
+          raise NotImplementedError
+        end
+
+        # @return [Symbol] always :svg for now; future formats (png, etc.)
+        #   would warrant a separate renderer family.
+        def output_format
+          OUTPUT_FORMAT
+        end
+
+        # Build the argv for the renderer. Subclasses return an Array
+        # suitable for `Open3.capture2e` (no shell interpolation).
+        # @param pdf_path [Pathname, String]
+        # @param page_num [Integer] 1-indexed
+        # @param out_path [Pathname, String]
+        # @return [Array<String>]
+        def build_command(pdf_path, page_num, out_path)
+          raise NotImplementedError
+        end
+
+        # @return [Boolean] true if the binary is on PATH
+        def available?
+          system("which", binary_name.to_s, out: "/dev/null", err: "/dev/null")
+        end
+
+        # Smoke-test the binary by actually rendering one page of the
+        # fixture PDF AND verifying the output format is consumable by
+        # the downstream `GridDetector` / `CellExtractor` pipeline.
+        #
+        # Three things can make a renderer unusable for this codebase:
+        #   1. Binary not on PATH (`available?` catches this).
+        #   2. Binary on PATH but silently broken (e.g. Ubuntu's
+        #      `mupdf-tools` is built without LCMS, so `mutool` warns
+        #      "ICC support is not available" and emits zero bytes for
+        #      ICC-profiled PDFs).
+        #   3. Binary works but emits a flat-path SVG that GridDetector
+        #      can't parse (mutool's format: `<path id="font_X_Y">`
+        #      directly in `<defs>`, no `<use>` references). The grid
+        #      detector requires the `<g id="glyph-N-M">` + `<use>` form
+        #      produced by pdftocairo / pdf2svg.
+        #
+        # The result is memoized per-renderer for the process lifetime —
+        # the binary's capabilities don't change mid-run.
+        #
+        # When no fixture PDF is available (e.g. installed gem without
+        # spec assets), degrades to `available?` — we can't smoke-test
+        # without input, so we trust the binary's presence on PATH.
+        #
+        # @param fixture_pdf [String, Pathname] small one-page PDF used
+        #   for the smoke render. Defaults to the project's
+        #   `basic_latin.pdf` spec fixture.
+        # @return [Boolean]
+        def works?(fixture_pdf: DEFAULT_SMOKE_FIXTURE)
+          if !available?
+            false
+          elsif !File.exist?(fixture_pdf.to_s)
+            true # no fixture to verify against; trust PATH
+          else
+            smoke_render_ok?(fixture_pdf)
+          end
+        end
+
+        # Render one page of `pdf_path` to `out_path` as SVG.
+        # @param pdf_path [Pathname, String]
+        # @param page_num [Integer] 1-indexed
+        # @param out_path [Pathname, String]
+        # @return [Symbol] :ok on success
+        # @raise [Ucode::PdfRenderError] on failure (non-zero exit,
+        #   output file missing, or binary unavailable)
+        def render(pdf_path, page_num, out_path)
+          unless available?
+            raise PdfRenderError.new(
+              "binary '#{binary_name}' not available on PATH",
+              context: { renderer: name, binary: binary_name },
+            )
+          end
+
+          out = Pathname.new(out_path)
+          out.dirname.mkpath
+
+          cmd = build_command(Pathname.new(pdf_path), page_num, out)
+          output, status = Open3.capture2e(*cmd)
+
+          unless status.success? && out.exist? && out.size.positive?
+            raise PdfRenderError.new(
+              "render failed for page #{page_num} of #{pdf_path} via '#{binary_name}'",
+              context: {
+                renderer: name,
+                binary: binary_name,
+                exit_status: status.exitstatus,
+                output: output,
+              },
+            )
+          end
+
+          :ok
+        end
+
+        # ---- Registry ----
+
+        # @return [Array<Class>] every known concrete renderer
+        def all
+          @all ||= KNOWN_RENDERERS.map { |n| Ucode::Glyphs.const_get(n) }.freeze
+        end
+
+        # @return [Array<Class>] renderers whose binary is installed
+        def available
+          all.select(&:available?)
+        end
+
+        # @return [Array<Class>] renderers that actually produce SVG in
+        #   the format `GridDetector` consumes (smoke-tested once per
+        #   process via `works?`, then cached). Subset of `available`.
+        def working
+          return @working if @working
+
+          @working = all.select(&:works?).freeze
+        end
+
+        # Clear the cached `working` list. Useful when the environment
+        # changes (e.g. a binary is installed mid-process) or in tests.
+        def reset_working_cache!
+          @working = nil
+        end
+
+        # @param name [Symbol, String]
+        # @return [Class, nil]
+        def find(name)
+          all.find { |r| r.renderer_name == name.to_sym }
+        end
+
+        # @return [Class, nil] the first working renderer; falls back to
+        #   the first available renderer if none have been smoke-tested
+        #   yet (preserves eager-init paths). nil if nothing is installed.
+        def default
+          working.first || available.first
+        end
+
+        private
+
+        # @param fixture_pdf [String] path to an existing PDF
+        # @return [Boolean] true iff rendering page 1 produces an SVG
+        #   with the `<g id="glyph-N-M">` + `<use>` form that
+        #   `GridDetector` requires.
+        def smoke_render_ok?(fixture_pdf)
+          Dir.mktmpdir("renderer-smoke-") do |dir|
+            out = File.join(dir, "smoke.svg")
+            begin
+              render(fixture_pdf, 1, out)
+            rescue PdfRenderError
+              break false
+            end
+            svg_has_pipeline_format?(out)
+          end
+        end
+
+        def svg_has_pipeline_format?(out_path)
+          return false unless File.exist?(out_path)
+          return false unless File.size(out_path).positive?
+
+          body = File.read(out_path)
+          body.include?("<svg") &&
+            body.include?("<use") &&
+            body.match?("id=\"glyph-\\d+-\\d+\"")
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/path_bbox.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Glyphs
+    # Estimates the axis-aligned bounding box of an SVG `<path>` `d`
+    # attribute by scanning every numeric coordinate pair in the path
+    # data. This is a conservative over-estimate: control points and
+    # implicit vertices are included, so the true curve bbox is always
+    # contained within the estimate. For grid detection and cell
+    # membership tests, the over-estimate is sufficient and avoids the
+    # cost of a Bezier solver.
+    #
+    # Only absolute coordinates are returned. Relative commands (lowercase
+    # `m`, `l`, `c`, …) are NOT supported — Code Charts SVGs from every
+    # supported renderer (pdftocairo, pdf2svg, dvisvgm, mutool) emit
+    # absolute commands. If relative commands appear, parse them via a
+    # proper SVG path parser before calling this.
+    module PathBbox
+      NUMBER = /-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?/.freeze
+
+      Result = Struct.new(:min_x, :min_y, :max_x, :max_y, keyword_init: true) do
+        def width
+          return nil if empty?
+
+          max_x - min_x
+        end
+
+        def height
+          return nil if empty?
+
+          max_y - min_y
+        end
+
+        def empty?
+          min_x.nil? || min_y.nil? || max_x.nil? || max_y.nil?
+        end
+      end
+
+      class << self
+        def estimate(path_d)
+          return Result.new if path_d.nil? || path_d.empty?
+
+          numbers = path_d.scan(NUMBER).map(&:to_f)
+          return Result.new if numbers.empty?
+
+          xs = []
+          ys = []
+          numbers.each_slice(2) do |x, y|
+            xs << x
+            ys << y
+          end
+          Result.new(
+            min_x: xs.min,
+            min_y: ys.min,
+            max_x: xs.max,
+            max_y: ys.max,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/pdf2svg_renderer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "ucode/glyphs/page_renderer"
+
+module Ucode
+  module Glyphs
+    # `pdf2svg` — simple, widely available. One SVG per page.
+    #
+    # Command: `pdf2svg <in.pdf> <out.svg> <page>`
+    class Pdf2svgRenderer < PageRenderer
+      class << self
+        def renderer_name
+          :pdf2svg
+        end
+
+        def binary_name
+          :pdf2svg
+        end
+
+        def build_command(pdf_path, page_num, out_path)
+          ["pdf2svg", pdf_path.to_s, out_path.to_s, page_num.to_s]
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/pdf_fetcher.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "open3"
+
+require "ucode/cache"
+require "ucode/fetch/code_charts"
+require "ucode/glyphs/monolith_page_map"
+
+module Ucode
+  module Glyphs
+    # Resolves a Unicode block to its source PDF on disk.
+    #
+    # Primary source: the per-block PDF cached at
+    # `<cache>/<version>/pdfs/U<XXXX>.pdf` (downloaded from
+    # `unicode.org/charts/PDF/` by `Ucode::Fetch::CodeCharts`).
+    #
+    # Fallback: slice the page range from the monolith `CodeCharts.pdf`.
+    # The page range is resolved by `MonolithPageMap` from the PDF's
+    # bookmark outline, cached under `data/codecharts_page_map.json`.
+    class PdfFetcher
+      # @param version [String] UCD version, used as the cache namespace.
+      # @param monolith_path [String, Pathname, nil] path to the full
+      #   `CodeCharts.pdf`. Pass nil to disable monolith fallback.
+      # @param blocks [Array<Ucode::Models::Block>] required for monolith
+      #   fallback — used to match bookmark titles to block first-cps.
+      # @param page_map_cache [String, Pathname, nil] where to read/write
+      #   the monolith page-map JSON cache.
+      def initialize(version, monolith_path: nil, blocks: [], page_map_cache: nil)
+        @version = version
+        @monolith_path = monolith_path && Pathname.new(monolith_path)
+        @blocks = blocks
+        @page_map_cache = page_map_cache
+      end
+
+      # Resolve the per-block PDF for `block_first_cp`, fetching from the
+      # network if missing. Returns the local PDF path, or nil if the
+      # block's PDF is unavailable (network failure + no monolith, or
+      # monolith lacks the requested block).
+      #
+      # @param block_first_cp [Integer] first codepoint of the block;
+      #   also the PDF's URL slug per unicode.org's naming convention.
+      # @param force [Boolean] re-download even if cached.
+      # @return [Pathname, nil]
+      def fetch(block_first_cp:, force: false)
+        path = per_block_path(block_first_cp)
+        return path if path.exist? && !force
+
+        download(block_first_cp)
+        return path if path.exist?
+
+        slice_from_monolith(block_first_cp)
+      end
+
+      private
+
+      def per_block_path(block_first_cp)
+        Cache.pdfs_dir(@version).join("U#{hex_slug(block_first_cp)}.pdf")
+      end
+
+      def hex_slug(cp)
+        cp.to_s(16).upcase.rjust(4, "0")
+      end
+
+      def download(block_first_cp)
+        Fetch::CodeCharts.call(@version, block_first_cps: [block_first_cp])
+      rescue StandardError => e
+        # Network failures fall through to monolith fallback. We do not
+        # swallow programming errors (NoMethodError etc.) — only fetch
+        # failures (network, checksum, HTTP).
+        return if e.is_a?(Ucode::FetchError)
+
+        raise
+      end
+
+      def slice_from_monolith(block_first_cp)
+        return unless @monolith_path&.exist?
+
+        entry = page_map[block_first_cp]
+        return unless entry && entry.start_page && entry.end_page
+
+        slice_pages(entry.start_page, entry.end_page, per_block_path(block_first_cp))
+      end
+
+      def page_map
+        @page_map ||= MonolithPageMap.load(
+          monolith_path: @monolith_path,
+          blocks: @blocks,
+          cache_path: @page_map_cache,
+        )
+      end
+
+      def slice_pages(start_page, end_page, out_path)
+        out_path.dirname.mkpath
+        cmd = ["pdftk", @monolith_path.to_s, "cat",
+               "#{start_page}-#{end_page}", "output", out_path.to_s]
+        _out, status = Open3.capture2e(*cmd)
+        status.success? ? out_path : nil
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/pdftocairo_renderer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "ucode/glyphs/page_renderer"
+
+module Ucode
+  module Glyphs
+    # `pdftocairo -svg` from the Poppler suite. Available on macOS via
+    # `brew install poppler`. Slower than `mutool` but widely available.
+    #
+    # Command: `pdftocairo -svg -f <n> -l <n> <in.pdf> <out.svg>`
+    #
+    # The `-f`/`-l` pair restricts rendering to one page (first/last).
+    class PdftocairoRenderer < PageRenderer
+      class << self
+        def renderer_name
+          :pdftocairo
+        end
+
+        def binary_name
+          :pdftocairo
+        end
+
+        def build_command(pdf_path, page_num, out_path)
+          ["pdftocairo", "-svg",
+           "-f", page_num.to_s,
+           "-l", page_num.to_s,
+           pdf_path.to_s, out_path.to_s]
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/real_fonts/block_coverage.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Ucode
+  module Glyphs
+    module RealFonts
+      # Per-block coverage row on a {FontCoverageReport}.
+      #
+      # `assigned` is the UCD-assigned codepoint count for this block
+      # (from {Unicode17Blocks}); `covered` is the count actually
+      # present in the font's cmap; `missing_cps` is the human-readable
+      # hex form (`U+XXXX`) of every assigned codepoint the font lacks,
+      # so a downstream consumer can audit gaps without re-walking the
+      # cmap.
+      class BlockCoverage < Lutaml::Model::Serializable
+        attribute :name, :string
+        attribute :first_cp, :integer
+        attribute :last_cp, :integer
+        attribute :assigned, :integer
+        attribute :covered, :integer
+        attribute :missing_cps, :string, collection: true, default: -> { [] }
+
+        key_value do
+          map "name",        to: :name
+          map "first_cp",    to: :first_cp
+          map "last_cp",     to: :last_cp
+          map "assigned",    to: :assigned
+          map "covered",     to: :covered
+          map "missing_cps", to: :missing_cps
+        end
+
+        def fill_ratio
+          return 0.0 if assigned.nil? || assigned.zero?
+
+          (covered.to_f / assigned).round(4)
+        end
+
+        def complete?
+          assigned.to_i.positive? && covered == assigned
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/real_fonts/coverage_auditor.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "time"
+
+require "fontisan"
+
+require_relative "block_coverage"
+require_relative "font_coverage_report"
+require_relative "unicode_17_blocks"
+
+module Ucode
+  module Glyphs
+    module RealFonts
+      # Builds a {FontCoverageReport} for a font on disk.
+      #
+      # Strategy:
+      #
+      #   1. Walk the font's cmap via fontisan to get the set of
+      #      codepoints the font actually has outlines for.
+      #   2. For each Unicode 17 block in {Unicode17Blocks}, intersect
+      #      the block's assigned-codepoint ranges against the cmap
+      #      set. The denominator (`assigned`) comes from our curated
+      #      ranges table — not from fontisan's UCD database, because
+      #      the UCD database is a separate download and its block
+      #      coverage for Unicode 17 is incomplete (it omits several
+      #      new blocks). The numerator (`covered`) and the
+      #      `missing_cps` list both come from the cmap walk.
+      #   3. Also call fontisan's {Fontisan::Commands::AuditCommand} in
+      #      brief mode for identity + total counts (no UCD dependency
+      #      in brief mode).
+      class CoverageAuditor
+        UCD_VERSION = "17.0.0"
+
+        # @param font_path [Pathname, String]
+        # @return [FontCoverageReport]
+        def audit(font_path)
+          font_path = Pathname(font_path)
+          fontisan_report = run_fontisan_audit(font_path)
+          cmap_codepoints = read_cmap_codepoints(font_path)
+          blocks = Unicode17Blocks::ALL.map do |block|
+            build_block_coverage(block, cmap_codepoints)
+          end
+
+          FontCoverageReport.new(**report_kwargs(font_path, fontisan_report,
+                                                 blocks))
+        end
+
+        private
+
+        # Brief mode is enough — we don't need fontisan's Aggregations
+        # extractor (we compute our own coverage from the curated
+        # Unicode17Blocks table) and brief mode skips the UCD database
+        # dependency that full mode requires.
+        def run_fontisan_audit(font_path)
+          Fontisan::Commands::AuditCommand.new(
+            font_path.to_s,
+            ucd_version: UCD_VERSION,
+            audit_brief: true,
+          ).run
+        end
+
+        def read_cmap_codepoints(font_path)
+          font = Fontisan::FontLoader.load(font_path.to_s)
+          cmap = font.table(Fontisan::Constants::CMAP_TAG)
+          return Set.new unless cmap
+
+          cmap.unicode_mappings.keys.to_set
+        end
+
+        def build_block_coverage(block, cmap_codepoints)
+          assigned_cps = block.assigned_ranges.flat_map(&:to_a)
+          covered = assigned_cps.select { |cp| cmap_codepoints.include?(cp) }
+
+          BlockCoverage.new(
+            name: block.name,
+            first_cp: block.first_cp,
+            last_cp: block.last_cp,
+            assigned: assigned_cps.length,
+            covered: covered.length,
+            missing_cps: missing_cps_for(assigned_cps, covered),
+          )
+        end
+
+        def missing_cps_for(assigned_cps, covered)
+          (assigned_cps - covered).map { |cp| format("U+%04X", cp) }
+        end
+
+        def identity_kwargs(font_path, fontisan_report)
+          {
+            source_file: font_path.basename.to_s,
+            source_format: fontisan_report.source_format,
+            family_name: fontisan_report.family_name,
+            full_name: fontisan_report.full_name,
+            postscript_name: fontisan_report.postscript_name,
+            version: fontisan_report.version,
+          }
+        end
+
+        def count_kwargs(fontisan_report, blocks)
+          {
+            total_codepoints: fontisan_report.total_codepoints,
+            total_glyphs: fontisan_report.total_glyphs,
+            ucd_version: UCD_VERSION,
+            blocks: blocks,
+          }
+        end
+
+        def report_kwargs(font_path, fontisan_report, blocks)
+          identity_kwargs(font_path, fontisan_report)
+            .merge(count_kwargs(fontisan_report, blocks))
+            .merge(generated_at: Time.now.utc.iso8601)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/real_fonts/font_coverage_report.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+require_relative "block_coverage"
+
+module Ucode
+  module Glyphs
+    module RealFonts
+      # Coverage report for a single font face. Produced by
+      # {CoverageAuditor} from a {Fontisan::Models::Audit::AuditReport}
+      # + the font's own cmap walk. Carries identity (so a consumer
+      # reading the file knows which font produced it), coverage
+      # totals, and the per-block detail restricted to Unicode 17 new
+      # blocks (older blocks are noise for this audit).
+      class FontCoverageReport < Lutaml::Model::Serializable
+        attribute :generated_at, :string
+        attribute :source_file, :string
+        attribute :source_format, :string
+        attribute :family_name, :string
+        attribute :full_name, :string
+        attribute :postscript_name, :string
+        attribute :version, :string
+        attribute :total_codepoints, :integer
+        attribute :total_glyphs, :integer
+        attribute :ucd_version, :string
+        attribute :blocks, BlockCoverage, collection: true, default: -> { [] }
+
+        key_value do
+          map "generated_at",      to: :generated_at
+          map "source_file",       to: :source_file
+          map "source_format",     to: :source_format
+          map "family_name",       to: :family_name
+          map "full_name",         to: :full_name
+          map "postscript_name",   to: :postscript_name
+          map "version",           to: :version
+          map "total_codepoints",  to: :total_codepoints
+          map "total_glyphs",      to: :total_glyphs
+          map "ucd_version",       to: :ucd_version
+          map "blocks",            to: :blocks
+        end
+      end
+    end
+  end
+end