ucode 0.2.2 → 0.2.3

data/lib/ucode/glyphs/monolith_page_map.rb

@@ -1,181 +0,0 @@
-# frozen_string_literal: true
-
-require "pathname"
-require "json"
-require "open3"
-
-module Ucode
-  module Glyphs
-    # Maps a Unicode block's first codepoint to its page range inside the
-    # monolith `CodeCharts.pdf` by parsing the PDF's bookmark outline and
-    # matching each bookmark title to a Block.name from `Blocks.txt`.
-    #
-    # Each chart cluster printed by the Unicode Consortium is a single
-    # bookmark entry:
-    #
-    #   BookmarkTitle: Greek and Coptic
-    #   BookmarkLevel: 1
-    #   BookmarkPageNumber: 415
-    #
-    # The cluster title usually equals a Block.name verbatim, but a few
-    # clusters carry a heading that prepends "C0 Controls and " /
-    # "C1 Controls and " to the block name. We resolve both forms.
-    #
-    # End-page of a cluster is one page before the next cluster's start
-    # page (last cluster's end-page is the PDF's last page).
-    #
-    # The map is cached as JSON at `data/codecharts_page_map.json` so
-    # we don't re-scan the 3,156-page monolith on every run.
-    class MonolithPageMap
-      BookmarkTitleRegex = /BookmarkTitle:\s*(.+)/.freeze
-      BookmarkPageRegex  = /BookmarkPageNumber:\s*(\d+)/.freeze
-      private_constant :BookmarkTitleRegex, :BookmarkPageRegex
-
-      # The Unicode charts print these multi-block clusters as a single
-      # chart page (the C0/C1 control chars are drawn alongside their
-      # block's other characters). Each cluster title maps to the single
-      # block it belongs to.
-      ClusterPrefixes = [
-        "C0 Controls and ",
-        "C1 Controls and ",
-      ].freeze
-      private_constant :ClusterPrefixes
-
-      MapEntry = Struct.new(:first_cp, :start_page, :end_page, keyword_init: true)
-
-      class << self
-        # Build the map by parsing the monolith's outline and matching
-        # each bookmark title to a Block.
-        #
-        # @param monolith_path [String, Pathname]
-        # @param blocks [Array<Ucode::Models::Block>] the parsed Blocks table
-        # @return [Hash{Integer => MapEntry}] keyed by block.range_first
-        def build(monolith_path:, blocks:)
-          name_to_first_cp = blocks.each_with_object({}) do |b, h|
-            h[b.name] = b.range_first
-          end
-          total_pages = page_count(monolith_path)
-          entries = parse_bookmarks(dump_bookmarks(monolith_path), name_to_first_cp)
-          attach_end_pages(entries, total_pages)
-          entries.each_with_object({}) do |e, h|
-            h[e.first_cp] = e
-          end
-        end
-
-        # Pure: parse a `pdftk dump_data` string into a list of
-        # MapEntry rows (without end_pages). Exposed for unit tests
-        # and any caller that already has the dump cached.
-        #
-        # @param dump [String] the raw `pdftk dump_data` output
-        # @param name_to_first_cp [Hash{String => Integer}]
-        # @return [Array<MapEntry>]
-        def parse_bookmarks(dump, name_to_first_cp)
-          entries = []
-          current_title = nil
-          dump.each_line do |line|
-            case line
-            when BookmarkTitleRegex
-              current_title = Regexp.last_match(1).strip
-            when BookmarkPageRegex
-              page = Regexp.last_match(1).to_i
-              cp = resolve_first_cp(current_title, name_to_first_cp)
-              entries << MapEntry.new(first_cp: cp, start_page: page) if cp
-              current_title = nil
-            end
-          end
-          entries.sort_by(&:start_page)
-        end
-
-        # Pure: attach end_pages by sorting entries and assigning each
-        # entry's end to one page before the next entry's start.
-        #
-        # @param entries [Array<MapEntry>]
-        # @param total_pages [Integer, nil] page count of the source PDF;
-        #   the last entry's end_page falls back to this when present.
-        # @return [Array<MapEntry>] the same entries, mutated with end_pages.
-        def attach_end_pages(entries, total_pages = nil)
-          sorted = entries.sort_by(&:start_page)
-          sorted.each_with_index do |entry, i|
-            next_entry = sorted[i + 1]
-            entry.end_page = next_entry ? next_entry.start_page - 1 : total_pages
-          end
-          sorted
-        end
-
-        # Load from cache, or build and cache.
-        # @param monolith_path [String, Pathname]
-        # @param blocks [Array<Ucode::Models::Block>]
-        # @param cache_path [String, Pathname, nil]
-        # @return [Hash{Integer => MapEntry}]
-        def load(monolith_path:, blocks:, cache_path: nil)
-          cache = cache_path && Pathname.new(cache_path)
-          if cache&.exist?
-            return load_from_json(cache.read)
-          end
-
-          map = build(monolith_path: monolith_path, blocks: blocks)
-          write_cache(map, cache) if cache
-          map
-        end
-
-        # Look up a block's page range by its first cp.
-        # @param map [Hash{Integer => MapEntry}]
-        # @param block_first_cp [Integer]
-        # @return [MapEntry, nil]
-        def range_for(map, block_first_cp)
-          map[block_first_cp]
-        end
-
-        # ---- I/O helpers (impure) --------------------------------------
-
-        def dump_bookmarks(monolith_path)
-          out, status = Open3.capture2e("pdftk", monolith_path.to_s, "dump_data")
-          return "" unless status.success?
-
-          out
-        end
-
-        def page_count(monolith_path)
-          out, status = Open3.capture2e("pdfinfo", monolith_path.to_s)
-          return nil unless status.success?
-
-          match = out.match(/^Pages:\s+(\d+)/)
-          match ? match[1].to_i : nil
-        end
-
-        private
-
-        def resolve_first_cp(title, name_to_first_cp)
-          return nil unless title
-
-          return name_to_first_cp[title] if name_to_first_cp.key?(title)
-
-          ClusterPrefixes.each do |prefix|
-            stripped = title.sub(/\A#{Regexp.escape(prefix)}/, "")
-            return name_to_first_cp[stripped] if name_to_first_cp.key?(stripped)
-          end
-
-          nil
-        end
-
-        def write_cache(map, cache_path)
-          payload = map.values.map { |e| { "first_cp" => e.first_cp,
-                                            "start_page" => e.start_page,
-                                            "end_page" => e.end_page } }
-          cache_path.dirname.mkpath
-          cache_path.write(JSON.pretty_generate(payload))
-        end
-
-        def load_from_json(json)
-          payload = JSON.parse(json)
-          payload.each_with_object({}) do |row, h|
-            entry = MapEntry.new(first_cp: row["first_cp"],
-                                 start_page: row["start_page"],
-                                 end_page: row["end_page"])
-            h[entry.first_cp] = entry
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/mutool_renderer.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-require "ucode/glyphs/page_renderer"
-
-module Ucode
-  module Glyphs
-    # `mutool draw` from MuPDF — typically the fastest and cleanest.
-    # Emits one `<svg>` per page with `<path>` vector data.
-    #
-    # Command: `mutool draw -F svg -o <out.svg> <in.pdf> <page>`
-    class MutoolRenderer < PageRenderer
-      class << self
-        def renderer_name
-          :mutool
-        end
-
-        def binary_name
-          :mutool
-        end
-
-        def build_command(pdf_path, page_num, out_path)
-          ["mutool", "draw", "-F", "svg", "-o", out_path.to_s,
-           pdf_path.to_s, page_num.to_s]
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/page_renderer.rb

@@ -1,234 +0,0 @@
-# frozen_string_literal: true
-
-require "open3"
-require "pathname"
-require "tmpdir"
-
-require "ucode/error"
-
-module Ucode
-  module Glyphs
-    # Strategy interface for PDF-page-to-SVG rendering.
-    #
-    # Subclasses implement `renderer_name`, `binary_name`, and
-    # `build_command`. The base class handles availability check,
-    # command execution, error handling, and the renderer registry.
-    #
-    # **OCP**: a new renderer is a new subclass file + one entry in
-    # `KNOWN_RENDERERS`. The base class and existing renderers are not
-    # modified.
-    #
-    # **Vector-only requirement**: every renderer here must emit SVG
-    # `<path>` elements (vector data) for the Code Charts PDFs, not
-    # raster images. Callers verify this via `path_count` on the output.
-    class PageRenderer
-      OUTPUT_FORMAT = :svg
-
-      # Fixture used by `works?` to smoke-test renderers. Resolved lazily
-      # so missing-fixture environments (installed gem without spec assets)
-      # don't fail at load time.
-      DEFAULT_SMOKE_FIXTURE =
-        File.expand_path("../../../spec/fixtures/pdfs/basic_latin.pdf", __dir__)
-
-      # Ordered list of known concrete renderer class names (as symbols),
-      # most-preferred first. Resolved lazily via `const_get` so that
-      # loading any one renderer does not eagerly load all of them — this
-      # avoids a circular require (each renderer file requires this file
-      # to inherit from PageRenderer).
-      KNOWN_RENDERERS = %i[
-        MutoolRenderer
-        Pdf2svgRenderer
-        DvisvgmRenderer
-        PdftocairoRenderer
-      ].freeze
-      private_constant :KNOWN_RENDERERS
-
-      class << self
-        # @return [Symbol] short identifier (e.g. :mutool)
-        def renderer_name
-          raise NotImplementedError
-        end
-
-        # @return [String, Symbol] the binary looked up on PATH
-        def binary_name
-          raise NotImplementedError
-        end
-
-        # @return [Symbol] always :svg for now; future formats (png, etc.)
-        #   would warrant a separate renderer family.
-        def output_format
-          OUTPUT_FORMAT
-        end
-
-        # Build the argv for the renderer. Subclasses return an Array
-        # suitable for `Open3.capture2e` (no shell interpolation).
-        # @param pdf_path [Pathname, String]
-        # @param page_num [Integer] 1-indexed
-        # @param out_path [Pathname, String]
-        # @return [Array<String>]
-        def build_command(pdf_path, page_num, out_path)
-          raise NotImplementedError
-        end
-
-        # @return [Boolean] true if the binary is on PATH. Returns
-        #   false on hosts without `which`/`where` or where the
-        #   binary isn't installed — the next renderer in
-        #   KNOWN_RENDERERS is tried.
-        def available?
-          if Gem.win_platform?
-            # `where` returns the first match path; exit status 0
-            # means the binary is found. Suppress stdout/stderr to
-            # avoid polluting test output.
-            system("where #{binary_name} >NUL 2>NUL")
-          else
-            system("which", binary_name.to_s,
-                   out: "/dev/null", err: "/dev/null")
-          end
-        rescue Errno::ENOENT, Errno::EINVAL
-          false
-        end
-
-        # Smoke-test the binary by actually rendering one page of the
-        # fixture PDF AND verifying the output format is consumable by
-        # the downstream `GridDetector` / `CellExtractor` pipeline.
-        #
-        # Three things can make a renderer unusable for this codebase:
-        #   1. Binary not on PATH (`available?` catches this).
-        #   2. Binary on PATH but silently broken (e.g. Ubuntu's
-        #      `mupdf-tools` is built without LCMS, so `mutool` warns
-        #      "ICC support is not available" and emits zero bytes for
-        #      ICC-profiled PDFs).
-        #   3. Binary works but emits a flat-path SVG that GridDetector
-        #      can't parse (mutool's format: `<path id="font_X_Y">`
-        #      directly in `<defs>`, no `<use>` references). The grid
-        #      detector requires the `<g id="glyph-N-M">` + `<use>` form
-        #      produced by pdftocairo / pdf2svg.
-        #
-        # The result is memoized per-renderer for the process lifetime —
-        # the binary's capabilities don't change mid-run.
-        #
-        # When no fixture PDF is available (e.g. installed gem without
-        # spec assets), degrades to `available?` — we can't smoke-test
-        # without input, so we trust the binary's presence on PATH.
-        #
-        # @param fixture_pdf [String, Pathname] small one-page PDF used
-        #   for the smoke render. Defaults to the project's
-        #   `basic_latin.pdf` spec fixture.
-        # @return [Boolean]
-        def works?(fixture_pdf: DEFAULT_SMOKE_FIXTURE)
-          if !available?
-            false
-          elsif !File.exist?(fixture_pdf.to_s)
-            true # no fixture to verify against; trust PATH
-          else
-            smoke_render_ok?(fixture_pdf)
-          end
-        end
-
-        # Render one page of `pdf_path` to `out_path` as SVG.
-        # @param pdf_path [Pathname, String]
-        # @param page_num [Integer] 1-indexed
-        # @param out_path [Pathname, String]
-        # @return [Symbol] :ok on success
-        # @raise [Ucode::PdfRenderError] on failure (non-zero exit,
-        #   output file missing, or binary unavailable)
-        def render(pdf_path, page_num, out_path)
-          unless available?
-            raise PdfRenderError.new(
-              "binary '#{binary_name}' not available on PATH",
-              context: { renderer: name, binary: binary_name },
-            )
-          end
-
-          out = Pathname.new(out_path)
-          out.dirname.mkpath
-
-          cmd = build_command(Pathname.new(pdf_path), page_num, out)
-          output, status = Open3.capture2e(*cmd)
-
-          unless status.success? && out.exist? && out.size.positive?
-            raise PdfRenderError.new(
-              "render failed for page #{page_num} of #{pdf_path} via '#{binary_name}'",
-              context: {
-                renderer: name,
-                binary: binary_name,
-                exit_status: status.exitstatus,
-                output: output,
-              },
-            )
-          end
-
-          :ok
-        end
-
-        # ---- Registry ----
-
-        # @return [Array<Class>] every known concrete renderer
-        def all
-          @all ||= KNOWN_RENDERERS.map { |n| Ucode::Glyphs.const_get(n) }.freeze
-        end
-
-        # @return [Array<Class>] renderers whose binary is installed
-        def available
-          all.select(&:available?)
-        end
-
-        # @return [Array<Class>] renderers that actually produce SVG in
-        #   the format `GridDetector` consumes (smoke-tested once per
-        #   process via `works?`, then cached). Subset of `available`.
-        def working
-          return @working if @working
-
-          @working = all.select(&:works?).freeze
-        end
-
-        # Clear the cached `working` list. Useful when the environment
-        # changes (e.g. a binary is installed mid-process) or in tests.
-        def reset_working_cache!
-          @working = nil
-        end
-
-        # @param name [Symbol, String]
-        # @return [Class, nil]
-        def find(name)
-          all.find { |r| r.renderer_name == name.to_sym }
-        end
-
-        # @return [Class, nil] the first working renderer; falls back to
-        #   the first available renderer if none have been smoke-tested
-        #   yet (preserves eager-init paths). nil if nothing is installed.
-        def default
-          working.first || available.first
-        end
-
-        private
-
-        # @param fixture_pdf [String] path to an existing PDF
-        # @return [Boolean] true iff rendering page 1 produces an SVG
-        #   with the `<g id="glyph-N-M">` + `<use>` form that
-        #   `GridDetector` requires.
-        def smoke_render_ok?(fixture_pdf)
-          Dir.mktmpdir("renderer-smoke-") do |dir|
-            out = File.join(dir, "smoke.svg")
-            begin
-              render(fixture_pdf, 1, out)
-            rescue PdfRenderError
-              break false
-            end
-            svg_has_pipeline_format?(out)
-          end
-        end
-
-        def svg_has_pipeline_format?(out_path)
-          return false unless File.exist?(out_path)
-          return false unless File.size(out_path).positive?
-
-          body = File.read(out_path)
-          body.include?("<svg") &&
-            body.include?("<use") &&
-            body.match?("id=\"glyph-\\d+-\\d+\"")
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/path_bbox.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-module Ucode
-  module Glyphs
-    # Estimates the axis-aligned bounding box of an SVG `<path>` `d`
-    # attribute by scanning every numeric coordinate pair in the path
-    # data. This is a conservative over-estimate: control points and
-    # implicit vertices are included, so the true curve bbox is always
-    # contained within the estimate. For grid detection and cell
-    # membership tests, the over-estimate is sufficient and avoids the
-    # cost of a Bezier solver.
-    #
-    # Only absolute coordinates are returned. Relative commands (lowercase
-    # `m`, `l`, `c`, …) are NOT supported — Code Charts SVGs from every
-    # supported renderer (pdftocairo, pdf2svg, dvisvgm, mutool) emit
-    # absolute commands. If relative commands appear, parse them via a
-    # proper SVG path parser before calling this.
-    module PathBbox
-      NUMBER = /-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?/.freeze
-
-      Result = Struct.new(:min_x, :min_y, :max_x, :max_y, keyword_init: true) do
-        def width
-          return nil if empty?
-
-          max_x - min_x
-        end
-
-        def height
-          return nil if empty?
-
-          max_y - min_y
-        end
-
-        def empty?
-          min_x.nil? || min_y.nil? || max_x.nil? || max_y.nil?
-        end
-      end
-
-      class << self
-        def estimate(path_d)
-          return Result.new if path_d.nil? || path_d.empty?
-
-          numbers = path_d.scan(NUMBER).map(&:to_f)
-          return Result.new if numbers.empty?
-
-          xs = []
-          ys = []
-          numbers.each_slice(2) do |x, y|
-            xs << x
-            ys << y
-          end
-          Result.new(
-            min_x: xs.min,
-            min_y: ys.min,
-            max_x: xs.max,
-            max_y: ys.max,
-          )
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/pdf2svg_renderer.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-require "ucode/glyphs/page_renderer"
-
-module Ucode
-  module Glyphs
-    # `pdf2svg` — simple, widely available. One SVG per page.
-    #
-    # Command: `pdf2svg <in.pdf> <out.svg> <page>`
-    class Pdf2svgRenderer < PageRenderer
-      class << self
-        def renderer_name
-          :pdf2svg
-        end
-
-        def binary_name
-          :pdf2svg
-        end
-
-        def build_command(pdf_path, page_num, out_path)
-          ["pdf2svg", pdf_path.to_s, out_path.to_s, page_num.to_s]
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/pdftocairo_renderer.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-require "ucode/glyphs/page_renderer"
-
-module Ucode
-  module Glyphs
-    # `pdftocairo -svg` from the Poppler suite. Available on macOS via
-    # `brew install poppler`. Slower than `mutool` but widely available.
-    #
-    # Command: `pdftocairo -svg -f <n> -l <n> <in.pdf> <out.svg>`
-    #
-    # The `-f`/`-l` pair restricts rendering to one page (first/last).
-    class PdftocairoRenderer < PageRenderer
-      class << self
-        def renderer_name
-          :pdftocairo
-        end
-
-        def binary_name
-          :pdftocairo
-        end
-
-        def build_command(pdf_path, page_num, out_path)
-          ["pdftocairo", "-svg",
-           "-f", page_num.to_s,
-           "-l", page_num.to_s,
-           pdf_path.to_s, out_path.to_s]
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/pipeline.rb

@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-require "pathname"
-
-require "ucode/cache"
-require "ucode/glyphs/pdf_fetcher"
-require "ucode/glyphs/writer"
-require "ucode/parsers"
-
-module Ucode
-  module Glyphs
-    # Assembles the per-block specs that {Glyphs::Writer#write_all} drains.
-    #
-    # Owns three pieces of orchestration that {Commands::GlyphsCommand}
-    # used to carry inline:
-    #
-    #   - block loading from {Cache.ucd_dir}/Blocks.txt (with an optional
-    #     block filter)
-    #   - PDF fetcher construction (with monolith fallback)
-    #   - the per-block page-map heuristic (per-block PDFs are page 1 =
-    #     title, page 2 = first chart page starting at the block's first
-    #     codepoint; true for most BMP blocks; multi-page blocks need a
-    #     richer resolver — mismatches yield placeholder SVGs only, never
-    #     wrong glyphs)
-    #
-    # The Command stays a thin wrapper that prints the experimental
-    # warning and wires the writer. See Candidate 3 of the 2026-06-29
-    # architecture review.
-    class Pipeline
-      # Path to the monolith fallback file when no per-block PDF is on
-      # disk yet. Overridable for tests.
-      DEFAULT_MONOLITH_PATH = "CodeCharts.pdf"
-      # Cache path for the page-map corpus. Overridable for tests.
-      DEFAULT_PAGE_MAP_CACHE = "data/codecharts_page_map.json"
-
-      Spec = Struct.new(:block, :pdf_path, :page_map, keyword_init: true)
-
-      # @param version [String] resolved UCD version (callers must
-      #   resolve via {VersionResolver.resolve} first)
-      # @param block_filter [Array<String>, nil] block ids to limit to;
-      #   nil = every block
-      # @param monolith_path [String, Pathname, nil] fallback monolith
-      # @param page_map_cache [String, Pathname] cache for the page map
-      def initialize(version:, block_filter: nil,
-                     monolith_path: DEFAULT_MONOLITH_PATH,
-                     page_map_cache: DEFAULT_PAGE_MAP_CACHE)
-        @version = version
-        @block_filter = block_filter
-        @monolith_path = monolith_path
-        @page_map_cache = page_map_cache
-      end
-
-      # Load every block from the cached Blocks.txt (filtered by
-      # `@block_filter` when set) and pair each one with a fetched PDF
-      # path and a page map. Blocks whose PDF cannot be fetched are
-      # silently dropped — the placeholder pass downstream covers them.
-      #
-      # @param force [Boolean] re-fetch PDFs even when cached
-      # @return [Array<Spec>]
-      def build_specs(force: false)
-        blocks = load_blocks
-        fetcher = build_fetcher(blocks)
-        blocks.map { |block| spec_for(block, fetcher, force) }.compact
-      end
-
-      private
-
-      def load_blocks
-        path = Cache.ucd_dir(@version).join("Blocks.txt")
-        return [] unless path.exist?
-
-        all = Parsers::Blocks.each_record(path).to_a
-        return all unless @block_filter && !@block_filter.empty?
-
-        filter_set = @block_filter.to_set
-        all.select { |block| filter_set.include?(block.id) }
-      end
-
-      def build_fetcher(blocks)
-        monolith = @monolith_path ? Pathname.new(@monolith_path) : nil
-        monolith = monolith.exist? ? monolith : nil
-        PdfFetcher.new(
-          @version,
-          monolith_path: monolith,
-          blocks: blocks,
-          page_map_cache: @page_map_cache,
-        )
-      end
-
-      def spec_for(block, fetcher, force)
-        pdf_path = fetcher.fetch(block_first_cp: block.range_first, force: force)
-        return nil unless pdf_path
-
-        Spec.new(block: block, pdf_path: pdf_path, page_map: page_map_for(block))
-      end
-
-      # Per-block PDFs are page 1 = title, page 2 = first chart page
-      # starting at the block's first codepoint. True for most BMP
-      # blocks; multi-page blocks (CJK) need a richer resolver.
-      def page_map_for(block)
-        { 2 => block.range_first }
-      end
-    end
-  end
-end