ucode 0.2.1 → 0.2.3

data/lib/ucode/glyphs/grid_detector.rb

@@ -1,165 +0,0 @@
-# frozen_string_literal: true
-
-require "nokogiri"
-
-require "ucode/glyphs/grid"
-require "ucode/glyphs/path_bbox"
-
-module Ucode
-  module Glyphs
-    # Detects the chart grid in a Code Charts PDF page rendered to SVG.
-    #
-    # The PDF page produced by pdftocairo / pdf2svg / dvisvgm contains
-    # every visible element (title, block name, row labels, codepoint
-    # digits, and the actual character glyphs) as positioned `<use>`
-    # references into a `<defs>` block of named glyph outlines. The
-    # character cells we want to extract correspond to glyphs whose
-    # bounding box is larger than every label or digit font on the
-    # page — the chart's character samples are drawn at a larger size
-    # than any of the surrounding text.
-    #
-    # Algorithm:
-    #   1. Walk `<defs>`, estimate each glyph's bbox via `PathBbox`.
-    #   2. Classify a glyph as "character-sized" when its width and
-    #      height both exceed `CharSizeThreshold` (default 8 pt).
-    #      This excludes title, row-label, and digit glyphs while
-    #      keeping every actual character sample — including pages
-    #      where the chart mixes multiple character fonts (e.g. the
-    #      Basic Latin page uses one font for punctuation/digits and
-    #      another for letters).
-    #   3. Collect every `<use>` that references a character-sized
-    #      glyph; these are the cell origins.
-    #   4. Cluster the Y values of those uses into rows, and within
-    #      each row cluster the X values into columns.
-    #   5. Drop rows whose column count diverges from the modal value
-    #      (these are footer/header artifacts, not chart rows).
-    #   6. Return a `Grid` value object anchored at the top-left cell
-    #      with uniform column/row pitches derived from the median
-    #      spacing between adjacent clusters.
-    #
-    # This is pure (no I/O). The detector takes a parsed Nokogiri
-    # document and returns a `Grid`.
-    class GridDetector
-      CharSizeThreshold = 8.0
-      ClusterEpsilon = 15.0
-      private_constant :CharSizeThreshold, :ClusterEpsilon
-
-      class << self
-        # @param doc [Nokogiri::XML::Document]
-        # @param block_first_cp [Integer] first codepoint of the block;
-        #   stored on the Grid so callers can map codepoint ↔ cell.
-        # @return [Ucode::Glyphs::Grid, nil] nil if no character grid
-        #   could be detected.
-        def detect(doc, block_first_cp:)
-          uses = collect_uses(doc)
-          return nil if uses.empty?
-
-          char_glyph_ids = char_sized_glyph_ids(doc)
-          return nil if char_glyph_ids.empty?
-
-          cell_uses = uses.select { |u| char_glyph_ids.include?(u.glyph_id) }
-          return nil if cell_uses.empty?
-
-          build_grid(cell_uses, block_first_cp)
-        end
-
-        private
-
-        UsePosition = Struct.new(:x, :y, :glyph_id, :set_id, keyword_init: true)
-
-        def collect_uses(doc)
-          doc.css("use").map do |node|
-            href = node["xlink:href"] || node["href"] || ""
-            glyph_id = href.sub(/\A#/, "")
-            match = glyph_id.match(/\Aglyph-(\d+)-(\d+)\z/)
-            next nil unless match
-
-            UsePosition.new(
-              x: node["x"].to_f,
-              y: node["y"].to_f,
-              glyph_id: glyph_id,
-              set_id: match[1].to_i,
-            )
-          end.compact
-        end
-
-        def char_sized_glyph_ids(doc)
-          doc.css("defs g[id^='glyph-']").each_with_object({}) do |g, acc|
-            id = g["id"]
-            next unless id =~ /\Aglyph-\d+-\d+\z/
-
-            paths = g.css("path")
-            next if paths.empty?
-
-            bbox = paths.map { |p| PathBbox.estimate(p["d"]) }.reject(&:empty?).reduce do |a, b|
-              PathBbox::Result.new(
-                min_x: [a.min_x, b.min_x].min,
-                min_y: [a.min_y, b.min_y].min,
-                max_x: [a.max_x, b.max_x].max,
-                max_y: [a.max_y, b.max_y].max,
-              )
-            end
-            next unless bbox
-
-            acc[id] = true if char_sized?(bbox)
-          end
-        end
-
-        def char_sized?(bbox)
-          bbox.width >= CharSizeThreshold && bbox.height >= CharSizeThreshold
-        end
-
-        def median(values)
-          return 0.0 if values.empty?
-
-          sorted = values.sort
-          mid = sorted.size / 2
-          sorted.size.even? ? (sorted[mid - 1] + sorted[mid]) / 2.0 : sorted[mid]
-        end
-
-        def build_grid(cell_uses, block_first_cp)
-          row_clusters = cluster_by_value(cell_uses, :y)
-          return nil if row_clusters.empty?
-
-          column_clusters = cluster_by_value(cell_uses, :x)
-          return nil if column_clusters.empty?
-
-          column_starts = column_clusters.map { |c| c.map(&:x).min }.sort
-          row_starts = row_clusters.map { |c| c.map(&:y).min }.sort
-
-          Grid.new(
-            origin_x: column_starts.first,
-            origin_y: row_starts.first,
-            column_pitch: median_pitch(column_starts),
-            row_pitch: median_pitch(row_starts),
-            columns: column_starts.size,
-            rows: row_starts.size,
-            block_first_cp: block_first_cp,
-          )
-        end
-
-        def cluster_by_value(items, attr)
-          sorted = items.sort_by { |i| i.public_send(attr) }
-          clusters = []
-          sorted.each do |item|
-            value = item.public_send(attr)
-            if clusters.empty? || (value - clusters.last[:max]).abs > ClusterEpsilon
-              clusters << { max: value, items: [item] }
-            else
-              clusters.last[:max] = value
-              clusters.last[:items] << item
-            end
-          end
-          clusters.map { |c| c[:items] }
-        end
-
-        def median_pitch(sorted_values)
-          return 0.0 if sorted_values.size < 2
-
-          pitches = sorted_values.each_cons(2).map { |a, b| b - a }
-          median(pitches)
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/monolith_page_map.rb

@@ -1,181 +0,0 @@
-# frozen_string_literal: true
-
-require "pathname"
-require "json"
-require "open3"
-
-module Ucode
-  module Glyphs
-    # Maps a Unicode block's first codepoint to its page range inside the
-    # monolith `CodeCharts.pdf` by parsing the PDF's bookmark outline and
-    # matching each bookmark title to a Block.name from `Blocks.txt`.
-    #
-    # Each chart cluster printed by the Unicode Consortium is a single
-    # bookmark entry:
-    #
-    #   BookmarkTitle: Greek and Coptic
-    #   BookmarkLevel: 1
-    #   BookmarkPageNumber: 415
-    #
-    # The cluster title usually equals a Block.name verbatim, but a few
-    # clusters carry a heading that prepends "C0 Controls and " /
-    # "C1 Controls and " to the block name. We resolve both forms.
-    #
-    # End-page of a cluster is one page before the next cluster's start
-    # page (last cluster's end-page is the PDF's last page).
-    #
-    # The map is cached as JSON at `data/codecharts_page_map.json` so
-    # we don't re-scan the 3,156-page monolith on every run.
-    class MonolithPageMap
-      BookmarkTitleRegex = /BookmarkTitle:\s*(.+)/.freeze
-      BookmarkPageRegex  = /BookmarkPageNumber:\s*(\d+)/.freeze
-      private_constant :BookmarkTitleRegex, :BookmarkPageRegex
-
-      # The Unicode charts print these multi-block clusters as a single
-      # chart page (the C0/C1 control chars are drawn alongside their
-      # block's other characters). Each cluster title maps to the single
-      # block it belongs to.
-      ClusterPrefixes = [
-        "C0 Controls and ",
-        "C1 Controls and ",
-      ].freeze
-      private_constant :ClusterPrefixes
-
-      MapEntry = Struct.new(:first_cp, :start_page, :end_page, keyword_init: true)
-
-      class << self
-        # Build the map by parsing the monolith's outline and matching
-        # each bookmark title to a Block.
-        #
-        # @param monolith_path [String, Pathname]
-        # @param blocks [Array<Ucode::Models::Block>] the parsed Blocks table
-        # @return [Hash{Integer => MapEntry}] keyed by block.range_first
-        def build(monolith_path:, blocks:)
-          name_to_first_cp = blocks.each_with_object({}) do |b, h|
-            h[b.name] = b.range_first
-          end
-          total_pages = page_count(monolith_path)
-          entries = parse_bookmarks(dump_bookmarks(monolith_path), name_to_first_cp)
-          attach_end_pages(entries, total_pages)
-          entries.each_with_object({}) do |e, h|
-            h[e.first_cp] = e
-          end
-        end
-
-        # Pure: parse a `pdftk dump_data` string into a list of
-        # MapEntry rows (without end_pages). Exposed for unit tests
-        # and any caller that already has the dump cached.
-        #
-        # @param dump [String] the raw `pdftk dump_data` output
-        # @param name_to_first_cp [Hash{String => Integer}]
-        # @return [Array<MapEntry>]
-        def parse_bookmarks(dump, name_to_first_cp)
-          entries = []
-          current_title = nil
-          dump.each_line do |line|
-            case line
-            when BookmarkTitleRegex
-              current_title = Regexp.last_match(1).strip
-            when BookmarkPageRegex
-              page = Regexp.last_match(1).to_i
-              cp = resolve_first_cp(current_title, name_to_first_cp)
-              entries << MapEntry.new(first_cp: cp, start_page: page) if cp
-              current_title = nil
-            end
-          end
-          entries.sort_by(&:start_page)
-        end
-
-        # Pure: attach end_pages by sorting entries and assigning each
-        # entry's end to one page before the next entry's start.
-        #
-        # @param entries [Array<MapEntry>]
-        # @param total_pages [Integer, nil] page count of the source PDF;
-        #   the last entry's end_page falls back to this when present.
-        # @return [Array<MapEntry>] the same entries, mutated with end_pages.
-        def attach_end_pages(entries, total_pages = nil)
-          sorted = entries.sort_by(&:start_page)
-          sorted.each_with_index do |entry, i|
-            next_entry = sorted[i + 1]
-            entry.end_page = next_entry ? next_entry.start_page - 1 : total_pages
-          end
-          sorted
-        end
-
-        # Load from cache, or build and cache.
-        # @param monolith_path [String, Pathname]
-        # @param blocks [Array<Ucode::Models::Block>]
-        # @param cache_path [String, Pathname, nil]
-        # @return [Hash{Integer => MapEntry}]
-        def load(monolith_path:, blocks:, cache_path: nil)
-          cache = cache_path && Pathname.new(cache_path)
-          if cache&.exist?
-            return load_from_json(cache.read)
-          end
-
-          map = build(monolith_path: monolith_path, blocks: blocks)
-          write_cache(map, cache) if cache
-          map
-        end
-
-        # Look up a block's page range by its first cp.
-        # @param map [Hash{Integer => MapEntry}]
-        # @param block_first_cp [Integer]
-        # @return [MapEntry, nil]
-        def range_for(map, block_first_cp)
-          map[block_first_cp]
-        end
-
-        # ---- I/O helpers (impure) --------------------------------------
-
-        def dump_bookmarks(monolith_path)
-          out, status = Open3.capture2e("pdftk", monolith_path.to_s, "dump_data")
-          return "" unless status.success?
-
-          out
-        end
-
-        def page_count(monolith_path)
-          out, status = Open3.capture2e("pdfinfo", monolith_path.to_s)
-          return nil unless status.success?
-
-          match = out.match(/^Pages:\s+(\d+)/)
-          match ? match[1].to_i : nil
-        end
-
-        private
-
-        def resolve_first_cp(title, name_to_first_cp)
-          return nil unless title
-
-          return name_to_first_cp[title] if name_to_first_cp.key?(title)
-
-          ClusterPrefixes.each do |prefix|
-            stripped = title.sub(/\A#{Regexp.escape(prefix)}/, "")
-            return name_to_first_cp[stripped] if name_to_first_cp.key?(stripped)
-          end
-
-          nil
-        end
-
-        def write_cache(map, cache_path)
-          payload = map.values.map { |e| { "first_cp" => e.first_cp,
-                                            "start_page" => e.start_page,
-                                            "end_page" => e.end_page } }
-          cache_path.dirname.mkpath
-          cache_path.write(JSON.pretty_generate(payload))
-        end
-
-        def load_from_json(json)
-          payload = JSON.parse(json)
-          payload.each_with_object({}) do |row, h|
-            entry = MapEntry.new(first_cp: row["first_cp"],
-                                 start_page: row["start_page"],
-                                 end_page: row["end_page"])
-            h[entry.first_cp] = entry
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/mutool_renderer.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-require "ucode/glyphs/page_renderer"
-
-module Ucode
-  module Glyphs
-    # `mutool draw` from MuPDF — typically the fastest and cleanest.
-    # Emits one `<svg>` per page with `<path>` vector data.
-    #
-    # Command: `mutool draw -F svg -o <out.svg> <in.pdf> <page>`
-    class MutoolRenderer < PageRenderer
-      class << self
-        def renderer_name
-          :mutool
-        end
-
-        def binary_name
-          :mutool
-        end
-
-        def build_command(pdf_path, page_num, out_path)
-          ["mutool", "draw", "-F", "svg", "-o", out_path.to_s,
-           pdf_path.to_s, page_num.to_s]
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/page_renderer.rb

@@ -1,221 +0,0 @@
-# frozen_string_literal: true
-
-require "open3"
-require "pathname"
-require "tmpdir"
-
-require "ucode/error"
-
-module Ucode
-  module Glyphs
-    # Strategy interface for PDF-page-to-SVG rendering.
-    #
-    # Subclasses implement `renderer_name`, `binary_name`, and
-    # `build_command`. The base class handles availability check,
-    # command execution, error handling, and the renderer registry.
-    #
-    # **OCP**: a new renderer is a new subclass file + one entry in
-    # `KNOWN_RENDERERS`. The base class and existing renderers are not
-    # modified.
-    #
-    # **Vector-only requirement**: every renderer here must emit SVG
-    # `<path>` elements (vector data) for the Code Charts PDFs, not
-    # raster images. Callers verify this via `path_count` on the output.
-    class PageRenderer
-      OUTPUT_FORMAT = :svg
-
-      # Fixture used by `works?` to smoke-test renderers. Resolved lazily
-      # so missing-fixture environments (installed gem without spec assets)
-      # don't fail at load time.
-      DEFAULT_SMOKE_FIXTURE =
-        File.expand_path("../../../spec/fixtures/pdfs/basic_latin.pdf", __dir__)
-
-      # Ordered list of known concrete renderer class names (as symbols),
-      # most-preferred first. Resolved lazily via `const_get` so that
-      # loading any one renderer does not eagerly load all of them — this
-      # avoids a circular require (each renderer file requires this file
-      # to inherit from PageRenderer).
-      KNOWN_RENDERERS = %i[
-        MutoolRenderer
-        Pdf2svgRenderer
-        DvisvgmRenderer
-        PdftocairoRenderer
-      ].freeze
-      private_constant :KNOWN_RENDERERS
-
-      class << self
-        # @return [Symbol] short identifier (e.g. :mutool)
-        def renderer_name
-          raise NotImplementedError
-        end
-
-        # @return [String, Symbol] the binary looked up on PATH
-        def binary_name
-          raise NotImplementedError
-        end
-
-        # @return [Symbol] always :svg for now; future formats (png, etc.)
-        #   would warrant a separate renderer family.
-        def output_format
-          OUTPUT_FORMAT
-        end
-
-        # Build the argv for the renderer. Subclasses return an Array
-        # suitable for `Open3.capture2e` (no shell interpolation).
-        # @param pdf_path [Pathname, String]
-        # @param page_num [Integer] 1-indexed
-        # @param out_path [Pathname, String]
-        # @return [Array<String>]
-        def build_command(pdf_path, page_num, out_path)
-          raise NotImplementedError
-        end
-
-        # @return [Boolean] true if the binary is on PATH
-        def available?
-          system("which", binary_name.to_s, out: "/dev/null", err: "/dev/null")
-        end
-
-        # Smoke-test the binary by actually rendering one page of the
-        # fixture PDF AND verifying the output format is consumable by
-        # the downstream `GridDetector` / `CellExtractor` pipeline.
-        #
-        # Three things can make a renderer unusable for this codebase:
-        #   1. Binary not on PATH (`available?` catches this).
-        #   2. Binary on PATH but silently broken (e.g. Ubuntu's
-        #      `mupdf-tools` is built without LCMS, so `mutool` warns
-        #      "ICC support is not available" and emits zero bytes for
-        #      ICC-profiled PDFs).
-        #   3. Binary works but emits a flat-path SVG that GridDetector
-        #      can't parse (mutool's format: `<path id="font_X_Y">`
-        #      directly in `<defs>`, no `<use>` references). The grid
-        #      detector requires the `<g id="glyph-N-M">` + `<use>` form
-        #      produced by pdftocairo / pdf2svg.
-        #
-        # The result is memoized per-renderer for the process lifetime —
-        # the binary's capabilities don't change mid-run.
-        #
-        # When no fixture PDF is available (e.g. installed gem without
-        # spec assets), degrades to `available?` — we can't smoke-test
-        # without input, so we trust the binary's presence on PATH.
-        #
-        # @param fixture_pdf [String, Pathname] small one-page PDF used
-        #   for the smoke render. Defaults to the project's
-        #   `basic_latin.pdf` spec fixture.
-        # @return [Boolean]
-        def works?(fixture_pdf: DEFAULT_SMOKE_FIXTURE)
-          if !available?
-            false
-          elsif !File.exist?(fixture_pdf.to_s)
-            true # no fixture to verify against; trust PATH
-          else
-            smoke_render_ok?(fixture_pdf)
-          end
-        end
-
-        # Render one page of `pdf_path` to `out_path` as SVG.
-        # @param pdf_path [Pathname, String]
-        # @param page_num [Integer] 1-indexed
-        # @param out_path [Pathname, String]
-        # @return [Symbol] :ok on success
-        # @raise [Ucode::PdfRenderError] on failure (non-zero exit,
-        #   output file missing, or binary unavailable)
-        def render(pdf_path, page_num, out_path)
-          unless available?
-            raise PdfRenderError.new(
-              "binary '#{binary_name}' not available on PATH",
-              context: { renderer: name, binary: binary_name },
-            )
-          end
-
-          out = Pathname.new(out_path)
-          out.dirname.mkpath
-
-          cmd = build_command(Pathname.new(pdf_path), page_num, out)
-          output, status = Open3.capture2e(*cmd)
-
-          unless status.success? && out.exist? && out.size.positive?
-            raise PdfRenderError.new(
-              "render failed for page #{page_num} of #{pdf_path} via '#{binary_name}'",
-              context: {
-                renderer: name,
-                binary: binary_name,
-                exit_status: status.exitstatus,
-                output: output,
-              },
-            )
-          end
-
-          :ok
-        end
-
-        # ---- Registry ----
-
-        # @return [Array<Class>] every known concrete renderer
-        def all
-          @all ||= KNOWN_RENDERERS.map { |n| Ucode::Glyphs.const_get(n) }.freeze
-        end
-
-        # @return [Array<Class>] renderers whose binary is installed
-        def available
-          all.select(&:available?)
-        end
-
-        # @return [Array<Class>] renderers that actually produce SVG in
-        #   the format `GridDetector` consumes (smoke-tested once per
-        #   process via `works?`, then cached). Subset of `available`.
-        def working
-          return @working if @working
-
-          @working = all.select(&:works?).freeze
-        end
-
-        # Clear the cached `working` list. Useful when the environment
-        # changes (e.g. a binary is installed mid-process) or in tests.
-        def reset_working_cache!
-          @working = nil
-        end
-
-        # @param name [Symbol, String]
-        # @return [Class, nil]
-        def find(name)
-          all.find { |r| r.renderer_name == name.to_sym }
-        end
-
-        # @return [Class, nil] the first working renderer; falls back to
-        #   the first available renderer if none have been smoke-tested
-        #   yet (preserves eager-init paths). nil if nothing is installed.
-        def default
-          working.first || available.first
-        end
-
-        private
-
-        # @param fixture_pdf [String] path to an existing PDF
-        # @return [Boolean] true iff rendering page 1 produces an SVG
-        #   with the `<g id="glyph-N-M">` + `<use>` form that
-        #   `GridDetector` requires.
-        def smoke_render_ok?(fixture_pdf)
-          Dir.mktmpdir("renderer-smoke-") do |dir|
-            out = File.join(dir, "smoke.svg")
-            begin
-              render(fixture_pdf, 1, out)
-            rescue PdfRenderError
-              break false
-            end
-            svg_has_pipeline_format?(out)
-          end
-        end
-
-        def svg_has_pipeline_format?(out_path)
-          return false unless File.exist?(out_path)
-          return false unless File.size(out_path).positive?
-
-          body = File.read(out_path)
-          body.include?("<svg") &&
-            body.include?("<use") &&
-            body.match?("id=\"glyph-\\d+-\\d+\"")
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/path_bbox.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-module Ucode
-  module Glyphs
-    # Estimates the axis-aligned bounding box of an SVG `<path>` `d`
-    # attribute by scanning every numeric coordinate pair in the path
-    # data. This is a conservative over-estimate: control points and
-    # implicit vertices are included, so the true curve bbox is always
-    # contained within the estimate. For grid detection and cell
-    # membership tests, the over-estimate is sufficient and avoids the
-    # cost of a Bezier solver.
-    #
-    # Only absolute coordinates are returned. Relative commands (lowercase
-    # `m`, `l`, `c`, …) are NOT supported — Code Charts SVGs from every
-    # supported renderer (pdftocairo, pdf2svg, dvisvgm, mutool) emit
-    # absolute commands. If relative commands appear, parse them via a
-    # proper SVG path parser before calling this.
-    module PathBbox
-      NUMBER = /-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?/.freeze
-
-      Result = Struct.new(:min_x, :min_y, :max_x, :max_y, keyword_init: true) do
-        def width
-          return nil if empty?
-
-          max_x - min_x
-        end
-
-        def height
-          return nil if empty?
-
-          max_y - min_y
-        end
-
-        def empty?
-          min_x.nil? || min_y.nil? || max_x.nil? || max_y.nil?
-        end
-      end
-
-      class << self
-        def estimate(path_d)
-          return Result.new if path_d.nil? || path_d.empty?
-
-          numbers = path_d.scan(NUMBER).map(&:to_f)
-          return Result.new if numbers.empty?
-
-          xs = []
-          ys = []
-          numbers.each_slice(2) do |x, y|
-            xs << x
-            ys << y
-          end
-          Result.new(
-            min_x: xs.min,
-            min_y: ys.min,
-            max_x: xs.max,
-            max_y: ys.max,
-          )
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/pdf2svg_renderer.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-require "ucode/glyphs/page_renderer"
-
-module Ucode
-  module Glyphs
-    # `pdf2svg` — simple, widely available. One SVG per page.
-    #
-    # Command: `pdf2svg <in.pdf> <out.svg> <page>`
-    class Pdf2svgRenderer < PageRenderer
-      class << self
-        def renderer_name
-          :pdf2svg
-        end
-
-        def binary_name
-          :pdf2svg
-        end
-
-        def build_command(pdf_path, page_num, out_path)
-          ["pdf2svg", pdf_path.to_s, out_path.to_s, page_num.to_s]
-        end
-      end
-    end
-  end
-end