ucode 0.2.3 → 0.3.0

data/lib/ucode/glyphs/embedded_fonts/content_stream_correlator.rb

@@ -6,28 +6,15 @@ module Ucode
       # Pillar 2 fallback: build a `{codepoint => gid}` map for a Type0
       # font whose PDF object graph has no `/ToUnicode` CMap stream.
       #
-      # The Code Charts draw every chart cell as a `<use>` element that
-      # references the font's GID via an `href` of the form
-      # `#font_<font_obj_id>_<gid>`. The chart also prints the row +
-      # column codepoint labels using one or more "label" fonts (small
-      # Latin glyphs) that show the hex codepoint as text. By clustering
-      # the labels positionally (Y-bucket for the row, X-bucket for the
-      # column) we recover the codepoint each cluster represents, then
-      # match each cluster positionally to the specimen glyph at the
-      # same Y/X position.
+      # Adapter for the `mutool draw -F svg` output format: parses
+      # `<use>` elements from the rendered PDF page SVG, partitions into
+      # labels and specimens by PDF font object ID (supplied via {Config}),
+      # then delegates matching to {PositionalMatcher}.
       #
-      # The algorithm generalizes the Tai Yo correlator that was tested
-      # against `data/pdfs/U1E6C0.pdf` (50/52 specimen codepoints
-      # matched, with the two missing being layout edge cases). The
-      # bucket sizes are configurable because some blocks use a tighter
-      # grid than others.
-      #
-      # Inputs are deliberately pure: a string of SVG markup plus a
-      # {Config}. The catalog is responsible for sourcing the SVG (by
-      # rendering the relevant PDF page(s) via `mutool draw -F svg`) and
-      # for knowing which font_obj_ids are labels vs specimen on that
-      # page. That keeps this class trivially testable with synthetic
-      # SVG fixtures.
+      # The SVG parsing (regex-based `<use>` extraction, HTML entity
+      # decoding) is the only piece of format-specific work here. The
+      # matching algorithm lives in {PositionalMatcher} and is shared
+      # with {TraceCorrelator}.
       class ContentStreamCorrelator
         # Per-font / per-block configuration.
         #
@@ -37,25 +24,13 @@ module Ucode
         #   whose glyphs are the specimens we want to attribute.
         # @!attribute page_numbers [Array<Integer>] 1-based PDF page
         #   numbers whose content streams reference the specimen font.
-        # @!attribute y_bucket [Float] vertical clustering granularity
-        #   in PDF points. Default 1.5 — matches mutool's text matrix
-        #   granularity for the row labels.
-        # @!attribute x_bucket [Float] horizontal clustering granularity
-        #   in PDF points. Default 50.0 — separates label clusters
-        #   within a row (labels are ~16pt wide, clusters ~60-160pt
-        #   apart).
         Config = Struct.new(
           :label_font_ids,
           :specimen_font_id,
           :page_numbers,
-          :y_bucket,
-          :x_bucket,
           keyword_init: true,
         )
 
-        DEFAULT_Y_BUCKET = 1.5
-        DEFAULT_X_BUCKET = 50.0
-
         # Internal value object for a parsed `<use>` element. Public so
         # the spec can construct realistic fixtures without re-implementing
         # the parser shape.
@@ -64,8 +39,6 @@ module Ucode
         # @param config [Config]
         def initialize(config)
           @config = config
-          @y_bucket = config.y_bucket || DEFAULT_Y_BUCKET
-          @x_bucket = config.x_bucket || DEFAULT_X_BUCKET
         end
 
         # @param svg [String] rendered PDF page(s) as SVG markup. May
@@ -77,29 +50,33 @@ module Ucode
           uses = parse_uses(svg)
           return {} if uses.empty?
 
-          partition_and_map(uses)
-        end
-
-        private
-
-        def partition_and_map(uses)
           labels, specimens = partition_uses(uses)
           return {} if labels.empty? || specimens.empty?
 
-          cp_per_cluster = decode_label_clusters(labels)
-          return {} if cp_per_cluster.empty?
-
-          build_mapping(cp_per_cluster, group_rows(specimens))
+          PositionalMatcher.match(
+            specimens.map { |u| to_position(u) },
+            labels.map { |u| to_position(u) },
+          )
         end
 
+        private
+
         def partition_uses(uses)
-          labels = uses.select do |u|
-            @config.label_font_ids.include?(u.font_id)
-          end
+          labels = uses.select { |u| @config.label_font_ids.include?(u.font_id) }
           specimens = uses.select { |u| u.font_id == @config.specimen_font_id }
           [labels, specimens]
         end
 
+        def to_position(use)
+          PositionalMatcher::Position.new(
+            x: use.x,
+            y: use.y,
+            font_ref: use.font_id,
+            glyph_id: use.gid,
+            text: decode_entities(use.text),
+          )
+        end
+
         # Match `<use .../>` elements and pull out the font_obj_id and
         # gid from the href, plus the text matrix's e and f terms (which
         # give the X/Y origin). The data-text attribute carries the
@@ -143,82 +120,6 @@ module Ucode
           attrs[/xlink:href="([^"]+)"/, 1] || attrs[/href="([^"]+)"/, 1]
         end
 
-        # Cluster label uses by quantized (Y, X) position. Within each
-        # cluster, members are sorted by X so that joined text reads
-        # left-to-right (hex codepoint string).
-        def decode_label_clusters(labels)
-          cluster_members = bucket_labels_by_position(labels)
-          decode_each_cluster(cluster_members)
-        end
-
-        def bucket_labels_by_position(labels)
-          clusters = Hash.new { |h, k| h[k] = [] }
-          labels.each do |label|
-            key = [bucket(label.y, @y_bucket), bucket(label.x, @x_bucket)]
-            clusters[key] << label
-          end
-          clusters
-        end
-
-        def decode_each_cluster(clusters)
-          clusters.each_with_object({}) do |(key, members), decoded|
-            text = members.sort_by(&:x).map { |m| decode_entities(m.text) }.join
-            next unless text.match?(/\A[0-9A-Fa-f]{4,6}\z/)
-
-            decoded[key] = text.to_i(16)
-          end
-        end
-
-        # Group any set of uses (labels or specimens) by Y-bucket; sort
-        # each row by X so positional matching is straightforward.
-        def group_rows(uses)
-          rows = Hash.new { |h, k| h[k] = [] }
-          uses.each do |u|
-            rows[bucket(u.y, @y_bucket)] << u
-          end
-          rows.each_value { |v| v.sort_by!(&:x) }
-          rows
-        end
-
-        # Within each Y-row, the rightmost label cluster is the
-        # specimen codepoint; the rightmost specimen glyph is the
-        # specimen GID. The preceding label clusters (if any) are
-        # cross-reference codepoints, matched positionally to the
-        # preceding specimen glyphs in the same row.
-        def build_mapping(cp_per_cluster, specimen_rows)
-          cp_rows = group_cps_by_row(cp_per_cluster)
-          cp_rows.keys.sort.each_with_object({}) do |yb, mapping|
-            assign_row(mapping, cp_rows[yb], specimen_rows[yb] || [])
-          end
-        end
-
-        def assign_row(mapping, cps, glyphs)
-          return if cps.empty? || glyphs.empty?
-
-          mapping[cps.last] = glyphs.last.gid
-          assign_xrefs(mapping, cps[0...-1], glyphs[0...-1])
-        end
-
-        def assign_xrefs(mapping, xref_cps, xref_glyphs)
-          xref_cps.each_with_index do |cp, i|
-            g = xref_glyphs[i]
-            mapping[cp] = g.gid if g
-          end
-        end
-
-        def group_cps_by_row(cp_per_cluster)
-          rows = Hash.new { |h, k| h[k] = [] }
-          cp_per_cluster.each do |(yb, xb), cp|
-            rows[yb] << [cp, xb]
-          end
-          rows.each_value { |v| v.sort_by! { |_, xb| xb } }
-          rows.transform_values { |v| v.map(&:first) }
-        end
-
-        def bucket(value, size)
-          (value / size).round * size
-        end
-
         def decode_entities(text)
           text.gsub(/&#x([0-9a-fA-F]+);/) { [$1.to_i(16)].pack("U") }
         end

data/lib/ucode/glyphs/embedded_fonts/font_entry.rb

@@ -5,7 +5,6 @@ require "pathname"
 require "tempfile"
 
 require "fontisan"
-require_relative "../../error"
 
 module Ucode
   module Glyphs

data/lib/ucode/glyphs/embedded_fonts/pdf_indexer.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+require "open3"
+require "pathname"
+
+module Ucode
+  module Glyphs
+    module EmbeddedFonts
+      # Walks the Code Charts PDF once via `mutool info` + `mutool show`
+      # and builds an Array of {RawFontDescriptor} — one per Type0 font
+      # that has the required descendant CIDFont, FontDescriptor, and
+      # FontFile2/3 + Identity CIDToGIDMap.
+      #
+      # Pure subprocess + parsing concern. Does NOT resolve codepoint →
+      # GID (that's {CodepointMapper}'s job). The descriptor carries
+      # every ref the mapper needs to do its work.
+      class PdfIndexer
+        # @param source [PdfLocation]
+        def initialize(source:)
+          @source = source
+        end
+
+        # @return [Array<RawFontDescriptor>]
+        def raw_descriptors
+          type0_refs = discover_type0_fonts
+          return [] if type0_refs.empty?
+
+          type0_dicts = fetch_objects(type0_refs.keys)
+          descendant_refs, = collect_refs(type0_refs, type0_dicts)
+          descendant_dicts = fetch_objects(descendant_refs)
+          fontdesc_dicts = fetch_fontdescs(descendant_dicts)
+
+          build_descriptors(type0_refs, type0_dicts, descendant_dicts, fontdesc_dicts)
+        end
+
+        def collect_refs(type0_refs, type0_dicts)
+          descendant_refs = []
+          tounicode_refs = []
+          type0_refs.each_key do |font_obj_id|
+            d = type0_dicts[font_obj_id] || {}
+            collect_ref(d["DescendantFonts"], descendant_refs)
+            collect_ref(d["ToUnicode"], tounicode_refs)
+          end
+          [descendant_refs, tounicode_refs]
+        end
+
+        def collect_ref(dict_value, acc)
+          ref = first_ref(dict_value)
+          acc << ref if ref
+        end
+
+        def fetch_fontdescs(descendant_dicts)
+          fontdesc_refs = []
+          descendant_dicts.each_value do |d|
+            collect_ref(d["FontDescriptor"], fontdesc_refs)
+          end
+          fetch_objects(fontdesc_refs)
+        end
+
+        # @return [Integer] total pages in the PDF
+        def page_count
+          @page_count ||= begin
+            m = mutool_info_text.match(/^Pages:\s+(\d+)/)
+            m ? m[1].to_i : 1
+          end
+        end
+
+        # @param base_font [String] e.g. "GPJAHB+WolofGaraySansSerif"
+        # @return [Boolean] true if this font appears on any page
+        def font_appears?(base_font)
+          font_entries_cache.key?(base_font)
+        end
+
+        private
+
+        def build_descriptors(type0_refs, type0_dicts, descendant_dicts, fontdesc_dicts)
+          type0_refs.filter_map do |font_obj_id, base_font|
+            build_descriptor(
+              font_obj_id, base_font, type0_dicts[font_obj_id] || {},
+              descendant_dicts, fontdesc_dicts,
+            )
+          end
+        end
+
+        def build_descriptor(font_obj_id, base_font, type0_dict,
+                             descendant_dicts, fontdesc_dicts)
+          desc_ref = first_ref(type0_dict["DescendantFonts"])
+          return nil unless desc_ref
+
+          tu_ref = first_ref(type0_dict["ToUnicode"])
+          desc_dict = descendant_dicts[desc_ref] || {}
+          fd_dict = fontdesc_for(desc_dict, fontdesc_dicts)
+          return nil unless fd_dict
+
+          fontfile_obj_id, fontfile_kind = resolve_fontfile(fd_dict)
+          return nil unless fontfile_obj_id
+
+          cid_map_kind = resolve_cid_to_gid(desc_dict)
+          return nil unless cid_map_kind
+
+          RawFontDescriptor.new(
+            base_font: base_font,
+            font_obj_id: font_obj_id,
+            fontfile_obj_id: fontfile_obj_id,
+            fontfile_kind: fontfile_kind,
+            tounicode_ref: tu_ref,
+            cid_map_kind: cid_map_kind,
+          )
+        end
+
+        def fontdesc_for(desc_dict, fontdesc_dicts)
+          fd_ref = first_ref(desc_dict["FontDescriptor"])
+          return nil unless fd_ref
+
+          fontdesc_dicts[fd_ref]
+        end
+
+        def resolve_fontfile(fd_dict)
+          if fd_dict.key?("FontFile2")
+            [first_ref(fd_dict["FontFile2"]), :ttf]
+          elsif fd_dict.key?("FontFile3")
+            [first_ref(fd_dict["FontFile3"]), :cff]
+          end
+        end
+
+        def resolve_cid_to_gid(desc_dict)
+          raw = desc_dict["CIDToGIDMap"]
+          return nil if raw.nil?
+
+          raw.to_s == "Identity" ? :identity : nil
+        end
+
+        # ---- mutool subprocess + dict parsing ----------------------------
+
+        def discover_type0_fonts
+          text = mutool_info_text
+          result = {}
+          seen = Set.new
+          text.each_line do |line|
+            next unless line.include?("Type0")
+
+            m = line.match(/Type0\s+'([^']+)'\s+\S+\s+\((\d+)\s+0\s+R\)/)
+            next unless m
+
+            font_obj_id = m[2].to_i
+            next if seen.include?(font_obj_id)
+
+            seen << font_obj_id
+            result[font_obj_id] = m[1]
+          end
+          result
+        end
+
+        def fetch_objects(obj_ids)
+          return {} if obj_ids.empty?
+
+          args = ["mutool", "show", "-g",
+                  @source.pdf_to_s].concat(obj_ids.map(&:to_s))
+          out, err, status = Open3.capture3(*args)
+          unless status.success?
+            raise Ucode::EmbeddedFontsMissingError,
+                  "mutool show failed: #{err.strip}"
+          end
+
+          parse_grep_output(out)
+        end
+
+        def parse_grep_output(text)
+          result = {}
+          text.each_line do |line|
+            m = line.match(/^(\d+)\s+0\s+obj\s+(.*)$/)
+            next unless m
+
+            result[m[1].to_i] = parse_dict(m[2])
+          end
+          result
+        end
+
+        # We don't try to fully parse the PDF dict grammar. Instead we
+        # regex each field we need directly out of the dict body.
+        def parse_dict(body)
+          body = body.to_s
+          {
+            "BaseFont" => field_match(body, %r{/BaseFont/([^\s/<>]+)}),
+            "DescendantFonts" => field_match(body,
+                                             %r{/DescendantFonts\s*\[\s*(\d+)\s+0\s+R\s*\]}),
+            "ToUnicode" => field_match(body, %r{/ToUnicode\s+(\d+)\s+0\s+R}),
+            "FontDescriptor" => field_match(body,
+                                            %r{/FontDescriptor\s+(\d+)\s+0\s+R}),
+            "FontFile2" => field_match(body, %r{/FontFile2\s+(\d+)\s+0\s+R}),
+            "FontFile3" => field_match(body, %r{/FontFile3\s+(\d+)\s+0\s+R}),
+            "CIDToGIDMap" => field_match(body,
+                                         %r{/CIDToGIDMap(?:/([^\s/<>]+)|\s+(\d+)\s+0\s+R)}),
+          }.compact
+        end
+
+        def field_match(body, regex)
+          m = body.match(regex)
+          return nil unless m
+
+          m.captures.compact.first
+        end
+
+        def first_ref(value)
+          return nil if value.nil? || value.empty?
+
+          Integer(value)
+        end
+
+        def mutool_info_text
+          @mutool_info_text ||= run_mutool_info
+        end
+
+        def run_mutool_info
+          out, err, status = Open3.capture3("mutool", "info", @source.pdf_to_s)
+          status.success? ? out + err : ""
+        end
+
+        def font_entries_cache
+          @font_entries_cache ||= begin
+            result = {}
+            mutool_info_text.each_line do |line|
+              next unless line.include?("Type0")
+
+              font_match = line.match(/Type0\s+'([^']+)'/)
+              next unless font_match
+
+              result[font_match[1]] = true
+            end
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/embedded_fonts/{source.rb → pdf_location.rb}

@@ -2,8 +2,6 @@
 
 require "pathname"
 
-require_relative "../../error"
-
 module Ucode
   module Glyphs
     module EmbeddedFonts
@@ -29,7 +27,7 @@ module Ucode
       # the BaseFont (e.g. `CIAIIP+Uni2000Generalpunctuation.ttf`).
       # Re-runs skip extraction when the cached file is newer than the
       # PDF.
-      class Source
+      class PdfLocation
         attr_reader :pdf_path, :cache_dir
 
         # @param pdf [String, Pathname, nil] path to a Code Charts PDF
@@ -41,8 +39,10 @@ module Ucode
         # @raise [Ucode::EmbeddedFontsMissingError] if the PDF is missing
         def initialize(pdf: nil, cache_dir: nil, env: ENV, gem_root: nil)
           @pdf_path = resolve_pdf(pdf, env, gem_root)
-          raise Ucode::EmbeddedFontsMissingError,
-                "Code Charts PDF not found at #{@pdf_path}" unless @pdf_path&.exist?
+          unless @pdf_path&.exist?
+            raise Ucode::EmbeddedFontsMissingError,
+                  "Code Charts PDF not found at #{@pdf_path}"
+          end
 
           @cache_dir = resolve_cache(cache_dir, env, gem_root)
           @cache_dir.mkpath unless @cache_dir.exist?

data/lib/ucode/glyphs/embedded_fonts/positional_matcher.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Glyphs
+    module EmbeddedFonts
+      # Shared positional matching for Code Charts specimen attribution.
+      #
+      # Both {ContentStreamCorrelator} and {TraceCorrelator} need the
+      # same algorithm: given a set of specimen glyphs and a set of hex
+      # codepoint labels with positions, match each specimen to its
+      # nearest valid label cluster by Euclidean distance.
+      #
+      # This module owns that algorithm. The input is format-agnostic —
+      # callers produce {Position} structs from their source format
+      # (SVG `<use>` elements or `mutool trace` XML) and delegate here.
+      #
+      # Handles both Code Charts layouts:
+      #
+      # 1. **List layout** — label to the LEFT of specimen at the same Y.
+      # 2. **Grid layout** — label ABOVE specimen (~12pt higher, same X).
+      #
+      # Greedy one-to-one matching: each GID and each codepoint is
+      # assigned at most once, so a specimen between two labels only
+      # claims the closer one.
+      module PositionalMatcher
+        # Value object: one positioned glyph with text content.
+        # font_ref is the font identifier (Integer obj-id for SVG,
+        # String font-name for trace); used only for partitioning by
+        # the caller, not by the matcher.
+        Position = Struct.new(
+          :x, :y, :font_ref, :glyph_id, :text, keyword_init: true,
+        )
+
+        DEFAULT_Y_BUCKET = 1.0
+        private_constant :DEFAULT_Y_BUCKET
+
+        # Adjacent label chars within one codepoint label are ~4-6 pt
+        # apart on X. Different columns are ~30+ pt apart. 10 pt
+        # cleanly separates within-label from between-column gaps.
+        X_GAP_THRESHOLD = 10.0
+        private_constant :X_GAP_THRESHOLD
+
+        # Maximum valid Unicode codepoint.
+        UNICODE_MAX = 0x10FFFF
+        private_constant :UNICODE_MAX
+
+        # Maximum Euclidean distance from a specimen to its matching
+        # label cluster. List-layout labels are ~21 pt to the left;
+        # grid-layout labels are ~12 pt above. Header/footer text is
+        # always > 30 pt away from any specimen.
+        MAX_MATCH_DISTANCE = 30.0
+        private_constant :MAX_MATCH_DISTANCE
+
+        module_function
+
+        # @param specimens [Array<Position>] positioned specimen glyphs
+        # @param labels [Array<Position>] positioned label chars
+        # @return [Hash{Integer=>Integer}] codepoint => gid
+        def match(specimens, labels)
+          return {} if specimens.empty? || labels.empty?
+
+          clusters = build_label_clusters(labels)
+          return {} if clusters.empty?
+
+          build_mapping(specimens, clusters)
+        end
+
+        # ---- Clustering --------------------------------------------------
+
+        def build_label_clusters(labels)
+          by_y = labels.group_by { |g| quantize(g.y, DEFAULT_Y_BUCKET) }
+
+          by_y.flat_map do |(_, glyphs)|
+            cluster_by_x_gap(glyphs.sort_by(&:x)).filter_map do |cluster|
+              build_cluster(cluster)
+            end
+          end
+        end
+
+        def cluster_by_x_gap(sorted_glyphs)
+          clusters = []
+          current = []
+
+          sorted_glyphs.each do |g|
+            if current.empty? || (g.x - current.last.x).abs < X_GAP_THRESHOLD
+              current << g
+            else
+              clusters << current if current.size > 1
+              current = [g]
+            end
+          end
+          clusters << current if current.size > 1
+          clusters
+        end
+
+        def build_cluster(cluster)
+          hex = cluster.map(&:text).join
+          return nil unless hex.match?(/\A[0-9A-Fa-f]{4,6}\z/)
+
+          cp = hex.to_i(16)
+          return nil unless cp <= UNICODE_MAX
+
+          {
+            x: cluster.sum(&:x) / cluster.size,
+            y: cluster.first.y,
+            codepoint: cp,
+          }
+        end
+
+        # ---- Matching ----------------------------------------------------
+
+        def build_mapping(specimens, clusters)
+          assigned_gids = Set.new
+          assigned_cps = Set.new
+          mapping = {}
+
+          pairs_by_distance(specimens, clusters).each do |spec_idx, cluster_idx, dist|
+            next if dist > MAX_MATCH_DISTANCE
+
+            spec = specimens[spec_idx]
+            cluster = clusters[cluster_idx]
+            next if assigned_gids.include?(spec.glyph_id)
+            next if assigned_cps.include?(cluster[:codepoint])
+
+            assigned_gids << spec.glyph_id
+            assigned_cps << cluster[:codepoint]
+            mapping[cluster[:codepoint]] = spec.glyph_id
+          end
+
+          mapping
+        end
+
+        def pairs_by_distance(specimens, clusters)
+          candidates = Array.new(clusters.size) do |ci|
+            specimen_distances(specimens, clusters, ci)
+          end
+
+          candidates.flatten(1).sort_by { |_, _, dist| dist }
+        end
+
+        def specimen_distances(specimens, clusters, cluster_idx)
+          cluster = clusters[cluster_idx]
+          specimens.each_with_index.map do |spec, spec_idx|
+            [spec_idx, cluster_idx, distance(spec, cluster)]
+          end
+        end
+
+        def distance(spec, cluster)
+          dx = spec.x - cluster[:x]
+          dy = spec.y - cluster[:y]
+          Math.sqrt(dx * dx + dy * dy)
+        end
+
+        def quantize(value, bucket_size)
+          return nil if value.nil?
+
+          (value / bucket_size).round * bucket_size
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/embedded_fonts/raw_font_descriptor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "ostruct"
+
+module Ucode
+  module Glyphs
+    module EmbeddedFonts
+      # Value object: one Type0 font discovered by {PdfIndexer}, carrying
+      # every ref the {CodepointMapper} needs to resolve codepoint → GID.
+      #
+      # Public so tests can construct realistic fixtures without going
+      # through the PDF subprocess layer.
+      RawFontDescriptor = Struct.new(
+        :base_font,
+        :font_obj_id,
+        :fontfile_obj_id,
+        :fontfile_kind,
+        :tounicode_ref,
+        :cid_map_kind,
+        keyword_init: true,
+      )
+    end
+  end
+end

data/lib/ucode/glyphs/embedded_fonts/renderer.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "svg"
-
 module Ucode
   module Glyphs
     module EmbeddedFonts