ucode 0.1.0

data/lib/ucode/glyphs/embedded_fonts/catalog.rb

@@ -0,0 +1,372 @@
+# frozen_string_literal: true
+
+require "open3"
+require "pathname"
+
+require_relative "../../error"
+require_relative "font_entry"
+require_relative "tounicode"
+
+module Ucode
+  module Glyphs
+    module EmbeddedFonts
+      # Walks the Code Charts PDF once and builds a global
+      # `{codepoint => FontEntry}` index.
+      #
+      # Discovery uses `mutool info` for the font list (one line per
+      # page-font), then `mutool show -g` to fetch the Type0 font dicts,
+      # their descendant CIDFont dicts, and the FontDescriptors — all in
+      # a handful of batched subprocess calls rather than one per font.
+      #
+      # For each Type0 font we then fetch its ToUnicode CMap stream
+      # (one `mutool show -b -o <tmpfile>` per font — these can't be
+      # batched because each is a separate stream) and parse it into a
+      # `{cid => codepoint}` map. With `/CIDToGIDMap /Identity` (the
+      # only form we currently support), `gid == cid`, so the per-font
+      # map is directly `{codepoint => gid}`.
+      #
+      # When multiple fonts cover the same codepoint (which happens for
+      # a handful of codepoints that appear in multiple blocks), the
+      # first font discovered wins. The discovery order follows the
+      # `mutool info` listing, which is page-major, so the earlier
+      # block's font wins — the expected behavior for the Code Charts.
+      class Catalog
+        # @param source [Source]
+        # @param correlator_configs [Hash{Integer=>ContentStreamCorrelator::Config}]
+        #   maps a Type0 font's PDF object ID to the pillar-2 config to
+        #   use when the font has no /ToUnicode CMap. Empty by default
+        #   — fonts without ToUnicode and without a config are skipped
+        #   (the v0.1 behavior).
+        def initialize(source, correlator_configs: {})
+          @source = source
+          @correlator_configs = correlator_configs
+          @index = nil
+        end
+
+        # @return [Hash{Integer=>FontEntry}] frozen codepoint → entry map
+        def index
+          @index ||= build_index.freeze
+        end
+
+        # @param codepoint [Integer]
+        # @return [FontEntry, nil]
+        def lookup(codepoint)
+          index[codepoint]
+        end
+
+        # @return [Array<Integer>] every codepoint this PDF covers
+        def codepoints
+          index.keys
+        end
+
+        # @return [Integer] number of codepoints covered
+        def size
+          index.size
+        end
+
+        # @return [Integer] number of Type0 fonts discovered
+        def font_count
+          font_entries.size
+        end
+
+        # @return [Array<FontEntry>] every font entry (one per Type0 font)
+        def font_entries
+          @font_entries ||= build_font_entries
+        end
+
+        private
+
+        def build_index
+          idx = {}
+          font_entries.each do |entry|
+            entry.codepoints.each do |cp|
+              idx[cp] ||= entry
+            end
+          end
+          idx
+        end
+
+        # Step 1: parse `mutool info` for the Type0 font list.
+        # Step 2: batch `mutool show -g` to get the Type0 dicts.
+        # Step 3: batch `mutool show -g` for the descendant CIDFont dicts.
+        # Step 4: batch `mutool show -g` for the FontDescriptors.
+        # Step 5: for each font, fetch + parse the ToUnicode CMap.
+        def build_font_entries
+          type0_refs = discover_type0_fonts
+          return [] if type0_refs.empty?
+
+          type0_dicts = fetch_objects(type0_refs.keys)
+          descendant_refs = []
+          tounicode_refs = []
+          type0_refs.each_key do |font_obj_id|
+            d = type0_dicts[font_obj_id] || {}
+            desc_ref = first_ref(d["DescendantFonts"])
+            tu_ref = first_ref(d["ToUnicode"])
+            descendant_refs << desc_ref if desc_ref
+            tounicode_refs << tu_ref if tu_ref
+          end
+
+          descendant_dicts = fetch_objects(descendant_refs)
+          fontdesc_refs = []
+          descendant_dicts.each_value do |d|
+            fd_ref = first_ref(d["FontDescriptor"])
+            fontdesc_refs << fd_ref if fd_ref
+          end
+
+          fontdesc_dicts = fetch_objects(fontdesc_refs)
+
+          # Walk again, now with all dicts in hand, and build entries.
+          entries = []
+          type0_refs.each do |font_obj_id, base_font|
+            entry = build_entry(
+              font_obj_id: font_obj_id,
+              base_font: base_font,
+              type0_dict: type0_dicts[font_obj_id],
+              descendant_dicts: descendant_dicts,
+              fontdesc_dicts: fontdesc_dicts,
+            )
+            entries << entry if entry
+          end
+          entries
+        end
+
+        # Parse `mutool info` output for Type0 fonts.
+        # Format per line: `\t<page>\t(<page_obj> 0 R):\tType0 '<name>' <enc> (<font_obj> 0 R)`
+        # Returns `{font_obj_id => base_font}` preserving first-seen order.
+        def discover_type0_fonts
+          # `mutool info` writes its report to STDERR, not STDOUT.
+          out, err, status = Open3.capture3("mutool", "info", @source.pdf_to_s)
+          unless status.success?
+            raise Ucode::EmbeddedFontsMissingError,
+                  "mutool info failed: #{(out + err).strip}"
+          end
+
+          text = out + err
+          result = {}
+          seen = Set.new
+          text.each_line do |line|
+            next unless line.include?("Type0")
+
+            # Font lines look like: "<page>\t(<pageobj> 0 R):\tType0 '<base>' <enc> (<fontobj> 0 R)"
+            m = line.match(/Type0\s+'([^']+)'\s+\S+\s+\((\d+)\s+0\s+R\)/)
+            next unless m
+
+            base_font = m[1]
+            font_obj_id = m[2].to_i
+            next if seen.include?(font_obj_id)
+
+            seen << font_obj_id
+            result[font_obj_id] = base_font
+          end
+          result
+        end
+
+        # Batch `mutool show -g` for many object numbers at once.
+        # Returns `{obj_id => parsed_dict_hash}`.
+        def fetch_objects(obj_ids)
+          return {} if obj_ids.empty?
+
+          args = ["mutool", "show", "-g",
+                  @source.pdf_to_s].concat(obj_ids.map(&:to_s))
+          out, err, status = Open3.capture3(*args)
+          unless status.success?
+            raise Ucode::EmbeddedFontsMissingError,
+                  "mutool show failed: #{err.strip}"
+          end
+
+          parse_grep_output(out)
+        end
+
+        # Parse the `mutool show -g` output: one `<id> 0 obj <<...>>` per line.
+        # The dictionary body is a flat string of `/Key value` pairs;
+        # value can be a number, name, string, array, or nested dict.
+        # We extract a small set of keys we care about and represent
+        # their values as strings (caller uses helpers like first_ref).
+        def parse_grep_output(text)
+          result = {}
+          text.each_line do |line|
+            m = line.match(/^(\d+)\s+0\s+obj\s+(.*)$/)
+            next unless m
+
+            obj_id = m[1].to_i
+            result[obj_id] = parse_dict(m[2])
+          end
+          result
+        end
+
+        # We don't try to fully parse the PDF dict grammar. Instead we
+        # regex each field we need directly out of the dict body. This
+        # is robust to `<<...>>`/`[...]` nesting and to `/Key/Value`
+        # pairs (no whitespace) that break naive whitespace-split parsers.
+        def parse_dict(body)
+          body = body.to_s
+          {
+            "BaseFont" => field_match(body, %r{/BaseFont/([^\s/<>]+)}),
+            "DescendantFonts" => field_match(body,
+                                             %r{/DescendantFonts\s*\[\s*(\d+)\s+0\s+R\s*\]}),
+            "ToUnicode" => field_match(body, %r{/ToUnicode\s+(\d+)\s+0\s+R}),
+            "FontDescriptor" => field_match(body,
+                                            %r{/FontDescriptor\s+(\d+)\s+0\s+R}),
+            "FontFile2" => field_match(body, %r{/FontFile2\s+(\d+)\s+0\s+R}),
+            "FontFile3" => field_match(body, %r{/FontFile3\s+(\d+)\s+0\s+R}),
+            "CIDToGIDMap" => field_match(body,
+                                         %r{/CIDToGIDMap(?:/([^\s/<>]+)|\s+(\d+)\s+0\s+R)}),
+          }.compact
+        end
+
+        def field_match(body, regex)
+          m = body.match(regex)
+          return nil unless m
+
+          m.captures.compact.first
+        end
+
+        # Cast a captured integer string into an Integer, tolerant of nil.
+        # {parse_dict}'s regexes already extract just the digit run.
+        def first_ref(value)
+          return nil if value.nil? || value.empty?
+
+          Integer(value)
+        end
+
+        def build_entry(font_obj_id:, base_font:, type0_dict:,
+                        descendant_dicts:, fontdesc_dicts:)
+          desc_ref = first_ref(type0_dict["DescendantFonts"])
+          tu_ref = first_ref(type0_dict["ToUnicode"])
+          return nil unless desc_ref
+
+          desc_dict = descendant_dicts[desc_ref] || {}
+          fd_dict = fontdesc_for(desc_dict, fontdesc_dicts)
+          return nil unless fd_dict
+
+          fontfile_obj_id, fontfile_kind = resolve_fontfile(fd_dict)
+          return nil unless fontfile_obj_id
+
+          cid_map_kind = resolve_cid_to_gid(desc_dict)
+          return nil unless cid_map_kind
+
+          cp_to_gid = build_codepoint_to_gid(
+            font_obj_id: font_obj_id,
+            tu_ref: tu_ref,
+            cid_map_kind: cid_map_kind,
+          )
+          return nil if cp_to_gid.empty?
+
+          FontEntry.new(
+            base_font: base_font,
+            font_obj_id: font_obj_id,
+            fontfile_obj_id: fontfile_obj_id,
+            fontfile_kind: fontfile_kind,
+            tounicode_obj_id: tu_ref,
+            cid_to_gid_map: cid_map_kind,
+            codepoint_to_gid: cp_to_gid.freeze,
+            source: @source,
+          )
+        end
+
+        def fontdesc_for(desc_dict, fontdesc_dicts)
+          fd_ref = first_ref(desc_dict["FontDescriptor"])
+          return nil unless fd_ref
+
+          fontdesc_dicts[fd_ref]
+        end
+
+        # Tier-1 path: parse the /ToUnicode CMap. Pillar-2 fallback:
+        # when no /ToUnicode is present, consult the correlator_configs
+        # registry — if the user supplied a config for this font, render
+        # the relevant page(s) to SVG and run positional correlation.
+        # Returns an empty hash when neither path produces a map (the
+        # caller treats that as "skip this font").
+        def build_codepoint_to_gid(font_obj_id:, tu_ref:, cid_map_kind:)
+          return {} if cid_map_kind != :identity
+
+          return codepoint_map_from_tounicode(tu_ref) if tu_ref
+
+          codepoint_map_from_correlator(font_obj_id)
+        end
+
+        def codepoint_map_from_tounicode(tu_ref)
+          cmap_text = fetch_tounicode(tu_ref)
+          build_codepoint_map(ToUnicode.parse(cmap_text), :identity)
+        end
+
+        def codepoint_map_from_correlator(font_obj_id)
+          config = @correlator_configs[font_obj_id]
+          return {} unless config
+
+          svg = render_pages(config.page_numbers)
+          ContentStreamCorrelator.new(config).correlate(svg)
+        end
+
+        def resolve_fontfile(fd_dict)
+          if fd_dict.key?("FontFile2")
+            [first_ref(fd_dict["FontFile2"]), :ttf]
+          elsif fd_dict.key?("FontFile3")
+            [first_ref(fd_dict["FontFile3"]), :cff]
+          end
+        end
+
+        def resolve_cid_to_gid(desc_dict)
+          raw = desc_dict["CIDToGIDMap"]
+          return nil if raw.nil?
+
+          # parse_dict captures the name without the leading slash, so
+          # "/Identity" comes through as "Identity". A stream-form map
+          # is captured as the integer obj id — not supported yet.
+          if raw.to_s == "Identity"
+            :identity
+          end
+        end
+
+        def fetch_tounicode(obj_id)
+          Tempfile.create("ucode-tounicode") do |tmp|
+            tmp.close
+            ok = system("mutool", "show", "-o", tmp.path, "-b",
+                        @source.pdf_to_s, obj_id.to_s,
+                        out: File::NULL, err: File::NULL)
+            unless ok
+              raise Ucode::EmbeddedFontsMissingError,
+                    "mutool show failed for ToUnicode obj=#{obj_id}"
+            end
+
+            File.binread(tmp.path).force_encoding("UTF-8")
+          end
+        end
+
+        # Render the given 1-based PDF pages to a single SVG string
+        # suitable for {ContentStreamCorrelator#correlate}. Each page
+        # is a separate `<svg>...</svg>` document; the correlator's
+        # `<use>` regex tolerates either a single concatenated blob or
+        # multiple documents. Output is captured from mutool's stdout.
+        def render_pages(page_numbers)
+          return "" if page_numbers.nil? || page_numbers.empty?
+
+          out, err, status = run_mutool_draw(page_numbers)
+          unless status.success?
+            raise Ucode::EmbeddedFontsMissingError,
+                  "mutool draw failed: #{err.strip}"
+          end
+
+          out
+        end
+
+        def run_mutool_draw(page_numbers)
+          Open3.capture3(
+            "mutool", "draw", "-F", "svg",
+            @source.pdf_to_s,
+            *page_numbers.map(&:to_s)
+          )
+        end
+
+        def build_codepoint_map(cid_to_cp, cid_map_kind)
+          return {} if cid_to_cp.empty? || cid_map_kind != :identity
+
+          # With /CIDToGIDMap /Identity, gid == cid.
+          cid_to_cp.each_with_object({}) do |(cid, cp), h|
+            h[cp] = cid
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/embedded_fonts/content_stream_correlator.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Glyphs
+    module EmbeddedFonts
+      # Pillar 2 fallback: build a `{codepoint => gid}` map for a Type0
+      # font whose PDF object graph has no `/ToUnicode` CMap stream.
+      #
+      # The Code Charts draw every chart cell as a `<use>` element that
+      # references the font's GID via an `href` of the form
+      # `#font_<font_obj_id>_<gid>`. The chart also prints the row +
+      # column codepoint labels using one or more "label" fonts (small
+      # Latin glyphs) that show the hex codepoint as text. By clustering
+      # the labels positionally (Y-bucket for the row, X-bucket for the
+      # column) we recover the codepoint each cluster represents, then
+      # match each cluster positionally to the specimen glyph at the
+      # same Y/X position.
+      #
+      # The algorithm generalizes the Tai Yo correlator that was tested
+      # against `data/pdfs/U1E6C0.pdf` (50/52 specimen codepoints
+      # matched, with the two missing being layout edge cases). The
+      # bucket sizes are configurable because some blocks use a tighter
+      # grid than others.
+      #
+      # Inputs are deliberately pure: a string of SVG markup plus a
+      # {Config}. The catalog is responsible for sourcing the SVG (by
+      # rendering the relevant PDF page(s) via `mutool draw -F svg`) and
+      # for knowing which font_obj_ids are labels vs specimen on that
+      # page. That keeps this class trivially testable with synthetic
+      # SVG fixtures.
+      class ContentStreamCorrelator
+        # Per-font / per-block configuration.
+        #
+        # @!attribute label_font_ids [Array<Integer>] Type0 font object
+        #   IDs whose glyphs print the hex codepoint labels on the page.
+        # @!attribute specimen_font_id [Integer] Type0 font object ID
+        #   whose glyphs are the specimens we want to attribute.
+        # @!attribute page_numbers [Array<Integer>] 1-based PDF page
+        #   numbers whose content streams reference the specimen font.
+        # @!attribute y_bucket [Float] vertical clustering granularity
+        #   in PDF points. Default 1.5 — matches mutool's text matrix
+        #   granularity for the row labels.
+        # @!attribute x_bucket [Float] horizontal clustering granularity
+        #   in PDF points. Default 50.0 — separates label clusters
+        #   within a row (labels are ~16pt wide, clusters ~60-160pt
+        #   apart).
+        Config = Struct.new(
+          :label_font_ids,
+          :specimen_font_id,
+          :page_numbers,
+          :y_bucket,
+          :x_bucket,
+          keyword_init: true,
+        )
+
+        DEFAULT_Y_BUCKET = 1.5
+        DEFAULT_X_BUCKET = 50.0
+
+        # Internal value object for a parsed `<use>` element. Public so
+        # the spec can construct realistic fixtures without re-implementing
+        # the parser shape.
+        Use = Struct.new(:font_id, :gid, :text, :x, :y, keyword_init: true)
+
+        # @param config [Config]
+        def initialize(config)
+          @config = config
+          @y_bucket = config.y_bucket || DEFAULT_Y_BUCKET
+          @x_bucket = config.x_bucket || DEFAULT_X_BUCKET
+        end
+
+        # @param svg [String] rendered PDF page(s) as SVG markup. May
+        #   contain multiple `<svg>` documents concatenated (one per
+        #   page); the regex scan handles either case.
+        # @return [Hash{Integer=>Integer}] codepoint => gid. Empty if
+        #   no clusters could be matched.
+        def correlate(svg)
+          uses = parse_uses(svg)
+          return {} if uses.empty?
+
+          partition_and_map(uses)
+        end
+
+        private
+
+        def partition_and_map(uses)
+          labels, specimens = partition_uses(uses)
+          return {} if labels.empty? || specimens.empty?
+
+          cp_per_cluster = decode_label_clusters(labels)
+          return {} if cp_per_cluster.empty?
+
+          build_mapping(cp_per_cluster, group_rows(specimens))
+        end
+
+        def partition_uses(uses)
+          labels = uses.select do |u|
+            @config.label_font_ids.include?(u.font_id)
+          end
+          specimens = uses.select { |u| u.font_id == @config.specimen_font_id }
+          [labels, specimens]
+        end
+
+        # Match `<use .../>` elements and pull out the font_obj_id and
+        # gid from the href, plus the text matrix's e and f terms (which
+        # give the X/Y origin). The data-text attribute carries the
+        # show-string as mutool emitted it (HTML-entity-encoded).
+        def parse_uses(svg)
+          svg.scan(%r{<use ([^/>]*?)/>}).filter_map do |(attrs_s)|
+            use_from_attrs(attrs_s)
+          end
+        end
+
+        def use_from_attrs(attrs)
+          font_match = match_font_ref(attrs)
+          return nil unless font_match
+
+          tm = attrs.match(
+            /matrix\([^,]+,[^,]+,[^,]+,[^,]+,([\d.-]+),([\d.-]+)\)/,
+          )
+          return nil unless tm
+
+          build_use(attrs, font_match, tm)
+        end
+
+        def match_font_ref(attrs)
+          href = extract_href(attrs)
+          return nil unless href
+
+          href.match(/#font_(\d+)_(\d+)\z/)
+        end
+
+        def build_use(attrs, font_match, transform)
+          Use.new(
+            font_id: font_match[1].to_i,
+            gid: font_match[2].to_i,
+            text: attrs[/data-text="([^"]*)"/, 1] || "",
+            x: transform[1].to_f,
+            y: transform[2].to_f,
+          )
+        end
+
+        def extract_href(attrs)
+          attrs[/xlink:href="([^"]+)"/, 1] || attrs[/href="([^"]+)"/, 1]
+        end
+
+        # Cluster label uses by quantized (Y, X) position. Within each
+        # cluster, members are sorted by X so that joined text reads
+        # left-to-right (hex codepoint string).
+        def decode_label_clusters(labels)
+          cluster_members = bucket_labels_by_position(labels)
+          decode_each_cluster(cluster_members)
+        end
+
+        def bucket_labels_by_position(labels)
+          clusters = Hash.new { |h, k| h[k] = [] }
+          labels.each do |label|
+            key = [bucket(label.y, @y_bucket), bucket(label.x, @x_bucket)]
+            clusters[key] << label
+          end
+          clusters
+        end
+
+        def decode_each_cluster(clusters)
+          clusters.each_with_object({}) do |(key, members), decoded|
+            text = members.sort_by(&:x).map { |m| decode_entities(m.text) }.join
+            next unless text.match?(/\A[0-9A-Fa-f]{4,6}\z/)
+
+            decoded[key] = text.to_i(16)
+          end
+        end
+
+        # Group any set of uses (labels or specimens) by Y-bucket; sort
+        # each row by X so positional matching is straightforward.
+        def group_rows(uses)
+          rows = Hash.new { |h, k| h[k] = [] }
+          uses.each do |u|
+            rows[bucket(u.y, @y_bucket)] << u
+          end
+          rows.each_value { |v| v.sort_by!(&:x) }
+          rows
+        end
+
+        # Within each Y-row, the rightmost label cluster is the
+        # specimen codepoint; the rightmost specimen glyph is the
+        # specimen GID. The preceding label clusters (if any) are
+        # cross-reference codepoints, matched positionally to the
+        # preceding specimen glyphs in the same row.
+        def build_mapping(cp_per_cluster, specimen_rows)
+          cp_rows = group_cps_by_row(cp_per_cluster)
+          cp_rows.keys.sort.each_with_object({}) do |yb, mapping|
+            assign_row(mapping, cp_rows[yb], specimen_rows[yb] || [])
+          end
+        end
+
+        def assign_row(mapping, cps, glyphs)
+          return if cps.empty? || glyphs.empty?
+
+          mapping[cps.last] = glyphs.last.gid
+          assign_xrefs(mapping, cps[0...-1], glyphs[0...-1])
+        end
+
+        def assign_xrefs(mapping, xref_cps, xref_glyphs)
+          xref_cps.each_with_index do |cp, i|
+            g = xref_glyphs[i]
+            mapping[cp] = g.gid if g
+          end
+        end
+
+        def group_cps_by_row(cp_per_cluster)
+          rows = Hash.new { |h, k| h[k] = [] }
+          cp_per_cluster.each do |(yb, xb), cp|
+            rows[yb] << [cp, xb]
+          end
+          rows.each_value { |v| v.sort_by! { |_, xb| xb } }
+          rows.transform_values { |v| v.map(&:first) }
+        end
+
+        def bucket(value, size)
+          (value / size).round * size
+        end
+
+        def decode_entities(text)
+          text.gsub(/&#x([0-9a-fA-F]+);/) { [$1.to_i(16)].pack("U") }
+        end
+      end
+    end
+  end
+end