ucode 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7dbe8b2ddc06a807988316b3800fd3aa886a15f3d6b9249f62770728aa3441c
-  data.tar.gz: 396ac3e45668194220bc26ff3d6e002c4942feb6d6c9a63ce0003a22ccfc1c8e
+  metadata.gz: 2d93808cc35d75099bb2028677c0ece5bcac1985cf93b20dbb3ba83d6161bd7a
+  data.tar.gz: 230980b63347f44f11b0cf5238836c0328aa0d46711baca05fec78830e5f67f3
 SHA512:
-  metadata.gz: 6f28010777955f1a11d03ed10c8edefc83ded96a0a307080b360c1a0918ca01a410e6571644e9cb4f924bbb1a67b676906f4252742b5cdcf1bc25c7df587e730
-  data.tar.gz: 764bd2ff3296fb44d26eabe2003d7e739a5b7baebafd3df80dc3521000958d5ad535f6d73d1eab63ebc7c039f0f8dda9c06d10851d6a8fda1d1a430ab060ea5c
+  metadata.gz: b1e56059471ba2fe204f018c41811f19f20f1cf1569ac9da91004031561f8826c64faba4d7ecd07963d273ca51631ec71589c74860e54521e6e1d30b4ec321f7
+  data.tar.gz: 2f30cd92e0ac72619250363c4b71321241042b3ade8a7eaaf20c8bfffa2c3d79be42414e1b872d1f0375c470a4c9b82825e8a518ec328022d6c6a8096b8211a6

data/config/unicode17_universal_glyph_set.yml

@@ -1,6 +1,6 @@
 ---
 unicode_version: 17.0.0
-ucode_version: 0.5.0
+ucode_version: 0.5.1
 generated_at: '2026-06-28T00:00:00Z'
 default_sources:
 - kind: fontist

data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb

@@ -56,11 +56,17 @@ module Ucode
         #   about block scope).
         # @param force_positional_for_font_ids [Set<Integer>] Type0
         #   font object IDs that always trigger positional attribution
+        # @param trace_cache [PageTraceCache, nil] consulted by
+        #   {#needs_positional?} for the partial-ToUnicode-coverage
+        #   case (the font has glyphs the CMap doesn't list). nil =
+        #   skip that check (legacy behavior).
         def initialize(strategies:, block_range: nil,
-                       force_positional_for_font_ids: Set.new)
+                       force_positional_for_font_ids: Set.new,
+                       trace_cache: nil)
           @strategies = strategies
           @block_range = block_range
           @force_positional_for_font_ids = force_positional_for_font_ids
+          @trace_cache = trace_cache
         end
 
         # Convenience builder — wires up the default 3-strategy chain
@@ -71,7 +77,9 @@ module Ucode
         # @param trace_cache [PageTraceCache, nil] when provided, the
         #   TraceStrategy shares this cache (lets the caller reuse the
         #   traced pages for downstream concerns like Catalog's
-        #   location lookup). nil = construct internally.
+        #   location lookup) AND the orchestrator consults it for
+        #   gap detection (ToUnicode partial-coverage case). nil =
+        #   construct internally.
         # @return [CodepointMapper]
         def self.build(source:, correlator_configs:, indexer:,
                        block_range: nil, force_positional_for_font_ids: Set.new,
@@ -93,7 +101,8 @@ module Ucode
           ]
           new(strategies: strategies,
               block_range: block_range,
-              force_positional_for_font_ids: force_positional_for_font_ids)
+              force_positional_for_font_ids: force_positional_for_font_ids,
+              trace_cache: trace_cache)
         end
 
         # @param descriptor [RawFontDescriptor]
@@ -136,7 +145,7 @@ module Ucode
           {}
         end
 
-        # Positional strategies are gated behind three conditions, any
+        # Positional strategies are gated behind four conditions, any
         # of which triggers them:
         #
         #   1. Caller explicitly listed this font in
@@ -146,10 +155,17 @@ module Ucode
         #   3. The intrinsic result fell entirely outside the
         #      caller's block scope — the font's CMap encoded the
         #      wrong codepoints (Option 1 auto-detect, e.g. U1F200).
+        #   4. The intrinsic result covered fewer glyphs than the
+        #      trace cache observes on the page — the font's CMap is
+        #      partial (Unicode Consortium ships orphan glyphs whose
+        #      CIDs the ToUnicode stream forgot to map, e.g. the
+        #      last glyph of Mongolian Supplement U+1166C and Myanmar
+        #      Extended-A U+116DA in the v17.0 charts).
         def needs_positional?(descriptor, intrinsic_result)
           return true if @force_positional_for_font_ids.include?(descriptor.font_obj_id)
           return true if intrinsic_result.empty?
           return true if intrinsic_out_of_scope?(intrinsic_result)
+          return true if intrinsic_has_uncovered_gids?(descriptor, intrinsic_result)
 
           false
         end
@@ -162,6 +178,21 @@ module Ucode
           intrinsic_result.keys.all? { |cp| !@block_range.include?(cp) }
         end
 
+        # Detects the partial-ToUnicode-coverage case: the font's
+        # CMap names N codepoints, but the page actually renders >N
+        # glyphs from this font (orphan glyphs whose CIDs the CMap
+        # forgot). Positional attribution via the chart's hex labels
+        # is the only way to recover them.
+        def intrinsic_has_uncovered_gids?(descriptor, intrinsic_result)
+          return false unless @trace_cache
+          return false if intrinsic_result.empty?
+
+          covered_gids = intrinsic_result.values.to_set
+          @trace_cache.distinct_gids_for(descriptor.base_font).any? do |gid|
+            !covered_gids.include?(gid)
+          end
+        end
+
         # Positional chain: union of all positional strategies' results.
         # Within positional, earlier strategies win on conflict (chain
         # order expresses caller preference — CorrelatorStrategy

data/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb

@@ -4,6 +4,7 @@ require "pathname"
 
 require "ucode/glyphs/embedded_fonts/mutool"
 require "ucode/glyphs/embedded_fonts/trace_parser"
+require "ucode/glyphs/embedded_fonts/trace_glyph"
 
 module Ucode
   module Glyphs
@@ -24,6 +25,15 @@ module Ucode
       # Lazy: glyphs are only fetched when a strategy first asks for
       # them. PDFs where every font has /ToUnicode never trigger the
       # trace at all.
+      #
+      # ## mutool font-name truncation
+      #
+      # mutool trace emits font names truncated to 31 chars (PDF
+      # base-font-name limit). The BaseFont dict may carry the full
+      # original name (e.g. `HBBJCP+Uni11660Mongoliansupplement`).
+      # All name comparisons in this class go through
+      # {TraceGlyph.name_match?} so the truncation doesn't silently
+      # break lookups for long-named fonts.
       class PageTraceCache
         # @param pdf [Pathname, String]
         # @param page_count [Integer] total pages in the PDF
@@ -55,7 +65,7 @@ module Ucode
           glyphs_by_page.each_with_index do |glyphs, idx|
             next if idx.zero?
 
-            present = glyphs.any? { |g| g.font_name == base_font }
+            present = glyphs.any? { |g| TraceGlyph.name_match?(g.font_name, base_font) }
             next unless present
 
             present_in_any = true
@@ -68,8 +78,27 @@ module Ucode
         # @return [Boolean] true if any page references this font
         def references_font?(base_font)
           glyphs_by_page.any? do |page_glyphs|
-            page_glyphs.any? { |g| g.font_name == base_font }
+            page_glyphs.any? { |g| TraceGlyph.name_match?(g.font_name, base_font) }
+          end
+        end
+
+        # Returns the set of distinct GIDs rendered on any page for
+        # the given font. Used by {CodepointMapper#needs_positional?}
+        # to detect the partial-ToUnicode-coverage case (font ships
+        # more glyphs than its CMap admits).
+        #
+        # @param base_font [String]
+        # @return [Set<Integer>]
+        def distinct_gids_for(base_font)
+          gids = Set.new
+          glyphs_by_page.each do |page_glyphs|
+            page_glyphs.each do |g|
+              next unless TraceGlyph.name_match?(g.font_name, base_font)
+
+              gids << g.gid if g.gid
+            end
           end
+          gids
         end
 
         # Locate the first occurrence of a specific (font, gid) pair
@@ -87,7 +116,7 @@ module Ucode
             next if idx.zero?
 
             match = page_glyphs.find do |g|
-              g.font_name == base_font && g.gid == gid
+              TraceGlyph.name_match?(g.font_name, base_font) && g.gid == gid
             end
             return { page: idx, x: match.x, y: match.y } if match
           end

data/lib/ucode/glyphs/embedded_fonts/trace_correlator.rb

@@ -48,7 +48,7 @@ module Ucode
         private
 
         def select_specimens(trace_glyphs)
-          trace_glyphs.select { |g| g.font_name == @specimen_font_name }
+          trace_glyphs.select { |g| TraceGlyph.name_match?(g.font_name, @specimen_font_name) }
         end
 
         def select_labels(trace_glyphs)
@@ -93,7 +93,7 @@ module Ucode
 
         def select_hex_candidates(trace_glyphs)
           trace_glyphs.select do |g|
-            g.font_name != @specimen_font_name &&
+            !TraceGlyph.name_match?(g.font_name, @specimen_font_name) &&
               g.unicode&.match?(/\A[0-9A-Fa-f]\z/)
           end
         end

data/lib/ucode/glyphs/embedded_fonts/trace_glyph.rb

@@ -3,6 +3,16 @@
 module Ucode
   module Glyphs
     module EmbeddedFonts
+      # mutool trace emits font names truncated to 31 characters
+      # (PDF base-font-name limit, see PDF 32000-1 §7.9.6). The
+      # BaseFont dict, however, may carry the full original name.
+      # `HBBJCP+Uni11660Mongoliansupplement` (34 chars) is emitted
+      # by mutool trace as `HBBJCP+Uni11660Mongoliansupplem`. The
+      # helpers below let callers compare trace-side names against
+      # catalog-side names without tripping on the truncation.
+      TRACE_NAME_LIMIT = 31
+      private_constant :TRACE_NAME_LIMIT
+
       # Value object for one glyph emitted by `mutool trace`.
       #
       # Each `<g>` element in the trace XML maps to one TraceGlyph:
@@ -21,7 +31,31 @@ module Ucode
         :y,
         :unicode,
         keyword_init: true,
-      )
+      ) do
+        class << self
+          # @param name [String, nil] a BaseFont name from `mutool info`
+          #   or a trace-emitted font name
+          # @return [String, nil] the name truncated to the trace limit
+          def normalize_name(name)
+            return nil if name.nil?
+
+            name.length <= TRACE_NAME_LIMIT ? name : name[0, TRACE_NAME_LIMIT]
+          end
+
+          # True when `a` and `b` resolve to the same normalized
+          # name. Treats `nil` as never matching (avoids accidental
+          # collision on missing names).
+          #
+          # @param a [String, nil]
+          # @param b [String, nil]
+          # @return [Boolean]
+          def name_match?(a, b)
+            return false if a.nil? || b.nil?
+
+            normalize_name(a) == normalize_name(b)
+          end
+        end
+      end
     end
   end
 end

data/lib/ucode/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ucode
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ucode
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Ribose Inc.