ucode 0.2.3 → 0.3.0

data/lib/ucode/glyphs/embedded_fonts/trace_correlator.rb

@@ -7,106 +7,91 @@ module Ucode
       # their Unicode codepoints via positional matching against hex
       # codepoint labels on the same chart page.
       #
-      # The Unicode Code Charts use two layouts:
+      # Adapter for the `mutool trace` XML format: parses {TraceGlyph}
+      # arrays, partitions into specimens and labels, auto-detects the
+      # label font by proximity, then delegates matching to
+      # {PositionalMatcher}.
       #
-      # 1. **List layout** (chart pages): the hex codepoint label (e.g.
-      #    "10D75") is printed to the LEFT of the specimen glyph at the
-      #    same Y baseline.
-      #
-      # 2. **Grid layout** (summary pages): the hex codepoint label is
-      #    printed directly ABOVE the specimen glyph (~12 pt higher on
-      #    Y, same X).
-      #
-      # Both layouts are handled by matching each specimen to the
-      # nearest valid label cluster by Euclidean distance, with a
-      # maximum match radius that excludes far-away header/footer text.
-      #
-      # The codepoint labels in every Unicode Code Charts PDF are set
-      # in a single dedicated label font (typically ArialNarrow).
-      # Character names, headers, and footers use other fonts. To avoid
-      # false matches from hex chars in those texts, the correlator
-      # auto-detects the label font as the non-specimen font that
-      # contributes the most hex-char glyphs.
-      #
-      # Matching is greedy one-to-one: each GID and each codepoint is
-      # assigned at most once, so a specimen that sits between two
-      # labels only claims the closer one.
-      #
-      # Pure logic — no I/O. The caller passes pre-parsed TraceGlyph
-      # arrays (typically from {TraceRunner} + {TraceParser}).
+      # The label font auto-detection is the only piece of "intelligence"
+      # in this adapter — everything else is format translation. The
+      # matching algorithm lives in {PositionalMatcher} and is shared
+      # with {ContentStreamCorrelator}.
       class TraceCorrelator
-        DEFAULT_Y_BUCKET = 1.0
-        private_constant :DEFAULT_Y_BUCKET
-
-        # Adjacent label chars within one codepoint label are ~4-6 pt
-        # apart on X. Different columns are ~30+ pt apart. 10 pt
-        # cleanly separates within-label from between-column gaps.
-        X_GAP_THRESHOLD = 10.0
-        private_constant :X_GAP_THRESHOLD
-
-        # Maximum valid Unicode codepoint. Filters out false labels
-        # that form hex strings from character-name fragments.
-        UNICODE_MAX = 0x10FFFF
-        private_constant :UNICODE_MAX
-
-        # Maximum Euclidean distance from a specimen to its matching
-        # label cluster. List-layout labels are ~21 pt to the left;
-        # grid-layout labels are ~12 pt above. Header/footer text is
-        # always > 30 pt away from any specimen.
-        MAX_MATCH_DISTANCE = 30.0
-        private_constant :MAX_MATCH_DISTANCE
+        # Proximity radius (in PDF points) for counting how often each
+        # non-specimen font's hex-char glyphs appear near a specimen.
+        # Code Charts dedicate one small font to the codepoint labels;
+        # body text and headers are farther away.
+        LABEL_PROXIMITY_RADIUS = 50.0
+        private_constant :LABEL_PROXIMITY_RADIUS
 
         # @param specimen_font_name [String] the BaseFont name of the
         #   CID font whose glyphs need correlation
         def initialize(specimen_font_name:)
           @specimen_font_name = specimen_font_name
-          @y_bucket = DEFAULT_Y_BUCKET
         end
 
         # @param trace_glyphs [Array<TraceGlyph>]
         # @return [Hash{Integer=>Integer}] codepoint => gid
         def correlate(trace_glyphs)
-          specimens = trace_glyphs.select { |g| g.font_name == @specimen_font_name }
+          specimens = select_specimens(trace_glyphs)
           return {} if specimens.empty?
 
-          label_font = detect_label_font(trace_glyphs)
-          return {} unless label_font
-
-          labels = trace_glyphs.select { |g| label_glyph?(g, label_font) }
+          labels = select_labels(trace_glyphs)
           return {} if labels.empty?
 
-          clusters = build_label_clusters(labels)
-          return {} if clusters.empty?
-
-          build_mapping(specimens, clusters)
+          PositionalMatcher.match(
+            specimens.map { |g| to_position(g) },
+            labels.map { |g| to_position(g) },
+          )
         end
 
         private
 
+        def select_specimens(trace_glyphs)
+          trace_glyphs.select { |g| g.font_name == @specimen_font_name }
+        end
+
+        def select_labels(trace_glyphs)
+          label_font = detect_label_font(trace_glyphs)
+          return [] unless label_font
+
+          trace_glyphs.select { |g| hex_char_from?(g, label_font) }
+        end
+
+        def hex_char_from?(glyph, font_name)
+          glyph.font_name == font_name && glyph.unicode&.match?(/\A[0-9A-Fa-f]\z/)
+        end
+
+        def to_position(glyph)
+          PositionalMatcher::Position.new(
+            x: glyph.x,
+            y: glyph.y,
+            font_ref: glyph.font_name,
+            glyph_id: glyph.gid,
+            text: glyph.unicode,
+          )
+        end
+
         # The label font is the non-specimen font whose hex-char glyphs
         # appear most often in close proximity to specimen glyphs.
         # Code Charts dedicate one small font to the codepoint labels;
         # body text, headers, and character names use other fonts that
         # may also contain hex chars but are not co-located with
-        # specimens (e.g. the index page has thousands of hex chars in
-        # MyriadPro-Light but zero specimens).
-        LABEL_PROXIMITY_RADIUS = 50.0
-        private_constant :LABEL_PROXIMITY_RADIUS
-
+        # specimens.
         def detect_label_font(trace_glyphs)
-          specimens = trace_glyphs.select { |g| g.font_name == @specimen_font_name }
+          specimens = select_specimens(trace_glyphs)
           return nil if specimens.empty?
 
-          non_specimen_hex = non_specimen_hex_glyphs(trace_glyphs)
-          return nil if non_specimen_hex.empty?
+          candidates = select_hex_candidates(trace_glyphs)
+          return nil if candidates.empty?
 
-          counts = proximity_counts(specimens, non_specimen_hex)
+          counts = proximity_counts(specimens, candidates)
           return nil if counts.empty?
 
           counts.max_by { |_, n| n }.first
         end
 
-        def non_specimen_hex_glyphs(trace_glyphs)
+        def select_hex_candidates(trace_glyphs)
           trace_glyphs.select do |g|
             g.font_name != @specimen_font_name &&
               g.unicode&.match?(/\A[0-9A-Fa-f]\z/)
@@ -118,112 +103,13 @@ module Ucode
           radius_sq = LABEL_PROXIMITY_RADIUS * LABEL_PROXIMITY_RADIUS
           specimens.each do |spec|
             candidates.each do |g|
-              counts[g.font_name] += 1 if within_radius?(spec, g, radius_sq)
+              dx = spec.x - g.x
+              dy = spec.y - g.y
+              counts[g.font_name] += 1 if dx * dx + dy * dy < radius_sq
             end
           end
           counts
         end
-
-        def within_radius?(spec, glyph, radius_sq)
-          dx = spec.x - glyph.x
-          dy = spec.y - glyph.y
-          dx * dx + dy * dy < radius_sq
-        end
-
-        def label_glyph?(glyph, label_font)
-          glyph.font_name == label_font &&
-            glyph.unicode&.match?(/\A[0-9A-Fa-f]\z/)
-        end
-
-        # Cluster labels by Y (row), then by X gap (column within row).
-        # Returns a flat array of {x:, y:, codepoint:} clusters.
-        def build_label_clusters(labels)
-          by_y = labels.group_by { |g| quantize(g.y, @y_bucket) }
-          by_y.flat_map { |(_, glyphs)| clusters_from_row(glyphs) }
-        end
-
-        def clusters_from_row(glyphs)
-          cluster_by_x_gap(glyphs.sort_by(&:x)).filter_map { |cluster| build_cluster(cluster) }
-        end
-
-        def build_cluster(cluster)
-          hex = cluster.map(&:unicode).join
-          return nil unless hex.match?(/\A[0-9A-Fa-f]{4,6}\z/)
-
-          cp = hex.to_i(16)
-          return nil unless cp <= UNICODE_MAX
-
-          {
-            x: cluster.sum(&:x) / cluster.size,
-            y: cluster.first.y,
-            codepoint: cp,
-          }
-        end
-
-        def cluster_by_x_gap(sorted_glyphs)
-          clusters = []
-          current = []
-
-          sorted_glyphs.each do |g|
-            if current.empty? || (g.x - current.last.x).abs < X_GAP_THRESHOLD
-              current << g
-            else
-              clusters << current if current.size > 1
-              current = [g]
-            end
-          end
-          clusters << current if current.size > 1
-          clusters
-        end
-
-        # Greedy one-to-one matching: each GID and each codepoint is
-        # assigned at most once. Candidate pairs are sorted by distance
-        # so the closest specimen-label pair always wins.
-        def build_mapping(specimens, clusters)
-          candidates = Array.new(clusters.size) { |ci| specimen_distances(specimens, clusters, ci) }
-
-          assigned_gids = Set.new
-          assigned_cps = Set.new
-          mapping = {}
-
-          pairs_by_distance(candidates).each do |spec_idx, cluster_idx, dist|
-            next if dist > MAX_MATCH_DISTANCE
-
-            spec = specimens[spec_idx]
-            cluster = clusters[cluster_idx]
-            next if assigned_gids.include?(spec.gid)
-            next if assigned_cps.include?(cluster[:codepoint])
-
-            assigned_gids << spec.gid
-            assigned_cps << cluster[:codepoint]
-            mapping[cluster[:codepoint]] = spec.gid
-          end
-
-          mapping
-        end
-
-        def specimen_distances(specimens, clusters, cluster_idx)
-          cluster = clusters[cluster_idx]
-          specimens.each_with_index.map do |spec, spec_idx|
-            [spec_idx, cluster_idx, distance(spec, cluster)]
-          end
-        end
-
-        def pairs_by_distance(candidates)
-          candidates.flatten(1).sort_by { |_, _, dist| dist }
-        end
-
-        def distance(spec, cluster)
-          dx = spec.x - cluster[:x]
-          dy = spec.y - cluster[:y]
-          Math.sqrt(dx * dx + dy * dy)
-        end
-
-        def quantize(value, bucket_size)
-          return nil if value.nil?
-
-          (value / bucket_size).round * bucket_size
-        end
       end
     end
   end

data/lib/ucode/glyphs/embedded_fonts/writer.rb

@@ -2,10 +2,6 @@
 
 require "pathname"
 
-require_relative "renderer"
-require_relative "../../repo/atomic_writes"
-require_relative "../../repo/paths"
-
 module Ucode
   module Glyphs
     module EmbeddedFonts

data/lib/ucode/glyphs/embedded_fonts.rb

@@ -36,12 +36,16 @@ module Ucode
     # `mutool info` (font enumeration) and `mutool show -b -o` (raw
     # stream extraction).
     module EmbeddedFonts
-      autoload :Source, "ucode/glyphs/embedded_fonts/source"
+      autoload :PdfLocation, "ucode/glyphs/embedded_fonts/pdf_location"
       autoload :ToUnicode, "ucode/glyphs/embedded_fonts/tounicode"
       autoload :FontEntry, "ucode/glyphs/embedded_fonts/font_entry"
+      autoload :RawFontDescriptor, "ucode/glyphs/embedded_fonts/raw_font_descriptor"
+      autoload :PdfIndexer, "ucode/glyphs/embedded_fonts/pdf_indexer"
+      autoload :CodepointMapper, "ucode/glyphs/embedded_fonts/codepoint_mapper"
       autoload :Catalog, "ucode/glyphs/embedded_fonts/catalog"
       autoload :ContentStreamCorrelator,
                "ucode/glyphs/embedded_fonts/content_stream_correlator"
+      autoload :PositionalMatcher, "ucode/glyphs/embedded_fonts/positional_matcher"
       autoload :TraceGlyph, "ucode/glyphs/embedded_fonts/trace_glyph"
       autoload :TraceParser, "ucode/glyphs/embedded_fonts/trace_parser"
       autoload :TraceCorrelator, "ucode/glyphs/embedded_fonts/trace_correlator"

data/lib/ucode/glyphs/resolver_factory.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Ucode
+  module Glyphs
+    # Single injection point for the 4-tier {Resolver}.
+    #
+    # Both CanonicalBuildCommand and UniversalSet::BuildCommand need the
+    # same shape: open a Database, load the SourceConfig, run a
+    # SourceBuilder, wrap the resulting tier-1 sources in a Resolver.
+    # Extracting it here gives tests one seam to mock (or bypass) and
+    # prevents drift between the two call sites.
+    module ResolverFactory
+      DEFAULT_INSTALL = false
+      private_constant :DEFAULT_INSTALL
+
+      # @param version [String] UCD version, used to open the Database
+      #   when one is not supplied.
+      # @param source_config_path [String, Pathname, nil] override path
+      #   to the Tier 1 font config YAML; nil uses the default.
+      # @param install [Boolean] pass through to SourceBuilder#tier1_sources
+      #   — whether to fontist-install missing fonts eagerly.
+      # @param database [Ucode::Database, nil] an already-open Database,
+      #   to skip re-opening when the caller already has one.
+      # @return [Ucode::Glyphs::Resolver]
+      def self.build(version:, source_config_path: nil,
+                     install: DEFAULT_INSTALL, database: nil)
+        db = database || Ucode::Database.open(version)
+        config = SourceConfig.new(path: resolve_config_path(source_config_path))
+        builder = SourceBuilder.new(config: config, database: db)
+        Resolver.new(sources: builder.tier1_sources(install: install))
+      end
+
+      # @api private
+      def self.resolve_config_path(path)
+        return SourceConfig::DEFAULT_PATH if path.nil?
+        return path if path.is_a?(Pathname)
+
+        Pathname.new(path)
+      end
+      private_class_method :resolve_config_path
+    end
+  end
+end

data/lib/ucode/glyphs/sources/pillar1_embedded_tounicode.rb

@@ -31,7 +31,7 @@ module Ucode
         # @param renderer [EmbeddedFonts::Renderer] the renderer to
         #   delegate to. Callers typically construct it with the
         #   {EmbeddedFonts::Catalog} built from the resolved Code
-        #   Charts {EmbeddedFonts::Source}. To enable pillar-2
+        #   Charts {EmbeddedFonts::PdfLocation}. To enable pillar-2
         #   fallback, that Catalog must be constructed with
         #   +correlator_configs:+.
         def initialize(renderer:)

data/lib/ucode/glyphs.rb

@@ -16,6 +16,7 @@ module Ucode
     autoload :RealFonts, "ucode/glyphs/real_fonts"
     autoload :Source, "ucode/glyphs/source"
     autoload :Resolver, "ucode/glyphs/resolver"
+    autoload :ResolverFactory, "ucode/glyphs/resolver_factory"
     autoload :SourceConfig, "ucode/glyphs/source_config"
     autoload :SourceBuilder, "ucode/glyphs/source_builder"
     autoload :Sources, "ucode/glyphs/sources"

data/lib/ucode/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ucode
-  VERSION = "0.2.3"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ucode
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-02 00:00:00.000000000 Z
+date: 2026-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -328,7 +328,19 @@ files:
 - lib/ucode/commands/universal_set.rb
 - lib/ucode/config.rb
 - lib/ucode/coordinator.rb
+- lib/ucode/coordinator/enrichment.rb
+- lib/ucode/coordinator/enrichment/bidi.rb
+- lib/ucode/coordinator/enrichment/binary.rb
+- lib/ucode/coordinator/enrichment/casing.rb
+- lib/ucode/coordinator/enrichment/cjk.rb
+- lib/ucode/coordinator/enrichment/display.rb
+- lib/ucode/coordinator/enrichment/emoji.rb
+- lib/ucode/coordinator/enrichment/identity.rb
+- lib/ucode/coordinator/enrichment/indic.rb
+- lib/ucode/coordinator/enrichment/names.rb
+- lib/ucode/coordinator/enrichment/segmentation.rb
 - lib/ucode/coordinator/indices.rb
+- lib/ucode/coordinator/range_lookup.rb
 - lib/ucode/database.rb
 - lib/ucode/db_builder.rb
 - lib/ucode/error.rb
@@ -343,10 +355,14 @@ files:
 - lib/ucode/glyphs.rb
 - lib/ucode/glyphs/embedded_fonts.rb
 - lib/ucode/glyphs/embedded_fonts/catalog.rb
+- lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb
 - lib/ucode/glyphs/embedded_fonts/content_stream_correlator.rb
 - lib/ucode/glyphs/embedded_fonts/font_entry.rb
+- lib/ucode/glyphs/embedded_fonts/pdf_indexer.rb
+- lib/ucode/glyphs/embedded_fonts/pdf_location.rb
+- lib/ucode/glyphs/embedded_fonts/positional_matcher.rb
+- lib/ucode/glyphs/embedded_fonts/raw_font_descriptor.rb
 - lib/ucode/glyphs/embedded_fonts/renderer.rb
-- lib/ucode/glyphs/embedded_fonts/source.rb
 - lib/ucode/glyphs/embedded_fonts/svg.rb
 - lib/ucode/glyphs/embedded_fonts/tounicode.rb
 - lib/ucode/glyphs/embedded_fonts/trace_correlator.rb
@@ -372,6 +388,7 @@ files:
 - lib/ucode/glyphs/real_fonts/unicode_17_blocks.rb
 - lib/ucode/glyphs/real_fonts/writer.rb
 - lib/ucode/glyphs/resolver.rb
+- lib/ucode/glyphs/resolver_factory.rb
 - lib/ucode/glyphs/source.rb
 - lib/ucode/glyphs/source_builder.rb
 - lib/ucode/glyphs/source_config.rb