ucode 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +10 -2
  3. data/README.md +66 -20
  4. data/Rakefile +18 -7
  5. data/config/unicode17_universal_glyph_set.yml +1 -1
  6. data/lib/ucode/audit/reference_factory.rb +1 -1
  7. data/lib/ucode/cli.rb +10 -42
  8. data/lib/ucode/code_chart/extractor.rb +3 -5
  9. data/lib/ucode/code_chart/provenance.rb +6 -5
  10. data/lib/ucode/code_chart/sidecar.rb +2 -2
  11. data/lib/ucode/code_chart/writer.rb +1 -1
  12. data/lib/ucode/code_chart.rb +1 -1
  13. data/lib/ucode/commands/build.rb +3 -26
  14. data/lib/ucode/commands/canonical_build.rb +1 -4
  15. data/lib/ucode/commands/fetch.rb +1 -1
  16. data/lib/ucode/commands/lookup.rb +1 -1
  17. data/lib/ucode/commands/parse.rb +1 -1
  18. data/lib/ucode/commands.rb +0 -1
  19. data/lib/ucode/coordinator/indices.rb +2 -2
  20. data/lib/ucode/error.rb +0 -8
  21. data/lib/ucode/fetch/code_charts.rb +2 -3
  22. data/lib/ucode/fetch/http.rb +12 -14
  23. data/lib/ucode/glyphs/embedded_fonts/catalog.rb +81 -4
  24. data/lib/ucode/glyphs/embedded_fonts/trace_correlator.rb +230 -0
  25. data/lib/ucode/glyphs/embedded_fonts/trace_glyph.rb +27 -0
  26. data/lib/ucode/glyphs/embedded_fonts/trace_parser.rb +50 -0
  27. data/lib/ucode/glyphs/embedded_fonts/trace_runner.rb +53 -0
  28. data/lib/ucode/glyphs/embedded_fonts.rb +4 -0
  29. data/lib/ucode/glyphs/pdf_fetcher.rb +7 -50
  30. data/lib/ucode/glyphs.rb +4 -14
  31. data/lib/ucode/repo/aggregate_writer.rb +1 -1
  32. data/lib/ucode/repo/writers/blocks_writer.rb +13 -13
  33. data/lib/ucode/repo/writers/enums_writer.rb +2 -2
  34. data/lib/ucode/repo/writers/indexes_writer.rb +4 -4
  35. data/lib/ucode/repo/writers/manifest_writer.rb +4 -4
  36. data/lib/ucode/repo/writers/named_sequences_writer.rb +1 -1
  37. data/lib/ucode/repo/writers/planes_writer.rb +17 -17
  38. data/lib/ucode/repo/writers/relationships_writer.rb +1 -1
  39. data/lib/ucode/repo/writers/scripts_writer.rb +6 -6
  40. data/lib/ucode/repo/writers.rb +1 -1
  41. data/lib/ucode/version.rb +1 -1
  42. data/lib/ucode.rb +0 -2
  43. data/ucode.gemspec +6 -1
  44. metadata +10 -19
  45. data/lib/ucode/commands/glyphs.rb +0 -94
  46. data/lib/ucode/glyphs/cell_extractor.rb +0 -130
  47. data/lib/ucode/glyphs/dvisvgm_renderer.rb +0 -29
  48. data/lib/ucode/glyphs/grid.rb +0 -30
  49. data/lib/ucode/glyphs/grid_detector.rb +0 -165
  50. data/lib/ucode/glyphs/monolith_page_map.rb +0 -181
  51. data/lib/ucode/glyphs/mutool_renderer.rb +0 -28
  52. data/lib/ucode/glyphs/page_renderer.rb +0 -221
  53. data/lib/ucode/glyphs/path_bbox.rb +0 -62
  54. data/lib/ucode/glyphs/pdf2svg_renderer.rb +0 -26
  55. data/lib/ucode/glyphs/pdftocairo_renderer.rb +0 -32
  56. data/lib/ucode/glyphs/pipeline.rb +0 -106
  57. data/lib/ucode/glyphs/writer.rb +0 -250
@@ -249,6 +249,7 @@ module Ucode
249
249
  font_obj_id: font_obj_id,
250
250
  tu_ref: tu_ref,
251
251
  cid_map_kind: cid_map_kind,
252
+ base_font: base_font,
252
253
  )
253
254
  return nil if cp_to_gid.empty?
254
255
 
@@ -275,14 +276,23 @@ module Ucode
275
276
  # when no /ToUnicode is present, consult the correlator_configs
276
277
  # registry — if the user supplied a config for this font, render
277
278
  # the relevant page(s) to SVG and run positional correlation.
278
- # Returns an empty hash when neither path produces a map (the
279
- # caller treats that as "skip this font").
280
- def build_codepoint_to_gid(font_obj_id:, tu_ref:, cid_map_kind:)
279
+ # Pillar-2b fallback: when no caller-supplied config either,
280
+ # auto-detect via `mutool trace` parse the structured text
281
+ # trace to build `{codepoint => gid}` from hex labels + specimen
282
+ # positions. Returns an empty hash when none of the paths
283
+ # produce a map (the caller treats that as "skip this font").
284
+ def build_codepoint_to_gid(font_obj_id:, tu_ref:, cid_map_kind:,
285
+ base_font: nil)
281
286
  return {} if cid_map_kind != :identity
282
287
 
283
288
  return codepoint_map_from_tounicode(tu_ref) if tu_ref
284
289
 
285
- codepoint_map_from_correlator(font_obj_id)
290
+ map = codepoint_map_from_correlator(font_obj_id)
291
+ return map unless map.empty?
292
+
293
+ return {} unless base_font
294
+
295
+ codepoint_map_from_trace(base_font, font_obj_id)
286
296
  end
287
297
 
288
298
  def codepoint_map_from_tounicode(tu_ref)
@@ -298,6 +308,73 @@ module Ucode
298
308
  ContentStreamCorrelator.new(config).correlate(svg)
299
309
  end
300
310
 
311
+ # Pillar-2b: auto-detect codepoint → GID via `mutool trace`.
312
+ # For CID-keyed fonts without /ToUnicode and without a
313
+ # caller-supplied correlator config, trace every page of the
314
+ # PDF and positionally match hex labels to specimen glyphs.
315
+ # `mutool info` only reports the first page per font, so tracing
316
+ # all pages is simpler and catches every chart page.
317
+ #
318
+ # Each page is correlated independently to prevent cross-page
319
+ # position interference (page coordinate systems overlap, so
320
+ # a label on page 3 could wrongly match a specimen on page 2).
321
+ # First match wins when a codepoint appears on multiple pages.
322
+ def codepoint_map_from_trace(base_font, _font_obj_id)
323
+ return {} unless font_appears_in_pdf?(base_font)
324
+
325
+ runner = TraceRunner.new(@source.pdf_path)
326
+ correlator = TraceCorrelator.new(specimen_font_name: base_font)
327
+
328
+ (1..page_count).each_with_object({}) do |page, mapping|
329
+ glyphs = runner.trace([page])
330
+ page_mapping = correlator.correlate(glyphs)
331
+ page_mapping.each do |cp, gid|
332
+ mapping[cp] ||= gid
333
+ end
334
+ end
335
+ end
336
+
337
+ def font_appears_in_pdf?(base_font)
338
+ font_entries_cache.key?(base_font)
339
+ end
340
+
341
+ # Lazy cache of {base_font => true} — which fonts `mutool info`
342
+ # reports in this PDF. We only need the key set, not page numbers,
343
+ # because {codepoint_map_from_trace} traces all pages regardless.
344
+ def font_entries_cache
345
+ @font_entries_cache ||= begin
346
+ result = {}
347
+ mutool_info_text.each_line do |line|
348
+ next unless line.include?("Type0")
349
+
350
+ font_match = line.match(/Type0\s+'([^']+)'/)
351
+ next unless font_match
352
+
353
+ result[font_match[1]] = true
354
+ end
355
+ result
356
+ end
357
+ end
358
+
359
+ # Total pages in the PDF, parsed from `mutool info`'s
360
+ # `Pages: N` line. Falls back to the first font page if parsing
361
+ # fails (so we still try at least one page).
362
+ def page_count
363
+ @page_count ||= begin
364
+ m = mutool_info_text.match(/^Pages:\s+(\d+)/)
365
+ m ? m[1].to_i : 1
366
+ end
367
+ end
368
+
369
+ def mutool_info_text
370
+ @mutool_info_text ||= run_mutool_info
371
+ end
372
+
373
+ def run_mutool_info
374
+ out, err, status = Open3.capture3("mutool", "info", @source.pdf_to_s)
375
+ status.success? ? out + err : ""
376
+ end
377
+
301
378
  def resolve_fontfile(fd_dict)
302
379
  if fd_dict.key?("FontFile2")
303
380
  [first_ref(fd_dict["FontFile2"]), :ttf]
@@ -0,0 +1,230 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ucode
4
+ module Glyphs
5
+ module EmbeddedFonts
6
+ # Correlates specimen glyphs (CID font without `/ToUnicode`) to
7
+ # their Unicode codepoints via positional matching against hex
8
+ # codepoint labels on the same chart page.
9
+ #
10
+ # The Unicode Code Charts use two layouts:
11
+ #
12
+ # 1. **List layout** (chart pages): the hex codepoint label (e.g.
13
+ # "10D75") is printed to the LEFT of the specimen glyph at the
14
+ # same Y baseline.
15
+ #
16
+ # 2. **Grid layout** (summary pages): the hex codepoint label is
17
+ # printed directly ABOVE the specimen glyph (~12 pt higher on
18
+ # Y, same X).
19
+ #
20
+ # Both layouts are handled by matching each specimen to the
21
+ # nearest valid label cluster by Euclidean distance, with a
22
+ # maximum match radius that excludes far-away header/footer text.
23
+ #
24
+ # The codepoint labels in every Unicode Code Charts PDF are set
25
+ # in a single dedicated label font (typically ArialNarrow).
26
+ # Character names, headers, and footers use other fonts. To avoid
27
+ # false matches from hex chars in those texts, the correlator
28
+ # auto-detects the label font as the non-specimen font that
29
+ # contributes the most hex-char glyphs.
30
+ #
31
+ # Matching is greedy one-to-one: each GID and each codepoint is
32
+ # assigned at most once, so a specimen that sits between two
33
+ # labels only claims the closer one.
34
+ #
35
+ # Pure logic — no I/O. The caller passes pre-parsed TraceGlyph
36
+ # arrays (typically from {TraceRunner} + {TraceParser}).
37
+ class TraceCorrelator
38
+ DEFAULT_Y_BUCKET = 1.0
39
+ private_constant :DEFAULT_Y_BUCKET
40
+
41
+ # Adjacent label chars within one codepoint label are ~4-6 pt
42
+ # apart on X. Different columns are ~30+ pt apart. 10 pt
43
+ # cleanly separates within-label from between-column gaps.
44
+ X_GAP_THRESHOLD = 10.0
45
+ private_constant :X_GAP_THRESHOLD
46
+
47
+ # Maximum valid Unicode codepoint. Filters out false labels
48
+ # that form hex strings from character-name fragments.
49
+ UNICODE_MAX = 0x10FFFF
50
+ private_constant :UNICODE_MAX
51
+
52
+ # Maximum Euclidean distance from a specimen to its matching
53
+ # label cluster. List-layout labels are ~21 pt to the left;
54
+ # grid-layout labels are ~12 pt above. Header/footer text is
55
+ # always > 30 pt away from any specimen.
56
+ MAX_MATCH_DISTANCE = 30.0
57
+ private_constant :MAX_MATCH_DISTANCE
58
+
59
+ # @param specimen_font_name [String] the BaseFont name of the
60
+ # CID font whose glyphs need correlation
61
+ def initialize(specimen_font_name:)
62
+ @specimen_font_name = specimen_font_name
63
+ @y_bucket = DEFAULT_Y_BUCKET
64
+ end
65
+
66
+ # @param trace_glyphs [Array<TraceGlyph>]
67
+ # @return [Hash{Integer=>Integer}] codepoint => gid
68
+ def correlate(trace_glyphs)
69
+ specimens = trace_glyphs.select { |g| g.font_name == @specimen_font_name }
70
+ return {} if specimens.empty?
71
+
72
+ label_font = detect_label_font(trace_glyphs)
73
+ return {} unless label_font
74
+
75
+ labels = trace_glyphs.select { |g| label_glyph?(g, label_font) }
76
+ return {} if labels.empty?
77
+
78
+ clusters = build_label_clusters(labels)
79
+ return {} if clusters.empty?
80
+
81
+ build_mapping(specimens, clusters)
82
+ end
83
+
84
+ private
85
+
86
+ # The label font is the non-specimen font whose hex-char glyphs
87
+ # appear most often in close proximity to specimen glyphs.
88
+ # Code Charts dedicate one small font to the codepoint labels;
89
+ # body text, headers, and character names use other fonts that
90
+ # may also contain hex chars but are not co-located with
91
+ # specimens (e.g. the index page has thousands of hex chars in
92
+ # MyriadPro-Light but zero specimens).
93
+ LABEL_PROXIMITY_RADIUS = 50.0
94
+ private_constant :LABEL_PROXIMITY_RADIUS
95
+
96
+ def detect_label_font(trace_glyphs)
97
+ specimens = trace_glyphs.select { |g| g.font_name == @specimen_font_name }
98
+ return nil if specimens.empty?
99
+
100
+ non_specimen_hex = non_specimen_hex_glyphs(trace_glyphs)
101
+ return nil if non_specimen_hex.empty?
102
+
103
+ counts = proximity_counts(specimens, non_specimen_hex)
104
+ return nil if counts.empty?
105
+
106
+ counts.max_by { |_, n| n }.first
107
+ end
108
+
109
+ def non_specimen_hex_glyphs(trace_glyphs)
110
+ trace_glyphs.select do |g|
111
+ g.font_name != @specimen_font_name &&
112
+ g.unicode&.match?(/\A[0-9A-Fa-f]\z/)
113
+ end
114
+ end
115
+
116
+ def proximity_counts(specimens, candidates)
117
+ counts = Hash.new(0)
118
+ radius_sq = LABEL_PROXIMITY_RADIUS * LABEL_PROXIMITY_RADIUS
119
+ specimens.each do |spec|
120
+ candidates.each do |g|
121
+ counts[g.font_name] += 1 if within_radius?(spec, g, radius_sq)
122
+ end
123
+ end
124
+ counts
125
+ end
126
+
127
+ def within_radius?(spec, glyph, radius_sq)
128
+ dx = spec.x - glyph.x
129
+ dy = spec.y - glyph.y
130
+ dx * dx + dy * dy < radius_sq
131
+ end
132
+
133
+ def label_glyph?(glyph, label_font)
134
+ glyph.font_name == label_font &&
135
+ glyph.unicode&.match?(/\A[0-9A-Fa-f]\z/)
136
+ end
137
+
138
+ # Cluster labels by Y (row), then by X gap (column within row).
139
+ # Returns a flat array of {x:, y:, codepoint:} clusters.
140
+ def build_label_clusters(labels)
141
+ by_y = labels.group_by { |g| quantize(g.y, @y_bucket) }
142
+ by_y.flat_map { |(_, glyphs)| clusters_from_row(glyphs) }
143
+ end
144
+
145
+ def clusters_from_row(glyphs)
146
+ cluster_by_x_gap(glyphs.sort_by(&:x)).filter_map { |cluster| build_cluster(cluster) }
147
+ end
148
+
149
+ def build_cluster(cluster)
150
+ hex = cluster.map(&:unicode).join
151
+ return nil unless hex.match?(/\A[0-9A-Fa-f]{4,6}\z/)
152
+
153
+ cp = hex.to_i(16)
154
+ return nil unless cp <= UNICODE_MAX
155
+
156
+ {
157
+ x: cluster.sum(&:x) / cluster.size,
158
+ y: cluster.first.y,
159
+ codepoint: cp,
160
+ }
161
+ end
162
+
163
+ def cluster_by_x_gap(sorted_glyphs)
164
+ clusters = []
165
+ current = []
166
+
167
+ sorted_glyphs.each do |g|
168
+ if current.empty? || (g.x - current.last.x).abs < X_GAP_THRESHOLD
169
+ current << g
170
+ else
171
+ clusters << current if current.size > 1
172
+ current = [g]
173
+ end
174
+ end
175
+ clusters << current if current.size > 1
176
+ clusters
177
+ end
178
+
179
+ # Greedy one-to-one matching: each GID and each codepoint is
180
+ # assigned at most once. Candidate pairs are sorted by distance
181
+ # so the closest specimen-label pair always wins.
182
+ def build_mapping(specimens, clusters)
183
+ candidates = Array.new(clusters.size) { |ci| specimen_distances(specimens, clusters, ci) }
184
+
185
+ assigned_gids = Set.new
186
+ assigned_cps = Set.new
187
+ mapping = {}
188
+
189
+ pairs_by_distance(candidates).each do |spec_idx, cluster_idx, dist|
190
+ next if dist > MAX_MATCH_DISTANCE
191
+
192
+ spec = specimens[spec_idx]
193
+ cluster = clusters[cluster_idx]
194
+ next if assigned_gids.include?(spec.gid)
195
+ next if assigned_cps.include?(cluster[:codepoint])
196
+
197
+ assigned_gids << spec.gid
198
+ assigned_cps << cluster[:codepoint]
199
+ mapping[cluster[:codepoint]] = spec.gid
200
+ end
201
+
202
+ mapping
203
+ end
204
+
205
+ def specimen_distances(specimens, clusters, cluster_idx)
206
+ cluster = clusters[cluster_idx]
207
+ specimens.each_with_index.map do |spec, spec_idx|
208
+ [spec_idx, cluster_idx, distance(spec, cluster)]
209
+ end
210
+ end
211
+
212
+ def pairs_by_distance(candidates)
213
+ candidates.flatten(1).sort_by { |_, _, dist| dist }
214
+ end
215
+
216
+ def distance(spec, cluster)
217
+ dx = spec.x - cluster[:x]
218
+ dy = spec.y - cluster[:y]
219
+ Math.sqrt(dx * dx + dy * dy)
220
+ end
221
+
222
+ def quantize(value, bucket_size)
223
+ return nil if value.nil?
224
+
225
+ (value / bucket_size).round * bucket_size
226
+ end
227
+ end
228
+ end
229
+ end
230
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ucode
4
+ module Glyphs
5
+ module EmbeddedFonts
6
+ # Value object for one glyph emitted by `mutool trace`.
7
+ #
8
+ # Each `<g>` element in the trace XML maps to one TraceGlyph:
9
+ #
10
+ # <g unicode="�" glyph="174" x="237.06" y="673.92" adv=".62"/>
11
+ #
12
+ # The `font_name` is inherited from the enclosing `<span>`:
13
+ #
14
+ # <span font="GPJAHB+WolofGaraySansSerif" ...>
15
+ # <g .../>
16
+ # </span>
17
+ TraceGlyph = Struct.new(
18
+ :font_name,
19
+ :gid,
20
+ :x,
21
+ :y,
22
+ :unicode,
23
+ keyword_init: true,
24
+ )
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "nokogiri"
4
+
5
+ module Ucode
6
+ module Glyphs
7
+ module EmbeddedFonts
8
+ # Parses the XML output of `mutool trace <pdf> <page>` into an
9
+ # array of {TraceGlyph} instances.
10
+ #
11
+ # The trace XML uses a flat `<span font="...">` → `<g glyph="..."
12
+ # x="..." y="..." unicode="..."/>` structure. Nokogiri walks
13
+ # the tree; the parser maps each `<g>` to a TraceGlyph,
14
+ # inheriting the font_name from the enclosing span.
15
+ #
16
+ # Pure function — no I/O, no PDF access. Callers inject the XML
17
+ # string (typically from {TraceRunner}).
18
+ module TraceParser
19
+ class << self
20
+ # @param xml [String] raw mutool trace XML
21
+ # @return [Array<TraceGlyph>] one per `<g>` element; empty
22
+ # if the XML is empty or has no `<g>` elements
23
+ def parse(xml)
24
+ return [] if xml.nil? || xml.strip.empty?
25
+
26
+ doc = Nokogiri::XML(xml)
27
+ doc.css("span").flat_map { |span| glyphs_in_span(span) }
28
+ end
29
+
30
+ private
31
+
32
+ def glyphs_in_span(span)
33
+ font_name = span[:font]
34
+ span.css("g").map { |g| build_glyph(font_name, g) }
35
+ end
36
+
37
+ def build_glyph(font_name, g)
38
+ TraceGlyph.new(
39
+ font_name: font_name,
40
+ gid: g[:glyph]&.to_i,
41
+ x: g[:x]&.to_f,
42
+ y: g[:y]&.to_f,
43
+ unicode: g[:unicode],
44
+ )
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "open3"
4
+ require "pathname"
5
+
6
+ require "ucode/error"
7
+
8
+ module Ucode
9
+ module Glyphs
10
+ module EmbeddedFonts
11
+ # Thin I/O wrapper around `mutool trace <pdf> <page>`.
12
+ #
13
+ # Runs mutool on the given pages, captures the XML output,
14
+ # delegates parsing to {TraceParser}, and returns a flat
15
+ # `Array<TraceGlyph>` across all pages.
16
+ #
17
+ # The only class in the trace pipeline that touches the
18
+ # filesystem / spawns subprocesses. Everything upstream
19
+ # (parser, correlator) is pure.
20
+ class TraceRunner
21
+ # @param pdf_path [Pathname, String]
22
+ def initialize(pdf_path)
23
+ @pdf_path = Pathname.new(pdf_path)
24
+ end
25
+
26
+ # @param page_numbers [Array<Integer>] 1-based PDF page numbers
27
+ # @return [Array<TraceGlyph>]
28
+ def trace(page_numbers)
29
+ page_numbers.flat_map { |page| trace_page(page) }
30
+ end
31
+
32
+ private
33
+
34
+ def trace_page(page)
35
+ xml = run_mutool(page)
36
+ TraceParser.parse(xml)
37
+ end
38
+
39
+ def run_mutool(page)
40
+ out, err, status = Open3.capture3(
41
+ "mutool", "trace", @pdf_path.to_s, page.to_s,
42
+ )
43
+ unless status.success?
44
+ raise Ucode::EmbeddedFontsMissingError,
45
+ "mutool trace failed: #{(out + err).strip}"
46
+ end
47
+
48
+ out + err
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
@@ -42,6 +42,10 @@ module Ucode
42
42
  autoload :Catalog, "ucode/glyphs/embedded_fonts/catalog"
43
43
  autoload :ContentStreamCorrelator,
44
44
  "ucode/glyphs/embedded_fonts/content_stream_correlator"
45
+ autoload :TraceGlyph, "ucode/glyphs/embedded_fonts/trace_glyph"
46
+ autoload :TraceParser, "ucode/glyphs/embedded_fonts/trace_parser"
47
+ autoload :TraceCorrelator, "ucode/glyphs/embedded_fonts/trace_correlator"
48
+ autoload :TraceRunner, "ucode/glyphs/embedded_fonts/trace_runner"
45
49
  autoload :Svg, "ucode/glyphs/embedded_fonts/svg"
46
50
  autoload :Renderer, "ucode/glyphs/embedded_fonts/renderer"
47
51
  autoload :Writer, "ucode/glyphs/embedded_fonts/writer"
@@ -1,42 +1,26 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "pathname"
4
- require "open3"
5
4
 
6
5
  require "ucode/cache"
7
6
  require "ucode/fetch/code_charts"
8
- require "ucode/glyphs/monolith_page_map"
9
7
 
10
8
  module Ucode
11
9
  module Glyphs
12
10
  # Resolves a Unicode block to its source PDF on disk.
13
11
  #
14
- # Primary source: the per-block PDF cached at
12
+ # Source: the per-block PDF cached at
15
13
  # `<cache>/<version>/pdfs/U<XXXX>.pdf` (downloaded from
16
14
  # `unicode.org/charts/PDF/` by `Ucode::Fetch::CodeCharts`).
17
- #
18
- # Fallback: slice the page range from the monolith `CodeCharts.pdf`.
19
- # The page range is resolved by `MonolithPageMap` from the PDF's
20
- # bookmark outline, cached under `data/codecharts_page_map.json`.
21
15
  class PdfFetcher
22
16
  # @param version [String] UCD version, used as the cache namespace.
23
- # @param monolith_path [String, Pathname, nil] path to the full
24
- # `CodeCharts.pdf`. Pass nil to disable monolith fallback.
25
- # @param blocks [Array<Ucode::Models::Block>] required for monolith
26
- # fallback — used to match bookmark titles to block first-cps.
27
- # @param page_map_cache [String, Pathname, nil] where to read/write
28
- # the monolith page-map JSON cache.
29
- def initialize(version, monolith_path: nil, blocks: [], page_map_cache: nil)
17
+ def initialize(version)
30
18
  @version = version
31
- @monolith_path = monolith_path && Pathname.new(monolith_path)
32
- @blocks = blocks
33
- @page_map_cache = page_map_cache
34
19
  end
35
20
 
36
21
  # Resolve the per-block PDF for `block_first_cp`, fetching from the
37
22
  # network if missing. Returns the local PDF path, or nil if the
38
- # block's PDF is unavailable (network failure + no monolith, or
39
- # monolith lacks the requested block).
23
+ # block's PDF is unavailable (network failure).
40
24
  #
41
25
  # @param block_first_cp [Integer] first codepoint of the block;
42
26
  # also the PDF's URL slug per unicode.org's naming convention.
@@ -47,9 +31,7 @@ module Ucode
47
31
  return path if path.exist? && !force
48
32
 
49
33
  download(block_first_cp)
50
- return path if path.exist?
51
-
52
- slice_from_monolith(block_first_cp)
34
+ path if path.exist?
53
35
  end
54
36
 
55
37
  private
@@ -65,38 +47,13 @@ module Ucode
65
47
  def download(block_first_cp)
66
48
  Fetch::CodeCharts.call(@version, block_first_cps: [block_first_cp])
67
49
  rescue StandardError => e
68
- # Network failures fall through to monolith fallback. We do not
69
- # swallow programming errors (NoMethodError etc.) — only fetch
70
- # failures (network, checksum, HTTP).
50
+ # Network failures return nil so callers can fall back to other
51
+ # tiers. We do not swallow programming errors (NoMethodError
52
+ # etc.) — only fetch failures (network, checksum, HTTP).
71
53
  return if e.is_a?(Ucode::FetchError)
72
54
 
73
55
  raise
74
56
  end
75
-
76
- def slice_from_monolith(block_first_cp)
77
- return unless @monolith_path&.exist?
78
-
79
- entry = page_map[block_first_cp]
80
- return unless entry && entry.start_page && entry.end_page
81
-
82
- slice_pages(entry.start_page, entry.end_page, per_block_path(block_first_cp))
83
- end
84
-
85
- def page_map
86
- @page_map ||= MonolithPageMap.load(
87
- monolith_path: @monolith_path,
88
- blocks: @blocks,
89
- cache_path: @page_map_cache,
90
- )
91
- end
92
-
93
- def slice_pages(start_page, end_page, out_path)
94
- out_path.dirname.mkpath
95
- cmd = ["pdftk", @monolith_path.to_s, "cat",
96
- "#{start_page}-#{end_page}", "output", out_path.to_s]
97
- _out, status = Open3.capture2e(*cmd)
98
- status.success? ? out_path : nil
99
- end
100
57
  end
101
58
  end
102
59
  end
data/lib/ucode/glyphs.rb CHANGED
@@ -3,24 +3,14 @@
3
3
  module Ucode
4
4
  # Glyphs — converts Code Charts PDF pages into per-codepoint SVGs.
5
5
  #
6
- # Pipeline: fetch per-block PDF render to SVG → detect grid → extract
7
- # cell normalize viewBoxwrite glyph.svg.
6
+ # The current pipeline is the 4-tier sourcing strategy:
7
+ # Tier 1 (real fonts)Pillar 1 (embedded CIDFont + ToUnicode) →
8
+ # Pillar 2 (positional correlation) → Pillar 3 (Last Resort UFO).
9
+ # See {EmbeddedFonts} for Pillar 1 + 2 and {LastResort} for Pillar 3.
8
10
  #
9
11
  # Vector extraction only. NEVER run OCR.
10
12
  module Glyphs
11
13
  autoload :PdfFetcher, "ucode/glyphs/pdf_fetcher"
12
- autoload :PageRenderer, "ucode/glyphs/page_renderer"
13
- autoload :MutoolRenderer, "ucode/glyphs/mutool_renderer"
14
- autoload :Pdf2svgRenderer, "ucode/glyphs/pdf2svg_renderer"
15
- autoload :DvisvgmRenderer, "ucode/glyphs/dvisvgm_renderer"
16
- autoload :PdftocairoRenderer, "ucode/glyphs/pdftocairo_renderer"
17
- autoload :Grid, "ucode/glyphs/grid"
18
- autoload :PathBbox, "ucode/glyphs/path_bbox"
19
- autoload :GridDetector, "ucode/glyphs/grid_detector"
20
- autoload :CellExtractor, "ucode/glyphs/cell_extractor"
21
- autoload :MonolithPageMap, "ucode/glyphs/monolith_page_map"
22
- autoload :Writer, "ucode/glyphs/writer"
23
- autoload :Pipeline, "ucode/glyphs/pipeline"
24
14
  autoload :LastResort, "ucode/glyphs/last_resort"
25
15
  autoload :EmbeddedFonts, "ucode/glyphs/embedded_fonts"
26
16
  autoload :RealFonts, "ucode/glyphs/real_fonts"
@@ -155,4 +155,4 @@ module Ucode
155
155
  end
156
156
  end
157
157
  end
158
- end
158
+ end
@@ -45,12 +45,12 @@ module Ucode
45
45
  path = Paths.blocks_index_path(@output_root)
46
46
  summary = @blocks.map do |block|
47
47
  {
48
- "id" => block.id,
49
- "name" => block.name,
50
- "first_cp" => block.range_first,
51
- "last_cp" => block.range_last,
48
+ "id" => block.id,
49
+ "name" => block.name,
50
+ "first_cp" => block.range_first,
51
+ "last_cp" => block.range_last,
52
52
  "plane_number" => block.plane_number,
53
- "age" => @block_ages[block.id],
53
+ "age" => @block_ages[block.id],
54
54
  }
55
55
  end
56
56
  write_atomic(path, to_pretty_json(summary)) ? 1 : 0
@@ -58,16 +58,16 @@ module Ucode
58
58
 
59
59
  def block_payload(block)
60
60
  to_pretty_json(
61
- "id" => block.id,
62
- "name" => block.name,
63
- "range_first" => block.range_first,
64
- "range_last" => block.range_last,
65
- "plane_number" => block.plane_number,
66
- "age" => @block_ages[block.id],
67
- "codepoint_ids" => (@block_codepoint_ids[block.id] || []),
61
+ "id" => block.id,
62
+ "name" => block.name,
63
+ "range_first" => block.range_first,
64
+ "range_last" => block.range_last,
65
+ "plane_number" => block.plane_number,
66
+ "age" => @block_ages[block.id],
67
+ "codepoint_ids" => @block_codepoint_ids[block.id] || [],
68
68
  )
69
69
  end
70
70
  end
71
71
  end
72
72
  end
73
- end
73
+ end
@@ -27,7 +27,7 @@ module Ucode
27
27
  def write
28
28
  path = Pathname(@output_root).join("enums.json")
29
29
  payload = {
30
- "properties" => @property_aliases.map(&:to_yaml_hash),
30
+ "properties" => @property_aliases.map(&:to_yaml_hash),
31
31
  "property_values" => @property_value_aliases.map(&:to_yaml_hash),
32
32
  }
33
33
  write_atomic(path, to_pretty_json(payload)) ? 1 : 0
@@ -35,4 +35,4 @@ module Ucode
35
35
  end
36
36
  end
37
37
  end
38
- end
38
+ end