ucode 0.3.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +45 -6
  3. data/config/unicode17_universal_glyph_set.yml +1 -1
  4. data/lib/ucode/cli.rb +201 -32
  5. data/lib/ucode/code_chart/batch_runner.rb +184 -0
  6. data/lib/ucode/code_chart/block_index.rb +89 -0
  7. data/lib/ucode/code_chart/coverage_gap_index.rb +115 -0
  8. data/lib/ucode/code_chart/extractor.rb +65 -13
  9. data/lib/ucode/code_chart/fetcher.rb +112 -0
  10. data/lib/ucode/code_chart/gap_analyzer/block_gap.rb +35 -0
  11. data/lib/ucode/code_chart/gap_analyzer/essenfont_manifest.rb +60 -0
  12. data/lib/ucode/code_chart/gap_analyzer/manifest.rb +39 -0
  13. data/lib/ucode/code_chart/gap_analyzer.rb +83 -0
  14. data/lib/ucode/code_chart/provenance.rb +99 -58
  15. data/lib/ucode/code_chart/sidecar.rb +3 -4
  16. data/lib/ucode/code_chart/verifier/builder.rb +49 -0
  17. data/lib/ucode/code_chart/verifier/mutool_strategy.rb +80 -0
  18. data/lib/ucode/code_chart/verifier/page_render_cache.rb +63 -0
  19. data/lib/ucode/code_chart/verifier/result.rb +27 -0
  20. data/lib/ucode/code_chart/verifier/resvg_strategy.rb +71 -0
  21. data/lib/ucode/code_chart/verifier/strategy.rb +73 -0
  22. data/lib/ucode/code_chart/verifier.rb +152 -0
  23. data/lib/ucode/code_chart/writer.rb +21 -14
  24. data/lib/ucode/code_chart.rb +6 -0
  25. data/lib/ucode/error.rb +16 -1
  26. data/lib/ucode/fetch/http.rb +23 -3
  27. data/lib/ucode/glyphs/embedded_fonts/catalog.rb +66 -3
  28. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/correlator_strategy.rb +46 -0
  29. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/strategy.rb +54 -0
  30. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/tounicode_strategy.rb +41 -0
  31. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/trace_strategy.rb +64 -0
  32. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb +150 -90
  33. data/lib/ucode/glyphs/embedded_fonts/mutool/draw.rb +26 -0
  34. data/lib/ucode/glyphs/embedded_fonts/mutool/info.rb +23 -0
  35. data/lib/ucode/glyphs/embedded_fonts/mutool/show.rb +36 -0
  36. data/lib/ucode/glyphs/embedded_fonts/mutool/trace.rb +26 -0
  37. data/lib/ucode/glyphs/embedded_fonts/mutool.rb +30 -0
  38. data/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb +112 -0
  39. data/lib/ucode/glyphs/embedded_fonts/pdf_indexer.rb +50 -56
  40. data/lib/ucode/glyphs/embedded_fonts/{pdf_location.rb → pdf_source.rb} +1 -1
  41. data/lib/ucode/glyphs/embedded_fonts/renderer.rb +19 -2
  42. data/lib/ucode/glyphs/embedded_fonts/trace_runner.rb +12 -28
  43. data/lib/ucode/glyphs/embedded_fonts.rb +3 -1
  44. data/lib/ucode/glyphs/real_fonts/coverage_auditor.rb +0 -4
  45. data/lib/ucode/glyphs/real_fonts/font_coverage_report.rb +0 -2
  46. data/lib/ucode/glyphs/real_fonts/writer.rb +0 -2
  47. data/lib/ucode/glyphs/source.rb +11 -1
  48. data/lib/ucode/glyphs/sources/pillar1_embedded_tounicode.rb +11 -6
  49. data/lib/ucode/version.rb +1 -1
  50. data/lib/ucode.rb +2 -0
  51. metadata +28 -74
  52. data/TODO.extract-code-chart/01-pdf-fetch-validation.md +0 -80
  53. data/TODO.extract-code-chart/02-block-name-resolver.md +0 -68
  54. data/TODO.extract-code-chart/03-codechart-namespace.md +0 -82
  55. data/TODO.extract-code-chart/04-codechart-extractor.md +0 -154
  56. data/TODO.extract-code-chart/05-provenance-and-sidecar.md +0 -147
  57. data/TODO.extract-code-chart/06-codechart-writer.md +0 -134
  58. data/TODO.extract-code-chart/07-codechart-cli.md +0 -135
  59. data/TODO.extract-code-chart/08-specs.md +0 -87
  60. data/TODO.full/00-README.md +0 -116
  61. data/TODO.full/01-panglyph-vision.md +0 -112
  62. data/TODO.full/02-panglyph-repo-bootstrap.md +0 -184
  63. data/TODO.full/03-panglyph-font-builder.md +0 -201
  64. data/TODO.full/04-panglyph-publish-pipeline.md +0 -126
  65. data/TODO.full/05-ucode-0-1-1-release.md +0 -139
  66. data/TODO.full/06-fontisan-remove-audit.md +0 -142
  67. data/TODO.full/07-fontisan-remove-ucd.md +0 -125
  68. data/TODO.full/08-archive-private-bin-build.md +0 -143
  69. data/TODO.full/09-archive-public-structure.md +0 -164
  70. data/TODO.full/10-fontist-org-woff-glyphs.md +0 -131
  71. data/TODO.full/11-fontist-org-audit-coverage.md +0 -140
  72. data/TODO.full/12-implementation-order.md +0 -216
  73. data/TODO.full/13-fontisan-font-writer-api.md +0 -189
  74. data/TODO.full/14-fontisan-table-writers.md +0 -66
  75. data/TODO.full/15-panglyph-builder-real.md +0 -82
  76. data/TODO.full/16-archive-public-sync-workflows.md +0 -167
  77. data/TODO.full/17-fontist-org-font-picker.md +0 -73
  78. data/TODO.full/18-comprehensive-spec-coverage.md +0 -64
  79. data/TODO.full/19-ucode-0-1-2-patch.md +0 -32
  80. data/TODO.full/20-fontisan-0-2-23-release.md +0 -52
  81. data/TODO.new/00-README.md +0 -96
  82. data/TODO.new/01-pillar-terminology-alignment.md +0 -69
  83. data/TODO.new/02-audit-schema-design.md +0 -255
  84. data/TODO.new/03-directory-output-spec.md +0 -203
  85. data/TODO.new/04-fontist-org-contract.md +0 -173
  86. data/TODO.new/05-baseline-unicode17-coverage-audit.md +0 -144
  87. data/TODO.new/06-audit-namespace-skeleton.md +0 -105
  88. data/TODO.new/07-audit-models-port.md +0 -132
  89. data/TODO.new/08-extractors-cheap-port.md +0 -113
  90. data/TODO.new/09-extractors-expensive-port.md +0 -99
  91. data/TODO.new/10-aggregations-ucd-rewrite.md +0 -168
  92. data/TODO.new/11-differ-and-library-auditor-port.md +0 -102
  93. data/TODO.new/12-formatters-port.md +0 -115
  94. data/TODO.new/13-directory-emitter.md +0 -147
  95. data/TODO.new/14-html-face-browser.md +0 -144
  96. data/TODO.new/15-html-library-browser.md +0 -102
  97. data/TODO.new/16-cli-audit-subcommands.md +0 -142
  98. data/TODO.new/17-fontisan-cleanup-audit.md +0 -147
  99. data/TODO.new/18-fontisan-cleanup-ucd.md +0 -156
  100. data/TODO.new/19-fontisan-docs-update.md +0 -155
  101. data/TODO.new/20-canonical-resolver-4-tier.md +0 -182
  102. data/TODO.new/21-canonical-unicode17-build.md +0 -148
  103. data/TODO.new/22-implementation-order.md +0 -176
  104. data/TODO.new/23-universal-glyph-set-source-map.md +0 -312
  105. data/TODO.new/24-universal-glyph-set-build.md +0 -189
  106. data/TODO.new/25-font-audit-against-universal-set.md +0 -195
  107. data/TODO.new/26-missing-glyph-reporter.md +0 -189
  108. data/TODO.new/27-fontist-org-consumer-integration.md +0 -200
  109. data/TODO.new/28-implementation-order-update.md +0 -187
  110. data/TODO.new/29-universal-set-curation-uc17.md +0 -312
  111. data/TODO.new/30-tier1-font-acquisition.md +0 -241
  112. data/TODO.new/31-universal-set-production-build.md +0 -205
  113. data/TODO.new/32-uc17-coverage-matrix.md +0 -165
  114. data/TODO.new/33-specialist-font-acquisition-refresh.md +0 -138
  115. data/TODO.new/34-pillar2-content-stream-correlator.md +0 -147
  116. data/TODO.new/35-universal-set-production-run.md +0 -160
  117. data/TODO.new/36-per-font-coverage-audit.md +0 -145
  118. data/TODO.new/37-coverage-highlight-reporter.md +0 -125
  119. data/TODO.new/38-fontist-org-glyph-consumer.md +0 -141
  120. data/TODO.new/39-implementation-order-update-32-38.md +0 -258
  121. data/TODO.new/40-archive-private-uses-ucode-audit.md +0 -124
  122. data/TODO.new/41-ucode-unicode-archive-bridge.md +0 -160
@@ -0,0 +1,152 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "pathname"
5
+
6
+ module Ucode
7
+ module CodeChart
8
+ # Pixel-diff verification for extracted SVG glyphs against the
9
+ # source PDF cell. REQ R4.
10
+ #
11
+ # ## Strategy chain (OCP)
12
+ #
13
+ # One renderer strategy per external tool. The strategy exposes
14
+ # three primitive operations: render an SVG to PNG, render a PDF
15
+ # page region to PNG, and compute a pixel-diff percentage between
16
+ # two PNGs. {Verifier} orchestrates these primitives; it has no
17
+ # knowledge of any specific CLI tool.
18
+ #
19
+ # Adding a new renderer (cairo, imagemagick, …) = one Strategy
20
+ # subclass + one entry in {Verifier::Builder.pick}. Verifier core
21
+ # never changes.
22
+ #
23
+ # ## Result types (see Verifier::Result)
24
+ #
25
+ # * `Pass` — diff < FAIL_THRESHOLD (default 1.0%).
26
+ # * `Fail` — diff ≥ threshold; carries the percent + diff
27
+ # artifact path for inspection.
28
+ # * `Skipped` — extractor produced no `source_page`/`source_cell`
29
+ # (e.g. ToUnicode-only path, or Last Resort placeholder), so no
30
+ # honest cell-diff is possible. Surfaces as a warning, NOT a
31
+ # pass.
32
+ class Verifier
33
+ autoload :Strategy, "ucode/code_chart/verifier/strategy"
34
+ autoload :ResvgStrategy, "ucode/code_chart/verifier/resvg_strategy"
35
+ autoload :MutoolStrategy, "ucode/code_chart/verifier/mutool_strategy"
36
+ autoload :Builder, "ucode/code_chart/verifier/builder"
37
+ autoload :PageRenderCache, "ucode/code_chart/verifier/page_render_cache"
38
+ autoload :Result, "ucode/code_chart/verifier/result"
39
+
40
+ DEFAULT_CELL_SIZE = 40.0
41
+ private_constant :DEFAULT_CELL_SIZE
42
+
43
+ # @param strategy [Strategy, nil] nil = {Builder.pick} auto.
44
+ # @param diff_dir [Pathname, String] where diff artifacts land
45
+ # @param threshold [Float, nil] override {Strategy::FAIL_THRESHOLD}
46
+ def initialize(diff_dir:, strategy: nil, threshold: nil)
47
+ @strategy = strategy || Builder.pick
48
+ @diff_dir = Pathname.new(diff_dir)
49
+ @threshold = threshold || (Strategy::FAIL_THRESHOLD if @strategy)
50
+ if @strategy
51
+ @page_cache = PageRenderCache.new(diff_dir: @diff_dir.join(".cache"),
52
+ strategy: @strategy)
53
+ end
54
+ end
55
+
56
+ # @return [Boolean] true iff a usable strategy was found
57
+ def available?
58
+ !@strategy.nil? && @strategy.available?
59
+ end
60
+
61
+ # @param result [Ucode::CodeChart::Extractor::Result]
62
+ # @param pdf_path [Pathname, String, nil] source PDF
63
+ # @return [Result::Pass, Result::Fail, Result::Skipped]
64
+ def verify(result, pdf_path:)
65
+ unless @strategy
66
+ return Result::Skipped.new(codepoint: result.codepoint,
67
+ reason: :no_strategy)
68
+ end
69
+
70
+ loc = extract_location(result)
71
+ unless loc
72
+ return Result::Skipped.new(codepoint: result.codepoint,
73
+ reason: :no_location)
74
+ end
75
+
76
+ path = Pathname.new(pdf_path)
77
+ unless path.exist?
78
+ return Result::Skipped.new(codepoint: result.codepoint,
79
+ reason: :no_pdf)
80
+ end
81
+
82
+ verify_with_location(result, path, loc)
83
+ end
84
+
85
+ private
86
+
87
+ def extract_location(result)
88
+ return nil unless result.source_page && result.source_cell
89
+
90
+ {
91
+ page: result.source_page,
92
+ x: result.source_cell[:x],
93
+ y: result.source_cell[:y],
94
+ }
95
+ end
96
+
97
+ def verify_with_location(result, pdf_path, loc)
98
+ svg_png = render_svg(result)
99
+ cell_png = render_cell(pdf_path, loc)
100
+
101
+ percent = @strategy.diff(svg_png, cell_png)
102
+ if percent < @threshold
103
+ Result::Pass.new(codepoint: result.codepoint, percent: percent)
104
+ else
105
+ artifact = write_artifact(result, svg_png, cell_png)
106
+ Result::Fail.new(codepoint: result.codepoint,
107
+ percent: percent,
108
+ diff_path: artifact)
109
+ end
110
+ ensure
111
+ [svg_png, cell_png].each { |p| FileUtils.rm_f(p.to_s) if p&.exist? }
112
+ end
113
+
114
+ def render_svg(result)
115
+ svg_path = @diff_dir.join("#{format_cp(result.codepoint)}.svg")
116
+ svg_path.dirname.mkpath
117
+ svg_path.write(result.svg)
118
+ png_path = Pathname.new("#{svg_path}.png")
119
+ @strategy.render_svg(svg_path, png_path)
120
+ png_path
121
+ ensure
122
+ FileUtils.rm_f(svg_path.to_s) if svg_path&.exist?
123
+ end
124
+
125
+ def render_cell(pdf_path, loc)
126
+ rect = cell_rect(loc)
127
+ png_path = @diff_dir.join(".cache",
128
+ "cell-#{loc[:page]}-" \
129
+ "#{format('%05.2f', loc[:x])}-" \
130
+ "#{format('%05.2f', loc[:y])}.png")
131
+ png_path.dirname.mkpath
132
+ @strategy.render_pdf_region(pdf_path, loc[:page], rect, png_path)
133
+ png_path
134
+ end
135
+
136
+ def cell_rect(loc)
137
+ half = DEFAULT_CELL_SIZE / 2
138
+ { x: loc[:x] - half, y: loc[:y] - half,
139
+ w: DEFAULT_CELL_SIZE, h: DEFAULT_CELL_SIZE }
140
+ end
141
+
142
+ def write_artifact(result, png_a, png_b)
143
+ dest = @diff_dir.join("#{format_cp(result.codepoint)}.diff.png")
144
+ @strategy.write_diff_artifact(png_a, png_b, dest) || dest
145
+ end
146
+
147
+ def format_cp(codepoint)
148
+ "U+#{codepoint.to_s(16).upcase.rjust(4, '0')}"
149
+ end
150
+ end
151
+ end
152
+ end
@@ -3,12 +3,6 @@
3
3
  require "digest"
4
4
  require "pathname"
5
5
 
6
- require "ucode/code_chart/extractor"
7
- require "ucode/code_chart/provenance"
8
- require "ucode/code_chart/sidecar"
9
- require "ucode/error"
10
- require "ucode/version_resolver"
11
-
12
6
  module Ucode
13
7
  module CodeChart
14
8
  # Orchestrates extraction + provenance sidecar writing for one
@@ -47,12 +41,19 @@ module Ucode
47
41
  # @param ucd_version [String, nil] UCD version to stamp on
48
42
  # provenance. nil = resolved via {VersionResolver.resolve(nil)}.
49
43
  # @param cache_dir [Pathname, String, nil] font-stream cache
50
- # directory for the EmbeddedFonts::PdfLocation.
44
+ # directory for the EmbeddedFonts::PdfSource.
51
45
  # @param now [Time, nil] timestamp override (for tests).
52
- # @param pillar3_source, tier1_sources: forwarded to the Extractor.
46
+ # @param pillar3_source, tier1_sources, assigned_only, codepoints:
47
+ # forwarded to the Extractor.
48
+ # @param extractor [Ucode::CodeChart::Extractor, nil] pre-built
49
+ # Extractor instance. When nil (default), the Writer constructs
50
+ # one from the other parameters. Injecting lets callers (tests)
51
+ # bypass the mutool-dependent default Extractor.
53
52
  def initialize(output_root:, pdf_path:, ucd_version: nil,
54
53
  cache_dir: nil, now: nil,
55
- pillar3_source: nil, tier1_sources: nil)
54
+ pillar3_source: nil, tier1_sources: nil,
55
+ assigned_only: false, codepoints: nil,
56
+ extractor: nil)
56
57
  @output_root = Pathname.new(output_root)
57
58
  @pdf_path = Pathname.new(pdf_path)
58
59
  @ucd_version = ucd_version || VersionResolver.resolve(nil)
@@ -60,6 +61,9 @@ module Ucode
60
61
  @now = now
61
62
  @pillar3_source = pillar3_source
62
63
  @tier1_sources = tier1_sources
64
+ @assigned_only = assigned_only
65
+ @codepoints = codepoints
66
+ @extractor = extractor
63
67
  end
64
68
 
65
69
  # Extracts every codepoint in `block` and writes `<block_id>/<cp>.svg`
@@ -72,27 +76,30 @@ module Ucode
72
76
  block_dir = @output_root.join(block.id)
73
77
  block_dir.mkpath
74
78
 
75
- pdf_sha = CodeChart.sha256_of(@pdf_path)
79
+ pdf_sha = Provenance.sha256_of(@pdf_path)
76
80
 
77
81
  sidecar = Sidecar.new(output_root: block_dir)
78
- extractor = Extractor.new(
82
+ extractor = @extractor || Extractor.new(
79
83
  block: block,
80
84
  pdf_path: @pdf_path,
81
85
  cache_dir: @cache_dir,
82
86
  pillar3_source: @pillar3_source,
83
87
  tier1_sources: @tier1_sources,
88
+ assigned_only: @assigned_only,
89
+ codepoints: @codepoints,
84
90
  )
85
-
86
91
  results = extractor.extract
87
92
  svgs = 0
88
93
  sidecars = 0
89
94
  results.each do |result|
90
95
  write_svg(block_dir, result)
91
96
  svgs += 1
92
- provenance = CodeChart.build(
97
+ provenance = Provenance.build(
93
98
  block: block, codepoint: result.codepoint,
94
99
  ucd_version: @ucd_version, pdf_path: @pdf_path,
95
- now: @now,
100
+ pdf_sha: pdf_sha, now: @now,
101
+ base_font: result.base_font, gid: result.gid,
102
+ source_page: result.source_page, source_cell: result.source_cell,
96
103
  )
97
104
  sidecar.write(provenance)
98
105
  sidecars += 1
@@ -31,9 +31,15 @@ module Ucode
31
31
  # (e.g. a future pure-Ruby PDF parser per ADR-0001) does not
32
32
  # change the public API.
33
33
  module CodeChart
34
+ autoload :BatchRunner, "ucode/code_chart/batch_runner"
35
+ autoload :BlockIndex, "ucode/code_chart/block_index"
36
+ autoload :CoverageGapIndex, "ucode/code_chart/coverage_gap_index"
34
37
  autoload :Extractor, "ucode/code_chart/extractor"
38
+ autoload :Fetcher, "ucode/code_chart/fetcher"
39
+ autoload :GapAnalyzer, "ucode/code_chart/gap_analyzer"
35
40
  autoload :Provenance, "ucode/code_chart/provenance"
36
41
  autoload :Sidecar, "ucode/code_chart/sidecar"
42
+ autoload :Verifier, "ucode/code_chart/verifier"
37
43
  autoload :Writer, "ucode/code_chart/writer"
38
44
  end
39
45
  end
data/lib/ucode/error.rb CHANGED
@@ -110,12 +110,27 @@ module Ucode
110
110
  # `mutool` is not installed on the PATH.
111
111
  class EmbeddedFontsMissingError < GlyphError; end
112
112
 
113
+ # A `mutool` subprocess exited non-zero. Distinct from
114
+ # {EmbeddedFontsMissingError}: this fires when mutool ran but failed
115
+ # (corrupt PDF, bad object ref, etc.). The context carries the argv.
116
+ class MutoolError < GlyphError; end
117
+
118
+ # CodeChart feature errors — concerns under `Ucode::CodeChart::*`.
119
+ # Distinct from the generic {GlyphError} hierarchy so callers can
120
+ # scope rescue clauses to feature failures.
121
+ class CodeChartError < GlyphError; end
122
+
113
123
  # The Code Charts PDF for a requested block cannot be obtained: the
114
124
  # network returned 4xx/5xx, the response wasn't application/pdf, or
115
125
  # the body didn't start with the `%PDF` magic. Distinct from
116
126
  # {EmbeddedFontsMissingError} (which fires when the file is already
117
127
  # on disk and we just can't open it): this fires at fetch time.
118
- class CodeChartNotFoundError < GlyphError; end
128
+ class CodeChartNotFoundError < CodeChartError; end
129
+
130
+ # A cached Code Charts PDF failed sha256 verification — the file on
131
+ # disk doesn't match the sidecar hash from the previous successful
132
+ # download. Implies external tampering or a filesystem fault.
133
+ class CodeChartChecksumError < CodeChartError; end
119
134
 
120
135
  # Pre-build validation failed for a universal-set build. The
121
136
  # context carries the failing checks so the CLI can render a
@@ -29,11 +29,18 @@ module Ucode
29
29
  # {Ucode::CodeChartNotFoundError} with the offending header
30
30
  # value in `context:` on failure. nil = no validation (the
31
31
  # default for non-PDF callers like UcdZip and UnihanZip).
32
+ # @param not_found_class [Class, nil] when set, HTTP 4xx
33
+ # responses raise this class (instantiated with message +
34
+ # context) instead of being treated as retriable transport
35
+ # errors. nil = 4xx is retriable like any other non-success.
32
36
  # @return [Pathname] destination path on success.
33
37
  # @raise [Ucode::NetworkError] if all retries fail.
34
38
  # @raise [Ucode::CodeChartNotFoundError] when `validate: :pdf`
35
39
  # and the response fails content validation.
36
- def get(url, dest:, retries: nil, timeout: nil, validate: nil)
40
+ # @raise [<not_found_class>] when `not_found_class:` is set
41
+ # and the server returns 4xx.
42
+ def get(url, dest:, retries: nil, timeout: nil, validate: nil,
43
+ not_found_class: nil)
37
44
  uri = url.is_a?(URI) ? url : URI(url)
38
45
  destination = Pathname.new(dest)
39
46
  destination.dirname.mkpath
@@ -44,7 +51,8 @@ module Ucode
44
51
 
45
52
  last_error = nil
46
53
  (attempts + 1).times do |attempt|
47
- response = stream_to(uri, destination, read_timeout)
54
+ response = stream_to(uri, destination, read_timeout,
55
+ not_found_class: not_found_class)
48
56
  validate_response!(validate, response, destination) if validate
49
57
  return destination
50
58
  rescue ValidationFailure => e
@@ -80,13 +88,15 @@ module Ucode
80
88
  end
81
89
  end
82
90
 
83
- def stream_to(uri, destination, read_timeout)
91
+ def stream_to(uri, destination, read_timeout, not_found_class: nil)
84
92
  response = nil
85
93
  Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
86
94
  read_timeout: read_timeout) do |http|
87
95
  request = Net::HTTP::Get.new(uri)
88
96
  http.request(request) do |r|
89
97
  unless r.is_a?(Net::HTTPSuccess)
98
+ raise ValidationFailure.new(not_found_error(not_found_class, uri, r)) if not_found_class && r.is_a?(Net::HTTPClientError)
99
+
90
100
  raise "HTTP #{r.code} #{r.message}"
91
101
  end
92
102
 
@@ -97,6 +107,16 @@ module Ucode
97
107
  response or raise "no response received"
98
108
  end
99
109
 
110
+ # Builds the not-found error (e.g. CodeChartNotFoundError)
111
+ # for a 4xx response, fed through ValidationFailure so the
112
+ # retry loop in `get` doesn't re-attempt a permanent miss.
113
+ def not_found_error(klass, uri, response)
114
+ klass.new(
115
+ "HTTP #{response.code} #{response.message}",
116
+ context: { url: uri.to_s, status: response.code.to_i },
117
+ )
118
+ end
119
+
100
120
  def write_body(response, destination)
101
121
  partial = destination.sub_ext("#{destination.extname}.part")
102
122
  File.open(partial, "wb") do |file|
@@ -17,12 +17,27 @@ module Ucode
17
17
  # discovered wins. Discovery order follows mutool info's page-major
18
18
  # listing, so earlier blocks' fonts win — the expected behavior.
19
19
  class Catalog
20
- # @param source [PdfLocation]
20
+ # @param source [PdfSource]
21
+ # @param block_range [Range<Integer>, nil] codepoint scope the
22
+ # caller is extracting. When present, drops an intrinsic
23
+ # strategy's result if it has zero in-block intersection,
24
+ # letting positional strategies take over (the U1F200 class
25
+ # of failure where the font's CMap encoded composing
26
+ # ideographs rather than the specimens). nil = legacy mode
27
+ # (intrinsic result always trusted).
28
+ # @param force_positional_for_font_ids [Set<Integer>] Type0
29
+ # font object IDs that always trigger positional attribution
30
+ # regardless of intrinsic success. Escape hatch for
31
+ # partial-overlap cases.
21
32
  # @param correlator_configs [Hash{Integer=>ContentStreamCorrelator::Config}]
22
33
  # maps a Type0 font's PDF object ID to the pillar-2 config to
23
34
  # use when the font has no /ToUnicode CMap. Empty by default.
24
- def initialize(source, correlator_configs: {})
35
+ def initialize(source, block_range: nil,
36
+ force_positional_for_font_ids: Set.new,
37
+ correlator_configs: {})
25
38
  @source = source
39
+ @block_range = block_range
40
+ @force_positional_for_font_ids = force_positional_for_font_ids
26
41
  @correlator_configs = correlator_configs
27
42
  @index = nil
28
43
  end
@@ -38,6 +53,38 @@ module Ucode
38
53
  index[codepoint]
39
54
  end
40
55
 
56
+ # Locate where in the PDF a codepoint's specimen was rendered.
57
+ # Returns `{page:, x:, y:}` (PDF user space, origin
58
+ # bottom-left) by joining `#lookup` with the trace cache's
59
+ # `(font, gid)` search. Nil when:
60
+ # * the codepoint isn't in this PDF, OR
61
+ # * the embedded font has no traced specimen (ToUnicode-only
62
+ # path with no positional correlation data), OR
63
+ # * the trace cache wasn't populated (lazy + never asked
64
+ # for any positional strategy).
65
+ #
66
+ # Memoized per-Catalog; the second call for the same
67
+ # codepoint is O(1).
68
+ #
69
+ # @param codepoint [Integer]
70
+ # @return [Hash{Symbol=>Integer, Float}, nil]
71
+ def location_for(codepoint)
72
+ @locations ||= {}
73
+ return @locations[codepoint] if @locations.key?(codepoint)
74
+
75
+ entry = lookup(codepoint)
76
+ result = if entry.nil?
77
+ nil
78
+ else
79
+ trace_cache.find_glyph(
80
+ base_font: entry.base_font,
81
+ gid: entry.gid_for(codepoint),
82
+ )
83
+ end
84
+ @locations[codepoint] = result
85
+ result
86
+ end
87
+
41
88
  # @return [Array<Integer>] every codepoint this PDF covers
42
89
  def codepoints
43
90
  index.keys
@@ -93,10 +140,26 @@ module Ucode
93
140
  end
94
141
 
95
142
  def mapper
96
- @mapper ||= CodepointMapper.new(
143
+ @mapper ||= CodepointMapper.build(
97
144
  source: @source,
145
+ block_range: @block_range,
146
+ force_positional_for_font_ids: @force_positional_for_font_ids,
98
147
  correlator_configs: @correlator_configs,
99
148
  indexer: indexer,
149
+ trace_cache: trace_cache,
150
+ )
151
+ end
152
+
153
+ # The PageTraceCache shared with the CodepointMapper. Lazily
154
+ # constructed on first reference; #location_for uses the same
155
+ # cache, so the per-page `mutool trace` cost is paid exactly
156
+ # once across the catalog + location lookups for a PDF.
157
+ #
158
+ # @return [PageTraceCache]
159
+ def trace_cache
160
+ @trace_cache ||= PageTraceCache.new(
161
+ pdf: @source.pdf_path,
162
+ page_count: indexer.page_count,
100
163
  )
101
164
  end
102
165
  end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
4
+ require "ucode/glyphs/embedded_fonts/content_stream_correlator"
5
+ require "ucode/glyphs/embedded_fonts/mutool"
6
+
7
+ module Ucode
8
+ module Glyphs
9
+ module EmbeddedFonts
10
+ class CodepointMapper
11
+ # Strategy 2 — caller-supplied pillar-2 config. Renders the
12
+ # configured pages to SVG and runs {ContentStreamCorrelator}
13
+ # to match specimen glyphs to hex labels positionally.
14
+ class CorrelatorStrategy < Strategy
15
+ # @param source [PdfSource]
16
+ # @param correlator_configs [Hash{Integer=>ContentStreamCorrelator::Config}]
17
+ # @param mutool_draw [Mutool::Draw]
18
+ def initialize(source:, correlator_configs:, mutool_draw:)
19
+ super()
20
+ @source = source
21
+ @correlator_configs = correlator_configs
22
+ @mutool_draw = mutool_draw
23
+ end
24
+
25
+ def supports?(descriptor)
26
+ descriptor.cid_map_kind == :identity &&
27
+ @correlator_configs.key?(descriptor.font_obj_id)
28
+ end
29
+
30
+ # @see Strategy#positional?
31
+ def positional?
32
+ true
33
+ end
34
+
35
+ def map(descriptor)
36
+ config = @correlator_configs[descriptor.font_obj_id]
37
+ return {} if config.page_numbers.nil? || config.page_numbers.empty?
38
+
39
+ svg = @mutool_draw.svg(@source.pdf_to_s, *config.page_numbers)
40
+ ContentStreamCorrelator.new(config).correlate(svg)
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ucode
4
+ module Glyphs
5
+ module EmbeddedFonts
6
+ class CodepointMapper
7
+ # Abstract base for the codepoint→GID resolution strategies.
8
+ #
9
+ # Each subclass owns ONE mutool subcommand and ONE collaborator.
10
+ # Subclasses declare their role via {#positional?}:
11
+ #
12
+ # * `positional? == false` — the strategy reads the font's own
13
+ # intrinsic mapping (ToUnicode CMap). Authoritative for
14
+ # cross-reference typography, but can be misleading when
15
+ # the embedded font's CMap encodes composing characters
16
+ # rather than the chart specimens (Enclosed Ideographic
17
+ # Supplement, where CJKSymbols' CIDs map to the inner CJK
18
+ # ideographs, not the squared characters themselves).
19
+ # * `positional? == true` — the strategy attributes glyphs to
20
+ # codepoints via chart-grid geometry (mutool trace, content
21
+ # stream correlation). Authoritative for in-block specimens.
22
+ #
23
+ # The orchestrator ({CodepointMapper}) partitions strategies by
24
+ # this predicate, gates positional strategies behind a
25
+ # block-scope check, and merges with positional precedence
26
+ # when both produce a result. Adding a new strategy = one
27
+ # subclass + one `positional?` override + one entry in the
28
+ # chain. No edit to CodepointMapper (Open/Closed Principle).
29
+ class Strategy
30
+ # @param descriptor [RawFontDescriptor]
31
+ def supports?(_descriptor)
32
+ raise NotImplementedError
33
+ end
34
+
35
+ # @param descriptor [RawFontDescriptor]
36
+ # @return [Hash{Integer=>Integer}] codepoint => gid; empty
37
+ # when the strategy cannot produce a mapping
38
+ def map(_descriptor)
39
+ raise NotImplementedError
40
+ end
41
+
42
+ # Declares whether this strategy attributes glyphs by chart
43
+ # geometry (true) or by the font's intrinsic mapping (false).
44
+ # The default is false; positional strategies override.
45
+ #
46
+ # @return [Boolean]
47
+ def positional?
48
+ false
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
4
+ require "ucode/glyphs/embedded_fonts/mutool"
5
+ require "ucode/glyphs/embedded_fonts/tounicode"
6
+
7
+ module Ucode
8
+ module Glyphs
9
+ module EmbeddedFonts
10
+ class CodepointMapper
11
+ # Strategy 1 — read the Type0 font's `/ToUnicode` CMap stream
12
+ # and parse it into a `{codepoint => gid}` map. Highest
13
+ # fidelity when present; usually missing for subsetted CID
14
+ # fonts (the case the trace fallback exists for).
15
+ class ToUnicodeStrategy < Strategy
16
+ # @param source [PdfSource]
17
+ # @param mutool_show [Mutool::Show]
18
+ def initialize(source:, mutool_show:)
19
+ super()
20
+ @source = source
21
+ @mutool_show = mutool_show
22
+ end
23
+
24
+ def supports?(descriptor)
25
+ descriptor.cid_map_kind == :identity &&
26
+ !descriptor.tounicode_ref.nil?
27
+ end
28
+
29
+ def map(descriptor)
30
+ cmap_text = @mutool_show.stream(@source.pdf_to_s,
31
+ descriptor.tounicode_ref)
32
+ cid_to_cp = ToUnicode.parse(cmap_text)
33
+ cid_to_cp.each_with_object({}) do |(cid, cp), h|
34
+ h[cp] = cid
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
4
+ require "ucode/glyphs/embedded_fonts/mutool"
5
+ require "ucode/glyphs/embedded_fonts/page_trace_cache"
6
+ require "ucode/glyphs/embedded_fonts/trace_correlator"
7
+
8
+ module Ucode
9
+ module Glyphs
10
+ module EmbeddedFonts
11
+ class CodepointMapper
12
+ # Strategy 3 — auto-detect via `mutool trace`. Last-resort
13
+ # fallback for CID fonts without /ToUnicode and without a
14
+ # caller-supplied correlator config.
15
+ #
16
+ # Consumes a {PageTraceCache} that traces each PDF page
17
+ # exactly once across all CID fonts. The pre-cache path was
18
+ # O(F × P) subprocess invocations; this is O(P).
19
+ #
20
+ # Correlates per page (Y positions are page-local — clustering
21
+ # across page boundaries would produce false matches).
22
+ class TraceStrategy < Strategy
23
+ # @param cache [PageTraceCache, nil] nil = strategy is a no-op
24
+ # @param indexer [PdfIndexer] for the cheap font_appears?
25
+ # precondition (avoids touching the cache for fonts the
26
+ # PDF doesn't reference at all)
27
+ def initialize(cache:, indexer:)
28
+ super()
29
+ @cache = cache
30
+ @indexer = indexer
31
+ end
32
+
33
+ def supports?(descriptor)
34
+ return false unless @cache
35
+ return false unless descriptor.cid_map_kind == :identity
36
+
37
+ @indexer.font_appears?(descriptor.base_font)
38
+ end
39
+
40
+ # @see Strategy#positional?
41
+ def positional?
42
+ true
43
+ end
44
+
45
+ def map(descriptor)
46
+ return {} unless @cache
47
+
48
+ correlator = TraceCorrelator.new(
49
+ specimen_font_name: descriptor.base_font,
50
+ )
51
+ mapping = {}
52
+ @cache.each_page_for(descriptor.base_font) do |_page, glyphs|
53
+ page_mapping = correlator.correlate(glyphs)
54
+ page_mapping.each do |cp, gid|
55
+ mapping[cp] ||= gid
56
+ end
57
+ end
58
+ mapping
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end