ucode 0.3.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +45 -6
- data/config/unicode17_universal_glyph_set.yml +1 -1
- data/lib/ucode/cli.rb +201 -32
- data/lib/ucode/code_chart/batch_runner.rb +184 -0
- data/lib/ucode/code_chart/block_index.rb +89 -0
- data/lib/ucode/code_chart/coverage_gap_index.rb +115 -0
- data/lib/ucode/code_chart/extractor.rb +65 -13
- data/lib/ucode/code_chart/fetcher.rb +112 -0
- data/lib/ucode/code_chart/gap_analyzer/block_gap.rb +35 -0
- data/lib/ucode/code_chart/gap_analyzer/essenfont_manifest.rb +60 -0
- data/lib/ucode/code_chart/gap_analyzer/manifest.rb +39 -0
- data/lib/ucode/code_chart/gap_analyzer.rb +83 -0
- data/lib/ucode/code_chart/provenance.rb +99 -58
- data/lib/ucode/code_chart/sidecar.rb +3 -4
- data/lib/ucode/code_chart/verifier/builder.rb +49 -0
- data/lib/ucode/code_chart/verifier/mutool_strategy.rb +80 -0
- data/lib/ucode/code_chart/verifier/page_render_cache.rb +63 -0
- data/lib/ucode/code_chart/verifier/result.rb +27 -0
- data/lib/ucode/code_chart/verifier/resvg_strategy.rb +71 -0
- data/lib/ucode/code_chart/verifier/strategy.rb +73 -0
- data/lib/ucode/code_chart/verifier.rb +152 -0
- data/lib/ucode/code_chart/writer.rb +21 -14
- data/lib/ucode/code_chart.rb +6 -0
- data/lib/ucode/error.rb +16 -1
- data/lib/ucode/fetch/http.rb +23 -3
- data/lib/ucode/glyphs/embedded_fonts/catalog.rb +66 -3
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/correlator_strategy.rb +46 -0
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/strategy.rb +54 -0
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/tounicode_strategy.rb +41 -0
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/trace_strategy.rb +64 -0
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb +150 -90
- data/lib/ucode/glyphs/embedded_fonts/mutool/draw.rb +26 -0
- data/lib/ucode/glyphs/embedded_fonts/mutool/info.rb +23 -0
- data/lib/ucode/glyphs/embedded_fonts/mutool/show.rb +36 -0
- data/lib/ucode/glyphs/embedded_fonts/mutool/trace.rb +26 -0
- data/lib/ucode/glyphs/embedded_fonts/mutool.rb +30 -0
- data/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb +112 -0
- data/lib/ucode/glyphs/embedded_fonts/pdf_indexer.rb +50 -56
- data/lib/ucode/glyphs/embedded_fonts/{pdf_location.rb → pdf_source.rb} +1 -1
- data/lib/ucode/glyphs/embedded_fonts/renderer.rb +19 -2
- data/lib/ucode/glyphs/embedded_fonts/trace_runner.rb +12 -28
- data/lib/ucode/glyphs/embedded_fonts.rb +3 -1
- data/lib/ucode/glyphs/real_fonts/coverage_auditor.rb +0 -4
- data/lib/ucode/glyphs/real_fonts/font_coverage_report.rb +0 -2
- data/lib/ucode/glyphs/real_fonts/writer.rb +0 -2
- data/lib/ucode/glyphs/source.rb +11 -1
- data/lib/ucode/glyphs/sources/pillar1_embedded_tounicode.rb +11 -6
- data/lib/ucode/version.rb +1 -1
- data/lib/ucode.rb +2 -0
- metadata +28 -74
- data/TODO.extract-code-chart/01-pdf-fetch-validation.md +0 -80
- data/TODO.extract-code-chart/02-block-name-resolver.md +0 -68
- data/TODO.extract-code-chart/03-codechart-namespace.md +0 -82
- data/TODO.extract-code-chart/04-codechart-extractor.md +0 -154
- data/TODO.extract-code-chart/05-provenance-and-sidecar.md +0 -147
- data/TODO.extract-code-chart/06-codechart-writer.md +0 -134
- data/TODO.extract-code-chart/07-codechart-cli.md +0 -135
- data/TODO.extract-code-chart/08-specs.md +0 -87
- data/TODO.full/00-README.md +0 -116
- data/TODO.full/01-panglyph-vision.md +0 -112
- data/TODO.full/02-panglyph-repo-bootstrap.md +0 -184
- data/TODO.full/03-panglyph-font-builder.md +0 -201
- data/TODO.full/04-panglyph-publish-pipeline.md +0 -126
- data/TODO.full/05-ucode-0-1-1-release.md +0 -139
- data/TODO.full/06-fontisan-remove-audit.md +0 -142
- data/TODO.full/07-fontisan-remove-ucd.md +0 -125
- data/TODO.full/08-archive-private-bin-build.md +0 -143
- data/TODO.full/09-archive-public-structure.md +0 -164
- data/TODO.full/10-fontist-org-woff-glyphs.md +0 -131
- data/TODO.full/11-fontist-org-audit-coverage.md +0 -140
- data/TODO.full/12-implementation-order.md +0 -216
- data/TODO.full/13-fontisan-font-writer-api.md +0 -189
- data/TODO.full/14-fontisan-table-writers.md +0 -66
- data/TODO.full/15-panglyph-builder-real.md +0 -82
- data/TODO.full/16-archive-public-sync-workflows.md +0 -167
- data/TODO.full/17-fontist-org-font-picker.md +0 -73
- data/TODO.full/18-comprehensive-spec-coverage.md +0 -64
- data/TODO.full/19-ucode-0-1-2-patch.md +0 -32
- data/TODO.full/20-fontisan-0-2-23-release.md +0 -52
- data/TODO.new/00-README.md +0 -96
- data/TODO.new/01-pillar-terminology-alignment.md +0 -69
- data/TODO.new/02-audit-schema-design.md +0 -255
- data/TODO.new/03-directory-output-spec.md +0 -203
- data/TODO.new/04-fontist-org-contract.md +0 -173
- data/TODO.new/05-baseline-unicode17-coverage-audit.md +0 -144
- data/TODO.new/06-audit-namespace-skeleton.md +0 -105
- data/TODO.new/07-audit-models-port.md +0 -132
- data/TODO.new/08-extractors-cheap-port.md +0 -113
- data/TODO.new/09-extractors-expensive-port.md +0 -99
- data/TODO.new/10-aggregations-ucd-rewrite.md +0 -168
- data/TODO.new/11-differ-and-library-auditor-port.md +0 -102
- data/TODO.new/12-formatters-port.md +0 -115
- data/TODO.new/13-directory-emitter.md +0 -147
- data/TODO.new/14-html-face-browser.md +0 -144
- data/TODO.new/15-html-library-browser.md +0 -102
- data/TODO.new/16-cli-audit-subcommands.md +0 -142
- data/TODO.new/17-fontisan-cleanup-audit.md +0 -147
- data/TODO.new/18-fontisan-cleanup-ucd.md +0 -156
- data/TODO.new/19-fontisan-docs-update.md +0 -155
- data/TODO.new/20-canonical-resolver-4-tier.md +0 -182
- data/TODO.new/21-canonical-unicode17-build.md +0 -148
- data/TODO.new/22-implementation-order.md +0 -176
- data/TODO.new/23-universal-glyph-set-source-map.md +0 -312
- data/TODO.new/24-universal-glyph-set-build.md +0 -189
- data/TODO.new/25-font-audit-against-universal-set.md +0 -195
- data/TODO.new/26-missing-glyph-reporter.md +0 -189
- data/TODO.new/27-fontist-org-consumer-integration.md +0 -200
- data/TODO.new/28-implementation-order-update.md +0 -187
- data/TODO.new/29-universal-set-curation-uc17.md +0 -312
- data/TODO.new/30-tier1-font-acquisition.md +0 -241
- data/TODO.new/31-universal-set-production-build.md +0 -205
- data/TODO.new/32-uc17-coverage-matrix.md +0 -165
- data/TODO.new/33-specialist-font-acquisition-refresh.md +0 -138
- data/TODO.new/34-pillar2-content-stream-correlator.md +0 -147
- data/TODO.new/35-universal-set-production-run.md +0 -160
- data/TODO.new/36-per-font-coverage-audit.md +0 -145
- data/TODO.new/37-coverage-highlight-reporter.md +0 -125
- data/TODO.new/38-fontist-org-glyph-consumer.md +0 -141
- data/TODO.new/39-implementation-order-update-32-38.md +0 -258
- data/TODO.new/40-archive-private-uses-ucode-audit.md +0 -124
- data/TODO.new/41-ucode-unicode-archive-bridge.md +0 -160
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "pathname"
|
|
5
|
+
|
|
6
|
+
module Ucode
|
|
7
|
+
module CodeChart
|
|
8
|
+
# Pixel-diff verification for extracted SVG glyphs against the
|
|
9
|
+
# source PDF cell. REQ R4.
|
|
10
|
+
#
|
|
11
|
+
# ## Strategy chain (OCP)
|
|
12
|
+
#
|
|
13
|
+
# One renderer strategy per external tool. The strategy exposes
|
|
14
|
+
# three primitive operations: render an SVG to PNG, render a PDF
|
|
15
|
+
# page region to PNG, and compute a pixel-diff percentage between
|
|
16
|
+
# two PNGs. {Verifier} orchestrates these primitives; it has no
|
|
17
|
+
# knowledge of any specific CLI tool.
|
|
18
|
+
#
|
|
19
|
+
# Adding a new renderer (cairo, imagemagick, …) = one Strategy
|
|
20
|
+
# subclass + one entry in {Verifier::Builder.pick}. Verifier core
|
|
21
|
+
# never changes.
|
|
22
|
+
#
|
|
23
|
+
# ## Result types (see Verifier::Result)
|
|
24
|
+
#
|
|
25
|
+
# * `Pass` — diff < FAIL_THRESHOLD (default 1.0%).
|
|
26
|
+
# * `Fail` — diff ≥ threshold; carries the percent + diff
|
|
27
|
+
# artifact path for inspection.
|
|
28
|
+
# * `Skipped` — extractor produced no `source_page`/`source_cell`
|
|
29
|
+
# (e.g. ToUnicode-only path, or Last Resort placeholder), so no
|
|
30
|
+
# honest cell-diff is possible. Surfaces as a warning, NOT a
|
|
31
|
+
# pass.
|
|
32
|
+
class Verifier
|
|
33
|
+
autoload :Strategy, "ucode/code_chart/verifier/strategy"
|
|
34
|
+
autoload :ResvgStrategy, "ucode/code_chart/verifier/resvg_strategy"
|
|
35
|
+
autoload :MutoolStrategy, "ucode/code_chart/verifier/mutool_strategy"
|
|
36
|
+
autoload :Builder, "ucode/code_chart/verifier/builder"
|
|
37
|
+
autoload :PageRenderCache, "ucode/code_chart/verifier/page_render_cache"
|
|
38
|
+
autoload :Result, "ucode/code_chart/verifier/result"
|
|
39
|
+
|
|
40
|
+
DEFAULT_CELL_SIZE = 40.0
|
|
41
|
+
private_constant :DEFAULT_CELL_SIZE
|
|
42
|
+
|
|
43
|
+
# @param strategy [Strategy, nil] nil = {Builder.pick} auto.
|
|
44
|
+
# @param diff_dir [Pathname, String] where diff artifacts land
|
|
45
|
+
# @param threshold [Float, nil] override {Strategy::FAIL_THRESHOLD}
|
|
46
|
+
def initialize(diff_dir:, strategy: nil, threshold: nil)
|
|
47
|
+
@strategy = strategy || Builder.pick
|
|
48
|
+
@diff_dir = Pathname.new(diff_dir)
|
|
49
|
+
@threshold = threshold || (Strategy::FAIL_THRESHOLD if @strategy)
|
|
50
|
+
if @strategy
|
|
51
|
+
@page_cache = PageRenderCache.new(diff_dir: @diff_dir.join(".cache"),
|
|
52
|
+
strategy: @strategy)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# @return [Boolean] true iff a usable strategy was found
|
|
57
|
+
def available?
|
|
58
|
+
!@strategy.nil? && @strategy.available?
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# @param result [Ucode::CodeChart::Extractor::Result]
|
|
62
|
+
# @param pdf_path [Pathname, String, nil] source PDF
|
|
63
|
+
# @return [Result::Pass, Result::Fail, Result::Skipped]
|
|
64
|
+
def verify(result, pdf_path:)
|
|
65
|
+
unless @strategy
|
|
66
|
+
return Result::Skipped.new(codepoint: result.codepoint,
|
|
67
|
+
reason: :no_strategy)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
loc = extract_location(result)
|
|
71
|
+
unless loc
|
|
72
|
+
return Result::Skipped.new(codepoint: result.codepoint,
|
|
73
|
+
reason: :no_location)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
path = Pathname.new(pdf_path)
|
|
77
|
+
unless path.exist?
|
|
78
|
+
return Result::Skipped.new(codepoint: result.codepoint,
|
|
79
|
+
reason: :no_pdf)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
verify_with_location(result, path, loc)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def extract_location(result)
|
|
88
|
+
return nil unless result.source_page && result.source_cell
|
|
89
|
+
|
|
90
|
+
{
|
|
91
|
+
page: result.source_page,
|
|
92
|
+
x: result.source_cell[:x],
|
|
93
|
+
y: result.source_cell[:y],
|
|
94
|
+
}
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def verify_with_location(result, pdf_path, loc)
|
|
98
|
+
svg_png = render_svg(result)
|
|
99
|
+
cell_png = render_cell(pdf_path, loc)
|
|
100
|
+
|
|
101
|
+
percent = @strategy.diff(svg_png, cell_png)
|
|
102
|
+
if percent < @threshold
|
|
103
|
+
Result::Pass.new(codepoint: result.codepoint, percent: percent)
|
|
104
|
+
else
|
|
105
|
+
artifact = write_artifact(result, svg_png, cell_png)
|
|
106
|
+
Result::Fail.new(codepoint: result.codepoint,
|
|
107
|
+
percent: percent,
|
|
108
|
+
diff_path: artifact)
|
|
109
|
+
end
|
|
110
|
+
ensure
|
|
111
|
+
[svg_png, cell_png].each { |p| FileUtils.rm_f(p.to_s) if p&.exist? }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def render_svg(result)
|
|
115
|
+
svg_path = @diff_dir.join("#{format_cp(result.codepoint)}.svg")
|
|
116
|
+
svg_path.dirname.mkpath
|
|
117
|
+
svg_path.write(result.svg)
|
|
118
|
+
png_path = Pathname.new("#{svg_path}.png")
|
|
119
|
+
@strategy.render_svg(svg_path, png_path)
|
|
120
|
+
png_path
|
|
121
|
+
ensure
|
|
122
|
+
FileUtils.rm_f(svg_path.to_s) if svg_path&.exist?
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def render_cell(pdf_path, loc)
|
|
126
|
+
rect = cell_rect(loc)
|
|
127
|
+
png_path = @diff_dir.join(".cache",
|
|
128
|
+
"cell-#{loc[:page]}-" \
|
|
129
|
+
"#{format('%05.2f', loc[:x])}-" \
|
|
130
|
+
"#{format('%05.2f', loc[:y])}.png")
|
|
131
|
+
png_path.dirname.mkpath
|
|
132
|
+
@strategy.render_pdf_region(pdf_path, loc[:page], rect, png_path)
|
|
133
|
+
png_path
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def cell_rect(loc)
|
|
137
|
+
half = DEFAULT_CELL_SIZE / 2
|
|
138
|
+
{ x: loc[:x] - half, y: loc[:y] - half,
|
|
139
|
+
w: DEFAULT_CELL_SIZE, h: DEFAULT_CELL_SIZE }
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def write_artifact(result, png_a, png_b)
|
|
143
|
+
dest = @diff_dir.join("#{format_cp(result.codepoint)}.diff.png")
|
|
144
|
+
@strategy.write_diff_artifact(png_a, png_b, dest) || dest
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def format_cp(codepoint)
|
|
148
|
+
"U+#{codepoint.to_s(16).upcase.rjust(4, '0')}"
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
@@ -3,12 +3,6 @@
|
|
|
3
3
|
require "digest"
|
|
4
4
|
require "pathname"
|
|
5
5
|
|
|
6
|
-
require "ucode/code_chart/extractor"
|
|
7
|
-
require "ucode/code_chart/provenance"
|
|
8
|
-
require "ucode/code_chart/sidecar"
|
|
9
|
-
require "ucode/error"
|
|
10
|
-
require "ucode/version_resolver"
|
|
11
|
-
|
|
12
6
|
module Ucode
|
|
13
7
|
module CodeChart
|
|
14
8
|
# Orchestrates extraction + provenance sidecar writing for one
|
|
@@ -47,12 +41,19 @@ module Ucode
|
|
|
47
41
|
# @param ucd_version [String, nil] UCD version to stamp on
|
|
48
42
|
# provenance. nil = resolved via {VersionResolver.resolve(nil)}.
|
|
49
43
|
# @param cache_dir [Pathname, String, nil] font-stream cache
|
|
50
|
-
# directory for the EmbeddedFonts::
|
|
44
|
+
# directory for the EmbeddedFonts::PdfSource.
|
|
51
45
|
# @param now [Time, nil] timestamp override (for tests).
|
|
52
|
-
# @param pillar3_source, tier1_sources
|
|
46
|
+
# @param pillar3_source, tier1_sources, assigned_only, codepoints:
|
|
47
|
+
# forwarded to the Extractor.
|
|
48
|
+
# @param extractor [Ucode::CodeChart::Extractor, nil] pre-built
|
|
49
|
+
# Extractor instance. When nil (default), the Writer constructs
|
|
50
|
+
# one from the other parameters. Injecting lets callers (tests)
|
|
51
|
+
# bypass the mutool-dependent default Extractor.
|
|
53
52
|
def initialize(output_root:, pdf_path:, ucd_version: nil,
|
|
54
53
|
cache_dir: nil, now: nil,
|
|
55
|
-
pillar3_source: nil, tier1_sources: nil
|
|
54
|
+
pillar3_source: nil, tier1_sources: nil,
|
|
55
|
+
assigned_only: false, codepoints: nil,
|
|
56
|
+
extractor: nil)
|
|
56
57
|
@output_root = Pathname.new(output_root)
|
|
57
58
|
@pdf_path = Pathname.new(pdf_path)
|
|
58
59
|
@ucd_version = ucd_version || VersionResolver.resolve(nil)
|
|
@@ -60,6 +61,9 @@ module Ucode
|
|
|
60
61
|
@now = now
|
|
61
62
|
@pillar3_source = pillar3_source
|
|
62
63
|
@tier1_sources = tier1_sources
|
|
64
|
+
@assigned_only = assigned_only
|
|
65
|
+
@codepoints = codepoints
|
|
66
|
+
@extractor = extractor
|
|
63
67
|
end
|
|
64
68
|
|
|
65
69
|
# Extracts every codepoint in `block` and writes `<block_id>/<cp>.svg`
|
|
@@ -72,27 +76,30 @@ module Ucode
|
|
|
72
76
|
block_dir = @output_root.join(block.id)
|
|
73
77
|
block_dir.mkpath
|
|
74
78
|
|
|
75
|
-
pdf_sha =
|
|
79
|
+
pdf_sha = Provenance.sha256_of(@pdf_path)
|
|
76
80
|
|
|
77
81
|
sidecar = Sidecar.new(output_root: block_dir)
|
|
78
|
-
extractor = Extractor.new(
|
|
82
|
+
extractor = @extractor || Extractor.new(
|
|
79
83
|
block: block,
|
|
80
84
|
pdf_path: @pdf_path,
|
|
81
85
|
cache_dir: @cache_dir,
|
|
82
86
|
pillar3_source: @pillar3_source,
|
|
83
87
|
tier1_sources: @tier1_sources,
|
|
88
|
+
assigned_only: @assigned_only,
|
|
89
|
+
codepoints: @codepoints,
|
|
84
90
|
)
|
|
85
|
-
|
|
86
91
|
results = extractor.extract
|
|
87
92
|
svgs = 0
|
|
88
93
|
sidecars = 0
|
|
89
94
|
results.each do |result|
|
|
90
95
|
write_svg(block_dir, result)
|
|
91
96
|
svgs += 1
|
|
92
|
-
provenance =
|
|
97
|
+
provenance = Provenance.build(
|
|
93
98
|
block: block, codepoint: result.codepoint,
|
|
94
99
|
ucd_version: @ucd_version, pdf_path: @pdf_path,
|
|
95
|
-
now: @now,
|
|
100
|
+
pdf_sha: pdf_sha, now: @now,
|
|
101
|
+
base_font: result.base_font, gid: result.gid,
|
|
102
|
+
source_page: result.source_page, source_cell: result.source_cell,
|
|
96
103
|
)
|
|
97
104
|
sidecar.write(provenance)
|
|
98
105
|
sidecars += 1
|
data/lib/ucode/code_chart.rb
CHANGED
|
@@ -31,9 +31,15 @@ module Ucode
|
|
|
31
31
|
# (e.g. a future pure-Ruby PDF parser per ADR-0001) does not
|
|
32
32
|
# change the public API.
|
|
33
33
|
module CodeChart
|
|
34
|
+
autoload :BatchRunner, "ucode/code_chart/batch_runner"
|
|
35
|
+
autoload :BlockIndex, "ucode/code_chart/block_index"
|
|
36
|
+
autoload :CoverageGapIndex, "ucode/code_chart/coverage_gap_index"
|
|
34
37
|
autoload :Extractor, "ucode/code_chart/extractor"
|
|
38
|
+
autoload :Fetcher, "ucode/code_chart/fetcher"
|
|
39
|
+
autoload :GapAnalyzer, "ucode/code_chart/gap_analyzer"
|
|
35
40
|
autoload :Provenance, "ucode/code_chart/provenance"
|
|
36
41
|
autoload :Sidecar, "ucode/code_chart/sidecar"
|
|
42
|
+
autoload :Verifier, "ucode/code_chart/verifier"
|
|
37
43
|
autoload :Writer, "ucode/code_chart/writer"
|
|
38
44
|
end
|
|
39
45
|
end
|
data/lib/ucode/error.rb
CHANGED
|
@@ -110,12 +110,27 @@ module Ucode
|
|
|
110
110
|
# `mutool` is not installed on the PATH.
|
|
111
111
|
class EmbeddedFontsMissingError < GlyphError; end
|
|
112
112
|
|
|
113
|
+
# A `mutool` subprocess exited non-zero. Distinct from
|
|
114
|
+
# {EmbeddedFontsMissingError}: this fires when mutool ran but failed
|
|
115
|
+
# (corrupt PDF, bad object ref, etc.). The context carries the argv.
|
|
116
|
+
class MutoolError < GlyphError; end
|
|
117
|
+
|
|
118
|
+
# CodeChart feature errors — concerns under `Ucode::CodeChart::*`.
|
|
119
|
+
# Distinct from the generic {GlyphError} hierarchy so callers can
|
|
120
|
+
# scope rescue clauses to feature failures.
|
|
121
|
+
class CodeChartError < GlyphError; end
|
|
122
|
+
|
|
113
123
|
# The Code Charts PDF for a requested block cannot be obtained: the
|
|
114
124
|
# network returned 4xx/5xx, the response wasn't application/pdf, or
|
|
115
125
|
# the body didn't start with the `%PDF` magic. Distinct from
|
|
116
126
|
# {EmbeddedFontsMissingError} (which fires when the file is already
|
|
117
127
|
# on disk and we just can't open it): this fires at fetch time.
|
|
118
|
-
class CodeChartNotFoundError <
|
|
128
|
+
class CodeChartNotFoundError < CodeChartError; end
|
|
129
|
+
|
|
130
|
+
# A cached Code Charts PDF failed sha256 verification — the file on
|
|
131
|
+
# disk doesn't match the sidecar hash from the previous successful
|
|
132
|
+
# download. Implies external tampering or a filesystem fault.
|
|
133
|
+
class CodeChartChecksumError < CodeChartError; end
|
|
119
134
|
|
|
120
135
|
# Pre-build validation failed for a universal-set build. The
|
|
121
136
|
# context carries the failing checks so the CLI can render a
|
data/lib/ucode/fetch/http.rb
CHANGED
|
@@ -29,11 +29,18 @@ module Ucode
|
|
|
29
29
|
# {Ucode::CodeChartNotFoundError} with the offending header
|
|
30
30
|
# value in `context:` on failure. nil = no validation (the
|
|
31
31
|
# default for non-PDF callers like UcdZip and UnihanZip).
|
|
32
|
+
# @param not_found_class [Class, nil] when set, HTTP 4xx
|
|
33
|
+
# responses raise this class (instantiated with message +
|
|
34
|
+
# context) instead of being treated as retriable transport
|
|
35
|
+
# errors. nil = 4xx is retriable like any other non-success.
|
|
32
36
|
# @return [Pathname] destination path on success.
|
|
33
37
|
# @raise [Ucode::NetworkError] if all retries fail.
|
|
34
38
|
# @raise [Ucode::CodeChartNotFoundError] when `validate: :pdf`
|
|
35
39
|
# and the response fails content validation.
|
|
36
|
-
|
|
40
|
+
# @raise [<not_found_class>] when `not_found_class:` is set
|
|
41
|
+
# and the server returns 4xx.
|
|
42
|
+
def get(url, dest:, retries: nil, timeout: nil, validate: nil,
|
|
43
|
+
not_found_class: nil)
|
|
37
44
|
uri = url.is_a?(URI) ? url : URI(url)
|
|
38
45
|
destination = Pathname.new(dest)
|
|
39
46
|
destination.dirname.mkpath
|
|
@@ -44,7 +51,8 @@ module Ucode
|
|
|
44
51
|
|
|
45
52
|
last_error = nil
|
|
46
53
|
(attempts + 1).times do |attempt|
|
|
47
|
-
response = stream_to(uri, destination, read_timeout
|
|
54
|
+
response = stream_to(uri, destination, read_timeout,
|
|
55
|
+
not_found_class: not_found_class)
|
|
48
56
|
validate_response!(validate, response, destination) if validate
|
|
49
57
|
return destination
|
|
50
58
|
rescue ValidationFailure => e
|
|
@@ -80,13 +88,15 @@ module Ucode
|
|
|
80
88
|
end
|
|
81
89
|
end
|
|
82
90
|
|
|
83
|
-
def stream_to(uri, destination, read_timeout)
|
|
91
|
+
def stream_to(uri, destination, read_timeout, not_found_class: nil)
|
|
84
92
|
response = nil
|
|
85
93
|
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
|
|
86
94
|
read_timeout: read_timeout) do |http|
|
|
87
95
|
request = Net::HTTP::Get.new(uri)
|
|
88
96
|
http.request(request) do |r|
|
|
89
97
|
unless r.is_a?(Net::HTTPSuccess)
|
|
98
|
+
raise ValidationFailure.new(not_found_error(not_found_class, uri, r)) if not_found_class && r.is_a?(Net::HTTPClientError)
|
|
99
|
+
|
|
90
100
|
raise "HTTP #{r.code} #{r.message}"
|
|
91
101
|
end
|
|
92
102
|
|
|
@@ -97,6 +107,16 @@ module Ucode
|
|
|
97
107
|
response or raise "no response received"
|
|
98
108
|
end
|
|
99
109
|
|
|
110
|
+
# Builds the not-found error (e.g. CodeChartNotFoundError)
|
|
111
|
+
# for a 4xx response, fed through ValidationFailure so the
|
|
112
|
+
# retry loop in `get` doesn't re-attempt a permanent miss.
|
|
113
|
+
def not_found_error(klass, uri, response)
|
|
114
|
+
klass.new(
|
|
115
|
+
"HTTP #{response.code} #{response.message}",
|
|
116
|
+
context: { url: uri.to_s, status: response.code.to_i },
|
|
117
|
+
)
|
|
118
|
+
end
|
|
119
|
+
|
|
100
120
|
def write_body(response, destination)
|
|
101
121
|
partial = destination.sub_ext("#{destination.extname}.part")
|
|
102
122
|
File.open(partial, "wb") do |file|
|
|
@@ -17,12 +17,27 @@ module Ucode
|
|
|
17
17
|
# discovered wins. Discovery order follows mutool info's page-major
|
|
18
18
|
# listing, so earlier blocks' fonts win — the expected behavior.
|
|
19
19
|
class Catalog
|
|
20
|
-
# @param source [
|
|
20
|
+
# @param source [PdfSource]
|
|
21
|
+
# @param block_range [Range<Integer>, nil] codepoint scope the
|
|
22
|
+
# caller is extracting. When present, drops an intrinsic
|
|
23
|
+
# strategy's result if it has zero in-block intersection,
|
|
24
|
+
# letting positional strategies take over (the U1F200 class
|
|
25
|
+
# of failure where the font's CMap encoded composing
|
|
26
|
+
# ideographs rather than the specimens). nil = legacy mode
|
|
27
|
+
# (intrinsic result always trusted).
|
|
28
|
+
# @param force_positional_for_font_ids [Set<Integer>] Type0
|
|
29
|
+
# font object IDs that always trigger positional attribution
|
|
30
|
+
# regardless of intrinsic success. Escape hatch for
|
|
31
|
+
# partial-overlap cases.
|
|
21
32
|
# @param correlator_configs [Hash{Integer=>ContentStreamCorrelator::Config}]
|
|
22
33
|
# maps a Type0 font's PDF object ID to the pillar-2 config to
|
|
23
34
|
# use when the font has no /ToUnicode CMap. Empty by default.
|
|
24
|
-
def initialize(source,
|
|
35
|
+
def initialize(source, block_range: nil,
|
|
36
|
+
force_positional_for_font_ids: Set.new,
|
|
37
|
+
correlator_configs: {})
|
|
25
38
|
@source = source
|
|
39
|
+
@block_range = block_range
|
|
40
|
+
@force_positional_for_font_ids = force_positional_for_font_ids
|
|
26
41
|
@correlator_configs = correlator_configs
|
|
27
42
|
@index = nil
|
|
28
43
|
end
|
|
@@ -38,6 +53,38 @@ module Ucode
|
|
|
38
53
|
index[codepoint]
|
|
39
54
|
end
|
|
40
55
|
|
|
56
|
+
# Locate where in the PDF a codepoint's specimen was rendered.
|
|
57
|
+
# Returns `{page:, x:, y:}` (PDF user space, origin
|
|
58
|
+
# bottom-left) by joining `#lookup` with the trace cache's
|
|
59
|
+
# `(font, gid)` search. Nil when:
|
|
60
|
+
# * the codepoint isn't in this PDF, OR
|
|
61
|
+
# * the embedded font has no traced specimen (ToUnicode-only
|
|
62
|
+
# path with no positional correlation data), OR
|
|
63
|
+
# * the trace cache wasn't populated (lazy + never asked
|
|
64
|
+
# for any positional strategy).
|
|
65
|
+
#
|
|
66
|
+
# Memoized per-Catalog; the second call for the same
|
|
67
|
+
# codepoint is O(1).
|
|
68
|
+
#
|
|
69
|
+
# @param codepoint [Integer]
|
|
70
|
+
# @return [Hash{Symbol=>Integer, Float}, nil]
|
|
71
|
+
def location_for(codepoint)
|
|
72
|
+
@locations ||= {}
|
|
73
|
+
return @locations[codepoint] if @locations.key?(codepoint)
|
|
74
|
+
|
|
75
|
+
entry = lookup(codepoint)
|
|
76
|
+
result = if entry.nil?
|
|
77
|
+
nil
|
|
78
|
+
else
|
|
79
|
+
trace_cache.find_glyph(
|
|
80
|
+
base_font: entry.base_font,
|
|
81
|
+
gid: entry.gid_for(codepoint),
|
|
82
|
+
)
|
|
83
|
+
end
|
|
84
|
+
@locations[codepoint] = result
|
|
85
|
+
result
|
|
86
|
+
end
|
|
87
|
+
|
|
41
88
|
# @return [Array<Integer>] every codepoint this PDF covers
|
|
42
89
|
def codepoints
|
|
43
90
|
index.keys
|
|
@@ -93,10 +140,26 @@ module Ucode
|
|
|
93
140
|
end
|
|
94
141
|
|
|
95
142
|
def mapper
|
|
96
|
-
@mapper ||= CodepointMapper.
|
|
143
|
+
@mapper ||= CodepointMapper.build(
|
|
97
144
|
source: @source,
|
|
145
|
+
block_range: @block_range,
|
|
146
|
+
force_positional_for_font_ids: @force_positional_for_font_ids,
|
|
98
147
|
correlator_configs: @correlator_configs,
|
|
99
148
|
indexer: indexer,
|
|
149
|
+
trace_cache: trace_cache,
|
|
150
|
+
)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# The PageTraceCache shared with the CodepointMapper. Lazily
|
|
154
|
+
# constructed on first reference; #location_for uses the same
|
|
155
|
+
# cache, so the per-page `mutool trace` cost is paid exactly
|
|
156
|
+
# once across the catalog + location lookups for a PDF.
|
|
157
|
+
#
|
|
158
|
+
# @return [PageTraceCache]
|
|
159
|
+
def trace_cache
|
|
160
|
+
@trace_cache ||= PageTraceCache.new(
|
|
161
|
+
pdf: @source.pdf_path,
|
|
162
|
+
page_count: indexer.page_count,
|
|
100
163
|
)
|
|
101
164
|
end
|
|
102
165
|
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
|
|
4
|
+
require "ucode/glyphs/embedded_fonts/content_stream_correlator"
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
6
|
+
|
|
7
|
+
module Ucode
|
|
8
|
+
module Glyphs
|
|
9
|
+
module EmbeddedFonts
|
|
10
|
+
class CodepointMapper
|
|
11
|
+
# Strategy 2 — caller-supplied pillar-2 config. Renders the
|
|
12
|
+
# configured pages to SVG and runs {ContentStreamCorrelator}
|
|
13
|
+
# to match specimen glyphs to hex labels positionally.
|
|
14
|
+
class CorrelatorStrategy < Strategy
|
|
15
|
+
# @param source [PdfSource]
|
|
16
|
+
# @param correlator_configs [Hash{Integer=>ContentStreamCorrelator::Config}]
|
|
17
|
+
# @param mutool_draw [Mutool::Draw]
|
|
18
|
+
def initialize(source:, correlator_configs:, mutool_draw:)
|
|
19
|
+
super()
|
|
20
|
+
@source = source
|
|
21
|
+
@correlator_configs = correlator_configs
|
|
22
|
+
@mutool_draw = mutool_draw
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def supports?(descriptor)
|
|
26
|
+
descriptor.cid_map_kind == :identity &&
|
|
27
|
+
@correlator_configs.key?(descriptor.font_obj_id)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @see Strategy#positional?
|
|
31
|
+
def positional?
|
|
32
|
+
true
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def map(descriptor)
|
|
36
|
+
config = @correlator_configs[descriptor.font_obj_id]
|
|
37
|
+
return {} if config.page_numbers.nil? || config.page_numbers.empty?
|
|
38
|
+
|
|
39
|
+
svg = @mutool_draw.svg(@source.pdf_to_s, *config.page_numbers)
|
|
40
|
+
ContentStreamCorrelator.new(config).correlate(svg)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ucode
|
|
4
|
+
module Glyphs
|
|
5
|
+
module EmbeddedFonts
|
|
6
|
+
class CodepointMapper
|
|
7
|
+
# Abstract base for the codepoint→GID resolution strategies.
|
|
8
|
+
#
|
|
9
|
+
# Each subclass owns ONE mutool subcommand and ONE collaborator.
|
|
10
|
+
# Subclasses declare their role via {#positional?}:
|
|
11
|
+
#
|
|
12
|
+
# * `positional? == false` — the strategy reads the font's own
|
|
13
|
+
# intrinsic mapping (ToUnicode CMap). Authoritative for
|
|
14
|
+
# cross-reference typography, but can be misleading when
|
|
15
|
+
# the embedded font's CMap encodes composing characters
|
|
16
|
+
# rather than the chart specimens (Enclosed Ideographic
|
|
17
|
+
# Supplement, where CJKSymbols' CIDs map to the inner CJK
|
|
18
|
+
# ideographs, not the squared characters themselves).
|
|
19
|
+
# * `positional? == true` — the strategy attributes glyphs to
|
|
20
|
+
# codepoints via chart-grid geometry (mutool trace, content
|
|
21
|
+
# stream correlation). Authoritative for in-block specimens.
|
|
22
|
+
#
|
|
23
|
+
# The orchestrator ({CodepointMapper}) partitions strategies by
|
|
24
|
+
# this predicate, gates positional strategies behind a
|
|
25
|
+
# block-scope check, and merges with positional precedence
|
|
26
|
+
# when both produce a result. Adding a new strategy = one
|
|
27
|
+
# subclass + one `positional?` override + one entry in the
|
|
28
|
+
# chain. No edit to CodepointMapper (Open/Closed Principle).
|
|
29
|
+
class Strategy
|
|
30
|
+
# @param descriptor [RawFontDescriptor]
|
|
31
|
+
def supports?(_descriptor)
|
|
32
|
+
raise NotImplementedError
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @param descriptor [RawFontDescriptor]
|
|
36
|
+
# @return [Hash{Integer=>Integer}] codepoint => gid; empty
|
|
37
|
+
# when the strategy cannot produce a mapping
|
|
38
|
+
def map(_descriptor)
|
|
39
|
+
raise NotImplementedError
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Declares whether this strategy attributes glyphs by chart
|
|
43
|
+
# geometry (true) or by the font's intrinsic mapping (false).
|
|
44
|
+
# The default is false; positional strategies override.
|
|
45
|
+
#
|
|
46
|
+
# @return [Boolean]
|
|
47
|
+
def positional?
|
|
48
|
+
false
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
|
|
4
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/tounicode"
|
|
6
|
+
|
|
7
|
+
module Ucode
|
|
8
|
+
module Glyphs
|
|
9
|
+
module EmbeddedFonts
|
|
10
|
+
class CodepointMapper
|
|
11
|
+
# Strategy 1 — read the Type0 font's `/ToUnicode` CMap stream
|
|
12
|
+
# and parse it into a `{codepoint => gid}` map. Highest
|
|
13
|
+
# fidelity when present; usually missing for subsetted CID
|
|
14
|
+
# fonts (the case the trace fallback exists for).
|
|
15
|
+
class ToUnicodeStrategy < Strategy
|
|
16
|
+
# @param source [PdfSource]
|
|
17
|
+
# @param mutool_show [Mutool::Show]
|
|
18
|
+
def initialize(source:, mutool_show:)
|
|
19
|
+
super()
|
|
20
|
+
@source = source
|
|
21
|
+
@mutool_show = mutool_show
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def supports?(descriptor)
|
|
25
|
+
descriptor.cid_map_kind == :identity &&
|
|
26
|
+
!descriptor.tounicode_ref.nil?
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def map(descriptor)
|
|
30
|
+
cmap_text = @mutool_show.stream(@source.pdf_to_s,
|
|
31
|
+
descriptor.tounicode_ref)
|
|
32
|
+
cid_to_cp = ToUnicode.parse(cmap_text)
|
|
33
|
+
cid_to_cp.each_with_object({}) do |(cid, cp), h|
|
|
34
|
+
h[cp] = cid
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
|
|
4
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/page_trace_cache"
|
|
6
|
+
require "ucode/glyphs/embedded_fonts/trace_correlator"
|
|
7
|
+
|
|
8
|
+
module Ucode
|
|
9
|
+
module Glyphs
|
|
10
|
+
module EmbeddedFonts
|
|
11
|
+
class CodepointMapper
|
|
12
|
+
# Strategy 3 — auto-detect via `mutool trace`. Last-resort
|
|
13
|
+
# fallback for CID fonts without /ToUnicode and without a
|
|
14
|
+
# caller-supplied correlator config.
|
|
15
|
+
#
|
|
16
|
+
# Consumes a {PageTraceCache} that traces each PDF page
|
|
17
|
+
# exactly once across all CID fonts. The pre-cache path was
|
|
18
|
+
# O(F × P) subprocess invocations; this is O(P).
|
|
19
|
+
#
|
|
20
|
+
# Correlates per page (Y positions are page-local — clustering
|
|
21
|
+
# across page boundaries would produce false matches).
|
|
22
|
+
class TraceStrategy < Strategy
|
|
23
|
+
# @param cache [PageTraceCache, nil] nil = strategy is a no-op
|
|
24
|
+
# @param indexer [PdfIndexer] for the cheap font_appears?
|
|
25
|
+
# precondition (avoids touching the cache for fonts the
|
|
26
|
+
# PDF doesn't reference at all)
|
|
27
|
+
def initialize(cache:, indexer:)
|
|
28
|
+
super()
|
|
29
|
+
@cache = cache
|
|
30
|
+
@indexer = indexer
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def supports?(descriptor)
|
|
34
|
+
return false unless @cache
|
|
35
|
+
return false unless descriptor.cid_map_kind == :identity
|
|
36
|
+
|
|
37
|
+
@indexer.font_appears?(descriptor.base_font)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @see Strategy#positional?
|
|
41
|
+
def positional?
|
|
42
|
+
true
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def map(descriptor)
|
|
46
|
+
return {} unless @cache
|
|
47
|
+
|
|
48
|
+
correlator = TraceCorrelator.new(
|
|
49
|
+
specimen_font_name: descriptor.base_font,
|
|
50
|
+
)
|
|
51
|
+
mapping = {}
|
|
52
|
+
@cache.each_page_for(descriptor.base_font) do |_page, glyphs|
|
|
53
|
+
page_mapping = correlator.correlate(glyphs)
|
|
54
|
+
page_mapping.each do |cp, gid|
|
|
55
|
+
mapping[cp] ||= gid
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
mapping
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|