ucode 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/config/unicode17_universal_glyph_set.yml +1 -1
- data/lib/ucode/code_chart/extractor.rb +7 -3
- data/lib/ucode/code_chart/writer.rb +1 -1
- data/lib/ucode/error.rb +5 -0
- data/lib/ucode/glyphs/embedded_fonts/catalog.rb +20 -3
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/correlator_strategy.rb +46 -0
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/strategy.rb +54 -0
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/tounicode_strategy.rb +41 -0
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/trace_strategy.rb +64 -0
- data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb +143 -90
- data/lib/ucode/glyphs/embedded_fonts/mutool/draw.rb +26 -0
- data/lib/ucode/glyphs/embedded_fonts/mutool/info.rb +23 -0
- data/lib/ucode/glyphs/embedded_fonts/mutool/show.rb +36 -0
- data/lib/ucode/glyphs/embedded_fonts/mutool/trace.rb +26 -0
- data/lib/ucode/glyphs/embedded_fonts/mutool.rb +30 -0
- data/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb +90 -0
- data/lib/ucode/glyphs/embedded_fonts/pdf_indexer.rb +50 -56
- data/lib/ucode/glyphs/embedded_fonts/{pdf_location.rb → pdf_source.rb} +1 -1
- data/lib/ucode/glyphs/embedded_fonts/trace_runner.rb +12 -28
- data/lib/ucode/glyphs/embedded_fonts.rb +3 -1
- data/lib/ucode/glyphs/real_fonts/coverage_auditor.rb +0 -4
- data/lib/ucode/glyphs/real_fonts/font_coverage_report.rb +0 -2
- data/lib/ucode/glyphs/real_fonts/writer.rb +0 -2
- data/lib/ucode/glyphs/sources/pillar1_embedded_tounicode.rb +1 -1
- data/lib/ucode/version.rb +1 -1
- metadata +13 -74
- data/TODO.extract-code-chart/01-pdf-fetch-validation.md +0 -80
- data/TODO.extract-code-chart/02-block-name-resolver.md +0 -68
- data/TODO.extract-code-chart/03-codechart-namespace.md +0 -82
- data/TODO.extract-code-chart/04-codechart-extractor.md +0 -154
- data/TODO.extract-code-chart/05-provenance-and-sidecar.md +0 -147
- data/TODO.extract-code-chart/06-codechart-writer.md +0 -134
- data/TODO.extract-code-chart/07-codechart-cli.md +0 -135
- data/TODO.extract-code-chart/08-specs.md +0 -87
- data/TODO.full/00-README.md +0 -116
- data/TODO.full/01-panglyph-vision.md +0 -112
- data/TODO.full/02-panglyph-repo-bootstrap.md +0 -184
- data/TODO.full/03-panglyph-font-builder.md +0 -201
- data/TODO.full/04-panglyph-publish-pipeline.md +0 -126
- data/TODO.full/05-ucode-0-1-1-release.md +0 -139
- data/TODO.full/06-fontisan-remove-audit.md +0 -142
- data/TODO.full/07-fontisan-remove-ucd.md +0 -125
- data/TODO.full/08-archive-private-bin-build.md +0 -143
- data/TODO.full/09-archive-public-structure.md +0 -164
- data/TODO.full/10-fontist-org-woff-glyphs.md +0 -131
- data/TODO.full/11-fontist-org-audit-coverage.md +0 -140
- data/TODO.full/12-implementation-order.md +0 -216
- data/TODO.full/13-fontisan-font-writer-api.md +0 -189
- data/TODO.full/14-fontisan-table-writers.md +0 -66
- data/TODO.full/15-panglyph-builder-real.md +0 -82
- data/TODO.full/16-archive-public-sync-workflows.md +0 -167
- data/TODO.full/17-fontist-org-font-picker.md +0 -73
- data/TODO.full/18-comprehensive-spec-coverage.md +0 -64
- data/TODO.full/19-ucode-0-1-2-patch.md +0 -32
- data/TODO.full/20-fontisan-0-2-23-release.md +0 -52
- data/TODO.new/00-README.md +0 -96
- data/TODO.new/01-pillar-terminology-alignment.md +0 -69
- data/TODO.new/02-audit-schema-design.md +0 -255
- data/TODO.new/03-directory-output-spec.md +0 -203
- data/TODO.new/04-fontist-org-contract.md +0 -173
- data/TODO.new/05-baseline-unicode17-coverage-audit.md +0 -144
- data/TODO.new/06-audit-namespace-skeleton.md +0 -105
- data/TODO.new/07-audit-models-port.md +0 -132
- data/TODO.new/08-extractors-cheap-port.md +0 -113
- data/TODO.new/09-extractors-expensive-port.md +0 -99
- data/TODO.new/10-aggregations-ucd-rewrite.md +0 -168
- data/TODO.new/11-differ-and-library-auditor-port.md +0 -102
- data/TODO.new/12-formatters-port.md +0 -115
- data/TODO.new/13-directory-emitter.md +0 -147
- data/TODO.new/14-html-face-browser.md +0 -144
- data/TODO.new/15-html-library-browser.md +0 -102
- data/TODO.new/16-cli-audit-subcommands.md +0 -142
- data/TODO.new/17-fontisan-cleanup-audit.md +0 -147
- data/TODO.new/18-fontisan-cleanup-ucd.md +0 -156
- data/TODO.new/19-fontisan-docs-update.md +0 -155
- data/TODO.new/20-canonical-resolver-4-tier.md +0 -182
- data/TODO.new/21-canonical-unicode17-build.md +0 -148
- data/TODO.new/22-implementation-order.md +0 -176
- data/TODO.new/23-universal-glyph-set-source-map.md +0 -312
- data/TODO.new/24-universal-glyph-set-build.md +0 -189
- data/TODO.new/25-font-audit-against-universal-set.md +0 -195
- data/TODO.new/26-missing-glyph-reporter.md +0 -189
- data/TODO.new/27-fontist-org-consumer-integration.md +0 -200
- data/TODO.new/28-implementation-order-update.md +0 -187
- data/TODO.new/29-universal-set-curation-uc17.md +0 -312
- data/TODO.new/30-tier1-font-acquisition.md +0 -241
- data/TODO.new/31-universal-set-production-build.md +0 -205
- data/TODO.new/32-uc17-coverage-matrix.md +0 -165
- data/TODO.new/33-specialist-font-acquisition-refresh.md +0 -138
- data/TODO.new/34-pillar2-content-stream-correlator.md +0 -147
- data/TODO.new/35-universal-set-production-run.md +0 -160
- data/TODO.new/36-per-font-coverage-audit.md +0 -145
- data/TODO.new/37-coverage-highlight-reporter.md +0 -125
- data/TODO.new/38-fontist-org-glyph-consumer.md +0 -141
- data/TODO.new/39-implementation-order-update-32-38.md +0 -258
- data/TODO.new/40-archive-private-uses-ucode-audit.md +0 -124
- data/TODO.new/41-ucode-unicode-archive-bridge.md +0 -160
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ebf9b5b2d2f0443697ac2cfe6f19ad8c5e1af310943ae4332eadc2df56e68c48
|
|
4
|
+
data.tar.gz: 2f14bc10e5d5012e4b17cd544443c59e1feadef979bec2144f11dbf53e14c32f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f2db668d6889a95d79ecaff540791d438044d2f9d066198b6255b6586d1f09b1433cf9d359ffa65e8b803e28c61b579bd9d08e3cfb35014ac3f49b0a0a48d351
|
|
7
|
+
data.tar.gz: 96a98950b3cfdca7fe6301647a997c9ca9fc96960a5af99ca934ca7e0066d19d7fd8c463910d50bb24f4b956e98f572cec9ef83dc30abbd03466baf1248c748b
|
|
@@ -30,7 +30,8 @@ module Ucode
|
|
|
30
30
|
# not checked out.
|
|
31
31
|
class Extractor
|
|
32
32
|
# Result of extracting one codepoint.
|
|
33
|
-
Result = Struct.new(:codepoint, :svg, :tier, :provenance,
|
|
33
|
+
Result = Struct.new(:codepoint, :svg, :tier, :provenance,
|
|
34
|
+
keyword_init: true)
|
|
34
35
|
|
|
35
36
|
# @param block [Ucode::Models::Block] block whose assigned
|
|
36
37
|
# codepoints will be extracted
|
|
@@ -100,10 +101,13 @@ module Ucode
|
|
|
100
101
|
end
|
|
101
102
|
|
|
102
103
|
def embedded_pillar_sources
|
|
103
|
-
embedded_source = Glyphs::EmbeddedFonts::
|
|
104
|
+
embedded_source = Glyphs::EmbeddedFonts::PdfSource.new(
|
|
104
105
|
pdf: @pdf_path, cache_dir: @cache_dir,
|
|
105
106
|
)
|
|
106
|
-
catalog = Glyphs::EmbeddedFonts::Catalog.new(
|
|
107
|
+
catalog = Glyphs::EmbeddedFonts::Catalog.new(
|
|
108
|
+
embedded_source,
|
|
109
|
+
block_range: (@block.range_first..@block.range_last),
|
|
110
|
+
)
|
|
107
111
|
renderer = Glyphs::EmbeddedFonts::Renderer.new(catalog)
|
|
108
112
|
[Glyphs::Sources::Pillar1EmbeddedTounicode.new(renderer: renderer)]
|
|
109
113
|
end
|
|
@@ -47,7 +47,7 @@ module Ucode
|
|
|
47
47
|
# @param ucd_version [String, nil] UCD version to stamp on
|
|
48
48
|
# provenance. nil = resolved via {VersionResolver.resolve(nil)}.
|
|
49
49
|
# @param cache_dir [Pathname, String, nil] font-stream cache
|
|
50
|
-
# directory for the EmbeddedFonts::
|
|
50
|
+
# directory for the EmbeddedFonts::PdfSource.
|
|
51
51
|
# @param now [Time, nil] timestamp override (for tests).
|
|
52
52
|
# @param pillar3_source, tier1_sources: forwarded to the Extractor.
|
|
53
53
|
def initialize(output_root:, pdf_path:, ucd_version: nil,
|
data/lib/ucode/error.rb
CHANGED
|
@@ -110,6 +110,11 @@ module Ucode
|
|
|
110
110
|
# `mutool` is not installed on the PATH.
|
|
111
111
|
class EmbeddedFontsMissingError < GlyphError; end
|
|
112
112
|
|
|
113
|
+
# A `mutool` subprocess exited non-zero. Distinct from
|
|
114
|
+
# {EmbeddedFontsMissingError}: this fires when mutool ran but failed
|
|
115
|
+
# (corrupt PDF, bad object ref, etc.). The context carries the argv.
|
|
116
|
+
class MutoolError < GlyphError; end
|
|
117
|
+
|
|
113
118
|
# The Code Charts PDF for a requested block cannot be obtained: the
|
|
114
119
|
# network returned 4xx/5xx, the response wasn't application/pdf, or
|
|
115
120
|
# the body didn't start with the `%PDF` magic. Distinct from
|
|
@@ -17,12 +17,27 @@ module Ucode
|
|
|
17
17
|
# discovered wins. Discovery order follows mutool info's page-major
|
|
18
18
|
# listing, so earlier blocks' fonts win — the expected behavior.
|
|
19
19
|
class Catalog
|
|
20
|
-
# @param source [
|
|
20
|
+
# @param source [PdfSource]
|
|
21
|
+
# @param block_range [Range<Integer>, nil] codepoint scope the
|
|
22
|
+
# caller is extracting. When present, drops an intrinsic
|
|
23
|
+
# strategy's result if it has zero in-block intersection,
|
|
24
|
+
# letting positional strategies take over (the U1F200 class
|
|
25
|
+
# of failure where the font's CMap encoded composing
|
|
26
|
+
# ideographs rather than the specimens). nil = legacy mode
|
|
27
|
+
# (intrinsic result always trusted).
|
|
28
|
+
# @param force_positional_for_font_ids [Set<Integer>] Type0
|
|
29
|
+
# font object IDs that always trigger positional attribution
|
|
30
|
+
# regardless of intrinsic success. Escape hatch for
|
|
31
|
+
# partial-overlap cases.
|
|
21
32
|
# @param correlator_configs [Hash{Integer=>ContentStreamCorrelator::Config}]
|
|
22
33
|
# maps a Type0 font's PDF object ID to the pillar-2 config to
|
|
23
34
|
# use when the font has no /ToUnicode CMap. Empty by default.
|
|
24
|
-
def initialize(source,
|
|
35
|
+
def initialize(source, block_range: nil,
|
|
36
|
+
force_positional_for_font_ids: Set.new,
|
|
37
|
+
correlator_configs: {})
|
|
25
38
|
@source = source
|
|
39
|
+
@block_range = block_range
|
|
40
|
+
@force_positional_for_font_ids = force_positional_for_font_ids
|
|
26
41
|
@correlator_configs = correlator_configs
|
|
27
42
|
@index = nil
|
|
28
43
|
end
|
|
@@ -93,8 +108,10 @@ module Ucode
|
|
|
93
108
|
end
|
|
94
109
|
|
|
95
110
|
def mapper
|
|
96
|
-
@mapper ||= CodepointMapper.
|
|
111
|
+
@mapper ||= CodepointMapper.build(
|
|
97
112
|
source: @source,
|
|
113
|
+
block_range: @block_range,
|
|
114
|
+
force_positional_for_font_ids: @force_positional_for_font_ids,
|
|
98
115
|
correlator_configs: @correlator_configs,
|
|
99
116
|
indexer: indexer,
|
|
100
117
|
)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
|
|
4
|
+
require "ucode/glyphs/embedded_fonts/content_stream_correlator"
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
6
|
+
|
|
7
|
+
module Ucode
|
|
8
|
+
module Glyphs
|
|
9
|
+
module EmbeddedFonts
|
|
10
|
+
class CodepointMapper
|
|
11
|
+
# Strategy 2 — caller-supplied pillar-2 config. Renders the
|
|
12
|
+
# configured pages to SVG and runs {ContentStreamCorrelator}
|
|
13
|
+
# to match specimen glyphs to hex labels positionally.
|
|
14
|
+
class CorrelatorStrategy < Strategy
|
|
15
|
+
# @param source [PdfSource]
|
|
16
|
+
# @param correlator_configs [Hash{Integer=>ContentStreamCorrelator::Config}]
|
|
17
|
+
# @param mutool_draw [Mutool::Draw]
|
|
18
|
+
def initialize(source:, correlator_configs:, mutool_draw:)
|
|
19
|
+
super()
|
|
20
|
+
@source = source
|
|
21
|
+
@correlator_configs = correlator_configs
|
|
22
|
+
@mutool_draw = mutool_draw
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def supports?(descriptor)
|
|
26
|
+
descriptor.cid_map_kind == :identity &&
|
|
27
|
+
@correlator_configs.key?(descriptor.font_obj_id)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @see Strategy#positional?
|
|
31
|
+
def positional?
|
|
32
|
+
true
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def map(descriptor)
|
|
36
|
+
config = @correlator_configs[descriptor.font_obj_id]
|
|
37
|
+
return {} if config.page_numbers.nil? || config.page_numbers.empty?
|
|
38
|
+
|
|
39
|
+
svg = @mutool_draw.svg(@source.pdf_to_s, *config.page_numbers)
|
|
40
|
+
ContentStreamCorrelator.new(config).correlate(svg)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ucode
|
|
4
|
+
module Glyphs
|
|
5
|
+
module EmbeddedFonts
|
|
6
|
+
class CodepointMapper
|
|
7
|
+
# Abstract base for the codepoint→GID resolution strategies.
|
|
8
|
+
#
|
|
9
|
+
# Each subclass owns ONE mutool subcommand and ONE collaborator.
|
|
10
|
+
# Subclasses declare their role via {#positional?}:
|
|
11
|
+
#
|
|
12
|
+
# * `positional? == false` — the strategy reads the font's own
|
|
13
|
+
# intrinsic mapping (ToUnicode CMap). Authoritative for
|
|
14
|
+
# cross-reference typography, but can be misleading when
|
|
15
|
+
# the embedded font's CMap encodes composing characters
|
|
16
|
+
# rather than the chart specimens (Enclosed Ideographic
|
|
17
|
+
# Supplement, where CJKSymbols' CIDs map to the inner CJK
|
|
18
|
+
# ideographs, not the squared characters themselves).
|
|
19
|
+
# * `positional? == true` — the strategy attributes glyphs to
|
|
20
|
+
# codepoints via chart-grid geometry (mutool trace, content
|
|
21
|
+
# stream correlation). Authoritative for in-block specimens.
|
|
22
|
+
#
|
|
23
|
+
# The orchestrator ({CodepointMapper}) partitions strategies by
|
|
24
|
+
# this predicate, gates positional strategies behind a
|
|
25
|
+
# block-scope check, and merges with positional precedence
|
|
26
|
+
# when both produce a result. Adding a new strategy = one
|
|
27
|
+
# subclass + one `positional?` override + one entry in the
|
|
28
|
+
# chain. No edit to CodepointMapper (Open/Closed Principle).
|
|
29
|
+
class Strategy
|
|
30
|
+
# @param descriptor [RawFontDescriptor]
|
|
31
|
+
def supports?(_descriptor)
|
|
32
|
+
raise NotImplementedError
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @param descriptor [RawFontDescriptor]
|
|
36
|
+
# @return [Hash{Integer=>Integer}] codepoint => gid; empty
|
|
37
|
+
# when the strategy cannot produce a mapping
|
|
38
|
+
def map(_descriptor)
|
|
39
|
+
raise NotImplementedError
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Declares whether this strategy attributes glyphs by chart
|
|
43
|
+
# geometry (true) or by the font's intrinsic mapping (false).
|
|
44
|
+
# The default is false; positional strategies override.
|
|
45
|
+
#
|
|
46
|
+
# @return [Boolean]
|
|
47
|
+
def positional?
|
|
48
|
+
false
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
|
|
4
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/tounicode"
|
|
6
|
+
|
|
7
|
+
module Ucode
|
|
8
|
+
module Glyphs
|
|
9
|
+
module EmbeddedFonts
|
|
10
|
+
class CodepointMapper
|
|
11
|
+
# Strategy 1 — read the Type0 font's `/ToUnicode` CMap stream
|
|
12
|
+
# and parse it into a `{codepoint => gid}` map. Highest
|
|
13
|
+
# fidelity when present; usually missing for subsetted CID
|
|
14
|
+
# fonts (the case the trace fallback exists for).
|
|
15
|
+
class ToUnicodeStrategy < Strategy
|
|
16
|
+
# @param source [PdfSource]
|
|
17
|
+
# @param mutool_show [Mutool::Show]
|
|
18
|
+
def initialize(source:, mutool_show:)
|
|
19
|
+
super()
|
|
20
|
+
@source = source
|
|
21
|
+
@mutool_show = mutool_show
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def supports?(descriptor)
|
|
25
|
+
descriptor.cid_map_kind == :identity &&
|
|
26
|
+
!descriptor.tounicode_ref.nil?
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def map(descriptor)
|
|
30
|
+
cmap_text = @mutool_show.stream(@source.pdf_to_s,
|
|
31
|
+
descriptor.tounicode_ref)
|
|
32
|
+
cid_to_cp = ToUnicode.parse(cmap_text)
|
|
33
|
+
cid_to_cp.each_with_object({}) do |(cid, cp), h|
|
|
34
|
+
h[cp] = cid
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
|
|
4
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/page_trace_cache"
|
|
6
|
+
require "ucode/glyphs/embedded_fonts/trace_correlator"
|
|
7
|
+
|
|
8
|
+
module Ucode
|
|
9
|
+
module Glyphs
|
|
10
|
+
module EmbeddedFonts
|
|
11
|
+
class CodepointMapper
|
|
12
|
+
# Strategy 3 — auto-detect via `mutool trace`. Last-resort
|
|
13
|
+
# fallback for CID fonts without /ToUnicode and without a
|
|
14
|
+
# caller-supplied correlator config.
|
|
15
|
+
#
|
|
16
|
+
# Consumes a {PageTraceCache} that traces each PDF page
|
|
17
|
+
# exactly once across all CID fonts. The pre-cache path was
|
|
18
|
+
# O(F × P) subprocess invocations; this is O(P).
|
|
19
|
+
#
|
|
20
|
+
# Correlates per page (Y positions are page-local — clustering
|
|
21
|
+
# across page boundaries would produce false matches).
|
|
22
|
+
class TraceStrategy < Strategy
|
|
23
|
+
# @param cache [PageTraceCache, nil] nil = strategy is a no-op
|
|
24
|
+
# @param indexer [PdfIndexer] for the cheap font_appears?
|
|
25
|
+
# precondition (avoids touching the cache for fonts the
|
|
26
|
+
# PDF doesn't reference at all)
|
|
27
|
+
def initialize(cache:, indexer:)
|
|
28
|
+
super()
|
|
29
|
+
@cache = cache
|
|
30
|
+
@indexer = indexer
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def supports?(descriptor)
|
|
34
|
+
return false unless @cache
|
|
35
|
+
return false unless descriptor.cid_map_kind == :identity
|
|
36
|
+
|
|
37
|
+
@indexer.font_appears?(descriptor.base_font)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @see Strategy#positional?
|
|
41
|
+
def positional?
|
|
42
|
+
true
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def map(descriptor)
|
|
46
|
+
return {} unless @cache
|
|
47
|
+
|
|
48
|
+
correlator = TraceCorrelator.new(
|
|
49
|
+
specimen_font_name: descriptor.base_font,
|
|
50
|
+
)
|
|
51
|
+
mapping = {}
|
|
52
|
+
@cache.each_page_for(descriptor.base_font) do |_page, glyphs|
|
|
53
|
+
page_mapping = correlator.correlate(glyphs)
|
|
54
|
+
page_mapping.each do |cp, gid|
|
|
55
|
+
mapping[cp] ||= gid
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
mapping
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -1,128 +1,181 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "
|
|
4
|
-
require "
|
|
5
|
-
require "
|
|
3
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/strategy"
|
|
4
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/tounicode_strategy"
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/correlator_strategy"
|
|
6
|
+
require "ucode/glyphs/embedded_fonts/codepoint_mapper/trace_strategy"
|
|
7
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
8
|
+
require "ucode/glyphs/embedded_fonts/page_trace_cache"
|
|
9
|
+
require "ucode/error"
|
|
6
10
|
|
|
7
11
|
module Ucode
|
|
8
12
|
module Glyphs
|
|
9
13
|
module EmbeddedFonts
|
|
10
|
-
# Resolves codepoint → GID for one Type0 font via a
|
|
14
|
+
# Resolves codepoint → GID for one Type0 font via a chain of
|
|
15
|
+
# {Strategy} subclasses.
|
|
11
16
|
#
|
|
12
|
-
#
|
|
13
|
-
# for pillar 1). Parsed by {ToUnicode}.
|
|
14
|
-
# 2. **Caller-supplied correlator config** (pillar 2) — render the
|
|
15
|
-
# font's pages to SVG and run {ContentStreamCorrelator}.
|
|
16
|
-
# 3. **Auto-detect via mutool trace** (pillar 2b) — trace every
|
|
17
|
-
# page and run {TraceCorrelator} positionally.
|
|
17
|
+
# The chain is partitioned by {Strategy#positional?}:
|
|
18
18
|
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
19
|
+
# * Intrinsic strategies (ToUnicodeStrategy) read the font's
|
|
20
|
+
# own CMap. Tried in chain order; the first non-empty result
|
|
21
|
+
# wins.
|
|
22
|
+
# * Positional strategies (CorrelatorStrategy, TraceStrategy)
|
|
23
|
+
# attribute glyphs by chart-grid geometry. Expensive — they
|
|
24
|
+
# shell out to mutool per page — so they only run when an
|
|
25
|
+
# intrinsic strategy cannot cover the requested block scope.
|
|
21
26
|
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
27
|
+
# When positional strategies run, their results merge over the
|
|
28
|
+
# intrinsic result with positional precedence: chart geometry is
|
|
29
|
+
# authoritative for in-block specimens, while a font's CMap can
|
|
30
|
+
# be misleading (Enclosed Ideographic Supplement, where the
|
|
31
|
+
# embedded CJKSymbols font's CMap maps its CIDs to the *composing*
|
|
32
|
+
# ideographs rather than the squared characters themselves).
|
|
33
|
+
#
|
|
34
|
+
# Two escape hatches let callers control the gating:
|
|
35
|
+
#
|
|
36
|
+
# * `block_range:` — when set, an intrinsic result with zero
|
|
37
|
+
# in-block intersection is treated as "wrong scope" and
|
|
38
|
+
# dropped, so positional strategies can take over. Auto-detect
|
|
39
|
+
# for the U1F200 class of failure (Option 1).
|
|
40
|
+
# * `force_positional_for_font_ids:` — Type0 font object IDs
|
|
41
|
+
# that always go through positional attribution, regardless
|
|
42
|
+
# of whether the intrinsic strategy succeeded. Escape hatch
|
|
43
|
+
# for partial-overlap cases where the CMap covers some
|
|
44
|
+
# in-block codepoints but positional attribution is still
|
|
45
|
+
# desired for the rest (Option 2).
|
|
46
|
+
#
|
|
47
|
+
# Adding a new strategy = one Strategy subclass + (if positional)
|
|
48
|
+
# a `positional?` override + one entry in the chain. No edit to
|
|
49
|
+
# {#map} (Open/Closed Principle).
|
|
26
50
|
class CodepointMapper
|
|
27
|
-
# @param
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
51
|
+
# @param strategies [Array<Strategy>] chain; partitioned by
|
|
52
|
+
# Strategy#positional? at map time
|
|
53
|
+
# @param block_range [Range<Integer>, nil] codepoint scope the
|
|
54
|
+
# caller is extracting. nil = intrinsic result is always
|
|
55
|
+
# trusted (legacy behavior; the caller doesn't know or care
|
|
56
|
+
# about block scope).
|
|
57
|
+
# @param force_positional_for_font_ids [Set<Integer>] Type0
|
|
58
|
+
# font object IDs that always trigger positional attribution
|
|
59
|
+
def initialize(strategies:, block_range: nil,
|
|
60
|
+
force_positional_for_font_ids: Set.new)
|
|
61
|
+
@strategies = strategies
|
|
62
|
+
@block_range = block_range
|
|
63
|
+
@force_positional_for_font_ids = force_positional_for_font_ids
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Convenience builder — wires up the default 3-strategy chain
|
|
67
|
+
# with default Mutool wrappers. Callers that need to inject
|
|
68
|
+
# stubs for tests should construct strategies directly and pass
|
|
69
|
+
# them to +#initialize+.
|
|
70
|
+
#
|
|
71
|
+
# @return [CodepointMapper]
|
|
72
|
+
def self.build(source:, correlator_configs:, indexer:,
|
|
73
|
+
block_range: nil, force_positional_for_font_ids: Set.new,
|
|
74
|
+
mutool_show: Mutool::Show.new,
|
|
75
|
+
mutool_draw: Mutool::Draw.new,
|
|
76
|
+
mutool_trace: Mutool::Trace.new)
|
|
77
|
+
trace_cache = PageTraceCache.new(
|
|
78
|
+
pdf: source.pdf_path,
|
|
79
|
+
page_count: indexer.page_count,
|
|
80
|
+
mutool: mutool_trace,
|
|
81
|
+
)
|
|
82
|
+
strategies = [
|
|
83
|
+
ToUnicodeStrategy.new(source: source, mutool_show: mutool_show),
|
|
84
|
+
CorrelatorStrategy.new(source: source,
|
|
85
|
+
correlator_configs: correlator_configs,
|
|
86
|
+
mutool_draw: mutool_draw),
|
|
87
|
+
TraceStrategy.new(cache: trace_cache, indexer: indexer),
|
|
88
|
+
]
|
|
89
|
+
new(strategies: strategies,
|
|
90
|
+
block_range: block_range,
|
|
91
|
+
force_positional_for_font_ids: force_positional_for_font_ids)
|
|
35
92
|
end
|
|
36
93
|
|
|
37
94
|
# @param descriptor [RawFontDescriptor]
|
|
38
|
-
# @return [Hash{Integer=>Integer}] codepoint => gid; empty
|
|
39
|
-
# no strategy produces a
|
|
95
|
+
# @return [Hash{Integer=>Integer}] codepoint => gid; empty
|
|
96
|
+
# when no strategy produces a mapping
|
|
40
97
|
def map(descriptor)
|
|
41
98
|
return {} unless descriptor.cid_map_kind == :identity
|
|
42
99
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
100
|
+
positional, intrinsic = partition_strategies
|
|
101
|
+
intrinsic_result = run_intrinsic(descriptor, intrinsic)
|
|
102
|
+
return intrinsic_result if positional.empty?
|
|
103
|
+
return intrinsic_result unless needs_positional?(descriptor,
|
|
104
|
+
intrinsic_result)
|
|
48
105
|
|
|
49
|
-
|
|
106
|
+
positional_result = run_positional(descriptor, positional)
|
|
107
|
+
merge_with_positional_precedence(intrinsic_result, positional_result)
|
|
50
108
|
end
|
|
51
109
|
|
|
52
110
|
private
|
|
53
111
|
|
|
54
|
-
#
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
build_codepoint_map(cid_to_cp)
|
|
112
|
+
# `Enumerable#partition` returns `[truthy_group, falsy_group]`,
|
|
113
|
+
# so the first element is positional strategies and the second
|
|
114
|
+
# is intrinsic. Naming them explicitly here avoids the
|
|
115
|
+
# destructure-order footgun.
|
|
116
|
+
def partition_strategies
|
|
117
|
+
positional, intrinsic = @strategies.partition(&:positional?)
|
|
118
|
+
[positional, intrinsic]
|
|
62
119
|
end
|
|
63
120
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
121
|
+
# Intrinsic chain: first non-empty result wins. preserves the
|
|
122
|
+
# pre-partition semantics so the legacy ToUnicode-only flow
|
|
123
|
+
# behaves identically when no positional strategies exist.
|
|
124
|
+
def run_intrinsic(descriptor, strategies)
|
|
125
|
+
strategies.each do |s|
|
|
126
|
+
next unless s.supports?(descriptor)
|
|
69
127
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
tmp.close
|
|
73
|
-
ok = system("mutool", "show", "-o", tmp.path, "-b",
|
|
74
|
-
@source.pdf_to_s, obj_id.to_s,
|
|
75
|
-
out: File::NULL, err: File::NULL)
|
|
76
|
-
unless ok
|
|
77
|
-
raise Ucode::EmbeddedFontsMissingError,
|
|
78
|
-
"mutool show failed for ToUnicode obj=#{obj_id}"
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
File.binread(tmp.path).force_encoding("UTF-8")
|
|
128
|
+
result = s.map(descriptor)
|
|
129
|
+
return result unless result.empty?
|
|
82
130
|
end
|
|
131
|
+
{}
|
|
83
132
|
end
|
|
84
133
|
|
|
85
|
-
#
|
|
134
|
+
# Positional strategies are gated behind three conditions, any
|
|
135
|
+
# of which triggers them:
|
|
136
|
+
#
|
|
137
|
+
# 1. Caller explicitly listed this font in
|
|
138
|
+
# `force_positional_for_font_ids` (Option 2 escape hatch).
|
|
139
|
+
# 2. No intrinsic strategy produced a mapping (legacy
|
|
140
|
+
# fallback for fonts without ToUnicode).
|
|
141
|
+
# 3. The intrinsic result fell entirely outside the
|
|
142
|
+
# caller's block scope — the font's CMap encoded the
|
|
143
|
+
# wrong codepoints (Option 1 auto-detect, e.g. U1F200).
|
|
144
|
+
def needs_positional?(descriptor, intrinsic_result)
|
|
145
|
+
return true if @force_positional_for_font_ids.include?(descriptor.font_obj_id)
|
|
146
|
+
return true if intrinsic_result.empty?
|
|
147
|
+
return true if intrinsic_out_of_scope?(intrinsic_result)
|
|
148
|
+
|
|
149
|
+
false
|
|
150
|
+
end
|
|
86
151
|
|
|
87
|
-
def
|
|
88
|
-
|
|
89
|
-
return {} unless config
|
|
152
|
+
def intrinsic_out_of_scope?(intrinsic_result)
|
|
153
|
+
return false unless @block_range
|
|
90
154
|
|
|
91
|
-
|
|
92
|
-
ContentStreamCorrelator.new(config).correlate(svg)
|
|
155
|
+
intrinsic_result.keys.none? { |cp| @block_range.include?(cp) }
|
|
93
156
|
end
|
|
94
157
|
|
|
95
|
-
|
|
96
|
-
|
|
158
|
+
# Positional chain: union of all positional strategies' results.
|
|
159
|
+
# Within positional, earlier strategies win on conflict (chain
|
|
160
|
+
# order expresses caller preference — CorrelatorStrategy
|
|
161
|
+
# before TraceStrategy when both are configured).
|
|
162
|
+
def run_positional(descriptor, strategies)
|
|
163
|
+
merged = {}
|
|
164
|
+
strategies.each do |s|
|
|
165
|
+
next unless s.supports?(descriptor)
|
|
97
166
|
|
|
98
|
-
|
|
99
|
-
"mutool", "draw", "-F", "svg",
|
|
100
|
-
@source.pdf_to_s,
|
|
101
|
-
*page_numbers.map(&:to_s),
|
|
102
|
-
)
|
|
103
|
-
unless status.success?
|
|
104
|
-
raise Ucode::EmbeddedFontsMissingError,
|
|
105
|
-
"mutool draw failed: #{err.strip}"
|
|
167
|
+
s.map(descriptor).each { |cp, gid| merged[cp] ||= gid }
|
|
106
168
|
end
|
|
107
|
-
|
|
108
|
-
out
|
|
169
|
+
merged
|
|
109
170
|
end
|
|
110
171
|
|
|
111
|
-
#
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
(1..@indexer.page_count).each_with_object({}) do |page, mapping|
|
|
120
|
-
glyphs = runner.trace([page])
|
|
121
|
-
page_mapping = correlator.correlate(glyphs)
|
|
122
|
-
page_mapping.each do |cp, gid|
|
|
123
|
-
mapping[cp] ||= gid
|
|
124
|
-
end
|
|
125
|
-
end
|
|
172
|
+
# Merge intrinsic and positional results. Positional wins on
|
|
173
|
+
# conflict (chart geometry is authoritative for in-block
|
|
174
|
+
# specimens).
|
|
175
|
+
def merge_with_positional_precedence(intrinsic, positional)
|
|
176
|
+
merged = intrinsic.dup
|
|
177
|
+
positional.each { |cp, gid| merged[cp] = gid }
|
|
178
|
+
merged
|
|
126
179
|
end
|
|
127
180
|
end
|
|
128
181
|
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
6
|
+
|
|
7
|
+
module Ucode
|
|
8
|
+
module Glyphs
|
|
9
|
+
module EmbeddedFonts
|
|
10
|
+
module Mutool
|
|
11
|
+
class Draw
|
|
12
|
+
def initialize(runner: SystemRunner.new)
|
|
13
|
+
@runner = runner
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def svg(pdf, *pages)
|
|
17
|
+
return "" if pages.empty?
|
|
18
|
+
|
|
19
|
+
@runner.run("mutool", "draw", "-F", "svg", pdf.to_s,
|
|
20
|
+
*pages.map(&:to_s))
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
|
|
5
|
+
require "ucode/glyphs/embedded_fonts/mutool"
|
|
6
|
+
|
|
7
|
+
module Ucode
|
|
8
|
+
module Glyphs
|
|
9
|
+
module EmbeddedFonts
|
|
10
|
+
module Mutool
|
|
11
|
+
class Info
|
|
12
|
+
def initialize(runner: SystemRunner.new)
|
|
13
|
+
@runner = runner
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call(pdf)
|
|
17
|
+
@runner.run("mutool", "info", pdf.to_s)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|