ucode 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +10 -2
- data/README.md +66 -20
- data/Rakefile +19 -8
- data/TODO.extract-code-chart/01-pdf-fetch-validation.md +80 -0
- data/TODO.extract-code-chart/02-block-name-resolver.md +68 -0
- data/TODO.extract-code-chart/03-codechart-namespace.md +82 -0
- data/TODO.extract-code-chart/04-codechart-extractor.md +154 -0
- data/TODO.extract-code-chart/05-provenance-and-sidecar.md +147 -0
- data/TODO.extract-code-chart/06-codechart-writer.md +134 -0
- data/TODO.extract-code-chart/07-codechart-cli.md +135 -0
- data/TODO.extract-code-chart/08-specs.md +87 -0
- data/config/unicode17_universal_glyph_set.yml +1 -1
- data/lib/ucode/audit/reference_factory.rb +1 -1
- data/lib/ucode/cli.rb +101 -0
- data/lib/ucode/code_chart/extractor.rb +120 -0
- data/lib/ucode/code_chart/provenance.rb +82 -0
- data/lib/ucode/code_chart/sidecar.rb +52 -0
- data/lib/ucode/code_chart/writer.rb +128 -0
- data/lib/ucode/code_chart.rb +39 -0
- data/lib/ucode/commands/fetch.rb +1 -1
- data/lib/ucode/commands/glyphs.rb +1 -1
- data/lib/ucode/commands/lookup.rb +1 -1
- data/lib/ucode/commands/parse.rb +1 -1
- data/lib/ucode/coordinator/indices.rb +2 -2
- data/lib/ucode/error.rb +11 -0
- data/lib/ucode/fetch/code_charts.rb +3 -4
- data/lib/ucode/fetch/http.rb +75 -7
- data/lib/ucode/glyphs/page_renderer.rb +15 -2
- data/lib/ucode/glyphs/pipeline.rb +1 -2
- data/lib/ucode/parsers/blocks.rb +34 -0
- data/lib/ucode/repo/aggregate_writer.rb +1 -1
- data/lib/ucode/repo/writers/blocks_writer.rb +13 -13
- data/lib/ucode/repo/writers/enums_writer.rb +2 -2
- data/lib/ucode/repo/writers/indexes_writer.rb +4 -4
- data/lib/ucode/repo/writers/manifest_writer.rb +4 -4
- data/lib/ucode/repo/writers/named_sequences_writer.rb +1 -1
- data/lib/ucode/repo/writers/planes_writer.rb +17 -17
- data/lib/ucode/repo/writers/relationships_writer.rb +1 -1
- data/lib/ucode/repo/writers/scripts_writer.rb +6 -6
- data/lib/ucode/repo/writers.rb +1 -1
- data/lib/ucode/version.rb +1 -1
- data/lib/ucode.rb +3 -0
- data/ucode.gemspec +6 -1
- metadata +19 -6
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
|
|
5
|
+
require "ucode/error"
|
|
6
|
+
require "ucode/glyphs/embedded_fonts/catalog"
|
|
7
|
+
require "ucode/glyphs/embedded_fonts/renderer"
|
|
8
|
+
require "ucode/glyphs/embedded_fonts/source"
|
|
9
|
+
require "ucode/glyphs/resolver"
|
|
10
|
+
require "ucode/glyphs/sources/pillar1_embedded_tounicode"
|
|
11
|
+
require "ucode/glyphs/sources/tier1_real_font"
|
|
12
|
+
|
|
13
|
+
module Ucode
|
|
14
|
+
module CodeChart
|
|
15
|
+
# Walks every assigned codepoint in a block and returns one
|
|
16
|
+
# {Result} per codepoint that any tier produced a glyph for.
|
|
17
|
+
#
|
|
18
|
+
# This is **not** a new extraction pipeline — it composes the
|
|
19
|
+
# existing {Ucode::Glyphs::Resolver} with per-block inputs
|
|
20
|
+
# (the block's Code Charts PDF + optionally Tier 1 and Pillar 3
|
|
21
|
+
# sources). The Resolver owns tier selection; the Extractor owns
|
|
22
|
+
# inputs.
|
|
23
|
+
#
|
|
24
|
+
# The REQ (R2) describes extraction via "locate the grid cell
|
|
25
|
+
# whose margin label matches the codepoint" — that was the v0.1
|
|
26
|
+
# retired approach (cell-border compositing). The current path
|
|
27
|
+
# is the embedded-font walk (Pillar 1, via {EmbeddedFonts::Catalog})
|
|
28
|
+
# with Pillar 2 (positional correlation) and Pillar 3 (Last Resort
|
|
29
|
+
# placeholders) as fallbacks.
|
|
30
|
+
#
|
|
31
|
+
# ## Tier selection
|
|
32
|
+
#
|
|
33
|
+
# Pillar 1 is always configured (the embedded font walk over the
|
|
34
|
+
# block's PDF). Tier 1 (real-font cmap) and Pillar 3 (Last
|
|
35
|
+
# Resort) are optional — the caller injects pre-built sources.
|
|
36
|
+
# This avoids forcing the Extractor to construct Last Resort
|
|
37
|
+
# eagerly, which would fail in environments where the UFO is
|
|
38
|
+
# not checked out.
|
|
39
|
+
class Extractor
|
|
40
|
+
# Result of extracting one codepoint.
|
|
41
|
+
Result = Struct.new(:codepoint, :svg, :tier, :provenance, keyword_init: true)
|
|
42
|
+
|
|
43
|
+
# @param block [Ucode::Models::Block] block whose assigned
|
|
44
|
+
# codepoints will be extracted
|
|
45
|
+
# @param pdf_path [Pathname, String] path to the per-block
|
|
46
|
+
# Code Charts PDF (downloaded by the caller; the Extractor
|
|
47
|
+
# doesn't fetch)
|
|
48
|
+
# @param cache_dir [Pathname, String, nil] directory for
|
|
49
|
+
# cached extracted font streams. nil = default
|
|
50
|
+
# (data/pdf-fonts/ relative to the gem root).
|
|
51
|
+
# @param tier1_sources [Array<Ucode::Glyphs::Source>, nil]
|
|
52
|
+
# optional Tier 1 sources (real-font cmap). nil = no Tier 1
|
|
53
|
+
# @param pillar3_source [Ucode::Glyphs::Source, nil] optional
|
|
54
|
+
# Pillar 3 (Last Resort) source. nil = no Pillar 3 fallback.
|
|
55
|
+
# Callers that want Last Resort placeholders inject the
|
|
56
|
+
# pre-built source here.
|
|
57
|
+
def initialize(block:, pdf_path:, cache_dir: nil,
|
|
58
|
+
tier1_sources: nil, pillar3_source: nil)
|
|
59
|
+
@block = block
|
|
60
|
+
@pdf_path = Pathname.new(pdf_path)
|
|
61
|
+
@cache_dir = cache_dir && Pathname.new(cache_dir)
|
|
62
|
+
@tier1_sources = tier1_sources || []
|
|
63
|
+
@pillar3_source = pillar3_source
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# @return [Array<Result>] one Result per codepoint that any
|
|
67
|
+
# tier produced a glyph for. Codepoints no tier can serve
|
|
68
|
+
# are silently skipped (no Result yielded).
|
|
69
|
+
def extract
|
|
70
|
+
resolver = build_resolver
|
|
71
|
+
results = []
|
|
72
|
+
each_codepoint do |cp|
|
|
73
|
+
resolver_result = resolver.resolve(cp)
|
|
74
|
+
next unless resolver_result&.svg
|
|
75
|
+
|
|
76
|
+
results << Result.new(
|
|
77
|
+
codepoint: cp,
|
|
78
|
+
svg: resolver_result.svg,
|
|
79
|
+
tier: resolver_result.tier,
|
|
80
|
+
provenance: resolver_result.provenance,
|
|
81
|
+
)
|
|
82
|
+
end
|
|
83
|
+
results
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
# Yields every codepoint in the block's range in ascending
|
|
89
|
+
# order. We yield the whole range because the Resolver's
|
|
90
|
+
# tiers handle unassigned codepoints — Pillar 3 (when
|
|
91
|
+
# configured) maps every codepoint via its Format 13 cmap,
|
|
92
|
+
# so unassigned slots get a placeholder. With no Pillar 3
|
|
93
|
+
# injected, only assigned codepoints (those the embedded
|
|
94
|
+
# font actually covers) yield Results; the rest are silently
|
|
95
|
+
# skipped, satisfying the REQ's "skip unassigned codepoints".
|
|
96
|
+
def each_codepoint(&)
|
|
97
|
+
return enum_for(:each_codepoint) unless block_given?
|
|
98
|
+
|
|
99
|
+
(@block.range_first..@block.range_last).each(&)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def build_resolver
|
|
103
|
+
sources = @tier1_sources.dup
|
|
104
|
+
sources.concat(embedded_pillar_sources)
|
|
105
|
+
sources << @pillar3_source if @pillar3_source
|
|
106
|
+
order = sources.map(&:tier).uniq
|
|
107
|
+
Glyphs::Resolver.new(sources: sources, order: order)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def embedded_pillar_sources
|
|
111
|
+
embedded_source = Glyphs::EmbeddedFonts::Source.new(
|
|
112
|
+
pdf: @pdf_path, cache_dir: @cache_dir,
|
|
113
|
+
)
|
|
114
|
+
catalog = Glyphs::EmbeddedFonts::Catalog.new(embedded_source)
|
|
115
|
+
renderer = Glyphs::EmbeddedFonts::Renderer.new(catalog)
|
|
116
|
+
[Glyphs::Sources::Pillar1EmbeddedTounicode.new(renderer: renderer)]
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "time"
|
|
5
|
+
|
|
6
|
+
require "ucode/version"
|
|
7
|
+
|
|
8
|
+
module Ucode
|
|
9
|
+
module CodeChart
|
|
10
|
+
# Per-codepoint provenance value object — every field the REQ
|
|
11
|
+
# (R5) requires in the sidecar JSON next to each extracted SVG.
|
|
12
|
+
#
|
|
13
|
+
# Single source of truth for the provenance schema: the
|
|
14
|
+
# {Sidecar} writer reads this Struct, the Writer constructs it.
|
|
15
|
+
# Adding a field is one place to change.
|
|
16
|
+
#
|
|
17
|
+
# `extractor_version` reads from `Ucode::VERSION` at construction
|
|
18
|
+
# so the field stays in sync with the gem's version bump — single
|
|
19
|
+
# source of truth.
|
|
20
|
+
#
|
|
21
|
+
# `extracted_at` is the extraction event timestamp (UTC ISO8601),
|
|
22
|
+
# not the file-write timestamp.
|
|
23
|
+
Provenance = Struct.new(
|
|
24
|
+
:codepoint,
|
|
25
|
+
:block,
|
|
26
|
+
:source_pdf_url,
|
|
27
|
+
:source_pdf_sha256,
|
|
28
|
+
:ucd_version,
|
|
29
|
+
:extracted_at,
|
|
30
|
+
:extractor_version,
|
|
31
|
+
keyword_init: true,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# Computes the source PDF's URL from a block name and first
|
|
35
|
+
# codepoint. Mirrors the per-block URL convention in
|
|
36
|
+
# {Ucode::Fetch::CodeCharts}: the hex representation of the
|
|
37
|
+
# codepoint, zero-padded to a minimum of 4 digits (e.g.
|
|
38
|
+
# `U0000.pdf` for BMP, `U10920.pdf` for Plane 1,
|
|
39
|
+
# `U100000.pdf` for Plane 16 SPUA-B).
|
|
40
|
+
#
|
|
41
|
+
# @param block_first_cp [Integer]
|
|
42
|
+
# @return [String]
|
|
43
|
+
def self.code_chart_url(block_first_cp)
|
|
44
|
+
slug = block_first_cp.to_s(16).upcase.rjust(4, "0")
|
|
45
|
+
"#{Ucode.configuration.charts_base_url}/U#{slug}.pdf"
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Builds a Provenance from the inputs the {Writer} has on hand
|
|
49
|
+
# (block, codepoint, ucd_version, pdf_path). Computes the PDF
|
|
50
|
+
# hash + URL once. The `extracted_at` timestamp is fixed at
|
|
51
|
+
# call time so re-running the same block produces identical
|
|
52
|
+
# provenance JSON for unchanged codepoints.
|
|
53
|
+
#
|
|
54
|
+
# @param block [Ucode::Models::Block]
|
|
55
|
+
# @param codepoint [Integer]
|
|
56
|
+
# @param ucd_version [String]
|
|
57
|
+
# @param pdf_path [Pathname, String]
|
|
58
|
+
# @param now [Time, nil] override for tests
|
|
59
|
+
# @return [Provenance]
|
|
60
|
+
def self.build(block:, codepoint:, ucd_version:, pdf_path:, now: nil)
|
|
61
|
+
path = Pathname.new(pdf_path)
|
|
62
|
+
Provenance.new(
|
|
63
|
+
codepoint: format("U+%04X", codepoint),
|
|
64
|
+
block: block.id,
|
|
65
|
+
source_pdf_url: code_chart_url(block.range_first),
|
|
66
|
+
source_pdf_sha256: sha256_of(path),
|
|
67
|
+
ucd_version: ucd_version,
|
|
68
|
+
extracted_at: (now || Time.now.utc).iso8601,
|
|
69
|
+
extractor_version: Ucode::VERSION,
|
|
70
|
+
)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @param path [Pathname]
|
|
74
|
+
# @return [String] hex digest, "" when the path doesn't exist
|
|
75
|
+
# (callers can decide how to handle a missing hash)
|
|
76
|
+
def self.sha256_of(path)
|
|
77
|
+
return "" unless path.exist?
|
|
78
|
+
|
|
79
|
+
Digest::SHA256.file(path).hexdigest
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "pathname"
|
|
5
|
+
|
|
6
|
+
require "ucode/repo/atomic_writes"
|
|
7
|
+
|
|
8
|
+
module Ucode
|
|
9
|
+
module CodeChart
|
|
10
|
+
# Writes a {Provenance} to disk as the sidecar JSON next to its
|
|
11
|
+
# corresponding SVG.
|
|
12
|
+
#
|
|
13
|
+
# Path: `<output_root>/<codepoint>.json` — colocated with the
|
|
14
|
+
# SVG so a downstream consumer can find both files by a single
|
|
15
|
+
# directory listing.
|
|
16
|
+
#
|
|
17
|
+
# Idempotent via {Ucode::Repo::AtomicWrites#write_atomic}: a
|
|
18
|
+
# re-write of byte-identical content is a no-op (no temp-file
|
|
19
|
+
# rename). Provenance JSON is canonical (sorted keys via Ruby's
|
|
20
|
+
# stdlib JSON), so the byte-equality test is sound.
|
|
21
|
+
class Sidecar
|
|
22
|
+
include Ucode::Repo::AtomicWrites
|
|
23
|
+
|
|
24
|
+
# @param output_root [Pathname, String] directory the SVG +
|
|
25
|
+
# sidecar live in. Parent directories are created on demand.
|
|
26
|
+
def initialize(output_root:)
|
|
27
|
+
@output_root = Pathname.new(output_root)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @param provenance [Ucode::CodeChart::Provenance]
|
|
31
|
+
# @return [Pathname] the written sidecar path
|
|
32
|
+
def write(provenance)
|
|
33
|
+
path = path_for(provenance)
|
|
34
|
+
payload = "#{JSON.pretty_generate(provenance.to_h)}\n"
|
|
35
|
+
write_atomic(path, payload)
|
|
36
|
+
path
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @param codepoint_id [String] e.g. "U+10920"
|
|
40
|
+
# @return [Pathname] the would-be path for a sidecar
|
|
41
|
+
def path_for_id(codepoint_id)
|
|
42
|
+
@output_root.join("#{codepoint_id}.json")
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def path_for(provenance)
|
|
48
|
+
path_for_id(provenance.codepoint)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "pathname"
|
|
5
|
+
|
|
6
|
+
require "ucode/code_chart/extractor"
|
|
7
|
+
require "ucode/code_chart/provenance"
|
|
8
|
+
require "ucode/code_chart/sidecar"
|
|
9
|
+
require "ucode/error"
|
|
10
|
+
require "ucode/version_resolver"
|
|
11
|
+
|
|
12
|
+
module Ucode
|
|
13
|
+
module CodeChart
|
|
14
|
+
# Orchestrates extraction + provenance sidecar writing for one
|
|
15
|
+
# block. The Writer is the **only thing that touches disk** in the
|
|
16
|
+
# CodeChart namespace; everything else is composition.
|
|
17
|
+
#
|
|
18
|
+
# Output layout (per block):
|
|
19
|
+
#
|
|
20
|
+
# <output_root>/<block_id>/<U+XXXX>.svg
|
|
21
|
+
# <output_root>/<block_id>/<U+XXXX>.json # provenance sidecar
|
|
22
|
+
#
|
|
23
|
+
# One folder per block keeps each block's output self-contained
|
|
24
|
+
# and discoverable — a downstream consumer (fontisan) can iterate
|
|
25
|
+
# a block's folder without scanning the whole tree.
|
|
26
|
+
#
|
|
27
|
+
# Idempotent: re-running `write` on the same inputs produces
|
|
28
|
+
# byte-identical files (SVGs via content check; sidecars via
|
|
29
|
+
# {Ucode::Repo::AtomicWrites#write_atomic}'s canonical-JSON
|
|
30
|
+
# byte-equality). The {Summary} tally distinguishes "first run"
|
|
31
|
+
# writes from no-op re-writes.
|
|
32
|
+
class Writer
|
|
33
|
+
# Per-block run summary. Returned from {#write}.
|
|
34
|
+
Summary = Struct.new(
|
|
35
|
+
:block,
|
|
36
|
+
:codepoints_extracted,
|
|
37
|
+
:svgs_written,
|
|
38
|
+
:sidecars_written,
|
|
39
|
+
:pdf_sha256,
|
|
40
|
+
keyword_init: true,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# @param output_root [Pathname, String] parent directory. The
|
|
44
|
+
# `<block_id>/` subdirectory is created inside it.
|
|
45
|
+
# @param pdf_path [Pathname, String] Code Charts PDF (already
|
|
46
|
+
# downloaded by the caller; Writer doesn't fetch).
|
|
47
|
+
# @param ucd_version [String, nil] UCD version to stamp on
|
|
48
|
+
# provenance. nil = resolved via {VersionResolver.resolve(nil)}.
|
|
49
|
+
# @param cache_dir [Pathname, String, nil] font-stream cache
|
|
50
|
+
# directory for the EmbeddedFonts::Source.
|
|
51
|
+
# @param now [Time, nil] timestamp override (for tests).
|
|
52
|
+
# @param pillar3_source, tier1_sources: forwarded to the Extractor.
|
|
53
|
+
def initialize(output_root:, pdf_path:, ucd_version: nil,
|
|
54
|
+
cache_dir: nil, now: nil,
|
|
55
|
+
pillar3_source: nil, tier1_sources: nil)
|
|
56
|
+
@output_root = Pathname.new(output_root)
|
|
57
|
+
@pdf_path = Pathname.new(pdf_path)
|
|
58
|
+
@ucd_version = ucd_version || VersionResolver.resolve(nil)
|
|
59
|
+
@cache_dir = cache_dir && Pathname.new(cache_dir)
|
|
60
|
+
@now = now
|
|
61
|
+
@pillar3_source = pillar3_source
|
|
62
|
+
@tier1_sources = tier1_sources
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Extracts every codepoint in `block` and writes `<block_id>/<cp>.svg`
|
|
66
|
+
# + `<block_id>/<cp>.json` under `@output_root`. Returns a
|
|
67
|
+
# {Summary} tally.
|
|
68
|
+
#
|
|
69
|
+
# @param block [Ucode::Models::Block]
|
|
70
|
+
# @return [Summary]
|
|
71
|
+
def write(block)
|
|
72
|
+
block_dir = @output_root.join(block.id)
|
|
73
|
+
block_dir.mkpath
|
|
74
|
+
|
|
75
|
+
pdf_sha = CodeChart.sha256_of(@pdf_path)
|
|
76
|
+
|
|
77
|
+
sidecar = Sidecar.new(output_root: block_dir)
|
|
78
|
+
extractor = Extractor.new(
|
|
79
|
+
block: block,
|
|
80
|
+
pdf_path: @pdf_path,
|
|
81
|
+
cache_dir: @cache_dir,
|
|
82
|
+
pillar3_source: @pillar3_source,
|
|
83
|
+
tier1_sources: @tier1_sources,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
results = extractor.extract
|
|
87
|
+
svgs = 0
|
|
88
|
+
sidecars = 0
|
|
89
|
+
results.each do |result|
|
|
90
|
+
write_svg(block_dir, result)
|
|
91
|
+
svgs += 1
|
|
92
|
+
provenance = CodeChart.build(
|
|
93
|
+
block: block, codepoint: result.codepoint,
|
|
94
|
+
ucd_version: @ucd_version, pdf_path: @pdf_path,
|
|
95
|
+
now: @now,
|
|
96
|
+
)
|
|
97
|
+
sidecar.write(provenance)
|
|
98
|
+
sidecars += 1
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
Summary.new(
|
|
102
|
+
block: block.id,
|
|
103
|
+
codepoints_extracted: results.size,
|
|
104
|
+
svgs_written: svgs,
|
|
105
|
+
sidecars_written: sidecars,
|
|
106
|
+
pdf_sha256: pdf_sha,
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
private
|
|
111
|
+
|
|
112
|
+
# Writes one SVG, skipping the write when the existing content
|
|
113
|
+
# is byte-identical (so mtime is preserved on idempotent
|
|
114
|
+
# re-runs — the Sidecar uses `Repo::AtomicWrites` for the same
|
|
115
|
+
# reason but at a different layer).
|
|
116
|
+
def write_svg(block_dir, result)
|
|
117
|
+
path = block_dir.join("#{format_cp(result.codepoint)}.svg")
|
|
118
|
+
return if path.exist? && path.read == result.svg
|
|
119
|
+
|
|
120
|
+
path.write(result.svg)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def format_cp(codepoint)
|
|
124
|
+
"U+#{codepoint.to_s(16).upcase.rjust(4, '0')}"
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ucode
|
|
4
|
+
# CodeChart — per-codepoint SVG glyph extraction from Unicode Code
|
|
5
|
+
# Charts PDFs.
|
|
6
|
+
#
|
|
7
|
+
# The "Code Chart donor" use case (essenfont consumer): for blocks
|
|
8
|
+
# where no OFL real-font covers the glyphs (Sidetic in Unicode 17,
|
|
9
|
+
# Egyptian Hieroglyphs Extended-B), the only canonical source is
|
|
10
|
+
# the Unicode Consortium's Code Chart PDF. This namespace turns one
|
|
11
|
+
# such PDF into a tree of standalone SVG files plus provenance
|
|
12
|
+
# sidecar JSON.
|
|
13
|
+
#
|
|
14
|
+
# ## Architecture (MECE)
|
|
15
|
+
#
|
|
16
|
+
# Every concern has exactly one home:
|
|
17
|
+
#
|
|
18
|
+
# * **Block metadata** (range + assigned codepoints) — Parsers::Blocks
|
|
19
|
+
# * **PDF download + cache** — Fetch::CodeCharts + Glyphs::PdfFetcher
|
|
20
|
+
# * **PDF object-graph walk + font extraction** — Glyphs::EmbeddedFonts::*
|
|
21
|
+
# * **Tier selection (Pillar 1 / 2 / 3)** — Glyphs::Resolver
|
|
22
|
+
# * **SVG conversion + y-flip + viewBox** — Glyphs::EmbeddedFonts::Svg
|
|
23
|
+
# * **Provenance schema** — CodeChart::Provenance (this namespace)
|
|
24
|
+
# * **Sidecar JSON write** — CodeChart::Sidecar (this namespace)
|
|
25
|
+
# * **Per-block orchestration + idempotent disk write** — CodeChart::Writer
|
|
26
|
+
# * **CLI dispatch** — Cli::CodeChartCmd
|
|
27
|
+
#
|
|
28
|
+
# CodeChart::* is the feature-facing namespace. It does not
|
|
29
|
+
# implement extraction, font parsing, or PDF I/O — it composes
|
|
30
|
+
# the existing infrastructure. Replacing the implementation
|
|
31
|
+
# (e.g. a future pure-Ruby PDF parser per ADR-0001) does not
|
|
32
|
+
# change the public API.
|
|
33
|
+
module CodeChart
|
|
34
|
+
autoload :Extractor, "ucode/code_chart/extractor"
|
|
35
|
+
autoload :Provenance, "ucode/code_chart/provenance"
|
|
36
|
+
autoload :Sidecar, "ucode/code_chart/sidecar"
|
|
37
|
+
autoload :Writer, "ucode/code_chart/writer"
|
|
38
|
+
end
|
|
39
|
+
end
|
data/lib/ucode/commands/fetch.rb
CHANGED
data/lib/ucode/commands/parse.rb
CHANGED
|
@@ -69,7 +69,7 @@ module Ucode
|
|
|
69
69
|
# `output/relationships/`
|
|
70
70
|
# @yieldparam records [Hash<Integer|String, Record|Array<Record>>]
|
|
71
71
|
# @return [Enumerator] when no block is given
|
|
72
|
-
def each_relationship(&
|
|
72
|
+
def each_relationship(&)
|
|
73
73
|
return enum_for(:each_relationship) unless block_given?
|
|
74
74
|
|
|
75
75
|
RELATIONSHIPS.each do |slug, field|
|
|
@@ -78,4 +78,4 @@ module Ucode
|
|
|
78
78
|
end
|
|
79
79
|
end
|
|
80
80
|
end
|
|
81
|
-
end
|
|
81
|
+
end
|
data/lib/ucode/error.rb
CHANGED
|
@@ -97,6 +97,10 @@ module Ucode
|
|
|
97
97
|
# Version string not in Config.known_versions.
|
|
98
98
|
class UnknownVersionError < LookupError; end
|
|
99
99
|
|
|
100
|
+
# Block identifier not present in the cached Blocks.txt. Carries the
|
|
101
|
+
# offending id and the path searched in `context:`.
|
|
102
|
+
class UnknownBlockError < LookupError; end
|
|
103
|
+
|
|
100
104
|
# Glyph pipeline failures.
|
|
101
105
|
class GlyphError < Error; end
|
|
102
106
|
|
|
@@ -114,6 +118,13 @@ module Ucode
|
|
|
114
118
|
# `mutool` is not installed on the PATH.
|
|
115
119
|
class EmbeddedFontsMissingError < GlyphError; end
|
|
116
120
|
|
|
121
|
+
# The Code Charts PDF for a requested block cannot be obtained: the
|
|
122
|
+
# network returned 4xx/5xx, the response wasn't application/pdf, or
|
|
123
|
+
# the body didn't start with the `%PDF` magic. Distinct from
|
|
124
|
+
# {EmbeddedFontsMissingError} (which fires when the file is already
|
|
125
|
+
# on disk and we just can't open it): this fires at fetch time.
|
|
126
|
+
class CodeChartNotFoundError < GlyphError; end
|
|
127
|
+
|
|
117
128
|
# Pre-build validation failed for a universal-set build. The
|
|
118
129
|
# context carries the failing checks so the CLI can render a
|
|
119
130
|
# useful diagnostic without re-running them. Distinct from
|
|
@@ -6,7 +6,7 @@ module Ucode
|
|
|
6
6
|
#
|
|
7
7
|
# URL pattern: `https://www.unicode.org/charts/PDF/U<XXXX>.pdf`
|
|
8
8
|
# where `XXXX` is the block's first codepoint zero-padded to 4 digits
|
|
9
|
-
# (5
|
|
9
|
+
# (5 digits for planes > 0).
|
|
10
10
|
module CodeCharts
|
|
11
11
|
class << self
|
|
12
12
|
# @param version [String] used as the on-disk path namespace; PDFs
|
|
@@ -29,7 +29,7 @@ module Ucode
|
|
|
29
29
|
next if dest.exist? && !force
|
|
30
30
|
|
|
31
31
|
url = "#{Ucode.configuration.charts_base_url}/#{filename}"
|
|
32
|
-
Http.get(url, dest: dest)
|
|
32
|
+
Http.get(url, dest: dest, validate: :pdf)
|
|
33
33
|
downloaded += 1
|
|
34
34
|
end
|
|
35
35
|
downloaded
|
|
@@ -48,8 +48,7 @@ module Ucode
|
|
|
48
48
|
private
|
|
49
49
|
|
|
50
50
|
def hex_pad(codepoint)
|
|
51
|
-
|
|
52
|
-
codepoint.to_s(16).upcase.rjust(width, "0")
|
|
51
|
+
codepoint.to_s(16).upcase.rjust(4, "0")
|
|
53
52
|
end
|
|
54
53
|
end
|
|
55
54
|
end
|
data/lib/ucode/fetch/http.rb
CHANGED
|
@@ -23,9 +23,17 @@ module Ucode
|
|
|
23
23
|
# directory is created if absent.
|
|
24
24
|
# @param retries [Integer, nil] override Config.http_retries.
|
|
25
25
|
# @param timeout [Integer, nil] override Config.http_timeout.
|
|
26
|
+
# @param validate [Symbol, nil] when `:pdf`, after a successful
|
|
27
|
+
# download verify (a) Content-Type starts with `application/pdf`
|
|
28
|
+
# and (b) the first 4 bytes of the body are `%PDF`. Raises
|
|
29
|
+
# {Ucode::CodeChartNotFoundError} with the offending header
|
|
30
|
+
# value in `context:` on failure. nil = no validation (the
|
|
31
|
+
# default for non-PDF callers like UcdZip and UnihanZip).
|
|
26
32
|
# @return [Pathname] destination path on success.
|
|
27
33
|
# @raise [Ucode::NetworkError] if all retries fail.
|
|
28
|
-
|
|
34
|
+
# @raise [Ucode::CodeChartNotFoundError] when `validate: :pdf`
|
|
35
|
+
# and the response fails content validation.
|
|
36
|
+
def get(url, dest:, retries: nil, timeout: nil, validate: nil)
|
|
29
37
|
uri = url.is_a?(URI) ? url : URI(url)
|
|
30
38
|
destination = Pathname.new(dest)
|
|
31
39
|
destination.dirname.mkpath
|
|
@@ -36,7 +44,11 @@ module Ucode
|
|
|
36
44
|
|
|
37
45
|
last_error = nil
|
|
38
46
|
(attempts + 1).times do |attempt|
|
|
39
|
-
|
|
47
|
+
response = stream_to(uri, destination, read_timeout)
|
|
48
|
+
validate_response!(validate, response, destination) if validate
|
|
49
|
+
return destination
|
|
50
|
+
rescue ValidationFailure => e
|
|
51
|
+
raise e.cause
|
|
40
52
|
rescue StandardError => e
|
|
41
53
|
last_error = e
|
|
42
54
|
sleep_for = backoff_sequence[attempt] || backoff_sequence.last
|
|
@@ -55,19 +67,34 @@ module Ucode
|
|
|
55
67
|
|
|
56
68
|
private
|
|
57
69
|
|
|
70
|
+
# Internal carrier for a validation failure inside a retry
|
|
71
|
+
# attempt. Re-raised from the loop so the response body (which
|
|
72
|
+
# is partial on retries) isn't double-validated against
|
|
73
|
+
# truncated bytes.
|
|
74
|
+
class ValidationFailure < StandardError
|
|
75
|
+
attr_reader :cause
|
|
76
|
+
|
|
77
|
+
def initialize(cause)
|
|
78
|
+
@cause = cause
|
|
79
|
+
super(cause.message)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
58
83
|
def stream_to(uri, destination, read_timeout)
|
|
84
|
+
response = nil
|
|
59
85
|
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
|
|
60
86
|
read_timeout: read_timeout) do |http|
|
|
61
87
|
request = Net::HTTP::Get.new(uri)
|
|
62
|
-
http.request(request) do |
|
|
63
|
-
unless
|
|
64
|
-
raise "HTTP #{
|
|
88
|
+
http.request(request) do |r|
|
|
89
|
+
unless r.is_a?(Net::HTTPSuccess)
|
|
90
|
+
raise "HTTP #{r.code} #{r.message}"
|
|
65
91
|
end
|
|
66
92
|
|
|
67
|
-
write_body(
|
|
93
|
+
write_body(r, destination)
|
|
94
|
+
response = r
|
|
68
95
|
end
|
|
69
96
|
end
|
|
70
|
-
|
|
97
|
+
response or raise "no response received"
|
|
71
98
|
end
|
|
72
99
|
|
|
73
100
|
def write_body(response, destination)
|
|
@@ -77,6 +104,47 @@ module Ucode
|
|
|
77
104
|
end
|
|
78
105
|
File.rename(partial.to_s, destination.to_s)
|
|
79
106
|
end
|
|
107
|
+
|
|
108
|
+
# Verifies Content-Type and magic bytes for a downloaded file.
|
|
109
|
+
# Raises ValidationFailure carrying a CodeChartNotFoundError so
|
|
110
|
+
# the retry loop in `get` doesn't re-attempt a download that's
|
|
111
|
+
# structurally invalid (only the transport is retriable).
|
|
112
|
+
def validate_response!(mode, response, destination)
|
|
113
|
+
case mode
|
|
114
|
+
when :pdf then validate_pdf!(response, destination)
|
|
115
|
+
else raise ArgumentError, "unknown validate mode: #{mode.inspect}"
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
PDF_CONTENT_TYPE_PREFIX = "application/pdf"
|
|
120
|
+
PDF_MAGIC = "%PDF"
|
|
121
|
+
private_constant :PDF_CONTENT_TYPE_PREFIX, :PDF_MAGIC
|
|
122
|
+
|
|
123
|
+
def validate_pdf!(response, destination)
|
|
124
|
+
content_type = response["Content-Type"].to_s
|
|
125
|
+
unless content_type.start_with?(PDF_CONTENT_TYPE_PREFIX)
|
|
126
|
+
raise ValidationFailure.new(
|
|
127
|
+
Ucode::CodeChartNotFoundError.new(
|
|
128
|
+
"expected Content-Type application/pdf, got #{content_type.inspect}",
|
|
129
|
+
context: { url: response.uri.to_s, content_type: content_type },
|
|
130
|
+
),
|
|
131
|
+
)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Re-open the destination file and peek at the first 4 bytes.
|
|
135
|
+
# The response body has already been written to disk by
|
|
136
|
+
# `stream_to`; we don't re-read from the response (which is
|
|
137
|
+
# consumed by then).
|
|
138
|
+
magic = File.open(destination, "rb") { |f| f.read(4) }
|
|
139
|
+
unless magic == PDF_MAGIC
|
|
140
|
+
raise ValidationFailure.new(
|
|
141
|
+
Ucode::CodeChartNotFoundError.new(
|
|
142
|
+
"expected %PDF magic bytes, got #{magic.inspect}",
|
|
143
|
+
context: { url: response.uri.to_s, magic: magic },
|
|
144
|
+
),
|
|
145
|
+
)
|
|
146
|
+
end
|
|
147
|
+
end
|
|
80
148
|
end
|
|
81
149
|
end
|
|
82
150
|
end
|
|
@@ -70,9 +70,22 @@ module Ucode
|
|
|
70
70
|
raise NotImplementedError
|
|
71
71
|
end
|
|
72
72
|
|
|
73
|
-
# @return [Boolean] true if the binary is on PATH
|
|
73
|
+
# @return [Boolean] true if the binary is on PATH. Returns
|
|
74
|
+
# false on hosts without `which`/`where` or where the
|
|
75
|
+
# binary isn't installed — the next renderer in
|
|
76
|
+
# KNOWN_RENDERERS is tried.
|
|
74
77
|
def available?
|
|
75
|
-
|
|
78
|
+
if Gem.win_platform?
|
|
79
|
+
# `where` returns the first match path; exit status 0
|
|
80
|
+
# means the binary is found. Suppress stdout/stderr to
|
|
81
|
+
# avoid polluting test output.
|
|
82
|
+
system("where #{binary_name} >NUL 2>NUL")
|
|
83
|
+
else
|
|
84
|
+
system("which", binary_name.to_s,
|
|
85
|
+
out: "/dev/null", err: "/dev/null")
|
|
86
|
+
end
|
|
87
|
+
rescue Errno::ENOENT, Errno::EINVAL
|
|
88
|
+
false
|
|
76
89
|
end
|
|
77
90
|
|
|
78
91
|
# Smoke-test the binary by actually rendering one page of the
|