ucode 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/TODO.extract-code-chart/01-pdf-fetch-validation.md +80 -0
- data/TODO.extract-code-chart/02-block-name-resolver.md +68 -0
- data/TODO.extract-code-chart/03-codechart-namespace.md +82 -0
- data/TODO.extract-code-chart/04-codechart-extractor.md +154 -0
- data/TODO.extract-code-chart/05-provenance-and-sidecar.md +147 -0
- data/TODO.extract-code-chart/06-codechart-writer.md +134 -0
- data/TODO.extract-code-chart/07-codechart-cli.md +135 -0
- data/TODO.extract-code-chart/08-specs.md +87 -0
- data/lib/ucode/audit/reference_factory.rb +66 -0
- data/lib/ucode/audit.rb +1 -0
- data/lib/ucode/cli.rb +134 -16
- data/lib/ucode/code_chart/extractor.rb +122 -0
- data/lib/ucode/code_chart/provenance.rb +81 -0
- data/lib/ucode/code_chart/sidecar.rb +52 -0
- data/lib/ucode/code_chart/writer.rb +128 -0
- data/lib/ucode/code_chart.rb +39 -0
- data/lib/ucode/commands/audit.rb +0 -1
- data/lib/ucode/commands/build.rb +4 -0
- data/lib/ucode/commands/canonical_build.rb +2 -3
- data/lib/ucode/commands/fetch.rb +12 -14
- data/lib/ucode/commands/glyphs.rb +25 -67
- data/lib/ucode/commands/lookup.rb +11 -11
- data/lib/ucode/commands/parse.rb +7 -5
- data/lib/ucode/commands/release.rb +0 -1
- data/lib/ucode/commands/universal_set.rb +10 -14
- data/lib/ucode/coordinator/indices.rb +38 -2
- data/lib/ucode/error.rb +11 -0
- data/lib/ucode/fetch/code_charts.rb +1 -1
- data/lib/ucode/fetch/http.rb +84 -14
- data/lib/ucode/glyphs/pipeline.rb +106 -0
- data/lib/ucode/glyphs.rb +1 -0
- data/lib/ucode/parsers/blocks.rb +34 -0
- data/lib/ucode/repo/aggregate_writer.rb +60 -298
- data/lib/ucode/repo/writers/blocks_writer.rb +73 -0
- data/lib/ucode/repo/writers/enums_writer.rb +38 -0
- data/lib/ucode/repo/writers/indexes_writer.rb +53 -0
- data/lib/ucode/repo/writers/manifest_writer.rb +78 -0
- data/lib/ucode/repo/writers/named_sequences_writer.rb +47 -0
- data/lib/ucode/repo/writers/planes_writer.rb +82 -0
- data/lib/ucode/repo/writers/relationships_writer.rb +71 -0
- data/lib/ucode/repo/writers/scripts_writer.rb +54 -0
- data/lib/ucode/repo/writers.rb +20 -0
- data/lib/ucode/repo.rb +1 -0
- data/lib/ucode/version.rb +1 -1
- data/lib/ucode.rb +3 -0
- data/ucode.gemspec +56 -0
- metadata +31 -5
- data/Gemfile.lock +0 -406
- data/lib/ucode/commands/audit/reference_builder.rb +0 -64
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
|
|
5
|
+
require "ucode/error"
|
|
6
|
+
require "ucode/glyphs/embedded_fonts/catalog"
|
|
7
|
+
require "ucode/glyphs/embedded_fonts/renderer"
|
|
8
|
+
require "ucode/glyphs/embedded_fonts/source"
|
|
9
|
+
require "ucode/glyphs/resolver"
|
|
10
|
+
require "ucode/glyphs/sources/pillar1_embedded_tounicode"
|
|
11
|
+
require "ucode/glyphs/sources/tier1_real_font"
|
|
12
|
+
|
|
13
|
+
module Ucode
|
|
14
|
+
module CodeChart
|
|
15
|
+
# Walks every assigned codepoint in a block and returns one
|
|
16
|
+
# {Result} per codepoint that any tier produced a glyph for.
|
|
17
|
+
#
|
|
18
|
+
# This is **not** a new extraction pipeline — it composes the
|
|
19
|
+
# existing {Ucode::Glyphs::Resolver} with per-block inputs
|
|
20
|
+
# (the block's Code Charts PDF + optionally Tier 1 and Pillar 3
|
|
21
|
+
# sources). The Resolver owns tier selection; the Extractor owns
|
|
22
|
+
# inputs.
|
|
23
|
+
#
|
|
24
|
+
# The REQ (R2) describes extraction via "locate the grid cell
|
|
25
|
+
# whose margin label matches the codepoint" — that was the v0.1
|
|
26
|
+
# retired approach (cell-border compositing). The current path
|
|
27
|
+
# is the embedded-font walk (Pillar 1, via {EmbeddedFonts::Catalog})
|
|
28
|
+
# with Pillar 2 (positional correlation) and Pillar 3 (Last Resort
|
|
29
|
+
# placeholders) as fallbacks.
|
|
30
|
+
#
|
|
31
|
+
# ## Tier selection
|
|
32
|
+
#
|
|
33
|
+
# Pillar 1 is always configured (the embedded font walk over the
|
|
34
|
+
# block's PDF). Tier 1 (real-font cmap) and Pillar 3 (Last
|
|
35
|
+
# Resort) are optional — the caller injects pre-built sources.
|
|
36
|
+
# This avoids forcing the Extractor to construct Last Resort
|
|
37
|
+
# eagerly, which would fail in environments where the UFO is
|
|
38
|
+
# not checked out.
|
|
39
|
+
class Extractor
|
|
40
|
+
# Result of extracting one codepoint.
|
|
41
|
+
Result = Struct.new(:codepoint, :svg, :tier, :provenance, keyword_init: true)
|
|
42
|
+
|
|
43
|
+
# @param block [Ucode::Models::Block] block whose assigned
|
|
44
|
+
# codepoints will be extracted
|
|
45
|
+
# @param pdf_path [Pathname, String] path to the per-block
|
|
46
|
+
# Code Charts PDF (downloaded by the caller; the Extractor
|
|
47
|
+
# doesn't fetch)
|
|
48
|
+
# @param cache_dir [Pathname, String, nil] directory for
|
|
49
|
+
# cached extracted font streams. nil = default
|
|
50
|
+
# (data/pdf-fonts/ relative to the gem root).
|
|
51
|
+
# @param tier1_sources [Array<Ucode::Glyphs::Source>, nil]
|
|
52
|
+
# optional Tier 1 sources (real-font cmap). nil = no Tier 1
|
|
53
|
+
# @param pillar3_source [Ucode::Glyphs::Source, nil] optional
|
|
54
|
+
# Pillar 3 (Last Resort) source. nil = no Pillar 3 fallback.
|
|
55
|
+
# Callers that want Last Resort placeholders inject the
|
|
56
|
+
# pre-built source here.
|
|
57
|
+
def initialize(block:, pdf_path:, cache_dir: nil,
|
|
58
|
+
tier1_sources: nil, pillar3_source: nil)
|
|
59
|
+
@block = block
|
|
60
|
+
@pdf_path = Pathname.new(pdf_path)
|
|
61
|
+
@cache_dir = cache_dir && Pathname.new(cache_dir)
|
|
62
|
+
@tier1_sources = tier1_sources || []
|
|
63
|
+
@pillar3_source = pillar3_source
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# @return [Array<Result>] one Result per codepoint that any
|
|
67
|
+
# tier produced a glyph for. Codepoints no tier can serve
|
|
68
|
+
# are silently skipped (no Result yielded).
|
|
69
|
+
def extract
|
|
70
|
+
resolver = build_resolver
|
|
71
|
+
results = []
|
|
72
|
+
each_codepoint do |cp|
|
|
73
|
+
resolver_result = resolver.resolve(cp)
|
|
74
|
+
next unless resolver_result&.svg
|
|
75
|
+
|
|
76
|
+
results << Result.new(
|
|
77
|
+
codepoint: cp,
|
|
78
|
+
svg: resolver_result.svg,
|
|
79
|
+
tier: resolver_result.tier,
|
|
80
|
+
provenance: resolver_result.provenance,
|
|
81
|
+
)
|
|
82
|
+
end
|
|
83
|
+
results
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
# Yields every codepoint in the block's range in ascending
|
|
89
|
+
# order. We yield the whole range because the Resolver's
|
|
90
|
+
# tiers handle unassigned codepoints — Pillar 3 (when
|
|
91
|
+
# configured) maps every codepoint via its Format 13 cmap,
|
|
92
|
+
# so unassigned slots get a placeholder. With no Pillar 3
|
|
93
|
+
# injected, only assigned codepoints (those the embedded
|
|
94
|
+
# font actually covers) yield Results; the rest are silently
|
|
95
|
+
# skipped, satisfying the REQ's "skip unassigned codepoints".
|
|
96
|
+
def each_codepoint
|
|
97
|
+
return enum_for(:each_codepoint) unless block_given?
|
|
98
|
+
|
|
99
|
+
(@block.range_first..@block.range_last).each do |cp|
|
|
100
|
+
yield cp
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def build_resolver
|
|
105
|
+
sources = @tier1_sources.dup
|
|
106
|
+
sources.concat(embedded_pillar_sources)
|
|
107
|
+
sources << @pillar3_source if @pillar3_source
|
|
108
|
+
order = sources.map(&:tier).uniq
|
|
109
|
+
Glyphs::Resolver.new(sources: sources, order: order)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def embedded_pillar_sources
|
|
113
|
+
embedded_source = Glyphs::EmbeddedFonts::Source.new(
|
|
114
|
+
pdf: @pdf_path, cache_dir: @cache_dir,
|
|
115
|
+
)
|
|
116
|
+
catalog = Glyphs::EmbeddedFonts::Catalog.new(embedded_source)
|
|
117
|
+
renderer = Glyphs::EmbeddedFonts::Renderer.new(catalog)
|
|
118
|
+
[Glyphs::Sources::Pillar1EmbeddedTounicode.new(renderer: renderer)]
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "time"
|
|
5
|
+
|
|
6
|
+
require "ucode/version"
|
|
7
|
+
|
|
8
|
+
module Ucode
|
|
9
|
+
module CodeChart
|
|
10
|
+
# Per-codepoint provenance value object — every field the REQ
|
|
11
|
+
# (R5) requires in the sidecar JSON next to each extracted SVG.
|
|
12
|
+
#
|
|
13
|
+
# Single source of truth for the provenance schema: the
|
|
14
|
+
# {Sidecar} writer reads this Struct, the Writer constructs it.
|
|
15
|
+
# Adding a field is one place to change.
|
|
16
|
+
#
|
|
17
|
+
# `extractor_version` reads from `Ucode::VERSION` at construction
|
|
18
|
+
# so the field stays in sync with the gem's version bump — single
|
|
19
|
+
# source of truth.
|
|
20
|
+
#
|
|
21
|
+
# `extracted_at` is the extraction event timestamp (UTC ISO8601),
|
|
22
|
+
# not the file-write timestamp.
|
|
23
|
+
Provenance = Struct.new(
|
|
24
|
+
:codepoint,
|
|
25
|
+
:block,
|
|
26
|
+
:source_pdf_url,
|
|
27
|
+
:source_pdf_sha256,
|
|
28
|
+
:ucd_version,
|
|
29
|
+
:extracted_at,
|
|
30
|
+
:extractor_version,
|
|
31
|
+
keyword_init: true,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# Computes the source PDF's URL from a block name and first
|
|
35
|
+
# codepoint. Mirrors the per-block URL convention in
|
|
36
|
+
# {Ucode::Fetch::CodeCharts}: 4-digit hex for BMP, 6-digit for
|
|
37
|
+
# supplementary planes.
|
|
38
|
+
#
|
|
39
|
+
# @param block_first_cp [Integer]
|
|
40
|
+
# @return [String]
|
|
41
|
+
def self.code_chart_url(block_first_cp)
|
|
42
|
+
width = block_first_cp > 0xFFFF ? 6 : 4
|
|
43
|
+
slug = block_first_cp.to_s(16).upcase.rjust(width, "0")
|
|
44
|
+
"#{Ucode.configuration.charts_base_url}/U#{slug}.pdf"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Builds a Provenance from the inputs the {Writer} has on hand
|
|
48
|
+
# (block, codepoint, ucd_version, pdf_path). Computes the PDF
|
|
49
|
+
# hash + URL once. The `extracted_at` timestamp is fixed at
|
|
50
|
+
# call time so re-running the same block produces identical
|
|
51
|
+
# provenance JSON for unchanged codepoints.
|
|
52
|
+
#
|
|
53
|
+
# @param block [Ucode::Models::Block]
|
|
54
|
+
# @param codepoint [Integer]
|
|
55
|
+
# @param ucd_version [String]
|
|
56
|
+
# @param pdf_path [Pathname, String]
|
|
57
|
+
# @param now [Time, nil] override for tests
|
|
58
|
+
# @return [Provenance]
|
|
59
|
+
def self.build(block:, codepoint:, ucd_version:, pdf_path:, now: nil)
|
|
60
|
+
path = Pathname.new(pdf_path)
|
|
61
|
+
Provenance.new(
|
|
62
|
+
codepoint: format("U+%04X", codepoint),
|
|
63
|
+
block: block.id,
|
|
64
|
+
source_pdf_url: code_chart_url(block.range_first),
|
|
65
|
+
source_pdf_sha256: sha256_of(path),
|
|
66
|
+
ucd_version: ucd_version,
|
|
67
|
+
extracted_at: (now || Time.now.utc).iso8601,
|
|
68
|
+
extractor_version: Ucode::VERSION,
|
|
69
|
+
)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# @param path [Pathname]
|
|
73
|
+
# @return [String] hex digest, "" when the path doesn't exist
|
|
74
|
+
# (callers can decide how to handle a missing hash)
|
|
75
|
+
def self.sha256_of(path)
|
|
76
|
+
return "" unless path.exist?
|
|
77
|
+
|
|
78
|
+
Digest::SHA256.file(path).hexdigest
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "pathname"
|
|
5
|
+
|
|
6
|
+
require "ucode/repo/atomic_writes"
|
|
7
|
+
|
|
8
|
+
module Ucode
|
|
9
|
+
module CodeChart
|
|
10
|
+
# Writes a {Provenance} to disk as the sidecar JSON next to its
|
|
11
|
+
# corresponding SVG.
|
|
12
|
+
#
|
|
13
|
+
# Path: `<output_root>/<codepoint>.json` — colocated with the
|
|
14
|
+
# SVG so a downstream consumer can find both files by a single
|
|
15
|
+
# directory listing.
|
|
16
|
+
#
|
|
17
|
+
# Idempotent via {Ucode::Repo::AtomicWrites#write_atomic}: a
|
|
18
|
+
# re-write of byte-identical content is a no-op (no temp-file
|
|
19
|
+
# rename). Provenance JSON is canonical (sorted keys via Ruby's
|
|
20
|
+
# stdlib JSON), so the byte-equality test is sound.
|
|
21
|
+
class Sidecar
|
|
22
|
+
include Ucode::Repo::AtomicWrites
|
|
23
|
+
|
|
24
|
+
# @param output_root [Pathname, String] directory the SVG +
|
|
25
|
+
# sidecar live in. Parent directories are created on demand.
|
|
26
|
+
def initialize(output_root:)
|
|
27
|
+
@output_root = Pathname.new(output_root)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @param provenance [Ucode::CodeChart::Provenance]
|
|
31
|
+
# @return [Pathname] the written sidecar path
|
|
32
|
+
def write(provenance)
|
|
33
|
+
path = path_for(provenance)
|
|
34
|
+
payload = JSON.pretty_generate(provenance.to_h) + "\n"
|
|
35
|
+
write_atomic(path, payload)
|
|
36
|
+
path
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @param codepoint_id [String] e.g. "U+10920"
|
|
40
|
+
# @return [Pathname] the would-be path for a sidecar
|
|
41
|
+
def path_for_id(codepoint_id)
|
|
42
|
+
@output_root.join("#{codepoint_id}.json")
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def path_for(provenance)
|
|
48
|
+
path_for_id(provenance.codepoint)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "pathname"
|
|
5
|
+
|
|
6
|
+
require "ucode/code_chart/extractor"
|
|
7
|
+
require "ucode/code_chart/provenance"
|
|
8
|
+
require "ucode/code_chart/sidecar"
|
|
9
|
+
require "ucode/error"
|
|
10
|
+
require "ucode/version_resolver"
|
|
11
|
+
|
|
12
|
+
module Ucode
|
|
13
|
+
module CodeChart
|
|
14
|
+
# Orchestrates extraction + provenance sidecar writing for one
|
|
15
|
+
# block. The Writer is the **only thing that touches disk** in the
|
|
16
|
+
# CodeChart namespace; everything else is composition.
|
|
17
|
+
#
|
|
18
|
+
# Output layout (per block):
|
|
19
|
+
#
|
|
20
|
+
# <output_root>/<block_id>/<U+XXXX>.svg
|
|
21
|
+
# <output_root>/<block_id>/<U+XXXX>.json # provenance sidecar
|
|
22
|
+
#
|
|
23
|
+
# One folder per block keeps each block's output self-contained
|
|
24
|
+
# and discoverable — a downstream consumer (fontisan) can iterate
|
|
25
|
+
# a block's folder without scanning the whole tree.
|
|
26
|
+
#
|
|
27
|
+
# Idempotent: re-running `write` on the same inputs produces
|
|
28
|
+
# byte-identical files (SVGs via content check; sidecars via
|
|
29
|
+
# {Ucode::Repo::AtomicWrites#write_atomic}'s canonical-JSON
|
|
30
|
+
# byte-equality). The {Summary} tally distinguishes "first run"
|
|
31
|
+
# writes from no-op re-writes.
|
|
32
|
+
class Writer
|
|
33
|
+
# Per-block run summary. Returned from {#write}.
|
|
34
|
+
Summary = Struct.new(
|
|
35
|
+
:block,
|
|
36
|
+
:codepoints_extracted,
|
|
37
|
+
:svgs_written,
|
|
38
|
+
:sidecars_written,
|
|
39
|
+
:pdf_sha256,
|
|
40
|
+
keyword_init: true,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# @param output_root [Pathname, String] parent directory. The
|
|
44
|
+
# `<block_id>/` subdirectory is created inside it.
|
|
45
|
+
# @param pdf_path [Pathname, String] Code Charts PDF (already
|
|
46
|
+
# downloaded by the caller; Writer doesn't fetch).
|
|
47
|
+
# @param ucd_version [String, nil] UCD version to stamp on
|
|
48
|
+
# provenance. nil = resolved via {VersionResolver.resolve(nil)}.
|
|
49
|
+
# @param cache_dir [Pathname, String, nil] font-stream cache
|
|
50
|
+
# directory for the EmbeddedFonts::Source.
|
|
51
|
+
# @param now [Time, nil] timestamp override (for tests).
|
|
52
|
+
# @param pillar3_source, tier1_sources: forwarded to the Extractor.
|
|
53
|
+
def initialize(output_root:, pdf_path:, ucd_version: nil,
|
|
54
|
+
cache_dir: nil, now: nil,
|
|
55
|
+
pillar3_source: nil, tier1_sources: nil)
|
|
56
|
+
@output_root = Pathname.new(output_root)
|
|
57
|
+
@pdf_path = Pathname.new(pdf_path)
|
|
58
|
+
@ucd_version = ucd_version || VersionResolver.resolve(nil)
|
|
59
|
+
@cache_dir = cache_dir && Pathname.new(cache_dir)
|
|
60
|
+
@now = now
|
|
61
|
+
@pillar3_source = pillar3_source
|
|
62
|
+
@tier1_sources = tier1_sources
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Extracts every codepoint in `block` and writes `<block_id>/<cp>.svg`
|
|
66
|
+
# + `<block_id>/<cp>.json` under `@output_root`. Returns a
|
|
67
|
+
# {Summary} tally.
|
|
68
|
+
#
|
|
69
|
+
# @param block [Ucode::Models::Block]
|
|
70
|
+
# @return [Summary]
|
|
71
|
+
def write(block)
|
|
72
|
+
block_dir = @output_root.join(block.id)
|
|
73
|
+
block_dir.mkpath
|
|
74
|
+
|
|
75
|
+
pdf_sha = CodeChart.sha256_of(@pdf_path)
|
|
76
|
+
|
|
77
|
+
sidecar = Sidecar.new(output_root: block_dir)
|
|
78
|
+
extractor = Extractor.new(
|
|
79
|
+
block: block,
|
|
80
|
+
pdf_path: @pdf_path,
|
|
81
|
+
cache_dir: @cache_dir,
|
|
82
|
+
pillar3_source: @pillar3_source,
|
|
83
|
+
tier1_sources: @tier1_sources,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
results = extractor.extract
|
|
87
|
+
svgs = 0
|
|
88
|
+
sidecars = 0
|
|
89
|
+
results.each do |result|
|
|
90
|
+
write_svg(block_dir, result)
|
|
91
|
+
svgs += 1
|
|
92
|
+
provenance = CodeChart.build(
|
|
93
|
+
block: block, codepoint: result.codepoint,
|
|
94
|
+
ucd_version: @ucd_version, pdf_path: @pdf_path,
|
|
95
|
+
now: @now,
|
|
96
|
+
)
|
|
97
|
+
sidecar.write(provenance)
|
|
98
|
+
sidecars += 1
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
Summary.new(
|
|
102
|
+
block: block.id,
|
|
103
|
+
codepoints_extracted: results.size,
|
|
104
|
+
svgs_written: svgs,
|
|
105
|
+
sidecars_written: sidecars,
|
|
106
|
+
pdf_sha256: pdf_sha,
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
private
|
|
111
|
+
|
|
112
|
+
# Writes one SVG, skipping the write when the existing content
|
|
113
|
+
# is byte-identical (so mtime is preserved on idempotent
|
|
114
|
+
# re-runs — the Sidecar uses `Repo::AtomicWrites` for the same
|
|
115
|
+
# reason but at a different layer).
|
|
116
|
+
def write_svg(block_dir, result)
|
|
117
|
+
path = block_dir.join("#{format_cp(result.codepoint)}.svg")
|
|
118
|
+
return if path.exist? && path.read == result.svg
|
|
119
|
+
|
|
120
|
+
path.write(result.svg)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def format_cp(codepoint)
|
|
124
|
+
"U+#{codepoint.to_s(16).upcase.rjust(4, '0')}"
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ucode
|
|
4
|
+
# CodeChart — per-codepoint SVG glyph extraction from Unicode Code
|
|
5
|
+
# Charts PDFs.
|
|
6
|
+
#
|
|
7
|
+
# The "Code Chart donor" use case (essenfont consumer): for blocks
|
|
8
|
+
# where no OFL real-font covers the glyphs (Sidetic in Unicode 17,
|
|
9
|
+
# Egyptian Hieroglyphs Extended-B), the only canonical source is
|
|
10
|
+
# the Unicode Consortium's Code Chart PDF. This namespace turns one
|
|
11
|
+
# such PDF into a tree of standalone SVG files plus provenance
|
|
12
|
+
# sidecar JSON.
|
|
13
|
+
#
|
|
14
|
+
# ## Architecture (MECE)
|
|
15
|
+
#
|
|
16
|
+
# Every concern has exactly one home:
|
|
17
|
+
#
|
|
18
|
+
# * **Block metadata** (range + assigned codepoints) — Parsers::Blocks
|
|
19
|
+
# * **PDF download + cache** — Fetch::CodeCharts + Glyphs::PdfFetcher
|
|
20
|
+
# * **PDF object-graph walk + font extraction** — Glyphs::EmbeddedFonts::*
|
|
21
|
+
# * **Tier selection (Pillar 1 / 2 / 3)** — Glyphs::Resolver
|
|
22
|
+
# * **SVG conversion + y-flip + viewBox** — Glyphs::EmbeddedFonts::Svg
|
|
23
|
+
# * **Provenance schema** — CodeChart::Provenance (this namespace)
|
|
24
|
+
# * **Sidecar JSON write** — CodeChart::Sidecar (this namespace)
|
|
25
|
+
# * **Per-block orchestration + idempotent disk write** — CodeChart::Writer
|
|
26
|
+
# * **CLI dispatch** — Cli::CodeChartCmd
|
|
27
|
+
#
|
|
28
|
+
# CodeChart::* is the feature-facing namespace. It does not
|
|
29
|
+
# implement extraction, font parsing, or PDF I/O — it composes
|
|
30
|
+
# the existing infrastructure. Replacing the implementation
|
|
31
|
+
# (e.g. a future pure-Ruby PDF parser per ADR-0001) does not
|
|
32
|
+
# change the public API.
|
|
33
|
+
module CodeChart
|
|
34
|
+
autoload :Extractor, "ucode/code_chart/extractor"
|
|
35
|
+
autoload :Provenance, "ucode/code_chart/provenance"
|
|
36
|
+
autoload :Sidecar, "ucode/code_chart/sidecar"
|
|
37
|
+
autoload :Writer, "ucode/code_chart/writer"
|
|
38
|
+
end
|
|
39
|
+
end
|
data/lib/ucode/commands/audit.rb
CHANGED
|
@@ -14,7 +14,6 @@ module Ucode
|
|
|
14
14
|
autoload :LibraryCommand, "ucode/commands/audit/library_command"
|
|
15
15
|
autoload :CompareCommand, "ucode/commands/audit/compare_command"
|
|
16
16
|
autoload :BrowserCommand, "ucode/commands/audit/browser_command"
|
|
17
|
-
autoload :ReferenceBuilder, "ucode/commands/audit/reference_builder"
|
|
18
17
|
end
|
|
19
18
|
end
|
|
20
19
|
end
|
data/lib/ucode/commands/build.rb
CHANGED
|
@@ -11,6 +11,10 @@ module Ucode
|
|
|
11
11
|
# parse → (optional) glyphs → (optional) site. Resumable: each step
|
|
12
12
|
# is idempotent and safe to re-run.
|
|
13
13
|
#
|
|
14
|
+
# Resolves the version intent once at the top and threads the
|
|
15
|
+
# resolved string through every sub-command. See Candidate 4 of the
|
|
16
|
+
# 2026-06-29 architecture review.
|
|
17
|
+
#
|
|
14
18
|
# **Glyph step is opt-in as of v0.1** because the SVG cell extractor
|
|
15
19
|
# is still experimental. Pass `include_glyphs: true` to enable it;
|
|
16
20
|
# otherwise the glyphs step is recorded as skipped.
|
|
@@ -39,7 +39,7 @@ module Ucode
|
|
|
39
39
|
# Missing pre-conditions cause silent fallthrough to lower tiers;
|
|
40
40
|
# the build report's `by_tier` totals surface what ran.
|
|
41
41
|
class CanonicalBuildCommand
|
|
42
|
-
# @param
|
|
42
|
+
# @param version [String] resolved UCD version
|
|
43
43
|
# @param output_root [String, Pathname]
|
|
44
44
|
# @param source_config_path [String, Pathname, nil] override the
|
|
45
45
|
# Tier 1 font config YAML; nil uses the default
|
|
@@ -54,9 +54,8 @@ module Ucode
|
|
|
54
54
|
# true. nil skips the block_coverage check.
|
|
55
55
|
# @return [Hash] { version:, codepoint_count:, report_path:,
|
|
56
56
|
# validation_report_path:, validation_passed: }
|
|
57
|
-
def call(
|
|
57
|
+
def call(version, output_root:, source_config_path: nil,
|
|
58
58
|
resolver: nil, validate: true, baseline: nil)
|
|
59
|
-
version = VersionResolver.resolve(version_intent)
|
|
60
59
|
root = Pathname.new(output_root)
|
|
61
60
|
|
|
62
61
|
resolved_resolver = resolver || build_resolver(version, source_config_path)
|
data/lib/ucode/commands/fetch.rb
CHANGED
|
@@ -5,7 +5,7 @@ require "pathname"
|
|
|
5
5
|
require "ucode/cache"
|
|
6
6
|
require "ucode/fetch"
|
|
7
7
|
require "ucode/glyphs/source_config"
|
|
8
|
-
require "ucode/
|
|
8
|
+
require "ucode/parsers"
|
|
9
9
|
|
|
10
10
|
module Ucode
|
|
11
11
|
module Commands
|
|
@@ -13,40 +13,38 @@ module Ucode
|
|
|
13
13
|
# per-version cache, plus the specialist Tier 1 fonts referenced by
|
|
14
14
|
# the curated source config.
|
|
15
15
|
#
|
|
16
|
-
# Thin shell over `Ucode::Fetch::*`. The command
|
|
17
|
-
#
|
|
18
|
-
#
|
|
16
|
+
# Thin shell over `Ucode::Fetch::*`. The command takes a resolved
|
|
17
|
+
# version string; CLI callers resolve via {VersionResolver.resolve}
|
|
18
|
+
# once and thread it through. See Candidate 4 of the 2026-06-29
|
|
19
|
+
# architecture review.
|
|
19
20
|
class FetchCommand
|
|
20
21
|
DEFAULT_SPECIALIST_FONTS_MANIFEST =
|
|
21
22
|
Ucode::Glyphs::SourceConfig::DEFAULT_PATH.dirname.join("specialist_fonts.yml")
|
|
22
23
|
private_constant :DEFAULT_SPECIALIST_FONTS_MANIFEST
|
|
23
24
|
|
|
24
|
-
# @param
|
|
25
|
+
# @param version [String] resolved UCD version
|
|
25
26
|
# @param force [Boolean]
|
|
26
27
|
# @return [Hash] { version:, ucd_dir: }
|
|
27
|
-
def fetch_ucd(
|
|
28
|
-
version = VersionResolver.resolve(version_intent)
|
|
28
|
+
def fetch_ucd(version, force: false)
|
|
29
29
|
Cache.ensure_version_dir!(version)
|
|
30
30
|
path = Fetch::UcdZip.call(version, force: force)
|
|
31
31
|
{ version: version, ucd_dir: path }
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
-
# @param
|
|
34
|
+
# @param version [String] resolved UCD version
|
|
35
35
|
# @param force [Boolean]
|
|
36
36
|
# @return [Hash] { version:, unihan_dir: }
|
|
37
|
-
def fetch_unihan(
|
|
38
|
-
version = VersionResolver.resolve(version_intent)
|
|
37
|
+
def fetch_unihan(version, force: false)
|
|
39
38
|
Cache.ensure_version_dir!(version)
|
|
40
39
|
path = Fetch::UnihanZip.call(version, force: force)
|
|
41
40
|
{ version: version, unihan_dir: path }
|
|
42
41
|
end
|
|
43
42
|
|
|
44
|
-
# @param
|
|
43
|
+
# @param version [String] resolved UCD version
|
|
45
44
|
# @param block_first_cps [Array<Integer>, nil] nil = all known blocks
|
|
46
45
|
# @param force [Boolean]
|
|
47
46
|
# @return [Hash] { version:, downloaded: }
|
|
48
|
-
def fetch_charts(
|
|
49
|
-
version = VersionResolver.resolve(version_intent)
|
|
47
|
+
def fetch_charts(version, block_first_cps: nil, force: false)
|
|
50
48
|
Cache.ensure_version_dir!(version)
|
|
51
49
|
|
|
52
50
|
cps = block_first_cps || default_block_first_cps(version)
|
|
@@ -95,4 +93,4 @@ module Ucode
|
|
|
95
93
|
end
|
|
96
94
|
end
|
|
97
95
|
end
|
|
98
|
-
end
|
|
96
|
+
end
|
|
@@ -1,19 +1,16 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "pathname"
|
|
4
|
-
require "set"
|
|
5
4
|
|
|
6
|
-
require "ucode/cache"
|
|
7
5
|
require "ucode/glyphs"
|
|
8
|
-
require "ucode/parsers"
|
|
9
|
-
require "ucode/version_resolver"
|
|
10
6
|
|
|
11
7
|
module Ucode
|
|
12
8
|
module Commands
|
|
13
9
|
# `ucode glyphs` — extract per-codepoint SVGs from Code Charts PDFs.
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
10
|
+
# Thin Thor-facing wrapper around {Ucode::Glyphs::Pipeline}:
|
|
11
|
+
# opt-in gate + experimental warning live here; the pipeline
|
|
12
|
+
# assembly (block loading, fetcher, per-block specs) lives in
|
|
13
|
+
# {Ucode::Glyphs::Pipeline}.
|
|
17
14
|
#
|
|
18
15
|
# **Status (v0.1): EXPERIMENTAL.** The cell-extraction pipeline
|
|
19
16
|
# currently includes cell-border decorations alongside the actual
|
|
@@ -23,16 +20,16 @@ module Ucode
|
|
|
23
20
|
# pipeline can be iterated on without churning the CLI surface, but
|
|
24
21
|
# callers MUST opt in via `include_glyphs: true` (CLI: `--include-glyphs`)
|
|
25
22
|
# and will receive a printed warning. Tracked for v0.2.
|
|
23
|
+
#
|
|
24
|
+
# Takes a resolved version string; CLI callers resolve via
|
|
25
|
+
# {VersionResolver.resolve} once and thread it through. See
|
|
26
|
+
# Candidate 4 of the 2026-06-29 architecture review.
|
|
26
27
|
class GlyphsCommand
|
|
27
28
|
ExperimentalWarning = "ucode glyphs is experimental in v0.1: " \
|
|
28
29
|
"extracted SVGs include cell-border decorations " \
|
|
29
30
|
"alongside the character outline."
|
|
30
31
|
private_constant :ExperimentalWarning
|
|
31
32
|
|
|
32
|
-
MonolithPath = "CodeCharts.pdf"
|
|
33
|
-
PageMapCache = "data/codecharts_page_map.json"
|
|
34
|
-
private_constant :MonolithPath, :PageMapCache
|
|
35
|
-
|
|
36
33
|
class << self
|
|
37
34
|
# @return [String] the experimental-status banner. Exposed so the
|
|
38
35
|
# CLI and BuildCommand surface the same message verbatim.
|
|
@@ -41,7 +38,7 @@ module Ucode
|
|
|
41
38
|
end
|
|
42
39
|
end
|
|
43
40
|
|
|
44
|
-
# @param
|
|
41
|
+
# @param version [String] resolved UCD version
|
|
45
42
|
# @param output_root [String, Pathname]
|
|
46
43
|
# @param block_filter [Array<String>, nil] block ids to limit to;
|
|
47
44
|
# nil = every block
|
|
@@ -55,75 +52,36 @@ module Ucode
|
|
|
55
52
|
# written here exactly once before work begins.
|
|
56
53
|
# @return [Hash] aggregated Writer tally + version, or a `skipped`
|
|
57
54
|
# payload when opt-in is false.
|
|
58
|
-
def call(
|
|
59
|
-
block_filter: nil, force: false,
|
|
55
|
+
def call(version, output_root:,
|
|
56
|
+
block_filter: nil, force: false,
|
|
57
|
+
monolith_path: Glyphs::Pipeline::DEFAULT_MONOLITH_PATH,
|
|
60
58
|
include_glyphs: false, warn: nil)
|
|
61
|
-
return skipped(
|
|
59
|
+
return skipped(version) unless include_glyphs
|
|
62
60
|
|
|
63
61
|
warn&.puts(ExperimentalWarning)
|
|
64
|
-
version = VersionResolver.resolve(version_intent)
|
|
65
|
-
root = Pathname.new(output_root)
|
|
66
62
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
63
|
+
pipeline = Glyphs::Pipeline.new(
|
|
64
|
+
version: version,
|
|
65
|
+
block_filter: block_filter,
|
|
66
|
+
monolith_path: monolith_path,
|
|
67
|
+
)
|
|
68
|
+
specs = pipeline.build_specs(force: force)
|
|
70
69
|
|
|
71
|
-
writer = Glyphs::Writer.new(
|
|
72
|
-
|
|
70
|
+
writer = Glyphs::Writer.new(
|
|
71
|
+
output_root: Pathname.new(output_root),
|
|
72
|
+
parallel_workers: workers,
|
|
73
|
+
)
|
|
73
74
|
tally = writer.write_all(specs)
|
|
74
75
|
tally.merge(version: version, block_count: specs.size)
|
|
75
76
|
end
|
|
76
77
|
|
|
77
78
|
private
|
|
78
79
|
|
|
79
|
-
def load_blocks(version, block_filter)
|
|
80
|
-
ucd_dir = Cache.ucd_dir(version)
|
|
81
|
-
path = ucd_dir.join("Blocks.txt")
|
|
82
|
-
return [] unless path.exist?
|
|
83
|
-
|
|
84
|
-
all = Parsers::Blocks.each_record(path).to_a
|
|
85
|
-
return all unless block_filter && !block_filter.empty?
|
|
86
|
-
|
|
87
|
-
filter_set = block_filter.to_set
|
|
88
|
-
all.select { |block| filter_set.include?(block.id) }
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
def build_fetcher(version, monolith_path, blocks)
|
|
92
|
-
monolith = Pathname.new(monolith_path)
|
|
93
|
-
monolith = monolith.exist? ? monolith : nil
|
|
94
|
-
Glyphs::PdfFetcher.new(
|
|
95
|
-
version,
|
|
96
|
-
monolith_path: monolith,
|
|
97
|
-
blocks: blocks,
|
|
98
|
-
page_map_cache: PageMapCache,
|
|
99
|
-
)
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
def spec_for(block, fetcher, force)
|
|
103
|
-
pdf_path = fetcher.fetch(block_first_cp: block.range_first, force: force)
|
|
104
|
-
return nil unless pdf_path
|
|
105
|
-
|
|
106
|
-
{ block: block, pdf_path: pdf_path, page_map: page_map_for(block) }
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
# Heuristic page map: per-block PDFs are page 1 = title, page 2 =
|
|
110
|
-
# first chart page starting at the block's first codepoint. True for
|
|
111
|
-
# most BMP blocks; multi-page blocks (CJK) need a richer resolver.
|
|
112
|
-
# Mismatches yield placeholder SVGs only — never wrong glyphs.
|
|
113
|
-
def page_map_for(block)
|
|
114
|
-
{ 2 => block.range_first }
|
|
115
|
-
end
|
|
116
|
-
|
|
117
80
|
def workers
|
|
118
81
|
Ucode.configuration.parallel_workers
|
|
119
82
|
end
|
|
120
83
|
|
|
121
|
-
def skipped(
|
|
122
|
-
version = begin
|
|
123
|
-
VersionResolver.resolve(version_intent)
|
|
124
|
-
rescue UnknownVersionError
|
|
125
|
-
version_intent
|
|
126
|
-
end
|
|
84
|
+
def skipped(version)
|
|
127
85
|
{
|
|
128
86
|
version: version,
|
|
129
87
|
skipped: true,
|
|
@@ -133,4 +91,4 @@ module Ucode
|
|
|
133
91
|
end
|
|
134
92
|
end
|
|
135
93
|
end
|
|
136
|
-
end
|
|
94
|
+
end
|