ucode 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +10 -2
- data/README.md +66 -20
- data/Rakefile +18 -7
- data/config/unicode17_universal_glyph_set.yml +1 -1
- data/lib/ucode/audit/reference_factory.rb +1 -1
- data/lib/ucode/cli.rb +10 -42
- data/lib/ucode/code_chart/extractor.rb +3 -5
- data/lib/ucode/code_chart/provenance.rb +6 -5
- data/lib/ucode/code_chart/sidecar.rb +2 -2
- data/lib/ucode/code_chart/writer.rb +1 -1
- data/lib/ucode/code_chart.rb +1 -1
- data/lib/ucode/commands/build.rb +3 -26
- data/lib/ucode/commands/canonical_build.rb +1 -4
- data/lib/ucode/commands/fetch.rb +1 -1
- data/lib/ucode/commands/lookup.rb +1 -1
- data/lib/ucode/commands/parse.rb +1 -1
- data/lib/ucode/commands.rb +0 -1
- data/lib/ucode/coordinator/indices.rb +2 -2
- data/lib/ucode/error.rb +0 -8
- data/lib/ucode/fetch/code_charts.rb +2 -3
- data/lib/ucode/fetch/http.rb +12 -14
- data/lib/ucode/glyphs/embedded_fonts/catalog.rb +81 -4
- data/lib/ucode/glyphs/embedded_fonts/trace_correlator.rb +230 -0
- data/lib/ucode/glyphs/embedded_fonts/trace_glyph.rb +27 -0
- data/lib/ucode/glyphs/embedded_fonts/trace_parser.rb +50 -0
- data/lib/ucode/glyphs/embedded_fonts/trace_runner.rb +53 -0
- data/lib/ucode/glyphs/embedded_fonts.rb +4 -0
- data/lib/ucode/glyphs/pdf_fetcher.rb +7 -50
- data/lib/ucode/glyphs.rb +4 -14
- data/lib/ucode/repo/aggregate_writer.rb +1 -1
- data/lib/ucode/repo/writers/blocks_writer.rb +13 -13
- data/lib/ucode/repo/writers/enums_writer.rb +2 -2
- data/lib/ucode/repo/writers/indexes_writer.rb +4 -4
- data/lib/ucode/repo/writers/manifest_writer.rb +4 -4
- data/lib/ucode/repo/writers/named_sequences_writer.rb +1 -1
- data/lib/ucode/repo/writers/planes_writer.rb +17 -17
- data/lib/ucode/repo/writers/relationships_writer.rb +1 -1
- data/lib/ucode/repo/writers/scripts_writer.rb +6 -6
- data/lib/ucode/repo/writers.rb +1 -1
- data/lib/ucode/version.rb +1 -1
- data/lib/ucode.rb +0 -2
- data/ucode.gemspec +6 -1
- metadata +10 -19
- data/lib/ucode/commands/glyphs.rb +0 -94
- data/lib/ucode/glyphs/cell_extractor.rb +0 -130
- data/lib/ucode/glyphs/dvisvgm_renderer.rb +0 -29
- data/lib/ucode/glyphs/grid.rb +0 -30
- data/lib/ucode/glyphs/grid_detector.rb +0 -165
- data/lib/ucode/glyphs/monolith_page_map.rb +0 -181
- data/lib/ucode/glyphs/mutool_renderer.rb +0 -28
- data/lib/ucode/glyphs/page_renderer.rb +0 -221
- data/lib/ucode/glyphs/path_bbox.rb +0 -62
- data/lib/ucode/glyphs/pdf2svg_renderer.rb +0 -26
- data/lib/ucode/glyphs/pdftocairo_renderer.rb +0 -32
- data/lib/ucode/glyphs/pipeline.rb +0 -106
- data/lib/ucode/glyphs/writer.rb +0 -250
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 409561757912083c19e4044c0ed37129945bf6de53bc3b029d349e4a8f16f10f
|
|
4
|
+
data.tar.gz: 85a06e0383587af4d8a88342974a58105423635b85212ecc7b1783268e6c5e2a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 85660ae16bbfa2632131888872ddebca9cdee45d26791837ddce2fa629e18a721c9701b023c5424e058165eac03d1d4e1d16bb2a6c0b582a8ef4c1e0104ecdf5
|
|
7
|
+
data.tar.gz: 411de21c9c5f3e46b559752d54462f02f4110aa59e261ee3ea6c19383ad9383fd43f013e1107fe59b3e6c3b296f906a8c99e7087da5f9c85f2b13ba385447b95
|
data/Gemfile
CHANGED
|
@@ -4,8 +4,13 @@ source "https://rubygems.org"
|
|
|
4
4
|
|
|
5
5
|
gemspec
|
|
6
6
|
|
|
7
|
+
# Rake must be in the default group (not :development) because the
|
|
8
|
+
# GHA release workflow runs `bundle exec rake release` to publish the
|
|
9
|
+
# gem. The release runner installs with `--without development`, so
|
|
10
|
+
# gems in the :development group are excluded.
|
|
11
|
+
gem "rake"
|
|
12
|
+
|
|
7
13
|
group :development do
|
|
8
|
-
gem "rake"
|
|
9
14
|
gem "rspec"
|
|
10
15
|
gem "rubocop"
|
|
11
16
|
gem "rubocop-performance"
|
|
@@ -19,4 +24,7 @@ end
|
|
|
19
24
|
# local sibling checkout, set FONTISAN_PATH before running bundle.
|
|
20
25
|
# FONTISAN_PATH=../fontisan bundle install
|
|
21
26
|
gem "fontisan", path: ENV["FONTISAN_PATH"] if ENV["FONTISAN_PATH"]
|
|
22
|
-
|
|
27
|
+
# Pin fontisan to 0.2.22 — 0.2.23+ removed
|
|
28
|
+
# `Fontisan::Commands::AuditCommand` and 0.4.x removed the Audit
|
|
29
|
+
# subsystem entirely. See ucode.gemspec for the rationale.
|
|
30
|
+
gem "fontisan", "= 0.2.22" unless ENV["FONTISAN_PATH"]
|
data/README.md
CHANGED
|
@@ -2,14 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
`ucode` is a Ruby toolkit for the Unicode Character Database (UCD). It turns the
|
|
4
4
|
official UCD text files into a structured, browsable dataset: one JSON document
|
|
5
|
-
per assigned codepoint, plus a Vitepress site for navigation.
|
|
5
|
+
per assigned codepoint, plus a Vitepress site for navigation. It also extracts
|
|
6
|
+
per-codepoint SVG glyphs from the Unicode Code Charts PDFs and audits font
|
|
7
|
+
coverage against the Unicode baseline.
|
|
6
8
|
|
|
7
|
-
> **Status (v0.1).** The JSON dataset, lookup index,
|
|
8
|
-
>
|
|
9
|
-
>
|
|
10
|
-
>
|
|
9
|
+
> **Status (v0.2.1).** The JSON dataset, lookup index, Vitepress site, and
|
|
10
|
+
> 4-tier glyph extraction pipeline are production-ready. The `ucode code-chart`
|
|
11
|
+
> subcommand extracts standalone SVGs + provenance sidecars from per-block
|
|
12
|
+
> Code Charts PDFs. Font coverage auditing (`ucode audit`) is production-ready.
|
|
11
13
|
|
|
12
|
-
## What you get (v0.
|
|
14
|
+
## What you get (v0.2)
|
|
13
15
|
|
|
14
16
|
- **Per-codepoint JSON** at `output/blocks/<BLOCK>/<U+XXXX>/index.json` with
|
|
15
17
|
full UCD properties, the human-curated relationships from `NamesList.txt`
|
|
@@ -21,6 +23,14 @@ per assigned codepoint, plus a Vitepress site for navigation.
|
|
|
21
23
|
relationships, named sequences, manifest.
|
|
22
24
|
- **SQLite lookup index** for fast codepoint → block/script/char queries.
|
|
23
25
|
- **Vitepress site** at `site/` for browsing Plane → Block → Character.
|
|
26
|
+
- **4-tier glyph extraction** — per-codepoint `glyph.svg` sourced from real
|
|
27
|
+
fonts (Tier 1), PDF-embedded fonts (Pillars 1+2), or Last Resort UFO
|
|
28
|
+
(Pillar 3).
|
|
29
|
+
- **Per-block Code Chart extraction** — `ucode code-chart extract` produces
|
|
30
|
+
standalone SVG + provenance JSON for every codepoint in a block.
|
|
31
|
+
- **Font coverage audit** — `ucode audit` compares a font's cmap against the
|
|
32
|
+
Unicode baseline and reports per-block coverage, missing codepoints, and
|
|
33
|
+
optional HTML browsers.
|
|
24
34
|
|
|
25
35
|
## Install
|
|
26
36
|
|
|
@@ -31,7 +41,7 @@ gem install ucode
|
|
|
31
41
|
Or in a Gemfile:
|
|
32
42
|
|
|
33
43
|
```ruby
|
|
34
|
-
gem "ucode", "~> 0.
|
|
44
|
+
gem "ucode", "~> 0.2"
|
|
35
45
|
```
|
|
36
46
|
|
|
37
47
|
## Quick start
|
|
@@ -107,7 +117,39 @@ Then:
|
|
|
107
117
|
cd site && npm install && npm run dev
|
|
108
118
|
```
|
|
109
119
|
|
|
110
|
-
##
|
|
120
|
+
## Code Chart extraction
|
|
121
|
+
|
|
122
|
+
Extract per-codepoint SVG glyphs from a Unicode Code Charts PDF, with
|
|
123
|
+
provenance sidecar JSON:
|
|
124
|
+
|
|
125
|
+
```sh
|
|
126
|
+
# Download the Code Charts PDF for a block
|
|
127
|
+
ucode code-chart fetch --block Sidetic
|
|
128
|
+
|
|
129
|
+
# Extract every codepoint as SVG + provenance JSON
|
|
130
|
+
ucode code-chart extract --block Sidetic --to /tmp/sidetic/
|
|
131
|
+
|
|
132
|
+
# List cached Code Charts PDFs
|
|
133
|
+
ucode code-chart list
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Output layout:
|
|
137
|
+
|
|
138
|
+
```
|
|
139
|
+
/tmp/sidetic/Sidetic/
|
|
140
|
+
U+10920.svg # vector glyph outline
|
|
141
|
+
U+10920.json # provenance: source PDF, sha256, version, timestamp
|
|
142
|
+
U+10921.svg
|
|
143
|
+
U+10921.json
|
|
144
|
+
...
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
The extractor uses the existing 4-tier glyph sourcing pipeline (Tier 1 →
|
|
148
|
+
Pillar 1 → Pillar 2 → Pillar 3). No new extraction logic — the
|
|
149
|
+
`ucode code-chart` subcommand is a thin CLI wrapper over `Ucode::CodeChart::Writer`,
|
|
150
|
+
which orchestrates `Ucode::Glyphs::Resolver` for each codepoint.
|
|
151
|
+
|
|
152
|
+
## Glyph extraction (4-tier pipeline)
|
|
111
153
|
|
|
112
154
|
The `ucode glyphs` command and the `--include-glyphs` flag on `ucode build`
|
|
113
155
|
are **opt-in and experimental in v0.1**. They emit per-codepoint `glyph.svg`
|
|
@@ -436,29 +478,33 @@ from the dataset.
|
|
|
436
478
|
|
|
437
479
|
## Architecture
|
|
438
480
|
|
|
439
|
-
|
|
481
|
+
Seven concerns, each isolated:
|
|
440
482
|
|
|
441
483
|
1. **`Ucode::Models`** — `lutaml-model` classes for every UCD aggregate.
|
|
442
484
|
2. **`Ucode::Parsers`** — one streaming parser per UCD text file.
|
|
443
485
|
3. **`Ucode::Coordinator`** — single-pass enrichment that merges indices
|
|
444
486
|
into each `CodePoint` as it streams.
|
|
445
|
-
4. **`Ucode::Repo`** — atomic, idempotent writers for the output tree
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
487
|
+
4. **`Ucode::Repo`** — atomic, idempotent writers for the output tree
|
|
488
|
+
(per-concern writers under `Repo::Writers::*`).
|
|
489
|
+
5. **`Ucode::Glyphs`** — 4-tier vector glyph extraction from Code Charts PDFs
|
|
490
|
+
(RealFonts, EmbeddedFonts, LastResort + Resolver).
|
|
491
|
+
6. **`Ucode::CodeChart`** — per-block SVG extraction + provenance sidecar
|
|
492
|
+
for the "Code Chart donor" use case.
|
|
493
|
+
7. **`Ucode::Site`** — Vitepress scaffold + config/page generator.
|
|
449
494
|
|
|
450
495
|
CLI is thin Thor dispatch over `Ucode::Commands::*`. Each command class
|
|
451
|
-
is a pure, in-process testable unit.
|
|
496
|
+
is a pure, in-process testable unit. Version resolution happens once
|
|
497
|
+
per CLI invocation and threads through to all sub-commands.
|
|
452
498
|
|
|
453
|
-
See `
|
|
454
|
-
`docs/
|
|
499
|
+
See `docs/architecture.md` for the canonical architecture reference and
|
|
500
|
+
`docs/adr/` for Architecture Decision Records.
|
|
455
501
|
|
|
456
502
|
## Authoritative source
|
|
457
503
|
|
|
458
|
-
ucode parses the **UCD text files** (
|
|
459
|
-
`
|
|
460
|
-
the human-curated relationship data
|
|
461
|
-
|
|
504
|
+
ucode parses the **UCD text files** (`UnicodeData.txt`, `NamesList.txt`,
|
|
505
|
+
`Blocks.txt`, etc.) per UAX #44. Never uses the flat XML dump — the text
|
|
506
|
+
files carry the human-curated relationship data that makes this project
|
|
507
|
+
valuable.
|
|
462
508
|
|
|
463
509
|
## License
|
|
464
510
|
|
data/Rakefile
CHANGED
|
@@ -4,15 +4,26 @@ require "rubygems"
|
|
|
4
4
|
require "rake"
|
|
5
5
|
require "bundler/gem_tasks"
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
begin
|
|
8
|
+
require "rspec/core/rake_task"
|
|
9
|
+
RSpec::Core::RakeTask.new(:spec)
|
|
10
|
+
rescue LoadError
|
|
11
|
+
# rspec is in the :development group; not available in the
|
|
12
|
+
# release runner (`bundle install --without development`).
|
|
13
|
+
end
|
|
9
14
|
|
|
10
|
-
|
|
11
|
-
|
|
15
|
+
begin
|
|
16
|
+
require "rubocop/rake_task"
|
|
17
|
+
RuboCop::RakeTask.new
|
|
18
|
+
rescue LoadError
|
|
19
|
+
end
|
|
12
20
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
21
|
+
begin
|
|
22
|
+
require "yard"
|
|
23
|
+
YARD::Rake::YardocTask.new do |t|
|
|
24
|
+
t.options = ["--output-dir", "docs/api"]
|
|
25
|
+
end
|
|
26
|
+
rescue LoadError
|
|
16
27
|
end
|
|
17
28
|
|
|
18
29
|
task default: %i[spec rubocop]
|
data/lib/ucode/cli.rb
CHANGED
|
@@ -109,34 +109,6 @@ module Ucode
|
|
|
109
109
|
puts JSON.pretty_generate(result)
|
|
110
110
|
end
|
|
111
111
|
|
|
112
|
-
# ─────────────── glyphs ───────────────
|
|
113
|
-
desc "glyphs [VERSION]", "Extract per-codepoint SVGs from Code Charts PDFs (experimental)"
|
|
114
|
-
long_desc <<~LONG
|
|
115
|
-
EXPERIMENTAL in v0.1. The cell extractor currently includes cell-border
|
|
116
|
-
decorations alongside the actual character outline, so the output is not
|
|
117
|
-
yet suitable for end-user display. Opt in with --include-glyphs to run
|
|
118
|
-
the pipeline anyway; otherwise it returns a skipped payload.
|
|
119
|
-
LONG
|
|
120
|
-
option :to, type: :string, default: "./output"
|
|
121
|
-
option :block, type: :array, desc: "Limit to these block ids"
|
|
122
|
-
option :force, type: :boolean, default: false
|
|
123
|
-
option :monolith, type: :string, default: "CodeCharts.pdf",
|
|
124
|
-
desc: "Path to CodeCharts.pdf for fallback slicing"
|
|
125
|
-
option :include_glyphs, type: :boolean, default: false,
|
|
126
|
-
desc: "Opt into the experimental v0.1 pipeline"
|
|
127
|
-
def glyphs(version = nil)
|
|
128
|
-
result = Commands::GlyphsCommand.new.call(
|
|
129
|
-
VersionResolver.resolve(version),
|
|
130
|
-
output_root: options[:to],
|
|
131
|
-
block_filter: options[:block],
|
|
132
|
-
force: options[:force],
|
|
133
|
-
monolith_path: options[:monolith],
|
|
134
|
-
include_glyphs: options[:include_glyphs],
|
|
135
|
-
warn: $stderr,
|
|
136
|
-
)
|
|
137
|
-
puts JSON.pretty_generate(result)
|
|
138
|
-
end
|
|
139
|
-
|
|
140
112
|
# ─────────────── site ───────────────
|
|
141
113
|
class Site < Thor
|
|
142
114
|
desc "init", "Copy the Vitepress scaffold into site/"
|
|
@@ -164,7 +136,7 @@ module Ucode
|
|
|
164
136
|
class CodeChartCmd < Thor
|
|
165
137
|
desc "fetch --block BLOCK [VERSION]", "Download the Code Charts PDF for a block"
|
|
166
138
|
option :block, type: :string, required: true,
|
|
167
|
-
|
|
139
|
+
desc: "Block identifier (e.g. Sidetic, Basic_Latin)"
|
|
168
140
|
def fetch(version = nil)
|
|
169
141
|
with_codechart_errors do
|
|
170
142
|
block_first_cp = resolve_block_first_cp!(options[:block], version)
|
|
@@ -179,7 +151,7 @@ module Ucode
|
|
|
179
151
|
desc "extract --block BLOCK --to DIR [VERSION]",
|
|
180
152
|
"Extract per-codepoint SVG + provenance sidecars from a Code Charts PDF"
|
|
181
153
|
option :block, type: :string, required: true,
|
|
182
|
-
|
|
154
|
+
desc: "Block identifier (e.g. Sidetic)"
|
|
183
155
|
option :to, type: :string, required: true,
|
|
184
156
|
desc: "Output directory (will contain <block_id>/<U+XXXX>.svg + .json)"
|
|
185
157
|
def extract(version = nil)
|
|
@@ -193,10 +165,12 @@ module Ucode
|
|
|
193
165
|
|
|
194
166
|
pdf = Ucode::Glyphs::PdfFetcher.new(version_str)
|
|
195
167
|
.fetch(block_first_cp: block_first_cp)
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
168
|
+
unless pdf
|
|
169
|
+
raise Ucode::CodeChartNotFoundError.new(
|
|
170
|
+
"Code Charts PDF unavailable for block #{block.id.inspect}",
|
|
171
|
+
context: { block_id: block.id, version: version_str },
|
|
172
|
+
)
|
|
173
|
+
end
|
|
200
174
|
|
|
201
175
|
writer = Ucode::CodeChart::Writer.new(
|
|
202
176
|
output_root: Pathname.new(options[:to]),
|
|
@@ -218,7 +192,7 @@ module Ucode
|
|
|
218
192
|
return
|
|
219
193
|
end
|
|
220
194
|
files.each do |f|
|
|
221
|
-
puts f.basename
|
|
195
|
+
puts f.basename
|
|
222
196
|
end
|
|
223
197
|
end
|
|
224
198
|
|
|
@@ -343,22 +317,16 @@ module Ucode
|
|
|
343
317
|
subcommand "cache", Cache
|
|
344
318
|
|
|
345
319
|
# ─────────────── build ───────────────
|
|
346
|
-
desc "build [VERSION]", "Full pipeline: fetch + parse +
|
|
320
|
+
desc "build [VERSION]", "Full pipeline: fetch + parse + site"
|
|
347
321
|
option :to, type: :string, default: "./output"
|
|
348
322
|
option :site, type: :string, default: nil, desc: "Build the site here (skipped if nil)"
|
|
349
|
-
option :monolith, type: :string, default: "CodeCharts.pdf"
|
|
350
323
|
option :force_fetch, type: :boolean, default: false
|
|
351
|
-
option :include_glyphs, type: :boolean, default: false,
|
|
352
|
-
desc: "Opt into the experimental v0.1 glyph step"
|
|
353
324
|
def build(version = nil)
|
|
354
325
|
result = Commands::BuildCommand.new.call(
|
|
355
326
|
version,
|
|
356
327
|
output_root: options[:to],
|
|
357
328
|
site_root: options[:site],
|
|
358
|
-
monolith_path: options[:monolith],
|
|
359
329
|
force_fetch: options[:force_fetch],
|
|
360
|
-
include_glyphs: options[:include_glyphs],
|
|
361
|
-
warn: $stderr,
|
|
362
330
|
)
|
|
363
331
|
puts JSON.pretty_generate(result)
|
|
364
332
|
end
|
|
@@ -93,12 +93,10 @@ module Ucode
|
|
|
93
93
|
# injected, only assigned codepoints (those the embedded
|
|
94
94
|
# font actually covers) yield Results; the rest are silently
|
|
95
95
|
# skipped, satisfying the REQ's "skip unassigned codepoints".
|
|
96
|
-
def each_codepoint
|
|
96
|
+
def each_codepoint(&)
|
|
97
97
|
return enum_for(:each_codepoint) unless block_given?
|
|
98
98
|
|
|
99
|
-
(@block.range_first..@block.range_last).each
|
|
100
|
-
yield cp
|
|
101
|
-
end
|
|
99
|
+
(@block.range_first..@block.range_last).each(&)
|
|
102
100
|
end
|
|
103
101
|
|
|
104
102
|
def build_resolver
|
|
@@ -119,4 +117,4 @@ module Ucode
|
|
|
119
117
|
end
|
|
120
118
|
end
|
|
121
119
|
end
|
|
122
|
-
end
|
|
120
|
+
end
|
|
@@ -33,14 +33,15 @@ module Ucode
|
|
|
33
33
|
|
|
34
34
|
# Computes the source PDF's URL from a block name and first
|
|
35
35
|
# codepoint. Mirrors the per-block URL convention in
|
|
36
|
-
# {Ucode::Fetch::CodeCharts}:
|
|
37
|
-
#
|
|
36
|
+
# {Ucode::Fetch::CodeCharts}: the hex representation of the
|
|
37
|
+
# codepoint, zero-padded to a minimum of 4 digits (e.g.
|
|
38
|
+
# `U0000.pdf` for BMP, `U10920.pdf` for Plane 1,
|
|
39
|
+
# `U100000.pdf` for Plane 16 SPUA-B).
|
|
38
40
|
#
|
|
39
41
|
# @param block_first_cp [Integer]
|
|
40
42
|
# @return [String]
|
|
41
43
|
def self.code_chart_url(block_first_cp)
|
|
42
|
-
|
|
43
|
-
slug = block_first_cp.to_s(16).upcase.rjust(width, "0")
|
|
44
|
+
slug = block_first_cp.to_s(16).upcase.rjust(4, "0")
|
|
44
45
|
"#{Ucode.configuration.charts_base_url}/U#{slug}.pdf"
|
|
45
46
|
end
|
|
46
47
|
|
|
@@ -78,4 +79,4 @@ module Ucode
|
|
|
78
79
|
Digest::SHA256.file(path).hexdigest
|
|
79
80
|
end
|
|
80
81
|
end
|
|
81
|
-
end
|
|
82
|
+
end
|
|
@@ -31,7 +31,7 @@ module Ucode
|
|
|
31
31
|
# @return [Pathname] the written sidecar path
|
|
32
32
|
def write(provenance)
|
|
33
33
|
path = path_for(provenance)
|
|
34
|
-
payload = JSON.pretty_generate(provenance.to_h)
|
|
34
|
+
payload = "#{JSON.pretty_generate(provenance.to_h)}\n"
|
|
35
35
|
write_atomic(path, payload)
|
|
36
36
|
path
|
|
37
37
|
end
|
|
@@ -49,4 +49,4 @@ module Ucode
|
|
|
49
49
|
end
|
|
50
50
|
end
|
|
51
51
|
end
|
|
52
|
-
end
|
|
52
|
+
end
|
data/lib/ucode/code_chart.rb
CHANGED
data/lib/ucode/commands/build.rb
CHANGED
|
@@ -8,36 +8,23 @@ require "ucode/version_resolver"
|
|
|
8
8
|
module Ucode
|
|
9
9
|
module Commands
|
|
10
10
|
# `ucode build` — full pipeline: fetch (ucd + unihan + charts) →
|
|
11
|
-
# parse →
|
|
12
|
-
# is idempotent and safe to re-run.
|
|
11
|
+
# parse → site. Resumable: each step is idempotent and safe to re-run.
|
|
13
12
|
#
|
|
14
13
|
# Resolves the version intent once at the top and threads the
|
|
15
|
-
# resolved string through every sub-command.
|
|
16
|
-
# 2026-06-29 architecture review.
|
|
17
|
-
#
|
|
18
|
-
# **Glyph step is opt-in as of v0.1** because the SVG cell extractor
|
|
19
|
-
# is still experimental. Pass `include_glyphs: true` to enable it;
|
|
20
|
-
# otherwise the glyphs step is recorded as skipped.
|
|
14
|
+
# resolved string through every sub-command.
|
|
21
15
|
class BuildCommand
|
|
22
16
|
# @param version_intent [nil, :default, :latest, String]
|
|
23
17
|
# @param output_root [String, Pathname]
|
|
24
18
|
# @param site_root [String, Pathname, nil] if nil, skip site build
|
|
25
|
-
# @param monolith_path [String, Pathname, nil] CodeCharts.pdf fallback
|
|
26
19
|
# @param force_fetch [Boolean] re-download sources
|
|
27
|
-
# @param include_glyphs [Boolean] opt into the experimental glyph
|
|
28
|
-
# step (default false)
|
|
29
|
-
# @param warn [IO, nil] forwarded to GlyphsCommand when enabled
|
|
30
20
|
# @return [Hash] aggregated step results
|
|
31
21
|
def call(version_intent, output_root:, site_root: nil,
|
|
32
|
-
|
|
33
|
-
include_glyphs: false, warn: nil)
|
|
22
|
+
force_fetch: false)
|
|
34
23
|
version = VersionResolver.resolve(version_intent)
|
|
35
24
|
steps = {}
|
|
36
25
|
|
|
37
26
|
steps[:fetch] = run_fetch(version, force: force_fetch)
|
|
38
27
|
steps[:parse] = ParseCommand.new.call(version, output_root: output_root)
|
|
39
|
-
steps[:glyphs] = run_glyphs(version, output_root, monolith_path,
|
|
40
|
-
include_glyphs: include_glyphs, warn: warn)
|
|
41
28
|
steps[:site] = run_site(output_root, site_root) if site_root
|
|
42
29
|
|
|
43
30
|
{ version: version, steps: steps }
|
|
@@ -54,16 +41,6 @@ module Ucode
|
|
|
54
41
|
}
|
|
55
42
|
end
|
|
56
43
|
|
|
57
|
-
def run_glyphs(version, output_root, monolith_path, include_glyphs:, warn:)
|
|
58
|
-
GlyphsCommand.new.call(
|
|
59
|
-
version,
|
|
60
|
-
output_root: output_root,
|
|
61
|
-
monolith_path: monolith_path || "CodeCharts.pdf",
|
|
62
|
-
include_glyphs: include_glyphs,
|
|
63
|
-
warn: warn,
|
|
64
|
-
)
|
|
65
|
-
end
|
|
66
|
-
|
|
67
44
|
def run_site(output_root, site_root)
|
|
68
45
|
SiteCommand.new.build(output_root: output_root, site_root: site_root)
|
|
69
46
|
end
|
|
@@ -20,10 +20,7 @@ module Ucode
|
|
|
20
20
|
# `index.json` + `glyph.svg` atomically, accumulate per-tier +
|
|
21
21
|
# per-block stats, and emit `output/build-report.json`.
|
|
22
22
|
#
|
|
23
|
-
# This is the
|
|
24
|
-
# in {GlyphsCommand}. The two coexist until the v0.1 pipeline is
|
|
25
|
-
# removed (TODOs 17-19); CanonicalBuildCommand is the path forward
|
|
26
|
-
# for production dataset runs.
|
|
23
|
+
# This is the production path for dataset runs.
|
|
27
24
|
#
|
|
28
25
|
# == Pre-conditions (per TODO 21)
|
|
29
26
|
#
|
data/lib/ucode/commands/fetch.rb
CHANGED
data/lib/ucode/commands/parse.rb
CHANGED
data/lib/ucode/commands.rb
CHANGED
|
@@ -9,7 +9,6 @@ module Ucode
|
|
|
9
9
|
module Commands
|
|
10
10
|
autoload :FetchCommand, "ucode/commands/fetch"
|
|
11
11
|
autoload :ParseCommand, "ucode/commands/parse"
|
|
12
|
-
autoload :GlyphsCommand, "ucode/commands/glyphs"
|
|
13
12
|
autoload :SiteCommand, "ucode/commands/site"
|
|
14
13
|
autoload :LookupCommand, "ucode/commands/lookup"
|
|
15
14
|
autoload :CacheCommand, "ucode/commands/cache"
|
|
@@ -69,7 +69,7 @@ module Ucode
|
|
|
69
69
|
# `output/relationships/`
|
|
70
70
|
# @yieldparam records [Hash<Integer|String, Record|Array<Record>>]
|
|
71
71
|
# @return [Enumerator] when no block is given
|
|
72
|
-
def each_relationship(&
|
|
72
|
+
def each_relationship(&)
|
|
73
73
|
return enum_for(:each_relationship) unless block_given?
|
|
74
74
|
|
|
75
75
|
RELATIONSHIPS.each do |slug, field|
|
|
@@ -78,4 +78,4 @@ module Ucode
|
|
|
78
78
|
end
|
|
79
79
|
end
|
|
80
80
|
end
|
|
81
|
-
end
|
|
81
|
+
end
|
data/lib/ucode/error.rb
CHANGED
|
@@ -24,8 +24,6 @@ module Ucode
|
|
|
24
24
|
# │ ├── Ucode::DatabaseSchemaError
|
|
25
25
|
# │ └── Ucode::UnknownVersionError
|
|
26
26
|
# └── Ucode::GlyphError
|
|
27
|
-
# ├── Ucode::PdfRenderError
|
|
28
|
-
# ├── Ucode::GridDetectionError
|
|
29
27
|
# ├── Ucode::LastResortMissingError
|
|
30
28
|
# ├── Ucode::EmbeddedFontsMissingError
|
|
31
29
|
# └── Ucode::UniversalSetPreBuildError
|
|
@@ -104,12 +102,6 @@ module Ucode
|
|
|
104
102
|
# Glyph pipeline failures.
|
|
105
103
|
class GlyphError < Error; end
|
|
106
104
|
|
|
107
|
-
# PDF → SVG rendering failure.
|
|
108
|
-
class PdfRenderError < GlyphError; end
|
|
109
|
-
|
|
110
|
-
# Grid detection couldn't anchor on codepoint labels.
|
|
111
|
-
class GridDetectionError < GlyphError; end
|
|
112
|
-
|
|
113
105
|
# The Last Resort Font UFO source cannot be located or is missing a
|
|
114
106
|
# required artifact (cmap-f13.ttx, font.ufo/glyphs/, contents.plist).
|
|
115
107
|
class LastResortMissingError < GlyphError; end
|
|
@@ -6,7 +6,7 @@ module Ucode
|
|
|
6
6
|
#
|
|
7
7
|
# URL pattern: `https://www.unicode.org/charts/PDF/U<XXXX>.pdf`
|
|
8
8
|
# where `XXXX` is the block's first codepoint zero-padded to 4 digits
|
|
9
|
-
# (5
|
|
9
|
+
# (5 digits for planes > 0).
|
|
10
10
|
module CodeCharts
|
|
11
11
|
class << self
|
|
12
12
|
# @param version [String] used as the on-disk path namespace; PDFs
|
|
@@ -48,8 +48,7 @@ module Ucode
|
|
|
48
48
|
private
|
|
49
49
|
|
|
50
50
|
def hex_pad(codepoint)
|
|
51
|
-
|
|
52
|
-
codepoint.to_s(16).upcase.rjust(width, "0")
|
|
51
|
+
codepoint.to_s(16).upcase.rjust(4, "0")
|
|
53
52
|
end
|
|
54
53
|
end
|
|
55
54
|
end
|
data/lib/ucode/fetch/http.rb
CHANGED
|
@@ -44,21 +44,19 @@ module Ucode
|
|
|
44
44
|
|
|
45
45
|
last_error = nil
|
|
46
46
|
(attempts + 1).times do |attempt|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
"
|
|
58
|
-
"#{e.class}: #{e.message}; retrying in #{sleep_for}s"
|
|
59
|
-
end
|
|
60
|
-
sleep(sleep_for)
|
|
47
|
+
response = stream_to(uri, destination, read_timeout)
|
|
48
|
+
validate_response!(validate, response, destination) if validate
|
|
49
|
+
return destination
|
|
50
|
+
rescue ValidationFailure => e
|
|
51
|
+
raise e.cause
|
|
52
|
+
rescue StandardError => e
|
|
53
|
+
last_error = e
|
|
54
|
+
sleep_for = backoff_sequence[attempt] || backoff_sequence.last
|
|
55
|
+
Ucode.configuration.logger&.warn do
|
|
56
|
+
"Http GET #{uri} failed (attempt #{attempt + 1}/#{attempts + 1}): " \
|
|
57
|
+
"#{e.class}: #{e.message}; retrying in #{sleep_for}s"
|
|
61
58
|
end
|
|
59
|
+
sleep(sleep_for)
|
|
62
60
|
end
|
|
63
61
|
|
|
64
62
|
raise Ucode::NetworkError.new(
|