ucode 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/config/unicode17_universal_glyph_set.yml +1 -1
  3. data/lib/ucode/code_chart/extractor.rb +7 -3
  4. data/lib/ucode/code_chart/writer.rb +1 -1
  5. data/lib/ucode/error.rb +5 -0
  6. data/lib/ucode/glyphs/embedded_fonts/catalog.rb +20 -3
  7. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/correlator_strategy.rb +46 -0
  8. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/strategy.rb +54 -0
  9. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/tounicode_strategy.rb +41 -0
  10. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper/trace_strategy.rb +64 -0
  11. data/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb +143 -90
  12. data/lib/ucode/glyphs/embedded_fonts/mutool/draw.rb +26 -0
  13. data/lib/ucode/glyphs/embedded_fonts/mutool/info.rb +23 -0
  14. data/lib/ucode/glyphs/embedded_fonts/mutool/show.rb +36 -0
  15. data/lib/ucode/glyphs/embedded_fonts/mutool/trace.rb +26 -0
  16. data/lib/ucode/glyphs/embedded_fonts/mutool.rb +30 -0
  17. data/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb +90 -0
  18. data/lib/ucode/glyphs/embedded_fonts/pdf_indexer.rb +50 -56
  19. data/lib/ucode/glyphs/embedded_fonts/{pdf_location.rb → pdf_source.rb} +1 -1
  20. data/lib/ucode/glyphs/embedded_fonts/trace_runner.rb +12 -28
  21. data/lib/ucode/glyphs/embedded_fonts.rb +3 -1
  22. data/lib/ucode/glyphs/real_fonts/coverage_auditor.rb +0 -4
  23. data/lib/ucode/glyphs/real_fonts/font_coverage_report.rb +0 -2
  24. data/lib/ucode/glyphs/real_fonts/writer.rb +0 -2
  25. data/lib/ucode/glyphs/sources/pillar1_embedded_tounicode.rb +1 -1
  26. data/lib/ucode/version.rb +1 -1
  27. metadata +13 -74
  28. data/TODO.extract-code-chart/01-pdf-fetch-validation.md +0 -80
  29. data/TODO.extract-code-chart/02-block-name-resolver.md +0 -68
  30. data/TODO.extract-code-chart/03-codechart-namespace.md +0 -82
  31. data/TODO.extract-code-chart/04-codechart-extractor.md +0 -154
  32. data/TODO.extract-code-chart/05-provenance-and-sidecar.md +0 -147
  33. data/TODO.extract-code-chart/06-codechart-writer.md +0 -134
  34. data/TODO.extract-code-chart/07-codechart-cli.md +0 -135
  35. data/TODO.extract-code-chart/08-specs.md +0 -87
  36. data/TODO.full/00-README.md +0 -116
  37. data/TODO.full/01-panglyph-vision.md +0 -112
  38. data/TODO.full/02-panglyph-repo-bootstrap.md +0 -184
  39. data/TODO.full/03-panglyph-font-builder.md +0 -201
  40. data/TODO.full/04-panglyph-publish-pipeline.md +0 -126
  41. data/TODO.full/05-ucode-0-1-1-release.md +0 -139
  42. data/TODO.full/06-fontisan-remove-audit.md +0 -142
  43. data/TODO.full/07-fontisan-remove-ucd.md +0 -125
  44. data/TODO.full/08-archive-private-bin-build.md +0 -143
  45. data/TODO.full/09-archive-public-structure.md +0 -164
  46. data/TODO.full/10-fontist-org-woff-glyphs.md +0 -131
  47. data/TODO.full/11-fontist-org-audit-coverage.md +0 -140
  48. data/TODO.full/12-implementation-order.md +0 -216
  49. data/TODO.full/13-fontisan-font-writer-api.md +0 -189
  50. data/TODO.full/14-fontisan-table-writers.md +0 -66
  51. data/TODO.full/15-panglyph-builder-real.md +0 -82
  52. data/TODO.full/16-archive-public-sync-workflows.md +0 -167
  53. data/TODO.full/17-fontist-org-font-picker.md +0 -73
  54. data/TODO.full/18-comprehensive-spec-coverage.md +0 -64
  55. data/TODO.full/19-ucode-0-1-2-patch.md +0 -32
  56. data/TODO.full/20-fontisan-0-2-23-release.md +0 -52
  57. data/TODO.new/00-README.md +0 -96
  58. data/TODO.new/01-pillar-terminology-alignment.md +0 -69
  59. data/TODO.new/02-audit-schema-design.md +0 -255
  60. data/TODO.new/03-directory-output-spec.md +0 -203
  61. data/TODO.new/04-fontist-org-contract.md +0 -173
  62. data/TODO.new/05-baseline-unicode17-coverage-audit.md +0 -144
  63. data/TODO.new/06-audit-namespace-skeleton.md +0 -105
  64. data/TODO.new/07-audit-models-port.md +0 -132
  65. data/TODO.new/08-extractors-cheap-port.md +0 -113
  66. data/TODO.new/09-extractors-expensive-port.md +0 -99
  67. data/TODO.new/10-aggregations-ucd-rewrite.md +0 -168
  68. data/TODO.new/11-differ-and-library-auditor-port.md +0 -102
  69. data/TODO.new/12-formatters-port.md +0 -115
  70. data/TODO.new/13-directory-emitter.md +0 -147
  71. data/TODO.new/14-html-face-browser.md +0 -144
  72. data/TODO.new/15-html-library-browser.md +0 -102
  73. data/TODO.new/16-cli-audit-subcommands.md +0 -142
  74. data/TODO.new/17-fontisan-cleanup-audit.md +0 -147
  75. data/TODO.new/18-fontisan-cleanup-ucd.md +0 -156
  76. data/TODO.new/19-fontisan-docs-update.md +0 -155
  77. data/TODO.new/20-canonical-resolver-4-tier.md +0 -182
  78. data/TODO.new/21-canonical-unicode17-build.md +0 -148
  79. data/TODO.new/22-implementation-order.md +0 -176
  80. data/TODO.new/23-universal-glyph-set-source-map.md +0 -312
  81. data/TODO.new/24-universal-glyph-set-build.md +0 -189
  82. data/TODO.new/25-font-audit-against-universal-set.md +0 -195
  83. data/TODO.new/26-missing-glyph-reporter.md +0 -189
  84. data/TODO.new/27-fontist-org-consumer-integration.md +0 -200
  85. data/TODO.new/28-implementation-order-update.md +0 -187
  86. data/TODO.new/29-universal-set-curation-uc17.md +0 -312
  87. data/TODO.new/30-tier1-font-acquisition.md +0 -241
  88. data/TODO.new/31-universal-set-production-build.md +0 -205
  89. data/TODO.new/32-uc17-coverage-matrix.md +0 -165
  90. data/TODO.new/33-specialist-font-acquisition-refresh.md +0 -138
  91. data/TODO.new/34-pillar2-content-stream-correlator.md +0 -147
  92. data/TODO.new/35-universal-set-production-run.md +0 -160
  93. data/TODO.new/36-per-font-coverage-audit.md +0 -145
  94. data/TODO.new/37-coverage-highlight-reporter.md +0 -125
  95. data/TODO.new/38-fontist-org-glyph-consumer.md +0 -141
  96. data/TODO.new/39-implementation-order-update-32-38.md +0 -258
  97. data/TODO.new/40-archive-private-uses-ucode-audit.md +0 -124
  98. data/TODO.new/41-ucode-unicode-archive-bridge.md +0 -160
@@ -1,135 +0,0 @@
1
- # TODO 07 — CodeChart CLI
2
-
3
- ## Status
4
-
5
- Pending. Depends on TODO 06 (Writer), TODO 04 (Extractor),
6
- TODO 02 (block name resolver).
7
-
8
- ## Goal
9
-
10
- `ucode code-chart fetch | extract | list` — the REQ (R4) commands.
11
- Thin Thor wrappers that delegate to the existing `CodeChart::*`
12
- modules. No orchestration logic in the CLI; every command is a
13
- single delegation.
14
-
15
- ## Files
16
-
17
- - `lib/ucode/cli.rb` — add the `CodeChartCmd` Thor subcommand class.
18
- - `spec/ucode/cli_spec.rb` (extend existing) — verify the new
19
- subcommand wires up.
20
-
21
- ## Design
22
-
23
- ### Subcommand shape
24
-
25
- ```ruby
26
- class Cli < Thor
27
- # …existing commands…
28
-
29
- class CodeChartCmd < Thor
30
- desc "fetch --block BLOCK", "Download the Code Charts PDF for a block"
31
- option :block, type: :string, required: true,
32
- desc: "Block identifier (e.g. Sidetic)"
33
- def fetch
34
- puts JSON.pretty_generate(
35
- Commands::FetchCommand.new.fetch_charts(
36
- VersionResolver.resolve(nil),
37
- block_first_cps: [block_first_cp!(options[:block])],
38
- ),
39
- )
40
- end
41
-
42
- desc "extract --block BLOCK --to DIR", "Extract per-codepoint SVG + provenance sidecars"
43
- option :block, type: :string, required: true,
44
- desc: "Block identifier (e.g. Sidetic)"
45
- option :to, type: :string, required: true,
46
- desc: "Output directory"
47
- def extract
48
- # ...
49
- end
50
-
51
- desc "list", "List blocks that have Code Charts PDFs available locally"
52
- def list
53
- # ...
54
- end
55
- end
56
-
57
- desc "code-chart", "Extract per-codepoint SVG glyphs from Unicode Code Charts PDFs"
58
- subcommand "code-chart", CodeChartCmd
59
- end
60
- ```
61
-
62
- ### `extract` flow
63
-
64
- ```ruby
65
- def extract
66
- Ucode::Commands::FetchCommand.new.fetch_charts(
67
- VersionResolver.resolve(nil),
68
- block_first_cps: [block_first_cp!(options[:block])],
69
- )
70
- blocks_txt = Ucode::Cache.ucd_dir(VersionResolver.resolve(nil)).join("Blocks.txt")
71
- block = Parsers::Blocks.find_by_name(blocks_txt, options[:block]) or
72
- raise Thor::Error, "Unknown block: #{options[:block].inspect}"
73
- pdf = Ucode::Glyphs::PdfFetcher.new(
74
- VersionResolver.resolve(nil),
75
- monolith_path: nil,
76
- blocks: [block],
77
- ).fetch(block_first_cp: block.range_first, force: false) or
78
- raise Thor::Error, "PDF unavailable for block #{options[:block]}"
79
-
80
- writer = Ucode::CodeChart::Writer.new(
81
- output_root: Pathname.new(options[:to]),
82
- pdf_path: pdf,
83
- blocks_txt: blocks_txt,
84
- )
85
- summary = writer.write(block)
86
- puts JSON.pretty_generate(summary.to_h.compact)
87
- end
88
-
89
- def block_first_cp!(block_id)
90
- cache = Ucode::Cache.ucd_dir(VersionResolver.resolve(nil))
91
- block = Ucode::Parsers::Blocks.find_by_name(cache.join("Blocks.txt"), block_id)
92
- raise Thor::Error, "Unknown block: #{block_id.inspect}" unless block
93
- block.range_first
94
- end
95
- ```
96
-
97
- ### Why `Ucode::Commands::FetchCommand.new.fetch_charts` for fetch
98
-
99
- `fetch_charts` is the existing CLI hook for "download a Code Charts
100
- PDF for these block first-cps." We just call it with the block's
101
- first cp. Reuse, don't reimplement.
102
-
103
- ### Why resolve version once at the top of `extract`
104
-
105
- Per Candidate 4 of the architecture review (`refactor/build-context-resolve-version-once`,
106
- merged): every CLI method resolves the version once and threads it
107
- through. This CLI method does the same — one call per invocation.
108
-
109
- ### Why no separate `Commands::CodeChartCommand` class
110
-
111
- Following the existing pattern (e.g. `Cli::Audit` calls
112
- `Commands::Audit::*Command` *only* when the logic is non-trivial).
113
- The CodeChart commands are trivial delegations — a one-liner each.
114
- The CLI methods call `CodeChart::Writer` and `CodeChart::Extractor`
115
- directly. Adding a `Commands::CodeChartCommand` class would be
116
- indirection without a payoff.
117
-
118
- If the extract logic grows (e.g. progress reporting, partial
119
- extraction), extract it into a Command class at that point.
120
-
121
- ## Acceptance
122
-
123
- - `ucode code-chart fetch --block Sidetic` downloads the PDF.
124
- - `ucode code-chart extract --block Sidetic --to /tmp/s/` extracts
125
- to the given directory.
126
- - `ucode code-chart list` prints available blocks.
127
- - Unknown block names produce a clean error, not a stack trace.
128
- - The CLI matches the REQ's signature exactly.
129
-
130
- ## Out of scope
131
-
132
- - `--version` flag (the REQ doesn't specify, and existing commands
133
- default to the configured version).
134
- - `--format svg|glif` (the REQ specifies SVG; `.glif` output is a
135
- future extension).
@@ -1,87 +0,0 @@
1
- # TODO 08 — CodeChart specs
2
-
3
- ## Status
4
-
5
- Pending. Depends on TODOs 01–07.
6
-
7
- ## Goal
8
-
9
- Comprehensive spec coverage for every new module. Per the project
10
- rules: real model instances (no doubles), tight focused tests,
11
- behavior assertions (not interaction counts).
12
-
13
- ## Files
14
-
15
- Already enumerated per TODO. Final spec coverage:
16
-
17
- - `spec/ucode/fetch/code_charts_spec.rb` — happy path + HTTP 4xx +
18
- wrong content-type + non-PDF body. (TODO 01)
19
- - `spec/ucode/parsers/blocks_spec.rb` (extend) — `find_by_name`
20
- happy path + nil on miss. (TODO 02)
21
- - `spec/ucode/code_chart/extractor_spec.rb` — constructor invariants,
22
- Resolver wiring, integration test against fixture PDF. (TODO 04)
23
- - `spec/ucode/code_chart/provenance_spec.rb` — value object
24
- construction + `to_h` schema. (TODO 05)
25
- - `spec/ucode/code_chart/sidecar_spec.rb` — write sidecar, idempotent
26
- re-write. (TODO 05)
27
- - `spec/ucode/code_chart/writer_spec.rb` — full lifecycle:
28
- extract → write → summary. Idempotent re-run produces byte-identical
29
- files. (TODO 06)
30
- - `spec/ucode/cli_spec.rb` (extend) — verify `ucode code-chart fetch`,
31
- `extract`, `list` wire up. (TODO 07)
32
-
33
- ## Design
34
-
35
- ### Fixture strategy
36
-
37
- The existing `spec/fixtures/pdfs/basic_latin.pdf` is the only PDF
38
- fixture in the repo. It's tiny and validates the integration path.
39
- The Sidetic + Egyptian Ext-B PDFs are large (whole-block) and would
40
- inflate the repo. The integration spec uses `basic_latin.pdf` to
41
- exercise the full pipeline; per-codepoint assertions cover
42
- representative cases.
43
-
44
- If Sidetic-specific behavior must be tested, a smaller fixture PDF
45
- cropped to ~5 codepoints would be the right tool — out of scope for
46
- this TODO.
47
-
48
- ### No doubles policy
49
-
50
- The project's `~/.claude/CLAUDE.md` rule: no doubles. All specs use
51
- real instances:
52
- - `Ucode::Models::Block.new(...)` for test blocks.
53
- - A temp directory + real `Blocks.txt` text for parser specs.
54
- - The real `Ucode::Glyphs::Resolver` for extractor specs.
55
-
56
- ### Idempotency assertion pattern
57
-
58
- `Writer#write` idempotency is asserted via byte-equality:
59
-
60
- ```ruby
61
- first_run = writer.write(block)
62
- first_size = File.stat(svg_path).size
63
- sleep 0.01 # ensure mtime changes would be detectable
64
- second_run = writer.write(block)
65
- second_size = File.stat(svg_path).size
66
- expect(second_size).to eq(first_size)
67
- expect(File.read(svg_path)).to eq(expected_svg_bytes)
68
- ```
69
-
70
- This is the existing pattern from `spec/ucode/repo/aggregate_writer_spec.rb`
71
- (idiempotency spec there). Reuse.
72
-
73
- ## Acceptance
74
-
75
- - `bundle exec rspec spec/ucode/code_chart/ spec/ucode/fetch/code_charts_spec.rb spec/ucode/parsers/blocks_spec.rb`
76
- passes 100%.
77
- - Coverage for the new files is ≥ 95% (per the project's per-file
78
- floor of 30% + the overall 80% minimum).
79
- - No doubles are introduced (verify with `grep -r "double(" spec/ucode/code_chart/`).
80
- - The integration spec exercises both the Extractor and Writer
81
- together end-to-end.
82
-
83
- ## Out of scope
84
-
85
- - Performance benchmarks — separate concern.
86
- - Sidetic-specific fixtures — requires PDF curation beyond the
87
- scope of this feature.
@@ -1,116 +0,0 @@
1
- # TODO.full — Master plan: panglyph + ucode release + fontisan cleanup + full pipeline
2
-
3
- This is the **shipping plan** for the full Fontist Unicode 17 stack:
4
-
5
- 1. **ucode** — Unicode data + audit tool + universal glyph set (SVGs)
6
- 2. **panglyph** (NEW) — assembles ucode's universal set into a single redistributable font
7
- 3. **fontisan** — slim font parsing + WOFF conversion library (audit + UCD removed)
8
- 4. **fontist-archive-private** — CI matrix runs ucode audit + fontisan convert per formula
9
- 5. **fontist-archive-public** — public artifacts: `coverage/` + `woff/` + `unicode/` + `panglyph/`
10
- 6. **fontist.org** — per-font unicode browser using WOFF (open-license) + ucode audit (all)
11
-
12
- The plan below covers everything that remains after TODO.new/ (which built the
13
- infrastructure). TODO.full/ is about wiring it all into a shippable product.
14
-
15
- ## Directives from user
16
-
17
- - **D0** — Define + build "Fontist universal glyph set for Unicode 17" as a
18
- single font, in a new repo `panglyph`. Uses fontisan to extract outlines
19
- from source fonts, assembles into one redistributable font file.
20
- - **D1** — Publish ucode as a patch release (0.1.0 → 0.1.1).
21
- - **D2** — Clean up fontisan: remove `AuditCommand` and UCD/UCDXML
22
- subsystems (now in ucode). Keep: font parsing primitives, ConvertCommand.
23
- - **D3** — Wire `fontist-archive-private` to use fontisan (WOFF) + ucode
24
- (audit) for ALL fonts. Wire `fontist-archive-public` to host all artifacts.
25
- - **D4** — Update fontist.org to consume fontist-archive-public: render
26
- per-font glyphs from WOFF (open-license), show per-font coverage from
27
- ucode audit (all fonts).
28
-
29
- ## File index
30
-
31
- ### Foundation
32
-
33
- - [01 — Panglyph vision: what the universal font is, why it exists](01-panglyph-vision.md)
34
- - [02 — Panglyph repo bootstrap (gem skeleton, CLI, CI)](02-panglyph-repo-bootstrap.md)
35
- - [03 — Panglyph font builder (outline extract + assemble + write)](03-panglyph-font-builder.md)
36
- - [04 — Panglyph publish pipeline (release artifacts to fontist-archive-public)](04-panglyph-publish-pipeline.md)
37
-
38
- ### Releases
39
-
40
- - [05 — ucode 0.1.1 patch release](05-ucode-0-1-1-release.md)
41
-
42
- ### Cleanup
43
-
44
- - [06 — fontisan: remove AuditCommand (and audit/ namespace)](06-fontisan-remove-audit.md)
45
- - [07 — fontisan: remove UCD/UCDXML subsystem](07-fontisan-remove-ucd.md)
46
-
47
- ### Pipeline
48
-
49
- - [08 — fontist-archive-private bin/build uses ucode audit + fontisan convert](08-archive-private-bin-build.md)
50
- - [09 — fontist-archive-public structure: unicode/ + panglyph/ + coverage/ + woff/](09-archive-public-structure.md)
51
-
52
- ### Consumer
53
-
54
- - [10 — fontist.org: per-font WOFF glyph rendering (open-license)](10-fontist-org-woff-glyphs.md)
55
- - [11 — fontist.org: per-font ucode audit rendering (ALL fonts)](11-fontist-org-audit-coverage.md)
56
-
57
- ### Sequencing
58
-
59
- - [12 — Implementation order (all directives)](12-implementation-order.md)
60
-
61
- ## Critical path (high-level)
62
-
63
- ```
64
- ┌──────────────────────────────────┐
65
- │ 05 ucode 0.1.1 patch release │ ← unblocks all downstream
66
- └──────────────┬───────────────────┘
67
-
68
- ┌──────────────┴───────────────────┐
69
- │ │
70
- ▼ ▼
71
- ┌────────────────────────┐ ┌────────────────────────┐
72
- │ 06 fontisan audit │ │ 01–04 panglyph │
73
- │ removal │ │ (new repo) │
74
- │ 07 fontisan UCD │ └────────────┬───────────┘
75
- │ removal │ │
76
- └────────────┬───────────┘ │
77
- │ │
78
- ▼ │
79
- ┌────────────────────────┐ │
80
- │ 08 archive-private │ │
81
- │ bin/build refactor │ │
82
- └────────────┬───────────┘ │
83
- │ │
84
- ▼ ▼
85
- ┌────────────────────────┐ ┌────────────────────────┐
86
- │ 09 archive-public │◄────────┤ 04 panglyph publish │
87
- │ structure │ │ to archive-public │
88
- └────────────┬───────────┘ └────────────────────────┘
89
-
90
-
91
- ┌────────────────────────┐
92
- │ 10 fontist.org WOFF │
93
- │ 11 fontist.org audit │
94
- └────────────────────────┘
95
- ```
96
-
97
- ## Repositories involved
98
-
99
- | Repo | Role | Branch / state |
100
- |---|---|---|
101
- | `fontist/ucode` | Unicode data + audit tool + universal glyph set | `fix/fontist-consumer-canonical-path` (PR #43) |
102
- | `fontist/panglyph` (NEW) | Universal font assembler | not yet created |
103
- | `fontist/fontisan` | Font parser + WOFF converter | `fix/ucdxml-real-shape-parsing` (cleanup target) |
104
- | `fontist/fontist-archive-private` | CI build env (per-formula) | main (uses old fontisan AuditCommand) |
105
- | `fontist/fontist-archive-public` | Public artifacts | main (no unicode/ or panglyph/ yet) |
106
- | `fontist/fontist.github.io` | Consumer site | `fix/unicode-char-page-fields` (PR #45) |
107
-
108
- ## Conventions
109
-
110
- - **PR-per-TODO** unless tightly coupled.
111
- - **Merging requires explicit user authorization per PR.**
112
- - **Never push tags directly.** Tag + `rake release` only after explicit user sign-off.
113
- - **No AI attribution** in commits, PRs, or release notes.
114
- - **Original block names verbatim** (`CJK_Ext_A`, never slugified) in source data.
115
- - **Vector-only glyph extraction.** No OCR.
116
- - Branch naming: `<repo-scope>/<track-slug>` (e.g. `audit/remove-audit-command`).
@@ -1,112 +0,0 @@
1
- # 01 — Panglyph vision: the Fontist universal Unicode 17 font
2
-
3
- ## What it is
4
-
5
- **panglyph** is a single redistributable font file that covers every
6
- assigned Unicode 17.0 codepoint (~299,382 glyphs). It's the **materialized
7
- form of ucode's universal glyph set**: where ucode produces one SVG per
8
- codepoint (sourced from Tier 1 fonts via fontisan), panglyph assembles
9
- those outlines into one OpenType/TrueType font that any application can
10
- install and use as a Unicode 17 fallback.
11
-
12
- Think "Noto Sans for everything" — except:
13
- - Sourced from many Tier 1 fonts (Noto family + FSung for CJK, Lentariso
14
- for Sidetic, Kedebideri for Beria Erfe, NotoSerifTaiYo, UniHieroglyphica,
15
- Egyptian Text, Symbola, BabelStone, etc.)
16
- - One font file, multiple script sources
17
- - Built reproducibly from ucode's per-block coverage matrix (TODO 32 in
18
- TODO.new/)
19
- - Open License (OFL for the assembled font, matching source licenses)
20
-
21
- ## Why it exists
22
-
23
- Today, no single font covers Unicode 17. Noto Sans comes closest but
24
- misses:
25
- - Rare UC17 additions (Sidetic, Beria Erfe, Tolong Siki, Tai Yo)
26
- - Egyptian Hieroglyphs Extended-B (needs UniHieroglyphica v16)
27
- - CJK Extension J (needs FSung)
28
- - Symbols for Legacy Computing Supplement (needs BabelStone)
29
-
30
- Users who want "Unicode 17 everywhere" must install 10+ fonts. panglyph
31
- collapses that to one.
32
-
33
- ## Use cases
34
-
35
- 1. **Browser fallback.** Browsers can be configured to use panglyph as
36
- the last-resort font. Any codepoint not covered by the active font
37
- gets panglyph's outline instead of tofu.
38
- 2. **OS-level Unicode 17 coverage.** Install once, every app gets full
39
- Unicode 17 rendering.
40
- 3. **Print/PDF embedding.** Designers can embed a single font for any
41
- Unicode 17 text.
42
- 4. **Search/indexing tools.** Text extraction tools that need glyph
43
- recognition for rare scripts get a uniform source.
44
- 5. **Fontist.org specimen rendering.** When fontist.org shows a char
45
- that the active font misses, fall back to panglyph instead of tofu.
46
-
47
- ## What it is NOT
48
-
49
- - **Not a replacement for source fonts.** panglyph is a fallback. Active
50
- fonts (the user's chosen Noto Sans, FSung, etc.) take priority.
51
- - **Not a font designer's tool.** It's a redistribution mechanism.
52
- - **Not a copy of Noto.** Different sources, different coverage policy.
53
- - **Not color emoji.** Vector outlines only (same as ucode's universal
54
- glyph set). Color emoji would need a separate TODO.
55
-
56
- ## Source policy
57
-
58
- panglyph is assembled from ucode's universal-set manifest
59
- (`output/universal_glyph_set/manifest.json`). For each codepoint:
60
-
61
- 1. Look up the Tier 1 source font (per `config/unicode17_universal_glyph_set.yml`)
62
- 2. Open the source font via fontisan
63
- 3. Extract the glyf outline (or CFF charstring for OTF) for the codepoint's GID
64
- 4. Copy the outline into panglyph's glyf table at the same GID
65
-
66
- Tier 2 (PDF-embedded extraction via correlate-v4 generalization) and
67
- Tier 3 (Last Resort tofu) are fallbacks when Tier 1 is unavailable.
68
- Tier 3 produces the recognizable "box with codepoint hex label" glyph
69
- familiar from Last Resort Font.
70
-
71
- ## Licensing
72
-
73
- panglyph's assembled font is **OFL**. Every source font in the universal
74
- set must be OFL (or compatible — Apache, MIT, BSD, CC0, UFL, Bitstream,
75
- GUST, CC-BY). Specialist fonts with proprietary licenses cannot be
76
- included; their codepoints fall back to pillar 2 or pillar 3.
77
-
78
- This is enforced at ucode's universal-set pre-check (TODO 35 in TODO.new/).
79
-
80
- ## Output formats
81
-
82
- | Format | Purpose |
83
- |---|---|
84
- | `panglyph-unicode17.ttf` | Installable system font (largest compatibility) |
85
- | `panglyph-unicode17.woff2` | Web font (smaller, used by fontist.org) |
86
- | `panglyph-unicode17.otf` | CFF-based variant (smaller for CJK-heavy ranges) |
87
-
88
- All three are produced by the build pipeline.
89
-
90
- ## Versioning
91
-
92
- - **`panglyph-unicode17-17.0.0.ofl`** — pinned to UCD 17.0.0
93
- - **`panglyph-unicode17-17.0.1.ofl`** — patch release (e.g. fixed an
94
- extraction bug); same Unicode data, regenerated glyphs
95
- - **`panglyph-unicode17-17.1.0.ofl`** — minor (new Tier 1 fonts added)
96
- - **`panglyph-unicode18-18.0.0.ofl`** — major (UCD 18 baseline)
97
-
98
- The first version tag is `v17.0.0` to match UCD.
99
-
100
- ## Deliverables
101
-
102
- - One redistributable font file (TTF + WOFF2 + OTF)
103
- - A SHA-256 manifest of source contributions (provenance)
104
- - An OFL license file
105
- - A coverage report (per-block % sourced from Tier 1 / Pillar 2 / Pillar 3)
106
-
107
- ## References
108
-
109
- - [TODO.new/32](../TODO.new/32-uc17-coverage-matrix.md) — Tier 1 source policy
110
- - [TODO.new/35](../TODO.new/35-universal-set-production-run.md) — universal-set SVGs (panglyph input)
111
- - [TODO.full/02](02-panglyph-repo-bootstrap.md) — repo skeleton
112
- - [TODO.full/03](03-panglyph-font-builder.md) — build implementation
@@ -1,184 +0,0 @@
1
- # 02 — Panglyph repo bootstrap
2
-
3
- ## Goal
4
-
5
- Create the `fontist/panglyph` repository skeleton: gem structure, CLI
6
- entry point, CI workflow, README. No build logic yet (that's TODO 03) —
7
- just the scaffold that lets development proceed.
8
-
9
- ## Why a separate repo
10
-
11
- panglyph consumes ucode's universal set as INPUT and produces a font
12
- as OUTPUT. The assembly logic (fontisan outline extraction + font
13
- writing + OFL packaging) is a distinct concern from ucode's Unicode
14
- data + audit work.
15
-
16
- Separation keeps:
17
- - ucode focused on Unicode data + audit
18
- - panglyph focused on font assembly
19
- - fontisan focused on parsing + conversion primitives
20
-
21
- ## Repository structure
22
-
23
- ```
24
- panglyph/
25
- ├── README.md # what panglyph is, how to build
26
- ├── LICENSE # OFL for the assembled font
27
- ├── CHANGELOG.md # version history
28
- ├── Gemfile # dev deps
29
- ├── panglyph.gemspec # gem metadata
30
- ├── lib/
31
- │ ├── panglyph.rb # top-level namespace + autoloads
32
- │ └── panglyph/
33
- │ ├── version.rb # VERSION = "17.0.0"
34
- │ ├── cli.rb # Thor CLI entry point
35
- │ ├── builder.rb # orchestrates the full build
36
- │ ├── outline_extractor.rb # extracts glyf/CFF outline via fontisan
37
- │ ├── font_assembler.rb # assembles outlines into a font
38
- │ ├── woff2_writer.rb # converts TTF → WOFF2 (via fontisan)
39
- │ ├── manifest_reader.rb # reads ucode's universal-set manifest
40
- │ ├── coverage_report.rb # emits per-block source breakdown
41
- │ └── publisher.rb # pushes artifacts to fontist-archive-public
42
- ├── exe/
43
- │ └── panglyph # CLI executable
44
- ├── spec/
45
- │ ├── spec_helper.rb
46
- │ └── panglyph/
47
- │ ├── builder_spec.rb
48
- │ ├── outline_extractor_spec.rb
49
- │ └── ...
50
- ├── data/
51
- │ └── OFL.txt # OFL license template
52
- ├── docs/
53
- │ ├── architecture.md # build pipeline reference
54
- │ └── coverage-policy.md # which fonts cover which blocks
55
- └── .github/
56
- └── workflows/
57
- └── build.yml # CI: build panglyph on tag push
58
- ```
59
-
60
- ## CLI surface
61
-
62
- ```
63
- $ panglyph --help
64
- panglyph commands:
65
- panglyph build [UCD_VERSION] # Build panglyph-unicode<version>.<ext>
66
- panglyph help [COMMAND] # Describe subcommands
67
- panglyph manifest [UCD_VERSION] # Print source contributions manifest
68
- panglyph publish [VERSION] # Publish built artifacts to archive-public
69
- panglyph validate [FONT_PATH] # Verify the built font against the universal set
70
- panglyph version # Print panglyph version
71
- ```
72
-
73
- ### `panglyph build`
74
-
75
- ```
76
- $ panglyph build 17.0.0
77
- → reads ucode's universal-set manifest (must already be built)
78
- → for each codepoint: extracts outline from the Tier 1 source font
79
- → assembles outlines into a TTF in memory
80
- → writes panglyph-unicode17-17.0.0.ttf
81
- → converts to panglyph-unicode17-17.0.0.woff2
82
- → emits coverage-report.json (per-block source breakdown)
83
- ```
84
-
85
- ### `panglyph validate`
86
-
87
- ```
88
- $ panglyph validate panglyph-unicode17-17.0.0.ttf
89
- → cmap-walks the built font
90
- → compares against ucode's universal-set codepoint list
91
- → reports: 299382 codepoints, 297415 covered (99.3%), 1967 missing
92
- → lists missing codepoints with their Tier 1 source (so the build can be fixed)
93
- ```
94
-
95
- ### `panglyph publish`
96
-
97
- ```
98
- $ panglyph publish 17.0.0
99
- → clones fontist-archive-public (shallow)
100
- → copies panglyph-unicode17-17.0.0.{ttf,woff2} to archive-public/panglyph/
101
- → updates archive-public/panglyph/manifest.json
102
- → commits + pushes via GHA bot token
103
- ```
104
-
105
- ## Dependencies
106
-
107
- ```ruby
108
- # panglyph.gemspec
109
- spec.add_dependency "fontisan", "~> 0.3" # font parsing + writing primitives
110
- spec.add_dependency "ucode", "~> 0.1" # universal-set manifest reader
111
- spec.add_dependency "thor", "~> 1.3" # CLI
112
- spec.add_dependency "json", "~> 2.0"
113
- spec.add_dependency "rubyzip", "~> 2.3" # OFL packaging
114
- ```
115
-
116
- fontisan needs font-WRITING primitives added (it currently only reads +
117
- converts). TODO 03 lists what fontisan needs to expose.
118
-
119
- ## CI workflow
120
-
121
- `.github/workflows/build.yml`:
122
-
123
- ```yaml
124
- name: Build panglyph
125
-
126
- on:
127
- push:
128
- tags: ['v*']
129
- workflow_dispatch:
130
- inputs:
131
- ucd_version:
132
- description: 'UCD version to build (e.g. 17.0.0)'
133
- required: true
134
- default: '17.0.0'
135
-
136
- jobs:
137
- build:
138
- runs-on: ubuntu-latest
139
- steps:
140
- - uses: actions/checkout@v4
141
- - uses: ruby/setup-ruby@v1
142
- with:
143
- ruby-version: '3.4'
144
- bundler-cache: true
145
- - name: Fetch ucode universal set
146
- run: |
147
- git clone --depth 1 https://github.com/fontist/ucode ucode-repo
148
- cd ucode-repo && bundle exec ucode universal-set build ${{ inputs.ucd_version }}
149
- tar czf /tmp/universal-set.tar.gz output/universal_glyph_set/
150
- - name: Build panglyph
151
- run: |
152
- bundle exec panglyph build ${{ inputs.ucd_version }} \
153
- --universal-set=/tmp/universal-set.tar.gz
154
- - name: Validate
155
- run: bundle exec panglyph validate panglyph-unicode*.ttf
156
- - name: Publish to fontist-archive-public
157
- env:
158
- GH_TOKEN: ${{ secrets.ARCHIVE_PUBLIC_BOT_TOKEN }}
159
- run: bundle exec panglyph publish ${{ inputs.ucd_version }}
160
- - uses: actions/upload-artifact@v4
161
- with:
162
- name: panglyph-${{ inputs.ucd_version }}
163
- path: |
164
- panglyph-unicode17-*.ttf
165
- panglyph-unicode17-*.woff2
166
- coverage-report.json
167
- ```
168
-
169
- ## Acceptance
170
-
171
- - [ ] `fontist/panglyph` repo exists
172
- - [ ] README.md explains what panglyph is + how to build (per TODO 01)
173
- - [ ] LICENSE is OFL
174
- - [ ] `bundle exec panglyph version` prints `17.0.0`
175
- - [ ] `bundle exec panglyph --help` lists build/manifest/publish/validate
176
- - [ ] CI workflow file exists and is syntactically valid
177
- - [ ] One trivial spec passes (`spec/panglyph/version_spec.rb`)
178
- - [ ] Repo is public
179
-
180
- ## References
181
-
182
- - [TODO 01](01-panglyph-vision.md) — vision
183
- - [TODO 03](03-panglyph-font-builder.md) — build implementation
184
- - [TODO.new/35](../TODO.new/35-universal-set-production-run.md) — input format