@oh-my-pi/snapcompact 16.2.6 → 16.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,19 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [16.2.7] - 2026-06-30
6
+
7
+ ### Added
8
+
9
+ - Added the `silver16-bw` shape backed by an embedded Silver TrueType font to support CJK and other non-Latin text.
10
+ - Added `resolveShapeForText` to support font-aware shape resolution.
11
+
12
+ ### Changed
13
+
14
+ - Improved non-ASCII text normalization by folding semantic emojis to ASCII labels (e.g., `[OK]`, `[WARN]`), dropping decorative emojis, and folding box-drawing symbols to ASCII skeletons.
15
+ - Enhanced missing glyph rendering to use the embedded Silver TrueType fallback per-character, including support for East Asian wide characters across two grid cells.
16
+ - Updated text wrapping, pagination, and provider shape geometries to support wide character footprints and updated X.org 8x13 font metrics.
17
+
5
18
  ## [16.1.23] - 2026-06-26
6
19
 
7
20
  ### Added
package/README.md CHANGED
@@ -9,7 +9,7 @@ Built for [oh-my-pi](https://github.com/can1357/oh-my-pi)'s compaction pipeline,
9
9
  ## How it works
10
10
 
11
11
  1. Discarded history is serialized to compact text (`serializeConversation`), with per-tool-result and per-argument character caps.
12
- 2. Text is normalized for the bundled bitmap fonts (`normalize`): ANSI sequences stripped, whitespace collapsed, newline runs folded into a single full-block glyph so line structure survives.
12
+ 2. Text is normalized for the selected native font (`normalize`): ANSI sequences stripped, whitespace collapsed, newline runs folded into a single full-block glyph, box drawing and compatibility symbols folded to ASCII, semantic emoji folded to ASCII labels, decorative emoji dropped, and non-Latin glyphs preserved when either the selected font or the embedded Silver fallback can render them.
13
13
  3. Pages of text are rasterized into PNG frames (`render` / `renderMany`). Frame width is fixed per shape; height hugs the rows actually printed, so a partially filled frame never bills blank pixel rows.
14
14
  4. Frames persist in the compaction entry's `preserveData` and are re-attached to the summary message on every context rebuild.
15
15
 
@@ -17,13 +17,15 @@ Frame shapes are provider-aware, chosen by SQuAD recall evals (see `research/`)
17
17
 
18
18
  | Reader | Default shape | Notes |
19
19
  | --- | --- | --- |
20
- | Anthropic | `6x12-dim` | X.org 6x12 glyphs, stopwords dimmed gray; high-res Claude lines get 1932px frames |
21
- | Google | `doc-8on16-sent-dim` @2048 | Two newspaper columns, sentence-hue ink; Gemini bills a fixed per-image budget, so larger frames are free chars |
22
- | OpenAI | `8on16-bw` | 8x13 glyphs on a patch-aligned 16px pitch, sent at `detail: "original"` |
20
+ | Anthropic | `11on16-bw` | X.org 8x13 glyphs on an 11px advance; high-res Claude lines get 1932px frames |
21
+ | Google | `8on22-bw` @2048 | X.org 8x13 glyphs on a 22px pitch; Gemini bills a fixed per-image budget, so larger frames are free chars |
22
+ | OpenAI | `8on22-bw` | X.org 8x13 glyphs on a 22px pitch, sent at `detail: "original"` |
23
23
  | Unknown | Anthropic shape | Per-provider image-count budgets guard against gateways that silently drop frames |
24
24
 
25
25
  `resolveShape({ api, id })` matches the model id, not just the wire API — a Claude routed through Vertex or OpenRouter keeps its Claude shape, priced for the gateway actually carrying the request.
26
26
 
27
+ Bitmap shapes keep their provider-tuned geometry and draw missing glyphs through the embedded Silver TrueType fallback one character at a time; East Asian (CJK/Kana/Hangul) glyphs render full-width across two cells so they stay legible in the narrow ASCII grid. Selecting `silver16-bw` uses Silver for the whole frame.
28
+
27
29
  ## Install
28
30
 
29
31
  ```sh
@@ -58,8 +60,8 @@ const result = await compact(preparation, { model });
58
60
 
59
61
  - **Compaction**: `compact`, `CompactionPreparation`, `CompactionResult`, `getPreservedArchive`, `images`, `historyBlocks`
60
62
  - **Rendering**: `render`, `renderMany`, `frames`, `geometry`
61
- - **Shapes**: `SHAPES`, `SHAPE_VARIANTS`, `resolveShape`, `idealShapeVariant`, `isShape`, `isShapeVariantName`
62
- - **Text**: `serializeConversation`, `normalize`, `dimStopwords`, `wrap`
63
+ - **Shapes**: `SHAPES`, `SHAPE_VARIANTS`, `resolveShape`, `resolveShapeForText`, `idealShapeVariant`, `isShape`, `isShapeVariantName`
64
+ - **Text**: `serializeConversation`, `normalize`, `scanRenderability`, `renderabilityProbeText`, `dimStopwords`, `wrap`
63
65
  - **Budgets**: `providerImageBudget`, `MAX_FRAMES_DEFAULT`, `FRAME_TOKEN_ESTIMATE`, `HQ_EDGE_FRAMES`
64
66
  - **File ops**: `createFileOps`, `computeFileLists`, `upsertFileOperations`
65
67
 
@@ -43,11 +43,11 @@
43
43
  * Frames persist in the compaction entry's `preserveData` and are
44
44
  * re-attached to the compaction summary message on every context rebuild.
45
45
  */
46
- import type { Api, ImageContent, Message, Model, TextContent } from "@oh-my-pi/pi-ai";
46
+ import type { Api, ImageContent, Message, TextContent } from "@oh-my-pi/pi-ai";
47
47
  /** One eval-validated frame shape: font, cell, ink, repetition, and size. */
48
48
  export interface Shape {
49
49
  /** Bundled font in the native renderer. */
50
- font: "5x8" | "8x8" | "6x12" | "8x13";
50
+ font: "5x8" | "8x8" | "6x12" | "8x13" | "silver";
51
51
  /** Target cell advance in pixels; differing from the font's natural cell
52
52
  * renders via Lanczos stretch (anti-aliased RGB frame). */
53
53
  cellWidth: number;
@@ -84,9 +84,10 @@ export type ShapeGeometry = Omit<Shape, "frameTokenEstimate" | "imageDetail">;
84
84
  * and `8x13` the X.org misc fonts, `8on16` 8x13 glyphs on an 8x16 cell pitch
85
85
  * (no stretch, extra leading), `8on22` the same glyphs on a 22px pitch (more
86
86
  * leading), `11on16` the same glyphs on an 11px advance (more tracking),
87
- * `doc-` prefixed shapes a two-column word-wrapped newspaper layout. Ink:
88
- * `sent` cycles six hues at sentence boundaries, `bw` is plain black, `-dim`
89
- * suffix prints stopwords in gray.
87
+ * `silver16` the embedded Silver TrueType font on a 16px grid for CJK and
88
+ * other non-Latin text, and `doc-` prefixed shapes a two-column word-wrapped
89
+ * newspaper layout. Ink: `sent` cycles six hues at sentence boundaries, `bw`
90
+ * is plain black, `-dim` suffix prints stopwords in gray.
90
91
  */
91
92
  export declare const SHAPE_VARIANTS: {
92
93
  readonly "8x8r-bw": {
@@ -197,6 +198,14 @@ export declare const SHAPE_VARIANTS: {
197
198
  readonly lineRepeat: 1;
198
199
  readonly frameSize: 1568;
199
200
  };
201
+ readonly "silver16-bw": {
202
+ readonly font: "silver";
203
+ readonly cellWidth: 16;
204
+ readonly cellHeight: 16;
205
+ readonly variant: "bw";
206
+ readonly lineRepeat: 1;
207
+ readonly frameSize: 1568;
208
+ };
200
209
  readonly "doc-8on16-bw": {
201
210
  readonly font: "8x13";
202
211
  readonly cellWidth: 8;
@@ -280,6 +289,14 @@ export interface ShapeTarget {
280
289
  * `Model` or any `{ api, id }` subset.
281
290
  */
282
291
  export declare function resolveShape(model?: ShapeTarget, variant?: ShapeVariantName | "auto"): Shape;
292
+ /**
293
+ * Pick the frame shape for `text` without changing the selected shape.
294
+ *
295
+ * Glyph-level Silver fallback happens during normalization/rendering, so this
296
+ * helper exists for callers that need a text-aware API name while preserving
297
+ * explicit and provider-selected shapes.
298
+ */
299
+ export declare function resolveShapeForText(_text: string, model?: ShapeTarget, variant?: ShapeVariantName | "auto"): Shape;
283
300
  /** Legacy frame edge in pixels (the 5x8 shape's eval-validated size). New
284
301
  * shapes carry their own `frameSize`. */
285
302
  export declare const FRAME_SIZE = 2576;
@@ -365,8 +382,8 @@ export interface Geometry {
365
382
  export interface Options<TMessage = Message> extends SerializeOptions {
366
383
  /** App-level message transformer (same contract as agent-core's `SummaryOptions.convertToLlm`). */
367
384
  convertToLlm?: ConvertToLlm<TMessage>;
368
- /** Model whose provider API selects the frame shape. */
369
- model?: Pick<Model, "api">;
385
+ /** Model whose provider API and id select the frame shape. */
386
+ model?: ShapeTarget;
370
387
  /** Explicit shape override; wins over `model`. */
371
388
  shape?: Shape;
372
389
  /** Frame edge in pixels. Defaults to the shape's `frameSize`. */
@@ -454,27 +471,24 @@ export declare function serializeConversation(messages: Message[], options?: Ser
454
471
  * entirely with pitch-black ink, so line structure survives whitespace
455
472
  * collapsing at a one-cell cost. */
456
473
  export declare const NEWLINE_GLYPH = "\u2588";
474
+ export interface NormalizeOptions {
475
+ /** Shape whose font is tried before the embedded Silver fallback. */
476
+ shape?: Pick<Shape, "font">;
477
+ /** Native font name when a full shape is not available. */
478
+ font?: Shape["font"];
479
+ }
457
480
  /**
458
481
  * Prepare text for printing: strip ANSI escape sequences, collapse horizontal
459
- * whitespace runs to single spaces and newline-bearing runs to one
460
- * {@link NEWLINE_GLYPH} (drawn as a pitch-black cell), then fold everything
461
- * outside the fonts' ASCII + Latin-1 coverage to ASCII approximations — first
462
- * through the {@link CHAR_FOLD} punctuation table, then via an NFKD
463
- * decomposition that recovers the ASCII skeleton of compatibility characters
464
- * (fullwidth, super/subscripts, ligatures, circled/math-styled alphanumerics,
465
- * Roman numerals, vulgar fractions). Unrenderable control/format/combining
466
- * characters are dropped without occupying a cell; `?` remains the fallback
467
- * for unsupported graphic characters. The zero-width ink toggles
468
- * {@link DIM_ON}/{@link DIM_OFF} pass through untouched.
482
+ * whitespace runs, fold unsupported symbols (including box drawing to ASCII),
483
+ * preserve Unicode glyphs that either the selected font or embedded Silver
484
+ * fallback can render, and drop decorative emoji instead of printing `?`.
469
485
  */
470
- export declare function normalize(text: string): string;
486
+ export declare function normalize(text: string, options?: NormalizeOptions): string;
471
487
  /**
472
- * Scan text to determine the proportion of graphic characters that will hit the
473
- * `?` fallback during {@link normalize}. Used as a preflight check to abort
474
- * snapcompact and fall back to the text summarizer when the input is heavily
475
- * non-renderable (e.g., CJK).
488
+ * Scan text with the same font-aware path as {@link normalize}; unsafe means
489
+ * more than 5% of graphic characters would hit the `?` fallback.
476
490
  */
477
- export declare function scanRenderability(text: string): {
491
+ export declare function scanRenderability(text: string, options?: NormalizeOptions): {
478
492
  isSafe: boolean;
479
493
  unrenderableRatio: number;
480
494
  };
@@ -492,7 +506,7 @@ export declare function dimStopwords(text: string): string;
492
506
  * markers count toward word length here; serialized history places them at
493
507
  * word boundaries, so the drift is at most one cell per affected line.
494
508
  */
495
- export declare function wrap(text: string, width: number): string[];
509
+ export declare function wrap(text: string, width: number, wideCells?: boolean): string[];
496
510
  export declare function geometry(shape: Shape, size?: number): Geometry;
497
511
  /** Render one snapcompact frame from already-normalized text. Doc shapes
498
512
  * (`columns === 2`) expect one page of `\n`-joined pre-wrapped lines. */
@@ -501,8 +515,8 @@ export declare function render(text: string, shape: Shape, size?: number): Promi
501
515
  export interface RenderManyOptions {
502
516
  /** Explicit shape; wins over `model`. */
503
517
  shape?: Shape;
504
- /** Model whose `api` selects the eval-optimal shape. */
505
- model?: Pick<Model, "api">;
518
+ /** Model whose provider API and id select the frame shape. */
519
+ model?: ShapeTarget;
506
520
  /** Frame edge in px; defaults to the shape's `frameSize`. */
507
521
  frameSize?: number;
508
522
  /** Hard cap on frames produced; omit for unbounded (caller decides usage). */
@@ -527,6 +541,8 @@ export declare function getPreservedArchive(preserveData: Record<string, unknown
527
541
  export declare function stripPreservedArchive(preserveData: Record<string, unknown> | undefined): Record<string, unknown> | undefined;
528
542
  /** Extract persisted archive source text as plain text for LLM summarization. */
529
543
  export declare function archiveSourceText(archive: Archive): string | undefined;
544
+ /** Build the text used to choose and preflight a font-aware snapcompact shape. */
545
+ export declare function renderabilityProbeText(serialized: string, previousPreserveData?: Record<string, unknown>, previousSummary?: string): string;
530
546
  /** Convert archive frames into LLM image blocks (oldest first). */
531
547
  export declare function images(archive: Archive): ImageContent[];
532
548
  /** Ordered archive blocks for a compaction summary message, oldest to newest:
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "type": "module",
3
3
  "name": "@oh-my-pi/snapcompact",
4
- "version": "16.2.6",
4
+ "version": "16.2.7",
5
5
  "description": "Bitmap-frame context compression for vision-capable LLMs",
6
6
  "homepage": "https://omp.sh",
7
7
  "author": "Can Boluk",
@@ -31,10 +31,10 @@
31
31
  "fmt": "biome format --write ."
32
32
  },
33
33
  "dependencies": {
34
- "@oh-my-pi/pi-ai": "16.2.6",
35
- "@oh-my-pi/pi-natives": "16.2.6",
36
- "@oh-my-pi/pi-utils": "16.2.6",
37
- "@oh-my-pi/pi-wire": "16.2.6"
34
+ "@oh-my-pi/pi-ai": "16.2.7",
35
+ "@oh-my-pi/pi-natives": "16.2.7",
36
+ "@oh-my-pi/pi-utils": "16.2.7",
37
+ "@oh-my-pi/pi-wire": "16.2.7"
38
38
  },
39
39
  "devDependencies": {
40
40
  "@types/bun": "^1.3.14"
@@ -44,8 +44,8 @@
44
44
  * re-attached to the compaction summary message on every context rebuild.
45
45
  */
46
46
 
47
- import type { Api, ImageContent, Message, Model, TextContent } from "@oh-my-pi/pi-ai";
48
- import { renderSnapcompactPng } from "@oh-my-pi/pi-natives";
47
+ import type { Api, ImageContent, Message, TextContent } from "@oh-my-pi/pi-ai";
48
+ import { renderSnapcompactPng, snapcompactSupportedChars } from "@oh-my-pi/pi-natives";
49
49
  import { formatGroupedPaths, prompt } from "@oh-my-pi/pi-utils";
50
50
  import { INTENT_FIELD } from "@oh-my-pi/pi-wire";
51
51
  import fileOperationsTemplate from "./prompts/file-operations.md" with { type: "text" };
@@ -58,7 +58,7 @@ import snapcompactSummaryPrompt from "./prompts/snapcompact-summary.md" with { t
58
58
  /** One eval-validated frame shape: font, cell, ink, repetition, and size. */
59
59
  export interface Shape {
60
60
  /** Bundled font in the native renderer. */
61
- font: "5x8" | "8x8" | "6x12" | "8x13";
61
+ font: "5x8" | "8x8" | "6x12" | "8x13" | "silver";
62
62
  /** Target cell advance in pixels; differing from the font's natural cell
63
63
  * renders via Lanczos stretch (anti-aliased RGB frame). */
64
64
  cellWidth: number;
@@ -97,9 +97,10 @@ export type ShapeGeometry = Omit<Shape, "frameTokenEstimate" | "imageDetail">;
97
97
  * and `8x13` the X.org misc fonts, `8on16` 8x13 glyphs on an 8x16 cell pitch
98
98
  * (no stretch, extra leading), `8on22` the same glyphs on a 22px pitch (more
99
99
  * leading), `11on16` the same glyphs on an 11px advance (more tracking),
100
- * `doc-` prefixed shapes a two-column word-wrapped newspaper layout. Ink:
101
- * `sent` cycles six hues at sentence boundaries, `bw` is plain black, `-dim`
102
- * suffix prints stopwords in gray.
100
+ * `silver16` the embedded Silver TrueType font on a 16px grid for CJK and
101
+ * other non-Latin text, and `doc-` prefixed shapes a two-column word-wrapped
102
+ * newspaper layout. Ink: `sent` cycles six hues at sentence boundaries, `bw`
103
+ * is plain black, `-dim` suffix prints stopwords in gray.
103
104
  */
104
105
  export const SHAPE_VARIANTS = {
105
106
  "8x8r-bw": { font: "8x8", cellWidth: 8, cellHeight: 8, variant: "bw", lineRepeat: 2, frameSize: 1568 },
@@ -147,6 +148,14 @@ export const SHAPE_VARIANTS = {
147
148
  lineRepeat: 1,
148
149
  frameSize: 1568,
149
150
  },
151
+ "silver16-bw": {
152
+ font: "silver",
153
+ cellWidth: 16,
154
+ cellHeight: 16,
155
+ variant: "bw",
156
+ lineRepeat: 1,
157
+ frameSize: 1568,
158
+ },
150
159
  "doc-8on16-bw": {
151
160
  font: "8x13",
152
161
  cellWidth: 8,
@@ -271,7 +280,7 @@ export function isShape(value: unknown): value is Shape {
271
280
  const variant = shape.variant;
272
281
  const detail = shape.imageDetail;
273
282
  return (
274
- (font === "5x8" || font === "8x8" || font === "6x12" || font === "8x13") &&
283
+ (font === "5x8" || font === "8x8" || font === "6x12" || font === "8x13" || font === "silver") &&
275
284
  typeof shape.cellWidth === "number" &&
276
285
  shape.cellWidth > 0 &&
277
286
  typeof shape.cellHeight === "number" &&
@@ -376,6 +385,17 @@ export function resolveShape(model?: ShapeTarget, variant?: ShapeVariantName | "
376
385
  return priceShape(ideal?.frameSize ? { ...base, frameSize: ideal.frameSize } : base, family);
377
386
  }
378
387
 
388
+ /**
389
+ * Pick the frame shape for `text` without changing the selected shape.
390
+ *
391
+ * Glyph-level Silver fallback happens during normalization/rendering, so this
392
+ * helper exists for callers that need a text-aware API name while preserving
393
+ * explicit and provider-selected shapes.
394
+ */
395
+ export function resolveShapeForText(_text: string, model?: ShapeTarget, variant?: ShapeVariantName | "auto"): Shape {
396
+ return resolveShape(model, variant);
397
+ }
398
+
379
399
  // ============================================================================
380
400
  // Constants
381
401
  // ============================================================================
@@ -492,8 +512,8 @@ export interface Geometry {
492
512
  export interface Options<TMessage = Message> extends SerializeOptions {
493
513
  /** App-level message transformer (same contract as agent-core's `SummaryOptions.convertToLlm`). */
494
514
  convertToLlm?: ConvertToLlm<TMessage>;
495
- /** Model whose provider API selects the frame shape. */
496
- model?: Pick<Model, "api">;
515
+ /** Model whose provider API and id select the frame shape. */
516
+ model?: ShapeTarget;
497
517
  /** Explicit shape override; wins over `model`. */
498
518
  shape?: Shape;
499
519
  /** Frame edge in pixels. Defaults to the shape's `frameSize`. */
@@ -918,6 +938,48 @@ const UNRENDERABLE = /[\p{Cc}\p{Mn}\p{Me}\p{Cs}]/u;
918
938
  * letter prints without the diacritic the bundled fonts cannot compose. */
919
939
  const COMBINING_MARKS = /\p{M}+/gu;
920
940
 
941
+ /** Status-like pictographs that carry meaning in tool output; all other emoji
942
+ * pictographs drop instead of burning cells as `?`. */
943
+ const EMOJI_FOLD: Record<string, string> = {
944
+ "✅": "[OK]",
945
+ "☑": "[OK]",
946
+ "✔": "[OK]",
947
+ "❌": "[FAIL]",
948
+ "❎": "[FAIL]",
949
+ "✖": "[FAIL]",
950
+ "⚠": "[WARN]",
951
+ "🚨": "[ALERT]",
952
+ ℹ: "[INFO]",
953
+ "🐛": "[BUG]",
954
+ "💥": "[CRASH]",
955
+ "🔥": "[HOT]",
956
+ "🔒": "[LOCK]",
957
+ "🔓": "[UNLOCK]",
958
+ "📁": "[DIR]",
959
+ "📂": "[DIR]",
960
+ "📄": "[FILE]",
961
+ "📝": "[NOTE]",
962
+ "🧪": "[TEST]",
963
+ "⏳": "[WAIT]",
964
+ "⌛": "[WAIT]",
965
+ "🚀": "[RUN]",
966
+ };
967
+
968
+ const EMOJI_PICTOGRAPH = /\p{Extended_Pictographic}/u;
969
+
970
+ export interface NormalizeOptions {
971
+ /** Shape whose font is tried before the embedded Silver fallback. */
972
+ shape?: Pick<Shape, "font">;
973
+ /** Native font name when a full shape is not available. */
974
+ font?: Shape["font"];
975
+ }
976
+
977
+ interface NormalizedText {
978
+ text: string;
979
+ totalGraphics: number;
980
+ fallbackCount: number;
981
+ }
982
+
921
983
  /**
922
984
  * Aggressive single-code-point ASCII fold via Unicode NFKD: decompose the
923
985
  * compatibility form (fullwidth, super/subscripts, ligatures, circled and
@@ -927,13 +989,17 @@ const COMBINING_MARKS = /\p{M}+/gu;
927
989
  * point has no decomposition or still leaves an undrawable glyph, so the
928
990
  * caller falls back to `?`.
929
991
  */
992
+ function isAsciiOrLatin1(cp: number): boolean {
993
+ return (cp >= 0x20 && cp < 0x7f) || (cp >= 0xa0 && cp <= 0xff);
994
+ }
995
+
930
996
  function foldToAscii(ch: string): string | undefined {
931
997
  const decomposed = ch.normalize("NFKD").replace(COMBINING_MARKS, "");
932
998
  if (decomposed === ch) return undefined;
933
999
  let out = "";
934
1000
  for (const part of decomposed) {
935
- const cp = part.codePointAt(0) as number;
936
- if ((cp >= 0x20 && cp < 0x7f) || (cp >= 0xa0 && cp <= 0xff)) {
1001
+ const cp = part.codePointAt(0);
1002
+ if (cp !== undefined && isAsciiOrLatin1(cp)) {
937
1003
  out += part;
938
1004
  continue;
939
1005
  }
@@ -944,90 +1010,126 @@ function foldToAscii(ch: string): string | undefined {
944
1010
  return out;
945
1011
  }
946
1012
 
947
- /**
948
- * Prepare text for printing: strip ANSI escape sequences, collapse horizontal
949
- * whitespace runs to single spaces and newline-bearing runs to one
950
- * {@link NEWLINE_GLYPH} (drawn as a pitch-black cell), then fold everything
951
- * outside the fonts' ASCII + Latin-1 coverage to ASCII approximations — first
952
- * through the {@link CHAR_FOLD} punctuation table, then via an NFKD
953
- * decomposition that recovers the ASCII skeleton of compatibility characters
954
- * (fullwidth, super/subscripts, ligatures, circled/math-styled alphanumerics,
955
- * Roman numerals, vulgar fractions). Unrenderable control/format/combining
956
- * characters are dropped without occupying a cell; `?` remains the fallback
957
- * for unsupported graphic characters. The zero-width ink toggles
958
- * {@link DIM_ON}/{@link DIM_OFF} pass through untouched.
959
- */
960
- export function normalize(text: string): string {
1013
+ function renderableUnicodeChars(chars: readonly string[], font: Shape["font"] | undefined): ReadonlySet<string> {
1014
+ if (chars.length === 0) return new Set();
1015
+ const text = chars.join("");
1016
+ const primaryFont = font ?? "5x8";
1017
+ const supported = new Set(snapcompactSupportedChars(primaryFont, text));
1018
+ if (primaryFont !== "silver") {
1019
+ for (const ch of snapcompactSupportedChars("silver", text)) supported.add(ch);
1020
+ }
1021
+ return supported;
1022
+ }
1023
+
1024
+ function normalizedInputChars(text: string): string[] {
961
1025
  const stripped = text.includes("\u001b") ? Bun.stripANSI(text) : text;
962
1026
  const collapsed = stripped
963
1027
  // A run of pure format chars (BOM is both \s and Cf) vanishes; only a
964
1028
  // run containing genuine whitespace separates words.
965
1029
  .replace(COLLAPSIBLE, run => (LINE_BREAK.test(run) ? NEWLINE_GLYPH : /[^\p{Cf}]/u.test(run) ? " " : ""))
966
1030
  .replace(EDGE_RUNS, "");
967
- let out = "";
968
- for (const ch of collapsed) {
969
- const cp = ch.codePointAt(0) as number;
970
- if ((cp >= 0x20 && cp < 0x7f) || (cp >= 0xa0 && cp <= 0xff)) {
971
- out += ch;
1031
+ return [...collapsed];
1032
+ }
1033
+
1034
+ function candidateUnicodeChars(chars: readonly string[]): string[] {
1035
+ const unique = new Set<string>();
1036
+ for (const ch of chars) {
1037
+ const cp = ch.codePointAt(0);
1038
+ if (cp === undefined || isAsciiOrLatin1(cp) || ch === DIM_ON || ch === DIM_OFF || ch === NEWLINE_GLYPH) {
972
1039
  continue;
973
1040
  }
974
- if (ch === DIM_ON || ch === DIM_OFF || ch === NEWLINE_GLYPH) {
975
- out += ch;
1041
+ if (
1042
+ CHAR_FOLD[ch] !== undefined ||
1043
+ (cp >= 0x2500 && cp <= 0x257f) ||
1044
+ EMOJI_FOLD[ch] !== undefined ||
1045
+ EMOJI_PICTOGRAPH.test(ch) ||
1046
+ foldToAscii(ch) !== undefined ||
1047
+ UNRENDERABLE.test(ch)
1048
+ ) {
976
1049
  continue;
977
1050
  }
978
- const fold = CHAR_FOLD[ch];
979
- if (fold !== undefined) {
980
- out += fold;
981
- } else if (cp >= 0x2500 && cp <= 0x257f) {
982
- // Box drawing: keep table skeletons legible.
983
- out += cp === 0x2502 || cp === 0x2503 ? "|" : cp === 0x2500 || cp === 0x2501 ? "-" : "+";
984
- } else {
985
- const folded = foldToAscii(ch);
986
- if (folded !== undefined) out += folded;
987
- else if (!UNRENDERABLE.test(ch)) out += "?";
988
- }
1051
+ unique.add(ch);
989
1052
  }
990
- return out;
1053
+ return [...unique];
991
1054
  }
992
1055
 
993
- /**
994
- * Scan text to determine the proportion of graphic characters that will hit the
995
- * `?` fallback during {@link normalize}. Used as a preflight check to abort
996
- * snapcompact and fall back to the text summarizer when the input is heavily
997
- * non-renderable (e.g., CJK).
998
- */
999
- export function scanRenderability(text: string): { isSafe: boolean; unrenderableRatio: number } {
1000
- const stripped = text.includes("\u001b") ? Bun.stripANSI(text) : text;
1001
- const collapsed = stripped
1002
- .replace(COLLAPSIBLE, run => (LINE_BREAK.test(run) ? NEWLINE_GLYPH : /[^\p{Cf}]/u.test(run) ? " " : ""))
1003
- .replace(EDGE_RUNS, "");
1056
+ function normalizeWithStats(text: string, options?: NormalizeOptions): NormalizedText {
1057
+ const chars = normalizedInputChars(text);
1058
+ const font = options?.font ?? options?.shape?.font;
1059
+ const supported = renderableUnicodeChars(candidateUnicodeChars(chars), font);
1060
+ const out: string[] = [];
1004
1061
  let totalGraphics = 0;
1005
1062
  let fallbackCount = 0;
1006
- for (const ch of collapsed) {
1007
- const cp = ch.codePointAt(0) as number;
1008
- if ((cp >= 0x20 && cp < 0x7f) || (cp >= 0xa0 && cp <= 0xff)) {
1063
+
1064
+ for (const ch of chars) {
1065
+ const cp = ch.codePointAt(0);
1066
+ if (cp === undefined) continue;
1067
+ if (isAsciiOrLatin1(cp)) {
1068
+ out.push(ch);
1009
1069
  totalGraphics++;
1010
1070
  continue;
1011
1071
  }
1012
1072
  if (ch === DIM_ON || ch === DIM_OFF || ch === NEWLINE_GLYPH) {
1073
+ out.push(ch);
1074
+ continue;
1075
+ }
1076
+ const emoji = EMOJI_FOLD[ch];
1077
+ if (emoji !== undefined) {
1078
+ out.push(emoji);
1079
+ totalGraphics++;
1013
1080
  continue;
1014
1081
  }
1015
1082
  const fold = CHAR_FOLD[ch];
1016
1083
  if (fold !== undefined) {
1084
+ out.push(fold);
1017
1085
  totalGraphics++;
1018
- } else if (cp >= 0x2500 && cp <= 0x257f) {
1086
+ continue;
1087
+ }
1088
+ if (cp >= 0x2500 && cp <= 0x257f) {
1089
+ out.push(cp === 0x2502 || cp === 0x2503 ? "|" : cp === 0x2500 || cp === 0x2501 ? "-" : "+");
1019
1090
  totalGraphics++;
1020
- } else {
1021
- const folded = foldToAscii(ch);
1022
- if (folded !== undefined) {
1023
- totalGraphics++;
1024
- } else if (!UNRENDERABLE.test(ch)) {
1025
- totalGraphics++;
1026
- fallbackCount++;
1027
- }
1091
+ continue;
1092
+ }
1093
+ if (!EMOJI_PICTOGRAPH.test(ch) && supported.has(ch)) {
1094
+ out.push(ch);
1095
+ totalGraphics++;
1096
+ continue;
1097
+ }
1098
+ const folded = foldToAscii(ch);
1099
+ if (folded !== undefined) {
1100
+ out.push(folded);
1101
+ totalGraphics++;
1102
+ } else if (EMOJI_PICTOGRAPH.test(ch)) {
1103
+ } else if (!UNRENDERABLE.test(ch)) {
1104
+ out.push("?");
1105
+ totalGraphics++;
1106
+ fallbackCount++;
1028
1107
  }
1029
1108
  }
1030
- const unrenderableRatio = totalGraphics > 0 ? fallbackCount / totalGraphics : 0;
1109
+
1110
+ return { text: out.join("").replace(/ +/g, " ").replace(EDGE_RUNS, ""), totalGraphics, fallbackCount };
1111
+ }
1112
+
1113
+ /**
1114
+ * Prepare text for printing: strip ANSI escape sequences, collapse horizontal
1115
+ * whitespace runs, fold unsupported symbols (including box drawing to ASCII),
1116
+ * preserve Unicode glyphs that either the selected font or embedded Silver
1117
+ * fallback can render, and drop decorative emoji instead of printing `?`.
1118
+ */
1119
+ export function normalize(text: string, options?: NormalizeOptions): string {
1120
+ return normalizeWithStats(text, options).text;
1121
+ }
1122
+
1123
+ /**
1124
+ * Scan text with the same font-aware path as {@link normalize}; unsafe means
1125
+ * more than 5% of graphic characters would hit the `?` fallback.
1126
+ */
1127
+ export function scanRenderability(
1128
+ text: string,
1129
+ options?: NormalizeOptions,
1130
+ ): { isSafe: boolean; unrenderableRatio: number } {
1131
+ const normalized = normalizeWithStats(text, options);
1132
+ const unrenderableRatio = normalized.totalGraphics > 0 ? normalized.fallbackCount / normalized.totalGraphics : 0;
1031
1133
  return { isSafe: unrenderableRatio <= 0.05, unrenderableRatio };
1032
1134
  }
1033
1135
 
@@ -1086,34 +1188,131 @@ export function dimStopwords(text: string): string {
1086
1188
  /** Char cells between the two doc columns (research exp14 `GUTTER`). */
1087
1189
  const DOC_GUTTER = 3;
1088
1190
 
1191
+ /** East Asian Wide / Fullwidth code points that occupy two grid cells when a
1192
+ * narrow bitmap shape draws them through the Silver fallback. Mirrors
1193
+ * `is_wide` in `crates/pi-natives/src/snapcompact.rs`; the two MUST stay in
1194
+ * sync or native layout and this capacity math disagree on cell counts. */
1195
+ function isWideCodePoint(cp: number): boolean {
1196
+ return (
1197
+ (cp >= 0x1100 && cp <= 0x115f) ||
1198
+ (cp >= 0x2e80 && cp <= 0x2eff) ||
1199
+ (cp >= 0x2f00 && cp <= 0x2fdf) ||
1200
+ (cp >= 0x3000 && cp <= 0x303e) ||
1201
+ (cp >= 0x3041 && cp <= 0x33ff) ||
1202
+ (cp >= 0x3400 && cp <= 0x4dbf) ||
1203
+ (cp >= 0x4e00 && cp <= 0x9fff) ||
1204
+ (cp >= 0xa000 && cp <= 0xa4cf) ||
1205
+ (cp >= 0xac00 && cp <= 0xd7a3) ||
1206
+ (cp >= 0xf900 && cp <= 0xfaff) ||
1207
+ (cp >= 0xfe30 && cp <= 0xfe4f) ||
1208
+ (cp >= 0xff00 && cp <= 0xff60) ||
1209
+ (cp >= 0xffe0 && cp <= 0xffe6) ||
1210
+ (cp >= 0x20000 && cp <= 0x2fffd) ||
1211
+ (cp >= 0x30000 && cp <= 0x3fffd)
1212
+ );
1213
+ }
1214
+
1215
+ /** Cells one character occupies: 0 for the zero-width dim toggles, 2 for wide
1216
+ * code points in narrow bitmap shapes, 1 otherwise. Mirrors native
1217
+ * `cell_units`. */
1218
+ function charCells(ch: string, wideCells: boolean): number {
1219
+ if (ch === DIM_ON || ch === DIM_OFF) return 0;
1220
+ const cp = ch.codePointAt(0);
1221
+ return wideCells && cp !== undefined && isWideCodePoint(cp) ? 2 : 1;
1222
+ }
1223
+
1224
+ /** Wide code points span two cells in every shape except the square-celled
1225
+ * Silver shape, which sizes each cell for a full-width glyph already. */
1226
+ function usesWideCells(shape: Pick<Shape, "font">): boolean {
1227
+ return shape.font !== "silver";
1228
+ }
1229
+
1230
+ /** Total grid cells a string occupies (ignoring row wrapping/pads). */
1231
+ function cellLength(text: string, wideCells: boolean): number {
1232
+ let cells = 0;
1233
+ for (const ch of text) cells += charCells(ch, wideCells);
1234
+ return cells;
1235
+ }
1236
+
1237
+ /** Longest prefix of `text` that fits `width` cells (at least one char). */
1238
+ function sliceCells(text: string, width: number, wideCells: boolean): string {
1239
+ let cells = 0;
1240
+ let out = "";
1241
+ let placed = false;
1242
+ for (const ch of text) {
1243
+ const w = charCells(ch, wideCells);
1244
+ if (placed && cells + w > width) break;
1245
+ out += ch;
1246
+ cells += w;
1247
+ if (w > 0) placed = true;
1248
+ }
1249
+ return out;
1250
+ }
1251
+
1252
+ /** Split `text` into pages that each fill at most `capacity` grid cells,
1253
+ * inserting a one-cell pad before a wide glyph that would straddle the right
1254
+ * edge (mirrors native `place_cell`). Pages are contiguous substrings, so each
1255
+ * renders independently starting at cell 0. A single char wider than the whole
1256
+ * budget still rides its page; the native renderer clips it. */
1257
+ function paginateCells(text: string, capacity: number, cols: number, wideCells: boolean): string[] {
1258
+ const chars = [...text];
1259
+ const pages: string[] = [];
1260
+ let start = 0;
1261
+ let cell = 0;
1262
+ let hasCell = false;
1263
+ for (let i = 0; i < chars.length; i++) {
1264
+ const w = charCells(chars[i] ?? "", wideCells);
1265
+ if (w === 0) continue;
1266
+ let at = cell;
1267
+ if (w === 2 && cols >= 2 && at % cols === cols - 1) at += 1;
1268
+ if (hasCell && at + w > capacity) {
1269
+ pages.push(chars.slice(start, i).join(""));
1270
+ start = i;
1271
+ at = 0;
1272
+ }
1273
+ cell = at + w;
1274
+ hasCell = true;
1275
+ }
1276
+ if (hasCell) pages.push(chars.slice(start).join(""));
1277
+ return pages;
1278
+ }
1279
+
1089
1280
  /**
1090
1281
  * Greedy word-wrap, no mid-word breaks (hard split only for width+ words) —
1091
1282
  * ported verbatim from `research/exp14_bestgpt.py` `wrap()`. Zero-width dim
1092
1283
  * markers count toward word length here; serialized history places them at
1093
1284
  * word boundaries, so the drift is at most one cell per affected line.
1094
1285
  */
1095
- export function wrap(text: string, width: number): string[] {
1286
+ export function wrap(text: string, width: number, wideCells = false): string[] {
1096
1287
  const lines: string[] = [];
1097
1288
  let cur = "";
1289
+ let curCells = 0;
1098
1290
  for (const token of text.split(/\s+/)) {
1099
1291
  if (token.length === 0) continue;
1100
1292
  let word = token;
1101
- while (word.length > width) {
1293
+ let wordCells = cellLength(word, wideCells);
1294
+ while (wordCells > width) {
1102
1295
  // Pathological; never hit on prose.
1103
1296
  if (cur) {
1104
1297
  lines.push(cur);
1105
1298
  cur = "";
1299
+ curCells = 0;
1106
1300
  }
1107
- lines.push(word.slice(0, width));
1108
- word = word.slice(width);
1301
+ const head = sliceCells(word, width, wideCells);
1302
+ lines.push(head);
1303
+ word = word.slice(head.length);
1304
+ wordCells = cellLength(word, wideCells);
1109
1305
  }
1110
1306
  if (!cur) {
1111
1307
  cur = word;
1112
- } else if (cur.length + 1 + word.length <= width) {
1308
+ curCells = wordCells;
1309
+ } else if (curCells + 1 + wordCells <= width) {
1113
1310
  cur += ` ${word}`;
1311
+ curCells += 1 + wordCells;
1114
1312
  } else {
1115
1313
  lines.push(cur);
1116
1314
  cur = word;
1315
+ curCells = wordCells;
1117
1316
  }
1118
1317
  }
1119
1318
  if (cur) lines.push(cur);
@@ -1126,8 +1325,8 @@ export function wrap(text: string, width: number): string[] {
1126
1325
  * Every input character lands on exactly one page (whitespace becomes the
1127
1326
  * wrap points).
1128
1327
  */
1129
- function docPages(normalized: string, geo: Geometry): string[] {
1130
- const lines = wrap(normalized, geo.cols);
1328
+ function docPages(normalized: string, geo: Geometry, wideCells: boolean): string[] {
1329
+ const lines = wrap(normalized, geo.cols, wideCells);
1131
1330
  const perPage = 2 * geo.rows;
1132
1331
  const pages: string[] = [];
1133
1332
  for (let offset = 0; offset < lines.length; offset += perPage) {
@@ -1165,18 +1364,35 @@ function nativeRenderOptions(shape: Shape, size: number) {
1165
1364
  };
1166
1365
  }
1167
1366
 
1168
- function renderedChars(text: string, shape: Shape, capacity: number): number {
1169
- let visible = text.length - (text.match(DIM_MARKERS)?.length ?? 0);
1170
- // Doc line separators consume no cell; in the grid they print as a blank.
1171
- if (shape.columns === 2) visible -= text.match(NEWLINES)?.length ?? 0;
1172
- return Math.min(visible, capacity);
1367
+ function renderedChars(text: string, shape: Shape, geo: Geometry): number {
1368
+ if (shape.columns === 2) {
1369
+ let visible = [...text].length - (text.match(DIM_MARKERS)?.length ?? 0);
1370
+ visible -= text.match(NEWLINES)?.length ?? 0;
1371
+ return Math.min(visible, geo.capacity);
1372
+ }
1373
+ // Grid: count visible chars that fit within the frame's cell budget, with
1374
+ // wide glyphs taking two cells (and a straddle pad) exactly as the renderer.
1375
+ const wideCells = usesWideCells(shape);
1376
+ let cell = 0;
1377
+ let count = 0;
1378
+ for (const ch of text) {
1379
+ const w = charCells(ch, wideCells);
1380
+ if (w === 0) continue;
1381
+ let at = cell;
1382
+ if (w === 2 && geo.cols >= 2 && at % geo.cols === geo.cols - 1) at += 1;
1383
+ if (at + w > geo.capacity) break;
1384
+ cell = at + w;
1385
+ count++;
1386
+ }
1387
+ return count;
1173
1388
  }
1174
1389
 
1175
1390
  /** Render one snapcompact frame from already-normalized text. Doc shapes
1176
1391
  * (`columns === 2`) expect one page of `\n`-joined pre-wrapped lines. */
1177
1392
  export async function render(text: string, shape: Shape, size: number = shape.frameSize): Promise<RenderedFrame> {
1178
- const { cols, rows, capacity } = geometry(shape, size);
1179
- const chars = renderedChars(text, shape, capacity);
1393
+ const geo = geometry(shape, size);
1394
+ const { cols, rows } = geo;
1395
+ const chars = renderedChars(text, shape, geo);
1180
1396
  const data = await renderSnapcompactPng(text, nativeRenderOptions(shape, size));
1181
1397
  return { data, cols, rows, chars };
1182
1398
  }
@@ -1197,8 +1413,8 @@ function pageFinisher(shape: Shape): (page: string) => string {
1197
1413
  export interface RenderManyOptions {
1198
1414
  /** Explicit shape; wins over `model`. */
1199
1415
  shape?: Shape;
1200
- /** Model whose `api` selects the eval-optimal shape. */
1201
- model?: Pick<Model, "api">;
1416
+ /** Model whose provider API and id select the frame shape. */
1417
+ model?: ShapeTarget;
1202
1418
  /** Frame edge in px; defaults to the shape's `frameSize`. */
1203
1419
  frameSize?: number;
1204
1420
  /** Hard cap on frames produced; omit for unbounded (caller decides usage). */
@@ -1210,27 +1426,26 @@ export interface RenderManyOptions {
1210
1426
  * (first page first). Empty/whitespace-only input yields no frames.
1211
1427
  */
1212
1428
  export async function renderMany(text: string, options?: RenderManyOptions): Promise<ImageContent[]> {
1213
- const shape = options?.shape ?? resolveShape(options?.model);
1429
+ const shape = options?.shape ?? resolveShapeForText(text, options?.model);
1214
1430
  const frameSize = options?.frameSize ?? shape.frameSize;
1215
1431
  const geo = geometry(shape, frameSize);
1216
- const normalized = normalize(text);
1432
+ const normalized = normalize(text, { shape });
1217
1433
  const cap = options?.maxFrames;
1218
1434
  // Build the per-frame texts in order first (cheap, synchronous), then fan
1219
1435
  // the native PNG renders out concurrently — render() is async/off-thread,
1220
1436
  // so awaiting each before starting the next leaves throughput on the table.
1221
1437
  const pageTexts: string[] = [];
1438
+ const wideCells = usesWideCells(shape);
1222
1439
  if (shape.columns === 2) {
1223
1440
  const finish = pageFinisher(shape);
1224
- for (const page of docPages(normalized, geo)) {
1441
+ for (const page of docPages(normalized, geo, wideCells)) {
1225
1442
  if (cap !== undefined && pageTexts.length >= cap) break;
1226
1443
  pageTexts.push(finish(page));
1227
1444
  }
1228
1445
  } else {
1229
- for (let offset = 0; offset < normalized.length; offset += geo.capacity) {
1446
+ for (const page of paginateCells(normalized, geo.capacity, geo.cols, wideCells)) {
1230
1447
  if (cap !== undefined && pageTexts.length >= cap) break;
1231
- let chunk = normalized.slice(offset, offset + geo.capacity);
1232
- if (shape.stopwordDim) chunk = dimStopwords(chunk);
1233
- pageTexts.push(chunk);
1448
+ pageTexts.push(shape.stopwordDim ? dimStopwords(page) : page);
1234
1449
  }
1235
1450
  }
1236
1451
  const rendered = await Promise.all(pageTexts.map(page => render(page, shape, frameSize)));
@@ -1246,11 +1461,12 @@ export async function renderMany(text: string, options?: RenderManyOptions): Pro
1246
1461
  * For doc shapes this wraps the text once and counts pages of `2 * rows`
1247
1462
  * lines; for grid shapes it divides by the frame capacity. */
1248
1463
  export function frames(text: string, options?: Pick<RenderManyOptions, "shape" | "model" | "frameSize">): number {
1249
- const shape = options?.shape ?? resolveShape(options?.model);
1464
+ const shape = options?.shape ?? resolveShapeForText(text, options?.model);
1250
1465
  const geo = geometry(shape, options?.frameSize ?? shape.frameSize);
1251
- const normalized = normalize(text);
1252
- if (shape.columns === 2) return Math.ceil(wrap(normalized, geo.cols).length / (2 * geo.rows));
1253
- return Math.ceil(normalized.length / geo.capacity);
1466
+ const normalized = normalize(text, { shape });
1467
+ const wideCells = usesWideCells(shape);
1468
+ if (shape.columns === 2) return Math.ceil(wrap(normalized, geo.cols, wideCells).length / (2 * geo.rows));
1469
+ return paginateCells(normalized, geo.capacity, geo.cols, wideCells).length;
1254
1470
  }
1255
1471
 
1256
1472
  // ============================================================================
@@ -1313,6 +1529,19 @@ export function archiveSourceText(archive: Archive): string | undefined {
1313
1529
  return text.length > 0 ? toPlainText(text) : undefined;
1314
1530
  }
1315
1531
 
1532
+ /** Build the text used to choose and preflight a font-aware snapcompact shape. */
1533
+ export function renderabilityProbeText(
1534
+ serialized: string,
1535
+ previousPreserveData?: Record<string, unknown>,
1536
+ previousSummary?: string,
1537
+ ): string {
1538
+ const previousArchive = getPreservedArchive(previousPreserveData);
1539
+ const previousText = previousArchive ? (archiveSourceText(previousArchive) ?? "") : "";
1540
+ if (previousText.length > 0) return `${previousText}${NEWLINE_GLYPH}${serialized}`;
1541
+ if (previousSummary) return `${previousSummary}${NEWLINE_GLYPH}${serialized}`;
1542
+ return serialized;
1543
+ }
1544
+
1316
1545
  /** Convert archive frames into LLM image blocks (oldest first). */
1317
1546
  export function images(archive: Archive): ImageContent[] {
1318
1547
  return archive.frames.map(frame => ({
@@ -1356,9 +1585,10 @@ export function historyBlocks(archive: Archive): (TextContent | ImageContent)[]
1356
1585
 
1357
1586
  /** Denser companion of `high` for the foveated archive middle: same family and
1358
1587
  * frame size (identical per-frame bill) but a tighter cell. Returns `high`
1359
- * unchanged for doc layouts or when no denser variant exists (foveation off). */
1588
+ * unchanged for doc layouts, TrueType Unicode shapes, or when no denser
1589
+ * variant exists (foveation off). */
1360
1590
  function denseCompanion(high: Shape, api: Api | undefined): Shape {
1361
- if (high.columns === 2) return high;
1591
+ if (high.columns === 2 || high.font === "silver") return high;
1362
1592
  const family = billingFamily(api);
1363
1593
  const low = priceShape({ ...SHAPE_VARIANTS[FAMILY_VARIANT_LOW[family]], frameSize: high.frameSize }, family);
1364
1594
  return geometry(low).capacity > geometry(high).capacity ? low : high;
@@ -1381,13 +1611,9 @@ interface ArchiveLayout {
1381
1611
  truncatedChars: number;
1382
1612
  }
1383
1613
 
1384
- /** Slice `text` into `capacity`-char frames at one shape (tier). */
1385
- function sliceFrames(text: string, capacity: number, shape: Shape): PlanFrame[] {
1386
- const out: PlanFrame[] = [];
1387
- for (let offset = 0; offset < text.length; offset += capacity) {
1388
- out.push({ text: text.slice(offset, offset + capacity), shape });
1389
- }
1390
- return out;
1614
+ /** Wrap each page string as a planned frame at one shape (tier). */
1615
+ function planFrames(pages: readonly string[], shape: Shape): PlanFrame[] {
1616
+ return pages.map(text => ({ text, shape }));
1391
1617
  }
1392
1618
 
1393
1619
  /**
@@ -1425,7 +1651,7 @@ function planArchive(text: string, high: Shape, low: Shape, maxFrames: number):
1425
1651
  // Doc layouts wrap (no char-slicing) and don't foveate: one tier, keep the
1426
1652
  // newest pages with the session head pinned, drop the oldest middle.
1427
1653
  if (high.columns === 2) {
1428
- const pages = docPages(imageText, geometry(high));
1654
+ const pages = docPages(imageText, geometry(high), usesWideCells(high));
1429
1655
  let kept = pages;
1430
1656
  let truncatedChars = 0;
1431
1657
  if (pages.length > maxFrames) {
@@ -1435,7 +1661,7 @@ function planArchive(text: string, high: Shape, low: Shape, maxFrames: number):
1435
1661
  }
1436
1662
  const flat = kept.map(page => page.replaceAll("\n", " ")).join(" ");
1437
1663
  return {
1438
- frames: kept.map(page => ({ text: page, shape: high })),
1664
+ frames: planFrames(kept, high),
1439
1665
  textHead,
1440
1666
  textTail,
1441
1667
  keptText: textHead + flat + textTail,
@@ -1443,10 +1669,12 @@ function planArchive(text: string, high: Shape, low: Shape, maxFrames: number):
1443
1669
  };
1444
1670
  }
1445
1671
 
1446
- // Grid: render all-HQ when the image region fits the budget outright.
1447
- if (Math.ceil(imageText.length / capHi) <= maxFrames) {
1672
+ // Grid: paginate the imaged region into HQ frames (cell-aware, so wide CJK
1673
+ // glyphs spanning two cells never overflow a frame's capacity).
1674
+ const hiPages = paginateCells(imageText, capHi, geometry(high).cols, usesWideCells(high));
1675
+ if (hiPages.length <= maxFrames) {
1448
1676
  return {
1449
- frames: sliceFrames(imageText, capHi, high),
1677
+ frames: planFrames(hiPages, high),
1450
1678
  textHead,
1451
1679
  textTail,
1452
1680
  keptText: textHead + imageText + textTail,
@@ -1457,22 +1685,23 @@ function planArchive(text: string, high: Shape, low: Shape, maxFrames: number):
1457
1685
  // Foveate the imaged middle: HQ edges, dense center, drop the oldest dense slice.
1458
1686
  const capLo = geometry(low).capacity;
1459
1687
  const imageEdgeFrames = Math.min(HQ_EDGE_FRAMES, Math.floor((maxFrames - 1) / 2));
1460
- const imageEdgeCap = imageEdgeFrames * capHi;
1461
- const imageHead = imageText.slice(0, imageEdgeCap);
1462
- const imageTail = imageEdgeCap > 0 ? imageText.slice(imageText.length - imageEdgeCap) : "";
1463
- let middleText = imageText.slice(imageEdgeCap, imageText.length - imageEdgeCap);
1688
+ const headPages = hiPages.slice(0, imageEdgeFrames);
1689
+ const tailPages = imageEdgeFrames > 0 ? hiPages.slice(hiPages.length - imageEdgeFrames) : [];
1690
+ const imageHead = headPages.join("");
1691
+ const imageTail = tailPages.join("");
1692
+ const middleSource = imageText.slice(imageHead.length, imageText.length - imageTail.length);
1693
+ let middlePages = paginateCells(middleSource, capLo, geometry(low).cols, usesWideCells(low));
1694
+ const middleBudget = maxFrames - 2 * imageEdgeFrames;
1464
1695
  let truncatedChars = 0;
1465
- const middleCap = (maxFrames - 2 * imageEdgeFrames) * capLo;
1466
- if (middleText.length > middleCap) {
1467
- truncatedChars = middleText.length - middleCap;
1468
- middleText = middleText.slice(truncatedChars);
1696
+ let middleText = middleSource;
1697
+ if (middlePages.length > middleBudget) {
1698
+ const dropped = middlePages.slice(0, middlePages.length - middleBudget).join("");
1699
+ truncatedChars = dropped.length;
1700
+ middleText = middleSource.slice(dropped.length);
1701
+ middlePages = middlePages.slice(middlePages.length - middleBudget);
1469
1702
  }
1470
1703
  return {
1471
- frames: [
1472
- ...sliceFrames(imageHead, capHi, high),
1473
- ...sliceFrames(middleText, capLo, low),
1474
- ...sliceFrames(imageTail, capHi, high),
1475
- ],
1704
+ frames: [...planFrames(headPages, high), ...planFrames(middlePages, low), ...planFrames(tailPages, high)],
1476
1705
  textHead,
1477
1706
  textTail,
1478
1707
  keptText: textHead + imageHead + middleText + imageTail + textTail,
@@ -1501,19 +1730,9 @@ export async function compact<TMessage = Message>(
1501
1730
  if (!firstKeptEntryId) {
1502
1731
  throw new Error("First kept entry has no ID - session may need migration");
1503
1732
  }
1504
- const baseShape = options?.shape ?? resolveShape(options?.model);
1505
- const frameSize = options?.frameSize ?? baseShape.frameSize;
1506
- const high = frameSize === baseShape.frameSize ? baseShape : { ...baseShape, frameSize };
1507
- const low = denseCompanion(high, options?.model?.api);
1508
- const geo = geometry(high);
1509
- // The engine default caps archive growth; a caller-supplied maxFrames only
1510
- // lowers it further (an upper limit), never raising it past the default.
1511
- const maxFrames = Math.max(1, Math.min(options?.maxFrames ?? MAX_FRAMES_DEFAULT, MAX_FRAMES_DEFAULT));
1512
-
1513
1733
  const messages = preparation.messagesToSummarize.concat(preparation.turnPrefixMessages);
1514
1734
  const llmMessages = (options?.convertToLlm ?? defaultConvertToLlm)(messages);
1515
- let archiveText = normalize(serializeConversation(llmMessages, options));
1516
-
1735
+ const serialized = serializeConversation(llmMessages, options);
1517
1736
  const previousArchive = getPreservedArchive(previousPreserveData);
1518
1737
  const previousText =
1519
1738
  previousArchive?.text ??
@@ -1522,8 +1741,20 @@ export async function compact<TMessage = Message>(
1522
1741
  .join(NEWLINE_GLYPH);
1523
1742
  const hasPreviousText = previousText.length > 0;
1524
1743
  const includedPreviousSummary = !hasPreviousText && !!previousSummary;
1744
+ const shapeProbeText = renderabilityProbeText(serialized, previousPreserveData, previousSummary);
1745
+ const baseShape = options?.shape ?? resolveShapeForText(shapeProbeText, options?.model);
1746
+ const frameSize = options?.frameSize ?? baseShape.frameSize;
1747
+ const high = frameSize === baseShape.frameSize ? baseShape : { ...baseShape, frameSize };
1748
+ const low = denseCompanion(high, options?.model?.api);
1749
+ const geo = geometry(high);
1750
+ // The engine default caps archive growth; a caller-supplied maxFrames only
1751
+ // lowers it further (an upper limit), never raising it past the default.
1752
+ const maxFrames = Math.max(1, Math.min(options?.maxFrames ?? MAX_FRAMES_DEFAULT, MAX_FRAMES_DEFAULT));
1753
+
1754
+ let archiveText = normalize(serialized, { shape: high });
1755
+
1525
1756
  if (includedPreviousSummary && previousSummary) {
1526
- const head = `[Summary of earlier history] ${normalize(previousSummary)}`;
1757
+ const head = `[Summary of earlier history] ${normalize(previousSummary, { shape: high })}`;
1527
1758
  archiveText = archiveText.length > 0 ? `${head} [Recent conversation] ${archiveText}` : head;
1528
1759
  }
1529
1760