@visual-z/xeno 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +498 -0
  2. package/docs/install.md +130 -0
  3. package/package.json +43 -0
  4. package/src/apps/approve.ts +49 -0
  5. package/src/apps/cli.ts +37 -0
  6. package/src/apps/commands.ts +226 -0
  7. package/src/apps/completions.ts +28 -0
  8. package/src/apps/input.ts +297 -0
  9. package/src/apps/render.ts +207 -0
  10. package/src/apps/shell.ts +2353 -0
  11. package/src/apps/startup.ts +1302 -0
  12. package/src/core/benchmark.ts +195 -0
  13. package/src/core/boundary.ts +174 -0
  14. package/src/core/churn.ts +59 -0
  15. package/src/core/completion-review.ts +66 -0
  16. package/src/core/context-policy.ts +60 -0
  17. package/src/core/context-window.ts +54 -0
  18. package/src/core/continuation.ts +114 -0
  19. package/src/core/cycle.ts +17 -0
  20. package/src/core/data-query.ts +201 -0
  21. package/src/core/deadline.ts +39 -0
  22. package/src/core/derive.ts +489 -0
  23. package/src/core/edit-plan.ts +200 -0
  24. package/src/core/environment.ts +29 -0
  25. package/src/core/event-validation.ts +99 -0
  26. package/src/core/events.ts +486 -0
  27. package/src/core/evidence-links.ts +71 -0
  28. package/src/core/evidence.ts +89 -0
  29. package/src/core/execution.ts +12 -0
  30. package/src/core/exhaustive.ts +18 -0
  31. package/src/core/experiment.ts +40 -0
  32. package/src/core/export.ts +169 -0
  33. package/src/core/file-references.ts +35 -0
  34. package/src/core/frontmatter.ts +213 -0
  35. package/src/core/handoff.ts +397 -0
  36. package/src/core/harness-comparison.ts +90 -0
  37. package/src/core/history-query.ts +103 -0
  38. package/src/core/history.ts +100 -0
  39. package/src/core/json-event.ts +135 -0
  40. package/src/core/json-pointer.ts +46 -0
  41. package/src/core/keymap.ts +241 -0
  42. package/src/core/observation-view.ts +87 -0
  43. package/src/core/outcome.ts +45 -0
  44. package/src/core/overflow.ts +59 -0
  45. package/src/core/permissions.ts +182 -0
  46. package/src/core/policy-evaluation.ts +55 -0
  47. package/src/core/progress.ts +101 -0
  48. package/src/core/project.ts +372 -0
  49. package/src/core/prompts.ts +83 -0
  50. package/src/core/queue.ts +60 -0
  51. package/src/core/read-bounds.ts +15 -0
  52. package/src/core/recall-analysis.ts +90 -0
  53. package/src/core/recall-trial.ts +153 -0
  54. package/src/core/response-recovery.ts +46 -0
  55. package/src/core/scheduling.ts +8 -0
  56. package/src/core/settings.ts +174 -0
  57. package/src/core/skills.ts +121 -0
  58. package/src/core/stall.ts +48 -0
  59. package/src/core/text-normalize.ts +81 -0
  60. package/src/core/theme.ts +179 -0
  61. package/src/core/tokens.ts +149 -0
  62. package/src/core/tool-contract.ts +102 -0
  63. package/src/core/tool-input.ts +38 -0
  64. package/src/core/transcript.ts +60 -0
  65. package/src/core/window-trial.ts +65 -0
  66. package/src/core/work-state.ts +103 -0
  67. package/src/core/yaml.ts +282 -0
  68. package/src/index.ts +10 -0
  69. package/src/runtime/auth.ts +255 -0
  70. package/src/runtime/benchmark.ts +187 -0
  71. package/src/runtime/catalog.ts +365 -0
  72. package/src/runtime/evidence-notes.ts +33 -0
  73. package/src/runtime/exec-sessions.ts +214 -0
  74. package/src/runtime/execution-control.ts +39 -0
  75. package/src/runtime/harness-comparison.ts +356 -0
  76. package/src/runtime/history-tools.ts +108 -0
  77. package/src/runtime/loop.ts +1983 -0
  78. package/src/runtime/model.ts +614 -0
  79. package/src/runtime/models-fetch.ts +155 -0
  80. package/src/runtime/oauth-device.ts +86 -0
  81. package/src/runtime/oauth.ts +759 -0
  82. package/src/runtime/openai-responses.ts +286 -0
  83. package/src/runtime/openai.ts +675 -0
  84. package/src/runtime/provider.ts +498 -0
  85. package/src/runtime/recall-trial.ts +226 -0
  86. package/src/runtime/resume.ts +66 -0
  87. package/src/runtime/retry.ts +297 -0
  88. package/src/runtime/tools.ts +1193 -0
  89. package/src/runtime/usage.ts +225 -0
  90. package/src/runtime/window-trial.ts +140 -0
  91. package/src/runtime/working-notes.ts +30 -0
  92. package/src/store/atomic.ts +31 -0
  93. package/src/store/auth-store.ts +50 -0
  94. package/src/store/blobs.ts +142 -0
  95. package/src/store/command.ts +75 -0
  96. package/src/store/config.ts +69 -0
  97. package/src/store/context-files.ts +356 -0
  98. package/src/store/file-index.ts +81 -0
  99. package/src/store/git-branch.ts +35 -0
  100. package/src/store/history-search.ts +19 -0
  101. package/src/store/log.ts +642 -0
  102. package/src/store/models.ts +41 -0
  103. package/src/store/observations.ts +181 -0
  104. package/src/store/query-source.ts +44 -0
  105. package/src/store/recovery.ts +39 -0
  106. package/src/store/replay.ts +63 -0
  107. package/src/store/resources.ts +361 -0
  108. package/src/store/restore.ts +302 -0
  109. package/src/store/search.ts +644 -0
  110. package/src/store/serial.ts +13 -0
  111. package/src/store/sessions.ts +463 -0
  112. package/src/store/trust.ts +69 -0
  113. package/src/store/workspace.ts +256 -0
  114. package/src/store/worktree.ts +233 -0
  115. package/src/tui/ansi.ts +67 -0
  116. package/src/tui/autocomplete.ts +29 -0
  117. package/src/tui/complete.ts +87 -0
  118. package/src/tui/diff.ts +160 -0
  119. package/src/tui/editor.ts +893 -0
  120. package/src/tui/exhaustive.ts +14 -0
  121. package/src/tui/fuzzy.ts +75 -0
  122. package/src/tui/host.ts +223 -0
  123. package/src/tui/input.ts +69 -0
  124. package/src/tui/keys.ts +378 -0
  125. package/src/tui/markdown.ts +17 -0
  126. package/src/tui/render.ts +153 -0
  127. package/src/tui/select.ts +240 -0
  128. package/src/tui/settings.ts +40 -0
  129. package/src/tui/statusline.ts +162 -0
  130. package/src/tui/style.ts +205 -0
  131. package/src/tui/terminal.ts +30 -0
  132. package/src/tui/text.ts +52 -0
  133. package/src/tui/toolview.ts +386 -0
  134. package/src/tui/transcript.ts +71 -0
  135. package/src/tui/tree.ts +405 -0
  136. package/src/tui/width.ts +567 -0
  137. package/src/vendor/pi-tui/LICENSE.upstream +21 -0
  138. package/src/vendor/pi-tui/VENDOR.md +23 -0
  139. package/src/vendor/pi-tui/alt-screen-search.ts +327 -0
  140. package/src/vendor/pi-tui/autocomplete.ts +835 -0
  141. package/src/vendor/pi-tui/components/alt-screen-flash.ts +51 -0
  142. package/src/vendor/pi-tui/components/box.ts +167 -0
  143. package/src/vendor/pi-tui/components/cancellable-loader.ts +40 -0
  144. package/src/vendor/pi-tui/components/editor.ts +2470 -0
  145. package/src/vendor/pi-tui/components/h-stack.ts +44 -0
  146. package/src/vendor/pi-tui/components/image.ts +127 -0
  147. package/src/vendor/pi-tui/components/input.ts +494 -0
  148. package/src/vendor/pi-tui/components/loader.ts +101 -0
  149. package/src/vendor/pi-tui/components/markdown.ts +1015 -0
  150. package/src/vendor/pi-tui/components/mouse-region.ts +33 -0
  151. package/src/vendor/pi-tui/components/scroll-view.ts +224 -0
  152. package/src/vendor/pi-tui/components/select-list.ts +273 -0
  153. package/src/vendor/pi-tui/components/settings-list.ts +328 -0
  154. package/src/vendor/pi-tui/components/spacer.ts +28 -0
  155. package/src/vendor/pi-tui/components/stack.ts +154 -0
  156. package/src/vendor/pi-tui/components/text.ts +107 -0
  157. package/src/vendor/pi-tui/components/truncated-text.ts +65 -0
  158. package/src/vendor/pi-tui/components/v-stack.ts +33 -0
  159. package/src/vendor/pi-tui/editor-component.ts +74 -0
  160. package/src/vendor/pi-tui/fuzzy.ts +138 -0
  161. package/src/vendor/pi-tui/index.ts +156 -0
  162. package/src/vendor/pi-tui/keybindings.ts +320 -0
  163. package/src/vendor/pi-tui/keys.ts +1401 -0
  164. package/src/vendor/pi-tui/kill-ring.ts +46 -0
  165. package/src/vendor/pi-tui/latex.ts +1506 -0
  166. package/src/vendor/pi-tui/layout-node.ts +51 -0
  167. package/src/vendor/pi-tui/layout.ts +449 -0
  168. package/src/vendor/pi-tui/native-modifiers.ts +13 -0
  169. package/src/vendor/pi-tui/native-module-path.ts +31 -0
  170. package/src/vendor/pi-tui/native-platform.ts +63 -0
  171. package/src/vendor/pi-tui/stdin-buffer.ts +444 -0
  172. package/src/vendor/pi-tui/terminal-colors.ts +73 -0
  173. package/src/vendor/pi-tui/terminal-image.ts +696 -0
  174. package/src/vendor/pi-tui/terminal.ts +547 -0
  175. package/src/vendor/pi-tui/tui-alt-screen.ts +1742 -0
  176. package/src/vendor/pi-tui/tui-main-screen.ts +655 -0
  177. package/src/vendor/pi-tui/tui.ts +1456 -0
  178. package/src/vendor/pi-tui/undo-stack.ts +28 -0
  179. package/src/vendor/pi-tui/utils.ts +1345 -0
  180. package/src/vendor/pi-tui/word-navigation.ts +117 -0
@@ -0,0 +1,179 @@
1
+ /**
2
+ * Theme schema and validation. Pure: a file is read elsewhere, this module
3
+ * only turns JSON into a complete Theme plus named problems.
4
+ *
5
+ * A broken theme must still produce a usable Theme (the dark builtin).
6
+ * Throwing here would take the agent down the moment the user edited a
7
+ * comma, which is when they most need it (feat-theming design).
8
+ */
9
+
10
+ export const THEME_TOKENS = [
11
+ "text",
12
+ "thinkingText",
13
+ "toolTitle",
14
+ "toolOutput",
15
+ "error",
16
+ "toolDiffAdded",
17
+ "toolDiffRemoved",
18
+ "status",
19
+ "prompt",
20
+ "selectedBg",
21
+ "muted",
22
+ "warning",
23
+ "mdHeading",
24
+ "mdCode",
25
+ "mdLink",
26
+ "userMessageBg",
27
+ "toolPendingBg",
28
+ "toolSuccessBg",
29
+ "toolErrorBg",
30
+ "thinkingLow",
31
+ "thinkingMedium",
32
+ "thinkingHigh",
33
+ "thinkingXhigh",
34
+ "thinkingMax",
35
+ ] as const;
36
+
37
+ export type ThemeToken = (typeof THEME_TOKENS)[number];
38
+
39
+ export type ThemeColors = { readonly [K in ThemeToken]: string };
40
+
41
+ export interface Theme {
42
+ readonly name: string;
43
+ readonly colors: ThemeColors;
44
+ }
45
+
46
+ export interface ThemeParse {
47
+ readonly theme: Theme;
48
+ readonly diagnostics: readonly string[];
49
+ }
50
+
51
+ /**
52
+ * Palette adapted from pi (earendil-works/pi, MIT), coding-agent theme/dark.json
53
+ * and light.json at 3390bd93630965a12a0a1a5c36ce890ec22f7e1d. Existing custom
54
+ * tokens remain accepted; missing presentation tokens inherit these defaults.
55
+ */
56
+ const DARK_COLORS: ThemeColors = {
57
+ "text": "38;2;212;212;212",
58
+ "thinkingText": "38;2;128;128;128",
59
+ "toolTitle": "38;2;212;212;212",
60
+ "toolOutput": "38;2;128;128;128",
61
+ "error": "38;2;204;102;102",
62
+ "toolDiffAdded": "38;2;181;189;104",
63
+ "toolDiffRemoved": "38;2;204;102;102",
64
+ "status": "38;2;128;128;128",
65
+ "prompt": "38;2;212;212;212",
66
+ "selectedBg": "48;2;58;58;74",
67
+ "muted": "38;2;128;128;128",
68
+ "warning": "38;2;255;255;0",
69
+ "mdHeading": "38;2;240;198;116",
70
+ "mdCode": "38;2;138;190;183",
71
+ "mdLink": "38;2;129;162;190",
72
+ "userMessageBg": "48;2;52;53;65",
73
+ "toolPendingBg": "48;2;40;40;50",
74
+ "toolSuccessBg": "48;2;40;50;40",
75
+ "toolErrorBg": "48;2;60;40;40",
76
+ "thinkingLow": "38;2;95;135;175",
77
+ "thinkingMedium": "38;2;129;162;190",
78
+ "thinkingHigh": "38;2;178;148;187",
79
+ "thinkingXhigh": "38;2;209;131;232",
80
+ "thinkingMax": "38;2;255;95;255"
81
+ };
82
+
83
+ /** Distinct from dark so `--theme light` is visible without OSC 11 probing. */
84
+ const LIGHT_COLORS: ThemeColors = {
85
+ "text": "38;2;31;35;40",
86
+ "thinkingText": "38;2;108;108;108",
87
+ "toolTitle": "38;2;31;35;40",
88
+ "toolOutput": "38;2;108;108;108",
89
+ "error": "38;2;170;85;85",
90
+ "toolDiffAdded": "38;2;88;132;88",
91
+ "toolDiffRemoved": "38;2;170;85;85",
92
+ "status": "38;2;108;108;108",
93
+ "prompt": "38;2;31;35;40",
94
+ "selectedBg": "48;2;208;208;224",
95
+ "muted": "38;2;108;108;108",
96
+ "warning": "38;2;154;115;38",
97
+ "mdHeading": "38;2;154;115;38",
98
+ "mdCode": "38;2;90;128;128",
99
+ "mdLink": "38;2;84;125;167",
100
+ "userMessageBg": "48;2;232;232;232",
101
+ "toolPendingBg": "48;2;232;232;240",
102
+ "toolSuccessBg": "48;2;232;240;232",
103
+ "toolErrorBg": "48;2;240;232;232",
104
+ "thinkingLow": "38;2;84;125;167",
105
+ "thinkingMedium": "38;2;90;128;128",
106
+ "thinkingHigh": "38;2;135;95;135",
107
+ "thinkingXhigh": "38;2;139;0;139",
108
+ "thinkingMax": "38;2;175;0;95"
109
+ };
110
+
111
+ export function builtinTheme(name: "dark" | "light"): Theme {
112
+ return name === "light"
113
+ ? { name: "light", colors: LIGHT_COLORS }
114
+ : { name: "dark", colors: DARK_COLORS };
115
+ }
116
+
117
+ export function parseTheme(json: unknown): ThemeParse {
118
+ const fallback = builtinTheme("dark");
119
+ if (typeof json !== "object" || json === null || Array.isArray(json)) {
120
+ return {
121
+ theme: fallback,
122
+ diagnostics: ["theme is not an object; using dark"],
123
+ };
124
+ }
125
+ const rec = json as { readonly [k: string]: unknown };
126
+ const diagnostics: string[] = [];
127
+ const name = typeof rec["name"] === "string" && rec["name"].length > 0 ? rec["name"] : "custom";
128
+ const rawColors = rec["colors"];
129
+ const src: { readonly [k: string]: unknown } =
130
+ typeof rawColors === "object" && rawColors !== null && !Array.isArray(rawColors)
131
+ ? (rawColors as { readonly [k: string]: unknown })
132
+ : (() => {
133
+ diagnostics.push("theme.colors is missing or not an object; filling from dark");
134
+ return {};
135
+ })();
136
+
137
+ for (const key of Object.keys(src)) {
138
+ if (!isThemeToken(key)) diagnostics.push(`unknown token: ${key}`);
139
+ }
140
+
141
+ const colors: { [K in ThemeToken]: string } = { ...fallback.colors };
142
+ for (const token of THEME_TOKENS) {
143
+ if (!(token in src)) continue;
144
+ const parsed = parseColor(src[token]);
145
+ if (parsed === null) {
146
+ diagnostics.push(`invalid colour for ${token}; using built-in`);
147
+ continue;
148
+ }
149
+ colors[token] = parsed;
150
+ }
151
+
152
+ return { theme: { name, colors }, diagnostics };
153
+ }
154
+
155
+ function isThemeToken(key: string): key is ThemeToken {
156
+ return (THEME_TOKENS as readonly string[]).includes(key);
157
+ }
158
+
159
+ /**
160
+ * Accept SGR params (`31`, `38;5;180`, `38;2;r;g;b`), `#RRGGBB`, a 256-colour
161
+ * index, or empty (terminal default). Anything else is invalid.
162
+ */
163
+ export function parseColor(value: unknown): string | null {
164
+ if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 255) {
165
+ return `38;5;${value}`;
166
+ }
167
+ if (typeof value !== "string") return null;
168
+ if (value === "") return "";
169
+ const hex = /^#([0-9a-fA-F]{6})$/.exec(value);
170
+ if (hex) {
171
+ const n = Number.parseInt(hex[1]!, 16);
172
+ const r = (n >> 16) & 255;
173
+ const g = (n >> 8) & 255;
174
+ const b = n & 255;
175
+ return `38;2;${r};${g};${b}`;
176
+ }
177
+ if (/^[0-9]+(?:;[0-9]+)*$/.test(value)) return value;
178
+ return null;
179
+ }
@@ -0,0 +1,149 @@
1
+ /**
2
+ * Token accounting.
3
+ *
4
+ * The default here is an ESTIMATE and is labelled as one. Real counting needs the
5
+ * provider's own counter (Anthropic's `messages.count_tokens`); a bundled
6
+ * tokenizer would drift from whatever model is actually in use and would be wrong
7
+ * in a way nobody notices. The boundary logic therefore takes a counter as a
8
+ * parameter, so wiring the real one in is a one-line change and nothing in the
9
+ * core pretends to be authoritative.
10
+ */
11
+
12
+ import type { Message } from "./project.ts";
13
+
14
+ export interface TokenCounter {
15
+ readonly kind: "estimate" | "exact";
16
+ /**
17
+ * Counts everything the API bills as prompt: tool schemas, the system
18
+ * prompt and the messages.
19
+ *
20
+ * Tools are a parameter because leaving them out made two sides of a
21
+ * boundary decision incomparable -- one came from the provider's billed
22
+ * figure, which includes schemas, the other from a count that silently
23
+ * skipped them, and the difference read as work the node had done.
24
+ */
25
+ count(messages: readonly Message[], system: string, tools?: readonly unknown[]): number;
26
+ }
27
+
28
+ /**
29
+ * Estimated tokens for one string.
30
+ *
31
+ * ASCII keeps the original chars/3.6: code and JSON tokenize more densely than
32
+ * prose, so this sits below the usual chars/4 rule, and every measured cut
33
+ * point so far was taken with it. Ideographic scripts are a different
34
+ * regime -- one Han character is roughly 0.6-1.5 tokens across current
35
+ * tokenizers, where chars/3.6 would say 0.28 -- so they are charged a full
36
+ * token each. Overestimating only moves a cut earlier; underestimating is a
37
+ * 400 from the provider. Other non-ASCII (accents, symbols, emoji) sits
38
+ * between the two at half a token per code point.
39
+ */
40
+ export function estimateText(text: string): number {
41
+ let ascii = 0;
42
+ let weighted = 0;
43
+ for (let i = 0; i < text.length; i++) {
44
+ const code = text.charCodeAt(i);
45
+ if (code < 0x80) { ascii++; continue; }
46
+ // A surrogate pair is one code point; charge it once, at its high half.
47
+ if (code >= 0xdc00 && code <= 0xdfff) continue;
48
+ const ideographic =
49
+ (code >= 0x2e80 && code <= 0x9fff) || // CJK radicals .. unified ideographs, kana, bopomofo
50
+ (code >= 0xac00 && code <= 0xd7af) || // Hangul syllables
51
+ (code >= 0xf900 && code <= 0xfaff) || // CJK compatibility ideographs
52
+ (code >= 0xff00 && code <= 0xffef); // full-width forms
53
+ weighted += ideographic ? 1 : 0.5;
54
+ }
55
+ return ascii / 3.6 + weighted;
56
+ }
57
+
58
+ /**
59
+ * estimateText across message and system text, plus serialized schemas and
60
+ * per-message framing. Expect roughly +/-20% on ASCII; treat the trigger
61
+ * threshold as approximate until an exact counter is wired in.
62
+ */
63
+ export const estimateTokens: TokenCounter = {
64
+ kind: "estimate",
65
+ count(messages, system, tools = []) {
66
+ // Serialized schemas are what goes on the wire, so their length is the
67
+ // right proxy -- and they are billed on every single request.
68
+ let tokens = estimateText(system) + (tools.length > 0 ? estimateText(JSON.stringify(tools)) : 0);
69
+ for (const m of messages) {
70
+ for (const b of m.content) {
71
+ if (b.type === "text") tokens += estimateText(b.text);
72
+ // Thinking is opaque here; its serialized size is the best proxy.
73
+ else if (b.type === "thinking") tokens += estimateText(JSON.stringify(b.raw));
74
+ else if (b.type === "tool_use") tokens += estimateText(b.name) + estimateText(JSON.stringify(b.input));
75
+ else if (b.type === "tool_result") tokens += estimateText(b.content);
76
+ }
77
+ // Per-message framing overhead, roughly.
78
+ tokens += 8 / 3.6;
79
+ }
80
+ return Math.ceil(tokens);
81
+ },
82
+ };
83
+
84
+ /**
85
+ * A counter whose every count is multiplied by `factor`.
86
+ *
87
+ * Used once the provider has rejected a prompt the estimate said would fit:
88
+ * that is a measurement that this content tokenizes `factor` times denser
89
+ * than estimated, and sizing the retained context with the unscaled counter
90
+ * would reproduce the overflow.
91
+ */
92
+ export function scaledCounter(counter: TokenCounter, factor: number): TokenCounter {
93
+ return { kind: "estimate", count: (messages, system, tools) => Math.ceil(counter.count(messages, system, tools) * factor) };
94
+ }
95
+
96
+ /**
97
+ * An anchor: the prompt size a provider actually charged, and how far into the
98
+ * conversation that measurement covered.
99
+ *
100
+ * Measuring the estimator against a real tokenizer showed it running between
101
+ * -21% and +38% across realistic content -- prose, source, JSON and YAML all
102
+ * tokenize at different densities, so no single chars-per-token constant fits
103
+ * them at once. Underestimating source by a fifth is the dangerous direction:
104
+ * a 30k trigger would fire at an actual 38k.
105
+ *
106
+ * Calibrating the constant would not fix that, because the error depends on
107
+ * what the content happens to be. Anchoring does: every response reports the
108
+ * prompt it was actually billed for, so everything up to that point is known
109
+ * exactly and only the delta since needs estimating. The error then scales
110
+ * with one turn instead of with the whole conversation.
111
+ */
112
+ export interface Anchor {
113
+ /** Sum of uncached, cache-write and cache-read input tokens. */
114
+ promptTokens: number;
115
+ /** Tokens the model generated, which become part of the next prompt. */
116
+ outputTokens: number;
117
+ /** Number of messages the measurement covered. */
118
+ messageCount: number;
119
+ }
120
+
121
+ /**
122
+ * Prompt size for the next request: exact up to the anchor, estimated after.
123
+ *
124
+ * Falls back to a plain estimate before the first response, which is the only
125
+ * point where nothing has been measured yet.
126
+ */
127
+ export function anchoredCount(
128
+ messages: readonly Message[],
129
+ system: string,
130
+ anchor: Anchor | null,
131
+ counter: TokenCounter = estimateTokens,
132
+ tools: readonly unknown[] = [],
133
+ ): { tokens: number; exactUpTo: number } {
134
+ if (anchor === null || anchor.messageCount > messages.length) {
135
+ return { tokens: counter.count(messages, system, tools), exactUpTo: 0 };
136
+ }
137
+ // The anchored prompt already includes the system prompt and the tools, so
138
+ // the delta is counted without either to avoid charging for them twice.
139
+ // The measured output is already the first assistant message after the
140
+ // measured prompt. Its billed count replaces, rather than adds to, an
141
+ // estimate of that same message (D1).
142
+ const outputIndex = messages[anchor.messageCount]?.role === "assistant" ? 1 : 0;
143
+ const since = messages.slice(anchor.messageCount + outputIndex);
144
+ const delta = counter.count(since, "", []);
145
+ return {
146
+ tokens: anchor.promptTokens + anchor.outputTokens + delta,
147
+ exactUpTo: anchor.promptTokens,
148
+ };
149
+ }
@@ -0,0 +1,102 @@
1
+ /** O.3: one declarative input contract serves model schemas and runtime admission. */
2
+ export type JsonValue = null | boolean | number | string | readonly JsonValue[] | { readonly [key: string]: JsonValue };
3
+ export type ToolDiagnostic = {
4
+ code: "invalid_input" | "version_mismatch" | "not_found" | "ambiguous_edit" | "permission_denied" | "execution_failed";
5
+ message: string;
6
+ recover?: { tool: string; input: { readonly [key: string]: JsonValue } };
7
+ };
8
+
9
+ export function isJsonValue(value: unknown): value is JsonValue {
10
+ if (value === null || typeof value === "string" || typeof value === "boolean") return true;
11
+ if (typeof value === "number") return Number.isFinite(value);
12
+ return typeof value === "object" && (Array.isArray(value) ? value : Object.values(value)).every(isJsonValue);
13
+ }
14
+
15
+ export function isToolDiagnostic(value: unknown): value is ToolDiagnostic {
16
+ if (value === null || typeof value !== "object") return false;
17
+ const item = value as Record<string, unknown>;
18
+ if (!["invalid_input", "version_mismatch", "not_found", "ambiguous_edit", "permission_denied", "execution_failed"].includes(String(item["code"])) || typeof item["message"] !== "string") return false;
19
+ const recover = item["recover"];
20
+ if (recover === undefined) return true;
21
+ if (recover === null || typeof recover !== "object") return false;
22
+ const recovery = recover as Record<string, unknown>;
23
+ return typeof recovery["tool"] === "string" && recovery["input"] !== null &&
24
+ typeof recovery["input"] === "object" && !Array.isArray(recovery["input"]) && isJsonValue(recovery["input"]);
25
+ }
26
+
27
+ import { editDistance } from "./text-normalize.ts";
28
+
29
+ /**
30
+ * Names models send for arguments that exist under another name here. Edit
31
+ * distance cannot see these (cwd and workdir share no letters in order);
32
+ * each entry is a spelling another harness uses for the same argument.
33
+ */
34
+ const SYNONYMS: Readonly<Record<string, readonly string[]>> = {
35
+ cwd: ["workdir"], dir: ["workdir", "path"], directory: ["workdir", "path"],
36
+ file: ["path"], file_path: ["path"], filepath: ["path"], filename: ["path"],
37
+ cmd: ["command"], script: ["command"],
38
+ text: ["content"], contents: ["content"], old_str: ["old_string"], new_str: ["new_string"],
39
+ query: ["pattern"], regex: ["pattern"], timeout_ms: ["timeout"], offset: ["start"], limit: ["end", "limit"],
40
+ };
41
+
42
+ /** The candidate nearest to `name`, when it is near enough to be the likely intent. */
43
+ export function closestName(name: string, candidates: readonly string[]): string | undefined {
44
+ const synonym = SYNONYMS[name.toLowerCase()]?.find((alias) => candidates.includes(alias));
45
+ if (synonym !== undefined) return synonym;
46
+ let best: string | undefined, distance = Infinity;
47
+ for (const candidate of candidates) {
48
+ const d = Math.min(editDistance(name.toLowerCase(), candidate.toLowerCase()),
49
+ // A common drift is a synonym sharing a stem (replaceAll / replace_all).
50
+ editDistance(name.toLowerCase().replace(/[_-]/g, ""), candidate.toLowerCase().replace(/[_-]/g, "")));
51
+ if (d < distance) { best = candidate; distance = d; }
52
+ }
53
+ return best !== undefined && distance <= Math.max(2, Math.floor(best.length / 2)) ? best : undefined;
54
+ }
55
+
56
+ /**
57
+ * Validate the schema vocabulary used by built-ins; semantic constraints stay with their owner.
58
+ *
59
+ * An argument the schema does not declare is an error, not noise: a model
60
+ * that sends `cwd` believes it changed directory, and dropping the field
61
+ * silently runs the command somewhere it did not intend. `hidden` names
62
+ * internal arguments accepted at the top level without being advertised.
63
+ */
64
+ export function inputError(schema: unknown, value: unknown, path = "input", hidden: readonly string[] = []): string | null {
65
+ if (schema === null || typeof schema !== "object") return null;
66
+ const s = schema as Record<string, unknown>;
67
+ const type = s["type"];
68
+ if (type === "object" && (value === null || typeof value !== "object" || Array.isArray(value))) return `${path} must be an object`;
69
+ if (type === "array" && !Array.isArray(value)) return `${path} must be an array`;
70
+ if (["string", "boolean", "number"].includes(String(type)) && typeof value !== type) return `${path} must be ${String(type)}`;
71
+ if (type === "integer" && (typeof value !== "number" || !Number.isSafeInteger(value))) return `${path} must be a positive integer or zero where allowed`;
72
+ if (typeof value === "number" && !Number.isFinite(value)) return `${path} must be finite`;
73
+ if (typeof value === "number" && typeof s["minimum"] === "number" && value < s["minimum"]) return `${path} must be >= ${s["minimum"]}`;
74
+ if (typeof value === "number" && typeof s["maximum"] === "number" && value > s["maximum"]) return `${path} must be <= ${s["maximum"]}`;
75
+ if (typeof value === "string" && typeof s["minLength"] === "number" && value.length < s["minLength"]) return `${path} must not be empty`;
76
+ if (Array.isArray(s["enum"]) && !s["enum"].includes(value)) return `${path} must be one of ${s["enum"].join(", ")}`;
77
+ if (type === "array" && Array.isArray(value)) {
78
+ for (let i = 0; i < value.length; i++) { const error = inputError(s["items"], value[i], `${path}[${i}]`); if (error) return error; }
79
+ }
80
+ if (type === "object" && value !== null && typeof value === "object") {
81
+ const object = value as Record<string, unknown>;
82
+ for (const key of Array.isArray(s["required"]) ? s["required"] : []) {
83
+ if (typeof key === "string" && object[key] === undefined) return `${path}.${key} is required`;
84
+ }
85
+ const properties = s["properties"];
86
+ if (properties !== null && typeof properties === "object") {
87
+ const declared = Object.keys(properties);
88
+ for (const key of Object.keys(object)) {
89
+ if (declared.includes(key) || hidden.includes(key) || object[key] === undefined) continue;
90
+ const near = closestName(key, declared);
91
+ return `unknown argument ${path === "input" ? key : `${path}.${key}`}` + (near === undefined ? "" : `; did you mean ${near}?`) +
92
+ ` Valid arguments: ${[...declared].sort().join(", ")}`;
93
+ }
94
+ }
95
+ if (properties !== null && typeof properties === "object") for (const [key, property] of Object.entries(properties)) {
96
+ if (object[key] === undefined) continue;
97
+ const error = inputError(property, object[key], `${path}.${key}`);
98
+ if (error) return error;
99
+ }
100
+ }
101
+ return null;
102
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Argument repair: the shapes models actually send, mapped onto the schema
3
+ * before validation.
4
+ *
5
+ * Every rule here is a drift observed in other harnesses, not a guess: pi's
6
+ * edit tool accepts `edits` sent as a JSON string or a single object
7
+ * (`coding-agent/src/core/tools/edit.ts` prepareArguments), and ForgeCode
8
+ * reports fewer call errors after aligning names with what models are trained
9
+ * on. A repaired call is still the model's call -- the log keeps the input
10
+ * as sent; this only decides how it is interpreted. Pure and idempotent, so
11
+ * admission and execution can both apply it.
12
+ */
13
+ type Input = Record<string, unknown>;
14
+
15
+ function parsedJson(value: unknown): unknown {
16
+ if (typeof value !== "string") return value;
17
+ const trimmed = value.trim();
18
+ if (!trimmed.startsWith("[") && !trimmed.startsWith("{")) return value;
19
+ try { return JSON.parse(trimmed); } catch { return value; }
20
+ }
21
+
22
+ function editEntry(value: unknown): unknown {
23
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return value;
24
+ const entry = { ...(value as Input) };
25
+ if (entry["old_string"] === undefined && entry["old"] !== undefined) { entry["old_string"] = entry["old"]; delete entry["old"]; }
26
+ if (entry["new_string"] === undefined && entry["new"] !== undefined) { entry["new_string"] = entry["new"]; delete entry["new"]; }
27
+ return entry;
28
+ }
29
+
30
+ export function repairToolInput(name: string, input: unknown): unknown {
31
+ if (name !== "edit" || input === null || typeof input !== "object" || Array.isArray(input)) return input;
32
+ const repaired = editEntry(input) as Input;
33
+ if (repaired["edits"] !== undefined) {
34
+ const edits = parsedJson(repaired["edits"]);
35
+ repaired["edits"] = (Array.isArray(edits) ? edits : [edits]).map(editEntry);
36
+ }
37
+ return repaired;
38
+ }
@@ -0,0 +1,60 @@
1
+ /** Human history is a separate pure log projection from the model's bounded context (D1, D8). */
2
+ import type { LogEvent } from "./events.ts";
3
+ import { assertNever } from "./exhaustive.ts";
4
+
5
+ type Identity = { id: string; revision: string };
6
+ export type TranscriptItem = Identity & (
7
+ | { kind: "user"; text: string; interrupted: boolean }
8
+ | { kind: "assistant" | "thinking" | "notice"; text: string }
9
+ | { kind: "tool"; callId: string; name: string; input: unknown; result: { text: string; ok: boolean } | null }
10
+ );
11
+
12
+ export function projectTranscript(events: readonly LogEvent[]): TranscriptItem[] {
13
+ const items: TranscriptItem[] = [];
14
+ const calls = new Map<string, number>();
15
+ const users = new Map<string, number>();
16
+ for (const event of events) {
17
+ const identity = { id: event.id, revision: event.id };
18
+ switch (event.t) {
19
+ case "user":
20
+ users.set(event.id, items.length);
21
+ items.push({ ...identity, kind: "user", text: event.text, interrupted: false });
22
+ break;
23
+ case "assistant":
24
+ event.blocks.forEach((block, i) => {
25
+ const id = `${event.id}:${i}`;
26
+ if (block.type === "tool_use") {
27
+ calls.set(block.call_id, items.length);
28
+ items.push({ id, revision: id, kind: "tool", callId: block.call_id, name: block.name, input: block.input, result: null });
29
+ } else items.push({ id, revision: id, kind: block.type === "text" ? "assistant" : "thinking", text: block.text });
30
+ });
31
+ break;
32
+ case "tool_result": {
33
+ const index = calls.get(event.call_id);
34
+ const call = index === undefined ? undefined : items[index];
35
+ if (call?.kind === "tool" && index !== undefined) {
36
+ items[index] = { ...call, revision: event.id, result: { text: event.content, ok: event.ok } };
37
+ } else items.push({ ...identity, kind: "notice", text: `Unpaired tool result ${event.call_id}: ${event.content}` });
38
+ break;
39
+ }
40
+ case "interrupt": {
41
+ const index = users.get(event.user_id);
42
+ const user = index === undefined ? undefined : items[index];
43
+ if (user?.kind === "user" && index !== undefined) items[index] = { ...user, revision: event.id, interrupted: true };
44
+ items.push({ ...identity, kind: "notice", text: "Turn interrupted" });
45
+ break;
46
+ }
47
+ case "cut": items.push({ ...identity, kind: "notice", text: `Context boundary · ${event.reason} · ${event.handoff_ref}` }); break;
48
+ case "rewind": items.push({ ...identity, kind: "notice", text: `Navigated context (${event.mode}) · ${event.prompt}` }); break;
49
+ case "spawn": items.push({ ...identity, kind: "notice", text: `Branch ${event.branch.node} · ${event.purpose}` }); break;
50
+ case "joined": items.push({ ...identity, kind: "notice", text: `Branch ${event.branch.node} finished · ${event.summary}` }); break;
51
+ case "adopt": items.push({ ...identity, kind: "notice", text: `Adopted branch ${event.branch.node}` }); break;
52
+ case "discard": items.push({ ...identity, kind: "notice", text: `Discarded branch ${event.branch.node} · ${event.reason}` }); break;
53
+ case "guidance": case "snapshot": case "claim": case "decision": case "pitfall": case "blocked": case "question": case "next_action": case "work_status": case "evidence_link":
54
+ // These remain inspectable via /state and /export; they are not chat messages.
55
+ break;
56
+ default: assertNever(event, "projectTranscript");
57
+ }
58
+ }
59
+ return items;
60
+ }
@@ -0,0 +1,65 @@
1
+ /** N.13: deterministic state maintenance, not a public benchmark score. */
2
+ import { seededRandom } from "./experiment.ts";
3
+
4
+ export interface WindowRecord { key: string; revision: number; value: string }
5
+ export interface WindowArtifact { schema: 1; policy: string; stage: number; records: WindowRecord[] }
6
+ export interface WindowPage { stage: number; path: string; body: string; expected: WindowArtifact }
7
+ export interface WindowFixture { seed: number; contract: string; pages: WindowPage[] }
8
+
9
+ export function windowFixture(seed: number, stages: number, paddingChars: number): WindowFixture {
10
+ if (!Number.isSafeInteger(seed) || !Number.isSafeInteger(stages) || stages < 1 || stages > 64 ||
11
+ !Number.isSafeInteger(paddingChars) || paddingChars < 0 || paddingChars > 1_000_000) throw new RangeError("invalid window fixture dimensions");
12
+ const random = seededRandom(seed), nonce = () => Math.floor(random() * 0x1_0000_0000).toString(16).padStart(8, "0");
13
+ const policy = `approved-highest-revision/${nonce()}`;
14
+ const origin = { key: "origin", revision: 0, value: `keep-é-中文-${nonce()}` };
15
+ const state = new Map<string, WindowRecord>([[origin.key, origin]]), pages: WindowPage[] = [];
16
+ const contract = "Maintain ledger.json as the deliverable across all stages, including context changes and restarts. " +
17
+ `Its only root fields are schema (1), policy (${JSON.stringify(policy)}), stage (the current integer), and records. ` +
18
+ "Records is an array sorted by key, each containing exactly key, revision, value. Begin with this permanent record: " + JSON.stringify(origin) + ". " +
19
+ "Each incoming page has UPDATE <JSON> lines and irrelevant TRACE lines. Accept only approved:true updates. " +
20
+ "For each key keep the highest approved revision ever seen; ignore older and unapproved revisions, and treat duplicate records idempotently. " +
21
+ "Never delete an earlier key. Preserve the original policy and permanent record exactly. " +
22
+ "For this controlled context experiment, read each incoming page in full with read (no ranges or search/query substitute), " +
23
+ "then apply its UPDATE records. Keep ledger.json current using write/edit and verify it with read. " +
24
+ "Do not copy TRACE data into the ledger. Use only the provided tools. Reply briefly when the stage is done.";
25
+ for (let stage = 0; stage < stages; stage++) {
26
+ const accepted = [0, 1].map(offset => ({ key: `service-${(stage + offset) % 6}`, revision: stage + 10,
27
+ value: `value-${stage}-${offset}-${nonce()}`, approved: true }));
28
+ const first = accepted[0]!, second = accepted[1]!;
29
+ const updates = [...accepted, { ...first, revision: first.revision - 1, value: "obsolete" },
30
+ { ...second, revision: second.revision + 1000, value: "unapproved", approved: false }, ...accepted];
31
+ for (const { key, revision, value, approved } of updates) if (approved && revision > (state.get(key)?.revision ?? -1)) state.set(key, { key, revision, value });
32
+ const lines: string[] = [];
33
+ for (let used = 0; used < paddingChars;) {
34
+ const trace = "TRACE " + Array.from({ length: 64 }, nonce).join("");
35
+ lines.push(trace); used += trace.length + 1;
36
+ }
37
+ // Distribute evidence through the page, including near the end; a head-only
38
+ // preview must not pass as a full observation.
39
+ for (let i = updates.length - 1; i >= 0; i--) lines.splice(Math.floor((i + 0.5) * lines.length / updates.length), 0, `UPDATE ${JSON.stringify(updates[i])}`);
40
+ const expected: WindowArtifact = { schema: 1, policy, stage, records: [...state.values()].map(record => ({ ...record })).sort((a, b) => a.key < b.key ? -1 : 1) };
41
+ pages.push({ stage, path: `incoming/batch-${String(stage).padStart(2, "0")}.txt`, body: lines.join("\n"), expected });
42
+ }
43
+ return { seed, contract, pages };
44
+ }
45
+
46
+ export function windowPrompt(fixture: WindowFixture, stage: number): string {
47
+ const page = fixture.pages[stage]; if (!page) throw new RangeError("unknown window stage");
48
+ return (stage === 0 ? fixture.contract + "\n\n" : "Continue under the original ledger rules. ") +
49
+ `Process stage ${stage}: read ${page.path} in full, update ledger.json, verify it, then finish this stage.`;
50
+ }
51
+
52
+ /** Structural equality ignores JSON property order only, never omitted/extra keys. */
53
+ export function verifyWindowArtifact(body: string | null, expected: WindowArtifact): { ok: boolean; reason: string } {
54
+ if (body === null) return { ok: false, reason: "missing ledger.json" };
55
+ let actual: unknown;
56
+ try { actual = JSON.parse(body); } catch { return { ok: false, reason: "invalid ledger JSON" }; }
57
+ const equal = (a: unknown, b: unknown): boolean => {
58
+ if (a === b) return true;
59
+ if (typeof a !== "object" || a === null || typeof b !== "object" || b === null) return false;
60
+ if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((v, i) => equal(v, b[i]));
61
+ const aa = a as Record<string, unknown>, bb = b as Record<string, unknown>;
62
+ return Object.keys(aa).length === Object.keys(bb).length && Object.keys(bb).every(key => Object.hasOwn(aa, key) && equal(aa[key], bb[key]));
63
+ };
64
+ return equal(actual, expected) ? { ok: true, reason: "exact artifact verified" } : { ok: false, reason: "ledger differs from independent expected state" };
65
+ }