pantheon-opencode 1.4.3 → 1.5.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pantheon/code-mode/README.md +93 -0
- package/.pantheon/code-mode/checkpoint-session.py +7 -0
- package/.pantheon/code-mode/checkpoint-session.sh +11 -0
- package/.pantheon/code-mode/checkpoint_session.py +481 -0
- package/.pantheon/code-mode/compress-inline.py +450 -0
- package/.pantheon/code-mode/eval-llm-judge.py +255 -0
- package/.pantheon/code-mode/eval-monte-carlo.py +368 -0
- package/.pantheon/code-mode/eval-run.py +197 -0
- package/.pantheon/code-mode/eval-static.py +321 -0
- package/.pantheon/code-mode/example-sync.sh +5 -0
- package/.pantheon/code-mode/manifest.json +21 -0
- package/.pantheon/code-mode/session-end-save.py +186 -0
- package/.pantheon/code-mode/session-end-save.sh +18 -0
- package/.pantheon/code-mode/test-allowed.sh +5 -0
- package/.pantheon/code-mode/test-auto-continue-checkpoint.sh +179 -0
- package/.pantheon/code-mode/test-env.sh +7 -0
- package/.pantheon/code-mode/test-prlimit.sh +9 -0
- package/.pantheon/code-mode/test-timeout.sh +6 -0
- package/CHANGELOG.md +1026 -0
- package/README.md +292 -1080
- package/README.pt-BR.md +308 -0
- package/bin/pantheon-init.mjs +207 -31
- package/opencode.json +1 -0
- package/package.json +34 -14
- package/plugin.json +15 -0
- package/pyproject.toml +65 -0
- package/scripts/_pantheon_paths.py +48 -7
- package/scripts/ci-validate-yaml.py +7 -2
- package/scripts/code_mode_server.py +451 -44
- package/scripts/cost.mjs +30 -14
- package/scripts/doctor.mjs +306 -129
- package/scripts/generate-preset-docs.mjs +127 -66
- package/scripts/generate-prompts.sh +2 -2
- package/scripts/install/cli-ui.mjs +23 -10
- package/scripts/install/config-migration.mjs +399 -0
- package/scripts/install/health-check.mjs +3 -3
- package/scripts/install/model-picker.mjs +72 -23
- package/scripts/install/opencode-version.mjs +47 -0
- package/scripts/install/opencode.mjs +396 -121
- package/scripts/install/plugin.mjs +89 -49
- package/scripts/install/shared.mjs +55 -159
- package/scripts/install/strings.mjs +129 -0
- package/scripts/install-mcp.mjs +45 -232
- package/scripts/manifest-inventory.mjs +410 -0
- package/scripts/mcp_persistence_server.py +798 -60
- package/scripts/mcp_resources_server.py +144 -40
- package/scripts/memory_mcp_server.py +2 -2
- package/scripts/package-evidence.mjs +242 -0
- package/scripts/probe-context-rehydrate.mjs +240 -0
- package/scripts/probe-pantheon-cost.mjs +313 -0
- package/scripts/recover-zenodo-deposition.mjs +146 -0
- package/scripts/release-beta-version.mjs +56 -60
- package/scripts/sync-artifacts.mjs +169 -0
- package/scripts/sync-tui.mjs +135 -42
- package/scripts/test-opencode-v1-v2-sandbox.sh +862 -0
- package/scripts/uninstall.mjs +298 -954
- package/scripts/validate-zenodo-release.mjs +154 -0
- package/scripts/validation-policy.mjs +7 -7
- package/scripts/version-check.mjs +113 -108
- package/scripts/versioning.mjs +230 -86
- package/scripts/zenodo-artifact.mjs +21 -0
- package/scripts/zenodo-release-state.mjs +53 -0
- package/src/agents/mnemosyne.md +1 -1
- package/src/agents/zeus.md +27 -16
- package/src/mcp/_pantheon_paths.py +48 -7
- package/src/mcp/code_mode_server.py +451 -44
- package/src/mcp/eval_store.py +1 -1
- package/src/mcp/mcp_persistence_server.py +798 -60
- package/src/mcp/mcp_resources_server.py +144 -40
- package/src/mcp/pantheon_vision_server.py +1 -1
- package/src/mcp/toon_codec.py +491 -0
- package/src/pantheon/auto-wake.ts +1 -5
- package/src/pantheon/background-job-board.ts +110 -2
- package/src/pantheon/compaction-handoff.ts +304 -0
- package/src/pantheon/cost-command.ts +356 -69
- package/src/pantheon/delegate-manager.ts +766 -0
- package/src/pantheon/delegation-classifier.ts +307 -0
- package/src/pantheon/delegation-enforce.ts +13 -12
- package/src/pantheon/delegation-finalize.ts +406 -0
- package/src/pantheon/delegation-notify.ts +212 -0
- package/src/pantheon/delegation-toast.ts +73 -0
- package/src/pantheon/delegation.ts +1337 -0
- package/src/pantheon/file-persistence-salvage.ts +64 -0
- package/src/pantheon/file-persistence.ts +194 -45
- package/src/pantheon/goal-loop.ts +2 -2
- package/src/pantheon/goal-store.ts +1 -1
- package/src/pantheon/hashline/tool.ts +2 -2
- package/src/pantheon/model-command.ts +2 -1
- package/src/pantheon/native-probe.ts +234 -0
- package/src/pantheon/native-task-status.ts +139 -0
- package/src/pantheon/permission-eval.ts +284 -0
- package/src/pantheon/provider-capabilities.ts +221 -0
- package/src/pantheon/provider-stream-tracker.ts +182 -0
- package/src/pantheon/shared-board.ts +37 -0
- package/src/pantheon/step-cap.ts +110 -0
- package/src/pantheon/task-result-guard.ts +107 -0
- package/src/pantheon/token-opt.ts +689 -0
- package/src/pantheon/tool-ceiling.ts +159 -0
- package/src/pantheon/toon-codec.ts +481 -0
- package/src/pantheon/tui-version-check.ts +1 -3
- package/src/pantheon/v2-bridge.ts +129 -0
- package/src/pantheon/v2-events.ts +148 -0
- package/src/pantheon/v2-hooks.ts +287 -0
- package/src/pantheon/v2-tool-definitions.ts +331 -0
- package/src/pantheon/zeus-delegate-with-retry.ts +406 -0
- package/src/plugin-v2/index.ts +21 -0
- package/src/plugin-v2.ts +824 -0
- package/src/plugin.ts +261 -51
- package/src/plugins/pantheon-hooks.ts +51 -0
- package/src/plugins/tui/dist/tui.d.ts +116 -24
- package/src/plugins/tui/dist/tui.d.ts.map +1 -1
- package/src/plugins/tui/dist/tui.js +416 -149
- package/src/plugins/tui/dist/tui.tsx +456 -87
- package/src/plugins/tui/package-lock.json +51 -51
- package/src/plugins/tui/package.json +7 -7
- package/src/plugins/tui/src/index.tsx +462 -95
- package/src/skills/README.md +0 -2
- package/src/pantheon/tool-context.ts +0 -41
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/** Host-real context ceiling probe for C9/tool filtering. */
|
|
2
|
+
|
|
3
|
+
import type { PluginInput } from '@opencode-ai/plugin'
|
|
4
|
+
import { createPantheonLogger, type PantheonLogger } from './logger.ts'
|
|
5
|
+
import type { NativeTaskStatus } from './native-task-status.ts'
|
|
6
|
+
|
|
7
|
+
export interface ToolCeilingSnapshot {
|
|
8
|
+
usage: { inputTokens: number; outputTokens: number }
|
|
9
|
+
limit: { contextTokens: number }
|
|
10
|
+
supportsToolFiltering: boolean
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export type ToolCeilingResult =
|
|
14
|
+
| {
|
|
15
|
+
status: 'OK'
|
|
16
|
+
ceiling: number
|
|
17
|
+
usage: ToolCeilingSnapshot['usage']
|
|
18
|
+
limit: ToolCeilingSnapshot['limit']
|
|
19
|
+
}
|
|
20
|
+
| { status: Exclude<NativeTaskStatus, 'OK'>; detail: string }
|
|
21
|
+
|
|
22
|
+
export type HostToolCeilingSource = () => unknown | Promise<unknown>
|
|
23
|
+
|
|
24
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
25
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function isFiniteNonNegative(value: unknown): value is number {
|
|
29
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function isSnapshot(value: unknown): value is ToolCeilingSnapshot {
|
|
33
|
+
if (!isRecord(value) || !isRecord(value.usage) || !isRecord(value.limit)) return false
|
|
34
|
+
return (
|
|
35
|
+
isFiniteNonNegative(value.usage.inputTokens) &&
|
|
36
|
+
isFiniteNonNegative(value.usage.outputTokens) &&
|
|
37
|
+
isFiniteNonNegative(value.limit.contextTokens) &&
|
|
38
|
+
typeof value.supportsToolFiltering === 'boolean'
|
|
39
|
+
)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Probe exact host usage/limit without estimation, fallback, or retry. */
|
|
43
|
+
export async function probeToolCeiling(source?: HostToolCeilingSource): Promise<ToolCeilingResult> {
|
|
44
|
+
if (source === undefined)
|
|
45
|
+
return { status: 'UNSUPPORTED', detail: 'host SDK exposes no context probe' }
|
|
46
|
+
|
|
47
|
+
let raw: unknown
|
|
48
|
+
try {
|
|
49
|
+
raw = await source()
|
|
50
|
+
} catch (error: unknown) {
|
|
51
|
+
return { status: 'UNAVAILABLE', detail: error instanceof Error ? error.message : String(error) }
|
|
52
|
+
}
|
|
53
|
+
if (!isSnapshot(raw))
|
|
54
|
+
return { status: 'CORRUPT_DATA', detail: 'host context probe returned malformed data' }
|
|
55
|
+
if (raw.supportsToolFiltering === false) {
|
|
56
|
+
return { status: 'UNSUPPORTED', detail: 'host SDK does not filter tools' }
|
|
57
|
+
}
|
|
58
|
+
const used = raw.usage.inputTokens + raw.usage.outputTokens
|
|
59
|
+
if (used > raw.limit.contextTokens) {
|
|
60
|
+
return { status: 'CORRUPT_DATA', detail: 'host usage exceeds context limit' }
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
status: 'OK',
|
|
64
|
+
ceiling: raw.limit.contextTokens - used,
|
|
65
|
+
usage: raw.usage,
|
|
66
|
+
limit: raw.limit,
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Read a ceiling from the actual OpenCode SDK event and provider catalogue.
|
|
72
|
+
*
|
|
73
|
+
* `message.updated` is the SDK's runtime usage source (`info.tokens`). The
|
|
74
|
+
* provider catalogue supplies the selected model's context limit and its
|
|
75
|
+
* `capabilities.toolcall` flag. This adapter intentionally does not accept a
|
|
76
|
+
* caller-provided snapshot/callback, so tests cannot accidentally turn a
|
|
77
|
+
* fixture into a claimed host probe.
|
|
78
|
+
*/
|
|
79
|
+
export async function probeToolCeilingFromHost(
|
|
80
|
+
client: PluginInput['client'],
|
|
81
|
+
event: unknown,
|
|
82
|
+
): Promise<ToolCeilingResult> {
|
|
83
|
+
if (!isRecord(event) || event.type !== 'message.updated' || !isRecord(event.properties)) {
|
|
84
|
+
return { status: 'UNSUPPORTED', detail: 'host SDK event has no message usage' }
|
|
85
|
+
}
|
|
86
|
+
const info = event.properties.info
|
|
87
|
+
if (
|
|
88
|
+
!isRecord(info) ||
|
|
89
|
+
typeof info.providerID !== 'string' ||
|
|
90
|
+
typeof info.modelID !== 'string' ||
|
|
91
|
+
!isRecord(info.tokens)
|
|
92
|
+
) {
|
|
93
|
+
return { status: 'UNSUPPORTED', detail: 'host SDK message has no token usage' }
|
|
94
|
+
}
|
|
95
|
+
const providerID = info.providerID as string
|
|
96
|
+
const modelID = info.modelID as string
|
|
97
|
+
const tokens = info.tokens as Record<string, unknown>
|
|
98
|
+
|
|
99
|
+
let providersResult: unknown
|
|
100
|
+
try {
|
|
101
|
+
providersResult = await client.config.providers()
|
|
102
|
+
} catch (error: unknown) {
|
|
103
|
+
return { status: 'UNAVAILABLE', detail: error instanceof Error ? error.message : String(error) }
|
|
104
|
+
}
|
|
105
|
+
if (!isRecord(providersResult) || providersResult.error !== undefined) {
|
|
106
|
+
return { status: 'UNAVAILABLE', detail: 'host SDK provider catalogue unavailable' }
|
|
107
|
+
}
|
|
108
|
+
const data = providersResult.data
|
|
109
|
+
if (!isRecord(data) || !Array.isArray(data.providers)) {
|
|
110
|
+
return { status: 'CORRUPT_DATA', detail: 'host SDK provider catalogue malformed' }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const provider = data.providers.find(
|
|
114
|
+
(candidate) =>
|
|
115
|
+
isRecord(candidate) &&
|
|
116
|
+
candidate.id === providerID &&
|
|
117
|
+
isRecord(candidate.models) &&
|
|
118
|
+
isRecord(candidate.models[modelID]),
|
|
119
|
+
)
|
|
120
|
+
if (!isRecord(provider) || !isRecord(provider.models)) {
|
|
121
|
+
return { status: 'UNSUPPORTED', detail: 'host SDK model metadata unavailable' }
|
|
122
|
+
}
|
|
123
|
+
const model = provider.models[modelID]
|
|
124
|
+
if (!isRecord(model) || !isRecord(model.limit) || !isRecord(model.capabilities)) {
|
|
125
|
+
return { status: 'UNSUPPORTED', detail: 'host SDK model ceiling capability unavailable' }
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
return probeToolCeiling(() => ({
|
|
129
|
+
usage: { inputTokens: tokens.input, outputTokens: tokens.output },
|
|
130
|
+
limit: { contextTokens: (model.limit as Record<string, unknown>).context },
|
|
131
|
+
supportsToolFiltering: (model.capabilities as Record<string, unknown>).toolcall,
|
|
132
|
+
}))
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const log = createPantheonLogger({ module: 'pantheon-tool-ceiling' })
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Handle the host usage event and retain the result for the compaction hook.
|
|
139
|
+
* Malformed SDK events are diagnostic-only: they must never break the event
|
|
140
|
+
* pipeline or create an entry under an invalid session identifier.
|
|
141
|
+
*/
|
|
142
|
+
export async function handleToolCeilingEvent(
|
|
143
|
+
client: PluginInput['client'],
|
|
144
|
+
event: unknown,
|
|
145
|
+
ceilings: Map<string, ToolCeilingResult>,
|
|
146
|
+
logger: Pick<PantheonLogger, 'warn'> = log,
|
|
147
|
+
): Promise<ToolCeilingResult> {
|
|
148
|
+
const sessionID = (
|
|
149
|
+
event as { properties?: { info?: { sessionID?: unknown } } } | null | undefined
|
|
150
|
+
)?.properties?.info?.sessionID
|
|
151
|
+
if (typeof sessionID !== 'string' || sessionID.trim() === '') {
|
|
152
|
+
logger.warn('[Pantheon Plugin] message.updated event has no valid sessionID')
|
|
153
|
+
return { status: 'UNSUPPORTED', detail: 'host SDK event has no valid sessionID' }
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const result = await probeToolCeilingFromHost(client, event)
|
|
157
|
+
ceilings.set(sessionID, result)
|
|
158
|
+
return result
|
|
159
|
+
}
|
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WS3 TOON codec (PR #94) — Token-Oriented Object Notation, TypeScript mirror.
|
|
3
|
+
*
|
|
4
|
+
* Minimal deterministic encoding for board signals, checkpoints and KV
|
|
5
|
+
* payloads. Semantics identical to JSON; structurally smaller (no braces,
|
|
6
|
+
* no per-key quotes, no commas); JSON fallback when the parser is
|
|
7
|
+
* absent/fails. Savings are per payload class (~11% content-dominated
|
|
8
|
+
* board records up to ~49% large tabular checkpoints — see
|
|
9
|
+
* `docs/ws3-token-opt-measurements.md` for the reproducible table).
|
|
10
|
+
* Byte-compatible with `src/mcp/toon_codec.py` on the
|
|
11
|
+
* supported subset (null/bool/int/string, dicts, lists, uniform tables).
|
|
12
|
+
*
|
|
13
|
+
* Pure TypeScript — zero dependencies, no LLM, no external gateway.
|
|
14
|
+
*
|
|
15
|
+
* @module toon-codec
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
export const TABLE_MARKER = '@table '
|
|
19
|
+
|
|
20
|
+
/** JSON-compatible value subset covered by the codec. */
|
|
21
|
+
export type ToonValue =
|
|
22
|
+
| null
|
|
23
|
+
| boolean
|
|
24
|
+
| number
|
|
25
|
+
| string
|
|
26
|
+
| { [key: string]: ToonValue }
|
|
27
|
+
| ToonValue[]
|
|
28
|
+
|
|
29
|
+
export interface ToonSizeReport {
|
|
30
|
+
jsonChars: number
|
|
31
|
+
toonChars: number
|
|
32
|
+
ratio: number
|
|
33
|
+
savedPct: number
|
|
34
|
+
jsonTokens: number
|
|
35
|
+
toonTokens: number
|
|
36
|
+
tokenRatio: number
|
|
37
|
+
tokenSavedPct: number
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Default cap on nested-block depth accepted by `toonDecode` (DoS guard). */
|
|
41
|
+
export const TOON_MAX_DEPTH = 100
|
|
42
|
+
|
|
43
|
+
/** Default cap on input length accepted by `toonDecode` (DoS guard). */
|
|
44
|
+
export const TOON_MAX_CHARS = 1000000
|
|
45
|
+
|
|
46
|
+
const BARE_KEY_RE = /^[A-Za-z0-9_][A-Za-z0-9_.@+~/-]*$/
|
|
47
|
+
const INT_RE = /^-?\d+$/
|
|
48
|
+
const FLOAT_RE = /^-?(?:\d+\.\d+|\d+\.\d*[eE][+-]?\d+|\d+[eE][+-]?\d+)$/
|
|
49
|
+
const KEYLIKE_RE = /^(?:"(?:[^"\\]|\\.)*"|[A-Za-z0-9_][A-Za-z0-9_.@+~/-]*)\s*:(?:\s+|$)/
|
|
50
|
+
const RESERVED = new Set(['null', 'true', 'false', '{}', '[]'])
|
|
51
|
+
|
|
52
|
+
function looksNumeric(text: string): boolean {
|
|
53
|
+
return INT_RE.test(text) || FLOAT_RE.test(text)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Nominal chars-per-token ratio (same fixed basis as token-opt metering). */
|
|
57
|
+
export const CHARS_PER_TOKEN = 4
|
|
58
|
+
|
|
59
|
+
function charsToTokens(chars: number): number {
|
|
60
|
+
if (chars <= 0) return 0
|
|
61
|
+
return Math.ceil(chars / CHARS_PER_TOKEN)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function quote(text: string): string {
|
|
65
|
+
return JSON.stringify(text)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function isBareValue(text: string, inTable = false): boolean {
|
|
69
|
+
if (text === '' || text !== text.trim()) return false
|
|
70
|
+
if (RESERVED.has(text) || looksNumeric(text)) return false
|
|
71
|
+
// In table cells a bare `|` would split the row — quote such values.
|
|
72
|
+
if (inTable && text.includes('|')) return false
|
|
73
|
+
if (text.includes('"') || text.includes('\\') || text.includes('\n') || text.includes('\r')) {
|
|
74
|
+
return false
|
|
75
|
+
}
|
|
76
|
+
return true
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function encodeScalar(value: ToonValue, itemPosition = false, tableCell = false): string {
|
|
80
|
+
if (value === null) return 'null'
|
|
81
|
+
if (value === true) return 'true'
|
|
82
|
+
if (value === false) return 'false'
|
|
83
|
+
if (typeof value === 'number') return String(value)
|
|
84
|
+
const text = String(value)
|
|
85
|
+
if (isBareValue(text, tableCell)) {
|
|
86
|
+
if (itemPosition && KEYLIKE_RE.test(text)) return quote(text)
|
|
87
|
+
return text
|
|
88
|
+
}
|
|
89
|
+
return quote(text)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function encodeKey(key: string): string {
|
|
93
|
+
return BARE_KEY_RE.test(key) ? key : quote(key)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function isPlainObject(value: ToonValue): value is { [key: string]: ToonValue } {
|
|
97
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function isUniformDictList(value: ToonValue): value is { [key: string]: ToonValue }[] {
|
|
101
|
+
if (!Array.isArray(value) || value.length === 0) return false
|
|
102
|
+
if (!value.every((item) => isPlainObject(item))) return false
|
|
103
|
+
const firstKeys = Object.keys(value[0] as { [key: string]: ToonValue })
|
|
104
|
+
if (firstKeys.length === 0) return false
|
|
105
|
+
return (value as { [key: string]: ToonValue }[]).every((item) => {
|
|
106
|
+
const keys = Object.keys(item)
|
|
107
|
+
return keys.length === firstKeys.length && keys.every((k, i) => k === firstKeys[i])
|
|
108
|
+
})
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Split a table row on `|` outside double quotes. */
|
|
112
|
+
function splitCells(line: string): string[] {
|
|
113
|
+
const cells: string[] = []
|
|
114
|
+
let current = ''
|
|
115
|
+
let inQuotes = false
|
|
116
|
+
let escaped = false
|
|
117
|
+
for (const char of line) {
|
|
118
|
+
if (escaped) {
|
|
119
|
+
current += char
|
|
120
|
+
escaped = false
|
|
121
|
+
} else if (char === '\\' && inQuotes) {
|
|
122
|
+
current += char
|
|
123
|
+
escaped = true
|
|
124
|
+
} else if (char === '"') {
|
|
125
|
+
current += char
|
|
126
|
+
inQuotes = !inQuotes
|
|
127
|
+
} else if (char === '|' && !inQuotes) {
|
|
128
|
+
cells.push(current)
|
|
129
|
+
current = ''
|
|
130
|
+
} else {
|
|
131
|
+
current += char
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
cells.push(current)
|
|
135
|
+
return cells
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Encode a value to a TOON string (deterministic). */
|
|
139
|
+
export function toonEncode(value: ToonValue): string {
|
|
140
|
+
return encodeValue(value, 0).join('\n')
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function encodeValue(value: ToonValue, indent: number): string[] {
|
|
144
|
+
const pad = ' '.repeat(indent)
|
|
145
|
+
if (value === null || typeof value !== 'object') return [pad + encodeScalar(value)]
|
|
146
|
+
if (isPlainObject(value)) {
|
|
147
|
+
const entries = Object.entries(value)
|
|
148
|
+
if (entries.length === 0) return [`${pad}{}`]
|
|
149
|
+
const lines: string[] = []
|
|
150
|
+
for (const [key, item] of entries) {
|
|
151
|
+
const encodedKey = encodeKey(key)
|
|
152
|
+
if (item === null || typeof item !== 'object') {
|
|
153
|
+
lines.push(`${pad}${encodedKey}: ${encodeScalar(item)}`)
|
|
154
|
+
} else if (isPlainObject(item) && Object.keys(item).length === 0) {
|
|
155
|
+
lines.push(`${pad}${encodedKey}: {}`)
|
|
156
|
+
} else if (Array.isArray(item) && item.length === 0) {
|
|
157
|
+
lines.push(`${pad}${encodedKey}: []`)
|
|
158
|
+
} else {
|
|
159
|
+
lines.push(`${pad}${encodedKey}:`)
|
|
160
|
+
lines.push(...encodeValue(item, indent + 2))
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return lines
|
|
164
|
+
}
|
|
165
|
+
if (value.length === 0) return [`${pad}[]`]
|
|
166
|
+
if (isUniformDictList(value)) {
|
|
167
|
+
const keys = Object.keys(value[0] as { [key: string]: ToonValue })
|
|
168
|
+
const lines = [pad + TABLE_MARKER + keys.map((k) => encodeKey(k)).join('|')]
|
|
169
|
+
for (const item of value) {
|
|
170
|
+
lines.push(
|
|
171
|
+
pad +
|
|
172
|
+
keys
|
|
173
|
+
.map((k) =>
|
|
174
|
+
encodeScalar((item as Record<string, ToonValue>)[k] as ToonValue, true, true),
|
|
175
|
+
)
|
|
176
|
+
.join('|'),
|
|
177
|
+
)
|
|
178
|
+
}
|
|
179
|
+
return lines
|
|
180
|
+
}
|
|
181
|
+
return (value as ToonValue[]).flatMap((item) => encodeItem(item, indent))
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function encodeItem(item: ToonValue, indent: number): string[] {
|
|
185
|
+
const pad = ' '.repeat(indent)
|
|
186
|
+
if (item === null || typeof item !== 'object') return [`${pad}- ${encodeScalar(item, true)}`]
|
|
187
|
+
if (isPlainObject(item) && Object.keys(item).length === 0) return [`${pad}- {}`]
|
|
188
|
+
if (Array.isArray(item) && item.length === 0) return [`${pad}- []`]
|
|
189
|
+
const sub = encodeValue(item, indent + 2)
|
|
190
|
+
const first = sub[0]?.trim() ?? ''
|
|
191
|
+
return [`${pad}- ${first}`, ...sub.slice(1)]
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function parseScalar(token: string): ToonValue {
|
|
195
|
+
if (token === 'null') return null
|
|
196
|
+
if (token === 'true') return true
|
|
197
|
+
if (token === 'false') return false
|
|
198
|
+
if (token === '{}') return {}
|
|
199
|
+
if (token === '[]') return []
|
|
200
|
+
if (INT_RE.test(token)) return Number.parseInt(token, 10)
|
|
201
|
+
if (FLOAT_RE.test(token)) return Number.parseFloat(token)
|
|
202
|
+
if (token.startsWith('"')) return JSON.parse(token) as ToonValue
|
|
203
|
+
return token
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function indentOf(line: string): number {
|
|
207
|
+
return line.length - line.trimStart().length
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Split `key: value` on the FIRST colon (values may contain colons).
|
|
212
|
+
* A quoted key is parsed FIRST (mirror of the py `raw_decode` fix): the
|
|
213
|
+
* colon is searched AFTER the closing quote, so `{"a:b": 2}` splits into
|
|
214
|
+
* key `a:b` instead of breaking inside the quotes.
|
|
215
|
+
*/
|
|
216
|
+
function splitKeyValue(text: string): [string, string] {
|
|
217
|
+
if (text.startsWith('"')) {
|
|
218
|
+
let i = 1
|
|
219
|
+
let closed = -1
|
|
220
|
+
while (i < text.length) {
|
|
221
|
+
const ch = text[i]
|
|
222
|
+
if (ch === '\\') {
|
|
223
|
+
i += 2
|
|
224
|
+
continue
|
|
225
|
+
}
|
|
226
|
+
if (ch === '"') {
|
|
227
|
+
closed = i
|
|
228
|
+
break
|
|
229
|
+
}
|
|
230
|
+
i += 1
|
|
231
|
+
}
|
|
232
|
+
if (closed < 0) throw new Error(`TOON: line without ':' separator: ${JSON.stringify(text)}`)
|
|
233
|
+
const rest = text.slice(closed + 1).trimStart()
|
|
234
|
+
if (!rest.startsWith(':')) {
|
|
235
|
+
throw new Error(`TOON: line without ':' separator: ${JSON.stringify(text)}`)
|
|
236
|
+
}
|
|
237
|
+
let key: unknown
|
|
238
|
+
try {
|
|
239
|
+
key = JSON.parse(text.slice(0, closed + 1)) as unknown
|
|
240
|
+
} catch {
|
|
241
|
+
throw new Error(`TOON: line without ':' separator: ${JSON.stringify(text)}`)
|
|
242
|
+
}
|
|
243
|
+
if (typeof key !== 'string' || key === '') {
|
|
244
|
+
throw new Error(`TOON: empty key in line: ${JSON.stringify(text)}`)
|
|
245
|
+
}
|
|
246
|
+
return [key, rest.slice(1).trim()]
|
|
247
|
+
}
|
|
248
|
+
const idx = text.indexOf(':')
|
|
249
|
+
if (idx < 0) throw new Error(`TOON: line without ':' separator: ${JSON.stringify(text)}`)
|
|
250
|
+
const rawKey = text.slice(0, idx).trim()
|
|
251
|
+
if (rawKey === '') throw new Error(`TOON: empty key in line: ${JSON.stringify(text)}`)
|
|
252
|
+
const key = rawKey.startsWith('"') ? (JSON.parse(rawKey) as string) : rawKey
|
|
253
|
+
return [key, text.slice(idx + 1).trim()]
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Decode a TOON string back to the identical value (throws on invalid).
|
|
258
|
+
* `maxDepth`/`maxChars` are DoS guards: oversized inputs fail with a
|
|
259
|
+
* controlled `Error` instead of a stack overflow / memory blowup.
|
|
260
|
+
*/
|
|
261
|
+
export function toonDecode(
|
|
262
|
+
text: string,
|
|
263
|
+
maxDepth: number = TOON_MAX_DEPTH,
|
|
264
|
+
maxChars: number = TOON_MAX_CHARS,
|
|
265
|
+
): ToonValue {
|
|
266
|
+
if (text.length > maxChars) {
|
|
267
|
+
throw new Error(`TOON: input too large (${text.length} chars > maxChars=${maxChars})`)
|
|
268
|
+
}
|
|
269
|
+
const lines = text.split('\n').filter((line) => line.trim() !== '')
|
|
270
|
+
if (lines.length === 0) throw new Error('TOON: empty input')
|
|
271
|
+
const first = lines[0]?.trim() ?? ''
|
|
272
|
+
if (first.startsWith('{') || first.startsWith('[')) {
|
|
273
|
+
try {
|
|
274
|
+
return JSON.parse(text) as ToonValue
|
|
275
|
+
} catch {
|
|
276
|
+
throw new Error('TOON: invalid JSON fallback')
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
const [value, nextIdx] = parseBlock(lines, 0, 0, 0, maxDepth)
|
|
280
|
+
if (nextIdx !== lines.length) throw new Error(`TOON: trailing content at line ${nextIdx + 1}`)
|
|
281
|
+
return value
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function parseBlock(
|
|
285
|
+
lines: string[],
|
|
286
|
+
idx: number,
|
|
287
|
+
indent: number,
|
|
288
|
+
depth: number,
|
|
289
|
+
maxDepth: number,
|
|
290
|
+
): [ToonValue, number] {
|
|
291
|
+
if (depth > maxDepth) throw new Error(`TOON: maxDepth=${maxDepth} exceeded`)
|
|
292
|
+
const stripped = lines[idx]?.trim() ?? ''
|
|
293
|
+
if (stripped.startsWith('@table ')) return parseTable(lines, idx, indent)
|
|
294
|
+
if (stripped.startsWith('- ') || stripped === '-')
|
|
295
|
+
return parseList(lines, idx, indent, depth, maxDepth)
|
|
296
|
+
return parseDict(lines, idx, indent, depth, maxDepth)
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function parseTable(lines: string[], idx: number, indent: number): [ToonValue, number] {
|
|
300
|
+
const headerLine = lines[idx]?.trim() ?? ''
|
|
301
|
+
const header = headerLine.slice(TABLE_MARKER.length)
|
|
302
|
+
const keys = splitCells(header).map((cell) =>
|
|
303
|
+
cell.startsWith('"') ? (JSON.parse(cell) as string) : cell,
|
|
304
|
+
)
|
|
305
|
+
if (keys.length === 0 || keys.some((k) => k === '')) {
|
|
306
|
+
throw new Error(`TOON: bad @table header at line ${idx + 1}`)
|
|
307
|
+
}
|
|
308
|
+
const rows: { [key: string]: ToonValue }[] = []
|
|
309
|
+
idx += 1
|
|
310
|
+
while (idx < lines.length && indentOf(lines[idx] as string) === indent) {
|
|
311
|
+
const rowStripped = (lines[idx] as string).trim()
|
|
312
|
+
if (rowStripped.startsWith('@table ') || rowStripped.startsWith('- ') || rowStripped === '-') {
|
|
313
|
+
break
|
|
314
|
+
}
|
|
315
|
+
if (rowStripped.includes(':') && KEYLIKE_RE.test(rowStripped)) break
|
|
316
|
+
const cells = splitCells(rowStripped)
|
|
317
|
+
if (cells.length !== keys.length) throw new Error(`TOON: row width mismatch at line ${idx + 1}`)
|
|
318
|
+
const row: { [key: string]: ToonValue } = {}
|
|
319
|
+
keys.forEach((k, i) => {
|
|
320
|
+
row[k] = parseScalar(cells[i] as string)
|
|
321
|
+
})
|
|
322
|
+
rows.push(row)
|
|
323
|
+
idx += 1
|
|
324
|
+
}
|
|
325
|
+
return [rows, idx]
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function parseDict(
|
|
329
|
+
lines: string[],
|
|
330
|
+
idx: number,
|
|
331
|
+
indent: number,
|
|
332
|
+
depth: number,
|
|
333
|
+
maxDepth: number,
|
|
334
|
+
): [ToonValue, number] {
|
|
335
|
+
const result: { [key: string]: ToonValue } = {}
|
|
336
|
+
while (idx < lines.length) {
|
|
337
|
+
const line = lines[idx] as string
|
|
338
|
+
if (line.trim() === '' || indentOf(line) !== indent) break
|
|
339
|
+
const stripped = line.trim()
|
|
340
|
+
if (stripped.startsWith('@table ') || stripped.startsWith('- ') || stripped === '-') break
|
|
341
|
+
const [key, rest] = splitKeyValue(stripped)
|
|
342
|
+
if (rest === '') {
|
|
343
|
+
idx += 1
|
|
344
|
+
const nestedLine = idx < lines.length ? (lines[idx] as string) : undefined
|
|
345
|
+
if (nestedLine === undefined || indentOf(nestedLine) <= indent) {
|
|
346
|
+
throw new Error(`TOON: missing nested block for key ${JSON.stringify(key)}`)
|
|
347
|
+
}
|
|
348
|
+
const [nested, next] = parseBlock(lines, idx, indentOf(nestedLine), depth + 1, maxDepth)
|
|
349
|
+
result[key] = nested
|
|
350
|
+
idx = next
|
|
351
|
+
} else {
|
|
352
|
+
result[key] = parseScalar(rest)
|
|
353
|
+
idx += 1
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
return [result, idx]
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function parseList(
|
|
360
|
+
lines: string[],
|
|
361
|
+
idx: number,
|
|
362
|
+
indent: number,
|
|
363
|
+
depth: number,
|
|
364
|
+
maxDepth: number,
|
|
365
|
+
): [ToonValue, number] {
|
|
366
|
+
const items: ToonValue[] = []
|
|
367
|
+
while (idx < lines.length) {
|
|
368
|
+
const line = lines[idx] as string
|
|
369
|
+
if (indentOf(line) !== indent) break
|
|
370
|
+
const stripped = line.trim()
|
|
371
|
+
if (!stripped.startsWith('- ') && stripped !== '-') break
|
|
372
|
+
const rest = stripped.slice(1).trim()
|
|
373
|
+
if (rest === '') {
|
|
374
|
+
idx += 1
|
|
375
|
+
const nestedLine = idx < lines.length ? (lines[idx] as string) : undefined
|
|
376
|
+
if (nestedLine === undefined || indentOf(nestedLine) <= indent) {
|
|
377
|
+
throw new Error(`TOON: missing nested block in list at line ${idx + 1}`)
|
|
378
|
+
}
|
|
379
|
+
const [nested, next] = parseBlock(lines, idx, indentOf(nestedLine), depth + 1, maxDepth)
|
|
380
|
+
items.push(nested)
|
|
381
|
+
idx = next
|
|
382
|
+
continue
|
|
383
|
+
}
|
|
384
|
+
if (KEYLIKE_RE.test(rest)) {
|
|
385
|
+
const [key, subRest] = splitKeyValue(rest)
|
|
386
|
+
const item: { [key: string]: ToonValue } = {}
|
|
387
|
+
if (subRest === '') {
|
|
388
|
+
idx += 1
|
|
389
|
+
const nestedLine = idx < lines.length ? (lines[idx] as string) : undefined
|
|
390
|
+
if (nestedLine === undefined || indentOf(nestedLine) <= indent) {
|
|
391
|
+
throw new Error(`TOON: missing nested block for key ${JSON.stringify(key)}`)
|
|
392
|
+
}
|
|
393
|
+
const [nested, next] = parseBlock(lines, idx, indentOf(nestedLine), depth + 1, maxDepth)
|
|
394
|
+
item[key] = nested
|
|
395
|
+
idx = next
|
|
396
|
+
} else {
|
|
397
|
+
item[key] = parseScalar(subRest)
|
|
398
|
+
idx += 1
|
|
399
|
+
}
|
|
400
|
+
while (idx < lines.length && indentOf(lines[idx] as string) > indent) {
|
|
401
|
+
const cont = (lines[idx] as string).trim()
|
|
402
|
+
const [subKey, subValue] = splitKeyValue(cont)
|
|
403
|
+
if (subValue === '') {
|
|
404
|
+
idx += 1
|
|
405
|
+
const nestedLine = idx < lines.length ? (lines[idx] as string) : undefined
|
|
406
|
+
if (nestedLine === undefined || indentOf(nestedLine) <= indent) {
|
|
407
|
+
throw new Error(`TOON: missing nested block for key ${JSON.stringify(subKey)}`)
|
|
408
|
+
}
|
|
409
|
+
const [nested, next] = parseBlock(lines, idx, indentOf(nestedLine), depth + 1, maxDepth)
|
|
410
|
+
item[subKey] = nested
|
|
411
|
+
idx = next
|
|
412
|
+
} else {
|
|
413
|
+
item[subKey] = parseScalar(subValue)
|
|
414
|
+
idx += 1
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
items.push(item)
|
|
418
|
+
} else {
|
|
419
|
+
items.push(parseScalar(rest))
|
|
420
|
+
idx += 1
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
return [items, idx]
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Decode TOON, falling back to JSON (mirrors "parser absent" mode).
|
|
428
|
+
* Throws a consistent `TOON:` error when the input is neither valid TOON
|
|
429
|
+
* nor valid JSON (never leaks the raw JSON parser message).
|
|
430
|
+
*/
|
|
431
|
+
export function toonDecodeAuto(
|
|
432
|
+
text: string,
|
|
433
|
+
maxDepth: number = TOON_MAX_DEPTH,
|
|
434
|
+
maxChars: number = TOON_MAX_CHARS,
|
|
435
|
+
): ToonValue {
|
|
436
|
+
const stripped = text.trim()
|
|
437
|
+
if (stripped.startsWith('{') || stripped.startsWith('[')) {
|
|
438
|
+
try {
|
|
439
|
+
return JSON.parse(text) as ToonValue
|
|
440
|
+
} catch {
|
|
441
|
+
throw new Error('TOON: invalid JSON fallback')
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
try {
|
|
445
|
+
return toonDecode(text, maxDepth, maxChars)
|
|
446
|
+
} catch {
|
|
447
|
+
try {
|
|
448
|
+
return JSON.parse(text) as ToonValue
|
|
449
|
+
} catch {
|
|
450
|
+
throw new Error(
|
|
451
|
+
`TOON: invalid input (neither TOON nor JSON): ${JSON.stringify(stripped.slice(0, 60))}`,
|
|
452
|
+
)
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* Compare JSON vs TOON sizes for a value.
|
|
459
|
+
* Reports BOTH chars and tokens: token ratios on tiny payloads suffer
|
|
460
|
+
* ceiling bias (`ceil` rounds 161→41 vs 143→36), so the char columns are
|
|
461
|
+
* the honest basis and the token columns show what metering will debit.
|
|
462
|
+
* Per-class numbers: see `docs/ws3-token-opt-measurements.md`.
|
|
463
|
+
*/
|
|
464
|
+
export function toonSizeReport(value: ToonValue): ToonSizeReport {
|
|
465
|
+
const compactJson = JSON.stringify(value)
|
|
466
|
+
const toon = toonEncode(value)
|
|
467
|
+
const ratio = compactJson.length > 0 ? toon.length / compactJson.length : 1
|
|
468
|
+
const jsonTokens = charsToTokens(compactJson.length)
|
|
469
|
+
const toonTokens = charsToTokens(toon.length)
|
|
470
|
+
const tokenRatio = jsonTokens > 0 ? toonTokens / jsonTokens : 1
|
|
471
|
+
return {
|
|
472
|
+
jsonChars: compactJson.length,
|
|
473
|
+
toonChars: toon.length,
|
|
474
|
+
ratio: Math.round(ratio * 10000) / 10000,
|
|
475
|
+
savedPct: Math.round((1 - ratio) * 10000) / 100,
|
|
476
|
+
jsonTokens,
|
|
477
|
+
toonTokens,
|
|
478
|
+
tokenRatio: Math.round(tokenRatio * 10000) / 10000,
|
|
479
|
+
tokenSavedPct: Math.round((1 - tokenRatio) * 10000) / 100,
|
|
480
|
+
}
|
|
481
|
+
}
|
|
@@ -44,9 +44,7 @@ function readVersion(pkgPath: string): string | null {
|
|
|
44
44
|
*
|
|
45
45
|
* @param logger - Logging function (defaults to console.warn)
|
|
46
46
|
*/
|
|
47
|
-
export function checkTuiVersionStaleness(
|
|
48
|
-
logger: (msg: string) => void = console.warn,
|
|
49
|
-
): void {
|
|
47
|
+
export function checkTuiVersionStaleness(logger: (msg: string) => void = console.warn): void {
|
|
50
48
|
try {
|
|
51
49
|
const configDir = resolveConfigDir()
|
|
52
50
|
if (!configDir) return // Not initialized — nothing to check
|