pantheon-opencode 1.4.3 → 1.5.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pantheon/code-mode/README.md +93 -0
- package/.pantheon/code-mode/checkpoint-session.py +7 -0
- package/.pantheon/code-mode/checkpoint-session.sh +11 -0
- package/.pantheon/code-mode/checkpoint_session.py +481 -0
- package/.pantheon/code-mode/compress-inline.py +450 -0
- package/.pantheon/code-mode/eval-llm-judge.py +255 -0
- package/.pantheon/code-mode/eval-monte-carlo.py +368 -0
- package/.pantheon/code-mode/eval-run.py +197 -0
- package/.pantheon/code-mode/eval-static.py +321 -0
- package/.pantheon/code-mode/example-sync.sh +5 -0
- package/.pantheon/code-mode/manifest.json +21 -0
- package/.pantheon/code-mode/session-end-save.py +186 -0
- package/.pantheon/code-mode/session-end-save.sh +18 -0
- package/.pantheon/code-mode/test-allowed.sh +5 -0
- package/.pantheon/code-mode/test-auto-continue-checkpoint.sh +179 -0
- package/.pantheon/code-mode/test-env.sh +7 -0
- package/.pantheon/code-mode/test-prlimit.sh +9 -0
- package/.pantheon/code-mode/test-timeout.sh +6 -0
- package/CHANGELOG.md +1026 -0
- package/README.md +292 -1080
- package/README.pt-BR.md +308 -0
- package/bin/pantheon-init.mjs +207 -31
- package/opencode.json +1 -0
- package/package.json +34 -14
- package/plugin.json +15 -0
- package/pyproject.toml +65 -0
- package/scripts/_pantheon_paths.py +48 -7
- package/scripts/ci-validate-yaml.py +7 -2
- package/scripts/code_mode_server.py +451 -44
- package/scripts/cost.mjs +30 -14
- package/scripts/doctor.mjs +306 -129
- package/scripts/generate-preset-docs.mjs +127 -66
- package/scripts/generate-prompts.sh +2 -2
- package/scripts/install/cli-ui.mjs +23 -10
- package/scripts/install/config-migration.mjs +399 -0
- package/scripts/install/health-check.mjs +3 -3
- package/scripts/install/model-picker.mjs +72 -23
- package/scripts/install/opencode-version.mjs +47 -0
- package/scripts/install/opencode.mjs +396 -121
- package/scripts/install/plugin.mjs +89 -49
- package/scripts/install/shared.mjs +55 -159
- package/scripts/install/strings.mjs +129 -0
- package/scripts/install-mcp.mjs +45 -232
- package/scripts/manifest-inventory.mjs +410 -0
- package/scripts/mcp_persistence_server.py +798 -60
- package/scripts/mcp_resources_server.py +144 -40
- package/scripts/memory_mcp_server.py +2 -2
- package/scripts/package-evidence.mjs +242 -0
- package/scripts/probe-context-rehydrate.mjs +240 -0
- package/scripts/probe-pantheon-cost.mjs +313 -0
- package/scripts/recover-zenodo-deposition.mjs +146 -0
- package/scripts/release-beta-version.mjs +56 -60
- package/scripts/sync-artifacts.mjs +169 -0
- package/scripts/sync-tui.mjs +135 -42
- package/scripts/test-opencode-v1-v2-sandbox.sh +862 -0
- package/scripts/uninstall.mjs +298 -954
- package/scripts/validate-zenodo-release.mjs +154 -0
- package/scripts/validation-policy.mjs +7 -7
- package/scripts/version-check.mjs +113 -108
- package/scripts/versioning.mjs +230 -86
- package/scripts/zenodo-artifact.mjs +21 -0
- package/scripts/zenodo-release-state.mjs +53 -0
- package/src/agents/mnemosyne.md +1 -1
- package/src/agents/zeus.md +27 -16
- package/src/mcp/_pantheon_paths.py +48 -7
- package/src/mcp/code_mode_server.py +451 -44
- package/src/mcp/eval_store.py +1 -1
- package/src/mcp/mcp_persistence_server.py +798 -60
- package/src/mcp/mcp_resources_server.py +144 -40
- package/src/mcp/pantheon_vision_server.py +1 -1
- package/src/mcp/toon_codec.py +491 -0
- package/src/pantheon/auto-wake.ts +1 -5
- package/src/pantheon/background-job-board.ts +110 -2
- package/src/pantheon/compaction-handoff.ts +304 -0
- package/src/pantheon/cost-command.ts +356 -69
- package/src/pantheon/delegate-manager.ts +766 -0
- package/src/pantheon/delegation-classifier.ts +307 -0
- package/src/pantheon/delegation-enforce.ts +13 -12
- package/src/pantheon/delegation-finalize.ts +406 -0
- package/src/pantheon/delegation-notify.ts +212 -0
- package/src/pantheon/delegation-toast.ts +73 -0
- package/src/pantheon/delegation.ts +1337 -0
- package/src/pantheon/file-persistence-salvage.ts +64 -0
- package/src/pantheon/file-persistence.ts +194 -45
- package/src/pantheon/goal-loop.ts +2 -2
- package/src/pantheon/goal-store.ts +1 -1
- package/src/pantheon/hashline/tool.ts +2 -2
- package/src/pantheon/model-command.ts +2 -1
- package/src/pantheon/native-probe.ts +234 -0
- package/src/pantheon/native-task-status.ts +139 -0
- package/src/pantheon/permission-eval.ts +284 -0
- package/src/pantheon/provider-capabilities.ts +221 -0
- package/src/pantheon/provider-stream-tracker.ts +182 -0
- package/src/pantheon/shared-board.ts +37 -0
- package/src/pantheon/step-cap.ts +110 -0
- package/src/pantheon/task-result-guard.ts +107 -0
- package/src/pantheon/token-opt.ts +689 -0
- package/src/pantheon/tool-ceiling.ts +159 -0
- package/src/pantheon/toon-codec.ts +481 -0
- package/src/pantheon/tui-version-check.ts +1 -3
- package/src/pantheon/v2-bridge.ts +129 -0
- package/src/pantheon/v2-events.ts +148 -0
- package/src/pantheon/v2-hooks.ts +287 -0
- package/src/pantheon/v2-tool-definitions.ts +331 -0
- package/src/pantheon/zeus-delegate-with-retry.ts +406 -0
- package/src/plugin-v2/index.ts +21 -0
- package/src/plugin-v2.ts +824 -0
- package/src/plugin.ts +261 -51
- package/src/plugins/pantheon-hooks.ts +51 -0
- package/src/plugins/tui/dist/tui.d.ts +116 -24
- package/src/plugins/tui/dist/tui.d.ts.map +1 -1
- package/src/plugins/tui/dist/tui.js +416 -149
- package/src/plugins/tui/dist/tui.tsx +456 -87
- package/src/plugins/tui/package-lock.json +51 -51
- package/src/plugins/tui/package.json +7 -7
- package/src/plugins/tui/src/index.tsx +462 -95
- package/src/skills/README.md +0 -2
- package/src/pantheon/tool-context.ts +0 -41
|
@@ -0,0 +1,491 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""WS3 TOON codec (PR #94) — Token-Oriented Object Notation.
|
|
3
|
+
|
|
4
|
+
Minimal deterministic encoding for board signals, checkpoints and KV
|
|
5
|
+
payloads. Semantics identical to JSON; structurally smaller (no braces,
|
|
6
|
+
no per-key quotes, no commas); JSON fallback when the parser is
|
|
7
|
+
absent/fails.
|
|
8
|
+
|
|
9
|
+
Only the standard library (``json`` + ``re``) — prohibited: external
|
|
10
|
+
gateway, generative embedding.
|
|
11
|
+
|
|
12
|
+
Encoding rules (mirrored byte-for-byte by ``src/pantheon/toon-codec.ts``):
|
|
13
|
+
|
|
14
|
+
- Scalars: ``None``/``True``/``False`` → ``null``/``true``/``false``;
|
|
15
|
+
numbers → shortest form; strings bare when unambiguous, else
|
|
16
|
+
JSON-quoted (quotes/backslash/newline, leading/trailing space,
|
|
17
|
+
``null``/``true``/``false``/numeric lookalikes, ``{}``/``[]``).
|
|
18
|
+
- Dicts: one ``key: value`` line each; nested values on deeper indent
|
|
19
|
+
(2 spaces); empty dict → ``{}``.
|
|
20
|
+
- Lists of scalars: ``- item`` lines; empty list → ``[]``.
|
|
21
|
+
- Lists of dicts with uniform keys: ``@table k1|k2`` header + ``v1|v2``
|
|
22
|
+
rows (savings are per payload class — ~11% on content-dominated board
|
|
23
|
+
records up to ~47% on large tabular checkpoints — see
|
|
24
|
+
``docs/ws3-token-opt-measurements.md`` for the reproducible table).
|
|
25
|
+
- Dict values never need quoting for ``:`` (split is on the FIRST colon);
|
|
26
|
+
list items / top-level scalars that look like ``key: ...`` are quoted.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import json
|
|
32
|
+
import re
|
|
33
|
+
from dataclasses import dataclass
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
TABLE_MARKER = "@table "
|
|
37
|
+
|
|
38
|
+
#: Default cap on nested-block depth accepted by ``toon_decode`` (DoS guard:
|
|
39
|
+
#: hostile deeply-nested input fails with ``ValueError``, never RecursionError).
|
|
40
|
+
TOON_MAX_DEPTH = 100
|
|
41
|
+
|
|
42
|
+
#: Default cap on input length accepted by ``toon_decode`` (DoS guard).
|
|
43
|
+
TOON_MAX_CHARS = 1_000_000
|
|
44
|
+
|
|
45
|
+
_BARE_KEY_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.@+~/-]*$")
|
|
46
|
+
_INT_RE = re.compile(r"^-?\d+$")
|
|
47
|
+
_FLOAT_RE = re.compile(r"^-?(?:\d+\.\d+|\d+\.\d*[eE][+-]?\d+|\d+[eE][+-]?\d+)$")
|
|
48
|
+
_KEYLIKE_RE = re.compile(
|
|
49
|
+
r'^(?:"(?:[^"\\]|\\.)*"|[A-Za-z0-9_][A-Za-z0-9_.@+~/\-]*)\s*:(?:\s+|$)'
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
_RESERVED_WORDS = frozenset({"null", "true", "false", "{}", "[]"})
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _looks_numeric(text: str) -> bool:
|
|
56
|
+
return bool(_INT_RE.match(text) or _FLOAT_RE.match(text))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _quote(text: str) -> str:
|
|
60
|
+
return json.dumps(text, ensure_ascii=False)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _bare_value(text: str, *, in_table: bool = False) -> bool:
|
|
64
|
+
"""Whether a string value is unambiguous without quotes in VALUE position.
|
|
65
|
+
|
|
66
|
+
In table cells (``in_table=True``) a bare ``|`` would split the row, so
|
|
67
|
+
values containing ``|`` must be quoted.
|
|
68
|
+
"""
|
|
69
|
+
if not text or text != text.strip():
|
|
70
|
+
return False
|
|
71
|
+
if text in _RESERVED_WORDS or _looks_numeric(text):
|
|
72
|
+
return False
|
|
73
|
+
if in_table and "|" in text:
|
|
74
|
+
return False
|
|
75
|
+
return not ('"' in text or "\\" in text or "\n" in text or "\r" in text)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _encode_scalar(
|
|
79
|
+
value: Any, *, item_position: bool = False, table_cell: bool = False
|
|
80
|
+
) -> str:
|
|
81
|
+
if value is None:
|
|
82
|
+
return "null"
|
|
83
|
+
if isinstance(value, bool):
|
|
84
|
+
return "true" if value else "false"
|
|
85
|
+
if isinstance(value, int):
|
|
86
|
+
return str(value)
|
|
87
|
+
if isinstance(value, float):
|
|
88
|
+
return repr(value)
|
|
89
|
+
text = str(value)
|
|
90
|
+
if _bare_value(text, in_table=table_cell) and not (
|
|
91
|
+
item_position and _KEYLIKE_RE.match(text)
|
|
92
|
+
):
|
|
93
|
+
# "- key: ..." reads as a dict item — quote to keep the string.
|
|
94
|
+
return text
|
|
95
|
+
return _quote(text)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _encode_key(key: str) -> str:
|
|
99
|
+
if _BARE_KEY_RE.match(key):
|
|
100
|
+
return key
|
|
101
|
+
return _quote(key)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _is_uniform_dict_list(value: Any) -> bool:
|
|
105
|
+
if not isinstance(value, list) or not value:
|
|
106
|
+
return False
|
|
107
|
+
if not all(isinstance(item, dict) for item in value):
|
|
108
|
+
return False
|
|
109
|
+
first_keys = list(value[0].keys())
|
|
110
|
+
if not first_keys:
|
|
111
|
+
return False
|
|
112
|
+
return all(list(item.keys()) == first_keys for item in value)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _split_cells(line: str) -> list[str]:
|
|
116
|
+
"""Split a table row on ``|`` outside double quotes."""
|
|
117
|
+
cells: list[str] = []
|
|
118
|
+
current: list[str] = []
|
|
119
|
+
in_quotes = False
|
|
120
|
+
escaped = False
|
|
121
|
+
for char in line:
|
|
122
|
+
if escaped:
|
|
123
|
+
current.append(char)
|
|
124
|
+
escaped = False
|
|
125
|
+
elif char == "\\" and in_quotes:
|
|
126
|
+
current.append(char)
|
|
127
|
+
escaped = True
|
|
128
|
+
elif char == '"':
|
|
129
|
+
current.append(char)
|
|
130
|
+
in_quotes = not in_quotes
|
|
131
|
+
elif char == "|" and not in_quotes:
|
|
132
|
+
cells.append("".join(current))
|
|
133
|
+
current = []
|
|
134
|
+
else:
|
|
135
|
+
current.append(char)
|
|
136
|
+
cells.append("".join(current))
|
|
137
|
+
return cells
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def toon_encode(value: Any) -> str:
|
|
141
|
+
"""Encode ``value`` to a TOON string (deterministic, no spaces wasted)."""
|
|
142
|
+
return "\n".join(_encode_value(value, 0))
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _encode_value(value: Any, indent: int) -> list[str]:
|
|
146
|
+
pad = " " * indent
|
|
147
|
+
if value is None or isinstance(value, (bool, int, float, str)):
|
|
148
|
+
return [pad + _encode_scalar(value)]
|
|
149
|
+
if isinstance(value, dict):
|
|
150
|
+
return _encode_dict(value, pad, indent)
|
|
151
|
+
if isinstance(value, list):
|
|
152
|
+
return _encode_list(value, pad, indent)
|
|
153
|
+
raise TypeError(f"TOON cannot encode {type(value).__name__}")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _encode_dict(value: dict[Any, Any], pad: str, indent: int) -> list[str]:
|
|
157
|
+
if not value:
|
|
158
|
+
return [pad + "{}"]
|
|
159
|
+
lines: list[str] = []
|
|
160
|
+
for key, item in value.items():
|
|
161
|
+
encoded_key = _encode_key(str(key))
|
|
162
|
+
if item is None or isinstance(item, (bool, int, float, str)):
|
|
163
|
+
lines.append(f"{pad}{encoded_key}: {_encode_scalar(item)}")
|
|
164
|
+
elif isinstance(item, dict) and not item:
|
|
165
|
+
lines.append(f"{pad}{encoded_key}: {{}}")
|
|
166
|
+
elif isinstance(item, list) and not item:
|
|
167
|
+
lines.append(f"{pad}{encoded_key}: []")
|
|
168
|
+
else:
|
|
169
|
+
lines.append(f"{pad}{encoded_key}:")
|
|
170
|
+
lines.extend(_encode_value(item, indent + 2))
|
|
171
|
+
return lines
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _encode_list(value: list[Any], pad: str, indent: int) -> list[str]:
|
|
175
|
+
if not value:
|
|
176
|
+
return [pad + "[]"]
|
|
177
|
+
if _is_uniform_dict_list(value):
|
|
178
|
+
return _encode_table(value, pad)
|
|
179
|
+
lines: list[str] = []
|
|
180
|
+
for item in value:
|
|
181
|
+
lines.extend(_encode_item(item, indent))
|
|
182
|
+
return lines
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _encode_table(value: list[Any], pad: str) -> list[str]:
|
|
186
|
+
first = value[0]
|
|
187
|
+
assert isinstance(first, dict)
|
|
188
|
+
keys = list(first.keys())
|
|
189
|
+
lines = [pad + TABLE_MARKER + "|".join(_encode_key(str(k)) for k in keys)]
|
|
190
|
+
for item in value:
|
|
191
|
+
assert isinstance(item, dict)
|
|
192
|
+
cells = [
|
|
193
|
+
_encode_scalar(item[k], item_position=True, table_cell=True) for k in keys
|
|
194
|
+
]
|
|
195
|
+
lines.append(pad + "|".join(cells))
|
|
196
|
+
return lines
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _encode_item(item: Any, indent: int) -> list[str]:
|
|
200
|
+
pad = " " * indent
|
|
201
|
+
if item is None or isinstance(item, (bool, int, float, str)):
|
|
202
|
+
return [f"{pad}- {_encode_scalar(item, item_position=True)}"]
|
|
203
|
+
if isinstance(item, dict) and not item:
|
|
204
|
+
return [f"{pad}- {{}}"]
|
|
205
|
+
if isinstance(item, list) and not item:
|
|
206
|
+
return [f"{pad}- []"]
|
|
207
|
+
sub = _encode_value(item, indent + 2)
|
|
208
|
+
first = sub[0].strip()
|
|
209
|
+
return [f"{pad}- {first}", *sub[1:]]
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _parse_scalar(token: str) -> Any:
|
|
213
|
+
if token == "null":
|
|
214
|
+
return None
|
|
215
|
+
if token in {"true", "false"}:
|
|
216
|
+
return token == "true"
|
|
217
|
+
if token in {"{}", "[]"}:
|
|
218
|
+
return {} if token == "{}" else []
|
|
219
|
+
if _looks_numeric(token):
|
|
220
|
+
return int(token) if _INT_RE.match(token) else float(token)
|
|
221
|
+
if token.startswith('"'):
|
|
222
|
+
return json.loads(token)
|
|
223
|
+
return token
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _indent_of(line: str) -> int:
|
|
227
|
+
return len(line) - len(line.lstrip(" "))
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _split_key_value(text: str) -> tuple[str, str]:
|
|
231
|
+
"""Split ``key: value`` on the FIRST colon (values may contain colons)."""
|
|
232
|
+
if text.startswith('"'):
|
|
233
|
+
try:
|
|
234
|
+
key, end = json.JSONDecoder().raw_decode(text)
|
|
235
|
+
except json.JSONDecodeError as exc:
|
|
236
|
+
raise ValueError(f"TOON: line without ':' separator: {text!r}") from exc
|
|
237
|
+
rest_text = text[end:].lstrip()
|
|
238
|
+
if not rest_text.startswith(":"):
|
|
239
|
+
raise ValueError(f"TOON: line without ':' separator: {text!r}")
|
|
240
|
+
if not isinstance(key, str) or not key:
|
|
241
|
+
raise ValueError(f"TOON: empty key in line: {text!r}")
|
|
242
|
+
return key, rest_text[1:].strip()
|
|
243
|
+
idx = text.find(":")
|
|
244
|
+
if idx < 0:
|
|
245
|
+
raise ValueError(f"TOON: line without ':' separator: {text!r}")
|
|
246
|
+
raw_key = text[:idx].strip()
|
|
247
|
+
if not raw_key:
|
|
248
|
+
raise ValueError(f"TOON: empty key in line: {text!r}")
|
|
249
|
+
key = raw_key
|
|
250
|
+
rest = text[idx + 1 :].strip()
|
|
251
|
+
return key, rest
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def toon_decode(
|
|
255
|
+
text: str, *, max_depth: int = TOON_MAX_DEPTH, max_chars: int = TOON_MAX_CHARS
|
|
256
|
+
) -> Any:
|
|
257
|
+
"""Decode a TOON string back to the identical value (raises ValueError).
|
|
258
|
+
|
|
259
|
+
``max_depth``/``max_chars`` are DoS guards: oversized inputs fail with a
|
|
260
|
+
controlled ``ValueError`` instead of ``RecursionError``/memory blowup.
|
|
261
|
+
"""
|
|
262
|
+
if len(text) > max_chars:
|
|
263
|
+
raise ValueError(
|
|
264
|
+
f"TOON: input too large ({len(text)} chars > max_chars={max_chars})"
|
|
265
|
+
)
|
|
266
|
+
lines = [line for line in text.split("\n") if line.strip() != ""]
|
|
267
|
+
if not lines:
|
|
268
|
+
raise ValueError("TOON: empty input")
|
|
269
|
+
stripped = lines[0].strip()
|
|
270
|
+
if stripped.startswith("{") or stripped.startswith("["):
|
|
271
|
+
try:
|
|
272
|
+
return json.loads(text)
|
|
273
|
+
except json.JSONDecodeError as exc:
|
|
274
|
+
raise ValueError(f"TOON: invalid JSON fallback: {exc}") from exc
|
|
275
|
+
value, next_idx = _parse_block(lines, 0, 0, 0, max_depth)
|
|
276
|
+
if next_idx != len(lines):
|
|
277
|
+
raise ValueError(f"TOON: trailing content at line {next_idx + 1}")
|
|
278
|
+
return value
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _parse_block(
|
|
282
|
+
lines: list[str], idx: int, indent: int, depth: int, max_depth: int
|
|
283
|
+
) -> tuple[Any, int]:
|
|
284
|
+
if depth > max_depth:
|
|
285
|
+
raise ValueError(f"TOON: max_depth={max_depth} exceeded")
|
|
286
|
+
stripped = lines[idx].strip()
|
|
287
|
+
if stripped.startswith("@table "):
|
|
288
|
+
return _parse_table(lines, idx, indent)
|
|
289
|
+
if stripped.startswith("- ") or stripped == "-":
|
|
290
|
+
return _parse_list(lines, idx, indent, depth, max_depth)
|
|
291
|
+
return _parse_dict(lines, idx, indent, depth, max_depth)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _parse_table(
|
|
295
|
+
lines: list[str], idx: int, indent: int
|
|
296
|
+
) -> tuple[list[dict[str, Any]], int]:
|
|
297
|
+
header = lines[idx].strip()[len(TABLE_MARKER) :]
|
|
298
|
+
keys = [
|
|
299
|
+
_parse_scalar(cell) if cell.startswith('"') else cell
|
|
300
|
+
for cell in _split_cells(header)
|
|
301
|
+
]
|
|
302
|
+
if not keys or any(not k for k in keys):
|
|
303
|
+
raise ValueError(f"TOON: bad @table header at line {idx + 1}")
|
|
304
|
+
rows: list[dict[str, Any]] = []
|
|
305
|
+
idx += 1
|
|
306
|
+
while idx < len(lines) and _indent_of(lines[idx]) == indent:
|
|
307
|
+
row_stripped = lines[idx].strip()
|
|
308
|
+
if row_stripped.startswith(("@table ", "- ")) or row_stripped == "-":
|
|
309
|
+
break
|
|
310
|
+
if ":" in row_stripped and _KEYLIKE_RE.match(row_stripped):
|
|
311
|
+
break
|
|
312
|
+
cells = _split_cells(row_stripped)
|
|
313
|
+
if len(cells) != len(keys):
|
|
314
|
+
raise ValueError(f"TOON: row width mismatch at line {idx + 1}")
|
|
315
|
+
rows.append({k: _parse_scalar(c) for k, c in zip(keys, cells, strict=True)})
|
|
316
|
+
idx += 1
|
|
317
|
+
return rows, idx
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _parse_dict(
|
|
321
|
+
lines: list[str], idx: int, indent: int, depth: int, max_depth: int
|
|
322
|
+
) -> tuple[dict[str, Any], int]:
|
|
323
|
+
result: dict[str, Any] = {}
|
|
324
|
+
while idx < len(lines):
|
|
325
|
+
line = lines[idx]
|
|
326
|
+
if line.strip() == "" or _indent_of(line) != indent:
|
|
327
|
+
break
|
|
328
|
+
stripped = line.strip()
|
|
329
|
+
if stripped.startswith(("@table ", "- ")) or stripped == "-":
|
|
330
|
+
break
|
|
331
|
+
key, rest = _split_key_value(stripped)
|
|
332
|
+
if rest == "":
|
|
333
|
+
idx += 1
|
|
334
|
+
if idx >= len(lines) or _indent_of(lines[idx]) <= indent:
|
|
335
|
+
raise ValueError(f"TOON: missing nested block for key {key!r}")
|
|
336
|
+
nested, idx = _parse_block(
|
|
337
|
+
lines, idx, _indent_of(lines[idx]), depth + 1, max_depth
|
|
338
|
+
)
|
|
339
|
+
result[key] = nested
|
|
340
|
+
else:
|
|
341
|
+
result[key] = _parse_scalar(rest)
|
|
342
|
+
idx += 1
|
|
343
|
+
return result, idx
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
@dataclass
|
|
347
|
+
class _ListParseContext:
|
|
348
|
+
"""Shared cursor for ``- key: value`` list-item parsing (PLR0913)."""
|
|
349
|
+
|
|
350
|
+
lines: list[str]
|
|
351
|
+
indent: int
|
|
352
|
+
depth: int
|
|
353
|
+
max_depth: int
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _parse_list_continuation(
|
|
357
|
+
ctx: _ListParseContext, idx: int, item: dict[str, Any]
|
|
358
|
+
) -> int:
|
|
359
|
+
"""Consume ``indent+`` continuation lines (``key: value``) of a dict item."""
|
|
360
|
+
lines, indent, depth, max_depth = ctx.lines, ctx.indent, ctx.depth, ctx.max_depth
|
|
361
|
+
while idx < len(lines) and _indent_of(lines[idx]) > indent:
|
|
362
|
+
cont = lines[idx].strip()
|
|
363
|
+
sub_key, sub_rest = _split_key_value(cont)
|
|
364
|
+
if sub_rest == "":
|
|
365
|
+
idx += 1
|
|
366
|
+
if idx >= len(lines) or _indent_of(lines[idx]) <= _indent_of(
|
|
367
|
+
lines[idx - 1]
|
|
368
|
+
):
|
|
369
|
+
raise ValueError(f"TOON: missing nested block for key {sub_key!r}")
|
|
370
|
+
nested, idx = _parse_block(
|
|
371
|
+
lines, idx, _indent_of(lines[idx]), depth + 1, max_depth
|
|
372
|
+
)
|
|
373
|
+
item[sub_key] = nested
|
|
374
|
+
else:
|
|
375
|
+
item[sub_key] = _parse_scalar(sub_rest)
|
|
376
|
+
idx += 1
|
|
377
|
+
return idx
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _parse_list_dict_item(
|
|
381
|
+
ctx: _ListParseContext, idx: int, rest: str
|
|
382
|
+
) -> tuple[dict[str, Any], int]:
|
|
383
|
+
"""Parse a ``- key: value`` dict item plus its continuation lines."""
|
|
384
|
+
lines, indent, depth, max_depth = ctx.lines, ctx.indent, ctx.depth, ctx.max_depth
|
|
385
|
+
key, sub_rest = _split_key_value(rest)
|
|
386
|
+
item: dict[str, Any] = {}
|
|
387
|
+
if sub_rest == "":
|
|
388
|
+
idx += 1
|
|
389
|
+
if idx >= len(lines) or _indent_of(lines[idx]) <= indent:
|
|
390
|
+
raise ValueError(f"TOON: missing nested block for key {key!r}")
|
|
391
|
+
nested, idx = _parse_block(
|
|
392
|
+
lines, idx, _indent_of(lines[idx]), depth + 1, max_depth
|
|
393
|
+
)
|
|
394
|
+
item[key] = nested
|
|
395
|
+
else:
|
|
396
|
+
item[key] = _parse_scalar(sub_rest)
|
|
397
|
+
idx += 1
|
|
398
|
+
idx = _parse_list_continuation(ctx, idx, item)
|
|
399
|
+
return item, idx
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _parse_list(
|
|
403
|
+
lines: list[str], idx: int, indent: int, depth: int, max_depth: int
|
|
404
|
+
) -> tuple[list[Any], int]:
|
|
405
|
+
items: list[Any] = []
|
|
406
|
+
while idx < len(lines):
|
|
407
|
+
line = lines[idx]
|
|
408
|
+
if _indent_of(line) != indent:
|
|
409
|
+
break
|
|
410
|
+
stripped = line.strip()
|
|
411
|
+
if not (stripped.startswith("- ") or stripped == "-"):
|
|
412
|
+
break
|
|
413
|
+
rest = stripped[1:].strip()
|
|
414
|
+
if rest == "":
|
|
415
|
+
idx += 1
|
|
416
|
+
if idx >= len(lines) or _indent_of(lines[idx]) <= indent:
|
|
417
|
+
raise ValueError(
|
|
418
|
+
f"TOON: missing nested block in list at line {idx + 1}"
|
|
419
|
+
)
|
|
420
|
+
nested, idx = _parse_block(
|
|
421
|
+
lines, idx, _indent_of(lines[idx]), depth + 1, max_depth
|
|
422
|
+
)
|
|
423
|
+
items.append(nested)
|
|
424
|
+
continue
|
|
425
|
+
if _KEYLIKE_RE.match(rest):
|
|
426
|
+
ctx = _ListParseContext(lines, indent, depth, max_depth)
|
|
427
|
+
item, idx = _parse_list_dict_item(ctx, idx, rest)
|
|
428
|
+
items.append(item)
|
|
429
|
+
else:
|
|
430
|
+
items.append(_parse_scalar(rest))
|
|
431
|
+
idx += 1
|
|
432
|
+
return items, idx
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def toon_decode_auto(
|
|
436
|
+
text: str, *, max_depth: int = TOON_MAX_DEPTH, max_chars: int = TOON_MAX_CHARS
|
|
437
|
+
) -> Any:
|
|
438
|
+
"""Decode TOON, falling back to JSON (mirrors "parser absent" mode).
|
|
439
|
+
|
|
440
|
+
Raises ValueError with a consistent ``TOON:`` message when the input is
|
|
441
|
+
neither valid TOON nor valid JSON (never leaks the raw JSON error).
|
|
442
|
+
"""
|
|
443
|
+
stripped = text.strip()
|
|
444
|
+
if stripped.startswith("{") or stripped.startswith("["):
|
|
445
|
+
try:
|
|
446
|
+
return json.loads(text)
|
|
447
|
+
except json.JSONDecodeError as exc:
|
|
448
|
+
raise ValueError(f"TOON: invalid JSON fallback: {exc}") from exc
|
|
449
|
+
try:
|
|
450
|
+
return toon_decode(text, max_depth=max_depth, max_chars=max_chars)
|
|
451
|
+
except ValueError:
|
|
452
|
+
pass
|
|
453
|
+
try:
|
|
454
|
+
return json.loads(text)
|
|
455
|
+
except json.JSONDecodeError as exc:
|
|
456
|
+
raise ValueError(
|
|
457
|
+
f"TOON: invalid input (neither TOON nor JSON): {stripped[:60]!r}"
|
|
458
|
+
) from exc
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _chars_to_tokens(chars: int) -> int:
|
|
462
|
+
"""Deterministic token estimate: ``ceil(chars / 4)`` (WS3 metering basis)."""
|
|
463
|
+
if chars <= 0:
|
|
464
|
+
return 0
|
|
465
|
+
return -(-chars // 4)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def toon_size_report(value: Any) -> dict[str, Any]:
|
|
469
|
+
"""Compare JSON vs TOON sizes for ``value``.
|
|
470
|
+
|
|
471
|
+
Reports BOTH chars and tokens: token ratios on tiny payloads suffer
|
|
472
|
+
ceiling bias (``ceil`` rounds 161→41 vs 143→36), so the char columns are
|
|
473
|
+
the honest basis and the token columns show what metering will debit.
|
|
474
|
+
Per-class numbers: see ``docs/ws3-token-opt-measurements.md``.
|
|
475
|
+
"""
|
|
476
|
+
compact_json = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
477
|
+
toon = toon_encode(value)
|
|
478
|
+
ratio = len(toon) / len(compact_json) if compact_json else 1.0
|
|
479
|
+
json_tokens = _chars_to_tokens(len(compact_json))
|
|
480
|
+
toon_tokens = _chars_to_tokens(len(toon))
|
|
481
|
+
token_ratio = toon_tokens / json_tokens if json_tokens else 1.0
|
|
482
|
+
return {
|
|
483
|
+
"json_chars": len(compact_json),
|
|
484
|
+
"toon_chars": len(toon),
|
|
485
|
+
"ratio": round(ratio, 4),
|
|
486
|
+
"saved_pct": round((1.0 - ratio) * 100.0, 2),
|
|
487
|
+
"json_tokens": json_tokens,
|
|
488
|
+
"toon_tokens": toon_tokens,
|
|
489
|
+
"token_ratio": round(token_ratio, 4),
|
|
490
|
+
"token_saved_pct": round((1.0 - token_ratio) * 100.0, 2),
|
|
491
|
+
}
|
|
@@ -78,11 +78,7 @@ export async function consumeWakeSignals(signalDir?: string): Promise<WakeSignal
|
|
|
78
78
|
|
|
79
79
|
// Step 3: Only add to results after successful rename
|
|
80
80
|
signals.push(data)
|
|
81
|
-
} catch {
|
|
82
|
-
// Rename failed (another consumer got there first) or file is corrupted.
|
|
83
|
-
// Skip silently in either case.
|
|
84
|
-
continue
|
|
85
|
-
}
|
|
81
|
+
} catch {}
|
|
86
82
|
}
|
|
87
83
|
|
|
88
84
|
return signals
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* @module background-job-board
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { mkdir, rename, writeFile } from 'node:fs/promises'
|
|
13
|
+
import { mkdir, rename, unlink, writeFile } from 'node:fs/promises'
|
|
14
14
|
import { dirname, join } from 'node:path'
|
|
15
15
|
|
|
16
16
|
import { createPantheonLogger } from './logger.ts'
|
|
@@ -200,6 +200,16 @@ export class BackgroundJobBoard {
|
|
|
200
200
|
|
|
201
201
|
// ─── Persistence ────────────────────────────────────────────────────
|
|
202
202
|
|
|
203
|
+
/** Maximum concurrent jobs per agent (mirrors the constructor option). */
|
|
204
|
+
get maxConcurrentPerAgent(): number {
|
|
205
|
+
return this.options.maxConcurrentPerAgent
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Total entries currently held (running + terminal + reconciled). */
|
|
209
|
+
size(): number {
|
|
210
|
+
return this.jobs.size
|
|
211
|
+
}
|
|
212
|
+
|
|
203
213
|
/** Inject a persistence adapter. Must be called before `recoverRunningJobs()`. */
|
|
204
214
|
setPersistence(adapter: PersistenceAdapter): void {
|
|
205
215
|
this.persistence = adapter
|
|
@@ -212,15 +222,28 @@ export class BackgroundJobBoard {
|
|
|
212
222
|
async recoverRunningJobs(): Promise<void> {
|
|
213
223
|
if (!this.persistence) return
|
|
214
224
|
const records = await this.persistence.loadAllJobs()
|
|
225
|
+
const recoveredIDs: string[] = []
|
|
215
226
|
for (const record of records) {
|
|
216
227
|
if (record.state === 'running') {
|
|
217
228
|
record.state = 'error'
|
|
218
229
|
record.lastStatusError = 'Process restarted — job marked as errored'
|
|
219
230
|
record.updatedAt = Date.now()
|
|
231
|
+
record.completedAt = Date.now()
|
|
220
232
|
record.totalErrors++
|
|
233
|
+
record.terminalUnreconciled = true
|
|
234
|
+
recoveredIDs.push(record.taskID)
|
|
221
235
|
}
|
|
222
236
|
this.jobs.set(record.taskID, record)
|
|
223
237
|
}
|
|
238
|
+
// Persist + signal + notify for each recovered job (write-ahead-log pattern)
|
|
239
|
+
for (const taskID of recoveredIDs) {
|
|
240
|
+
const job = this.jobs.get(taskID)
|
|
241
|
+
if (job) {
|
|
242
|
+
await this.persistRecord(job)
|
|
243
|
+
await this.writeSignal(job)
|
|
244
|
+
this.notifyTerminal(job.taskID)
|
|
245
|
+
}
|
|
246
|
+
}
|
|
224
247
|
}
|
|
225
248
|
|
|
226
249
|
// ─── Core Operations ────────────────────────────────────────────────
|
|
@@ -230,6 +253,16 @@ export class BackgroundJobBoard {
|
|
|
230
253
|
* The job starts in the `running` state with a generated alias.
|
|
231
254
|
*/
|
|
232
255
|
async registerLaunch(input: LaunchInput): Promise<BackgroundJobRecord> {
|
|
256
|
+
// Atomic concurrency check: validate BEFORE any mutation or await.
|
|
257
|
+
// JavaScript's single-threaded event loop guarantees this check + set
|
|
258
|
+
// cannot be interleaved with another registerLaunch call.
|
|
259
|
+
if (!this.canDispatch(input.agent)) {
|
|
260
|
+
throw new Error(
|
|
261
|
+
`Concurrency limit reached for agent "${input.agent}" ` +
|
|
262
|
+
`(${this.getRunningCount(input.agent)} running, max ${this.options.maxConcurrentPerAgent})`,
|
|
263
|
+
)
|
|
264
|
+
}
|
|
265
|
+
|
|
233
266
|
const prefix = getAgentPrefix(input.agent)
|
|
234
267
|
let sessionCounters = this.aliasCounters.get(input.parentSessionID)
|
|
235
268
|
if (!sessionCounters) {
|
|
@@ -262,6 +295,20 @@ export class BackgroundJobBoard {
|
|
|
262
295
|
return record
|
|
263
296
|
}
|
|
264
297
|
|
|
298
|
+
/**
|
|
299
|
+
* Register a launch only when taskID has not already been claimed.
|
|
300
|
+
*
|
|
301
|
+
* The map check and set happen before the first await, so JavaScript's
|
|
302
|
+
* single-threaded event loop makes this critical section atomic even when
|
|
303
|
+
* two event handlers call it concurrently. The existing record is returned
|
|
304
|
+
* for the losing caller.
|
|
305
|
+
*/
|
|
306
|
+
async registerLaunchIfAbsent(input: LaunchInput): Promise<BackgroundJobRecord> {
|
|
307
|
+
const existing = this.jobs.get(input.taskID)
|
|
308
|
+
if (existing !== undefined) return existing
|
|
309
|
+
return this.registerLaunch(input)
|
|
310
|
+
}
|
|
311
|
+
|
|
265
312
|
/**
|
|
266
313
|
* Update a job's state to a terminal state.
|
|
267
314
|
*
|
|
@@ -410,7 +457,12 @@ export class BackgroundJobBoard {
|
|
|
410
457
|
job.terminalUnreconciled = false
|
|
411
458
|
|
|
412
459
|
await this.persistRecord(job)
|
|
413
|
-
|
|
460
|
+
// P1-1: reconcile is an acknowledgment, NOT a new terminal transition
|
|
461
|
+
// (onTerminal does not re-fire). Re-writing a `.signal.json` here leaked
|
|
462
|
+
// orphan signals with `state:"reconciled"` after the finalize→reconcile
|
|
463
|
+
// cycle deleted them. Instead, remove any still-pending wake signal so
|
|
464
|
+
// reconcile leaves no `<alias>.signal.json` behind.
|
|
465
|
+
await this.deleteSignal(job.alias)
|
|
414
466
|
|
|
415
467
|
return job
|
|
416
468
|
}
|
|
@@ -599,6 +651,62 @@ export class BackgroundJobBoard {
|
|
|
599
651
|
}
|
|
600
652
|
}
|
|
601
653
|
|
|
654
|
+
/**
|
|
655
|
+
* Delete the auto-wake signal file for an alias (`<alias>.signal.json`).
|
|
656
|
+
* No-op (false) when no signalDir is configured or the file is absent.
|
|
657
|
+
* The delegate manager calls this on the spot right after auto-reconcile.
|
|
658
|
+
*/
|
|
659
|
+
async deleteSignal(alias: string): Promise<boolean> {
|
|
660
|
+
const signalDir = this.options.signalDir
|
|
661
|
+
if (!signalDir) return false
|
|
662
|
+
const signalPath = join(signalDir, `${alias}.signal.json`)
|
|
663
|
+
try {
|
|
664
|
+
await unlink(signalPath)
|
|
665
|
+
return true
|
|
666
|
+
} catch {
|
|
667
|
+
return false
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/**
|
|
672
|
+
* Aggressive prune of finished jobs: keep only the `keepNewest` most
|
|
673
|
+
* recently updated completed/error/cancelled/reconciled jobs, evict the
|
|
674
|
+
* rest (memory + persistence). Running jobs are never touched.
|
|
675
|
+
* Returns the number of evicted jobs.
|
|
676
|
+
*/
|
|
677
|
+
async pruneCompleted(keepNewest: number): Promise<number> {
|
|
678
|
+
const done = Array.from(this.jobs.values())
|
|
679
|
+
.filter((j) => TERMINAL_STATES.has(j.state) || j.state === 'reconciled')
|
|
680
|
+
.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
681
|
+
const evict = done.slice(Math.max(0, keepNewest))
|
|
682
|
+
for (const job of evict) {
|
|
683
|
+
this.jobs.delete(job.taskID)
|
|
684
|
+
await this.deletePersisted(job.taskID)
|
|
685
|
+
}
|
|
686
|
+
return evict.length
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Hard entry cap: while the board holds more than `maxEntries` jobs,
|
|
691
|
+
* evict the oldest finished job (by updatedAt). Running jobs are never
|
|
692
|
+
* evicted — when only running jobs remain the cap cannot shrink further
|
|
693
|
+
* and the loop stops. Returns the number of evicted jobs.
|
|
694
|
+
*/
|
|
695
|
+
async enforceEntryCap(maxEntries: number): Promise<number> {
|
|
696
|
+
let evicted = 0
|
|
697
|
+
for (;;) {
|
|
698
|
+
if (this.jobs.size <= Math.max(0, maxEntries)) break
|
|
699
|
+
const oldest = Array.from(this.jobs.values())
|
|
700
|
+
.filter((j) => TERMINAL_STATES.has(j.state) || j.state === 'reconciled')
|
|
701
|
+
.sort((a, b) => a.updatedAt - b.updatedAt)[0]
|
|
702
|
+
if (!oldest) break
|
|
703
|
+
this.jobs.delete(oldest.taskID)
|
|
704
|
+
await this.deletePersisted(oldest.taskID)
|
|
705
|
+
evicted += 1
|
|
706
|
+
}
|
|
707
|
+
return evicted
|
|
708
|
+
}
|
|
709
|
+
|
|
602
710
|
// ─── Internal Helpers ────────────────────────────────────────────────
|
|
603
711
|
|
|
604
712
|
private async persistRecord(record: BackgroundJobRecord): Promise<void> {
|