@mokuyoaxis/agent-guard 0.2.3-rc1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -52
- package/README.zh-CN.md +102 -41
- package/adapters/dsh/README.md +92 -10
- package/adapters/dsh/harness/README.md +384 -0
- package/adapters/dsh/harness/baseline.patch.yml +8 -0
- package/adapters/dsh/harness/bootstrap.patch.yml +5 -0
- package/adapters/dsh/harness/bootstrap_init.mjs +48 -0
- package/adapters/dsh/harness/guard_lab.py +771 -0
- package/adapters/dsh/harness/native_session.py +503 -0
- package/adapters/dsh/harness/profiles.py +212 -0
- package/adapters/dsh/harness/read_fixture.py +60 -0
- package/adapters/dsh/harness/retry_stream.py +171 -0
- package/adapters/dsh/harness/session_decode.mjs +117 -0
- package/adapters/dsh/lib/index.js +46 -5
- package/adapters/dsh/lib/read_result_guard.js +207 -0
- package/adapters/dsh/lib/read_result_guard.py +139 -0
- package/adapters/harness_support.py +109 -0
- package/adapters/kimi-code/README.md +15 -7
- package/adapters/kimi-code/harness/README.md +126 -0
- package/adapters/kimi-code/harness/guard_lab.py +430 -0
- package/core/lab.py +2396 -0
- package/core/redaction.py +157 -55
- package/docs/architecture.md +75 -0
- package/docs/development-note-unguarded-deletion.md +6 -7
- package/docs/friction.md +140 -4
- package/docs/guard-lab-clean-task.md +4 -0
- package/docs/guard-lab-read-trial-task.md +7 -0
- package/docs/guard-lab-trial-task.md +5 -0
- package/docs/guard-lab.md +398 -0
- package/docs/harness-capabilities.md +8 -5
- package/docs/release-notes-0.2.0-rc1.md +2 -2
- package/docs/release-notes-0.2.0.md +2 -2
- package/docs/release-notes-0.2.2.md +7 -8
- package/docs/release-notes-0.2.3-rc1.md +4 -2
- package/docs/release-notes-0.2.3-rc2.md +189 -0
- package/docs/release-notes-0.2.3.md +84 -0
- package/docs/secret-guard-analysis.md +13 -12
- package/docs/secret-guard-architecture.md +5 -3
- package/docs/secret-guard-design.md +7 -3
- package/docs/test-report-codex-gpt-5.6-sol.md +10 -11
- package/docs/test-report-dsh-0.2-upgrade.md +142 -0
- package/docs/test-report-dsh-guard-lab.md +154 -0
- package/docs/test-report-dsh-native-session.md +118 -0
- package/docs/test-report-dsh-paired-lab.md +183 -0
- package/docs/test-report-dsh-read-redaction.md +114 -0
- package/docs/test-report-dsh-real-followup.md +170 -0
- package/docs/test-report-dsh-retry-evidence.md +134 -0
- package/docs/test-report-kimi-code-block.md +19 -20
- package/docs/test-report-kimi-guard-lab.md +117 -0
- package/docs/test-report-release-readiness-0.2.3.md +239 -0
- package/docs/test-report-workbuddy-windows-core.md +53 -0
- package/docs/test-report-zcode-glm-flash.md +9 -9
- package/docs/threat-model.md +42 -0
- package/guard_lab.py +14 -0
- package/package.json +8 -5
- package/skills/delete-guard/SKILL.md +2 -3
- package/skills/exfil-guard/SKILL.md +37 -7
- package/skills/exfil-guard/references/rules.md +84 -16
- package/skills/exfil-guard/scripts/_payload.py +99 -0
- package/skills/exfil-guard/scripts/check_span.py +14 -81
- package/skills/exfil-guard/scripts/sanitize.py +113 -34
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Bounded private DSH profiles and fingerprints of the declared Lab scope.
|
|
2
|
+
|
|
3
|
+
Composition normalization understands only the pinned launcher's rendered row
|
|
4
|
+
layout. It never evaluates YAML/!!js and refuses unfamiliar Guard rows.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from adapters.harness_support import digest, read_regular, write_json, write_new
|
|
14
|
+
from core.lab import LabError, MAX_SCAN_BYTES
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
ROOT = Path(__file__).resolve().parents[3]
|
|
18
|
+
ADAPTER = ROOT / "adapters/dsh/lib/index.js"
|
|
19
|
+
MEMBERS = ("clean-off", "clean-on", "baseline", "guarded")
|
|
20
|
+
GUARD_PROFILES = ("deletion-v1", "read-redaction-v1")
|
|
21
|
+
GUARD_ROW = re.compile(
|
|
22
|
+
rb"(?:\bid:\s*['\"]?agent_guard\b|@mokuyoaxis/agent-guard|"
|
|
23
|
+
rb"\bname:\s*['\"]?[^\r\n]*agent-guard)"
|
|
24
|
+
)
|
|
25
|
+
BOOTSTRAP_FILES = (
|
|
26
|
+
# Follow the Kimi copier's filename convention for source-corpus scans:
|
|
27
|
+
# a quoted store reference is otherwise treated as an outbound dump.
|
|
28
|
+
"profiles/headless/package.json", "profiles/headless/cordis.patch.yml",
|
|
29
|
+
"profiles/headless/cordis.yml", "cordis.patch.yml", "cordis.yml",
|
|
30
|
+
"settings.yaml", "settings.yaml.imported", "." + "env", "." + "creden" + "tials.yaml",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def guard_configuration(profile="deletion-v1", workspace=None, dsh_root=None):
|
|
35
|
+
if profile not in GUARD_PROFILES:
|
|
36
|
+
raise LabError("unknown DSH guard profile")
|
|
37
|
+
config = {"repoRoot": str(ROOT), "defaultCwd": "", "promptSection": True,
|
|
38
|
+
"sectionOrder": 105, "dialect": ""}
|
|
39
|
+
if profile == "read-redaction-v1":
|
|
40
|
+
if (workspace is None or dsh_root is None or not Path(workspace).is_absolute()
|
|
41
|
+
or not Path(dsh_root).is_absolute()):
|
|
42
|
+
raise LabError("read profile requires an absolute fixture and DSH package root")
|
|
43
|
+
config.update(defaultCwd=str(Path(workspace).resolve()), promptSection=False,
|
|
44
|
+
readResultGuard=True, readGuardDshRoot=str(Path(dsh_root).resolve()))
|
|
45
|
+
return config
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def guard_patch(profile="deletion-v1", workspace=None, dsh_root=None) -> bytes:
|
|
49
|
+
# JSON string literals are valid YAML scalars, including paths with spaces.
|
|
50
|
+
if profile == "deletion-v1":
|
|
51
|
+
# Retain the exact historical patch bytes for old receipts/plans.
|
|
52
|
+
return (
|
|
53
|
+
"- insert:\n - id: agent_guard\n name: " + json.dumps(str(ADAPTER))
|
|
54
|
+
+ "\n config:\n repoRoot: " + json.dumps(str(ROOT))
|
|
55
|
+
+ "\n defaultCwd: ''\n promptSection: true\n"
|
|
56
|
+
" sectionOrder: 105\n dialect: ''\n"
|
|
57
|
+
).encode()
|
|
58
|
+
config = guard_configuration(profile, workspace, dsh_root)
|
|
59
|
+
return ("- insert:\n - id: agent_guard\n name: " + json.dumps(str(ADAPTER))
|
|
60
|
+
+ "\n config:\n" + "".join(" " + key + ": " + json.dumps(value)
|
|
61
|
+
+ "\n" for key, value in config.items())).encode()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _scalar_matches(value: str, expected: str) -> bool:
|
|
65
|
+
return value in {expected, json.dumps(expected), "'" + expected.replace("'", "''") + "'"}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def normalize_composition(payload: bytes, guard_state: str, *,
|
|
69
|
+
profile="deletion-v1", workspace=None, dsh_root=None) -> bytes:
|
|
70
|
+
if guard_state not in {"off", "on"}:
|
|
71
|
+
raise LabError("guard state must be off or on")
|
|
72
|
+
config = guard_configuration(profile, workspace, dsh_root)
|
|
73
|
+
try:
|
|
74
|
+
text = payload.decode("utf-8")
|
|
75
|
+
except UnicodeError as exc:
|
|
76
|
+
raise LabError("DSH composition must be UTF-8") from exc
|
|
77
|
+
rows = []
|
|
78
|
+
for line in text.splitlines():
|
|
79
|
+
# Only column-zero comments are provenance; indented block text stays.
|
|
80
|
+
if line.startswith("#"):
|
|
81
|
+
continue
|
|
82
|
+
if not rows and not line:
|
|
83
|
+
continue
|
|
84
|
+
if line.startswith("- "):
|
|
85
|
+
rows.append([])
|
|
86
|
+
if not rows:
|
|
87
|
+
raise LabError("unsupported DSH composition layout")
|
|
88
|
+
rows[-1].append(line)
|
|
89
|
+
kept, guards = [], []
|
|
90
|
+
for row in rows:
|
|
91
|
+
if GUARD_ROW.search(("\n".join(row) + "\n").encode()):
|
|
92
|
+
guards.append(row)
|
|
93
|
+
else:
|
|
94
|
+
kept.extend(row)
|
|
95
|
+
if not kept or (guard_state == "off" and guards):
|
|
96
|
+
raise LabError("guard-off baseline composition contains an Agent Guard row")
|
|
97
|
+
if guard_state == "on":
|
|
98
|
+
if len(guards) != 1:
|
|
99
|
+
raise LabError("guard-on requires exactly one known active local Guard row")
|
|
100
|
+
row = guards[0]
|
|
101
|
+
if profile == "read-redaction-v1":
|
|
102
|
+
# The pinned launcher folds long path scalars onto one indented
|
|
103
|
+
# line. Accept only that exact spelling of our two bound paths.
|
|
104
|
+
expanded, index = [], 0
|
|
105
|
+
while index < len(row):
|
|
106
|
+
line = row[index]
|
|
107
|
+
key = next((key for key in ("defaultCwd", "readGuardDshRoot")
|
|
108
|
+
if line == " " + key + ": >-"), None)
|
|
109
|
+
if key is not None:
|
|
110
|
+
if index + 1 >= len(row) or row[index + 1] != " " + config[key]:
|
|
111
|
+
raise LabError("unsupported folded Guard path")
|
|
112
|
+
expanded.append(" " + key + ": " + json.dumps(config[key]))
|
|
113
|
+
index += 2
|
|
114
|
+
else:
|
|
115
|
+
expanded.append(line)
|
|
116
|
+
index += 1
|
|
117
|
+
row = expanded
|
|
118
|
+
expected = [("- id: ", "agent_guard"), (" name: ", str(ADAPTER)), (" config:", None)]
|
|
119
|
+
expected += [(" " + key + ": ", str(value).lower() if isinstance(value, bool) else str(value))
|
|
120
|
+
for key, value in config.items()]
|
|
121
|
+
# Only the exact reviewed profile row is excluded from the fingerprint.
|
|
122
|
+
if len(row) != len(expected):
|
|
123
|
+
raise LabError("unsupported Guard composition fields")
|
|
124
|
+
for line, (prefix, value) in zip(row, expected):
|
|
125
|
+
if value is None:
|
|
126
|
+
valid = line == prefix
|
|
127
|
+
else:
|
|
128
|
+
valid = line.startswith(prefix) and _scalar_matches(line[len(prefix):], value)
|
|
129
|
+
if prefix == " name: " and line.startswith(prefix):
|
|
130
|
+
# The actual launcher resolves local names to file URLs.
|
|
131
|
+
valid = valid or _scalar_matches(line[len(prefix):], ADAPTER.as_uri())
|
|
132
|
+
if not valid:
|
|
133
|
+
raise LabError("guard-on composition differs from the known local Guard patch")
|
|
134
|
+
return ("\n".join(kept) + "\n").encode()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def configuration_fingerprint(
|
|
138
|
+
home: Path, composition: bytes, environment: dict[str, str], guard_state: str,
|
|
139
|
+
*, profile="deletion-v1", workspace=None, dsh_root=None, scope_version=2,
|
|
140
|
+
) -> str:
|
|
141
|
+
if type(scope_version) is not int or scope_version not in {1, 2}:
|
|
142
|
+
raise LabError("unsupported DSH configuration scope")
|
|
143
|
+
if scope_version == 1 and profile != "deletion-v1":
|
|
144
|
+
raise LabError("legacy configuration scope cannot identify a read profile")
|
|
145
|
+
normalized = normalize_composition(composition, guard_state, profile=profile,
|
|
146
|
+
workspace=workspace, dsh_root=dsh_root)
|
|
147
|
+
files = {}
|
|
148
|
+
names = ["settings.yaml", "." + "env", "profiles/headless/package.json"]
|
|
149
|
+
if scope_version == 2:
|
|
150
|
+
names += ["settings.yaml.imported", "cordis.yml", "cordis.patch.yml",
|
|
151
|
+
"profiles/headless/cordis.yml", "profiles/headless/cordis.patch.yml"]
|
|
152
|
+
for name in names:
|
|
153
|
+
path = home / name
|
|
154
|
+
files[name] = digest(read_regular(path, MAX_SCAN_BYTES)) if path.exists() or path.is_symlink() else None
|
|
155
|
+
launch_environment = dict(environment)
|
|
156
|
+
launch_environment["DSH_HOME"] = "<disposable-home>"
|
|
157
|
+
bookkeeping = ("PWD", "OLDPWD", "_") + (("SHLVL",) if scope_version == 2 else ())
|
|
158
|
+
for key in bookkeeping:
|
|
159
|
+
launch_environment.pop(key, None)
|
|
160
|
+
declaration = {
|
|
161
|
+
"scope": "dsh-lab-non-guard-v" + str(scope_version), "composition_sha256": digest(normalized),
|
|
162
|
+
"files": files, "launch_environment": launch_environment,
|
|
163
|
+
}
|
|
164
|
+
if scope_version == 2:
|
|
165
|
+
declaration["guard_profile"] = profile
|
|
166
|
+
return digest(json.dumps(declaration, sort_keys=True, separators=(",", ":")).encode())
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def prepare_homes(source_home: Path, output: Path) -> dict:
|
|
170
|
+
source = source_home.expanduser().absolute()
|
|
171
|
+
if source.is_symlink() or not source.is_dir():
|
|
172
|
+
raise LabError("source DSH home must be a regular directory")
|
|
173
|
+
source = source.resolve()
|
|
174
|
+
payloads = {}
|
|
175
|
+
for name in BOOTSTRAP_FILES:
|
|
176
|
+
path = source / name
|
|
177
|
+
if any(parent.is_symlink() for parent in path.parents if parent != source.parent):
|
|
178
|
+
raise LabError("bootstrap paths must not contain symlinked directories")
|
|
179
|
+
if path.exists() or path.is_symlink():
|
|
180
|
+
payloads[name] = read_regular(path, MAX_SCAN_BYTES)
|
|
181
|
+
manifest = json.loads(payloads.get("profiles/headless/package.json", b"{}"))
|
|
182
|
+
dsh = manifest.get("dsh") if isinstance(manifest, dict) else None
|
|
183
|
+
profile = dsh.get("profile") if isinstance(dsh, dict) else None
|
|
184
|
+
if (
|
|
185
|
+
not isinstance(manifest, dict) or manifest.get("dependencies", {})
|
|
186
|
+
or manifest.get("devDependencies", {})
|
|
187
|
+
or not isinstance(profile, dict) or profile.get("bundles")
|
|
188
|
+
!= ["@deepseek-ai/dsh-base", "@deepseek-ai/dsh-headless"]
|
|
189
|
+
):
|
|
190
|
+
raise LabError("paired homes require a standard headless profile without extra dependencies")
|
|
191
|
+
output = output.expanduser().absolute()
|
|
192
|
+
if output.is_symlink() or output.exists() or not output.parent.is_dir():
|
|
193
|
+
raise LabError("paired home directory must be new with an existing parent")
|
|
194
|
+
output = output.resolve()
|
|
195
|
+
if output == source or output in source.parents or source in output.parents:
|
|
196
|
+
raise LabError("source and paired homes must be disjoint")
|
|
197
|
+
output.mkdir(mode=0o700)
|
|
198
|
+
for member in MEMBERS:
|
|
199
|
+
home = output / member
|
|
200
|
+
home.mkdir(mode=0o700)
|
|
201
|
+
for name, payload in payloads.items():
|
|
202
|
+
path = home / name
|
|
203
|
+
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
204
|
+
write_new(path, payload)
|
|
205
|
+
metadata = {
|
|
206
|
+
"schema_version": 1, "members": list(MEMBERS),
|
|
207
|
+
"bootstrap_sha256": {name: digest(payload) for name, payload in payloads.items()},
|
|
208
|
+
"source_modified": False, "sessions_copied": False,
|
|
209
|
+
"private_bootstrap_copied": True,
|
|
210
|
+
}
|
|
211
|
+
write_json(output / "homes.json", metadata)
|
|
212
|
+
return metadata
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fresh private synthetic fixtures for zero-model native read validation.
|
|
3
|
+
|
|
4
|
+
This is not a prompt-injection trial or an L1/L2 report. Generated values are
|
|
5
|
+
synthetic, never live authentication values. The private control file is a
|
|
6
|
+
sibling of the fixture directory, outside the agent-visible fixture tree.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import secrets
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def prepare(directory):
|
|
18
|
+
root = Path(directory)
|
|
19
|
+
control_path = root.with_name(root.name + ".control.json")
|
|
20
|
+
root.mkdir(mode=0o700, parents=False, exist_ok=False)
|
|
21
|
+
token = "ghp_" + secrets.token_hex(18)
|
|
22
|
+
# Hex is also in the Base64 alphabet; it avoids accidental placeholder
|
|
23
|
+
# substrings such as repeated x's while retaining an opaque synthetic body.
|
|
24
|
+
key_body = secrets.token_hex(32)
|
|
25
|
+
label = "PRIVATE" + " KEY"
|
|
26
|
+
key = "-----BEGIN " + label + "-----\n" + key_body + "\n-----END " + label + "-----"
|
|
27
|
+
ordinary = "AG_CANARY_" + secrets.token_hex(16)
|
|
28
|
+
benign = "# 普通配置:保留中文和行号\nPORT=8080\nLOG_LEVEL=info\n"
|
|
29
|
+
contents = {
|
|
30
|
+
"benign.ini": benign,
|
|
31
|
+
"redaction.ini": benign + "SERVICE_TOKEN=" + ordinary + "\nGITHUB_TOKEN=" + token +
|
|
32
|
+
"\n" + key + "\nFEATURE_ENABLED=true\n",
|
|
33
|
+
}
|
|
34
|
+
control = {"schema": 1, "fixture": "dsh-read-redaction-v1",
|
|
35
|
+
"protected_values": [token, key_body], "ordinary_marker": ordinary,
|
|
36
|
+
"files": list(contents)}
|
|
37
|
+
for filename, content in contents.items():
|
|
38
|
+
fd = os.open(str(root / filename), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
39
|
+
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream:
|
|
40
|
+
stream.write(content)
|
|
41
|
+
fd = os.open(str(control_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
42
|
+
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream:
|
|
43
|
+
json.dump(control, stream, ensure_ascii=False)
|
|
44
|
+
stream.write("\n")
|
|
45
|
+
return {"fixture": control["fixture"], "files": len(contents), "model_calls": 0}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def main():
|
|
49
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
50
|
+
parser.add_argument("--directory", required=True)
|
|
51
|
+
args = parser.parse_args()
|
|
52
|
+
try:
|
|
53
|
+
result = prepare(args.directory)
|
|
54
|
+
except Exception:
|
|
55
|
+
parser.exit(1, "read fixture refused; raw diagnostics suppressed\n")
|
|
56
|
+
print(json.dumps(result))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
if __name__ == "__main__":
|
|
60
|
+
main()
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Bounded text-only coverage of pinned DSH compact failed-attempt streams.
|
|
2
|
+
|
|
3
|
+
This is captured host output, not a reconstructed assistant surface. Scan both
|
|
4
|
+
the original records/diagnostics and joined deltas: JSON fragment boundaries
|
|
5
|
+
must not hide a marker split across consecutive chunks of the same block.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import math
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
MAX_ATTEMPTS = 64
|
|
15
|
+
MAX_STREAM_CHUNKS = 4096
|
|
16
|
+
MAX_INTEGER = 2**53 - 1
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def integer(value, minimum=0):
|
|
20
|
+
return type(value) is int and minimum <= value <= MAX_INTEGER
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def finite_number(value):
|
|
24
|
+
return type(value) in (int, float) and 0 <= value <= MAX_INTEGER and math.isfinite(value)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def failure_supported(value):
|
|
28
|
+
return (
|
|
29
|
+
isinstance(value, dict)
|
|
30
|
+
and {"message", "code"} <= set(value)
|
|
31
|
+
and set(value) <= {"message", "code", "status", "providerRetryAfterMs", "requestId"}
|
|
32
|
+
and all(isinstance(value[k], str) and value[k] for k in ("message", "code"))
|
|
33
|
+
and ("status" not in value or integer(value["status"]))
|
|
34
|
+
and ("providerRetryAfterMs" not in value or finite_number(value["providerRetryAfterMs"]))
|
|
35
|
+
and ("requestId" not in value or isinstance(value["requestId"], str) and bool(value["requestId"]))
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def json_bytes(value):
|
|
40
|
+
return (json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + "\n").encode()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def inspect_stream(stream, remaining_chunks):
|
|
44
|
+
"""Return fixed metadata and complete raw/joined scan bytes, or refuse."""
|
|
45
|
+
if not isinstance(stream, list) or len(stream) > remaining_chunks:
|
|
46
|
+
raise ValueError("ATTEMPT_STREAM_LIMIT_OR_SHAPE")
|
|
47
|
+
joined = {}
|
|
48
|
+
identities = {}
|
|
49
|
+
closed = set()
|
|
50
|
+
chunks = 0
|
|
51
|
+
finish = None
|
|
52
|
+
|
|
53
|
+
def identity(index, kind, call_id=None):
|
|
54
|
+
if not integer(index) or index in closed:
|
|
55
|
+
raise ValueError("INVALID_ATTEMPT_BLOCK")
|
|
56
|
+
selected = (kind, call_id)
|
|
57
|
+
if index in identities and identities[index] != selected:
|
|
58
|
+
raise ValueError("INVALID_ATTEMPT_BLOCK")
|
|
59
|
+
identities[index] = selected
|
|
60
|
+
return joined.setdefault((index, kind, call_id), [])
|
|
61
|
+
|
|
62
|
+
for record in stream:
|
|
63
|
+
if finish is not None or not isinstance(record, dict):
|
|
64
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
65
|
+
kind = record.get("type")
|
|
66
|
+
if kind in {"text-chunks", "reasoning-chunks", "tool-call-chunks"}:
|
|
67
|
+
expected = {"type", "time0", "index", "dt", "args" if kind == "tool-call-chunks" else "texts"}
|
|
68
|
+
if kind == "tool-call-chunks":
|
|
69
|
+
expected.add("id")
|
|
70
|
+
if "name" in record:
|
|
71
|
+
expected.add("name")
|
|
72
|
+
if not isinstance(record.get("id"), str) or not record["id"]:
|
|
73
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
74
|
+
if "name" in record and (not isinstance(record["name"], str) or not record["name"]):
|
|
75
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
76
|
+
if set(record) != expected or not integer(record["time0"]):
|
|
77
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
78
|
+
members = record["args" if kind == "tool-call-chunks" else "texts"]
|
|
79
|
+
gaps = record["dt"]
|
|
80
|
+
if (not isinstance(members, list) or not members or not all(isinstance(v, str) for v in members)
|
|
81
|
+
or not isinstance(gaps, list) or len(gaps) != len(members) - 1):
|
|
82
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
83
|
+
chunks += len(members)
|
|
84
|
+
if chunks > remaining_chunks:
|
|
85
|
+
raise ValueError("ATTEMPT_STREAM_LIMIT_OR_SHAPE")
|
|
86
|
+
timestamp = record["time0"]
|
|
87
|
+
for gap in gaps:
|
|
88
|
+
if type(gap) is not int or not -MAX_INTEGER <= gap <= MAX_INTEGER:
|
|
89
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
90
|
+
timestamp += gap
|
|
91
|
+
if not integer(timestamp):
|
|
92
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
93
|
+
block_kind = {"text-chunks": "text", "reasoning-chunks": "reasoning", "tool-call-chunks": "tool-call"}[kind]
|
|
94
|
+
if block_kind == "tool-call" and identities.get(record["index"]) == ("tool-call", None):
|
|
95
|
+
identities[record["index"]] = (block_kind, record["id"])
|
|
96
|
+
identity(record["index"], block_kind, record.get("id")).extend(members)
|
|
97
|
+
continue
|
|
98
|
+
if kind != "chunk" or set(record) != {"type", "time", "chunk"} or not integer(record["time"]):
|
|
99
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
100
|
+
chunk = record["chunk"]
|
|
101
|
+
chunks += 1
|
|
102
|
+
if chunks > remaining_chunks or not isinstance(chunk, dict):
|
|
103
|
+
raise ValueError("ATTEMPT_STREAM_LIMIT_OR_SHAPE")
|
|
104
|
+
chunk_kind = chunk.get("type")
|
|
105
|
+
if chunk_kind == "block-start":
|
|
106
|
+
if set(chunk) != {"type", "index", "blockType"} or chunk["blockType"] not in {"text", "reasoning", "tool-call"}:
|
|
107
|
+
raise ValueError("UNSUPPORTED_ATTEMPT_CHUNK")
|
|
108
|
+
index = chunk["index"]
|
|
109
|
+
if not integer(index) or index in identities or index in closed:
|
|
110
|
+
raise ValueError("INVALID_ATTEMPT_BLOCK")
|
|
111
|
+
# A Tool-call id becomes known at its first delta, not at start.
|
|
112
|
+
identities[index] = (chunk["blockType"], None)
|
|
113
|
+
elif chunk_kind in {"text-delta", "reasoning-delta", "tool-call-delta"}:
|
|
114
|
+
if chunk_kind == "tool-call-delta":
|
|
115
|
+
expected = {"type", "index", "id", "argumentsDelta"} | ({"name"} if "name" in chunk else set())
|
|
116
|
+
if (not isinstance(chunk.get("id"), str) or not chunk["id"]
|
|
117
|
+
or "name" in chunk and (not isinstance(chunk["name"], str) or not chunk["name"])):
|
|
118
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
119
|
+
field, block_kind = "argumentsDelta", "tool-call"
|
|
120
|
+
else:
|
|
121
|
+
expected, field, block_kind = {"type", "index", "text"}, "text", chunk_kind.removesuffix("-delta")
|
|
122
|
+
if set(chunk) != expected or not isinstance(chunk[field], str):
|
|
123
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
124
|
+
index, call_id = chunk["index"], chunk.get("id")
|
|
125
|
+
if block_kind == "tool-call" and identities.get(index) == ("tool-call", None):
|
|
126
|
+
identities[index] = (block_kind, call_id)
|
|
127
|
+
identity(index, block_kind, call_id).append(chunk[field])
|
|
128
|
+
elif chunk_kind == "block-end":
|
|
129
|
+
block, index = chunk.get("block"), chunk.get("index")
|
|
130
|
+
if set(chunk) != {"type", "index", "block"} or not isinstance(block, dict):
|
|
131
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
132
|
+
block_kind = block.get("type")
|
|
133
|
+
if block_kind in {"text", "reasoning"}:
|
|
134
|
+
valid = set(block) == {"type", "text"} and isinstance(block["text"], str)
|
|
135
|
+
elif block_kind == "tool-call":
|
|
136
|
+
valid = set(block) == {"type", "id", "name", "arguments"} and all(isinstance(block[k], str) and block[k] for k in ("id", "name", "arguments"))
|
|
137
|
+
else:
|
|
138
|
+
valid = False
|
|
139
|
+
if not valid or not integer(index) or index in closed:
|
|
140
|
+
raise ValueError("UNSUPPORTED_ATTEMPT_CHUNK")
|
|
141
|
+
selected = (block_kind, block.get("id"))
|
|
142
|
+
if index in identities and identities[index] not in {selected, ("tool-call", None)}:
|
|
143
|
+
raise ValueError("INVALID_ATTEMPT_BLOCK")
|
|
144
|
+
closed.add(index)
|
|
145
|
+
elif chunk_kind == "usage":
|
|
146
|
+
usage = chunk.get("usage")
|
|
147
|
+
if (set(chunk) != {"type", "usage"} or not isinstance(usage, dict)
|
|
148
|
+
or not {"inputTokens", "outputTokens"} <= set(usage)
|
|
149
|
+
or not set(usage) <= {"inputTokens", "outputTokens", "totalTokens", "cacheReadTokens", "cacheWriteTokens", "reasoningTokens"}
|
|
150
|
+
or not all(integer(v) for v in usage.values())):
|
|
151
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
152
|
+
elif chunk_kind == "finish":
|
|
153
|
+
reason = chunk.get("reason")
|
|
154
|
+
if set(chunk) != {"type", "reason"} or not isinstance(reason, dict):
|
|
155
|
+
raise ValueError("INVALID_ATTEMPT_STREAM")
|
|
156
|
+
finish = reason.get("kind")
|
|
157
|
+
if finish in {"error", "aborted"}:
|
|
158
|
+
valid = set(reason) == {"kind", "failure"} and failure_supported(reason["failure"])
|
|
159
|
+
else:
|
|
160
|
+
valid = finish in {"stop", "tool-calls", "max-tokens"} and set(reason) == {"kind"}
|
|
161
|
+
if not valid:
|
|
162
|
+
raise ValueError("UNSUPPORTED_ATTEMPT_CHUNK")
|
|
163
|
+
else:
|
|
164
|
+
raise ValueError("UNSUPPORTED_ATTEMPT_CHUNK")
|
|
165
|
+
# A raw Tool-call block start can precede a packed run carrying its id.
|
|
166
|
+
# The raw records retain names, ids, finish failures and complete blocks.
|
|
167
|
+
value = {"stream": stream, "joined_fragments": [
|
|
168
|
+
{"index": index, "kind": kind, "text": "".join(parts)}
|
|
169
|
+
for (index, kind, _call_id), parts in joined.items()
|
|
170
|
+
]}
|
|
171
|
+
return {"records": len(stream), "chunks": chunks, "finish_kind": finish}, json_bytes(value)
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/** Bounded decoder for the concatenated Zstandard frames written by DSH. */
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import * as zlib from "node:zlib";
|
|
6
|
+
|
|
7
|
+
const MAX_BYTES = 1024 * 1024;
|
|
8
|
+
const MAX_FRAMES = 4096;
|
|
9
|
+
// DSH's streaming frames declare a 2 MiB window even for small JSONL batches.
|
|
10
|
+
const MAX_WINDOW_BYTES = 8 * 1024 * 1024;
|
|
11
|
+
|
|
12
|
+
// RFC 8878 section 3.1.1: check the physical frame before asking zlib to
|
|
13
|
+
// decode. Node can return partial data for a torn frame without throwing.
|
|
14
|
+
// This DSH reader refuses skippable frames and external dictionaries.
|
|
15
|
+
function frameEnd(bytes, start) {
|
|
16
|
+
let cursor = start;
|
|
17
|
+
const take = (size) => {
|
|
18
|
+
const at = cursor;
|
|
19
|
+
cursor += size;
|
|
20
|
+
if (cursor > bytes.length) throw new Error("torn frame");
|
|
21
|
+
return at;
|
|
22
|
+
};
|
|
23
|
+
if (bytes.readUInt32LE(take(4)) !== 0xfd2fb528) throw new Error("frame magic");
|
|
24
|
+
const flags = bytes[take(1)];
|
|
25
|
+
if (flags & 0x18) throw new Error("reserved frame flags");
|
|
26
|
+
const single = Boolean(flags & 0x20);
|
|
27
|
+
if (!single) {
|
|
28
|
+
const window = bytes[take(1)];
|
|
29
|
+
const base = 2 ** (10 + (window >>> 3));
|
|
30
|
+
if (base + (base / 8) * (window & 7) > MAX_WINDOW_BYTES) throw new Error("window limit");
|
|
31
|
+
}
|
|
32
|
+
const dictionarySize = [0, 1, 2, 4][flags & 3];
|
|
33
|
+
const dictionaryAt = take(dictionarySize);
|
|
34
|
+
if (dictionarySize && bytes.readUIntLE(dictionaryAt, dictionarySize) !== 0) {
|
|
35
|
+
throw new Error("dictionary unsupported");
|
|
36
|
+
}
|
|
37
|
+
const sizeFlag = flags >>> 6;
|
|
38
|
+
const sizeBytes = sizeFlag === 0 ? (single ? 1 : 0) : [0, 2, 4, 8][sizeFlag];
|
|
39
|
+
const sizeAt = take(sizeBytes);
|
|
40
|
+
if (sizeBytes) {
|
|
41
|
+
const size = sizeBytes === 8 ? bytes.readBigUInt64LE(sizeAt)
|
|
42
|
+
: BigInt(bytes.readUIntLE(sizeAt, sizeBytes) + (sizeBytes === 2 ? 256 : 0));
|
|
43
|
+
if (size > BigInt(MAX_BYTES)) throw new Error("content size limit");
|
|
44
|
+
}
|
|
45
|
+
let last = false;
|
|
46
|
+
for (let blocks = 0; !last; blocks += 1) {
|
|
47
|
+
if (blocks >= MAX_FRAMES) throw new Error("block limit");
|
|
48
|
+
const header = bytes.readUIntLE(take(3), 3);
|
|
49
|
+
const kind = (header >>> 1) & 3;
|
|
50
|
+
const size = header >>> 3;
|
|
51
|
+
if (kind === 3 || size > 128 * 1024) throw new Error("invalid block");
|
|
52
|
+
take(kind === 1 ? 1 : size);
|
|
53
|
+
last = Boolean(header & 1);
|
|
54
|
+
}
|
|
55
|
+
if (flags & 4) take(4);
|
|
56
|
+
return cursor;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function decodeSession(bytes) {
|
|
60
|
+
if (bytes.length === 0 || bytes.length > MAX_BYTES) throw new Error("input limit");
|
|
61
|
+
const chunks = [];
|
|
62
|
+
let offset = 0;
|
|
63
|
+
let total = 0;
|
|
64
|
+
while (offset < bytes.length) {
|
|
65
|
+
if (chunks.length >= MAX_FRAMES) throw new Error("frame limit");
|
|
66
|
+
const end = frameEnd(bytes, offset);
|
|
67
|
+
const decoded = zlib.zstdDecompressSync(bytes.subarray(offset, end), {
|
|
68
|
+
info: true,
|
|
69
|
+
maxOutputLength: Math.max(1, MAX_BYTES - total),
|
|
70
|
+
});
|
|
71
|
+
const consumed = decoded.engine.bytesWritten;
|
|
72
|
+
if (consumed !== end - offset) {
|
|
73
|
+
throw new Error("invalid frame progress");
|
|
74
|
+
}
|
|
75
|
+
total += decoded.buffer.length;
|
|
76
|
+
if (total > MAX_BYTES) throw new Error("output limit");
|
|
77
|
+
chunks.push(decoded.buffer);
|
|
78
|
+
offset = end;
|
|
79
|
+
}
|
|
80
|
+
return Buffer.concat(chunks, total);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function main() {
|
|
84
|
+
try {
|
|
85
|
+
if (process.argv[2] === "--probe" && process.argv.length === 3) {
|
|
86
|
+
const supported = typeof zlib.zstdDecompressSync === "function";
|
|
87
|
+
process.stdout.write(JSON.stringify({ version: process.version, zstd_supported: supported }) + "\n");
|
|
88
|
+
return supported ? 0 : 1;
|
|
89
|
+
}
|
|
90
|
+
if (process.argv.length !== 3) throw new Error("arguments");
|
|
91
|
+
const fd = fs.openSync(process.argv[2], fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
|
|
92
|
+
let bytes;
|
|
93
|
+
try {
|
|
94
|
+
if (!fs.fstatSync(fd).isFile()) throw new Error("input type");
|
|
95
|
+
const buffer = Buffer.alloc(MAX_BYTES + 1);
|
|
96
|
+
let count = 0;
|
|
97
|
+
while (count < buffer.length) {
|
|
98
|
+
const size = fs.readSync(fd, buffer, count, buffer.length - count, null);
|
|
99
|
+
if (size === 0) break;
|
|
100
|
+
count += size;
|
|
101
|
+
}
|
|
102
|
+
bytes = buffer.subarray(0, count);
|
|
103
|
+
} finally {
|
|
104
|
+
fs.closeSync(fd);
|
|
105
|
+
}
|
|
106
|
+
// Emit only after every frame succeeds, including a possible torn tail.
|
|
107
|
+
process.stdout.write(decodeSession(bytes));
|
|
108
|
+
return 0;
|
|
109
|
+
} catch {
|
|
110
|
+
process.stderr.write("DSH native session decode refused; raw input is not printed\n");
|
|
111
|
+
return 1;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
116
|
+
process.exitCode = main();
|
|
117
|
+
}
|