@mokuyoaxis/agent-guard 0.2.3-rc2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -47
- package/README.zh-CN.md +71 -39
- package/adapters/dsh/README.md +15 -10
- package/adapters/dsh/harness/README.md +68 -12
- package/adapters/dsh/harness/bootstrap.patch.yml +5 -0
- package/adapters/dsh/harness/bootstrap_init.mjs +48 -0
- package/adapters/dsh/harness/guard_lab.py +191 -29
- package/adapters/dsh/harness/native_session.py +41 -20
- package/adapters/dsh/harness/profiles.py +74 -22
- package/adapters/kimi-code/README.md +8 -7
- package/core/lab.py +24 -1
- package/core/redaction.py +157 -55
- package/docs/development-note-unguarded-deletion.md +6 -7
- package/docs/friction.md +68 -4
- package/docs/guard-lab-read-trial-task.md +7 -0
- package/docs/guard-lab.md +7 -0
- package/docs/harness-capabilities.md +1 -1
- package/docs/release-notes-0.2.0-rc1.md +2 -2
- package/docs/release-notes-0.2.0.md +2 -2
- package/docs/release-notes-0.2.2.md +7 -8
- package/docs/release-notes-0.2.3-rc1.md +4 -2
- package/docs/release-notes-0.2.3.md +51 -12
- package/docs/secret-guard-analysis.md +13 -12
- package/docs/secret-guard-architecture.md +5 -3
- package/docs/secret-guard-design.md +7 -3
- package/docs/test-report-codex-gpt-5.6-sol.md +10 -11
- package/docs/test-report-kimi-code-block.md +19 -20
- package/docs/test-report-release-readiness-0.2.3.md +78 -0
- package/docs/test-report-workbuddy-windows-core.md +53 -0
- package/docs/test-report-zcode-glm-flash.md +9 -9
- package/package.json +1 -1
- package/skills/delete-guard/SKILL.md +2 -3
- package/skills/exfil-guard/SKILL.md +37 -7
- package/skills/exfil-guard/references/rules.md +84 -16
- package/skills/exfil-guard/scripts/_payload.py +99 -0
- package/skills/exfil-guard/scripts/check_span.py +14 -81
- package/skills/exfil-guard/scripts/sanitize.py +113 -34
|
@@ -34,7 +34,7 @@ from adapters.dsh.harness.native_session import ( # noqa: E402
|
|
|
34
34
|
PARSER_VERSION, capture_session, check_decoder, session_inventory, split_session,
|
|
35
35
|
)
|
|
36
36
|
from adapters.dsh.harness.profiles import ( # noqa: E402
|
|
37
|
-
ADAPTER, MEMBERS, configuration_fingerprint, guard_patch,
|
|
37
|
+
ADAPTER, MEMBERS, GUARD_PROFILES, configuration_fingerprint, guard_patch,
|
|
38
38
|
normalize_composition, prepare_homes,
|
|
39
39
|
)
|
|
40
40
|
|
|
@@ -42,9 +42,34 @@ from adapters.dsh.harness.profiles import ( # noqa: E402
|
|
|
42
42
|
TESTED_VERSION = "0.2.0-rc.2"
|
|
43
43
|
HOST_CONTRACTS = {"0.1.5-rc.1": (3, 4), "0.2.0-rc.2": (4, 5)}
|
|
44
44
|
BASELINE_PATCH = Path(__file__).with_name("baseline.patch.yml")
|
|
45
|
+
BOOTSTRAP_PATCH = Path(__file__).with_name("bootstrap.patch.yml")
|
|
46
|
+
BOOTSTRAP_INIT = Path(__file__).with_name("bootstrap_init.mjs")
|
|
47
|
+
RECEIPT_VERSION = 6
|
|
45
48
|
MAX_RECEIPT_BYTES = 64 * 1024
|
|
46
49
|
|
|
47
50
|
|
|
51
|
+
def _profile_options(profile, workspace, dsh_root):
|
|
52
|
+
return {"profile": profile, "workspace": workspace, "dsh_root": dsh_root}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _launcher_root(executable: str, version: str) -> Path:
|
|
56
|
+
for parent in Path(executable).resolve().parents:
|
|
57
|
+
manifest = parent / "package.json"
|
|
58
|
+
if manifest.is_file():
|
|
59
|
+
value = json.loads(_read_regular(manifest, MAX_RECEIPT_BYTES))
|
|
60
|
+
if value.get("name") == "@deepseek-ai/dsh" and value.get("version") == version:
|
|
61
|
+
return parent
|
|
62
|
+
raise LabError("bootstrap requires the reviewed installed DSH package launcher")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _guard_assets() -> dict:
|
|
66
|
+
names = ["adapters/dsh/lib/index.js", "adapters/dsh/lib/read_result_guard.js",
|
|
67
|
+
"adapters/dsh/lib/read_result_guard.py", "skills/exfil-guard/scripts/sanitize.py",
|
|
68
|
+
"skills/exfil-guard/scripts/_payload.py", "skills/exfil-guard/scripts/_bootstrap.py"]
|
|
69
|
+
names.extend(str(path.relative_to(ROOT)) for path in sorted((ROOT / "core").glob("*.py")))
|
|
70
|
+
return {name: _digest(_read_regular(ROOT / name, MAX_SCAN_BYTES)) for name in names}
|
|
71
|
+
|
|
72
|
+
|
|
48
73
|
def _environment(home: Path) -> dict[str, str]:
|
|
49
74
|
environment = dict(os.environ)
|
|
50
75
|
# A previous sentinel/workspace must not redirect this independent trial.
|
|
@@ -59,6 +84,7 @@ def _environment(home: Path) -> dict[str, str]:
|
|
|
59
84
|
def _host_options(
|
|
60
85
|
executable: str, home: Path, patches: Sequence[Path],
|
|
61
86
|
guard_state: str = "off", guard_patch_file: Optional[Path] = None,
|
|
87
|
+
*, guard_profile="deletion-v1", workspace=None, dsh_root=None,
|
|
62
88
|
) -> tuple[list[str], dict[str, str]]:
|
|
63
89
|
if os.name != "posix":
|
|
64
90
|
raise LabError("this DSH runner currently supports POSIX hosts only")
|
|
@@ -74,7 +100,8 @@ def _host_options(
|
|
|
74
100
|
argv = [resolved, "--profile", "headless"]
|
|
75
101
|
if guard_state not in {"off", "on"} or (guard_state == "on") != (guard_patch_file is not None):
|
|
76
102
|
raise LabError("guard-on requires the generated local guard patch; guard-off omits it")
|
|
77
|
-
|
|
103
|
+
expected_patch = guard_patch(guard_profile, workspace, dsh_root)
|
|
104
|
+
if guard_patch_file is not None and _read_regular(guard_patch_file, MAX_RECEIPT_BYTES) != expected_patch:
|
|
78
105
|
raise LabError("select the exact generated local Guard patch")
|
|
79
106
|
layers = (*patches, BASELINE_PATCH, *((guard_patch_file,) if guard_patch_file else ()))
|
|
80
107
|
for patch in layers:
|
|
@@ -87,6 +114,7 @@ def _host_options(
|
|
|
87
114
|
def _config_dump(
|
|
88
115
|
argv: Sequence[str], cwd: Path, environment: dict[str, str],
|
|
89
116
|
guard_state: str = "off",
|
|
117
|
+
*, guard_profile="deletion-v1", workspace=None, dsh_root=None,
|
|
90
118
|
) -> bytes:
|
|
91
119
|
result, output = _capture_process(
|
|
92
120
|
[*argv, "--dump-config"], cwd, environment, 15,
|
|
@@ -96,7 +124,8 @@ def _config_dump(
|
|
|
96
124
|
or result["output_limit_exceeded"] or not output["stdout"].strip()
|
|
97
125
|
):
|
|
98
126
|
raise LabError("DSH config preflight failed; raw diagnostics are not printed")
|
|
99
|
-
normalize_composition(output["stdout"], guard_state
|
|
127
|
+
normalize_composition(output["stdout"], guard_state,
|
|
128
|
+
**_profile_options(guard_profile, workspace, dsh_root))
|
|
100
129
|
return output["stdout"]
|
|
101
130
|
|
|
102
131
|
|
|
@@ -104,9 +133,11 @@ def check_host(
|
|
|
104
133
|
executable: str, home: Path, patches: Sequence[Path] = (),
|
|
105
134
|
node_executable: str = "node",
|
|
106
135
|
guard_state: str = "off", guard_patch_file: Optional[Path] = None,
|
|
136
|
+
*, guard_profile="deletion-v1", workspace=None, dsh_root=None,
|
|
107
137
|
) -> tuple[dict, list[str], dict[str, str]]:
|
|
108
138
|
home = home.expanduser().resolve()
|
|
109
|
-
|
|
139
|
+
context = dict(guard_profile=guard_profile, workspace=workspace, dsh_root=dsh_root)
|
|
140
|
+
argv, environment = _host_options(executable, home, patches, guard_state, guard_patch_file, **context)
|
|
110
141
|
result, output = _capture_process([argv[0], "--version"], home.resolve(), environment, 10)
|
|
111
142
|
version = output["stdout"].strip().decode("ascii", errors="replace")
|
|
112
143
|
if (
|
|
@@ -115,24 +146,73 @@ def check_host(
|
|
|
115
146
|
or version not in HOST_CONTRACTS
|
|
116
147
|
):
|
|
117
148
|
raise LabError("DSH version preflight requires a reviewed exact host version")
|
|
118
|
-
|
|
149
|
+
if guard_profile == "read-redaction-v1" and _launcher_root(argv[0], version) != Path(dsh_root).resolve():
|
|
150
|
+
raise LabError("read profile root must match the selected DSH launcher")
|
|
151
|
+
configuration = _config_dump(argv, home.resolve(), environment, guard_state, **context)
|
|
119
152
|
decoder, _ = check_decoder(node_executable, home, environment)
|
|
120
153
|
return {
|
|
121
154
|
"harness_version": version,
|
|
122
155
|
"native_format_version": HOST_CONTRACTS[version][0],
|
|
123
156
|
"profile": "headless", "guard_state": guard_state.upper(),
|
|
124
157
|
"guard_state_source": "user-declared",
|
|
158
|
+
"guard_profile": guard_profile, "native_parser_version": PARSER_VERSION,
|
|
125
159
|
"composition_bytes": len(configuration),
|
|
126
160
|
"composition_sha256": _digest(configuration),
|
|
127
161
|
"telemetry_disabled": True, "configuration_retained": False,
|
|
128
162
|
"model_identity_source": "user-declared",
|
|
129
163
|
"native_decoder": decoder,
|
|
130
|
-
"non_guard_configuration_sha256": configuration_fingerprint(home, configuration, environment, guard_state
|
|
131
|
-
|
|
164
|
+
"non_guard_configuration_sha256": configuration_fingerprint(home, configuration, environment, guard_state,
|
|
165
|
+
**_profile_options(guard_profile, workspace, dsh_root)),
|
|
166
|
+
"configuration_scope": "dsh-lab-non-guard-v2",
|
|
132
167
|
"guard_adapter_sha256": _digest(_read_regular(ADAPTER, MAX_SCAN_BYTES)),
|
|
168
|
+
"guard_assets": _guard_assets(),
|
|
133
169
|
}, argv, environment
|
|
134
170
|
|
|
135
171
|
|
|
172
|
+
def _initialize_home(executable, home, patches, node_executable) -> dict:
|
|
173
|
+
preflight, argv, environment = check_host(executable, home, patches, node_executable)
|
|
174
|
+
if session_inventory(home):
|
|
175
|
+
raise LabError("bootstrap requires a fresh disposable home")
|
|
176
|
+
if preflight["harness_version"] == "0.1.5-rc.1":
|
|
177
|
+
return {"method": "legacy-host-no-import", "model_tasks_launched": 0}
|
|
178
|
+
root = _launcher_root(argv[0], preflight["harness_version"])
|
|
179
|
+
legacy = home / "settings.yaml"
|
|
180
|
+
imported = home / "settings.yaml.imported"
|
|
181
|
+
legacy_payload = _read_regular(legacy, MAX_SCAN_BYTES) if legacy.exists() else None
|
|
182
|
+
if legacy_payload is not None and imported.exists():
|
|
183
|
+
raise LabError("ambiguous legacy bootstrap documents")
|
|
184
|
+
init_argv = [node_executable, str(BOOTSTRAP_INIT), str(root),
|
|
185
|
+
*(str(path.expanduser().absolute()) for path in patches), str(BASELINE_PATCH), str(BOOTSTRAP_PATCH)]
|
|
186
|
+
capture = home / ".lab-bootstrap"
|
|
187
|
+
if capture.exists() or capture.is_symlink():
|
|
188
|
+
raise LabError("bootstrap capture already exists; use fresh copied homes")
|
|
189
|
+
capture.mkdir(mode=0o700)
|
|
190
|
+
fingerprints = []
|
|
191
|
+
for index in range(2):
|
|
192
|
+
result, streams = _capture_process(init_argv, home, environment, 60)
|
|
193
|
+
for name, payload in streams.items():
|
|
194
|
+
_write_new(capture / (str(index + 1) + "-" + name + ".bin"), payload)
|
|
195
|
+
_write_json(capture / (str(index + 1) + "-receipt.json"), {
|
|
196
|
+
"host": result, "model_tasks_launched": 0,
|
|
197
|
+
"streams": {name: {"bytes": len(payload), "sha256": _digest(payload)}
|
|
198
|
+
for name, payload in streams.items()}})
|
|
199
|
+
if (result["exit_code"] != 0 or any(result[key] for key in
|
|
200
|
+
("timed_out", "start_failed", "output_limit_exceeded"))
|
|
201
|
+
or any(b"was not imported" in data for data in streams.values())
|
|
202
|
+
or session_inventory(home)):
|
|
203
|
+
raise LabError("native bootstrap failed or created a session; no model task was launched by the Lab")
|
|
204
|
+
if legacy.exists() or (legacy_payload is not None
|
|
205
|
+
and _read_regular(imported, MAX_SCAN_BYTES) != legacy_payload):
|
|
206
|
+
raise LabError("legacy bootstrap import did not complete")
|
|
207
|
+
value, _, _ = check_host(executable, home, patches, node_executable)
|
|
208
|
+
fingerprints.append(value["non_guard_configuration_sha256"])
|
|
209
|
+
if fingerprints[0] != fingerprints[1]:
|
|
210
|
+
raise LabError("native bootstrap settings did not stabilize")
|
|
211
|
+
return {"method": "native-import-with-driver-disabled-v1", "passes": 2,
|
|
212
|
+
"model_tasks_launched": 0, "sessions_created": 0,
|
|
213
|
+
"settings_stable": True, "legacy_imported": legacy_payload is not None}
|
|
214
|
+
|
|
215
|
+
|
|
136
216
|
def run_trial(
|
|
137
217
|
output_dir: Path, home: Path, executable: str, case_id: str,
|
|
138
218
|
task_file: Path, model_label: str, trial_group: str,
|
|
@@ -141,6 +221,7 @@ def run_trial(
|
|
|
141
221
|
node_executable: str = "node",
|
|
142
222
|
guard_state: str = "off", guard_patch_file: Optional[Path] = None,
|
|
143
223
|
expected_configuration_sha256: Optional[str] = None,
|
|
224
|
+
*, guard_profile="deletion-v1", dsh_root=None,
|
|
144
225
|
) -> dict:
|
|
145
226
|
if case_id not in {"clean", "injection-probe"}:
|
|
146
227
|
raise LabError("this runner supports clean and injection-probe only")
|
|
@@ -148,6 +229,8 @@ def run_trial(
|
|
|
148
229
|
raise LabError("host timeout must be between 1 and 600 seconds")
|
|
149
230
|
if attack_sample not in INJECTION_SAMPLES or (case_id != "injection-probe" and attack_sample != "direct-v1"):
|
|
150
231
|
raise LabError("select a known attack sample for injection-probe only")
|
|
232
|
+
if case_id == "injection-probe" and ((attack_sample == "read-redaction-v1") != (guard_profile == "read-redaction-v1")):
|
|
233
|
+
raise LabError("read redaction profile and attack sample must be selected together")
|
|
151
234
|
if not TRIAL_LABEL_PATTERN.fullmatch(model_label):
|
|
152
235
|
raise LabError("model label must follow the Lab public-label format")
|
|
153
236
|
if not TRIAL_GROUP_PATTERN.fullmatch(trial_group):
|
|
@@ -166,7 +249,10 @@ def run_trial(
|
|
|
166
249
|
output = output.resolve()
|
|
167
250
|
if output == home or output in home.parents or home in output.parents:
|
|
168
251
|
raise LabError("trial output and the selected DSH home must be disjoint")
|
|
169
|
-
|
|
252
|
+
context = dict(guard_profile=guard_profile, workspace=output / "fixture", dsh_root=dsh_root)
|
|
253
|
+
preflight, argv, environment = check_host(executable, home, patches, node_executable, guard_state, guard_patch_file, **context)
|
|
254
|
+
if preflight["harness_version"] == "0.2.0-rc.2" and (home / "settings.yaml").exists():
|
|
255
|
+
raise LabError("initialize the copied modern DSH home before freezing a trial")
|
|
170
256
|
if expected_configuration_sha256 is not None and preflight["non_guard_configuration_sha256"] != expected_configuration_sha256:
|
|
171
257
|
raise LabError("paired non-guard settings changed; no host task was launched")
|
|
172
258
|
before = session_inventory(home)
|
|
@@ -200,12 +286,14 @@ def run_trial(
|
|
|
200
286
|
request_stop(evidence)
|
|
201
287
|
composition_unchanged = False
|
|
202
288
|
try:
|
|
203
|
-
after = _config_dump(argv, home, environment, guard_state)
|
|
289
|
+
after = _config_dump(argv, home, environment, guard_state, **context)
|
|
204
290
|
composition_unchanged = (
|
|
205
291
|
_digest(after) == preflight["composition_sha256"]
|
|
206
|
-
and configuration_fingerprint(home, after, environment, guard_state
|
|
292
|
+
and configuration_fingerprint(home, after, environment, guard_state,
|
|
293
|
+
**_profile_options(guard_profile, output / "fixture", dsh_root))
|
|
207
294
|
== preflight["non_guard_configuration_sha256"]
|
|
208
295
|
and _digest(_read_regular(ADAPTER, MAX_SCAN_BYTES)) == preflight["guard_adapter_sha256"]
|
|
296
|
+
and _guard_assets() == preflight["guard_assets"]
|
|
209
297
|
)
|
|
210
298
|
except LabError:
|
|
211
299
|
pass
|
|
@@ -219,7 +307,7 @@ def run_trial(
|
|
|
219
307
|
)
|
|
220
308
|
captures.update(native_captures)
|
|
221
309
|
receipt = {
|
|
222
|
-
"schema_version":
|
|
310
|
+
"schema_version": RECEIPT_VERSION,
|
|
223
311
|
"run_id": prepared["run_id"], "case_id": case_id,
|
|
224
312
|
**({"attack_sample": prepared["attack_sample"]} if "attack_sample" in prepared else {}),
|
|
225
313
|
"task_sha256": _digest(task), "task_bytes": len(task),
|
|
@@ -255,10 +343,19 @@ def _read_capture(output_dir: Path) -> tuple[dict, dict, dict, Optional[dict], b
|
|
|
255
343
|
run = json.loads(_read_regular(output / "evidence" / "run.json", MAX_RECEIPT_BYTES))
|
|
256
344
|
if not isinstance(receipt, dict) or not isinstance(run, dict):
|
|
257
345
|
raise LabError("capture metadata must be JSON objects")
|
|
258
|
-
if type(receipt.get("schema_version")) is not int or receipt["schema_version"] not in {1, 2, 3, 4, 5}:
|
|
346
|
+
if type(receipt.get("schema_version")) is not int or receipt["schema_version"] not in {1, 2, 3, 4, 5, 6}:
|
|
259
347
|
raise LabError("unsupported capture receipt version")
|
|
260
348
|
host_version = receipt.get("preflight", {}).get("harness_version")
|
|
261
349
|
native_format = 4 if receipt["schema_version"] == 5 else 3
|
|
350
|
+
if receipt["schema_version"] == 6:
|
|
351
|
+
preflight = receipt.get("preflight", {})
|
|
352
|
+
native_format = preflight.get("native_format_version")
|
|
353
|
+
if (type(native_format) is not int or native_format not in {3, 4}
|
|
354
|
+
or type(preflight.get("native_parser_version")) is not int
|
|
355
|
+
or preflight["native_parser_version"] != 3
|
|
356
|
+
or preflight.get("configuration_scope") != "dsh-lab-non-guard-v2"
|
|
357
|
+
or preflight.get("guard_profile") not in GUARD_PROFILES):
|
|
358
|
+
raise LabError("capture parser/profile/format contract does not match")
|
|
262
359
|
if receipt["schema_version"] >= 3:
|
|
263
360
|
expected_host = "0.2.0-rc.2" if native_format == 4 else "0.1.5-rc.1"
|
|
264
361
|
if host_version != expected_host or (native_format == 4 and
|
|
@@ -305,7 +402,7 @@ def _read_capture(output_dir: Path) -> tuple[dict, dict, dict, Optional[dict], b
|
|
|
305
402
|
if "session" not in payloads or native.get("status") != "CAPTURED" or native.get("compression") not in {"none", "zstd"}:
|
|
306
403
|
raise LabError("native capture is missing its original artifact or format")
|
|
307
404
|
summary, roles = split_session(payloads["session_jsonl"], Path(run["fixture_dir"]), receipt["task_sha256"],
|
|
308
|
-
parser_version=
|
|
405
|
+
parser_version=3 if receipt["schema_version"] == 6 else (2 if receipt["schema_version"] >= 4 else 1),
|
|
309
406
|
native_format_version=native_format)
|
|
310
407
|
if native.get("parser") != summary or native.get("recognized") is not summary["recognized"]:
|
|
311
408
|
raise LabError("native session interpretation differs from the receipt")
|
|
@@ -359,12 +456,13 @@ NATIVE_CHANNELS = sorted([
|
|
|
359
456
|
def compare_trials(baseline_dir: Path, guarded_dir: Path) -> dict:
|
|
360
457
|
result = compare_runs(baseline_dir / "evidence", guarded_dir / "evidence")
|
|
361
458
|
valid = True
|
|
459
|
+
contracts = []
|
|
362
460
|
for label, directory, state in (("baseline", baseline_dir, "OFF"), ("guarded", guarded_dir, "ON")):
|
|
363
461
|
try:
|
|
364
462
|
receipt, run, _, _, ready = _read_capture(directory)
|
|
365
463
|
report = result[label]
|
|
366
464
|
if (
|
|
367
|
-
receipt["schema_version"] not in {3, 4, 5} or not ready
|
|
465
|
+
receipt["schema_version"] not in {3, 4, 5, 6} or not ready
|
|
368
466
|
or receipt["preflight"]["guard_state"] != state
|
|
369
467
|
or not receipt["composition_unchanged"]
|
|
370
468
|
or run["case_id"] != "injection-probe"
|
|
@@ -373,11 +471,18 @@ def compare_trials(baseline_dir: Path, guarded_dir: Path) -> dict:
|
|
|
373
471
|
or any(receipt["host"][key] for key in ("timed_out", "start_failed", "output_limit_exceeded"))
|
|
374
472
|
):
|
|
375
473
|
raise LabError("trial is outside the paired native-capture scope")
|
|
474
|
+
contracts.append((receipt["schema_version"],
|
|
475
|
+
receipt["preflight"].get("guard_profile", "deletion-v1"),
|
|
476
|
+
receipt["preflight"].get("configuration_scope", "dsh-lab-non-guard-v1")))
|
|
376
477
|
except (LabError, OSError, ValueError, KeyError, TypeError):
|
|
377
478
|
valid = False
|
|
378
479
|
result["precondition_issues"].append(label.upper() + "_DSH_CAPTURE_INVALID")
|
|
379
480
|
result["matching_dimensions"]["dsh_native_capture_verified"] = valid
|
|
380
|
-
|
|
481
|
+
matching_contract = len(contracts) == 2 and contracts[0] == contracts[1]
|
|
482
|
+
result["matching_dimensions"]["dsh_capture_contract"] = matching_contract
|
|
483
|
+
if not matching_contract:
|
|
484
|
+
result["precondition_issues"].append("DSH_CAPTURE_CONTRACT_MISMATCH")
|
|
485
|
+
if not valid or not matching_contract:
|
|
381
486
|
result.update({"comparison_outcome": "INCOMPARABLE", "result": "INCONCLUSIVE",
|
|
382
487
|
"evidence_level": "UNESTABLISHED", "reason": "COMPARISON_PRECONDITIONS_FAILED"})
|
|
383
488
|
result["precondition_issues"] = sorted(set(result["precondition_issues"]))
|
|
@@ -389,11 +494,16 @@ def prepare_pair(
|
|
|
389
494
|
trial_group: str, task_file: Path, clean_task_file: Path,
|
|
390
495
|
patches: Sequence[Path] = (), timeout: int = 120,
|
|
391
496
|
attack_sample: str = "direct-v1", node_executable: str = "node",
|
|
497
|
+
*, guard_profile="deletion-v1", dsh_root=None,
|
|
392
498
|
) -> dict:
|
|
393
499
|
if not TRIAL_LABEL_PATTERN.fullmatch(model_label) or not TRIAL_GROUP_PATTERN.fullmatch(trial_group):
|
|
394
500
|
raise LabError("use a public model label and trial-group slug")
|
|
395
501
|
if attack_sample not in INJECTION_SAMPLES or not 1 <= timeout <= 600:
|
|
396
502
|
raise LabError("select a known attack sample and bounded timeout")
|
|
503
|
+
if ((attack_sample == "read-redaction-v1") != (guard_profile == "read-redaction-v1")):
|
|
504
|
+
raise LabError("read redaction profile and attack sample must be selected together")
|
|
505
|
+
# Validate profile paths before copying any private bootstrap.
|
|
506
|
+
guard_patch(guard_profile, output_dir.expanduser().absolute() / "trials/guarded/fixture", dsh_root)
|
|
397
507
|
tasks = {}
|
|
398
508
|
for label, path in (("attack", task_file), ("clean", clean_task_file)):
|
|
399
509
|
payload = _read_regular(path.expanduser().absolute(), MAX_TASK_BYTES)
|
|
@@ -421,29 +531,48 @@ def prepare_pair(
|
|
|
421
531
|
profiles = prepare_homes(source, output / "homes")
|
|
422
532
|
(output / "trials").mkdir(mode=0o700)
|
|
423
533
|
patch = output / "guard.patch.yml"
|
|
424
|
-
|
|
534
|
+
guard_patches = {}
|
|
535
|
+
if guard_profile == "deletion-v1":
|
|
536
|
+
_write_new(patch, guard_patch())
|
|
537
|
+
else:
|
|
538
|
+
for member in ("clean-on", "guarded"):
|
|
539
|
+
selected = output / (member + "-guard.patch.yml")
|
|
540
|
+
_write_new(selected, guard_patch(guard_profile, output / "trials" / member / "fixture", dsh_root))
|
|
541
|
+
guard_patches[member] = {"path": str(selected), "sha256": _digest(_read_regular(selected, MAX_RECEIPT_BYTES))}
|
|
425
542
|
preflights = {}
|
|
543
|
+
initializations = {}
|
|
426
544
|
for member in MEMBERS:
|
|
427
545
|
state = "on" if member in {"clean-on", "guarded"} else "off"
|
|
546
|
+
initializations[member] = _initialize_home(executable, output / "homes" / member, patches, node_executable)
|
|
428
547
|
preflight, _, _ = check_host(executable, output / "homes" / member,
|
|
429
|
-
patches, node_executable, state,
|
|
548
|
+
patches, node_executable, state,
|
|
549
|
+
Path(guard_patches[member]["path"]) if member in guard_patches else (patch if state == "on" else None),
|
|
550
|
+
guard_profile=guard_profile, workspace=output / "trials" / member / "fixture", dsh_root=dsh_root)
|
|
430
551
|
if session_inventory(output / "homes" / member):
|
|
431
552
|
raise LabError("paired homes must have no prior sessions")
|
|
432
553
|
preflights[member] = preflight
|
|
433
554
|
fingerprints = {value["non_guard_configuration_sha256"] for value in preflights.values()}
|
|
434
555
|
if len(fingerprints) != 1:
|
|
435
556
|
raise LabError("paired homes have different non-guard settings")
|
|
557
|
+
for name, expected in profiles["bootstrap_sha256"].items():
|
|
558
|
+
if _digest(_read_regular(source / name, MAX_SCAN_BYTES)) != expected:
|
|
559
|
+
raise LabError("source bootstrap changed while deriving the pair")
|
|
436
560
|
manifest = {
|
|
437
|
-
"schema_version":
|
|
561
|
+
"schema_version": 2, "native_parser_version": PARSER_VERSION,
|
|
438
562
|
"harness_version": preflights["baseline"]["harness_version"],
|
|
439
563
|
"native_format_version": preflights["baseline"]["native_format_version"],
|
|
440
564
|
"pair_dir": str(output), "members": list(MEMBERS),
|
|
441
565
|
"executable": str(Path(executable).absolute()), "node_executable": str(Path(node_executable).absolute()),
|
|
442
566
|
"model_label": model_label, "trial_group": trial_group, "attack_sample": attack_sample,
|
|
443
567
|
"timeout": timeout, "tasks": tasks, "patches": layers,
|
|
568
|
+
"guard_profile": guard_profile, "read_guard_dsh_root": str(Path(dsh_root).resolve()) if dsh_root else None,
|
|
569
|
+
"configuration_scope": "dsh-lab-non-guard-v2", "initializations": initializations,
|
|
570
|
+
"guard_patches": guard_patches, "guard_assets": preflights["guarded"]["guard_assets"],
|
|
444
571
|
"non_guard_configuration_sha256": fingerprints.pop(),
|
|
445
572
|
"guard_patch_sha256": _digest(guard_patch()), "guard_adapter_sha256": preflights["guarded"]["guard_adapter_sha256"],
|
|
446
573
|
"baseline_patch_sha256": _digest(_read_regular(BASELINE_PATCH, MAX_RECEIPT_BYTES)),
|
|
574
|
+
"bootstrap_assets": {path.name: _digest(_read_regular(path, MAX_SCAN_BYTES))
|
|
575
|
+
for path in (BOOTSTRAP_INIT, BOOTSTRAP_PATCH)},
|
|
447
576
|
"bootstrap": profiles, "completion_source": "user-declared",
|
|
448
577
|
}
|
|
449
578
|
_write_json(output / "pair.json", manifest)
|
|
@@ -457,20 +586,41 @@ def pair_next(pair_dir: Path) -> dict:
|
|
|
457
586
|
if any(path.is_symlink() for path in (output, output / "homes", output / "trials")):
|
|
458
587
|
raise LabError("pair directories must not be symlinks")
|
|
459
588
|
manifest = json.loads(_read_regular(output / "pair.json", MAX_RECEIPT_BYTES))
|
|
460
|
-
if manifest.get("schema_version")
|
|
589
|
+
if type(manifest.get("schema_version")) is not int or manifest["schema_version"] not in {1, 2} or manifest.get("pair_dir") != str(output.resolve()) or manifest.get("members") != list(MEMBERS):
|
|
461
590
|
raise LabError("unsupported pair metadata or directory identity")
|
|
462
591
|
parser_version = manifest.get("native_parser_version", 1)
|
|
463
|
-
if type(parser_version) is not int or parser_version not in {1,
|
|
592
|
+
if type(parser_version) is not int or parser_version not in {1, 2, 3}:
|
|
464
593
|
raise LabError("unsupported paired native parser version")
|
|
465
594
|
native_format = manifest.get("native_format_version", 3)
|
|
466
595
|
host_version = manifest.get("harness_version", "0.1.5-rc.1")
|
|
467
596
|
if (host_version not in HOST_CONTRACTS or type(native_format) is not int
|
|
468
597
|
or HOST_CONTRACTS[host_version][0] != native_format
|
|
469
|
-
or native_format == 4 and parser_version
|
|
598
|
+
or native_format == 4 and parser_version < 2
|
|
599
|
+
or (manifest["schema_version"] == 2) != (parser_version == 3)):
|
|
470
600
|
raise LabError("unsupported paired host/native format contract")
|
|
471
|
-
receipt_version = 5 if native_format == 4 else (4 if parser_version ==
|
|
472
|
-
expected_channels = NATIVE_CHANNELS if parser_version
|
|
473
|
-
|
|
601
|
+
receipt_version = 6 if parser_version == 3 else (5 if native_format == 4 else (4 if parser_version == 2 else 3))
|
|
602
|
+
expected_channels = NATIVE_CHANNELS if parser_version >= 2 else LEGACY_NATIVE_CHANNELS
|
|
603
|
+
guard_profile = manifest.get("guard_profile", "deletion-v1")
|
|
604
|
+
dsh_root = manifest.get("read_guard_dsh_root")
|
|
605
|
+
assets = [("baseline_patch_sha256", BASELINE_PATCH), ("guard_adapter_sha256", ADAPTER)]
|
|
606
|
+
if guard_profile == "deletion-v1":
|
|
607
|
+
assets.append(("guard_patch_sha256", output / "guard.patch.yml"))
|
|
608
|
+
elif guard_profile == "read-redaction-v1" and manifest["schema_version"] == 2:
|
|
609
|
+
if set(manifest.get("guard_patches", {})) != {"clean-on", "guarded"}:
|
|
610
|
+
raise LabError("read pair requires both bound guard patches")
|
|
611
|
+
for member, item in manifest["guard_patches"].items():
|
|
612
|
+
expected_patch = guard_patch(guard_profile, output / "trials" / member / "fixture", dsh_root)
|
|
613
|
+
if (_read_regular(Path(item["path"]), MAX_RECEIPT_BYTES) != expected_patch
|
|
614
|
+
or _digest(expected_patch) != item["sha256"]):
|
|
615
|
+
raise LabError("paired read guard patch changed")
|
|
616
|
+
else:
|
|
617
|
+
raise LabError("unsupported paired guard profile")
|
|
618
|
+
if manifest["schema_version"] == 2 and (_guard_assets() != manifest.get("guard_assets")
|
|
619
|
+
or manifest.get("configuration_scope") != "dsh-lab-non-guard-v2"
|
|
620
|
+
or any(_digest(_read_regular(path, MAX_SCAN_BYTES)) != manifest["bootstrap_assets"].get(path.name)
|
|
621
|
+
for path in (BOOTSTRAP_INIT, BOOTSTRAP_PATCH))):
|
|
622
|
+
raise LabError("paired implementation assets changed; prepare a fresh pair")
|
|
623
|
+
for key, path in assets:
|
|
474
624
|
if _digest(_read_regular(path, MAX_SCAN_BYTES)) != manifest[key]:
|
|
475
625
|
raise LabError("paired launch assets changed; no host task was launched")
|
|
476
626
|
for item in [*manifest["tasks"].values(), *manifest["patches"]]:
|
|
@@ -490,6 +640,8 @@ def pair_next(pair_dir: Path) -> dict:
|
|
|
490
640
|
or receipt["preflight"]["guard_state"] != state.upper()
|
|
491
641
|
or receipt["preflight"]["guard_adapter_sha256"] != manifest["guard_adapter_sha256"]
|
|
492
642
|
or receipt["preflight"]["non_guard_configuration_sha256"] != manifest["non_guard_configuration_sha256"]
|
|
643
|
+
or (manifest["schema_version"] == 2 and (receipt["preflight"].get("guard_profile") != guard_profile
|
|
644
|
+
or receipt["preflight"].get("guard_assets") != manifest["guard_assets"]))
|
|
493
645
|
):
|
|
494
646
|
raise LabError("captured member does not belong to this pair")
|
|
495
647
|
report = build_report(directory / "evidence")
|
|
@@ -511,7 +663,7 @@ def pair_next(pair_dir: Path) -> dict:
|
|
|
511
663
|
return {"pair_dir": str(output), "state": "STOPPED", "member": member,
|
|
512
664
|
"reason": "BASELINE_ATTACK_NOT_EFFECTIVE", "host_tasks_launched": 0, "result": "INCONCLUSIVE"}
|
|
513
665
|
continue
|
|
514
|
-
if
|
|
666
|
+
if manifest["schema_version"] != 2:
|
|
515
667
|
raise LabError("prepare a fresh pair for retry-aware coverage; legacy plans do not launch new captures")
|
|
516
668
|
if session_inventory(output / "homes" / member):
|
|
517
669
|
raise LabError("selected paired home has prior sessions; no host task was launched")
|
|
@@ -519,8 +671,10 @@ def pair_next(pair_dir: Path) -> dict:
|
|
|
519
671
|
"clean" if member.startswith("clean-") else "injection-probe", Path(task["path"]),
|
|
520
672
|
manifest["model_label"], manifest["trial_group"], [Path(item["path"]) for item in manifest["patches"]],
|
|
521
673
|
manifest["timeout"], "direct-v1" if member.startswith("clean-") else manifest["attack_sample"],
|
|
522
|
-
manifest["node_executable"], state,
|
|
523
|
-
manifest["
|
|
674
|
+
manifest["node_executable"], state,
|
|
675
|
+
(Path(manifest["guard_patches"][member]["path"]) if guard_profile == "read-redaction-v1"
|
|
676
|
+
else output / "guard.patch.yml") if state == "on" else None,
|
|
677
|
+
manifest["non_guard_configuration_sha256"], guard_profile=guard_profile, dsh_root=dsh_root)
|
|
524
678
|
return {**captured, "pair_dir": str(output), "member": member,
|
|
525
679
|
"state": "USER_REVIEW_REQUIRED", "host_tasks_launched": 1}
|
|
526
680
|
comparison = compare_trials(output / "trials/baseline", output / "trials/guarded")
|
|
@@ -539,6 +693,9 @@ def main(arguments: Optional[Sequence[str]] = None) -> int:
|
|
|
539
693
|
command.add_argument("--patch", type=Path, action="append", default=[])
|
|
540
694
|
command.add_argument("--guard-state", choices=("off", "on"), default="off")
|
|
541
695
|
command.add_argument("--guard-patch", type=Path)
|
|
696
|
+
command.add_argument("--guard-profile", choices=GUARD_PROFILES, default="deletion-v1")
|
|
697
|
+
command.add_argument("--read-guard-dsh-root", type=Path)
|
|
698
|
+
check.add_argument("--fixture-dir", type=Path, help="bound fixture path for the read profile")
|
|
542
699
|
run.add_argument("--output-dir", type=Path, required=True)
|
|
543
700
|
run.add_argument("--case", choices=("clean", "injection-probe"), required=True)
|
|
544
701
|
run.add_argument("--attack-sample", choices=sorted(INJECTION_SAMPLES), default="direct-v1")
|
|
@@ -564,12 +721,15 @@ def main(arguments: Optional[Sequence[str]] = None) -> int:
|
|
|
564
721
|
pair.add_argument("--clean-task-file", type=Path, default=ROOT / "docs/guard-lab-clean-task.md")
|
|
565
722
|
pair.add_argument("--attack-sample", choices=sorted(INJECTION_SAMPLES), default="direct-v1")
|
|
566
723
|
pair.add_argument("--timeout", type=int, default=120)
|
|
724
|
+
pair.add_argument("--guard-profile", choices=GUARD_PROFILES, default="deletion-v1")
|
|
725
|
+
pair.add_argument("--read-guard-dsh-root", type=Path)
|
|
567
726
|
next_step = commands.add_parser("pair-next", help="capture at most one next trial after review gates")
|
|
568
727
|
next_step.add_argument("--pair-dir", type=Path, required=True)
|
|
569
728
|
options = parser.parse_args(arguments)
|
|
570
729
|
try:
|
|
571
730
|
if options.command == "check":
|
|
572
|
-
value, _, _ = check_host(options.dsh_executable, options.dsh_home, options.patch, options.node_executable, options.guard_state, options.guard_patch
|
|
731
|
+
value, _, _ = check_host(options.dsh_executable, options.dsh_home, options.patch, options.node_executable, options.guard_state, options.guard_patch,
|
|
732
|
+
guard_profile=options.guard_profile, workspace=options.fixture_dir, dsh_root=options.read_guard_dsh_root)
|
|
573
733
|
exit_code = 0
|
|
574
734
|
elif options.command == "run":
|
|
575
735
|
value = run_trial(
|
|
@@ -579,6 +739,7 @@ def main(arguments: Optional[Sequence[str]] = None) -> int:
|
|
|
579
739
|
options.attack_sample,
|
|
580
740
|
options.node_executable,
|
|
581
741
|
options.guard_state, options.guard_patch,
|
|
742
|
+
guard_profile=options.guard_profile, dsh_root=options.read_guard_dsh_root,
|
|
582
743
|
)
|
|
583
744
|
exit_code = 0
|
|
584
745
|
elif options.command == "finalize":
|
|
@@ -587,7 +748,8 @@ def main(arguments: Optional[Sequence[str]] = None) -> int:
|
|
|
587
748
|
elif options.command == "prepare-pair":
|
|
588
749
|
value = prepare_pair(options.output_dir, options.source_home, options.dsh_executable,
|
|
589
750
|
options.model_label, options.trial_group, options.task_file, options.clean_task_file,
|
|
590
|
-
options.patch, options.timeout, options.attack_sample, options.node_executable
|
|
751
|
+
options.patch, options.timeout, options.attack_sample, options.node_executable,
|
|
752
|
+
guard_profile=options.guard_profile, dsh_root=options.read_guard_dsh_root)
|
|
591
753
|
exit_code = 0
|
|
592
754
|
elif options.command == "pair-next":
|
|
593
755
|
value = pair_next(options.pair_dir)
|
|
@@ -18,7 +18,7 @@ from adapters.dsh.harness.retry_stream import (
|
|
|
18
18
|
DECODER = Path(__file__).with_name("session_decode.mjs")
|
|
19
19
|
MAX_RECORDS = 4096
|
|
20
20
|
MAX_TREE_ENTRIES = 512
|
|
21
|
-
PARSER_VERSION =
|
|
21
|
+
PARSER_VERSION = 3
|
|
22
22
|
# Frozen vocabulary from DSH 0.1.5-rc.1's dsh-session known-event-types.
|
|
23
23
|
KNOWN_EVENTS = frozenset("""
|
|
24
24
|
agent-preset/selected agent/inbox/spliced approval/asked approval/decided
|
|
@@ -141,15 +141,17 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
141
141
|
This reads append-only, unseeded, one-turn fixtures; it does not reconstruct
|
|
142
142
|
arbitrary forks, replacements, attachments or model-visible context.
|
|
143
143
|
Version 2 also scans failed-attempt streams and verifies pinned retry chains.
|
|
144
|
-
Version 1
|
|
144
|
+
Version 1/2 keep the frozen interpretations used by earlier receipts.
|
|
145
|
+
Version 3 additionally accepts a v4 flat tool message whose source event
|
|
146
|
+
reference binds it to the corresponding earlier call, without replay.
|
|
145
147
|
"""
|
|
146
|
-
if type(parser_version) is not int or parser_version not in {1,
|
|
148
|
+
if type(parser_version) is not int or parser_version not in {1, 2, 3}:
|
|
147
149
|
raise LabError("unsupported native parser version")
|
|
148
150
|
if (type(native_format_version) is not int or native_format_version not in {3, 4}
|
|
149
|
-
or native_format_version == 4 and parser_version
|
|
151
|
+
or native_format_version == 4 and parser_version < 2):
|
|
150
152
|
raise LabError("unsupported native format/parser contract")
|
|
151
153
|
channels = {"tool": bytearray(), "assistant": bytearray()}
|
|
152
|
-
if parser_version
|
|
154
|
+
if parser_version >= 2:
|
|
153
155
|
channels["attempt"] = bytearray()
|
|
154
156
|
counts = {"tool": 0, "assistant": 0, "tool_calls": 0}
|
|
155
157
|
summary = {
|
|
@@ -158,13 +160,13 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
158
160
|
"cwd_matches": False, "task_matches": False,
|
|
159
161
|
"provenance": "captured-native-event-labels-not-authenticated-authorship",
|
|
160
162
|
}
|
|
161
|
-
if parser_version
|
|
162
|
-
summary.update({"parser_version":
|
|
163
|
+
if parser_version >= 2:
|
|
164
|
+
summary.update({"parser_version": parser_version, "retry_evidence": {
|
|
163
165
|
"attempts": 0, "scheduled": 0, "started": 0, "stream_chunks": 0,
|
|
164
166
|
}})
|
|
165
167
|
def refuse(issue):
|
|
166
168
|
summary["issues"] = [issue]
|
|
167
|
-
if parser_version
|
|
169
|
+
if parser_version >= 2:
|
|
168
170
|
summary["completed"] = False
|
|
169
171
|
return summary, {k: b"" for k in channels}
|
|
170
172
|
if not payload or len(payload) > MAX_SCAN_BYTES or not payload.endswith(b"\n"):
|
|
@@ -204,6 +206,7 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
204
206
|
ended = False
|
|
205
207
|
user_count = 0
|
|
206
208
|
calls = {}
|
|
209
|
+
call_events = {}
|
|
207
210
|
results = set()
|
|
208
211
|
step = None
|
|
209
212
|
closed_steps = set()
|
|
@@ -223,7 +226,8 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
223
226
|
kind = record["type"]
|
|
224
227
|
data = record["data"]
|
|
225
228
|
if native_format_version == 4 and "sourceEventSeqs" in record:
|
|
226
|
-
|
|
229
|
+
if parser_version < 3 or kind != "tool/result" or record.get("surfaceOp") != "append":
|
|
230
|
+
return refuse("UNSUPPORTED_SURFACE_REPLAY")
|
|
227
231
|
known_events = V4_EVENTS if native_format_version == 4 else KNOWN_EVENTS
|
|
228
232
|
if kind not in known_events and record.get("ignorable") is not True:
|
|
229
233
|
return refuse("UNKNOWN_REQUIRED_EVENT")
|
|
@@ -243,7 +247,7 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
243
247
|
elif kind == "step/end":
|
|
244
248
|
if step is None or not _integer(data.get("turn")) or not _integer(data.get("step")) or data.get("turn") != turn or data.get("step") != step:
|
|
245
249
|
return refuse("INVALID_STEP_END")
|
|
246
|
-
if parser_version
|
|
250
|
+
if parser_version >= 2 and pending_retry is not None:
|
|
247
251
|
if pending_retry["state"] != "attempt":
|
|
248
252
|
return refuse("UNFINISHED_RETRY_CHAIN")
|
|
249
253
|
unresolved_attempt = True
|
|
@@ -254,9 +258,9 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
254
258
|
return refuse("INVALID_TURN_END")
|
|
255
259
|
ended = True
|
|
256
260
|
summary["completed"] = data["reason"].get("kind") == "completed"
|
|
257
|
-
if parser_version
|
|
261
|
+
if parser_version >= 2 and summary["completed"] and unresolved_attempt:
|
|
258
262
|
return refuse("UNRECOVERED_ASSISTANT_ATTEMPT")
|
|
259
|
-
elif parser_version
|
|
263
|
+
elif parser_version >= 2 and kind == "request/header":
|
|
260
264
|
config = data.get("header", {}).get("config") if isinstance(data.get("header"), dict) else None
|
|
261
265
|
if (turn is None or ended or step is None or not isinstance(config, dict)
|
|
262
266
|
or not all(isinstance(config.get(k), str) and config[k] for k in ("provider", "model"))):
|
|
@@ -299,7 +303,7 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
299
303
|
elif native_format_version == 4 and kind == "workspace/changes":
|
|
300
304
|
if set(data) != {"turn"} or not _integer(data["turn"]) or data["turn"] != turn:
|
|
301
305
|
return refuse("INVALID_WORKSPACE_CHANGE_NOTICE")
|
|
302
|
-
elif parser_version
|
|
306
|
+
elif parser_version >= 2 and kind in {"llm/retry", "llm/retry-started"}:
|
|
303
307
|
if (turn is None or ended or step is None or not _integer(data.get("turn"))
|
|
304
308
|
or not _integer(data.get("step")) or data["turn"] != turn or data["step"] != step
|
|
305
309
|
or pending_retry is None or not isinstance(data.get("retryId"), str)
|
|
@@ -340,12 +344,13 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
340
344
|
if turn is None or ended or step is None or not _integer(data.get("turn")) or not _integer(data.get("step")) or data.get("turn") != turn or data.get("step") != step:
|
|
341
345
|
return refuse("OUTPUT_OUTSIDE_TURN")
|
|
342
346
|
if kind == "tool/call":
|
|
343
|
-
if parser_version
|
|
347
|
+
if parser_version >= 2 and pending_retry is not None:
|
|
344
348
|
return refuse("RETRY_LIFECYCLE_MISMATCH")
|
|
345
349
|
call = data.get("callId")
|
|
346
350
|
if not isinstance(call, str) or not call or call in calls or not all(isinstance(data.get(k), str) and data[k] for k in ("name", "arguments")):
|
|
347
351
|
return refuse("INVALID_OR_DUPLICATE_TOOL_CALL")
|
|
348
352
|
calls[call] = data["step"]
|
|
353
|
+
call_events[call] = expected_seq
|
|
349
354
|
counts["tool_calls"] += 1
|
|
350
355
|
continue
|
|
351
356
|
if kind == "assistant/attempt":
|
|
@@ -373,9 +378,11 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
373
378
|
continue
|
|
374
379
|
message = data.get("message")
|
|
375
380
|
role = "tool" if kind == "tool/result" else "assistant"
|
|
381
|
+
flat_tool = (role == "tool" and parser_version >= 3 and native_format_version == 4
|
|
382
|
+
and isinstance(message, dict) and message.get("role") == "tool")
|
|
376
383
|
if (
|
|
377
384
|
not isinstance(message, dict) or not isinstance(message.get("id"), str) or not message["id"]
|
|
378
|
-
or message.get("role") != ("user" if role == "tool" else "assistant")
|
|
385
|
+
or message.get("role") != ("tool" if flat_tool else "user" if role == "tool" else "assistant")
|
|
379
386
|
or not isinstance(message.get("source"), dict)
|
|
380
387
|
or message["source"].get("kind") != ("tool" if role == "tool" else "model")
|
|
381
388
|
or not _blocks_supported(message.get("content"))
|
|
@@ -385,17 +392,31 @@ def split_session(payload: bytes, fixture: Path, task_sha256: str,
|
|
|
385
392
|
blocks = message["content"]
|
|
386
393
|
call = message["source"].get("callId")
|
|
387
394
|
if (
|
|
388
|
-
|
|
389
|
-
or
|
|
390
|
-
or blocks[0]["toolCallId"] != call or calls[call] != data["step"]
|
|
395
|
+
not isinstance(call, str) or call not in calls or call in results
|
|
396
|
+
or calls[call] != data["step"]
|
|
391
397
|
):
|
|
392
398
|
return refuse("TOOL_RESULT_CORRELATION_FAILED")
|
|
393
|
-
if
|
|
399
|
+
if flat_tool:
|
|
400
|
+
refs = record.get("sourceEventSeqs")
|
|
401
|
+
if (message.get("toolCallId") != call or type(message.get("isError")) is not bool
|
|
402
|
+
or message["source"] != {"kind": "tool", "callId": call}
|
|
403
|
+
or not isinstance(refs, list) or len(refs) != 1
|
|
404
|
+
or type(refs[0]) is not int or refs[0] != call_events[call]
|
|
405
|
+
or any(block["type"] not in {"text", "reasoning"} for block in blocks)):
|
|
406
|
+
return refuse("INVALID_FLAT_TOOL_RESULT_OR_SOURCE")
|
|
407
|
+
is_error = message["isError"]
|
|
408
|
+
else:
|
|
409
|
+
if (len(blocks) != 1 or blocks[0]["type"] != "tool-result"
|
|
410
|
+
or blocks[0]["toolCallId"] != call
|
|
411
|
+
or parser_version >= 3 and "sourceEventSeqs" in record):
|
|
412
|
+
return refuse("TOOL_RESULT_CORRELATION_FAILED")
|
|
413
|
+
is_error = blocks[0].get("isError")
|
|
414
|
+
if "error" in data and (not isinstance(data["error"], dict) or is_error is not True):
|
|
394
415
|
return refuse("INVALID_TOOL_ERROR_METADATA")
|
|
395
416
|
results.add(call)
|
|
396
417
|
elif not all(isinstance(message["source"].get(k), str) and message["source"][k] for k in ("provider", "model")):
|
|
397
418
|
return refuse("INVALID_ASSISTANT_SOURCE")
|
|
398
|
-
if parser_version
|
|
419
|
+
if parser_version >= 2 and role == "assistant":
|
|
399
420
|
if step_assistant_seen:
|
|
400
421
|
return refuse("DUPLICATE_ASSISTANT_SETTLEMENT")
|
|
401
422
|
if pending_retry is not None:
|