@mokuyoaxis/agent-guard 0.2.2 → 0.2.3-rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -19
- package/README.zh-CN.md +52 -15
- package/adapters/claude/README.md +5 -5
- package/adapters/dsh/README.md +81 -4
- package/adapters/dsh/harness/README.md +328 -0
- package/adapters/dsh/harness/baseline.patch.yml +8 -0
- package/adapters/dsh/harness/guard_lab.py +609 -0
- package/adapters/dsh/harness/native_session.py +482 -0
- package/adapters/dsh/harness/profiles.py +160 -0
- package/adapters/dsh/harness/read_fixture.py +60 -0
- package/adapters/dsh/harness/retry_stream.py +171 -0
- package/adapters/dsh/harness/session_decode.mjs +117 -0
- package/adapters/dsh/lib/index.js +46 -5
- package/adapters/dsh/lib/read_result_guard.js +207 -0
- package/adapters/dsh/lib/read_result_guard.py +139 -0
- package/adapters/harness_support.py +109 -0
- package/adapters/kimi-code/README.md +7 -0
- package/adapters/kimi-code/harness/README.md +126 -0
- package/adapters/kimi-code/harness/guard_lab.py +430 -0
- package/core/classifier.py +90 -23
- package/core/dialects.py +6 -5
- package/core/lab.py +2373 -0
- package/docs/architecture.md +102 -12
- package/docs/compatibility.md +10 -5
- package/docs/friction.md +95 -0
- package/docs/guard-lab-clean-task.md +4 -0
- package/docs/guard-lab-trial-task.md +5 -0
- package/docs/guard-lab.md +391 -0
- package/docs/harness-capabilities.md +7 -4
- package/docs/release-notes-0.2.3-rc1.md +92 -0
- package/docs/release-notes-0.2.3-rc2.md +189 -0
- package/docs/release-notes-0.2.3.md +45 -0
- package/docs/test-report-dsh-0.2-upgrade.md +142 -0
- package/docs/test-report-dsh-guard-lab.md +154 -0
- package/docs/test-report-dsh-native-session.md +118 -0
- package/docs/test-report-dsh-paired-lab.md +183 -0
- package/docs/test-report-dsh-read-redaction.md +114 -0
- package/docs/test-report-dsh-real-followup.md +170 -0
- package/docs/test-report-dsh-retry-evidence.md +134 -0
- package/docs/test-report-kimi-guard-lab.md +117 -0
- package/docs/test-report-release-readiness-0.2.3.md +161 -0
- package/docs/test-report-zcode-glm-flash.md +8 -1
- package/docs/threat-model.md +42 -0
- package/guard_lab.py +14 -0
- package/package.json +8 -5
|
@@ -0,0 +1,609 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Capture, review and compare bounded DSH headless Lab trials.
|
|
3
|
+
|
|
4
|
+
This runner deliberately keeps real-host completion user-declared. A zero
|
|
5
|
+
exit status and a nonempty capture are prerequisites, not proof of completion.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
import shutil
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
from typing import Optional, Sequence
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
ROOT = Path(__file__).resolve().parents[3]
|
|
21
|
+
sys.path.insert(0, str(ROOT))
|
|
22
|
+
|
|
23
|
+
from core.lab import ( # noqa: E402
|
|
24
|
+
INJECTION_SAMPLES, LabError, MAX_SCAN_BYTES, MAX_TASK_BYTES, TRIAL_GROUP_PATTERN,
|
|
25
|
+
TRIAL_LABEL_PATTERN, arm_observer, build_report, compare_runs,
|
|
26
|
+
prepare_run, record_host_result, request_stop, scan_bytes,
|
|
27
|
+
)
|
|
28
|
+
from adapters.harness_support import ( # noqa: E402
|
|
29
|
+
capture_process as _capture_process, digest as _digest,
|
|
30
|
+
read_regular as _read_regular, write_new as _write_new,
|
|
31
|
+
write_json as _write_json,
|
|
32
|
+
)
|
|
33
|
+
from adapters.dsh.harness.native_session import ( # noqa: E402
|
|
34
|
+
PARSER_VERSION, capture_session, check_decoder, session_inventory, split_session,
|
|
35
|
+
)
|
|
36
|
+
from adapters.dsh.harness.profiles import ( # noqa: E402
|
|
37
|
+
ADAPTER, MEMBERS, configuration_fingerprint, guard_patch,
|
|
38
|
+
normalize_composition, prepare_homes,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
TESTED_VERSION = "0.2.0-rc.2"
|
|
43
|
+
HOST_CONTRACTS = {"0.1.5-rc.1": (3, 4), "0.2.0-rc.2": (4, 5)}
|
|
44
|
+
BASELINE_PATCH = Path(__file__).with_name("baseline.patch.yml")
|
|
45
|
+
MAX_RECEIPT_BYTES = 64 * 1024
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _environment(home: Path) -> dict[str, str]:
|
|
49
|
+
environment = dict(os.environ)
|
|
50
|
+
# A previous sentinel/workspace must not redirect this independent trial.
|
|
51
|
+
for name in tuple(environment):
|
|
52
|
+
if name.startswith("AGENT_GUARD_"):
|
|
53
|
+
environment.pop(name)
|
|
54
|
+
environment["DSH_HOME"] = str(home)
|
|
55
|
+
environment["DSH_TELEMETRY_DISABLED"] = "1"
|
|
56
|
+
return environment
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _host_options(
|
|
60
|
+
executable: str, home: Path, patches: Sequence[Path],
|
|
61
|
+
guard_state: str = "off", guard_patch_file: Optional[Path] = None,
|
|
62
|
+
) -> tuple[list[str], dict[str, str]]:
|
|
63
|
+
if os.name != "posix":
|
|
64
|
+
raise LabError("this DSH runner currently supports POSIX hosts only")
|
|
65
|
+
resolved = shutil.which(executable)
|
|
66
|
+
if not resolved:
|
|
67
|
+
raise LabError("DSH executable was not found")
|
|
68
|
+
resolved = str(Path(resolved).absolute())
|
|
69
|
+
home = home.expanduser().resolve()
|
|
70
|
+
if not (home / "profiles" / "headless" / "package.json").is_file():
|
|
71
|
+
raise LabError("select an already initialized disposable headless DSH home")
|
|
72
|
+
if home == (Path.home() / ".dsh").resolve():
|
|
73
|
+
raise LabError("use a disposable DSH home, not the default user home")
|
|
74
|
+
argv = [resolved, "--profile", "headless"]
|
|
75
|
+
if guard_state not in {"off", "on"} or (guard_state == "on") != (guard_patch_file is not None):
|
|
76
|
+
raise LabError("guard-on requires the generated local guard patch; guard-off omits it")
|
|
77
|
+
if guard_patch_file is not None and _read_regular(guard_patch_file, MAX_RECEIPT_BYTES) != guard_patch():
|
|
78
|
+
raise LabError("select the exact generated local Guard patch")
|
|
79
|
+
layers = (*patches, BASELINE_PATCH, *((guard_patch_file,) if guard_patch_file else ()))
|
|
80
|
+
for patch in layers:
|
|
81
|
+
selected = patch.expanduser().absolute()
|
|
82
|
+
_read_regular(selected, MAX_RECEIPT_BYTES)
|
|
83
|
+
argv.extend(["--patch", str(selected)])
|
|
84
|
+
return argv, _environment(home)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _config_dump(
|
|
88
|
+
argv: Sequence[str], cwd: Path, environment: dict[str, str],
|
|
89
|
+
guard_state: str = "off",
|
|
90
|
+
) -> bytes:
|
|
91
|
+
result, output = _capture_process(
|
|
92
|
+
[*argv, "--dump-config"], cwd, environment, 15,
|
|
93
|
+
)
|
|
94
|
+
if (
|
|
95
|
+
result["exit_code"] != 0 or result["timed_out"]
|
|
96
|
+
or result["output_limit_exceeded"] or not output["stdout"].strip()
|
|
97
|
+
):
|
|
98
|
+
raise LabError("DSH config preflight failed; raw diagnostics are not printed")
|
|
99
|
+
normalize_composition(output["stdout"], guard_state)
|
|
100
|
+
return output["stdout"]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def check_host(
|
|
104
|
+
executable: str, home: Path, patches: Sequence[Path] = (),
|
|
105
|
+
node_executable: str = "node",
|
|
106
|
+
guard_state: str = "off", guard_patch_file: Optional[Path] = None,
|
|
107
|
+
) -> tuple[dict, list[str], dict[str, str]]:
|
|
108
|
+
home = home.expanduser().resolve()
|
|
109
|
+
argv, environment = _host_options(executable, home, patches, guard_state, guard_patch_file)
|
|
110
|
+
result, output = _capture_process([argv[0], "--version"], home.resolve(), environment, 10)
|
|
111
|
+
version = output["stdout"].strip().decode("ascii", errors="replace")
|
|
112
|
+
if (
|
|
113
|
+
result["exit_code"] != 0 or result["timed_out"]
|
|
114
|
+
or result["output_limit_exceeded"]
|
|
115
|
+
or version not in HOST_CONTRACTS
|
|
116
|
+
):
|
|
117
|
+
raise LabError("DSH version preflight requires a reviewed exact host version")
|
|
118
|
+
configuration = _config_dump(argv, home.resolve(), environment, guard_state)
|
|
119
|
+
decoder, _ = check_decoder(node_executable, home, environment)
|
|
120
|
+
return {
|
|
121
|
+
"harness_version": version,
|
|
122
|
+
"native_format_version": HOST_CONTRACTS[version][0],
|
|
123
|
+
"profile": "headless", "guard_state": guard_state.upper(),
|
|
124
|
+
"guard_state_source": "user-declared",
|
|
125
|
+
"composition_bytes": len(configuration),
|
|
126
|
+
"composition_sha256": _digest(configuration),
|
|
127
|
+
"telemetry_disabled": True, "configuration_retained": False,
|
|
128
|
+
"model_identity_source": "user-declared",
|
|
129
|
+
"native_decoder": decoder,
|
|
130
|
+
"non_guard_configuration_sha256": configuration_fingerprint(home, configuration, environment, guard_state),
|
|
131
|
+
"configuration_scope": "dsh-lab-non-guard-v1",
|
|
132
|
+
"guard_adapter_sha256": _digest(_read_regular(ADAPTER, MAX_SCAN_BYTES)),
|
|
133
|
+
}, argv, environment
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def run_trial(
|
|
137
|
+
output_dir: Path, home: Path, executable: str, case_id: str,
|
|
138
|
+
task_file: Path, model_label: str, trial_group: str,
|
|
139
|
+
patches: Sequence[Path] = (), timeout: int = 120,
|
|
140
|
+
attack_sample: str = "direct-v1",
|
|
141
|
+
node_executable: str = "node",
|
|
142
|
+
guard_state: str = "off", guard_patch_file: Optional[Path] = None,
|
|
143
|
+
expected_configuration_sha256: Optional[str] = None,
|
|
144
|
+
) -> dict:
|
|
145
|
+
if case_id not in {"clean", "injection-probe"}:
|
|
146
|
+
raise LabError("this runner supports clean and injection-probe only")
|
|
147
|
+
if not 1 <= timeout <= 600:
|
|
148
|
+
raise LabError("host timeout must be between 1 and 600 seconds")
|
|
149
|
+
if attack_sample not in INJECTION_SAMPLES or (case_id != "injection-probe" and attack_sample != "direct-v1"):
|
|
150
|
+
raise LabError("select a known attack sample for injection-probe only")
|
|
151
|
+
if not TRIAL_LABEL_PATTERN.fullmatch(model_label):
|
|
152
|
+
raise LabError("model label must follow the Lab public-label format")
|
|
153
|
+
if not TRIAL_GROUP_PATTERN.fullmatch(trial_group):
|
|
154
|
+
raise LabError("trial group must follow the Lab slug format")
|
|
155
|
+
task = _read_regular(task_file.expanduser().absolute(), MAX_TASK_BYTES)
|
|
156
|
+
try:
|
|
157
|
+
prompt = task.decode("utf-8")
|
|
158
|
+
except UnicodeError as exc:
|
|
159
|
+
raise LabError("task must be UTF-8") from exc
|
|
160
|
+
if not prompt.strip() or "\x00" in prompt:
|
|
161
|
+
raise LabError("task must be nonempty text without NUL bytes")
|
|
162
|
+
output = output_dir.expanduser().absolute()
|
|
163
|
+
home = home.expanduser().resolve()
|
|
164
|
+
if output.is_symlink() or output.exists() or not output.parent.is_dir():
|
|
165
|
+
raise LabError("output directory must be new with an existing parent")
|
|
166
|
+
output = output.resolve()
|
|
167
|
+
if output == home or output in home.parents or home in output.parents:
|
|
168
|
+
raise LabError("trial output and the selected DSH home must be disjoint")
|
|
169
|
+
preflight, argv, environment = check_host(executable, home, patches, node_executable, guard_state, guard_patch_file)
|
|
170
|
+
if expected_configuration_sha256 is not None and preflight["non_guard_configuration_sha256"] != expected_configuration_sha256:
|
|
171
|
+
raise LabError("paired non-guard settings changed; no host task was launched")
|
|
172
|
+
before = session_inventory(home)
|
|
173
|
+
selected_node = str(Path(shutil.which(node_executable)).absolute())
|
|
174
|
+
output.mkdir(mode=0o700)
|
|
175
|
+
capture = output / "capture"
|
|
176
|
+
capture.mkdir(mode=0o700)
|
|
177
|
+
metadata = {}
|
|
178
|
+
if case_id == "injection-probe":
|
|
179
|
+
metadata = {
|
|
180
|
+
"harness_label": "DSH", "harness_version": preflight["harness_version"],
|
|
181
|
+
"model_label": model_label, "guard_state": guard_state,
|
|
182
|
+
"trial_group": trial_group, "task_file": task_file,
|
|
183
|
+
"non_guard_configuration_sha256": preflight["non_guard_configuration_sha256"],
|
|
184
|
+
}
|
|
185
|
+
prepared = prepare_run(
|
|
186
|
+
output / "fixture", output / "evidence", case_id,
|
|
187
|
+
model_usage="real", attack_sample=attack_sample, **metadata,
|
|
188
|
+
)
|
|
189
|
+
if metadata and prepared["trial"]["task_sha256"] != _digest(task):
|
|
190
|
+
raise LabError("task changed during preparation; no host task was launched")
|
|
191
|
+
evidence = Path(prepared["evidence_dir"])
|
|
192
|
+
fixture = Path(prepared["fixture_dir"])
|
|
193
|
+
# Startup handles its own failure/stop request. Only a ready observer can
|
|
194
|
+
# be stopped here; otherwise cleanup would hide the original startup error.
|
|
195
|
+
arm_observer(evidence, timeout + 15)
|
|
196
|
+
try:
|
|
197
|
+
# Task bytes used for hashing are exactly the bytes passed to the host.
|
|
198
|
+
result, streams = _capture_process([*argv, prompt], fixture, environment, timeout)
|
|
199
|
+
finally:
|
|
200
|
+
request_stop(evidence)
|
|
201
|
+
composition_unchanged = False
|
|
202
|
+
try:
|
|
203
|
+
after = _config_dump(argv, home, environment, guard_state)
|
|
204
|
+
composition_unchanged = (
|
|
205
|
+
_digest(after) == preflight["composition_sha256"]
|
|
206
|
+
and configuration_fingerprint(home, after, environment, guard_state)
|
|
207
|
+
== preflight["non_guard_configuration_sha256"]
|
|
208
|
+
and _digest(_read_regular(ADAPTER, MAX_SCAN_BYTES)) == preflight["guard_adapter_sha256"]
|
|
209
|
+
)
|
|
210
|
+
except LabError:
|
|
211
|
+
pass
|
|
212
|
+
captures = {}
|
|
213
|
+
for name, payload in streams.items():
|
|
214
|
+
_write_new(capture / (name + ".bin"), payload)
|
|
215
|
+
captures[name] = {"bytes": len(payload), "sha256": _digest(payload)}
|
|
216
|
+
native, native_captures = capture_session(
|
|
217
|
+
home, before, capture, fixture, _digest(task), selected_node, environment,
|
|
218
|
+
native_format_version=preflight["native_format_version"],
|
|
219
|
+
)
|
|
220
|
+
captures.update(native_captures)
|
|
221
|
+
receipt = {
|
|
222
|
+
"schema_version": HOST_CONTRACTS[preflight["harness_version"]][1],
|
|
223
|
+
"run_id": prepared["run_id"], "case_id": case_id,
|
|
224
|
+
**({"attack_sample": prepared["attack_sample"]} if "attack_sample" in prepared else {}),
|
|
225
|
+
"task_sha256": _digest(task), "task_bytes": len(task),
|
|
226
|
+
"model_label": model_label, "trial_group": trial_group,
|
|
227
|
+
"task_text_retained": "session" in captures, "preflight": preflight,
|
|
228
|
+
"composition_unchanged": composition_unchanged,
|
|
229
|
+
"host": result, "captures": captures,
|
|
230
|
+
"native_session": native,
|
|
231
|
+
"completion": "USER_REVIEW_REQUIRED",
|
|
232
|
+
"raw_host_output_retained": True,
|
|
233
|
+
}
|
|
234
|
+
_write_json(capture / "receipt.json", receipt)
|
|
235
|
+
report = build_report(evidence)
|
|
236
|
+
return {
|
|
237
|
+
"output_dir": str(output), "run_id": prepared["run_id"],
|
|
238
|
+
"case_id": case_id, "completion": receipt["completion"],
|
|
239
|
+
"host": result, "composition_unchanged": composition_unchanged,
|
|
240
|
+
"native_session": native,
|
|
241
|
+
"report_result": report["result"],
|
|
242
|
+
"raw_host_output_retained": True,
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _read_capture(output_dir: Path) -> tuple[dict, dict, dict, Optional[dict], bool]:
|
|
247
|
+
output = output_dir.expanduser().absolute()
|
|
248
|
+
if (
|
|
249
|
+
output.is_symlink() or (output / "capture").is_symlink()
|
|
250
|
+
or (output / "evidence").is_symlink()
|
|
251
|
+
):
|
|
252
|
+
raise LabError("trial directories must not be symlinks")
|
|
253
|
+
capture = output / "capture"
|
|
254
|
+
receipt = json.loads(_read_regular(capture / "receipt.json", MAX_RECEIPT_BYTES))
|
|
255
|
+
run = json.loads(_read_regular(output / "evidence" / "run.json", MAX_RECEIPT_BYTES))
|
|
256
|
+
if not isinstance(receipt, dict) or not isinstance(run, dict):
|
|
257
|
+
raise LabError("capture metadata must be JSON objects")
|
|
258
|
+
if type(receipt.get("schema_version")) is not int or receipt["schema_version"] not in {1, 2, 3, 4, 5}:
|
|
259
|
+
raise LabError("unsupported capture receipt version")
|
|
260
|
+
host_version = receipt.get("preflight", {}).get("harness_version")
|
|
261
|
+
native_format = 4 if receipt["schema_version"] == 5 else 3
|
|
262
|
+
if receipt["schema_version"] >= 3:
|
|
263
|
+
expected_host = "0.2.0-rc.2" if native_format == 4 else "0.1.5-rc.1"
|
|
264
|
+
if host_version != expected_host or (native_format == 4 and
|
|
265
|
+
receipt["preflight"].get("native_format_version") != 4):
|
|
266
|
+
raise LabError("capture host and native format contract do not match")
|
|
267
|
+
if receipt.get("run_id") != run.get("run_id") or receipt.get("case_id") != run.get("case_id"):
|
|
268
|
+
raise LabError("capture and Lab run identities do not match")
|
|
269
|
+
if (
|
|
270
|
+
Path(run["evidence_dir"]).resolve() != (output / "evidence").resolve()
|
|
271
|
+
or Path(run["fixture_dir"]).resolve() != (output / "fixture").resolve()
|
|
272
|
+
or run.get("case_id") not in {"clean", "injection-probe"}
|
|
273
|
+
):
|
|
274
|
+
raise LabError("capture paths or case do not match this trial")
|
|
275
|
+
payloads = {}
|
|
276
|
+
names = {"stdout": "stdout.bin", "stderr": "stderr.bin"}
|
|
277
|
+
if receipt["schema_version"] >= 2:
|
|
278
|
+
for name, filename in (("session", "session.bin"), ("session_jsonl", "session.jsonl")):
|
|
279
|
+
if name in receipt["captures"]:
|
|
280
|
+
names[name] = filename
|
|
281
|
+
for name, filename in names.items():
|
|
282
|
+
payload = _read_regular(capture / filename, MAX_SCAN_BYTES)
|
|
283
|
+
expected = receipt["captures"][name]
|
|
284
|
+
if expected != {"bytes": len(payload), "sha256": _digest(payload)}:
|
|
285
|
+
raise LabError("host capture integrity check failed")
|
|
286
|
+
payloads[name] = payload
|
|
287
|
+
roles = None
|
|
288
|
+
native_ready = False
|
|
289
|
+
if isinstance(run.get("trial"), dict) and receipt["task_sha256"] != run["trial"]["task_sha256"]:
|
|
290
|
+
raise LabError("capture task fingerprint differs from the Lab trial")
|
|
291
|
+
if receipt["schema_version"] >= 3 and isinstance(run.get("trial"), dict):
|
|
292
|
+
trial = run["trial"]
|
|
293
|
+
if any(trial.get(key) != value for key, value in {
|
|
294
|
+
"harness_label": "DSH", "harness_version": host_version,
|
|
295
|
+
"guard_state": receipt["preflight"]["guard_state"],
|
|
296
|
+
"non_guard_configuration_sha256": receipt["preflight"]["non_guard_configuration_sha256"],
|
|
297
|
+
"model_label": receipt["model_label"], "trial_group": receipt["trial_group"],
|
|
298
|
+
}.items()):
|
|
299
|
+
raise LabError("capture configuration or identity differs from the Lab trial")
|
|
300
|
+
if receipt["schema_version"] >= 2:
|
|
301
|
+
native = receipt["native_session"]
|
|
302
|
+
if not isinstance(native, dict):
|
|
303
|
+
raise LabError("native capture metadata must be an object")
|
|
304
|
+
if "session_jsonl" in payloads:
|
|
305
|
+
if "session" not in payloads or native.get("status") != "CAPTURED" or native.get("compression") not in {"none", "zstd"}:
|
|
306
|
+
raise LabError("native capture is missing its original artifact or format")
|
|
307
|
+
summary, roles = split_session(payloads["session_jsonl"], Path(run["fixture_dir"]), receipt["task_sha256"],
|
|
308
|
+
parser_version=PARSER_VERSION if receipt["schema_version"] >= 4 else 1,
|
|
309
|
+
native_format_version=native_format)
|
|
310
|
+
if native.get("parser") != summary or native.get("recognized") is not summary["recognized"]:
|
|
311
|
+
raise LabError("native session interpretation differs from the receipt")
|
|
312
|
+
native_ready = native.get("status") == "CAPTURED" and summary["recognized"] and summary["completed"]
|
|
313
|
+
return receipt, run, payloads, roles, native_ready
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def finalize_trial(output_dir: Path, status: str) -> dict:
|
|
317
|
+
output = output_dir.expanduser().absolute()
|
|
318
|
+
receipt, run, payloads, roles, native_ready = _read_capture(output)
|
|
319
|
+
if status == "completed" and receipt["schema_version"] >= 2 and not native_ready:
|
|
320
|
+
raise LabError("missing, ambiguous, unsupported or unfinished native session cannot be completed")
|
|
321
|
+
host = receipt["host"]
|
|
322
|
+
if status == "completed" and (
|
|
323
|
+
host["exit_code"] != 0 or host["timed_out"] or host["start_failed"]
|
|
324
|
+
or host["output_limit_exceeded"] or not receipt["composition_unchanged"]
|
|
325
|
+
or not payloads["stdout"].strip()
|
|
326
|
+
):
|
|
327
|
+
raise LabError("failed, empty, changed or incomplete capture cannot be declared completed")
|
|
328
|
+
if host["timed_out"] and status != "timed-out":
|
|
329
|
+
raise LabError("a timed-out capture must be recorded as timed-out")
|
|
330
|
+
exit_code = host["exit_code"]
|
|
331
|
+
if not isinstance(exit_code, int) or not 0 <= exit_code <= 255:
|
|
332
|
+
exit_code = None
|
|
333
|
+
evidence = output / "evidence"
|
|
334
|
+
record_host_result(evidence, status, exit_code)
|
|
335
|
+
for name in ("stdout", "stderr"):
|
|
336
|
+
scan_bytes(evidence, payloads[name], "host-output", source_kind="dsh-headless-" + name)
|
|
337
|
+
if roles is not None and receipt["native_session"].get("recognized"):
|
|
338
|
+
scan_bytes(evidence, roles["tool"], "tool-output", source_kind="dsh-native-tool-result")
|
|
339
|
+
scan_bytes(evidence, roles["assistant"], "model-output", source_kind="dsh-native-assistant-content")
|
|
340
|
+
if receipt["schema_version"] >= 4:
|
|
341
|
+
scan_bytes(evidence, roles["attempt"], "host-output", source_kind="dsh-native-assistant-attempt-stream")
|
|
342
|
+
elif "session_jsonl" in payloads:
|
|
343
|
+
scan_bytes(evidence, payloads["session_jsonl"], "host-output", source_kind="dsh-native-unrecognized-session")
|
|
344
|
+
return build_report(evidence)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
LEGACY_NATIVE_CHANNELS = sorted([
|
|
348
|
+
{"stage": "host-output", "source_kind": "dsh-headless-stdout"},
|
|
349
|
+
{"stage": "host-output", "source_kind": "dsh-headless-stderr"},
|
|
350
|
+
{"stage": "tool-output", "source_kind": "dsh-native-tool-result"},
|
|
351
|
+
{"stage": "model-output", "source_kind": "dsh-native-assistant-content"},
|
|
352
|
+
], key=lambda item: (item["stage"], item["source_kind"]))
|
|
353
|
+
NATIVE_CHANNELS = sorted([
|
|
354
|
+
*LEGACY_NATIVE_CHANNELS,
|
|
355
|
+
{"stage": "host-output", "source_kind": "dsh-native-assistant-attempt-stream"},
|
|
356
|
+
], key=lambda item: (item["stage"], item["source_kind"]))
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def compare_trials(baseline_dir: Path, guarded_dir: Path) -> dict:
|
|
360
|
+
result = compare_runs(baseline_dir / "evidence", guarded_dir / "evidence")
|
|
361
|
+
valid = True
|
|
362
|
+
for label, directory, state in (("baseline", baseline_dir, "OFF"), ("guarded", guarded_dir, "ON")):
|
|
363
|
+
try:
|
|
364
|
+
receipt, run, _, _, ready = _read_capture(directory)
|
|
365
|
+
report = result[label]
|
|
366
|
+
if (
|
|
367
|
+
receipt["schema_version"] not in {3, 4, 5} or not ready
|
|
368
|
+
or receipt["preflight"]["guard_state"] != state
|
|
369
|
+
or not receipt["composition_unchanged"]
|
|
370
|
+
or run["case_id"] != "injection-probe"
|
|
371
|
+
or report["scanned_channels"] != (NATIVE_CHANNELS if receipt["schema_version"] >= 4 else LEGACY_NATIVE_CHANNELS)
|
|
372
|
+
or receipt["host"]["exit_code"] != 0
|
|
373
|
+
or any(receipt["host"][key] for key in ("timed_out", "start_failed", "output_limit_exceeded"))
|
|
374
|
+
):
|
|
375
|
+
raise LabError("trial is outside the paired native-capture scope")
|
|
376
|
+
except (LabError, OSError, ValueError, KeyError, TypeError):
|
|
377
|
+
valid = False
|
|
378
|
+
result["precondition_issues"].append(label.upper() + "_DSH_CAPTURE_INVALID")
|
|
379
|
+
result["matching_dimensions"]["dsh_native_capture_verified"] = valid
|
|
380
|
+
if not valid:
|
|
381
|
+
result.update({"comparison_outcome": "INCOMPARABLE", "result": "INCONCLUSIVE",
|
|
382
|
+
"evidence_level": "UNESTABLISHED", "reason": "COMPARISON_PRECONDITIONS_FAILED"})
|
|
383
|
+
result["precondition_issues"] = sorted(set(result["precondition_issues"]))
|
|
384
|
+
return result
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def prepare_pair(
|
|
388
|
+
output_dir: Path, source_home: Path, executable: str, model_label: str,
|
|
389
|
+
trial_group: str, task_file: Path, clean_task_file: Path,
|
|
390
|
+
patches: Sequence[Path] = (), timeout: int = 120,
|
|
391
|
+
attack_sample: str = "direct-v1", node_executable: str = "node",
|
|
392
|
+
) -> dict:
|
|
393
|
+
if not TRIAL_LABEL_PATTERN.fullmatch(model_label) or not TRIAL_GROUP_PATTERN.fullmatch(trial_group):
|
|
394
|
+
raise LabError("use a public model label and trial-group slug")
|
|
395
|
+
if attack_sample not in INJECTION_SAMPLES or not 1 <= timeout <= 600:
|
|
396
|
+
raise LabError("select a known attack sample and bounded timeout")
|
|
397
|
+
tasks = {}
|
|
398
|
+
for label, path in (("attack", task_file), ("clean", clean_task_file)):
|
|
399
|
+
payload = _read_regular(path.expanduser().absolute(), MAX_TASK_BYTES)
|
|
400
|
+
text = payload.decode("utf-8")
|
|
401
|
+
if not text.strip() or "\x00" in text:
|
|
402
|
+
raise LabError("tasks must be nonempty UTF-8 without NUL bytes")
|
|
403
|
+
tasks[label] = {"path": str(path.expanduser().absolute()), "sha256": _digest(payload)}
|
|
404
|
+
layers = [{"path": str(path.expanduser().absolute()),
|
|
405
|
+
"sha256": _digest(_read_regular(path.expanduser().absolute(), MAX_RECEIPT_BYTES))} for path in patches]
|
|
406
|
+
output = output_dir.expanduser().absolute()
|
|
407
|
+
if output.exists() or output.is_symlink() or not output.parent.is_dir():
|
|
408
|
+
raise LabError("pair directory must be new with an existing parent")
|
|
409
|
+
if source_home.expanduser().is_symlink():
|
|
410
|
+
raise LabError("source DSH home must not be a symlink")
|
|
411
|
+
source = source_home.expanduser().resolve()
|
|
412
|
+
output = output.resolve()
|
|
413
|
+
if output == source or output in source.parents or source in output.parents:
|
|
414
|
+
raise LabError("pair directory and source home must be disjoint")
|
|
415
|
+
# Check external inputs before creating any private profiles.
|
|
416
|
+
executable = shutil.which(executable)
|
|
417
|
+
node_executable = shutil.which(node_executable)
|
|
418
|
+
if not executable or not node_executable:
|
|
419
|
+
raise LabError("DSH and decoder executables must be available")
|
|
420
|
+
output.mkdir(mode=0o700)
|
|
421
|
+
profiles = prepare_homes(source, output / "homes")
|
|
422
|
+
(output / "trials").mkdir(mode=0o700)
|
|
423
|
+
patch = output / "guard.patch.yml"
|
|
424
|
+
_write_new(patch, guard_patch())
|
|
425
|
+
preflights = {}
|
|
426
|
+
for member in MEMBERS:
|
|
427
|
+
state = "on" if member in {"clean-on", "guarded"} else "off"
|
|
428
|
+
preflight, _, _ = check_host(executable, output / "homes" / member,
|
|
429
|
+
patches, node_executable, state, patch if state == "on" else None)
|
|
430
|
+
if session_inventory(output / "homes" / member):
|
|
431
|
+
raise LabError("paired homes must have no prior sessions")
|
|
432
|
+
preflights[member] = preflight
|
|
433
|
+
fingerprints = {value["non_guard_configuration_sha256"] for value in preflights.values()}
|
|
434
|
+
if len(fingerprints) != 1:
|
|
435
|
+
raise LabError("paired homes have different non-guard settings")
|
|
436
|
+
manifest = {
|
|
437
|
+
"schema_version": 1, "native_parser_version": PARSER_VERSION,
|
|
438
|
+
"harness_version": preflights["baseline"]["harness_version"],
|
|
439
|
+
"native_format_version": preflights["baseline"]["native_format_version"],
|
|
440
|
+
"pair_dir": str(output), "members": list(MEMBERS),
|
|
441
|
+
"executable": str(Path(executable).absolute()), "node_executable": str(Path(node_executable).absolute()),
|
|
442
|
+
"model_label": model_label, "trial_group": trial_group, "attack_sample": attack_sample,
|
|
443
|
+
"timeout": timeout, "tasks": tasks, "patches": layers,
|
|
444
|
+
"non_guard_configuration_sha256": fingerprints.pop(),
|
|
445
|
+
"guard_patch_sha256": _digest(guard_patch()), "guard_adapter_sha256": preflights["guarded"]["guard_adapter_sha256"],
|
|
446
|
+
"baseline_patch_sha256": _digest(_read_regular(BASELINE_PATCH, MAX_RECEIPT_BYTES)),
|
|
447
|
+
"bootstrap": profiles, "completion_source": "user-declared",
|
|
448
|
+
}
|
|
449
|
+
_write_json(output / "pair.json", manifest)
|
|
450
|
+
return {"pair_dir": str(output), "state": "READY", "host_tasks_launched": 0,
|
|
451
|
+
"non_guard_configuration_sha256": manifest["non_guard_configuration_sha256"],
|
|
452
|
+
"members": list(MEMBERS), "source_modified": False}
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def pair_next(pair_dir: Path) -> dict:
|
|
456
|
+
output = pair_dir.expanduser().absolute()
|
|
457
|
+
if any(path.is_symlink() for path in (output, output / "homes", output / "trials")):
|
|
458
|
+
raise LabError("pair directories must not be symlinks")
|
|
459
|
+
manifest = json.loads(_read_regular(output / "pair.json", MAX_RECEIPT_BYTES))
|
|
460
|
+
if manifest.get("schema_version") != 1 or manifest.get("pair_dir") != str(output.resolve()) or manifest.get("members") != list(MEMBERS):
|
|
461
|
+
raise LabError("unsupported pair metadata or directory identity")
|
|
462
|
+
parser_version = manifest.get("native_parser_version", 1)
|
|
463
|
+
if type(parser_version) is not int or parser_version not in {1, PARSER_VERSION}:
|
|
464
|
+
raise LabError("unsupported paired native parser version")
|
|
465
|
+
native_format = manifest.get("native_format_version", 3)
|
|
466
|
+
host_version = manifest.get("harness_version", "0.1.5-rc.1")
|
|
467
|
+
if (host_version not in HOST_CONTRACTS or type(native_format) is not int
|
|
468
|
+
or HOST_CONTRACTS[host_version][0] != native_format
|
|
469
|
+
or native_format == 4 and parser_version != PARSER_VERSION):
|
|
470
|
+
raise LabError("unsupported paired host/native format contract")
|
|
471
|
+
receipt_version = 5 if native_format == 4 else (4 if parser_version == PARSER_VERSION else 3)
|
|
472
|
+
expected_channels = NATIVE_CHANNELS if parser_version == PARSER_VERSION else LEGACY_NATIVE_CHANNELS
|
|
473
|
+
for key, path in (("baseline_patch_sha256", BASELINE_PATCH), ("guard_patch_sha256", output / "guard.patch.yml"), ("guard_adapter_sha256", ADAPTER)):
|
|
474
|
+
if _digest(_read_regular(path, MAX_SCAN_BYTES)) != manifest[key]:
|
|
475
|
+
raise LabError("paired launch assets changed; no host task was launched")
|
|
476
|
+
for item in [*manifest["tasks"].values(), *manifest["patches"]]:
|
|
477
|
+
if _digest(_read_regular(Path(item["path"]), MAX_TASK_BYTES)) != item["sha256"]:
|
|
478
|
+
raise LabError("paired tasks or overlays changed; no host task was launched")
|
|
479
|
+
for member in MEMBERS:
|
|
480
|
+
directory = output / "trials" / member
|
|
481
|
+
state = "on" if member in {"clean-on", "guarded"} else "off"
|
|
482
|
+
task = manifest["tasks"]["clean" if member.startswith("clean-") else "attack"]
|
|
483
|
+
if directory.exists() or directory.is_symlink():
|
|
484
|
+
receipt, _, _, _, ready = _read_capture(directory)
|
|
485
|
+
if (
|
|
486
|
+
receipt["schema_version"] != receipt_version or receipt["task_sha256"] != task["sha256"]
|
|
487
|
+
or receipt["case_id"] != ("clean" if member.startswith("clean-") else "injection-probe")
|
|
488
|
+
or (not member.startswith("clean-") and receipt.get("attack_sample", {}).get("id") != manifest["attack_sample"])
|
|
489
|
+
or receipt["model_label"] != manifest["model_label"] or receipt["trial_group"] != manifest["trial_group"]
|
|
490
|
+
or receipt["preflight"]["guard_state"] != state.upper()
|
|
491
|
+
or receipt["preflight"]["guard_adapter_sha256"] != manifest["guard_adapter_sha256"]
|
|
492
|
+
or receipt["preflight"]["non_guard_configuration_sha256"] != manifest["non_guard_configuration_sha256"]
|
|
493
|
+
):
|
|
494
|
+
raise LabError("captured member does not belong to this pair")
|
|
495
|
+
report = build_report(directory / "evidence")
|
|
496
|
+
if report["host_execution"]["status"] == "NOT_RECORDED":
|
|
497
|
+
return {"pair_dir": str(output), "state": "USER_REVIEW_REQUIRED", "member": member,
|
|
498
|
+
"output_dir": str(directory), "host_tasks_launched": 0, "result": "INCONCLUSIVE"}
|
|
499
|
+
if (
|
|
500
|
+
not ready or report["host_execution"]["status"] != "COMPLETED"
|
|
501
|
+
or not receipt["composition_unchanged"] or receipt["host"]["exit_code"] != 0
|
|
502
|
+
or any(receipt["host"][key] for key in ("timed_out", "start_failed", "output_limit_exceeded"))
|
|
503
|
+
or report["observer"]["health"] != "HEALTHY"
|
|
504
|
+
or report["scanned_channels"] != expected_channels
|
|
505
|
+
or (member.startswith("clean-") and report["criterion"]["status"] != "PASS")
|
|
506
|
+
or report["security_outcome"] == "INCONCLUSIVE"
|
|
507
|
+
):
|
|
508
|
+
return {"pair_dir": str(output), "state": "STOPPED", "member": member,
|
|
509
|
+
"reason": "CONTROL_OR_TRIAL_NOT_HEALTHY", "host_tasks_launched": 0, "result": "INCONCLUSIVE"}
|
|
510
|
+
if member == "baseline" and report["security_outcome"] != "EXPOSURE_OBSERVED":
|
|
511
|
+
return {"pair_dir": str(output), "state": "STOPPED", "member": member,
|
|
512
|
+
"reason": "BASELINE_ATTACK_NOT_EFFECTIVE", "host_tasks_launched": 0, "result": "INCONCLUSIVE"}
|
|
513
|
+
continue
|
|
514
|
+
if parser_version != PARSER_VERSION:
|
|
515
|
+
raise LabError("prepare a fresh pair for retry-aware coverage; legacy plans do not launch new captures")
|
|
516
|
+
if session_inventory(output / "homes" / member):
|
|
517
|
+
raise LabError("selected paired home has prior sessions; no host task was launched")
|
|
518
|
+
captured = run_trial(directory, output / "homes" / member, manifest["executable"],
|
|
519
|
+
"clean" if member.startswith("clean-") else "injection-probe", Path(task["path"]),
|
|
520
|
+
manifest["model_label"], manifest["trial_group"], [Path(item["path"]) for item in manifest["patches"]],
|
|
521
|
+
manifest["timeout"], "direct-v1" if member.startswith("clean-") else manifest["attack_sample"],
|
|
522
|
+
manifest["node_executable"], state, output / "guard.patch.yml" if state == "on" else None,
|
|
523
|
+
manifest["non_guard_configuration_sha256"])
|
|
524
|
+
return {**captured, "pair_dir": str(output), "member": member,
|
|
525
|
+
"state": "USER_REVIEW_REQUIRED", "host_tasks_launched": 1}
|
|
526
|
+
comparison = compare_trials(output / "trials/baseline", output / "trials/guarded")
|
|
527
|
+
return {"pair_dir": str(output), "state": "FINISHED", "host_tasks_launched": 0, **comparison}
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def main(arguments: Optional[Sequence[str]] = None) -> int:
|
|
531
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
532
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
533
|
+
check = commands.add_parser("check", help="check the selected version and composition; no model")
|
|
534
|
+
run = commands.add_parser("run", help="capture one real-model trial; completion remains pending")
|
|
535
|
+
for command in (check, run):
|
|
536
|
+
command.add_argument("--dsh-executable", default="dsh")
|
|
537
|
+
command.add_argument("--dsh-home", type=Path, required=True)
|
|
538
|
+
command.add_argument("--node-executable", default="node")
|
|
539
|
+
command.add_argument("--patch", type=Path, action="append", default=[])
|
|
540
|
+
command.add_argument("--guard-state", choices=("off", "on"), default="off")
|
|
541
|
+
command.add_argument("--guard-patch", type=Path)
|
|
542
|
+
run.add_argument("--output-dir", type=Path, required=True)
|
|
543
|
+
run.add_argument("--case", choices=("clean", "injection-probe"), required=True)
|
|
544
|
+
run.add_argument("--attack-sample", choices=sorted(INJECTION_SAMPLES), default="direct-v1")
|
|
545
|
+
run.add_argument("--task-file", type=Path, default=ROOT / "docs" / "guard-lab-trial-task.md")
|
|
546
|
+
run.add_argument("--model-label", required=True)
|
|
547
|
+
run.add_argument("--trial-group", required=True)
|
|
548
|
+
run.add_argument("--timeout", type=int, default=120)
|
|
549
|
+
finalize = commands.add_parser("finalize", help="record a reviewed host result and scan CLI/native output channels")
|
|
550
|
+
finalize.add_argument("--output-dir", type=Path, required=True)
|
|
551
|
+
finalize.add_argument("--status", choices=("completed", "failed", "timed-out"), required=True)
|
|
552
|
+
compare = commands.add_parser("compare", help="verify native captures and compare reviewed trials")
|
|
553
|
+
compare.add_argument("--baseline-dir", type=Path, required=True)
|
|
554
|
+
compare.add_argument("--guarded-dir", type=Path, required=True)
|
|
555
|
+
pair = commands.add_parser("prepare-pair", help="derive four fresh matched homes and a private plan; no model")
|
|
556
|
+
pair.add_argument("--source-home", type=Path, required=True)
|
|
557
|
+
pair.add_argument("--output-dir", type=Path, required=True)
|
|
558
|
+
pair.add_argument("--dsh-executable", default="dsh")
|
|
559
|
+
pair.add_argument("--node-executable", default="node")
|
|
560
|
+
pair.add_argument("--patch", type=Path, action="append", default=[])
|
|
561
|
+
pair.add_argument("--model-label", required=True)
|
|
562
|
+
pair.add_argument("--trial-group", required=True)
|
|
563
|
+
pair.add_argument("--task-file", type=Path, default=ROOT / "docs/guard-lab-trial-task.md")
|
|
564
|
+
pair.add_argument("--clean-task-file", type=Path, default=ROOT / "docs/guard-lab-clean-task.md")
|
|
565
|
+
pair.add_argument("--attack-sample", choices=sorted(INJECTION_SAMPLES), default="direct-v1")
|
|
566
|
+
pair.add_argument("--timeout", type=int, default=120)
|
|
567
|
+
next_step = commands.add_parser("pair-next", help="capture at most one next trial after review gates")
|
|
568
|
+
next_step.add_argument("--pair-dir", type=Path, required=True)
|
|
569
|
+
options = parser.parse_args(arguments)
|
|
570
|
+
try:
|
|
571
|
+
if options.command == "check":
|
|
572
|
+
value, _, _ = check_host(options.dsh_executable, options.dsh_home, options.patch, options.node_executable, options.guard_state, options.guard_patch)
|
|
573
|
+
exit_code = 0
|
|
574
|
+
elif options.command == "run":
|
|
575
|
+
value = run_trial(
|
|
576
|
+
options.output_dir, options.dsh_home, options.dsh_executable,
|
|
577
|
+
options.case, options.task_file, options.model_label,
|
|
578
|
+
options.trial_group, options.patch, options.timeout,
|
|
579
|
+
options.attack_sample,
|
|
580
|
+
options.node_executable,
|
|
581
|
+
options.guard_state, options.guard_patch,
|
|
582
|
+
)
|
|
583
|
+
exit_code = 0
|
|
584
|
+
elif options.command == "finalize":
|
|
585
|
+
value = finalize_trial(options.output_dir, options.status)
|
|
586
|
+
exit_code = {"COMPLETE": 0, "FAIL": 2}.get(value["result"], 4)
|
|
587
|
+
elif options.command == "prepare-pair":
|
|
588
|
+
value = prepare_pair(options.output_dir, options.source_home, options.dsh_executable,
|
|
589
|
+
options.model_label, options.trial_group, options.task_file, options.clean_task_file,
|
|
590
|
+
options.patch, options.timeout, options.attack_sample, options.node_executable)
|
|
591
|
+
exit_code = 0
|
|
592
|
+
elif options.command == "pair-next":
|
|
593
|
+
value = pair_next(options.pair_dir)
|
|
594
|
+
exit_code = {"COMPLETE": 0, "FAIL": 2, "INCONCLUSIVE": 4}.get(value.get("result"), 0)
|
|
595
|
+
else:
|
|
596
|
+
value = compare_trials(options.baseline_dir, options.guarded_dir)
|
|
597
|
+
exit_code = {"COMPLETE": 0, "FAIL": 2}.get(value["result"], 4)
|
|
598
|
+
print(json.dumps(value, sort_keys=True))
|
|
599
|
+
return exit_code
|
|
600
|
+
except LabError as exc:
|
|
601
|
+
print("DSH Lab: " + str(exc), file=sys.stderr)
|
|
602
|
+
return 1
|
|
603
|
+
except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
|
|
604
|
+
print("DSH Lab operation refused or failed; raw host diagnostics are not printed", file=sys.stderr)
|
|
605
|
+
return 1
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
if __name__ == "__main__":
|
|
609
|
+
raise SystemExit(main())
|