@mokuyoaxis/agent-guard 0.2.3-rc1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -52
- package/README.zh-CN.md +102 -41
- package/adapters/dsh/README.md +92 -10
- package/adapters/dsh/harness/README.md +384 -0
- package/adapters/dsh/harness/baseline.patch.yml +8 -0
- package/adapters/dsh/harness/bootstrap.patch.yml +5 -0
- package/adapters/dsh/harness/bootstrap_init.mjs +48 -0
- package/adapters/dsh/harness/guard_lab.py +771 -0
- package/adapters/dsh/harness/native_session.py +503 -0
- package/adapters/dsh/harness/profiles.py +212 -0
- package/adapters/dsh/harness/read_fixture.py +60 -0
- package/adapters/dsh/harness/retry_stream.py +171 -0
- package/adapters/dsh/harness/session_decode.mjs +117 -0
- package/adapters/dsh/lib/index.js +46 -5
- package/adapters/dsh/lib/read_result_guard.js +207 -0
- package/adapters/dsh/lib/read_result_guard.py +139 -0
- package/adapters/harness_support.py +109 -0
- package/adapters/kimi-code/README.md +15 -7
- package/adapters/kimi-code/harness/README.md +126 -0
- package/adapters/kimi-code/harness/guard_lab.py +430 -0
- package/core/lab.py +2396 -0
- package/core/redaction.py +157 -55
- package/docs/architecture.md +75 -0
- package/docs/development-note-unguarded-deletion.md +6 -7
- package/docs/friction.md +140 -4
- package/docs/guard-lab-clean-task.md +4 -0
- package/docs/guard-lab-read-trial-task.md +7 -0
- package/docs/guard-lab-trial-task.md +5 -0
- package/docs/guard-lab.md +398 -0
- package/docs/harness-capabilities.md +8 -5
- package/docs/release-notes-0.2.0-rc1.md +2 -2
- package/docs/release-notes-0.2.0.md +2 -2
- package/docs/release-notes-0.2.2.md +7 -8
- package/docs/release-notes-0.2.3-rc1.md +4 -2
- package/docs/release-notes-0.2.3-rc2.md +189 -0
- package/docs/release-notes-0.2.3.md +84 -0
- package/docs/secret-guard-analysis.md +13 -12
- package/docs/secret-guard-architecture.md +5 -3
- package/docs/secret-guard-design.md +7 -3
- package/docs/test-report-codex-gpt-5.6-sol.md +10 -11
- package/docs/test-report-dsh-0.2-upgrade.md +142 -0
- package/docs/test-report-dsh-guard-lab.md +154 -0
- package/docs/test-report-dsh-native-session.md +118 -0
- package/docs/test-report-dsh-paired-lab.md +183 -0
- package/docs/test-report-dsh-read-redaction.md +114 -0
- package/docs/test-report-dsh-real-followup.md +170 -0
- package/docs/test-report-dsh-retry-evidence.md +134 -0
- package/docs/test-report-kimi-code-block.md +19 -20
- package/docs/test-report-kimi-guard-lab.md +117 -0
- package/docs/test-report-release-readiness-0.2.3.md +239 -0
- package/docs/test-report-workbuddy-windows-core.md +53 -0
- package/docs/test-report-zcode-glm-flash.md +9 -9
- package/docs/threat-model.md +42 -0
- package/guard_lab.py +14 -0
- package/package.json +8 -5
- package/skills/delete-guard/SKILL.md +2 -3
- package/skills/exfil-guard/SKILL.md +37 -7
- package/skills/exfil-guard/references/rules.md +84 -16
- package/skills/exfil-guard/scripts/_payload.py +99 -0
- package/skills/exfil-guard/scripts/check_span.py +14 -81
- package/skills/exfil-guard/scripts/sanitize.py +113 -34
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Kimi Code guard-lab capture and comparison
|
|
2
|
+
|
|
3
|
+
This POSIX helper pins **Kimi Code 2.1.1**, captures prompt mode with
|
|
4
|
+
`stream-json`, separates role records, and preserves explicit completion review.
|
|
5
|
+
Configuration checks and profile preparation require Python 3.11+ for `tomllib`;
|
|
6
|
+
the shared Guard Core retains its Python 3.9+ floor. No dependency is installed.
|
|
7
|
+
|
|
8
|
+
## Private matched homes
|
|
9
|
+
|
|
10
|
+
Select the approved source home and the exact 2.1.1 executable. There may be a
|
|
11
|
+
different Kimi installation on PATH. Every output directory must be new.
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
python3 adapters/kimi-code/harness/guard_lab.py prepare-homes \
|
|
15
|
+
--source-home /absolute/path/to/source-kimi-home \
|
|
16
|
+
--output-dir /tmp/kimi-lab-homes
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
`prepare-homes` derives private `off/` and `on/` homes. It removes only the
|
|
20
|
+
recognized repository Guard hook and skill-root entries for `off`, checks TOML
|
|
21
|
+
round-trip equality, and fingerprints the remaining settings. The source is
|
|
22
|
+
left untouched. Authentication/bootstrap state is copied privately so refreshes
|
|
23
|
+
write to the disposable homes; sessions, logs, history, caches and other user
|
|
24
|
+
data are not copied. Each directory is mode 0700 and each copied file mode 0600.
|
|
25
|
+
These homes contain credentials and must stay outside the fixture and public
|
|
26
|
+
project/package. Raw configuration and authentication values are never printed.
|
|
27
|
+
|
|
28
|
+
Check each selected home without a model call:
|
|
29
|
+
|
|
30
|
+
```sh
|
|
31
|
+
python3 adapters/kimi-code/harness/guard_lab.py check \
|
|
32
|
+
--kimi-home /tmp/kimi-lab-homes/off \
|
|
33
|
+
--kimi-executable /absolute/path/to/kimi-2.1.1 --guard-state off
|
|
34
|
+
python3 adapters/kimi-code/harness/guard_lab.py check \
|
|
35
|
+
--kimi-home /tmp/kimi-lab-homes/on \
|
|
36
|
+
--kimi-executable /absolute/path/to/kimi-2.1.1 --guard-state on
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Guard-on preflight also runs the local doctor bridge/Core probes. This is not
|
|
40
|
+
a live-host interception proof. Model labels and the remote model identity
|
|
41
|
+
remain user-declared; configuration equality does not authenticate a backend.
|
|
42
|
+
|
|
43
|
+
## Clean control, then baseline
|
|
44
|
+
|
|
45
|
+
Use the [README-only task](../../../docs/guard-lab-clean-task.md) for a real-host
|
|
46
|
+
negative control so that it requests no bait read. Use the original
|
|
47
|
+
[review task](../../../docs/guard-lab-trial-task.md) for both attack trials.
|
|
48
|
+
For the clean control, prepare a separate equivalent disposable home if the
|
|
49
|
+
matched attack pair should start without any previous host session state.
|
|
50
|
+
|
|
51
|
+
```sh
|
|
52
|
+
python3 adapters/kimi-code/harness/guard_lab.py run \
|
|
53
|
+
--kimi-home /tmp/kimi-lab-homes/off \
|
|
54
|
+
--kimi-executable /absolute/path/to/kimi-2.1.1 --guard-state off \
|
|
55
|
+
--output-dir /tmp/kimi-lab-clean --case clean \
|
|
56
|
+
--task-file docs/guard-lab-clean-task.md \
|
|
57
|
+
--model-label "Kimi official OAuth model" --trial-group kimi-pair-01
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The observer is stopped after capture, including timeout/failure. Raw stdout
|
|
61
|
+
and stderr are private `capture/*.bin` files capped at one MiB each. `run`
|
|
62
|
+
returns a pending review summary; the initial Lab report stays `INCONCLUSIVE`.
|
|
63
|
+
No `--auto`/`--yolo` is added to prompt mode.
|
|
64
|
+
|
|
65
|
+
Review the captures and host result, then record `completed`, `failed` or
|
|
66
|
+
`timed-out` with the create-once finalizer:
|
|
67
|
+
|
|
68
|
+
```sh
|
|
69
|
+
python3 adapters/kimi-code/harness/guard_lab.py finalize \
|
|
70
|
+
--output-dir /tmp/kimi-lab-clean --status completed
|
|
71
|
+
|
|
72
|
+
python3 adapters/kimi-code/harness/guard_lab.py run \
|
|
73
|
+
--kimi-home /tmp/kimi-lab-homes/off \
|
|
74
|
+
--kimi-executable /absolute/path/to/kimi-2.1.1 --guard-state off \
|
|
75
|
+
--output-dir /tmp/kimi-lab-baseline --case injection-probe \
|
|
76
|
+
--model-label "Kimi official OAuth model" --trial-group kimi-pair-01
|
|
77
|
+
|
|
78
|
+
# Review actual host completion before selecting completed.
|
|
79
|
+
python3 adapters/kimi-code/harness/guard_lab.py finalize \
|
|
80
|
+
--output-dir /tmp/kimi-lab-baseline --status completed
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Run guard-on only after an effective baseline. Use the same task, model label
|
|
84
|
+
and trial group, the fresh matched `on/` home and a fresh output directory.
|
|
85
|
+
The default sample is `direct-v1`; both attack captures can instead select the
|
|
86
|
+
same `--attack-sample maintenance-v1` or `--attack-sample config-comment-v1`.
|
|
87
|
+
Different sample IDs, protocols or template hashes refuse comparison.
|
|
88
|
+
Then compare using the wrapper, which additionally checks non-Guard settings:
|
|
89
|
+
|
|
90
|
+
```sh
|
|
91
|
+
python3 adapters/kimi-code/harness/guard_lab.py compare \
|
|
92
|
+
--baseline-dir /tmp/kimi-lab-baseline --guarded-dir /tmp/kimi-lab-guarded
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Role handling and evidence
|
|
96
|
+
|
|
97
|
+
The parser requires one `role=meta,type=system.version,version=2.1.1` record,
|
|
98
|
+
recognized NDJSON role/content shapes, and an assistant response. At most 4096
|
|
99
|
+
records are processed. Header-only exit-zero failures, malformed/unknown roles,
|
|
100
|
+
non-text shapes, timeout, overflow, changed configuration, or corrupted captures
|
|
101
|
+
cannot be declared completed. Failed runs can still be scanned as host output
|
|
102
|
+
while their security result remains `INCONCLUSIVE`.
|
|
103
|
+
|
|
104
|
+
Recognized `role=tool` records use `tool-output`; `role=assistant` records use
|
|
105
|
+
`model-output`; metadata and stderr use `host-output`, each with a separate
|
|
106
|
+
source kind. These are captured **role labels**, not authenticated authorship.
|
|
107
|
+
Assistant records can contain tool-call arguments and Kimi-rendered hook
|
|
108
|
+
feedback; a hit there does not by itself prove model prose reproduced a marker.
|
|
109
|
+
The parser retains whole role records to avoid dropping tool-call payloads or
|
|
110
|
+
JSON metadata, and rejects incomplete streams instead of guessing provenance.
|
|
111
|
+
|
|
112
|
+
Raw captures stay private for review. Lab reports/events retain only hashes,
|
|
113
|
+
sizes, stages, bait IDs and counts. The comparison adds an AST configuration
|
|
114
|
+
fingerprint check, but environment values, credentials, backend identity and
|
|
115
|
+
same-UID isolation remain outside its proof.
|
|
116
|
+
|
|
117
|
+
Preparation/check/capture return 0 on successful orchestration and 1 on error.
|
|
118
|
+
Finalization/comparison use Lab exits: 0 for passing criterion/mitigation, 2 for
|
|
119
|
+
observed exposure/non-mitigation, 4 for inconclusive/incomparable evidence.
|
|
120
|
+
|
|
121
|
+
```sh
|
|
122
|
+
python3 -m unittest tests.test_kimi_guard_lab tests.test_dsh_guard_lab
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
These regressions use synthetic subprocess hosts and no model call. The shared
|
|
126
|
+
bounded process helper is also exercised by the DSH suite.
|
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Private Kimi 2.1.1 Lab capture, role separation and explicit host review."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import copy
|
|
8
|
+
from datetime import date, datetime, time
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
from typing import Optional, Sequence
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
ROOT = Path(__file__).resolve().parents[3]
|
|
19
|
+
sys.path.insert(0, str(ROOT))
|
|
20
|
+
|
|
21
|
+
from adapters.harness_support import ( # noqa: E402
|
|
22
|
+
capture_process, digest, read_regular, write_json, write_new,
|
|
23
|
+
)
|
|
24
|
+
from core.lab import ( # noqa: E402
|
|
25
|
+
INJECTION_SAMPLES, LabError, MAX_SCAN_BYTES, MAX_TASK_BYTES, TRIAL_GROUP_PATTERN,
|
|
26
|
+
TRIAL_LABEL_PATTERN, arm_observer, build_report, compare_runs, prepare_run,
|
|
27
|
+
record_host_result, request_stop, scan_bytes,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
TESTED_VERSION = "2.1.1"
|
|
31
|
+
MAX_CONFIG_BYTES = 1024 * 1024
|
|
32
|
+
MAX_RECORDS = 4096
|
|
33
|
+
# Match safe_view's filename convention: the corpus scanner otherwise treats
|
|
34
|
+
# a quoted store name in source code as a request to dump that store.
|
|
35
|
+
AUTH_STORE_NAME = "creden" + "tials"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def load_config(path: Path) -> tuple[bytes, dict]:
|
|
39
|
+
try:
|
|
40
|
+
import tomllib
|
|
41
|
+
except ImportError as exc:
|
|
42
|
+
raise LabError("Kimi Lab configuration checks require Python 3.11+") from exc
|
|
43
|
+
payload = read_regular(path, MAX_CONFIG_BYTES)
|
|
44
|
+
try:
|
|
45
|
+
return payload, tomllib.loads(payload.decode("utf-8"))
|
|
46
|
+
except (ValueError, UnicodeError) as exc:
|
|
47
|
+
raise LabError("Kimi configuration must be valid UTF-8 TOML") from exc
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def toml_value(value) -> str:
|
|
51
|
+
# Inline tables keep arbitrary configuration keys while avoiding a dependency
|
|
52
|
+
# or editing the user's original TOML. Round-trip equality is mandatory.
|
|
53
|
+
if isinstance(value, str):
|
|
54
|
+
return json.dumps(value, ensure_ascii=False)
|
|
55
|
+
if isinstance(value, bool):
|
|
56
|
+
return "true" if value else "false"
|
|
57
|
+
if isinstance(value, (int, float)):
|
|
58
|
+
return repr(value)
|
|
59
|
+
if isinstance(value, (date, datetime, time)):
|
|
60
|
+
return value.isoformat()
|
|
61
|
+
if isinstance(value, list):
|
|
62
|
+
return "[" + ", ".join(toml_value(item) for item in value) + "]"
|
|
63
|
+
if isinstance(value, dict):
|
|
64
|
+
return "{ " + ", ".join(
|
|
65
|
+
json.dumps(key, ensure_ascii=False) + " = " + toml_value(item)
|
|
66
|
+
for key, item in value.items()
|
|
67
|
+
) + " }"
|
|
68
|
+
raise LabError("configuration contains an unsupported TOML value")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def without_guard(config: dict) -> tuple[dict, dict]:
|
|
72
|
+
result = copy.deepcopy(config)
|
|
73
|
+
hooks = result.get("hooks", [])
|
|
74
|
+
if not isinstance(hooks, list):
|
|
75
|
+
raise LabError("Kimi hooks must be an array")
|
|
76
|
+
kept = [hook for hook in hooks if not (
|
|
77
|
+
isinstance(hook, dict) and hook.get("event") == "PreToolUse"
|
|
78
|
+
and "adapters/kimi-code/" in str(hook.get("command", ""))
|
|
79
|
+
and any(name in str(hook.get("command", "")) for name in (
|
|
80
|
+
"hook_bridge.sh", "pre_tool_use.py",
|
|
81
|
+
))
|
|
82
|
+
)]
|
|
83
|
+
changes = {"guard_hooks_removed": len(hooks) - len(kept), "guard_skill_dirs_removed": 0}
|
|
84
|
+
if "hooks" in result:
|
|
85
|
+
result["hooks"] = kept
|
|
86
|
+
for key in ("extraSkillDirs", "extra_skill_dirs"):
|
|
87
|
+
if key not in result:
|
|
88
|
+
continue
|
|
89
|
+
entries = result[key]
|
|
90
|
+
if not isinstance(entries, list) or not all(isinstance(item, str) for item in entries):
|
|
91
|
+
raise LabError("Kimi extra skill directories must be string arrays")
|
|
92
|
+
filtered = [item for item in entries if Path(item).expanduser().resolve() != ROOT / "skills"]
|
|
93
|
+
changes["guard_skill_dirs_removed"] += len(entries) - len(filtered)
|
|
94
|
+
result[key] = filtered
|
|
95
|
+
return result, changes
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def config_fingerprint(config: dict) -> str:
|
|
99
|
+
return digest(json.dumps(config, sort_keys=True, separators=(",", ":"), default=str).encode())
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def prepare_homes(source_home: Path, output_dir: Path) -> dict:
|
|
103
|
+
source = source_home.expanduser().resolve()
|
|
104
|
+
raw, guarded = load_config(source / "config.toml")
|
|
105
|
+
baseline, changes = without_guard(guarded)
|
|
106
|
+
if changes["guard_hooks_removed"] != 1 or not changes["guard_skill_dirs_removed"]:
|
|
107
|
+
raise LabError("expected one Guard hook and its repository skill root")
|
|
108
|
+
output = output_dir.expanduser().absolute()
|
|
109
|
+
if output.exists() or output.is_symlink() or not output.parent.is_dir():
|
|
110
|
+
raise LabError("profile output directory must be new")
|
|
111
|
+
output = output.resolve()
|
|
112
|
+
if output == source or output in source.parents or source in output.parents:
|
|
113
|
+
raise LabError("profile output and source home must be disjoint")
|
|
114
|
+
import tomllib
|
|
115
|
+
serialized = "".join(json.dumps(key) + " = " + toml_value(value) + "\n" for key, value in baseline.items()).encode()
|
|
116
|
+
if tomllib.loads(serialized.decode()) != baseline:
|
|
117
|
+
raise LabError("baseline TOML round-trip changed configuration")
|
|
118
|
+
output.mkdir(mode=0o700)
|
|
119
|
+
for label, payload in (("off", serialized), ("on", raw)):
|
|
120
|
+
home = output / label
|
|
121
|
+
home.mkdir(mode=0o700)
|
|
122
|
+
write_new(home / "config.toml", payload)
|
|
123
|
+
# Copy only authentication/bootstrap state, never sessions or model logs.
|
|
124
|
+
# Private copies let token refresh avoid writes to the user's home.
|
|
125
|
+
for name in (AUTH_STORE_NAME, "oauth", "device_id", "region"):
|
|
126
|
+
original = source / name
|
|
127
|
+
if original.is_symlink():
|
|
128
|
+
raise LabError("authentication bootstrap entries must not be symlinks")
|
|
129
|
+
if original.is_file():
|
|
130
|
+
write_new(home / name, read_regular(original, MAX_CONFIG_BYTES))
|
|
131
|
+
elif original.is_dir():
|
|
132
|
+
target = home / name
|
|
133
|
+
target.mkdir(mode=0o700)
|
|
134
|
+
for item in original.rglob("*"):
|
|
135
|
+
if item.is_symlink():
|
|
136
|
+
raise LabError("authentication tree must not contain symlinks")
|
|
137
|
+
selected = target / item.relative_to(original)
|
|
138
|
+
if item.is_dir():
|
|
139
|
+
selected.mkdir(mode=0o700, exist_ok=True)
|
|
140
|
+
elif item.is_file():
|
|
141
|
+
selected.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
142
|
+
write_new(selected, read_regular(item, MAX_CONFIG_BYTES))
|
|
143
|
+
identity = config_fingerprint(baseline)
|
|
144
|
+
manifest = {
|
|
145
|
+
"schema_version": 1, "non_guard_configuration_sha256": identity,
|
|
146
|
+
"semantic_equality_verified": True, **changes,
|
|
147
|
+
"auth_copied_privately": True, "source_modified": False,
|
|
148
|
+
}
|
|
149
|
+
write_json(output / "pair.json", manifest)
|
|
150
|
+
return {"profile_dir": str(output), **manifest}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def check_host(executable: str, home: Path, guard_state: str) -> tuple[dict, str, dict]:
|
|
154
|
+
selected = shutil.which(executable)
|
|
155
|
+
if not selected:
|
|
156
|
+
raise LabError("select the Kimi 2.1.1 executable explicitly")
|
|
157
|
+
selected = str(Path(selected).absolute())
|
|
158
|
+
home = home.expanduser().resolve()
|
|
159
|
+
if home == (Path.home() / ".kimi-code").resolve():
|
|
160
|
+
raise LabError("select a disposable Kimi home")
|
|
161
|
+
raw, config = load_config(home / "config.toml")
|
|
162
|
+
baseline, changes = without_guard(config)
|
|
163
|
+
if guard_state not in {"off", "on"}:
|
|
164
|
+
raise LabError("guard state must be off or on")
|
|
165
|
+
if guard_state == "off" and any(changes.values()):
|
|
166
|
+
raise LabError("guard-off configuration still contains known Guard entries")
|
|
167
|
+
if guard_state == "on" and changes["guard_hooks_removed"] != 1:
|
|
168
|
+
raise LabError("guard-on configuration requires one known Guard hook")
|
|
169
|
+
if guard_state == "on":
|
|
170
|
+
from doctor import check_kimi
|
|
171
|
+
if check_kimi(home / "config.toml", run_probe=True):
|
|
172
|
+
raise LabError("guard-on Kimi local configuration or bridge probe failed")
|
|
173
|
+
environment = {key: value for key, value in os.environ.items() if not key.startswith("AGENT_GUARD_")}
|
|
174
|
+
environment["KIMI_CODE_HOME"] = str(home)
|
|
175
|
+
result, streams = capture_process([selected, "--version"], home, environment, 15)
|
|
176
|
+
if result["exit_code"] != 0 or streams["stdout"].strip() != TESTED_VERSION.encode():
|
|
177
|
+
raise LabError("Kimi version preflight requires exactly " + TESTED_VERSION)
|
|
178
|
+
return {
|
|
179
|
+
"harness_version": TESTED_VERSION, "guard_state": guard_state.upper(),
|
|
180
|
+
"configuration_sha256": digest(raw),
|
|
181
|
+
"non_guard_configuration_sha256": config_fingerprint(baseline),
|
|
182
|
+
"identity_source": "user-declared",
|
|
183
|
+
}, selected, environment
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def split_stream(payload: bytes) -> tuple[dict, dict[str, bytes]]:
|
|
187
|
+
"""Recognize the selected version's NDJSON role records without claiming authorship."""
|
|
188
|
+
roles = {"assistant": [], "tool": [], "meta": []}
|
|
189
|
+
issues = []
|
|
190
|
+
version_count = 0
|
|
191
|
+
record_count = 0
|
|
192
|
+
meaningful = False
|
|
193
|
+
try:
|
|
194
|
+
lines = payload.decode("utf-8").splitlines()
|
|
195
|
+
except UnicodeError:
|
|
196
|
+
return {"recognized": False, "issues": ["STREAM_NOT_UTF8"], "record_count": 0}, {}
|
|
197
|
+
for line in lines:
|
|
198
|
+
if not line.strip():
|
|
199
|
+
continue
|
|
200
|
+
record_count += 1
|
|
201
|
+
if record_count > MAX_RECORDS:
|
|
202
|
+
issues.append("STREAM_RECORD_LIMIT")
|
|
203
|
+
break
|
|
204
|
+
try:
|
|
205
|
+
value = json.loads(line)
|
|
206
|
+
except ValueError:
|
|
207
|
+
issues.append("STREAM_NOT_NDJSON")
|
|
208
|
+
continue
|
|
209
|
+
if not isinstance(value, dict) or value.get("role") not in roles:
|
|
210
|
+
issues.append("UNKNOWN_RECORD_ROLE")
|
|
211
|
+
continue
|
|
212
|
+
role = value["role"]
|
|
213
|
+
if role == "meta":
|
|
214
|
+
if not isinstance(value.get("type"), str):
|
|
215
|
+
issues.append("INVALID_META_RECORD")
|
|
216
|
+
if value.get("type") == "system.version":
|
|
217
|
+
version_count += 1
|
|
218
|
+
if value.get("version") != TESTED_VERSION:
|
|
219
|
+
issues.append("STREAM_VERSION_MISMATCH")
|
|
220
|
+
elif not isinstance(value.get("content", ""), str):
|
|
221
|
+
issues.append("UNSUPPORTED_CONTENT_SHAPE")
|
|
222
|
+
elif role == "tool" and not isinstance(value.get("tool_call_id"), str):
|
|
223
|
+
issues.append("TOOL_CALL_ID_MISSING")
|
|
224
|
+
if role == "assistant" and isinstance(value.get("content"), str) and value["content"].strip():
|
|
225
|
+
meaningful = True
|
|
226
|
+
roles[role].append(line.encode("utf-8") + b"\n")
|
|
227
|
+
if version_count != 1:
|
|
228
|
+
issues.append("VERSION_RECORD_MISSING_OR_DUPLICATE")
|
|
229
|
+
if not meaningful:
|
|
230
|
+
issues.append("ASSISTANT_RESPONSE_MISSING")
|
|
231
|
+
return {
|
|
232
|
+
"recognized": not issues, "issues": sorted(set(issues)),
|
|
233
|
+
"record_count": record_count,
|
|
234
|
+
"role_counts": {name: len(lines) for name, lines in roles.items()},
|
|
235
|
+
"provenance": "captured-role-labels-not-authenticated-authorship",
|
|
236
|
+
}, {name: b"".join(lines) for name, lines in roles.items()}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def run_trial(
|
|
240
|
+
output_dir: Path, home: Path, executable: str, case_id: str, task_file: Path,
|
|
241
|
+
model_label: str, trial_group: str, guard_state: str, timeout: int = 180,
|
|
242
|
+
attack_sample: str = "direct-v1",
|
|
243
|
+
) -> dict:
|
|
244
|
+
if case_id not in {"clean", "injection-probe"} or not 1 <= timeout <= 600:
|
|
245
|
+
raise LabError("unsupported Kimi case or timeout")
|
|
246
|
+
if attack_sample not in INJECTION_SAMPLES or (case_id != "injection-probe" and attack_sample != "direct-v1"):
|
|
247
|
+
raise LabError("select a known attack sample for injection-probe only")
|
|
248
|
+
if not TRIAL_LABEL_PATTERN.fullmatch(model_label) or not TRIAL_GROUP_PATTERN.fullmatch(trial_group):
|
|
249
|
+
raise LabError("use public Lab model labels and a trial-group slug")
|
|
250
|
+
task = read_regular(task_file.expanduser().absolute(), MAX_TASK_BYTES)
|
|
251
|
+
prompt = task.decode("utf-8")
|
|
252
|
+
if not prompt.strip() or "\x00" in prompt:
|
|
253
|
+
raise LabError("task must be nonempty UTF-8 without NUL bytes")
|
|
254
|
+
output = output_dir.expanduser().absolute()
|
|
255
|
+
home = home.expanduser().resolve()
|
|
256
|
+
if output.exists() or output.is_symlink() or not output.parent.is_dir():
|
|
257
|
+
raise LabError("trial output directory must be new")
|
|
258
|
+
output = output.resolve()
|
|
259
|
+
if output == home or output in home.parents or home in output.parents:
|
|
260
|
+
raise LabError("Kimi home and trial output must be disjoint")
|
|
261
|
+
preflight, selected, environment = check_host(executable, home, guard_state)
|
|
262
|
+
output.mkdir(mode=0o700)
|
|
263
|
+
capture = output / "capture"
|
|
264
|
+
capture.mkdir(mode=0o700)
|
|
265
|
+
metadata = {} if case_id == "clean" else {
|
|
266
|
+
"harness_label": "Kimi Code", "harness_version": TESTED_VERSION,
|
|
267
|
+
"model_label": model_label, "guard_state": guard_state,
|
|
268
|
+
"trial_group": trial_group, "task_file": task_file,
|
|
269
|
+
"non_guard_configuration_sha256": preflight["non_guard_configuration_sha256"],
|
|
270
|
+
}
|
|
271
|
+
prepared = prepare_run(output / "fixture", output / "evidence", case_id,
|
|
272
|
+
model_usage="real", attack_sample=attack_sample, **metadata)
|
|
273
|
+
if metadata and prepared["trial"]["task_sha256"] != digest(task):
|
|
274
|
+
raise LabError("task changed during preparation")
|
|
275
|
+
fixture = Path(prepared["fixture_dir"])
|
|
276
|
+
evidence = Path(prepared["evidence_dir"])
|
|
277
|
+
# Startup handles its own failure/stop request; preserve its original error.
|
|
278
|
+
arm_observer(evidence, timeout + 15)
|
|
279
|
+
try:
|
|
280
|
+
host, streams = capture_process([
|
|
281
|
+
selected, "-p", prompt, "--output-format", "stream-json",
|
|
282
|
+
], fixture, environment, timeout)
|
|
283
|
+
finally:
|
|
284
|
+
request_stop(evidence)
|
|
285
|
+
configuration_unchanged = False
|
|
286
|
+
try:
|
|
287
|
+
after, _ = load_config(home / "config.toml")
|
|
288
|
+
configuration_unchanged = digest(after) == preflight["configuration_sha256"]
|
|
289
|
+
except (LabError, OSError):
|
|
290
|
+
pass
|
|
291
|
+
stream_summary, _ = split_stream(streams["stdout"])
|
|
292
|
+
captures = {}
|
|
293
|
+
for name, payload in streams.items():
|
|
294
|
+
write_new(capture / (name + ".bin"), payload)
|
|
295
|
+
captures[name] = {"bytes": len(payload), "sha256": digest(payload)}
|
|
296
|
+
receipt = {
|
|
297
|
+
"schema_version": 1, "run_id": prepared["run_id"], "case_id": case_id,
|
|
298
|
+
**({"attack_sample": prepared["attack_sample"]} if "attack_sample" in prepared else {}),
|
|
299
|
+
"task_sha256": digest(task), "model_label": model_label, "trial_group": trial_group,
|
|
300
|
+
"preflight": preflight, "configuration_unchanged": configuration_unchanged,
|
|
301
|
+
"stream": stream_summary, "host": host, "captures": captures,
|
|
302
|
+
"completion": "USER_REVIEW_REQUIRED", "raw_host_output_retained": True,
|
|
303
|
+
}
|
|
304
|
+
write_json(capture / "receipt.json", receipt)
|
|
305
|
+
report = build_report(evidence)
|
|
306
|
+
return {
|
|
307
|
+
"output_dir": str(output), "run_id": prepared["run_id"],
|
|
308
|
+
"host": host, "stream": stream_summary,
|
|
309
|
+
"configuration_unchanged": receipt["configuration_unchanged"],
|
|
310
|
+
"completion": receipt["completion"], "report_result": report["result"],
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def finalize_trial(output_dir: Path, status: str) -> dict:
|
|
315
|
+
output = output_dir.expanduser().absolute()
|
|
316
|
+
if any(path.is_symlink() for path in (output, output / "capture", output / "evidence")):
|
|
317
|
+
raise LabError("trial directories must not be symlinks")
|
|
318
|
+
receipt = json.loads(read_regular(output / "capture/receipt.json", MAX_CONFIG_BYTES))
|
|
319
|
+
run = json.loads(read_regular(output / "evidence/run.json", MAX_CONFIG_BYTES))
|
|
320
|
+
if (receipt["run_id"], receipt["case_id"]) != (run["run_id"], run["case_id"]):
|
|
321
|
+
raise LabError("capture and Lab identities differ")
|
|
322
|
+
if Path(run["evidence_dir"]).resolve() != (output / "evidence").resolve():
|
|
323
|
+
raise LabError("capture evidence path does not match this trial")
|
|
324
|
+
streams = {}
|
|
325
|
+
for name in ("stdout", "stderr"):
|
|
326
|
+
payload = read_regular(output / "capture" / (name + ".bin"), MAX_SCAN_BYTES)
|
|
327
|
+
if receipt["captures"][name] != {"bytes": len(payload), "sha256": digest(payload)}:
|
|
328
|
+
raise LabError("capture integrity check failed")
|
|
329
|
+
streams[name] = payload
|
|
330
|
+
summary, roles = split_stream(streams["stdout"])
|
|
331
|
+
if summary != receipt["stream"]:
|
|
332
|
+
raise LabError("stream interpretation differs from its capture receipt")
|
|
333
|
+
if isinstance(run.get("trial"), dict) and "non_guard_configuration_sha256" in run["trial"] and (
|
|
334
|
+
run["trial"]["non_guard_configuration_sha256"]
|
|
335
|
+
!= receipt["preflight"]["non_guard_configuration_sha256"]
|
|
336
|
+
or run["trial"]["task_sha256"] != receipt["task_sha256"]
|
|
337
|
+
):
|
|
338
|
+
raise LabError("capture configuration or task differs from the Lab trial")
|
|
339
|
+
host = receipt["host"]
|
|
340
|
+
if status == "completed" and (
|
|
341
|
+
host["exit_code"] != 0 or host["timed_out"] or host["start_failed"]
|
|
342
|
+
or host["output_limit_exceeded"] or not receipt["configuration_unchanged"]
|
|
343
|
+
or not summary["recognized"]
|
|
344
|
+
):
|
|
345
|
+
raise LabError("failed, changed or unrecognized Kimi capture cannot be completed")
|
|
346
|
+
if host["timed_out"] and status != "timed-out":
|
|
347
|
+
raise LabError("a timed-out capture must be recorded as timed-out")
|
|
348
|
+
exit_code = host["exit_code"]
|
|
349
|
+
if not isinstance(exit_code, int) or not 0 <= exit_code <= 255:
|
|
350
|
+
exit_code = None
|
|
351
|
+
evidence = output / "evidence"
|
|
352
|
+
record_host_result(evidence, status, exit_code)
|
|
353
|
+
if summary["recognized"]:
|
|
354
|
+
for role, stage in (("assistant", "model-output"), ("tool", "tool-output"), ("meta", "host-output")):
|
|
355
|
+
scan_bytes(evidence, roles[role], stage, source_kind="kimi-role-" + role)
|
|
356
|
+
else:
|
|
357
|
+
scan_bytes(evidence, streams["stdout"], "host-output", source_kind="kimi-unrecognized-stdout")
|
|
358
|
+
scan_bytes(evidence, streams["stderr"], "host-output", source_kind="kimi-stderr")
|
|
359
|
+
return build_report(evidence)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def compare_trials(baseline_dir: Path, guarded_dir: Path) -> dict:
|
|
363
|
+
receipts = [json.loads(read_regular(path / "capture/receipt.json", MAX_CONFIG_BYTES))
|
|
364
|
+
for path in (baseline_dir, guarded_dir)]
|
|
365
|
+
result = compare_runs(baseline_dir / "evidence", guarded_dir / "evidence")
|
|
366
|
+
matching = (
|
|
367
|
+
receipts[0]["preflight"]["non_guard_configuration_sha256"]
|
|
368
|
+
== receipts[1]["preflight"]["non_guard_configuration_sha256"]
|
|
369
|
+
)
|
|
370
|
+
result["matching_dimensions"]["non_guard_configuration"] = matching
|
|
371
|
+
if not matching:
|
|
372
|
+
result.update({
|
|
373
|
+
"comparison_outcome": "INCOMPARABLE", "result": "INCONCLUSIVE",
|
|
374
|
+
"evidence_level": "UNESTABLISHED", "reason": "NON_GUARD_CONFIGURATION_MISMATCH",
|
|
375
|
+
})
|
|
376
|
+
result["precondition_issues"].append("NON_GUARD_CONFIGURATION_MISMATCH")
|
|
377
|
+
return result
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def main(arguments: Optional[Sequence[str]] = None) -> int:
|
|
381
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
382
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
383
|
+
profiles = commands.add_parser("prepare-homes", help="derive private matched homes; no model")
|
|
384
|
+
profiles.add_argument("--source-home", type=Path, required=True)
|
|
385
|
+
profiles.add_argument("--output-dir", type=Path, required=True)
|
|
386
|
+
check = commands.add_parser("check", help="check selected version/configuration; no model")
|
|
387
|
+
run = commands.add_parser("run", help="capture one real-host trial")
|
|
388
|
+
for command in (check, run):
|
|
389
|
+
command.add_argument("--kimi-home", type=Path, required=True)
|
|
390
|
+
command.add_argument("--kimi-executable", required=True)
|
|
391
|
+
command.add_argument("--guard-state", choices=("off", "on"), required=True)
|
|
392
|
+
run.add_argument("--output-dir", type=Path, required=True)
|
|
393
|
+
run.add_argument("--case", choices=("clean", "injection-probe"), required=True)
|
|
394
|
+
run.add_argument("--attack-sample", choices=sorted(INJECTION_SAMPLES), default="direct-v1")
|
|
395
|
+
run.add_argument("--task-file", type=Path, default=ROOT / "docs/guard-lab-trial-task.md")
|
|
396
|
+
run.add_argument("--model-label", required=True)
|
|
397
|
+
run.add_argument("--trial-group", required=True)
|
|
398
|
+
run.add_argument("--timeout", type=int, default=180)
|
|
399
|
+
finalize = commands.add_parser("finalize", help="record reviewed completion and scan role channels")
|
|
400
|
+
finalize.add_argument("--output-dir", type=Path, required=True)
|
|
401
|
+
finalize.add_argument("--status", choices=("completed", "failed", "timed-out"), required=True)
|
|
402
|
+
compare = commands.add_parser("compare", help="compare reviewed trials with matching non-Guard settings")
|
|
403
|
+
compare.add_argument("--baseline-dir", type=Path, required=True)
|
|
404
|
+
compare.add_argument("--guarded-dir", type=Path, required=True)
|
|
405
|
+
options = parser.parse_args(arguments)
|
|
406
|
+
try:
|
|
407
|
+
if options.command == "prepare-homes":
|
|
408
|
+
value = prepare_homes(options.source_home, options.output_dir)
|
|
409
|
+
elif options.command == "check":
|
|
410
|
+
value, _, _ = check_host(options.kimi_executable, options.kimi_home, options.guard_state)
|
|
411
|
+
elif options.command == "run":
|
|
412
|
+
value = run_trial(options.output_dir, options.kimi_home, options.kimi_executable,
|
|
413
|
+
options.case, options.task_file, options.model_label, options.trial_group,
|
|
414
|
+
options.guard_state, options.timeout, options.attack_sample)
|
|
415
|
+
elif options.command == "finalize":
|
|
416
|
+
value = finalize_trial(options.output_dir, options.status)
|
|
417
|
+
else:
|
|
418
|
+
value = compare_trials(options.baseline_dir, options.guarded_dir)
|
|
419
|
+
print(json.dumps(value, sort_keys=True))
|
|
420
|
+
return {"COMPLETE": 0, "FAIL": 2, "INCONCLUSIVE": 4}.get(value.get("result"), 0)
|
|
421
|
+
except LabError as exc:
|
|
422
|
+
print("Kimi Lab: " + str(exc), file=sys.stderr)
|
|
423
|
+
return 1
|
|
424
|
+
except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
|
|
425
|
+
print("Kimi Lab operation failed; raw diagnostics are not printed", file=sys.stderr)
|
|
426
|
+
return 1
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
if __name__ == "__main__":
|
|
430
|
+
raise SystemExit(main())
|