@cairnkeep/cli 2.21.1 → 2.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/docs/README.md +2 -0
- package/docs/agent-memory-protocol.md +146 -0
- package/docs/agents.md +7 -0
- package/docs/context-packs.md +10 -0
- package/docs/learning/CURRICULUM-MAP.md +3 -2
- package/docs/learning/FEATURE-GUIDE.md +7 -1
- package/docs/learning/lessons/L00-why-cairnkeep.md +1 -1
- package/docs/learning/lessons/L01-safe-trial.md +3 -3
- package/docs/learning/lessons/L02-installation.md +2 -2
- package/docs/learning/lessons/L03-first-project.md +1 -1
- package/docs/learning/lessons/L16-evaluation.md +23 -0
- package/docs/learning/lessons/L19-validated-skill-improvement.md +1 -1
- package/docs/learning/lessons/L20-mcp-tool-profiles.md +1 -1
- package/docs/learning/lessons/L21-context-packs.md +5 -1
- package/docs/learning/lessons/L22-native-windows.md +3 -2
- package/docs/learning/lessons/L23-guided-setup.md +1 -1
- package/docs/learning/lessons/L24-okf-exchange.md +1 -1
- package/docs/learning/lessons/L25-playbooks.md +1 -1
- package/docs/learning/lessons/L26-context-intelligence.md +1 -1
- package/docs/learning/lessons/L27-ecosystem-evidence.md +1 -1
- package/docs/learning/lessons/L28-security-assurance.md +14 -4
- package/docs/learning/tracks/evidence-and-evaluation.md +4 -0
- package/docs/learning/video-scripts/V16-memory-protocol-audit.md +32 -0
- package/docs/learning/video-scripts/V28-security-assurance.md +13 -0
- package/docs/operating.md +8 -0
- package/docs/privacy-and-data-flow.md +9 -0
- package/docs/releasing.md +39 -5
- package/docs/research/release-ledger-task.md +36 -0
- package/docs/security-assurance.md +11 -0
- package/examples/eval/README.md +13 -0
- package/examples/eval/bundled-fake.json +1 -1
- package/examples/eval/protocol-codex.jsonl +5 -0
- package/examples/eval/protocol-narration.json +12 -0
- package/examples/eval/protocol-project-search.json +13 -0
- package/mcp-memory-server/dist/context-pack-lock.js +48 -0
- package/mcp-memory-server/dist/context-pack.js +2 -27
- package/mcp-memory-server/dist/eval-cli.js +25 -2
- package/mcp-memory-server/dist/eval-codex.js +163 -0
- package/mcp-memory-server/dist/eval-protocol.js +200 -0
- package/mcp-memory-server/package.json +3 -1
- package/package.json +1 -1
- package/scripts/cairn-cli.mjs +1 -1
- package/scripts/completion.sh +9 -4
- package/scripts/spikes/verify-release-ledger.mjs +46 -0
- package/scripts/test-completion.sh +7 -3
- package/scripts/test-ecosystem-spikes.sh +1 -0
- package/scripts/test-learning-docs.sh +8 -0
- package/scripts/test-package-install.sh +24 -1
- package/scripts/verify-security-baseline.mjs +70 -9
- package/scripts/windows-platform.mjs +2 -1
package/docs/releasing.md
CHANGED
|
@@ -3,6 +3,23 @@
|
|
|
3
3
|
Publishing is driven by a GitHub Release. Maintainers do not publish the package
|
|
4
4
|
from a workstation.
|
|
5
5
|
|
|
6
|
+
## Advisory Scorecard recovery
|
|
7
|
+
|
|
8
|
+
The repository's Actions policy must allow the exact pinned
|
|
9
|
+
`ossf/scorecard-action@<full-SHA-from-scorecard.yml>` reference. An entry ending
|
|
10
|
+
in `scorecard-action/*` does not allow the root action. Keep SHA pinning required;
|
|
11
|
+
do not enable all third-party actions to recover this one workflow.
|
|
12
|
+
|
|
13
|
+
If GitHub rejects a run before creating any jobs and refuses to retry it, fix
|
|
14
|
+
the repository allowlist and start a fresh advisory analysis explicitly:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
gh workflow run scorecard.yml --ref main
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
This does not bypass required platform, CodeQL or dependency-review gates and
|
|
21
|
+
does not require republishing the npm package.
|
|
22
|
+
|
|
6
23
|
## Release contract
|
|
7
24
|
|
|
8
25
|
1. Update the root `package.json` version and `CHANGELOG.md` on a pull request.
|
|
@@ -52,11 +69,28 @@ different version or tree.
|
|
|
52
69
|
|
|
53
70
|
The repository must provide an `NPM_TOKEN` Actions secret authorized to publish
|
|
54
71
|
`@cairnkeep/cli`. Keep that credential out of local files and rotate it according
|
|
55
|
-
to the npm account's security policy.
|
|
56
|
-
`contents:
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
72
|
+
to the npm account's security policy. Workflow defaults grant only
|
|
73
|
+
`contents: read`; publication authority is explicit and job-scoped:
|
|
74
|
+
|
|
75
|
+
| Job | Token permissions | Purpose |
|
|
76
|
+
|---|---|---|
|
|
77
|
+
| `prepare` | `contents: read`, `actions: read` | Check the tag and download the successful exact-tree CI candidate; no publication or signing authority |
|
|
78
|
+
| `npm` | `contents: write`, `id-token: write` | Attach verified release assets and request npm provenance; registry publication uses the separate `NPM_TOKEN` |
|
|
79
|
+
| `containers` | `contents: read`, `packages: write`, `attestations: write`, `id-token: write`, `artifact-metadata: write` | Read source, publish OCI images, sign/upload provenance and create the pinned action's registry storage record |
|
|
80
|
+
|
|
81
|
+
Each job declares its complete permission map. Unspecified token permissions
|
|
82
|
+
are denied rather than inherited; see [GitHub's permission semantics](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#permissions).
|
|
83
|
+
The container grant retains `artifact-metadata: write` because the pinned
|
|
84
|
+
[attestation implementation](https://github.com/actions/attest/blob/508db95dd578ae2727ebd6217d5ba78e4fbda05d/README.md#usage)
|
|
85
|
+
creates a storage record. Do not remove required provenance permissions merely
|
|
86
|
+
to improve an advisory score.
|
|
87
|
+
|
|
88
|
+
The parsed-YAML baseline rejects workflow-wide writes, unexpected publication
|
|
89
|
+
jobs and any missing or excessive grant in these maps. Its positive control
|
|
90
|
+
and mutation tests verify the declared contract, not GitHub's execution or a
|
|
91
|
+
future privileged publication. Keep the required platform/security gates and
|
|
92
|
+
verify the next normal release's publication and attestations separately.
|
|
93
|
+
CI-only permission changes do not require republishing an immutable version.
|
|
60
94
|
|
|
61
95
|
GitHub creates each new GHCR package as private. After the first container
|
|
62
96
|
release, a package administrator must change both `cairnkeep` and
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Bounded release-ledger task
|
|
2
|
+
|
|
3
|
+
Use a disposable project, Node standard APIs and ESM only. Implement
|
|
4
|
+
`src/release-ledger.mjs` exporting `createLedger()`. The returned API exposes
|
|
5
|
+
`append(input)`, `finalize(packageName, version)`, `get(packageName, version)`
|
|
6
|
+
and `list()`.
|
|
7
|
+
|
|
8
|
+
- Package names match `^[a-z][a-z0-9-]*$`; versions are numeric x.y.z. Notes
|
|
9
|
+
are non-empty arrays of non-empty strings, preserving order. Invalid input
|
|
10
|
+
throws Error with code `INVALID_PACKAGE`, `INVALID_VERSION` or `INVALID_NOTES`.
|
|
11
|
+
- `append({package,version,notes})` creates a draft with monotonically
|
|
12
|
+
increasing sequence starting at 1. A pair is unique forever; duplicates
|
|
13
|
+
throw `DUPLICATE_RELEASE`.
|
|
14
|
+
- Entries have exactly package, version, notes, sequence, state and optional
|
|
15
|
+
checksum. Draft state is `draft`; finalized state is `final`.
|
|
16
|
+
- `finalize` is idempotent. Checksum is lowercase SHA-256 of
|
|
17
|
+
`JSON.stringify({package,version,notes,sequence})` in that property order.
|
|
18
|
+
Missing releases throw `NOT_FOUND`.
|
|
19
|
+
- `get` and `list` return detached copies including nested notes. `list` is
|
|
20
|
+
sequence ordered. Add and run self-tests without changing this brief.
|
|
21
|
+
|
|
22
|
+
After the task closes, run the maintained independent grader from outside the
|
|
23
|
+
project, with an external process timeout:
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
node scripts/spikes/verify-release-ledger.mjs /PATH/TO/LAB/src/release-ledger.mjs
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
This imports and executes the selected artifact; it is not a safe reader for
|
|
30
|
+
arbitrary untrusted code. Use a lab with no production secrets or services.
|
|
31
|
+
The grader checks 15 bounded behaviors, not all possible inputs or security.
|
|
32
|
+
Do not expose its implementation to the task before grading. A model could
|
|
33
|
+
deliberately game any visible rubric; this is a correctness smoke test, not
|
|
34
|
+
adversarial certification. Run protocol observation separately using the
|
|
35
|
+
[memory audit](../agent-memory-protocol.md). Keep raw exports private; publish
|
|
36
|
+
only appropriately reviewed aggregate evidence. One run is not an A/B study.
|
|
@@ -46,6 +46,7 @@ be enabled and evaluated independently.
|
|
|
46
46
|
|---|---|---|
|
|
47
47
|
| Traversal, alternate separators, drive paths, ADS, forbidden Windows characters/device aliases, invalid Unicode, or case collisions in portable data | One cross-platform path validator is used by context packs, OKF, artifacts, and evaluations; deterministic adversarial cases run in the smoke suite | Filesystems can have additional locale- or mount-specific semantics; containment checks remain required at every write |
|
|
48
48
|
| Symlink or special-file substitution | Canonical roots, non-symlink regular-file checks, atomic replacement, and existing symlink/race regression tests | A hostile process running as the same account can race or replace resources outside Cairnkeep's process boundary |
|
|
49
|
+
| Concurrent context-pack project pointer updates | Exclusive directory locks, bounded Windows contention retries, unsafe-lock rejection, injected denial controls and native cross-process exclusion tests | A persistent lock can block progress; recovery never infers abandonment or steals the lock |
|
|
49
50
|
| Unauthenticated or rebound HTTP access | HTTP refuses to start without a token, compares it in constant time, validates `Host`, denies CORS by default, and bounds headers, request time, keep-alive time, and bodies to 8 MiB | Bearer tokens are not user identities; deploy TLS at the ingress for non-loopback traffic |
|
|
50
51
|
| Excessive MCP authority | Explicit MCP annotations, restrictive profiles, capability intersection, and separate HTTP consent for sensitive optional tools | The default profile remains `full` for compatibility; operators should choose `read-only` or `custom` for observation-only clients |
|
|
51
52
|
| Malicious repository or context content influencing an agent | Memory results are locators, skills need digest-bound approval, pack content stays read-only, and maintained source must be verified | Prompt injection is not solved; the harness/model must continue treating retrieved text as untrusted data |
|
|
@@ -105,6 +106,16 @@ provenance, produces CycloneDX 1.6 and container SBOMs, and attests container
|
|
|
105
106
|
provenance. A digest proves content integrity, not publisher identity; verify
|
|
106
107
|
the repository, release tag, registry namespace, and provenance together.
|
|
107
108
|
|
|
109
|
+
Workflow token defaults are read-only. Publication grants are explicit per job:
|
|
110
|
+
candidate preparation receives only source/Actions read access, npm receives
|
|
111
|
+
release-attachment and provenance authority, and container publication receives
|
|
112
|
+
registry/signing/attestation/storage-record authority without release-write
|
|
113
|
+
access. See the exact maps in [Releasing](releasing.md). The security baseline
|
|
114
|
+
rejects excess and missing grants with positive and adversarial controls.
|
|
115
|
+
This reduces unnecessary token authority; it neither isolates steps within a
|
|
116
|
+
job nor constrains the separate npm publishing secret. Static checks do not
|
|
117
|
+
prove a future privileged publish run succeeds.
|
|
118
|
+
|
|
108
119
|
## Deliberate non-goals and residual risks
|
|
109
120
|
|
|
110
121
|
- Cairnkeep is not a sandbox, endpoint-security product, secret manager, TLS
|
package/examples/eval/README.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Offline evaluation fixture
|
|
2
2
|
|
|
3
|
+
The separate `protocol-project-search.json`, `protocol-narration.json` and
|
|
4
|
+
native-format `protocol-codex.jsonl`
|
|
5
|
+
controls exercise `cairn eval protocol`, without running an adapter. See the
|
|
6
|
+
[memory protocol guide](../../docs/agent-memory-protocol.md). These synthetic
|
|
7
|
+
controls demonstrate parser behavior, not live model quality.
|
|
8
|
+
|
|
9
|
+
`scripts/spikes/verify-release-ledger.mjs MODULE_PATH` separately grades the
|
|
10
|
+
bounded release-ledger coding task with 15 deterministic artifact checks. It
|
|
11
|
+
imports and executes the explicitly selected module: use an isolated lab and
|
|
12
|
+
an external timeout, never untrusted production files. Its score is not a
|
|
13
|
+
protocol score, a comprehensive security audit or an A/B comparison.
|
|
14
|
+
The [task brief](../../docs/research/release-ledger-task.md) defines the lab.
|
|
15
|
+
|
|
3
16
|
This directory contains Cairnkeep's deterministic, network-free evaluation
|
|
4
17
|
population. It validates the evaluation framework; its designed differences are
|
|
5
18
|
not performance, efficiency, quality, or live-product evidence.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"identifier":"cairn-offline-fake-v1","package_version":"2.
|
|
1
|
+
{"identifier":"cairn-offline-fake-v1","package_version":"2.22.1","schema_version":1,"task_set_digest":"13e95379f35839593047ec34521a65a8fa3ab92bd3993bca9dc8bf5c4a9ff36f"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
{"type":"thread.started","thread_id":"synthetic-control"}
|
|
2
|
+
{"type":"turn.started"}
|
|
3
|
+
{"type":"item.started","item":{"id":"a","type":"mcp_tool_call","server":"cairn-memory","tool":"memory_search","arguments":{"scope":"project","query":"release ledger"},"status":"in_progress"}}
|
|
4
|
+
{"type":"item.completed","item":{"id":"a","type":"mcp_tool_call","server":"cairn-memory","tool":"memory_search","arguments":{"scope":"project","query":"release ledger"},"status":"completed","result":{"content":[],"structured_content":{"mode":"substring","count":0,"results":[]}}}}
|
|
5
|
+
{"type":"turn.completed","usage":{"input_tokens":10,"cached_input_tokens":0,"output_tokens":1}}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": 1,
|
|
3
|
+
"session_id": "protocol-negative-control",
|
|
4
|
+
"harness": "pi",
|
|
5
|
+
"project_root": "/synthetic/project",
|
|
6
|
+
"started_at": "2026-10-07T00:00:00.000Z",
|
|
7
|
+
"ended_at": "2026-10-07T00:01:00.000Z",
|
|
8
|
+
"events": [
|
|
9
|
+
{"sequence": 0, "kind": "model_output", "payload": {"text": "I called memory_search with scope project and found the ledger conventions."}}
|
|
10
|
+
],
|
|
11
|
+
"capture": {"captured_at": "2026-10-07T00:01:00.000Z", "omitted_reasoning_blocks": 0, "omitted_unknown_records": 0, "truncated": false}
|
|
12
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": 1,
|
|
3
|
+
"session_id": "protocol-positive-control",
|
|
4
|
+
"harness": "pi",
|
|
5
|
+
"project_root": "/synthetic/project",
|
|
6
|
+
"started_at": "2026-10-07T00:00:00.000Z",
|
|
7
|
+
"ended_at": "2026-10-07T00:01:00.000Z",
|
|
8
|
+
"events": [
|
|
9
|
+
{"sequence": 0, "kind": "tool_invocation", "payload": {"call_id": "search-1", "tool_name": "memory_search", "input": {"scope": "project", "query": "ledger validation conventions"}}},
|
|
10
|
+
{"sequence": 1, "kind": "tool_result", "payload": {"call_id": "search-1", "is_error": false, "output": "{\"mode\":\"substring\",\"count\":0,\"results\":[]}"}}
|
|
11
|
+
],
|
|
12
|
+
"capture": {"captured_at": "2026-10-07T00:01:00.000Z", "omitted_reasoning_blocks": 0, "omitted_unknown_records": 0, "truncated": false}
|
|
13
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { lstatSync } from "node:fs";
|
|
2
|
+
import { mkdir, rm } from "node:fs/promises";
|
|
3
|
+
const filesystem = {
|
|
4
|
+
platform: process.platform,
|
|
5
|
+
create: (path) => mkdir(path, { mode: 0o700 }),
|
|
6
|
+
inspect: (path) => lstatSync(path),
|
|
7
|
+
remove: (path) => rm(path, { recursive: true, force: true }),
|
|
8
|
+
wait: (milliseconds) => new Promise((resolveWait) => setTimeout(resolveWait, milliseconds)),
|
|
9
|
+
};
|
|
10
|
+
// The internal filesystem seam lets offline controls reproduce platform errors
|
|
11
|
+
// without replacing process.platform or weakening the real filesystem checks.
|
|
12
|
+
export async function acquireContextPackPointerLock(lock, operations = filesystem) {
|
|
13
|
+
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
14
|
+
try {
|
|
15
|
+
await operations.create(lock);
|
|
16
|
+
return () => operations.remove(lock);
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
const code = error.code;
|
|
20
|
+
const windowsContention = operations.platform === "win32" && code === "EPERM";
|
|
21
|
+
if (code !== "EEXIST" && !windowsContention)
|
|
22
|
+
throw error;
|
|
23
|
+
let info;
|
|
24
|
+
try {
|
|
25
|
+
info = operations.inspect(lock);
|
|
26
|
+
}
|
|
27
|
+
catch (inspectionError) {
|
|
28
|
+
const inspectionCode = inspectionError.code;
|
|
29
|
+
if (inspectionCode === "ENOENT") {
|
|
30
|
+
// A Windows delete-pending directory can temporarily appear
|
|
31
|
+
// absent; pace this case under the same bounded retry budget.
|
|
32
|
+
if (windowsContention)
|
|
33
|
+
await operations.wait(10);
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
if (operations.platform === "win32" && ["EPERM", "EBUSY"].includes(inspectionCode ?? "")) {
|
|
37
|
+
await operations.wait(10);
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
throw inspectionError;
|
|
41
|
+
}
|
|
42
|
+
if (!info.isDirectory() || info.isSymbolicLink())
|
|
43
|
+
throw new Error("Context pack project pointer lock is unsafe.");
|
|
44
|
+
await operations.wait(10);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
throw new Error("Context pack project pointer is locked; retry after the active update finishes.");
|
|
48
|
+
}
|
|
@@ -11,6 +11,7 @@ import { doctorProgressiveContextCache, loadProgressiveContext, publicTree, sele
|
|
|
11
11
|
import { indexOkfBundle, validateOkfBundle, validatedOkfFileContent } from "./okf.js";
|
|
12
12
|
import { atomicReplace, hardenPrivatePath, privatePathIsSafe } from "./platform-security.js";
|
|
13
13
|
import { portablePathCollisionKey, portableRelativePathIssue } from "./path-security.js";
|
|
14
|
+
import { acquireContextPackPointerLock } from "./context-pack-lock.js";
|
|
14
15
|
const execFileAsync = promisify(execFile);
|
|
15
16
|
export const CONTEXT_PACK_MANIFEST = "context-pack.json";
|
|
16
17
|
export const CONTEXT_PACK_MAX_FILE_BYTES = 1024 * 1024;
|
|
@@ -611,36 +612,10 @@ export function readProjectPointer(options) {
|
|
|
611
612
|
throw new Error("Context pack project pointer is unsafe.");
|
|
612
613
|
return parseProjectPointer(JSON.parse(readFileSync(identity.path, "utf8")), identity.id);
|
|
613
614
|
}
|
|
614
|
-
function delay(milliseconds) {
|
|
615
|
-
return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds));
|
|
616
|
-
}
|
|
617
615
|
async function acquirePointerLock(options) {
|
|
618
616
|
const identity = projectIdentity(options);
|
|
619
617
|
await mkdir(dirname(identity.path), { recursive: true, mode: 0o700 });
|
|
620
|
-
|
|
621
|
-
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
622
|
-
try {
|
|
623
|
-
await mkdir(lock, { mode: 0o700 });
|
|
624
|
-
return () => rm(lock, { recursive: true, force: true });
|
|
625
|
-
}
|
|
626
|
-
catch (error) {
|
|
627
|
-
if (error.code !== "EEXIST")
|
|
628
|
-
throw error;
|
|
629
|
-
let info;
|
|
630
|
-
try {
|
|
631
|
-
info = lstatSync(lock);
|
|
632
|
-
}
|
|
633
|
-
catch (inspectionError) {
|
|
634
|
-
if (inspectionError.code === "ENOENT")
|
|
635
|
-
continue;
|
|
636
|
-
throw inspectionError;
|
|
637
|
-
}
|
|
638
|
-
if (!info.isDirectory() || info.isSymbolicLink())
|
|
639
|
-
throw new Error("Context pack project pointer lock is unsafe.");
|
|
640
|
-
await delay(10);
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
|
-
throw new Error("Context pack project pointer is locked; retry after the active update finishes.");
|
|
618
|
+
return acquireContextPackPointerLock(`${identity.path}.lock`);
|
|
644
619
|
}
|
|
645
620
|
async function mutateProjectPointer(options, mutate) {
|
|
646
621
|
const release = await acquirePointerLock(options);
|
|
@@ -8,19 +8,21 @@ import { diagnoseEvalReport, readEvalReport, renderEvalReport, } from "./eval-re
|
|
|
8
8
|
import { buildAblationArms, runCapabilityAblation, runTwoPassExperiment, } from "./eval-runner.js";
|
|
9
9
|
import { canonicalJson, EVAL_SCHEMA_VERSION } from "./eval-schema.js";
|
|
10
10
|
import { privatePathIsSafe } from "./platform-security.js";
|
|
11
|
+
import { auditMemoryProtocol, readProtocolTrajectory, readCodexProtocol } from "./eval-protocol.js";
|
|
11
12
|
process.stdout.on("error", (error) => {
|
|
12
13
|
if (error.code === "EPIPE")
|
|
13
14
|
process.exit(0);
|
|
14
15
|
throw error;
|
|
15
16
|
});
|
|
16
|
-
const publicCommands = new Set(["validate", "run", "ablate", "report", "prune", "delete"]);
|
|
17
|
+
const publicCommands = new Set(["validate", "run", "ablate", "report", "prune", "delete", "protocol"]);
|
|
18
|
+
const protocolFlags = new Set(["--trajectory", "--codex-jsonl", "--capture-authorized", "--json"]);
|
|
17
19
|
const validateFlags = new Set(["--task-set", "--adapter", "--output", "--repetitions", "--seed", "--json"]);
|
|
18
20
|
const runFlags = new Set([...validateFlags, "--yes"]);
|
|
19
21
|
const ablateFlags = new Set([...runFlags, "--disable"]);
|
|
20
22
|
const reportFlags = new Set(["--experiment", "--json"]);
|
|
21
23
|
const pruneFlags = new Set(["--older-than-days", "--dry-run", "--json"]);
|
|
22
24
|
const deleteFlags = new Set(["--experiment", "--dry-run", "--json"]);
|
|
23
|
-
const valueFlags = new Set(["--task-set", "--adapter", "--output", "--repetitions", "--seed", "--disable", "--experiment", "--older-than-days"]);
|
|
25
|
+
const valueFlags = new Set(["--task-set", "--adapter", "--output", "--repetitions", "--seed", "--disable", "--experiment", "--older-than-days", "--trajectory", "--codex-jsonl"]);
|
|
24
26
|
const EXPERIMENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
|
25
27
|
const DEFAULT_REPORT_BYTES = 16 * 1024 * 1024;
|
|
26
28
|
const DEFAULT_RETENTION_DAYS = 30;
|
|
@@ -36,9 +38,14 @@ Usage:
|
|
|
36
38
|
cairn eval report --experiment ID [--json]
|
|
37
39
|
cairn eval prune [--older-than-days N] [--dry-run] [--json]
|
|
38
40
|
cairn eval delete --experiment ID [--dry-run] [--json]
|
|
41
|
+
cairn eval protocol --trajectory PATH [--capture-authorized] [--json]
|
|
42
|
+
cairn eval protocol --codex-jsonl PATH [--capture-authorized] [--json]
|
|
39
43
|
|
|
40
44
|
Evaluation is disabled unless CAIRN_EVAL is explicitly enabled. Live harness
|
|
41
45
|
commands remain operator-owned; validate resolves inputs without executing one.
|
|
46
|
+
Protocol audits normalized trajectories or single-turn Codex exec JSONL offline.
|
|
47
|
+
No transcript execution, capture activation, network requests or writes occur.
|
|
48
|
+
Capture authorization is a caller assertion; protocol compliance is not task quality.
|
|
42
49
|
`;
|
|
43
50
|
}
|
|
44
51
|
function valueAfter(args, flag) {
|
|
@@ -466,6 +473,22 @@ async function main() {
|
|
|
466
473
|
disabled(json);
|
|
467
474
|
return;
|
|
468
475
|
}
|
|
476
|
+
if (command === "protocol") {
|
|
477
|
+
assertKnown(args, protocolFlags);
|
|
478
|
+
if (args.includes("--trajectory") === args.includes("--codex-jsonl")) {
|
|
479
|
+
throw new Error("Supply exactly one of --trajectory or --codex-jsonl.");
|
|
480
|
+
}
|
|
481
|
+
const loaded = args.includes("--codex-jsonl") ? readCodexProtocol(requireValue(args, "--codex-jsonl"))
|
|
482
|
+
: readProtocolTrajectory(requireValue(args, "--trajectory"));
|
|
483
|
+
const value = auditMemoryProtocol(loaded.session, loaded.digest, args.includes("--capture-authorized"));
|
|
484
|
+
process.stdout.write(`${json ? JSON.stringify(value) : [
|
|
485
|
+
`Memory protocol: ${value.status}`,
|
|
486
|
+
...value.checks.map(check => `${check.id}: ${check.status} (${check.code})`),
|
|
487
|
+
"Local evidence only; task quality and authenticated consent are not measured.",
|
|
488
|
+
].join("\n")}\n`);
|
|
489
|
+
process.exitCode = value.status === "fail" ? 1 : value.status === "inconclusive" ? 3 : 0;
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
469
492
|
if (command === "validate") {
|
|
470
493
|
assertKnown(args, validateFlags);
|
|
471
494
|
const plan = loadPlan(args);
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { canonicalJson } from "./eval-schema.js";
|
|
2
|
+
function record(value) {
|
|
3
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : undefined;
|
|
4
|
+
}
|
|
5
|
+
function invalid() { throw new Error("protocol_input_invalid_or_unsafe"); }
|
|
6
|
+
/** Explicit single-turn exec JSONL export; never an internal rollout reader. */
|
|
7
|
+
export function normalizeCodexExec(text) {
|
|
8
|
+
try {
|
|
9
|
+
if (Buffer.byteLength(text, "utf8") > 16 * 1024 * 1024)
|
|
10
|
+
invalid();
|
|
11
|
+
const lines = text.split("\n");
|
|
12
|
+
if (lines.at(-1) === "" || lines.at(-1) === "\r")
|
|
13
|
+
lines.pop();
|
|
14
|
+
if (!lines.length || lines.length > 50_000)
|
|
15
|
+
invalid();
|
|
16
|
+
const events = [];
|
|
17
|
+
const items = new Map();
|
|
18
|
+
let thread = false, turn = false, ended = false, gaps = 0, reasoning = 0;
|
|
19
|
+
const emit = (kind, payload) => {
|
|
20
|
+
events.push({ sequence: events.length, kind, payload });
|
|
21
|
+
};
|
|
22
|
+
const extra = (value, fields) => {
|
|
23
|
+
if (Object.keys(value).some(key => !fields.includes(key)))
|
|
24
|
+
gaps++;
|
|
25
|
+
};
|
|
26
|
+
for (const line of lines) {
|
|
27
|
+
const event = record(JSON.parse(line));
|
|
28
|
+
if (!event || typeof event.type !== "string")
|
|
29
|
+
invalid();
|
|
30
|
+
if (event.type === "thread.started") {
|
|
31
|
+
if (thread || turn || ended || typeof event.thread_id !== "string" || !event.thread_id)
|
|
32
|
+
invalid();
|
|
33
|
+
thread = true;
|
|
34
|
+
extra(event, ["type", "thread_id"]);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
if (event.type === "turn.started") {
|
|
38
|
+
if (turn || ended)
|
|
39
|
+
invalid();
|
|
40
|
+
if (!thread)
|
|
41
|
+
gaps++;
|
|
42
|
+
turn = true;
|
|
43
|
+
extra(event, ["type"]);
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
if (event.type === "turn.completed" || event.type === "turn.failed") {
|
|
47
|
+
if (ended)
|
|
48
|
+
invalid();
|
|
49
|
+
if (!turn || event.type === "turn.failed")
|
|
50
|
+
gaps++;
|
|
51
|
+
ended = true;
|
|
52
|
+
extra(event, ["type", "usage", "error"]);
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
if (event.type === "error") {
|
|
56
|
+
gaps++;
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (!["item.started", "item.updated", "item.completed"].includes(event.type)) {
|
|
60
|
+
gaps++;
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (ended)
|
|
64
|
+
invalid();
|
|
65
|
+
if (!turn)
|
|
66
|
+
gaps++;
|
|
67
|
+
extra(event, ["type", "item"]);
|
|
68
|
+
const item = record(event.item);
|
|
69
|
+
if (!item || typeof item.id !== "string" || !item.id || item.id.length > 1024 || typeof item.type !== "string")
|
|
70
|
+
invalid();
|
|
71
|
+
const id = item.id;
|
|
72
|
+
const complete = event.type === "item.completed";
|
|
73
|
+
const previous = items.get(id);
|
|
74
|
+
if (previous && (previous.complete || previous.type !== item.type || event.type === "item.started"))
|
|
75
|
+
invalid();
|
|
76
|
+
const tool = ["mcp_tool_call", "command_execution", "file_change", "web_search"].includes(item.type);
|
|
77
|
+
let identity = item.type;
|
|
78
|
+
let name = item.type;
|
|
79
|
+
let input = {};
|
|
80
|
+
let output = null;
|
|
81
|
+
if (item.type === "mcp_tool_call") {
|
|
82
|
+
extra(item, ["id", "type", "server", "tool", "arguments", "result", "error", "status"]);
|
|
83
|
+
if (typeof item.server !== "string" || !item.server || typeof item.tool !== "string" || !item.tool
|
|
84
|
+
|| !record(item.arguments))
|
|
85
|
+
invalid();
|
|
86
|
+
name = `mcp__${item.server}__${item.tool}`;
|
|
87
|
+
input = item.arguments;
|
|
88
|
+
identity = canonicalJson({ server: item.server, tool: item.tool, arguments: input });
|
|
89
|
+
const result = record(item.result);
|
|
90
|
+
if (result) {
|
|
91
|
+
extra(result, ["content", "structured_content", "_meta"]);
|
|
92
|
+
output = result.structured_content === undefined || result.structured_content === null
|
|
93
|
+
? { content: result.content } : { structuredContent: result.structured_content };
|
|
94
|
+
}
|
|
95
|
+
// Native CLI snapshots serialize absent result/error as null.
|
|
96
|
+
if (!complete && (item.result != null || item.error != null))
|
|
97
|
+
invalid();
|
|
98
|
+
}
|
|
99
|
+
else if (item.type === "command_execution") {
|
|
100
|
+
extra(item, ["id", "type", "command", "aggregated_output", "exit_code", "status"]);
|
|
101
|
+
if (typeof item.command !== "string")
|
|
102
|
+
invalid();
|
|
103
|
+
identity = canonicalJson({ command: item.command });
|
|
104
|
+
}
|
|
105
|
+
else if (item.type === "file_change") {
|
|
106
|
+
extra(item, ["id", "type", "changes", "status"]);
|
|
107
|
+
if (!Array.isArray(item.changes))
|
|
108
|
+
invalid();
|
|
109
|
+
}
|
|
110
|
+
else if (item.type === "web_search") {
|
|
111
|
+
extra(item, ["id", "type", "query"]);
|
|
112
|
+
if (typeof item.query !== "string")
|
|
113
|
+
invalid();
|
|
114
|
+
}
|
|
115
|
+
else if (item.type === "agent_message" || item.type === "reasoning") {
|
|
116
|
+
extra(item, ["id", "type", "text"]);
|
|
117
|
+
if (typeof item.text !== "string")
|
|
118
|
+
invalid();
|
|
119
|
+
if (item.type === "reasoning" && !previous)
|
|
120
|
+
reasoning++;
|
|
121
|
+
}
|
|
122
|
+
else if (item.type === "todo_list") {
|
|
123
|
+
extra(item, ["id", "type", "items"]);
|
|
124
|
+
if (!Array.isArray(item.items))
|
|
125
|
+
invalid();
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
gaps++;
|
|
129
|
+
}
|
|
130
|
+
if (previous && previous.identity !== identity)
|
|
131
|
+
invalid();
|
|
132
|
+
if (tool) {
|
|
133
|
+
if (!previous) {
|
|
134
|
+
// MCP/command starts carry dispatch order. A terminal-only
|
|
135
|
+
// record cannot establish it, but must still expose writes.
|
|
136
|
+
if (["mcp_tool_call", "command_execution"].includes(item.type) && event.type !== "item.started")
|
|
137
|
+
gaps++;
|
|
138
|
+
emit("tool_invocation", { call_id: id, tool_name: name, input });
|
|
139
|
+
}
|
|
140
|
+
if (["mcp_tool_call", "command_execution", "file_change"].includes(item.type)) {
|
|
141
|
+
if (complete ? !["completed", "failed"].includes(String(item.status)) : item.status !== "in_progress")
|
|
142
|
+
invalid();
|
|
143
|
+
}
|
|
144
|
+
if (complete)
|
|
145
|
+
emit("tool_result", { call_id: id, output, is_error: item.status === "failed" || item.error != null });
|
|
146
|
+
}
|
|
147
|
+
items.set(id, { type: item.type, identity, complete });
|
|
148
|
+
}
|
|
149
|
+
if (!thread || !turn || !ended || [...items.values()].some(item => !item.complete))
|
|
150
|
+
gaps++;
|
|
151
|
+
if (events.length > 50_000)
|
|
152
|
+
invalid();
|
|
153
|
+
// Timestamps/project identity are not supplied by exec JSONL. These
|
|
154
|
+
// placeholders are private adapter scaffolding, never output claims.
|
|
155
|
+
const time = "1970-01-01T00:00:00.000Z";
|
|
156
|
+
return { schema_version: 1, session_id: "codex-exec-export", harness: "codex", project_root: "not-supplied",
|
|
157
|
+
started_at: time, ended_at: time, events,
|
|
158
|
+
capture: { captured_at: time, omitted_reasoning_blocks: reasoning, omitted_unknown_records: gaps, truncated: false } };
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return invalid();
|
|
162
|
+
}
|
|
163
|
+
}
|