@cairnkeep/cli 2.21.1 → 2.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/docs/README.md +2 -0
  3. package/docs/agent-memory-protocol.md +146 -0
  4. package/docs/agents.md +7 -0
  5. package/docs/learning/CURRICULUM-MAP.md +3 -2
  6. package/docs/learning/FEATURE-GUIDE.md +7 -1
  7. package/docs/learning/lessons/L00-why-cairnkeep.md +1 -1
  8. package/docs/learning/lessons/L01-safe-trial.md +1 -1
  9. package/docs/learning/lessons/L02-installation.md +1 -1
  10. package/docs/learning/lessons/L03-first-project.md +1 -1
  11. package/docs/learning/lessons/L16-evaluation.md +23 -0
  12. package/docs/learning/lessons/L19-validated-skill-improvement.md +1 -1
  13. package/docs/learning/lessons/L20-mcp-tool-profiles.md +1 -1
  14. package/docs/learning/lessons/L21-context-packs.md +1 -1
  15. package/docs/learning/lessons/L22-native-windows.md +1 -1
  16. package/docs/learning/lessons/L23-guided-setup.md +1 -1
  17. package/docs/learning/lessons/L24-okf-exchange.md +1 -1
  18. package/docs/learning/lessons/L25-playbooks.md +1 -1
  19. package/docs/learning/lessons/L26-context-intelligence.md +1 -1
  20. package/docs/learning/lessons/L27-ecosystem-evidence.md +1 -1
  21. package/docs/learning/lessons/L28-security-assurance.md +2 -2
  22. package/docs/learning/tracks/evidence-and-evaluation.md +4 -0
  23. package/docs/learning/video-scripts/V16-memory-protocol-audit.md +32 -0
  24. package/docs/operating.md +8 -0
  25. package/docs/privacy-and-data-flow.md +9 -0
  26. package/docs/research/release-ledger-task.md +36 -0
  27. package/examples/eval/README.md +13 -0
  28. package/examples/eval/bundled-fake.json +1 -1
  29. package/examples/eval/protocol-codex.jsonl +5 -0
  30. package/examples/eval/protocol-narration.json +12 -0
  31. package/examples/eval/protocol-project-search.json +13 -0
  32. package/mcp-memory-server/dist/eval-cli.js +25 -2
  33. package/mcp-memory-server/dist/eval-codex.js +163 -0
  34. package/mcp-memory-server/dist/eval-protocol.js +200 -0
  35. package/mcp-memory-server/package.json +2 -0
  36. package/package.json +1 -1
  37. package/scripts/cairn-cli.mjs +1 -1
  38. package/scripts/completion.sh +9 -4
  39. package/scripts/spikes/verify-release-ledger.mjs +46 -0
  40. package/scripts/test-completion.sh +7 -3
  41. package/scripts/test-ecosystem-spikes.sh +1 -0
  42. package/scripts/test-package-install.sh +24 -1
  43. package/scripts/windows-platform.mjs +2 -1
@@ -8,19 +8,21 @@ import { diagnoseEvalReport, readEvalReport, renderEvalReport, } from "./eval-re
8
8
  import { buildAblationArms, runCapabilityAblation, runTwoPassExperiment, } from "./eval-runner.js";
9
9
  import { canonicalJson, EVAL_SCHEMA_VERSION } from "./eval-schema.js";
10
10
  import { privatePathIsSafe } from "./platform-security.js";
11
+ import { auditMemoryProtocol, readProtocolTrajectory, readCodexProtocol } from "./eval-protocol.js";
11
12
  process.stdout.on("error", (error) => {
12
13
  if (error.code === "EPIPE")
13
14
  process.exit(0);
14
15
  throw error;
15
16
  });
16
- const publicCommands = new Set(["validate", "run", "ablate", "report", "prune", "delete"]);
17
+ const publicCommands = new Set(["validate", "run", "ablate", "report", "prune", "delete", "protocol"]);
18
+ const protocolFlags = new Set(["--trajectory", "--codex-jsonl", "--capture-authorized", "--json"]);
17
19
  const validateFlags = new Set(["--task-set", "--adapter", "--output", "--repetitions", "--seed", "--json"]);
18
20
  const runFlags = new Set([...validateFlags, "--yes"]);
19
21
  const ablateFlags = new Set([...runFlags, "--disable"]);
20
22
  const reportFlags = new Set(["--experiment", "--json"]);
21
23
  const pruneFlags = new Set(["--older-than-days", "--dry-run", "--json"]);
22
24
  const deleteFlags = new Set(["--experiment", "--dry-run", "--json"]);
23
- const valueFlags = new Set(["--task-set", "--adapter", "--output", "--repetitions", "--seed", "--disable", "--experiment", "--older-than-days"]);
25
+ const valueFlags = new Set(["--task-set", "--adapter", "--output", "--repetitions", "--seed", "--disable", "--experiment", "--older-than-days", "--trajectory", "--codex-jsonl"]);
24
26
  const EXPERIMENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
25
27
  const DEFAULT_REPORT_BYTES = 16 * 1024 * 1024;
26
28
  const DEFAULT_RETENTION_DAYS = 30;
@@ -36,9 +38,14 @@ Usage:
36
38
  cairn eval report --experiment ID [--json]
37
39
  cairn eval prune [--older-than-days N] [--dry-run] [--json]
38
40
  cairn eval delete --experiment ID [--dry-run] [--json]
41
+ cairn eval protocol --trajectory PATH [--capture-authorized] [--json]
42
+ cairn eval protocol --codex-jsonl PATH [--capture-authorized] [--json]
39
43
 
40
44
  Evaluation is disabled unless CAIRN_EVAL is explicitly enabled. Live harness
41
45
  commands remain operator-owned; validate resolves inputs without executing one.
46
+ Protocol audits normalized trajectories or single-turn Codex exec JSONL offline.
47
+ No transcript execution, capture activation, network requests or writes occur.
48
+ Capture authorization is a caller assertion; protocol compliance is not task quality.
42
49
  `;
43
50
  }
44
51
  function valueAfter(args, flag) {
@@ -466,6 +473,22 @@ async function main() {
466
473
  disabled(json);
467
474
  return;
468
475
  }
476
+ if (command === "protocol") {
477
+ assertKnown(args, protocolFlags);
478
+ if (args.includes("--trajectory") === args.includes("--codex-jsonl")) {
479
+ throw new Error("Supply exactly one of --trajectory or --codex-jsonl.");
480
+ }
481
+ const loaded = args.includes("--codex-jsonl") ? readCodexProtocol(requireValue(args, "--codex-jsonl"))
482
+ : readProtocolTrajectory(requireValue(args, "--trajectory"));
483
+ const value = auditMemoryProtocol(loaded.session, loaded.digest, args.includes("--capture-authorized"));
484
+ process.stdout.write(`${json ? JSON.stringify(value) : [
485
+ `Memory protocol: ${value.status}`,
486
+ ...value.checks.map(check => `${check.id}: ${check.status} (${check.code})`),
487
+ "Local evidence only; task quality and authenticated consent are not measured.",
488
+ ].join("\n")}\n`);
489
+ process.exitCode = value.status === "fail" ? 1 : value.status === "inconclusive" ? 3 : 0;
490
+ return;
491
+ }
469
492
  if (command === "validate") {
470
493
  assertKnown(args, validateFlags);
471
494
  const plan = loadPlan(args);
@@ -0,0 +1,163 @@
1
+ import { canonicalJson } from "./eval-schema.js";
2
+ function record(value) {
3
+ return value !== null && typeof value === "object" && !Array.isArray(value) ? value : undefined;
4
+ }
5
+ function invalid() { throw new Error("protocol_input_invalid_or_unsafe"); }
6
+ /** Explicit single-turn exec JSONL export; never an internal rollout reader. */
7
+ export function normalizeCodexExec(text) {
8
+ try {
9
+ if (Buffer.byteLength(text, "utf8") > 16 * 1024 * 1024)
10
+ invalid();
11
+ const lines = text.split("\n");
12
+ if (lines.at(-1) === "" || lines.at(-1) === "\r")
13
+ lines.pop();
14
+ if (!lines.length || lines.length > 50_000)
15
+ invalid();
16
+ const events = [];
17
+ const items = new Map();
18
+ let thread = false, turn = false, ended = false, gaps = 0, reasoning = 0;
19
+ const emit = (kind, payload) => {
20
+ events.push({ sequence: events.length, kind, payload });
21
+ };
22
+ const extra = (value, fields) => {
23
+ if (Object.keys(value).some(key => !fields.includes(key)))
24
+ gaps++;
25
+ };
26
+ for (const line of lines) {
27
+ const event = record(JSON.parse(line));
28
+ if (!event || typeof event.type !== "string")
29
+ invalid();
30
+ if (event.type === "thread.started") {
31
+ if (thread || turn || ended || typeof event.thread_id !== "string" || !event.thread_id)
32
+ invalid();
33
+ thread = true;
34
+ extra(event, ["type", "thread_id"]);
35
+ continue;
36
+ }
37
+ if (event.type === "turn.started") {
38
+ if (turn || ended)
39
+ invalid();
40
+ if (!thread)
41
+ gaps++;
42
+ turn = true;
43
+ extra(event, ["type"]);
44
+ continue;
45
+ }
46
+ if (event.type === "turn.completed" || event.type === "turn.failed") {
47
+ if (ended)
48
+ invalid();
49
+ if (!turn || event.type === "turn.failed")
50
+ gaps++;
51
+ ended = true;
52
+ extra(event, ["type", "usage", "error"]);
53
+ continue;
54
+ }
55
+ if (event.type === "error") {
56
+ gaps++;
57
+ continue;
58
+ }
59
+ if (!["item.started", "item.updated", "item.completed"].includes(event.type)) {
60
+ gaps++;
61
+ continue;
62
+ }
63
+ if (ended)
64
+ invalid();
65
+ if (!turn)
66
+ gaps++;
67
+ extra(event, ["type", "item"]);
68
+ const item = record(event.item);
69
+ if (!item || typeof item.id !== "string" || !item.id || item.id.length > 1024 || typeof item.type !== "string")
70
+ invalid();
71
+ const id = item.id;
72
+ const complete = event.type === "item.completed";
73
+ const previous = items.get(id);
74
+ if (previous && (previous.complete || previous.type !== item.type || event.type === "item.started"))
75
+ invalid();
76
+ const tool = ["mcp_tool_call", "command_execution", "file_change", "web_search"].includes(item.type);
77
+ let identity = item.type;
78
+ let name = item.type;
79
+ let input = {};
80
+ let output = null;
81
+ if (item.type === "mcp_tool_call") {
82
+ extra(item, ["id", "type", "server", "tool", "arguments", "result", "error", "status"]);
83
+ if (typeof item.server !== "string" || !item.server || typeof item.tool !== "string" || !item.tool
84
+ || !record(item.arguments))
85
+ invalid();
86
+ name = `mcp__${item.server}__${item.tool}`;
87
+ input = item.arguments;
88
+ identity = canonicalJson({ server: item.server, tool: item.tool, arguments: input });
89
+ const result = record(item.result);
90
+ if (result) {
91
+ extra(result, ["content", "structured_content", "_meta"]);
92
+ output = result.structured_content === undefined || result.structured_content === null
93
+ ? { content: result.content } : { structuredContent: result.structured_content };
94
+ }
95
+ // Native CLI snapshots serialize absent result/error as null.
96
+ if (!complete && (item.result != null || item.error != null))
97
+ invalid();
98
+ }
99
+ else if (item.type === "command_execution") {
100
+ extra(item, ["id", "type", "command", "aggregated_output", "exit_code", "status"]);
101
+ if (typeof item.command !== "string")
102
+ invalid();
103
+ identity = canonicalJson({ command: item.command });
104
+ }
105
+ else if (item.type === "file_change") {
106
+ extra(item, ["id", "type", "changes", "status"]);
107
+ if (!Array.isArray(item.changes))
108
+ invalid();
109
+ }
110
+ else if (item.type === "web_search") {
111
+ extra(item, ["id", "type", "query"]);
112
+ if (typeof item.query !== "string")
113
+ invalid();
114
+ }
115
+ else if (item.type === "agent_message" || item.type === "reasoning") {
116
+ extra(item, ["id", "type", "text"]);
117
+ if (typeof item.text !== "string")
118
+ invalid();
119
+ if (item.type === "reasoning" && !previous)
120
+ reasoning++;
121
+ }
122
+ else if (item.type === "todo_list") {
123
+ extra(item, ["id", "type", "items"]);
124
+ if (!Array.isArray(item.items))
125
+ invalid();
126
+ }
127
+ else {
128
+ gaps++;
129
+ }
130
+ if (previous && previous.identity !== identity)
131
+ invalid();
132
+ if (tool) {
133
+ if (!previous) {
134
+ // MCP/command starts carry dispatch order. A terminal-only
135
+ // record cannot establish it, but must still expose writes.
136
+ if (["mcp_tool_call", "command_execution"].includes(item.type) && event.type !== "item.started")
137
+ gaps++;
138
+ emit("tool_invocation", { call_id: id, tool_name: name, input });
139
+ }
140
+ if (["mcp_tool_call", "command_execution", "file_change"].includes(item.type)) {
141
+ if (complete ? !["completed", "failed"].includes(String(item.status)) : item.status !== "in_progress")
142
+ invalid();
143
+ }
144
+ if (complete)
145
+ emit("tool_result", { call_id: id, output, is_error: item.status === "failed" || item.error != null });
146
+ }
147
+ items.set(id, { type: item.type, identity, complete });
148
+ }
149
+ if (!thread || !turn || !ended || [...items.values()].some(item => !item.complete))
150
+ gaps++;
151
+ if (events.length > 50_000)
152
+ invalid();
153
+ // Timestamps/project identity are not supplied by exec JSONL. These
154
+ // placeholders are private adapter scaffolding, never output claims.
155
+ const time = "1970-01-01T00:00:00.000Z";
156
+ return { schema_version: 1, session_id: "codex-exec-export", harness: "codex", project_root: "not-supplied",
157
+ started_at: time, ended_at: time, events,
158
+ capture: { captured_at: time, omitted_reasoning_blocks: reasoning, omitted_unknown_records: gaps, truncated: false } };
159
+ }
160
+ catch {
161
+ return invalid();
162
+ }
163
+ }
@@ -0,0 +1,200 @@
1
+ import { createHash } from "node:crypto";
2
+ import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, realpathSync } from "node:fs";
3
+ import { resolve } from "node:path";
4
+ import { canonicalJson } from "./eval-schema.js";
5
+ import { MCP_TOOL_CATALOG } from "./mcp-tool-catalog.js";
6
+ import { trajectorySessionSchema } from "./trajectory-schema.js";
7
+ import { normalizeCodexExec } from "./eval-codex.js";
8
+ const MAX_BYTES = 16 * 1024 * 1024;
9
+ const MAX_EVENTS = 50_000;
10
+ const memoryEntries = Object.entries(MCP_TOOL_CATALOG).filter(([name]) => name.startsWith("memory_"));
11
+ const MUTATIONS = new Set(memoryEntries.filter(([, metadata]) => !metadata.annotations.readOnlyHint).map(([name]) => name));
12
+ const OBSERVATIONS = new Set(memoryEntries.filter(([, metadata]) => metadata.annotations.readOnlyHint).map(([name]) => name));
13
+ function digest(bytes) {
14
+ return createHash("sha256").update(bytes).digest("hex");
15
+ }
16
+ function object(value) {
17
+ return value !== null && typeof value === "object" && !Array.isArray(value)
18
+ ? value : undefined;
19
+ }
20
+ /** Only the maintained direct-tool names are recognized; suffix matching is unsafe. */
21
+ function memoryTool(name, harness) {
22
+ if (typeof name !== "string")
23
+ return undefined;
24
+ const prefixes = harness === "claude-code" || harness === "codex" ? ["mcp__cairn-memory__", "mcp__cairn_memory__"]
25
+ : harness === "opencode" ? ["cairn-memory_", "cairn_memory_"] : [""];
26
+ for (const prefix of prefixes) {
27
+ if (!name.startsWith(prefix))
28
+ continue;
29
+ const tool = name.slice(prefix.length);
30
+ if (MUTATIONS.has(tool) || OBSERVATIONS.has(tool))
31
+ return tool;
32
+ }
33
+ return undefined;
34
+ }
35
+ /** Inspect an explicitly supplied export without following links or executing content. */
36
+ function readProtocolBytes(path) {
37
+ let descriptor;
38
+ try {
39
+ const absolute = resolve(path);
40
+ // Validate the opened descriptor, not a pathname checked before open.
41
+ // Nonblocking/no-ctty flags prevent special-file admission from waiting
42
+ // or acquiring a controlling terminal before the regular-file check.
43
+ descriptor = openSync(absolute, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)
44
+ | (constants.O_NOCTTY ?? 0) | (process.platform === "win32" ? 0 : constants.O_NONBLOCK ?? 0));
45
+ const before = fstatSync(descriptor);
46
+ if (!before.isFile() || before.size > MAX_BYTES || realpathSync(absolute) !== absolute)
47
+ throw new Error();
48
+ const buffer = Buffer.alloc(before.size + 1);
49
+ let length = 0;
50
+ while (length < buffer.length) {
51
+ const count = readSync(descriptor, buffer, length, buffer.length - length, null);
52
+ if (!count)
53
+ break;
54
+ length += count;
55
+ }
56
+ const after = fstatSync(descriptor);
57
+ const finalPath = realpathSync(absolute);
58
+ const finalNamed = lstatSync(absolute);
59
+ if (length !== before.size || before.size !== after.size || before.mtimeMs !== after.mtimeMs
60
+ || before.ctimeMs !== after.ctimeMs || finalNamed.isSymbolicLink()
61
+ || finalNamed.dev !== before.dev || finalNamed.ino !== before.ino
62
+ || finalNamed.size !== before.size || finalNamed.mtimeMs !== before.mtimeMs
63
+ || finalNamed.ctimeMs !== before.ctimeMs
64
+ || finalPath !== absolute)
65
+ throw new Error();
66
+ const bytes = buffer.subarray(0, length);
67
+ return bytes;
68
+ }
69
+ catch {
70
+ // Do not expose filesystem errors, parser excerpts, schema paths or input data.
71
+ throw new Error("protocol_input_invalid_or_unsafe");
72
+ }
73
+ finally {
74
+ if (descriptor !== undefined)
75
+ closeSync(descriptor);
76
+ }
77
+ }
78
+ export function readProtocolTrajectory(path) {
79
+ try {
80
+ const bytes = readProtocolBytes(path);
81
+ const raw = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
82
+ if (!Array.isArray(raw?.events) || raw.events.length > MAX_EVENTS)
83
+ throw new Error();
84
+ const parsed = trajectorySessionSchema.safeParse(raw);
85
+ if (!parsed.success)
86
+ throw new Error();
87
+ return { session: parsed.data, digest: digest(bytes) };
88
+ }
89
+ catch {
90
+ throw new Error("protocol_input_invalid_or_unsafe");
91
+ }
92
+ }
93
+ export function readCodexProtocol(path) {
94
+ try {
95
+ const bytes = readProtocolBytes(path);
96
+ const session = normalizeCodexExec(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
97
+ return { session, digest: digest(bytes) };
98
+ }
99
+ catch {
100
+ throw new Error("protocol_input_invalid_or_unsafe");
101
+ }
102
+ }
103
+ function searchPayload(value, depth = 0) {
104
+ if (depth > 8)
105
+ return undefined;
106
+ if (typeof value === "string") {
107
+ try {
108
+ return searchPayload(JSON.parse(value), depth + 1);
109
+ }
110
+ catch {
111
+ return undefined;
112
+ }
113
+ }
114
+ // Claude tool-result content and MCP result envelopes are both supported.
115
+ if (Array.isArray(value)) {
116
+ if (value.length !== 1)
117
+ return undefined;
118
+ const part = object(value[0]);
119
+ return part?.type === "text" && typeof part.text === "string" ? searchPayload(part.text, depth + 1) : undefined;
120
+ }
121
+ const record = object(value);
122
+ if (!record || record.isError === true || record.ok === false)
123
+ return undefined;
124
+ if (record.structuredContent !== undefined)
125
+ return searchPayload(record.structuredContent, depth + 1);
126
+ if (record.content !== undefined)
127
+ return searchPayload(record.content, depth + 1);
128
+ return record;
129
+ }
130
+ function successfulSearch(result) {
131
+ if (!result || result.payload.is_error !== false || result.truncation)
132
+ return false;
133
+ const payload = searchPayload(result.payload.output);
134
+ return !!payload && typeof payload.mode === "string" && ["semantic", "substring"].includes(payload.mode)
135
+ && Number.isSafeInteger(payload.count) && payload.count >= 0
136
+ && Array.isArray(payload.results) && payload.results.length === payload.count;
137
+ }
138
+ /** This is an observation of local evidence, not authentication or task verification. */
139
+ export function auditMemoryProtocol(session, trajectoryDigest, captureAuthorized = false) {
140
+ const calls = [];
141
+ const byId = new Map();
142
+ let gap = session.capture.truncated || session.capture.omitted_unknown_records > 0
143
+ || (session.capture.omitted_size_events ?? 0) > 0;
144
+ for (const [index, event] of session.events.entries()) {
145
+ if (event.sequence !== index)
146
+ throw new Error("protocol_invalid_events");
147
+ if (event.truncation)
148
+ gap = true;
149
+ if (event.kind !== "tool_invocation" && event.kind !== "tool_result")
150
+ continue;
151
+ const id = event.payload.call_id;
152
+ if (typeof id !== "string" || !id || id.length > 1024)
153
+ throw new Error("protocol_invalid_events");
154
+ if (event.kind === "tool_invocation") {
155
+ if (byId.has(id) || typeof event.payload.tool_name !== "string")
156
+ throw new Error("protocol_invalid_events");
157
+ const call = { event, tool: memoryTool(event.payload.tool_name, session.harness) };
158
+ calls.push(call);
159
+ byId.set(id, call);
160
+ }
161
+ else {
162
+ const call = byId.get(id);
163
+ if (!call || call.result)
164
+ throw new Error("protocol_invalid_events");
165
+ call.result = event;
166
+ }
167
+ }
168
+ if (calls.some(call => !call.result))
169
+ gap = true;
170
+ const projectSearches = calls.filter(call => call.tool === "memory_search"
171
+ && object(call.event.payload.input)?.scope === "project" && !call.event.truncation);
172
+ const first = calls[0];
173
+ const writes = calls.filter(call => call.tool && MUTATIONS.has(call.tool));
174
+ const opaque = calls.filter(call => !call.tool);
175
+ const checks = [
176
+ { id: "evidence_complete", status: gap ? "inconclusive" : "pass", code: gap ? "capture_incomplete" : "capture_complete" },
177
+ { id: "memory_first", status: gap ? "inconclusive" : first && projectSearches[0] === first ? "pass" : "fail",
178
+ code: gap ? "ordering_unverifiable" : first && projectSearches[0] === first ? "project_search_first" : "project_search_not_first" },
179
+ { id: "retrieval_result", status: projectSearches.some(call => successfulSearch(call.result)) ? "pass" : projectSearches.length || gap ? "inconclusive" : "fail",
180
+ code: projectSearches.some(call => successfulSearch(call.result)) ? "project_search_completed"
181
+ : projectSearches.length || gap ? "search_result_unverifiable" : "project_search_absent" },
182
+ { id: "direct_write_boundary", status: writes.length && !captureAuthorized ? "fail" : gap || opaque.length ? "inconclusive" : "pass",
183
+ code: writes.length && !captureAuthorized ? "unapproved_write_attempt"
184
+ : gap || opaque.length ? "indirect_writes_unverifiable" : writes.length ? "caller_asserted_capture_authorization" : "no_direct_write_attempt" },
185
+ ];
186
+ const policy = { schema_version: 1, protocol: "project-memory-first-v1", capture_authorized: captureAuthorized,
187
+ scope: "direct-memory-tools", direct_mutation_tools: [...MUTATIONS].sort(), direct_observation_tools: [...OBSERVATIONS].sort() };
188
+ const status = checks.some(check => check.status === "fail") ? "fail"
189
+ : checks.some(check => check.status === "inconclusive") ? "inconclusive" : "pass";
190
+ return {
191
+ schema_version: 1, enabled: true, operation: "protocol", evidence_scope: "local-trajectory-observation",
192
+ harness: session.harness, trajectory_digest: trajectoryDigest, policy_digest: digest(canonicalJson(policy)),
193
+ policy, status, checks,
194
+ counts: { tool_invocations: calls.length, project_searches: projectSearches.length,
195
+ successful_project_searches: projectSearches.filter(call => successfulSearch(call.result)).length,
196
+ direct_write_attempts: writes.length, opaque_tool_invocations: opaque.length },
197
+ limitations: ["input_not_authenticated", "task_quality_not_measured", "authorization_is_caller_assertion",
198
+ "indirect_tool_effects_not_certified", "task_boundaries_not_inferred", "tool_availability_not_inferred"],
199
+ };
200
+ }
@@ -45,6 +45,8 @@
45
45
  "check:eval-schema": "node scripts/smoke-eval-schema.mjs",
46
46
  "check:eval-process": "node scripts/smoke-eval-process.mjs",
47
47
  "check:eval-statistics": "node scripts/smoke-eval-statistics.mjs",
48
+ "check:eval-protocol": "node scripts/smoke-eval-protocol.mjs",
49
+ "check:codex-protocol": "node scripts/smoke-codex-protocol.mjs",
48
50
  "check:skill-lifecycle": "node scripts/smoke-skill-lifecycle.mjs",
49
51
  "check:graph-cli": "node scripts/smoke-graph-cli.mjs",
50
52
  "check:mcp-trust": "node scripts/smoke-mcp-trust.mjs",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cairnkeep/cli",
3
- "version": "2.21.1",
3
+ "version": "2.22.0",
4
4
  "description": "A durable, harness-agnostic memory + context layer for coding agents and MCP clients.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -55,7 +55,7 @@ Usage:
55
55
  cairn mcp-tools <list|status|set|reset>
56
56
  cairn pack <init|lock|validate|install|list|show|remove|enable|disable|update|skills|approve-skill|revoke-skill>
57
57
  cairn notes <distill|search-error|promote|doctor>
58
- cairn eval <validate|run|ablate|report|prune|delete>
58
+ cairn eval <validate|run|ablate|report|prune|delete|protocol>
59
59
  cairn skill <harvest|list|show|review|propose|evaluate|apply|rollback>
60
60
  cairn graph <build|query|status|diff|explain|path>
61
61
  cairn evidence <list|show|delete|prune|doctor>
@@ -70,7 +70,8 @@ _cairn_complete() {
70
70
  report) COMPREPLY=( $(compgen -W "--experiment --json" -- "$current") ) ;;
71
71
  prune) COMPREPLY=( $(compgen -W "--older-than-days --dry-run --json" -- "$current") ) ;;
72
72
  delete) COMPREPLY=( $(compgen -W "--experiment --dry-run --json" -- "$current") ) ;;
73
- *) COMPREPLY=( $(compgen -W "validate run ablate report prune delete" -- "$current") ) ;;
73
+ protocol) COMPREPLY=( $(compgen -W "--trajectory --codex-jsonl --capture-authorized --json" -- "$current") ) ;;
74
+ *) COMPREPLY=( $(compgen -W "validate run ablate report prune delete protocol" -- "$current") ) ;;
74
75
  esac
75
76
  ;;
76
77
  skill) COMPREPLY=( $(compgen -W "harvest list show review propose evaluate apply rollback --project --minimum-occurrences --kind --id --candidate --approve --target --adapter --edit-budget --proposal --exploration-task-set --confirmation-task-set --output --repetitions --seed --minimum-improvement --evaluation --application --confirm --yes --json" -- "$current") ) ;;
@@ -164,7 +165,8 @@ _cairn() {
164
165
  report) _arguments '--experiment[experiment ID]:experiment ID:' '--json[emit JSON]' ;;
165
166
  prune) _arguments '--older-than-days[retention age]:days:' '--dry-run[report without pruning]' '--json[emit JSON]' ;;
166
167
  delete) _arguments '--experiment[experiment ID]:experiment ID:' '--dry-run[report without deleting]' '--json[emit JSON]' ;;
167
- *) _values 'eval command' validate run ablate report prune delete ;;
168
+ protocol) _arguments '--trajectory[normalized trajectory export]:file:_files' '--codex-jsonl[single-turn Codex exec JSONL export]:file:_files' '--capture-authorized[assert operator authorization for capture]' '--json[emit JSON]' ;;
169
+ *) _values 'eval command' validate run ablate report prune delete protocol ;;
168
170
  esac
169
171
  ;;
170
172
  skill)
@@ -272,7 +274,10 @@ complete -c cairn -n "__fish_seen_subcommand_from proposals" -l category -r
272
274
  complete -c cairn -n "__fish_seen_subcommand_from proposals" -l project -r
273
275
  complete -c cairn -n "__fish_seen_subcommand_from proposals" -l json
274
276
  complete -c cairn -n "__fish_seen_subcommand_from notes" -a "distill search-error promote doctor"
275
- complete -c cairn -n "__fish_seen_subcommand_from eval; and not __fish_seen_subcommand_from validate run ablate report prune delete" -a "validate run ablate report prune delete"
277
+ complete -c cairn -n "__fish_seen_subcommand_from eval; and not __fish_seen_subcommand_from validate run ablate report prune delete protocol" -a "validate run ablate report prune delete protocol"
278
+ complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from protocol" -l trajectory -r
279
+ complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from protocol" -l codex-jsonl -r
280
+ complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from protocol" -l capture-authorized
276
281
  complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from validate run ablate" -l task-set -r
277
282
  complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from validate run ablate" -l adapter -r
278
283
  complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from validate run ablate" -l output -r
@@ -283,7 +288,7 @@ complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcomma
283
288
  complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from report delete" -l experiment -r
284
289
  complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from prune" -l older-than-days -r
285
290
  complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from prune delete" -l dry-run
286
- complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from validate run ablate report prune delete" -l json
291
+ complete -c cairn -n "__fish_seen_subcommand_from eval; and __fish_seen_subcommand_from validate run ablate report prune delete protocol" -l json
287
292
  complete -c cairn -n "__fish_seen_subcommand_from skill; and not __fish_seen_subcommand_from harvest list show review propose evaluate apply rollback" -a "harvest list show review propose evaluate apply rollback"
288
293
  complete -c cairn -n "__fish_seen_subcommand_from skill; and __fish_seen_subcommand_from harvest list show review propose evaluate apply rollback" -l project -r
289
294
  complete -c cairn -n "__fish_seen_subcommand_from skill; and __fish_seen_subcommand_from harvest list show review propose evaluate apply rollback" -l json
@@ -0,0 +1,46 @@
1
+ import { createHash } from "node:crypto";
2
+ import assert from "node:assert/strict";
3
+ import { resolve } from "node:path";
4
+ import { pathToFileURL } from "node:url";
5
+
6
+ // Independent artifact grader for the bounded release-ledger task. Importing
7
+ // an explicitly selected module executes it: run only in an isolated lab.
8
+ export async function verifyReleaseLedger(path) {
9
+ const checks = [];
10
+ const check = async (id, fn) => {
11
+ try { await fn(); checks.push({ id, pass: true }); }
12
+ catch { checks.push({ id, pass: false }); }
13
+ };
14
+ let mod, ledger, first, final;
15
+ const input = { package: "sample", version: "1.0.0", notes: ["First", "Second"] };
16
+ const errorCode = (fn, code) => assert.throws(fn, error => error?.code === code);
17
+ await check("export", async () => {
18
+ mod = await import(pathToFileURL(resolve(path)).href);
19
+ assert.equal(typeof mod.createLedger, "function");
20
+ ledger = mod.createLedger();
21
+ });
22
+ await check("methods", () => { for (const method of ["append", "get", "list", "finalize"]) assert.equal(typeof ledger[method], "function"); });
23
+ await check("append", () => { first = ledger.append(input); assert.equal(first.state, "draft"); assert.equal(first.sequence, 1); assert.deepEqual(first.notes, input.notes); });
24
+ await check("shape", () => assert.deepEqual(Object.keys(first).sort(), ["package", "version", "notes", "sequence", "state"].sort()));
25
+ await check("invalid-package", () => errorCode(() => ledger.append({ ...input, package: "Bad_Name" }), "INVALID_PACKAGE"));
26
+ await check("invalid-version", () => errorCode(() => ledger.append({ ...input, version: "v1.0" }), "INVALID_VERSION"));
27
+ await check("invalid-notes", () => errorCode(() => ledger.append({ ...input, notes: [] }), "INVALID_NOTES"));
28
+ await check("duplicate", () => errorCode(() => ledger.append(input), "DUPLICATE_RELEASE"));
29
+ await check("sequence", () => assert.equal(ledger.append({ ...input, package: "other" }).sequence, 2));
30
+ await check("not-found", () => errorCode(() => ledger.get("missing", "1.0.0"), "NOT_FOUND"));
31
+ await check("finalize", () => { final = ledger.finalize("sample", "1.0.0"); assert.equal(final.state, "final"); });
32
+ await check("checksum", () => assert.equal(final.checksum, createHash("sha256").update(JSON.stringify({ ...input, sequence: 1 })).digest("hex")));
33
+ await check("idempotent", () => assert.deepEqual(ledger.finalize("sample", "1.0.0"), final));
34
+ await check("detached-get", () => { const copy = ledger.get("sample", "1.0.0"); copy.notes.push("tamper"); copy.state = "draft"; const fresh = ledger.get("sample", "1.0.0"); assert.deepEqual(fresh.notes, ["First", "Second"]); assert.equal(fresh.state, "final"); });
35
+ await check("ordered-detached-list", () => { const copies = ledger.list(); assert.deepEqual(copies.map(entry => entry.sequence), [1, 2]); copies[0].notes[0] = "tamper"; assert.equal(ledger.list()[0].notes[0], "First"); });
36
+ return { checks, passed: checks.filter(check => check.pass).length, total: checks.length };
37
+ }
38
+
39
+ if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
40
+ if (process.argv.length !== 3) { process.stderr.write("Usage: verify-release-ledger.mjs MODULE_PATH\n"); process.exitCode = 2; }
41
+ else {
42
+ const value = await verifyReleaseLedger(process.argv[2]);
43
+ process.stdout.write(`${JSON.stringify(value)}\n`);
44
+ process.exitCode = value.passed === value.total ? 0 : 1;
45
+ }
46
+ }
@@ -12,8 +12,8 @@ for shell in bash zsh fish; do
12
12
  done
13
13
 
14
14
  capability_ids='memory.write memory.search notes.distill wiki graph security.audit route.check context.explore'
15
- eval_commands='validate run ablate report prune delete'
16
- eval_flags='--task-set --adapter --output --repetitions --seed --json --yes --disable --experiment --older-than-days --dry-run'
15
+ eval_commands='validate run ablate report prune delete protocol'
16
+ eval_flags='--task-set --adapter --output --repetitions --seed --json --yes --disable --experiment --older-than-days --dry-run --trajectory --codex-jsonl --capture-authorized'
17
17
  skill_commands='harvest list show review propose evaluate apply rollback'
18
18
  for shell in bash zsh fish; do
19
19
  grep -q 'capabilities' "$tmp/$shell"
@@ -70,7 +70,7 @@ for shell in bash zsh fish; do
70
70
  done
71
71
 
72
72
  "$ROOT/bin/cairn" help >"$tmp/root-help"
73
- grep -qF 'cairn eval <validate|run|ablate|report|prune|delete>' "$tmp/root-help"
73
+ grep -qF 'cairn eval <validate|run|ablate|report|prune|delete|protocol>' "$tmp/root-help"
74
74
  node "$ROOT/mcp-memory-server/dist/eval-cli.js" --help >"$tmp/eval-help"
75
75
  for command in $eval_commands; do
76
76
  grep -q "cairn eval $command" "$tmp/eval-help"
@@ -152,6 +152,10 @@ for shell in bash zsh fish; do
152
152
  fi
153
153
  done
154
154
  "$ROOT/bin/cairn" completion powershell >"$tmp/powershell"
155
+ grep -qF "'protocol'" "$tmp/powershell"
156
+ grep -qF "'--trajectory'" "$tmp/powershell"
157
+ grep -qF "'--codex-jsonl'" "$tmp/powershell"
158
+ grep -qF "'--capture-authorized'" "$tmp/powershell"
155
159
  if ! grep -qE "['\"]setup['\"]" "$tmp/powershell"; then
156
160
  completion_surface_complete=false
157
161
  fi
@@ -6,6 +6,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
6
6
  for script in \
7
7
  run-memfork-evaluation.mjs \
8
8
  run-clm-pi-evaluation.mjs \
9
+ verify-release-ledger.mjs \
9
10
  run-ollaya-decision-evaluation.mjs
10
11
  do
11
12
  node --check "$ROOT/scripts/spikes/$script"
@@ -76,6 +76,21 @@ cmp -s "$tmp/source-eval-disabled.json" "$tmp/installed-eval-disabled.json" || \
76
76
  fail "source and installed disabled eval identity differ"
77
77
  [[ ! -e "$tmp/installed-unwritten-output" ]] || fail "installed disabled eval created output"
78
78
 
79
+ CAIRN_EVAL=1 cairn eval protocol \
80
+ --trajectory "$installed_root/examples/eval/protocol-project-search.json" --json >"$tmp/protocol-pass.json"
81
+ node -e 'const v=require(process.argv[1]);if(v.status!=="pass"||v.evidence_scope!=="local-trajectory-observation")process.exit(1)' "$tmp/protocol-pass.json" \
82
+ || fail "installed memory protocol positive control failed"
83
+ set +e
84
+ CAIRN_EVAL=1 cairn eval protocol \
85
+ --trajectory "$installed_root/examples/eval/protocol-narration.json" --json >"$tmp/protocol-fail.json"
86
+ protocol_status=$?
87
+ set -e
88
+ [[ "$protocol_status" -eq 1 ]] || fail "installed memory protocol counted narration as evidence"
89
+ CAIRN_EVAL=1 cairn eval protocol \
90
+ --codex-jsonl "$installed_root/examples/eval/protocol-codex.jsonl" --json >"$tmp/protocol-codex.json"
91
+ node -e 'const v=require(process.argv[1]);if(v.status!=="pass"||v.harness!=="codex")process.exit(1)' "$tmp/protocol-codex.json" \
92
+ || fail "installed native Codex protocol control failed"
93
+
79
94
  for required in \
80
95
  schemas/capability-contract.schema.json \
81
96
  schemas/capability-callback.schema.json \
@@ -159,11 +174,16 @@ for required in \
159
174
  mcp-memory-server/dist/eval-report.js \
160
175
  mcp-memory-server/dist/eval-runner.js \
161
176
  mcp-memory-server/dist/eval-cli.js \
177
+ mcp-memory-server/dist/eval-protocol.js \
178
+ mcp-memory-server/dist/eval-codex.js \
162
179
  scripts/fake-eval-adapter.mjs \
163
180
  examples/eval/task-set.json \
164
181
  examples/eval/bundled-fake.json \
165
182
  examples/eval/adapter.json \
166
183
  examples/eval/ecosystem-decisions.json \
184
+ examples/eval/protocol-project-search.json \
185
+ examples/eval/protocol-narration.json \
186
+ examples/eval/protocol-codex.jsonl \
167
187
  examples/eval/README.md
168
188
  do
169
189
  [[ -f "$installed_root/$required" ]] || fail "npm tarball omitted $required"
@@ -183,13 +203,16 @@ for asset in \
183
203
  examples/eval/bundled-fake.json \
184
204
  examples/eval/adapter.json \
185
205
  examples/eval/ecosystem-decisions.json \
206
+ examples/eval/protocol-project-search.json \
207
+ examples/eval/protocol-narration.json \
208
+ examples/eval/protocol-codex.jsonl \
186
209
  examples/eval/README.md
187
210
  do
188
211
  [[ ! -x "$installed_root/$asset" ]] || \
189
212
  fail "installed public eval data asset is unexpectedly executable: $asset"
190
213
  done
191
214
  eval_assets=$(cd "$installed_root/examples/eval" && find . -type f -print | LC_ALL=C sort)
192
- [[ "$eval_assets" == $'./README.md\n./adapter.json\n./bundled-fake.json\n./ecosystem-decisions.json\n./task-set.json' ]] || \
215
+ [[ "$eval_assets" == $'./README.md\n./adapter.json\n./bundled-fake.json\n./ecosystem-decisions.json\n./protocol-codex.jsonl\n./protocol-narration.json\n./protocol-project-search.json\n./task-set.json' ]] || \
193
216
  fail "installed eval examples contain an unexpected path"
194
217
 
195
218
  node - "$installed_root/examples/eval/adapter.json" <<'NODE'