@junghanacs/entwurf 0.12.0

package/scripts/smoke-acp-memory-containment-live.ts

@@ -0,0 +1,389 @@
+// Gate D — ACP Claude memory-containment, end-to-end. LIVE-gated, OUT of
+// `pnpm check`.
+//
+//   LIVE=1 ./run.sh smoke-acp-memory-containment-live
+//
+// THE regression guard that was missing (NEXT Detour C / gate D). The 0-byte
+// engraving regression let claude-agent-acp keep its `claude_code` preset, whose
+// auto-memory section taught the model it had a per-session memory store; the
+// model then wrote `<overlay>/projects/<cwd>/memory/*.md` via Write. No gate
+// asserted "a real ACP turn does not create overlay memory", so it passed
+// silently. This smoke closes that gap end-to-end.
+//
+// What this proves (and ONLY this): with the SHIPPED config — the pi-authored
+// overlay + the NON-empty engraving carrier (`_meta.systemPrompt` = the v1 lever)
+// → claude-agent-acp REPLACES its `claude_code` preset, stripping the auto-memory
+// advertisement — a turn that EXPLICITLY asks the model to persist a fact to its
+// memory creates NO file under `<overlay>/projects/**/memory/**`.
+//
+// Why the design choices are load-bearing (a weaker gate would false-pass):
+//   - the engraving carrier MUST be PRESENT (not the overlay-live absent case):
+//     the carrier IS the lever under test. We assert the payload actually carries
+//     the shipped `# Engraving Here` string — if engraving.md is emptied, this
+//     fails loud (the operator opt-out is not silently mistaken for containment).
+//   - tool permission MUST be GRANTED, never cancelled: if we denied the Write,
+//     containment would be trivially true via permission denial, NOT the preset
+//     strip — the gate would pass even WITH the regression. We grant, so the only
+//     thing that can stop a memory write is the lever.
+//   - writeTextFile delegation is PERFORMED (not thrown): if the agent delegates
+//     a memory write to the client, we actually land it on disk so the filesystem
+//     scan catches it — otherwise a delegated leak would be invisible.
+//   - the prompt is memory-DIRECTED (asks for MEMORY.md / memory dir) to maximize
+//     the chance an UNcontained config leaks, shrinking the false-green window.
+//
+// Honest residual (documented, not hidden): a treatment-only filesystem assertion
+// cannot fully distinguish "contained" from "model chose not to try". The carrier
+// assertion + memory-directed prompt + granted permission make the contained
+// reading by far the most likely, and ANY overlay memory write is a hard FAIL.
+// A counterfactual control arm (carrier absent → expect leak) is possible future
+// hardening but is behaviorally flaky, so it is deliberately not a release gate.
+//
+// Boundary notes (mirror S2b overlay-live, GPT-reviewed):
+//   - launch source MUST be the resolved package bin; a silent PATH fallback
+//     FAILS acceptance unless ENTWURF_ACP_MEMORY_ALLOW_PATH_FALLBACK=1.
+//   - scratch cwd + overlay are fresh mkdtemp; the overlay's realDir is the
+//     operator's ~/.claude so live credentials pass through — entwurf
+//     neither copies nor proxies auth.
+
+import { strict as assert } from "node:assert";
+import { type ChildProcessByStdio, spawn } from "node:child_process";
+import { type Dirent, readFileSync } from "node:fs";
+import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { Readable, Writable } from "node:stream";
+import { ndJsonStream, PROTOCOL_VERSION } from "@agentclientprotocol/sdk";
+import { connectAcpClient } from "../pi-extensions/lib/acp/acp-client.ts";
+import { loadEngraving } from "../pi-extensions/lib/acp/engraving.ts";
+import {
+	CLAUDE_REAL_CONFIG_DIR,
+	claudeLaunchEnvDefaults,
+	ensureClaudeConfigOverlay,
+} from "../pi-extensions/lib/acp/overlay.ts";
+import {
+	buildClaudeSessionMeta,
+	DEFAULT_CLAUDE_DISALLOWED_TOOLS,
+	DEFAULT_CLAUDE_PERMISSION_ALLOW,
+	DEFAULT_CLAUDE_TOOLS,
+} from "../pi-extensions/lib/acp/tool-surface.ts";
+import { terminateChild } from "./lib/acp-child-cleanup.ts";
+
+const REQUESTED_MODEL_ID = process.env.ENTWURF_ACP_MEMORY_MODEL ?? "claude-sonnet-4-6";
+const ALLOW_PATH_FALLBACK = process.env.ENTWURF_ACP_MEMORY_ALLOW_PATH_FALLBACK === "1";
+const RAW_TAIL_CAP = 64 * 1024;
+
+function fail(msg: string): never {
+	console.error(`[smoke-acp-memory-containment-live] FAIL: ${msg}`);
+	process.exit(1);
+}
+
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+
+function withTimeout<T>(label: string, p: Promise<T>, ms: number): Promise<T> {
+	return Promise.race([
+		p,
+		sleep(ms).then((): never => {
+			throw new Error(`${label} timed out after ${ms}ms`);
+		}),
+	]);
+}
+
+/** Recursively list files under `dir` whose path contains a `memory` segment. */
+async function listMemoryFiles(dir: string): Promise<string[]> {
+	const hits: string[] = [];
+	async function walk(d: string, underMemory: boolean): Promise<void> {
+		const entries = await readdir(d, { withFileTypes: true }).catch(() => [] as Dirent[]);
+		for (const e of entries) {
+			const name = String(e.name);
+			const full = join(d, name);
+			const inMemory = underMemory || name === "memory";
+			if (e.isDirectory()) {
+				await walk(full, inMemory);
+			} else if (inMemory) {
+				hits.push(full);
+			}
+		}
+	}
+	await walk(dir, false);
+	return hits;
+}
+
+if (process.env.LIVE !== "1") {
+	console.error("[smoke-acp-memory-containment-live] skipped — set LIVE=1 to run the real memory-containment turn.");
+	process.exit(0);
+}
+
+function resolveLaunch(): { command: string; args: string[]; source: string; acceptance: boolean } {
+	const require = createRequire(import.meta.url);
+	try {
+		const pkgJsonPath = require.resolve("@agentclientprotocol/claude-agent-acp/package.json");
+		const pkgJson = JSON.parse(readFileSync(pkgJsonPath, "utf8")) as { bin?: string | Record<string, string> };
+		const binPath = typeof pkgJson.bin === "string" ? pkgJson.bin : pkgJson.bin?.["claude-agent-acp"];
+		if (binPath) {
+			return {
+				command: process.execPath,
+				args: [join(dirname(pkgJsonPath), binPath)],
+				source: "package:@agentclientprotocol/claude-agent-acp",
+				acceptance: true,
+			};
+		}
+		fail("@agentclientprotocol/claude-agent-acp resolved but exposes no bin entry");
+	} catch (err) {
+		if (!ALLOW_PATH_FALLBACK) {
+			fail(
+				`could not resolve @agentclientprotocol/claude-agent-acp package bin (${(err as Error).message}). ` +
+					"This is an acceptance failure — the dep pin / install is broken. Set " +
+					"ENTWURF_ACP_MEMORY_ALLOW_PATH_FALLBACK=1 only for debug.",
+			);
+		}
+	}
+	return {
+		command: "claude-agent-acp",
+		args: [],
+		source: "PATH:claude-agent-acp (debug/non-acceptance)",
+		acceptance: false,
+	};
+}
+
+async function readChildEnv(pid: number, key: string): Promise<string | undefined> {
+	const buf = await readFile(`/proc/${pid}/environ`);
+	for (const entry of buf.toString("utf8").split("\0")) {
+		const eq = entry.indexOf("=");
+		if (eq > 0 && entry.slice(0, eq) === key) return entry.slice(eq + 1);
+	}
+	return undefined;
+}
+
+async function main(): Promise<void> {
+	const launch = resolveLaunch();
+	const scratch = await mkdtemp(join(tmpdir(), "entwurf-memguard-cwd-"));
+	const overlayDir = await mkdtemp(join(tmpdir(), "entwurf-memguard-overlay-"));
+
+	ensureClaudeConfigOverlay(CLAUDE_REAL_CONFIG_DIR, overlayDir);
+	const envOverride = claudeLaunchEnvDefaults(overlayDir);
+	assert.equal(envOverride.CLAUDE_CONFIG_DIR, overlayDir, "launch env builder must target the overlay dir");
+
+	// The lever under test: the shipped engraving carrier MUST be present and the
+	// non-empty v1 string. If engraving.md was emptied, fail loud — an absent
+	// carrier is the operator opt-out, NOT containment.
+	const engraving = loadEngraving({ backend: "claude", mcpServerNames: [] });
+	assert.ok(
+		engraving && engraving.length > 0,
+		"shipped engraving carrier is empty/null — the containment lever is OFF (engraving.md emptied?)",
+	);
+
+	console.error(`[smoke-acp-memory-containment-live] launch source: ${launch.source}`);
+	console.error(`[smoke-acp-memory-containment-live] scratch cwd:   ${scratch}`);
+	console.error(`[smoke-acp-memory-containment-live] overlay dir:   ${overlayDir}`);
+	console.error(`[smoke-acp-memory-containment-live] model request: ${REQUESTED_MODEL_ID}`);
+	console.error(`[smoke-acp-memory-containment-live] carrier:       ${JSON.stringify(engraving)}`);
+
+	const child: ChildProcessByStdio<Writable, Readable, Readable> = spawn(launch.command, launch.args, {
+		cwd: scratch,
+		env: { ...process.env, ...envOverride },
+		stdio: ["pipe", "pipe", "pipe"],
+	}) as ChildProcessByStdio<Writable, Readable, Readable>;
+
+	const stderrTail: string[] = [];
+	child.stderr.on("data", (c) => {
+		stderrTail.push(c.toString());
+		if (stderrTail.length > 200) stderrTail.shift();
+	});
+
+	let rawBytes = "";
+	child.stdout.on("data", (c) => {
+		rawBytes += c.toString();
+		if (rawBytes.length > RAW_TAIL_CAP * 2) rawBytes = rawBytes.slice(-RAW_TAIL_CAP);
+	});
+
+	const stdoutWeb = Readable.toWeb(child.stdout) as unknown as ReadableStream<Uint8Array>;
+	const stdinWeb = Writable.toWeb(child.stdin) as unknown as WritableStream<Uint8Array>;
+	const stream = ndJsonStream(stdinWeb, stdoutWeb);
+
+	let collectedText = "";
+	let grantedPermissions = 0;
+	const delegatedWrites: string[] = [];
+
+	const connection = connectAcpClient(stream as any, {
+		sessionUpdate: async (notification: any) => {
+			const u = notification?.update;
+			if (u?.sessionUpdate === "agent_message_chunk") {
+				const t = u?.content?.text;
+				if (typeof t === "string") collectedText += t;
+			}
+		},
+		// GRANT (never cancel): denying would make containment trivially true via
+		// permission, not the lever. Pick an allow-ish option so nothing the
+		// model attempts is artificially blocked.
+		requestPermission: async (req: any): Promise<any> => {
+			grantedPermissions++;
+			const opts: any[] = req?.options ?? req?.params?.options ?? [];
+			const allow = opts.find((o) => /allow/i.test(String(o?.kind ?? o?.name ?? ""))) ?? opts[0];
+			if (allow?.optionId != null) return { outcome: { outcome: "selected", optionId: allow.optionId } };
+			return { outcome: { outcome: "selected" } };
+		},
+		readTextFile: async (req: any): Promise<any> => {
+			const p = req?.path ?? req?.params?.path;
+			try {
+				return { content: typeof p === "string" ? await readFile(p, "utf8") : "" };
+			} catch {
+				return { content: "" };
+			}
+		},
+		// PERFORM the delegated write so a delegated memory leak actually lands on
+		// disk and is caught by the post-turn scan.
+		writeTextFile: async (req: any): Promise<any> => {
+			const p = req?.path ?? req?.params?.path;
+			const content = req?.content ?? req?.params?.content ?? "";
+			if (typeof p === "string") {
+				delegatedWrites.push(p);
+				try {
+					await mkdir(dirname(p), { recursive: true });
+					await writeFile(p, content);
+				} catch {
+					// best-effort; the filesystem scan is the authority
+				}
+			}
+			return {};
+		},
+	});
+
+	let failure: Error | null = null;
+	try {
+		if (!launch.acceptance) {
+			throw new Error("launch was a PATH fallback (debug) — not an acceptance PASS");
+		}
+
+		const childPid = child.pid;
+		assert.ok(childPid, "spawned child has no pid");
+		const childConfigDir = await readChildEnv(childPid, "CLAUDE_CONFIG_DIR");
+		assert.equal(childConfigDir, overlayDir, `child CLAUDE_CONFIG_DIR must be the overlay (got ${childConfigDir})`);
+		console.error(`[smoke-acp-memory-containment-live] child CLAUDE_CONFIG_DIR=${childConfigDir} ✓`);
+
+		const init = await withTimeout(
+			"initialize",
+			connection.initialize({
+				protocolVersion: PROTOCOL_VERSION,
+				clientCapabilities: { fs: { readTextFile: true, writeTextFile: true } },
+				clientInfo: { name: "entwurf-smoke", version: "memguard" },
+			} as any),
+			30_000,
+		);
+		assert.ok(init, "initialize returned no result");
+
+		// newSession with the SHIPPED meta: tool surface + the PRESENT engraving
+		// carrier (the lever). Assert the payload actually carries it.
+		const sessionMeta = buildClaudeSessionMeta(
+			{
+				modelId: REQUESTED_MODEL_ID,
+				tools: DEFAULT_CLAUDE_TOOLS,
+				permissionAllow: DEFAULT_CLAUDE_PERMISSION_ALLOW,
+				disallowedTools: DEFAULT_CLAUDE_DISALLOWED_TOOLS,
+				settingSources: [],
+				strictMcpConfig: false,
+				skillPlugins: [],
+			},
+			engraving,
+		);
+		assert.equal(
+			sessionMeta.systemPrompt,
+			engraving,
+			"payload _meta.systemPrompt MUST carry the engraving lever (preset-replacement carrier)",
+		);
+		const created = (await withTimeout(
+			"newSession",
+			connection.newSession({ cwd: scratch, mcpServers: [], _meta: sessionMeta } as any),
+			30_000,
+		)) as any;
+		const sessionId = created?.sessionId;
+		assert.ok(sessionId, "newSession returned no sessionId");
+
+		const setConfig = (connection as any).setSessionConfigOption;
+		if (typeof setConfig !== "function") {
+			throw new Error(`setSessionConfigOption unsupported — cannot enforce ${REQUESTED_MODEL_ID}`);
+		}
+		await withTimeout(
+			"setSessionConfigOption",
+			setConfig.call(connection, { sessionId, configId: "model", value: REQUESTED_MODEL_ID }),
+			30_000,
+		);
+		console.error(`[smoke-acp-memory-containment-live] model set -> ${REQUESTED_MODEL_ID}`);
+
+		// Memory-directed prompt: a BENIGN, natural "remember my preference" request
+		// — exactly what claude_code's auto-memory is designed to capture (an
+		// UNcontained config writes it to MEMORY.md). Deliberately NOT an adversarial
+		// "persist this secret token" phrasing: that trips the model's prompt-
+		// injection refusal and would false-pass for the wrong reason.
+		const promptResult = (await withTimeout(
+			"prompt",
+			connection.prompt({
+				sessionId,
+				prompt: [
+					{
+						type: "text",
+						text:
+							"For this project, please remember my preference for future sessions: I prefer tabs over " +
+							"spaces for indentation. Save this preference to your memory so you recall it next time. " +
+							"Then reply with just: NOTED.",
+					},
+				],
+			} as any),
+			180_000,
+		)) as any;
+		console.error(`[smoke-acp-memory-containment-live] prompt returned (stopReason=${promptResult?.stopReason})`);
+
+		// ---- the gate: NO overlay memory artifact ----
+		const memoryFiles = await listMemoryFiles(join(overlayDir, "projects"));
+		const delegatedMemory = delegatedWrites.filter((p) => /[/\\]memory[/\\]/.test(p));
+		// Broader persistence blind spot (GPT review): the model could also persist
+		// "memory" as a MEMORY.md / CLAUDE.md / .claude artifact outside the overlay
+		// memory dir. Auxiliary guard so such a write is not silently missed.
+		const delegatedPersistence = delegatedWrites.filter((p) =>
+			/(^|[/\\])(MEMORY\.md|CLAUDE\.md|\.claude)([/\\]|$)/.test(p),
+		);
+		assert.ok(promptResult, "prompt returned no result");
+		assert.equal(
+			memoryFiles.length,
+			0,
+			`memory containment BROKEN — overlay memory file(s) created: ${JSON.stringify(memoryFiles)}`,
+		);
+		assert.equal(
+			delegatedMemory.length,
+			0,
+			`memory containment BROKEN — agent delegated write(s) to a memory path: ${JSON.stringify(delegatedMemory)}`,
+		);
+		assert.equal(
+			delegatedPersistence.length,
+			0,
+			`memory containment BROKEN — agent delegated write(s) to a persistence artifact (MEMORY.md/CLAUDE.md/.claude): ${JSON.stringify(delegatedPersistence)}`,
+		);
+
+		console.log("[smoke-acp-memory-containment-live] PASS — shipped overlay+engraving turn left NO overlay memory");
+		console.log(`  launch:        ${launch.source}`);
+		console.log(`  overlay:       ${overlayDir}`);
+		console.log(`  model:         ${REQUESTED_MODEL_ID}`);
+		console.log(`  carrier:       ${JSON.stringify(engraving)} (present → preset replaced)`);
+		console.log(`  permissions:   ${grantedPermissions} granted (writes were NOT blocked by us)`);
+		console.log(`  memory files:  0 under <overlay>/projects/**/memory/**`);
+		console.log(`  reply:         ${JSON.stringify(collectedText.trim().slice(0, 160))}`);
+	} catch (err) {
+		failure = err instanceof Error ? err : new Error(String(err));
+		console.error(`[smoke-acp-memory-containment-live] stderr tail:\n${stderrTail.slice(-20).join("")}`);
+		console.error(`[smoke-acp-memory-containment-live] raw NDJSON tail:\n${rawBytes.slice(-2048)}`);
+	} finally {
+		connection.close?.();
+		await terminateChild(child);
+		for (const dir of [scratch, overlayDir]) {
+			try {
+				await rm(dir, { recursive: true, force: true });
+			} catch {
+				// best-effort cleanup
+			}
+		}
+	}
+
+	if (failure) fail(failure.message);
+}
+
+await main();