@kolisachint/hoocode-agent 0.5.2 → 0.5.4

package/dist/core/extensions/plugins/smoke.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"smoke.d.ts","sourceRoot":"","sources":["../../../../src/core/extensions/plugins/smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAOH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAoBtD,MAAM,WAAW,YAAY;IAC5B,0EAA0E;IAC1E,IAAI,CAAC,EAAE,OAAO,CAAC;CACf;AAoID,qGAAqG;AACrG,wBAAsB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAsE5G","sourcesContent":["/**\n * G3 — behavioral smoke for a plugin's executable capabilities.\n *\n * G1 and G2 read files. G3 is the first gate that finds out whether the thing\n * actually runs, which is the most common real failure: a plugin that installs\n * cleanly and breaks the session at the next tool call.\n *\n * **What this is and is not.** Hooks run with `cwd`, `HOME`, `TMPDIR` and the\n * plugin data dir redirected into a throwaway directory, and with a hard\n * timeout. That reduces blast radius; it is *not* containment. Without OS-level\n * sandboxing a shell command can still write wherever it likes, and claiming\n * otherwise in a confirmation prompt would be worse than saying nothing — the\n * whole point of showing gate results to a human is that they are true.\n *\n * **Why it runs before the human confirms.** It does execute not-yet-approved\n * code, which is a real cost. Against it: the code was authored in this session\n * from the user's own request rather than fetched from anywhere, G2 has already\n * screened it for destructive shapes, and absent the smoke test the very same\n * command runs moments later anyway — unscreened, unredirected, and in the real\n * working directory. Running it once under redirection to find out whether it\n * even works is the smaller risk. G3 is therefore authored-only: it is never\n * applied to a marketplace plugin, where the code is someone else's and\n * executing it pre-consent would not be defensible.\n */\n\nimport { spawn } from \"node:child_process\";\nimport { mkdtempSync, rmSync } from \"node:fs\";\nimport { tmpdir } from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline\";\nimport type { GateFinding } from \"./gates.js\";\nimport { pluginVariables } from \"./index.js\";\nimport type { NormalizedPlugin } from \"./manifest.js\";\n\n/** Per-capability budget. Long enough for a real server to boot, short enough not to stall a turn. */\nconst SMOKE_TIMEOUT_MS = 10_000;\n\n/** Env override, so tests can exercise the timeout path without waiting for it. */\nfunction smokeTimeoutMs(): number {\n\tconst raw = Number(process.env.HOOCODE_PLUGIN_SMOKE_TIMEOUT_MS);\n\treturn Number.isFinite(raw) && raw > 0 ? raw : SMOKE_TIMEOUT_MS;\n}\n\n/** Synthetic payloads, one per hook event, matching what the bridge really sends. */\nconst SYNTHETIC_PAYLOADS: Record<string, unknown> = {\n\tPreToolUse: { hook_event_name: \"PreToolUse\", tool_name: \"read\", tool_input: { file_path: \"smoke.txt\" } },\n\tPostToolUse: { hook_event_name: \"PostToolUse\", tool_name: \"read\", tool_input: {}, tool_response: \"\" },\n\tUserPromptSubmit: { hook_event_name: \"UserPromptSubmit\", prompt: \"smoke test\" },\n\tSessionStart: { hook_event_name: \"SessionStart\", source: \"startup\" },\n\tStop: { hook_event_name: \"Stop\" },\n};\n\nexport interface SmokeOptions {\n\t/** Skip the whole gate (no UI to report into, or an explicit opt-out). */\n\tskip?: boolean;\n}\n\n/** Environment for a smoke run: the plugin's own variables, redirected at a scratch dir. */\nfunction smokeEnv(plugin: NormalizedPlugin, sandbox: string): NodeJS.ProcessEnv {\n\treturn {\n\t\t...process.env,\n\t\t...pluginVariables(plugin.root, path.join(sandbox, \"data\")),\n\t\tHOME: sandbox,\n\t\tTMPDIR: sandbox,\n\t};\n}\n\nfunction runOnce(\n\tcommand: string,\n\tpayload: unknown,\n\tcwd: string,\n\tenv: NodeJS.ProcessEnv,\n): Promise<{ code: number | null; timedOut: boolean; output: string; spawnError?: string }> {\n\treturn new Promise((resolve) => {\n\t\tlet settled = false;\n\t\tlet output = \"\";\n\t\tconst child = spawn(command, { shell: true, cwd, env });\n\t\tconst timer = setTimeout(() => {\n\t\t\tif (settled) return;\n\t\t\tsettled = true;\n\t\t\tchild.kill(\"SIGKILL\");\n\t\t\tresolve({ code: null, timedOut: true, output });\n\t\t}, smokeTimeoutMs());\n\t\ttimer.unref?.();\n\n\t\tchild.stdout?.on(\"data\", (d) => {\n\t\t\toutput += d.toString().slice(0, 2000);\n\t\t});\n\t\tchild.stderr?.on(\"data\", (d) => {\n\t\t\toutput += d.toString().slice(0, 2000);\n\t\t});\n\t\tchild.on(\"error\", (e) => {\n\t\t\tif (settled) return;\n\t\t\tsettled = true;\n\t\t\tclearTimeout(timer);\n\t\t\tresolve({ code: null, timedOut: false, output, spawnError: String(e) });\n\t\t});\n\t\tchild.on(\"close\", (code) => {\n\t\t\tif (settled) return;\n\t\t\tsettled = true;\n\t\t\tclearTimeout(timer);\n\t\t\tresolve({ code, timedOut: false, output });\n\t\t});\n\n\t\t// A command that ignores stdin (`exit 0`) closes the pipe before we write,\n\t\t// and the resulting EPIPE surfaces asynchronously — a try/catch around the\n\t\t// write does not see it. Swallow it on the stream instead; not reading the\n\t\t// payload is a legitimate thing for a hook to do.\n\t\tchild.stdin?.on(\"error\", () => {});\n\t\tchild.stdin?.end(`${JSON.stringify(payload)}\\n`);\n\t});\n}\n\n/**\n * Complete an MCP handshake against a candidate server, then kill it.\n *\n * Hand-rolled rather than reusing `connectMcpServer`: that registers the\n * connection in a module-level map and terminates any existing entry with the\n * same name, so smoke-testing a draft server would tear down a live one the\n * session is using.\n */\nfunction probeMcpServer(\n\tcommand: string,\n\targs: string[],\n\tenv: NodeJS.ProcessEnv,\n\tcwd: string,\n): Promise<{ ok: boolean; detail: string; startFailed?: boolean }> {\n\treturn new Promise((resolve) => {\n\t\tlet settled = false;\n\t\tlet stderr = \"\";\n\t\tconst child = spawn(command, args, { cwd, env, stdio: [\"pipe\", \"pipe\", \"pipe\"] });\n\t\tconst finish = (ok: boolean, detail: string, startFailed = false) => {\n\t\t\tif (settled) return;\n\t\t\tsettled = true;\n\t\t\tclearTimeout(timer);\n\t\t\tchild.kill(\"SIGKILL\");\n\t\t\tresolve({ ok, detail, startFailed });\n\t\t};\n\t\tconst timer = setTimeout(() => finish(false, \"no response to initialize in time\"), smokeTimeoutMs());\n\t\ttimer.unref?.();\n\n\t\tchild.on(\"error\", (e) => finish(false, `could not start: ${e}`, true));\n\t\tchild.on(\"close\", (code) =>\n\t\t\tfinish(false, `exited (${code}) before completing the handshake${stderr ? `: ${stderr.slice(0, 300)}` : \"\"}`),\n\t\t);\n\t\tchild.stderr?.on(\"data\", (d) => {\n\t\t\tstderr += d.toString().slice(0, 1000);\n\t\t});\n\n\t\tchild.stdin?.on(\"error\", () => {});\n\t\tconst rl = readline.createInterface({ input: child.stdout! });\n\t\trl.on(\"line\", (line) => {\n\t\t\tlet msg: { id?: number; result?: unknown; error?: { message?: string } };\n\t\t\ttry {\n\t\t\t\tmsg = JSON.parse(line);\n\t\t\t} catch {\n\t\t\t\treturn; // servers sometimes log plain text on stdout\n\t\t\t}\n\t\t\tif (msg.error) return finish(false, `server error: ${msg.error.message ?? \"unknown\"}`);\n\t\t\tif (msg.id === 1) {\n\t\t\t\t// Per the spec the client acknowledges initialize before anything else;\n\t\t\t\t// strict servers refuse tools/list without it.\n\t\t\t\tchild.stdin?.write(`${JSON.stringify({ jsonrpc: \"2.0\", method: \"notifications/initialized\" })}\\n`);\n\t\t\t\tchild.stdin?.write(`${JSON.stringify({ jsonrpc: \"2.0\", id: 2, method: \"tools/list\", params: {} })}\\n`);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tif (msg.id === 2) {\n\t\t\t\tconst tools = (msg.result as { tools?: unknown[] } | undefined)?.tools ?? [];\n\t\t\t\tfinish(true, `handshake ok, ${tools.length} tool(s)`);\n\t\t\t}\n\t\t});\n\n\t\tchild.stdin?.write(\n\t\t\t`${JSON.stringify({\n\t\t\t\tjsonrpc: \"2.0\",\n\t\t\t\tid: 1,\n\t\t\t\tmethod: \"initialize\",\n\t\t\t\tparams: {\n\t\t\t\t\tprotocolVersion: \"2024-11-05\",\n\t\t\t\t\tcapabilities: { tools: {} },\n\t\t\t\t\tclientInfo: { name: \"hoocode-smoke\", version: \"1.0.0\" },\n\t\t\t\t},\n\t\t\t})}\\n`,\n\t\t);\n\t});\n}\n\n/** Run G3 over a plugin's executable capabilities. Returns findings; empty means nothing to test. */\nexport async function runSmokeGate(plugin: NormalizedPlugin, opts: SmokeOptions = {}): Promise<GateFinding[]> {\n\tif (opts.skip) return [];\n\tconst findings: GateFinding[] = [];\n\tconst hasExecutable = !!plugin.hooks || !!plugin.mcpServers;\n\tif (!hasExecutable) return findings;\n\n\tconst sandbox = mkdtempSync(path.join(tmpdir(), \"hoo-smoke-\"));\n\tconst env = smokeEnv(plugin, sandbox);\n\ttry {\n\t\tfor (const [event, groups] of Object.entries(plugin.hooks ?? {})) {\n\t\t\tconst payload = SYNTHETIC_PAYLOADS[event] ?? { hook_event_name: event };\n\t\t\tfor (const group of groups) {\n\t\t\t\tfor (const cmd of group.hooks) {\n\t\t\t\t\tconst res = await runOnce(cmd.command, payload, sandbox, env);\n\t\t\t\t\tconst label = `hook ${event}`;\n\t\t\t\t\tif (res.spawnError) {\n\t\t\t\t\t\t// Consistent with G2: a missing binary is a documented-prerequisite\n\t\t\t\t\t\t// problem, not a broken plugin. Erroring here would contradict the\n\t\t\t\t\t\t// warning G2 already issues for the very same condition.\n\t\t\t\t\t\tfindings.push({\n\t\t\t\t\t\t\tgate: \"G3\",\n\t\t\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\t\t\tmessage: `${label} could not start: ${res.spawnError}`,\n\t\t\t\t\t\t});\n\t\t\t\t\t} else if (res.timedOut) {\n\t\t\t\t\t\tfindings.push({\n\t\t\t\t\t\t\tgate: \"G3\",\n\t\t\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\t\t\tmessage: `${label} did not finish in time — it would stall every matching tool call.`,\n\t\t\t\t\t\t});\n\t\t\t\t\t} else if (res.code === 0 || (event === \"PreToolUse\" && res.code === 2)) {\n\t\t\t\t\t\t// Exit 2 from PreToolUse is the documented \"block\" decision, not a\n\t\t\t\t\t\t// failure: the hook ran and made a call.\n\t\t\t\t\t\tfindings.push({ gate: \"G3\", severity: \"info\", message: `${label} ran cleanly (exit ${res.code}).` });\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// A non-zero exit may just mean the synthetic payload is not one this\n\t\t\t\t\t\t// hook handles, so it is reported rather than fatal.\n\t\t\t\t\t\tfindings.push({\n\t\t\t\t\t\t\tgate: \"G3\",\n\t\t\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\t\t\tmessage: `${label} exited ${res.code} on a synthetic ${event} payload${res.output.trim() ? `: ${res.output.trim().slice(0, 200)}` : \"\"}`,\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor (const [name, raw] of Object.entries(plugin.mcpServers ?? {})) {\n\t\t\tconst cfg = raw as { command?: unknown; args?: unknown; env?: unknown };\n\t\t\tif (typeof cfg.command !== \"string\") continue; // remote transports are not spawned\n\t\t\tconst res = await probeMcpServer(\n\t\t\t\tcfg.command,\n\t\t\t\tArray.isArray(cfg.args) ? cfg.args.map(String) : [],\n\t\t\t\t{ ...env, ...(cfg.env && typeof cfg.env === \"object\" ? (cfg.env as NodeJS.ProcessEnv) : {}) },\n\t\t\t\tplugin.root,\n\t\t\t);\n\t\t\tfindings.push({\n\t\t\t\tgate: \"G3\",\n\t\t\t\t// A server that never starts is the missing-binary case again — warn.\n\t\t\t\t// One that starts and then fails the handshake is genuinely broken and\n\t\t\t\t// would take the session down at connect time.\n\t\t\t\tseverity: res.ok ? \"info\" : res.startFailed ? \"warning\" : \"error\",\n\t\t\t\tmessage: `mcp server \"${name}\": ${res.detail}`,\n\t\t\t});\n\t\t}\n\t} finally {\n\t\trmSync(sandbox, { recursive: true, force: true });\n\t}\n\n\treturn findings;\n}\n"]}

package/dist/core/extensions/plugins/smoke.js

@@ -0,0 +1,231 @@
+/**
+ * G3 — behavioral smoke for a plugin's executable capabilities.
+ *
+ * G1 and G2 read files. G3 is the first gate that finds out whether the thing
+ * actually runs, which is the most common real failure: a plugin that installs
+ * cleanly and breaks the session at the next tool call.
+ *
+ * **What this is and is not.** Hooks run with `cwd`, `HOME`, `TMPDIR` and the
+ * plugin data dir redirected into a throwaway directory, and with a hard
+ * timeout. That reduces blast radius; it is *not* containment. Without OS-level
+ * sandboxing a shell command can still write wherever it likes, and claiming
+ * otherwise in a confirmation prompt would be worse than saying nothing — the
+ * whole point of showing gate results to a human is that they are true.
+ *
+ * **Why it runs before the human confirms.** It does execute not-yet-approved
+ * code, which is a real cost. Against it: the code was authored in this session
+ * from the user's own request rather than fetched from anywhere, G2 has already
+ * screened it for destructive shapes, and absent the smoke test the very same
+ * command runs moments later anyway — unscreened, unredirected, and in the real
+ * working directory. Running it once under redirection to find out whether it
+ * even works is the smaller risk. G3 is therefore authored-only: it is never
+ * applied to a marketplace plugin, where the code is someone else's and
+ * executing it pre-consent would not be defensible.
+ */
+import { spawn } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import * as path from "node:path";
+import * as readline from "node:readline";
+import { pluginVariables } from "./index.js";
+/** Per-capability budget. Long enough for a real server to boot, short enough not to stall a turn. */
+const SMOKE_TIMEOUT_MS = 10_000;
+/** Env override, so tests can exercise the timeout path without waiting for it. */
+function smokeTimeoutMs() {
+    const raw = Number(process.env.HOOCODE_PLUGIN_SMOKE_TIMEOUT_MS);
+    return Number.isFinite(raw) && raw > 0 ? raw : SMOKE_TIMEOUT_MS;
+}
+/** Synthetic payloads, one per hook event, matching what the bridge really sends. */
+const SYNTHETIC_PAYLOADS = {
+    PreToolUse: { hook_event_name: "PreToolUse", tool_name: "read", tool_input: { file_path: "smoke.txt" } },
+    PostToolUse: { hook_event_name: "PostToolUse", tool_name: "read", tool_input: {}, tool_response: "" },
+    UserPromptSubmit: { hook_event_name: "UserPromptSubmit", prompt: "smoke test" },
+    SessionStart: { hook_event_name: "SessionStart", source: "startup" },
+    Stop: { hook_event_name: "Stop" },
+};
+/** Environment for a smoke run: the plugin's own variables, redirected at a scratch dir. */
+function smokeEnv(plugin, sandbox) {
+    return {
+        ...process.env,
+        ...pluginVariables(plugin.root, path.join(sandbox, "data")),
+        HOME: sandbox,
+        TMPDIR: sandbox,
+    };
+}
+function runOnce(command, payload, cwd, env) {
+    return new Promise((resolve) => {
+        let settled = false;
+        let output = "";
+        const child = spawn(command, { shell: true, cwd, env });
+        const timer = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            child.kill("SIGKILL");
+            resolve({ code: null, timedOut: true, output });
+        }, smokeTimeoutMs());
+        timer.unref?.();
+        child.stdout?.on("data", (d) => {
+            output += d.toString().slice(0, 2000);
+        });
+        child.stderr?.on("data", (d) => {
+            output += d.toString().slice(0, 2000);
+        });
+        child.on("error", (e) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve({ code: null, timedOut: false, output, spawnError: String(e) });
+        });
+        child.on("close", (code) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve({ code, timedOut: false, output });
+        });
+        // A command that ignores stdin (`exit 0`) closes the pipe before we write,
+        // and the resulting EPIPE surfaces asynchronously — a try/catch around the
+        // write does not see it. Swallow it on the stream instead; not reading the
+        // payload is a legitimate thing for a hook to do.
+        child.stdin?.on("error", () => { });
+        child.stdin?.end(`${JSON.stringify(payload)}\n`);
+    });
+}
+/**
+ * Complete an MCP handshake against a candidate server, then kill it.
+ *
+ * Hand-rolled rather than reusing `connectMcpServer`: that registers the
+ * connection in a module-level map and terminates any existing entry with the
+ * same name, so smoke-testing a draft server would tear down a live one the
+ * session is using.
+ */
+function probeMcpServer(command, args, env, cwd) {
+    return new Promise((resolve) => {
+        let settled = false;
+        let stderr = "";
+        const child = spawn(command, args, { cwd, env, stdio: ["pipe", "pipe", "pipe"] });
+        const finish = (ok, detail, startFailed = false) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            child.kill("SIGKILL");
+            resolve({ ok, detail, startFailed });
+        };
+        const timer = setTimeout(() => finish(false, "no response to initialize in time"), smokeTimeoutMs());
+        timer.unref?.();
+        child.on("error", (e) => finish(false, `could not start: ${e}`, true));
+        child.on("close", (code) => finish(false, `exited (${code}) before completing the handshake${stderr ? `: ${stderr.slice(0, 300)}` : ""}`));
+        child.stderr?.on("data", (d) => {
+            stderr += d.toString().slice(0, 1000);
+        });
+        child.stdin?.on("error", () => { });
+        const rl = readline.createInterface({ input: child.stdout });
+        rl.on("line", (line) => {
+            let msg;
+            try {
+                msg = JSON.parse(line);
+            }
+            catch {
+                return; // servers sometimes log plain text on stdout
+            }
+            if (msg.error)
+                return finish(false, `server error: ${msg.error.message ?? "unknown"}`);
+            if (msg.id === 1) {
+                // Per the spec the client acknowledges initialize before anything else;
+                // strict servers refuse tools/list without it.
+                child.stdin?.write(`${JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" })}\n`);
+                child.stdin?.write(`${JSON.stringify({ jsonrpc: "2.0", id: 2, method: "tools/list", params: {} })}\n`);
+                return;
+            }
+            if (msg.id === 2) {
+                const tools = msg.result?.tools ?? [];
+                finish(true, `handshake ok, ${tools.length} tool(s)`);
+            }
+        });
+        child.stdin?.write(`${JSON.stringify({
+            jsonrpc: "2.0",
+            id: 1,
+            method: "initialize",
+            params: {
+                protocolVersion: "2024-11-05",
+                capabilities: { tools: {} },
+                clientInfo: { name: "hoocode-smoke", version: "1.0.0" },
+            },
+        })}\n`);
+    });
+}
+/** Run G3 over a plugin's executable capabilities. Returns findings; empty means nothing to test. */
+export async function runSmokeGate(plugin, opts = {}) {
+    if (opts.skip)
+        return [];
+    const findings = [];
+    const hasExecutable = !!plugin.hooks || !!plugin.mcpServers;
+    if (!hasExecutable)
+        return findings;
+    const sandbox = mkdtempSync(path.join(tmpdir(), "hoo-smoke-"));
+    const env = smokeEnv(plugin, sandbox);
+    try {
+        for (const [event, groups] of Object.entries(plugin.hooks ?? {})) {
+            const payload = SYNTHETIC_PAYLOADS[event] ?? { hook_event_name: event };
+            for (const group of groups) {
+                for (const cmd of group.hooks) {
+                    const res = await runOnce(cmd.command, payload, sandbox, env);
+                    const label = `hook ${event}`;
+                    if (res.spawnError) {
+                        // Consistent with G2: a missing binary is a documented-prerequisite
+                        // problem, not a broken plugin. Erroring here would contradict the
+                        // warning G2 already issues for the very same condition.
+                        findings.push({
+                            gate: "G3",
+                            severity: "warning",
+                            message: `${label} could not start: ${res.spawnError}`,
+                        });
+                    }
+                    else if (res.timedOut) {
+                        findings.push({
+                            gate: "G3",
+                            severity: "error",
+                            message: `${label} did not finish in time — it would stall every matching tool call.`,
+                        });
+                    }
+                    else if (res.code === 0 || (event === "PreToolUse" && res.code === 2)) {
+                        // Exit 2 from PreToolUse is the documented "block" decision, not a
+                        // failure: the hook ran and made a call.
+                        findings.push({ gate: "G3", severity: "info", message: `${label} ran cleanly (exit ${res.code}).` });
+                    }
+                    else {
+                        // A non-zero exit may just mean the synthetic payload is not one this
+                        // hook handles, so it is reported rather than fatal.
+                        findings.push({
+                            gate: "G3",
+                            severity: "warning",
+                            message: `${label} exited ${res.code} on a synthetic ${event} payload${res.output.trim() ? `: ${res.output.trim().slice(0, 200)}` : ""}`,
+                        });
+                    }
+                }
+            }
+        }
+        for (const [name, raw] of Object.entries(plugin.mcpServers ?? {})) {
+            const cfg = raw;
+            if (typeof cfg.command !== "string")
+                continue; // remote transports are not spawned
+            const res = await probeMcpServer(cfg.command, Array.isArray(cfg.args) ? cfg.args.map(String) : [], { ...env, ...(cfg.env && typeof cfg.env === "object" ? cfg.env : {}) }, plugin.root);
+            findings.push({
+                gate: "G3",
+                // A server that never starts is the missing-binary case again — warn.
+                // One that starts and then fails the handshake is genuinely broken and
+                // would take the session down at connect time.
+                severity: res.ok ? "info" : res.startFailed ? "warning" : "error",
+                message: `mcp server "${name}": ${res.detail}`,
+            });
+        }
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
+    return findings;
+}
+//# sourceMappingURL=smoke.js.map

package/dist/core/extensions/plugins/smoke.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"smoke.js","sourceRoot":"","sources":["../../../../src/core/extensions/plugins/smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAG7C,sGAAsG;AACtG,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,mFAAmF;AACnF,SAAS,cAAc,GAAW;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAChE,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC;AAAA,CAChE;AAED,qFAAqF;AACrF,MAAM,kBAAkB,GAA4B;IACnD,UAAU,EAAE,EAAE,eAAe,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE;IACxG,WAAW,EAAE,EAAE,eAAe,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE;IACrG,gBAAgB,EAAE,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,EAAE,YAAY,EAAE;IAC/E,YAAY,EAAE,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE;IACpE,IAAI,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE;CACjC,CAAC;AAOF,4FAA4F;AAC5F,SAAS,QAAQ,CAAC,MAAwB,EAAE,OAAe,EAAqB;IAC/E,OAAO;QACN,GAAG,OAAO,CAAC,GAAG;QACd,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC3D,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,OAAO;KACf,CAAC;AAAA,CACF;AAED,SAAS,OAAO,CACf,OAAe,EACf,OAAgB,EAChB,GAAW,EACX,GAAsB,EACqE;IAC3F,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/B,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC;YAC9B,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAAA,CAChD,EAAE,cAAc,EAAE,CAAC,CAAC;QACrB,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAEhB,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAAA,CACtC,CAAC,CAAC;QACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAAA,CACtC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YACxB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAAA,CACxE,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAAA,CAC3C,CAAC,CAAC;QAEH,2EAA2E;QAC3E,6EAA2E;QAC3E,2EAA2E;QAC3E,kDAAkD;QAClD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC,CAAC,CAAC;QACnC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAAA,CACjD,CAAC,CAAC;AAAA,CACH;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CACtB,OAAe,EACf,IAAc,EACd,GAAsB,EACtB,GAAW,EACuD;IAClE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/B,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,CAAC,EAAW,EAAE,MAAc,EAAE,WAAW,GAAG,KAAK,EAAE,EAAE,CAAC;YACpE,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,OAAO,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAAA,CACrC,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,mCAAmC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC;QACrG,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAEhB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,oBAAoB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;QACvE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAC1B,MAAM,CAAC,KAAK,EAAE,WAAW,IAAI,oCAAoC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAC7G,CAAC;QACF,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAAA,CACtC,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAO,EAAE,CAAC,CAAC;QAC9D,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YACvB,IAAI,GAAoE,CAAC;YACzE,IAAI,CAAC;gBACJ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACR,OAAO,CAAC,6CAA6C;YACtD,CAAC;YACD,IAAI,GAAG,CAAC,KAAK;gBAAE,OAAO,MAAM,CAAC,KAAK,EAAE,iBAAiB,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,SAAS,EAAE,CAAC,CAAC;YACvF,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;gBAClB,wEAAwE;gBACxE,+CAA+C;gBAC/C,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAC,IAAI,CAAC,CAAC;gBACnG,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACvG,OAAO;YACR,CAAC;YACD,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;gBAClB,MAAM,KAAK,GAAI,GAAG,CAAC,MAA4C,EAAE,KAAK,IAAI,EAAE,CAAC;gBAC7E,MAAM,CAAC,IAAI,EAAE,iBAAiB,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;YACvD,CAAC;QAAA,CACD,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,EAAE,KAAK,CACjB,GAAG,IAAI,CAAC,SAAS,CAAC;YACjB,OAAO,EAAE,KAAK;YACd,EAAE,EAAE,CAAC;YACL,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE;gBACP,eAAe,EAAE,YAAY;gBAC7B,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3B,UAAU,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE;aACvD;SACD,CAAC,IAAI,CACN,CAAC;IAAA,CACF,CAAC,CAAC;AAAA,CACH;AAED,qGAAqG;AACrG,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAwB,EAAE,IAAI,GAAiB,EAAE,EAA0B;IAC7G,IAAI,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACzB,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;IAC5D,IAAI,CAAC,aAAa;QAAE,OAAO,QAAQ,CAAC;IAEpC,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtC,IAAI,CAAC;QACJ,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;YAClE,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;YACxE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC5B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;oBAC9D,MAAM,KAAK,GAAG,QAAQ,KAAK,EAAE,CAAC;oBAC9B,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;wBACpB,oEAAoE;wBACpE,mEAAmE;wBACnE,yDAAyD;wBACzD,QAAQ,CAAC,IAAI,CAAC;4BACb,IAAI,EAAE,IAAI;4BACV,QAAQ,EAAE,SAAS;4BACnB,OAAO,EAAE,GAAG,KAAK,qBAAqB,GAAG,CAAC,UAAU,EAAE;yBACtD,CAAC,CAAC;oBACJ,CAAC;yBAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;wBACzB,QAAQ,CAAC,IAAI,CAAC;4BACb,IAAI,EAAE,IAAI;4BACV,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,GAAG,KAAK,sEAAoE;yBACrF,CAAC,CAAC;oBACJ,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,YAAY,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC;wBACzE,mEAAmE;wBACnE,yCAAyC;wBACzC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,sBAAsB,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;oBACtG,CAAC;yBAAM,CAAC;wBACP,sEAAsE;wBACtE,qDAAqD;wBACrD,QAAQ,CAAC,IAAI,CAAC;4BACb,IAAI,EAAE,IAAI;4BACV,QAAQ,EAAE,SAAS;4BACnB,OAAO,EAAE,GAAG,KAAK,WAAW,GAAG,CAAC,IAAI,mBAAmB,KAAK,WAAW,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;yBACxI,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACnE,MAAM,GAAG,GAAG,GAA2D,CAAC;YACxE,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;gBAAE,SAAS,CAAC,oCAAoC;YACnF,MAAM,GAAG,GAAG,MAAM,cAAc,CAC/B,GAAG,CAAC,OAAO,EACX,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EACnD,EAAE,GAAG,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAE,GAAG,CAAC,GAAyB,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAC7F,MAAM,CAAC,IAAI,CACX,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC;gBACb,IAAI,EAAE,IAAI;gBACV,wEAAsE;gBACtE,uEAAuE;gBACvE,+CAA+C;gBAC/C,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO;gBACjE,OAAO,EAAE,eAAe,IAAI,MAAM,GAAG,CAAC,MAAM,EAAE;aAC9C,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,OAAO,QAAQ,CAAC;AAAA,CAChB","sourcesContent":["/**\n * G3 — behavioral smoke for a plugin's executable capabilities.\n *\n * G1 and G2 read files. G3 is the first gate that finds out whether the thing\n * actually runs, which is the most common real failure: a plugin that installs\n * cleanly and breaks the session at the next tool call.\n *\n * **What this is and is not.** Hooks run with `cwd`, `HOME`, `TMPDIR` and the\n * plugin data dir redirected into a throwaway directory, and with a hard\n * timeout. That reduces blast radius; it is *not* containment. Without OS-level\n * sandboxing a shell command can still write wherever it likes, and claiming\n * otherwise in a confirmation prompt would be worse than saying nothing — the\n * whole point of showing gate results to a human is that they are true.\n *\n * **Why it runs before the human confirms.** It does execute not-yet-approved\n * code, which is a real cost. Against it: the code was authored in this session\n * from the user's own request rather than fetched from anywhere, G2 has already\n * screened it for destructive shapes, and absent the smoke test the very same\n * command runs moments later anyway — unscreened, unredirected, and in the real\n * working directory. Running it once under redirection to find out whether it\n * even works is the smaller risk. G3 is therefore authored-only: it is never\n * applied to a marketplace plugin, where the code is someone else's and\n * executing it pre-consent would not be defensible.\n */\n\nimport { spawn } from \"node:child_process\";\nimport { mkdtempSync, rmSync } from \"node:fs\";\nimport { tmpdir } from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline\";\nimport type { GateFinding } from \"./gates.js\";\nimport { pluginVariables } from \"./index.js\";\nimport type { NormalizedPlugin } from \"./manifest.js\";\n\n/** Per-capability budget. Long enough for a real server to boot, short enough not to stall a turn. */\nconst SMOKE_TIMEOUT_MS = 10_000;\n\n/** Env override, so tests can exercise the timeout path without waiting for it. */\nfunction smokeTimeoutMs(): number {\n\tconst raw = Number(process.env.HOOCODE_PLUGIN_SMOKE_TIMEOUT_MS);\n\treturn Number.isFinite(raw) && raw > 0 ? raw : SMOKE_TIMEOUT_MS;\n}\n\n/** Synthetic payloads, one per hook event, matching what the bridge really sends. */\nconst SYNTHETIC_PAYLOADS: Record<string, unknown> = {\n\tPreToolUse: { hook_event_name: \"PreToolUse\", tool_name: \"read\", tool_input: { file_path: \"smoke.txt\" } },\n\tPostToolUse: { hook_event_name: \"PostToolUse\", tool_name: \"read\", tool_input: {}, tool_response: \"\" },\n\tUserPromptSubmit: { hook_event_name: \"UserPromptSubmit\", prompt: \"smoke test\" },\n\tSessionStart: { hook_event_name: \"SessionStart\", source: \"startup\" },\n\tStop: { hook_event_name: \"Stop\" },\n};\n\nexport interface SmokeOptions {\n\t/** Skip the whole gate (no UI to report into, or an explicit opt-out). */\n\tskip?: boolean;\n}\n\n/** Environment for a smoke run: the plugin's own variables, redirected at a scratch dir. */\nfunction smokeEnv(plugin: NormalizedPlugin, sandbox: string): NodeJS.ProcessEnv {\n\treturn {\n\t\t...process.env,\n\t\t...pluginVariables(plugin.root, path.join(sandbox, \"data\")),\n\t\tHOME: sandbox,\n\t\tTMPDIR: sandbox,\n\t};\n}\n\nfunction runOnce(\n\tcommand: string,\n\tpayload: unknown,\n\tcwd: string,\n\tenv: NodeJS.ProcessEnv,\n): Promise<{ code: number | null; timedOut: boolean; output: string; spawnError?: string }> {\n\treturn new Promise((resolve) => {\n\t\tlet settled = false;\n\t\tlet output = \"\";\n\t\tconst child = spawn(command, { shell: true, cwd, env });\n\t\tconst timer = setTimeout(() => {\n\t\t\tif (settled) return;\n\t\t\tsettled = true;\n\t\t\tchild.kill(\"SIGKILL\");\n\t\t\tresolve({ code: null, timedOut: true, output });\n\t\t}, smokeTimeoutMs());\n\t\ttimer.unref?.();\n\n\t\tchild.stdout?.on(\"data\", (d) => {\n\t\t\toutput += d.toString().slice(0, 2000);\n\t\t});\n\t\tchild.stderr?.on(\"data\", (d) => {\n\t\t\toutput += d.toString().slice(0, 2000);\n\t\t});\n\t\tchild.on(\"error\", (e) => {\n\t\t\tif (settled) return;\n\t\t\tsettled = true;\n\t\t\tclearTimeout(timer);\n\t\t\tresolve({ code: null, timedOut: false, output, spawnError: String(e) });\n\t\t});\n\t\tchild.on(\"close\", (code) => {\n\t\t\tif (settled) return;\n\t\t\tsettled = true;\n\t\t\tclearTimeout(timer);\n\t\t\tresolve({ code, timedOut: false, output });\n\t\t});\n\n\t\t// A command that ignores stdin (`exit 0`) closes the pipe before we write,\n\t\t// and the resulting EPIPE surfaces asynchronously — a try/catch around the\n\t\t// write does not see it. Swallow it on the stream instead; not reading the\n\t\t// payload is a legitimate thing for a hook to do.\n\t\tchild.stdin?.on(\"error\", () => {});\n\t\tchild.stdin?.end(`${JSON.stringify(payload)}\\n`);\n\t});\n}\n\n/**\n * Complete an MCP handshake against a candidate server, then kill it.\n *\n * Hand-rolled rather than reusing `connectMcpServer`: that registers the\n * connection in a module-level map and terminates any existing entry with the\n * same name, so smoke-testing a draft server would tear down a live one the\n * session is using.\n */\nfunction probeMcpServer(\n\tcommand: string,\n\targs: string[],\n\tenv: NodeJS.ProcessEnv,\n\tcwd: string,\n): Promise<{ ok: boolean; detail: string; startFailed?: boolean }> {\n\treturn new Promise((resolve) => {\n\t\tlet settled = false;\n\t\tlet stderr = \"\";\n\t\tconst child = spawn(command, args, { cwd, env, stdio: [\"pipe\", \"pipe\", \"pipe\"] });\n\t\tconst finish = (ok: boolean, detail: string, startFailed = false) => {\n\t\t\tif (settled) return;\n\t\t\tsettled = true;\n\t\t\tclearTimeout(timer);\n\t\t\tchild.kill(\"SIGKILL\");\n\t\t\tresolve({ ok, detail, startFailed });\n\t\t};\n\t\tconst timer = setTimeout(() => finish(false, \"no response to initialize in time\"), smokeTimeoutMs());\n\t\ttimer.unref?.();\n\n\t\tchild.on(\"error\", (e) => finish(false, `could not start: ${e}`, true));\n\t\tchild.on(\"close\", (code) =>\n\t\t\tfinish(false, `exited (${code}) before completing the handshake${stderr ? `: ${stderr.slice(0, 300)}` : \"\"}`),\n\t\t);\n\t\tchild.stderr?.on(\"data\", (d) => {\n\t\t\tstderr += d.toString().slice(0, 1000);\n\t\t});\n\n\t\tchild.stdin?.on(\"error\", () => {});\n\t\tconst rl = readline.createInterface({ input: child.stdout! });\n\t\trl.on(\"line\", (line) => {\n\t\t\tlet msg: { id?: number; result?: unknown; error?: { message?: string } };\n\t\t\ttry {\n\t\t\t\tmsg = JSON.parse(line);\n\t\t\t} catch {\n\t\t\t\treturn; // servers sometimes log plain text on stdout\n\t\t\t}\n\t\t\tif (msg.error) return finish(false, `server error: ${msg.error.message ?? \"unknown\"}`);\n\t\t\tif (msg.id === 1) {\n\t\t\t\t// Per the spec the client acknowledges initialize before anything else;\n\t\t\t\t// strict servers refuse tools/list without it.\n\t\t\t\tchild.stdin?.write(`${JSON.stringify({ jsonrpc: \"2.0\", method: \"notifications/initialized\" })}\\n`);\n\t\t\t\tchild.stdin?.write(`${JSON.stringify({ jsonrpc: \"2.0\", id: 2, method: \"tools/list\", params: {} })}\\n`);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tif (msg.id === 2) {\n\t\t\t\tconst tools = (msg.result as { tools?: unknown[] } | undefined)?.tools ?? [];\n\t\t\t\tfinish(true, `handshake ok, ${tools.length} tool(s)`);\n\t\t\t}\n\t\t});\n\n\t\tchild.stdin?.write(\n\t\t\t`${JSON.stringify({\n\t\t\t\tjsonrpc: \"2.0\",\n\t\t\t\tid: 1,\n\t\t\t\tmethod: \"initialize\",\n\t\t\t\tparams: {\n\t\t\t\t\tprotocolVersion: \"2024-11-05\",\n\t\t\t\t\tcapabilities: { tools: {} },\n\t\t\t\t\tclientInfo: { name: \"hoocode-smoke\", version: \"1.0.0\" },\n\t\t\t\t},\n\t\t\t})}\\n`,\n\t\t);\n\t});\n}\n\n/** Run G3 over a plugin's executable capabilities. Returns findings; empty means nothing to test. */\nexport async function runSmokeGate(plugin: NormalizedPlugin, opts: SmokeOptions = {}): Promise<GateFinding[]> {\n\tif (opts.skip) return [];\n\tconst findings: GateFinding[] = [];\n\tconst hasExecutable = !!plugin.hooks || !!plugin.mcpServers;\n\tif (!hasExecutable) return findings;\n\n\tconst sandbox = mkdtempSync(path.join(tmpdir(), \"hoo-smoke-\"));\n\tconst env = smokeEnv(plugin, sandbox);\n\ttry {\n\t\tfor (const [event, groups] of Object.entries(plugin.hooks ?? {})) {\n\t\t\tconst payload = SYNTHETIC_PAYLOADS[event] ?? { hook_event_name: event };\n\t\t\tfor (const group of groups) {\n\t\t\t\tfor (const cmd of group.hooks) {\n\t\t\t\t\tconst res = await runOnce(cmd.command, payload, sandbox, env);\n\t\t\t\t\tconst label = `hook ${event}`;\n\t\t\t\t\tif (res.spawnError) {\n\t\t\t\t\t\t// Consistent with G2: a missing binary is a documented-prerequisite\n\t\t\t\t\t\t// problem, not a broken plugin. Erroring here would contradict the\n\t\t\t\t\t\t// warning G2 already issues for the very same condition.\n\t\t\t\t\t\tfindings.push({\n\t\t\t\t\t\t\tgate: \"G3\",\n\t\t\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\t\t\tmessage: `${label} could not start: ${res.spawnError}`,\n\t\t\t\t\t\t});\n\t\t\t\t\t} else if (res.timedOut) {\n\t\t\t\t\t\tfindings.push({\n\t\t\t\t\t\t\tgate: \"G3\",\n\t\t\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\t\t\tmessage: `${label} did not finish in time — it would stall every matching tool call.`,\n\t\t\t\t\t\t});\n\t\t\t\t\t} else if (res.code === 0 || (event === \"PreToolUse\" && res.code === 2)) {\n\t\t\t\t\t\t// Exit 2 from PreToolUse is the documented \"block\" decision, not a\n\t\t\t\t\t\t// failure: the hook ran and made a call.\n\t\t\t\t\t\tfindings.push({ gate: \"G3\", severity: \"info\", message: `${label} ran cleanly (exit ${res.code}).` });\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// A non-zero exit may just mean the synthetic payload is not one this\n\t\t\t\t\t\t// hook handles, so it is reported rather than fatal.\n\t\t\t\t\t\tfindings.push({\n\t\t\t\t\t\t\tgate: \"G3\",\n\t\t\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\t\t\tmessage: `${label} exited ${res.code} on a synthetic ${event} payload${res.output.trim() ? `: ${res.output.trim().slice(0, 200)}` : \"\"}`,\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor (const [name, raw] of Object.entries(plugin.mcpServers ?? {})) {\n\t\t\tconst cfg = raw as { command?: unknown; args?: unknown; env?: unknown };\n\t\t\tif (typeof cfg.command !== \"string\") continue; // remote transports are not spawned\n\t\t\tconst res = await probeMcpServer(\n\t\t\t\tcfg.command,\n\t\t\t\tArray.isArray(cfg.args) ? cfg.args.map(String) : [],\n\t\t\t\t{ ...env, ...(cfg.env && typeof cfg.env === \"object\" ? (cfg.env as NodeJS.ProcessEnv) : {}) },\n\t\t\t\tplugin.root,\n\t\t\t);\n\t\t\tfindings.push({\n\t\t\t\tgate: \"G3\",\n\t\t\t\t// A server that never starts is the missing-binary case again — warn.\n\t\t\t\t// One that starts and then fails the handshake is genuinely broken and\n\t\t\t\t// would take the session down at connect time.\n\t\t\t\tseverity: res.ok ? \"info\" : res.startFailed ? \"warning\" : \"error\",\n\t\t\t\tmessage: `mcp server \"${name}\": ${res.detail}`,\n\t\t\t});\n\t\t}\n\t} finally {\n\t\trmSync(sandbox, { recursive: true, force: true });\n\t}\n\n\treturn findings;\n}\n"]}

package/dist/core/extensions/plugins/trigger-eval.d.ts

@@ -0,0 +1,136 @@
+/**
+ * G4 — trigger eval: does an authored skill fire on the situations it is for,
+ * and stay quiet otherwise?
+ *
+ * G1–G3 ask whether a plugin is well-formed, safe, and runnable. None of them
+ * ask whether it is any *good*, and for a passive capability the whole of "good"
+ * is the `description`: it is the only thing the model sees when deciding
+ * whether to reach for the skill. A skill that parses, validates, and never
+ * triggers is dead weight that costs context on every request.
+ *
+ * ## The circularity trap
+ *
+ * The tempting design is to generate test prompts from the description and then
+ * check that the description matches them. That measures self-consistency and
+ * always passes. So gold prompts are **supplied**, not generated (`eval/triggers.json`
+ * in the plugin, or authored alongside it), and the negatives largely come for
+ * free: a prompt whose right answer is a *different* capability is a negative for
+ * this one. Discriminating against real siblings is the question that matters —
+ * "does it fire" is easy, "does it fire instead of the wrong thing" is not.
+ *
+ * ## Reuse of the search eval harness
+ *
+ * §4.2 asks G4 to reuse `core/search/eval-harness.ts`. Its three *rules* are
+ * adopted verbatim below and are the reason this module looks the way it does:
+ * pin what you measured, record it machine-readably, and never let a degraded
+ * run read like a real one. Its *machinery* could not be called: that harness
+ * pins a corpus by checking out a git worktree and stamps provenance with
+ * embsearch daemon state, and G4's corpus is a handful of description strings
+ * with no repo and no embedder. Calling it would have meant a git worktree per
+ * skill eval. The pinning here is therefore a content hash of exactly what was
+ * judged — same guarantee, appropriate mechanism.
+ *
+ * See docs/plugin-system-architecture.md §4.2.
+ */
+import type { CapabilityDoc } from "../../capabilities/registry.js";
+import type { GateFinding } from "./gates.js";
+import type { NormalizedPlugin } from "./manifest.js";
+/** Where a plugin declares the prompts its capabilities should (and should not) fire on. */
+export declare const TRIGGER_GOLD_FILE: string;
+export interface TriggerCase {
+    prompt: string;
+    /** Capability name this prompt should select, or null when nothing should fire. */
+    expect: string | null;
+}
+/** One capability offered to the judge as a candidate. */
+export interface TriggerCandidate {
+    name: string;
+    description: string;
+    /** False for the distractors drawn from other plugins. */
+    own: boolean;
+}
+export interface TriggerJudgeRequest {
+    candidates: TriggerCandidate[];
+    prompts: string[];
+}
+/** The judge's answer per prompt: the selected candidate name, or null for none. */
+export type TriggerJudgeVerdict = string | null;
+/**
+ * The single model call, injected.
+ *
+ * Injected rather than imported so the scoring, the gold-set handling and the
+ * gate mapping are testable without a model — and so a caller with no model
+ * configured gets an honest `not-run` instead of a gate that quietly passes.
+ */
+export type TriggerJudge = (request: TriggerJudgeRequest) => Promise<TriggerJudgeVerdict[]>;
+export interface TriggerCaseResult {
+    prompt: string;
+    expected: string | null;
+    actual: string | null;
+    correct: boolean;
+}
+export interface TriggerEvalRecord {
+    /** Content hash of the candidates and cases judged — the pin. */
+    corpusHash: string;
+    timestampMs: number;
+    pluginId: string;
+    candidateCount: number;
+    caseCount: number;
+    /** Fires when it should: TP / (TP + FN). Undefined when there were no positives. */
+    recall?: number;
+    /** Quiet when it should be: TN / (TN + FP). Undefined when there were no negatives. */
+    specificity?: number;
+    results: TriggerCaseResult[];
+}
+export type TriggerEvalOutcome = {
+    status: "not-run";
+    reason: string;
+} | {
+    status: "ran";
+    record: TriggerEvalRecord;
+};
+/** Read the plugin's declared gold set, or undefined when it has none. */
+export declare function loadTriggerCases(root: string): TriggerCase[] | undefined;
+/**
+ * The plugin's own passive capabilities, as judge candidates.
+ *
+ * Skills only. Commands are invoked by name (`/foo`) rather than selected from a
+ * description, so "does the description fire" is not a question about them, and
+ * scoring them would dilute the metric with cases that cannot fail.
+ */
+export declare function ownCandidates(plugin: NormalizedPlugin): TriggerCandidate[];
+/**
+ * Sibling capabilities to put in front of the judge alongside the plugin's own.
+ *
+ * Without them the judge picks from a list of one and "does it fire" is nearly
+ * free. The interesting failure is a description broad enough to win prompts
+ * that belong to something else, and that only shows up against real competition.
+ */
+export declare function distractorCandidates(exclude: string, docs: readonly CapabilityDoc[]): TriggerCandidate[];
+/**
+ * Pin: a hash of exactly what was judged.
+ *
+ * Over candidates *and* cases, because both move the number. A rerun after a
+ * description edit is measuring something else, and a record that cannot say so
+ * is the failure mode the search harness was built to prevent.
+ */
+export declare function triggerCorpusHash(candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[]): string;
+/**
+ * Score a plugin's passive capabilities against its gold set.
+ *
+ * Never throws and never fabricates: no judge, no gold set, or no candidates all
+ * return `not-run` with the reason. A judge that answers the wrong number of
+ * prompts is also `not-run` — a partial alignment between prompts and verdicts
+ * would silently score the wrong pairs.
+ */
+export declare function runTriggerEval(pluginId: string, candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[] | undefined, judge: TriggerJudge | undefined): Promise<TriggerEvalOutcome>;
+/**
+ * Map an outcome onto gate findings.
+ *
+ * `not-run` is `info`, never a pass and never a failure: G4 is opt-in and needs a
+ * model, so a machine without one must not be blocked from publishing — but the
+ * record has to say the check did not happen rather than leaving a silence that
+ * reads like a green.
+ */
+export declare function triggerFindings(outcome: TriggerEvalOutcome): GateFinding[];
+//# sourceMappingURL=trigger-eval.d.ts.map

package/dist/core/extensions/plugins/trigger-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trigger-eval.d.ts","sourceRoot":"","sources":["../../../../src/core/extensions/plugins/trigger-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAEpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtD,4FAA4F;AAC5F,eAAO,MAAM,iBAAiB,QAAqC,CAAC;AAEpE,MAAM,WAAW,WAAW;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,mFAAmF;IACnF,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,GAAG,EAAE,OAAO,CAAC;CACb;AAED,MAAM,WAAW,mBAAmB;IACnC,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,oFAAoF;AACpF,MAAM,MAAM,mBAAmB,GAAG,MAAM,GAAG,IAAI,CAAC;AAEhD;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,mBAAmB,KAAK,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;AAE5F,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IACjC,mEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,oFAAoF;IACpF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,iBAAiB,EAAE,CAAC;CAC7B;AAED,MAAM,MAAM,kBAAkB,GAAG;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAOtH,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,GAAG,SAAS,CAkBxE;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,CAI1E;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,aAAa,EAAE,GAAG,gBAAgB,EAAE,CAIxG;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,SAAS,gBAAgB,EAAE,EAAE,KAAK,EAAE,SAAS,WAAW,EAAE,GAAG,MAAM,CAOhH;AAMD;;;;;;;GAOG;AACH,wBAAsB,cAAc,CACnC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,SAAS,gBAAgB,EAAE,EACvC,KAAK,EAAE,SAAS,WAAW,EAAE,GAAG,SAAS,EACzC,KAAK,EAAE,YAAY,GAAG,SAAS,GAC7B,OAAO,CAAC,kBAAkB,CAAC,CA4D7B;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAsC1E","sourcesContent":["/**\n * G4 — trigger eval: does an authored skill fire on the situations it is for,\n * and stay quiet otherwise?\n *\n * G1–G3 ask whether a plugin is well-formed, safe, and runnable. None of them\n * ask whether it is any *good*, and for a passive capability the whole of \"good\"\n * is the `description`: it is the only thing the model sees when deciding\n * whether to reach for the skill. A skill that parses, validates, and never\n * triggers is dead weight that costs context on every request.\n *\n * ## The circularity trap\n *\n * The tempting design is to generate test prompts from the description and then\n * check that the description matches them. That measures self-consistency and\n * always passes. So gold prompts are **supplied**, not generated (`eval/triggers.json`\n * in the plugin, or authored alongside it), and the negatives largely come for\n * free: a prompt whose right answer is a *different* capability is a negative for\n * this one. Discriminating against real siblings is the question that matters —\n * \"does it fire\" is easy, \"does it fire instead of the wrong thing\" is not.\n *\n * ## Reuse of the search eval harness\n *\n * §4.2 asks G4 to reuse `core/search/eval-harness.ts`. Its three *rules* are\n * adopted verbatim below and are the reason this module looks the way it does:\n * pin what you measured, record it machine-readably, and never let a degraded\n * run read like a real one. Its *machinery* could not be called: that harness\n * pins a corpus by checking out a git worktree and stamps provenance with\n * embsearch daemon state, and G4's corpus is a handful of description strings\n * with no repo and no embedder. Calling it would have meant a git worktree per\n * skill eval. The pinning here is therefore a content hash of exactly what was\n * judged — same guarantee, appropriate mechanism.\n *\n * See docs/plugin-system-architecture.md §4.2.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readFileSync } from \"node:fs\";\nimport * as path from \"node:path\";\nimport type { CapabilityDoc } from \"../../capabilities/registry.js\";\nimport { loadSkillsFromDir } from \"../../skills.js\";\nimport type { GateFinding } from \"./gates.js\";\nimport type { NormalizedPlugin } from \"./manifest.js\";\n\n/** Where a plugin declares the prompts its capabilities should (and should not) fire on. */\nexport const TRIGGER_GOLD_FILE = path.join(\"eval\", \"triggers.json\");\n\nexport interface TriggerCase {\n\tprompt: string;\n\t/** Capability name this prompt should select, or null when nothing should fire. */\n\texpect: string | null;\n}\n\n/** One capability offered to the judge as a candidate. */\nexport interface TriggerCandidate {\n\tname: string;\n\tdescription: string;\n\t/** False for the distractors drawn from other plugins. */\n\town: boolean;\n}\n\nexport interface TriggerJudgeRequest {\n\tcandidates: TriggerCandidate[];\n\tprompts: string[];\n}\n\n/** The judge's answer per prompt: the selected candidate name, or null for none. */\nexport type TriggerJudgeVerdict = string | null;\n\n/**\n * The single model call, injected.\n *\n * Injected rather than imported so the scoring, the gold-set handling and the\n * gate mapping are testable without a model — and so a caller with no model\n * configured gets an honest `not-run` instead of a gate that quietly passes.\n */\nexport type TriggerJudge = (request: TriggerJudgeRequest) => Promise<TriggerJudgeVerdict[]>;\n\nexport interface TriggerCaseResult {\n\tprompt: string;\n\texpected: string | null;\n\tactual: string | null;\n\tcorrect: boolean;\n}\n\nexport interface TriggerEvalRecord {\n\t/** Content hash of the candidates and cases judged — the pin. */\n\tcorpusHash: string;\n\ttimestampMs: number;\n\tpluginId: string;\n\tcandidateCount: number;\n\tcaseCount: number;\n\t/** Fires when it should: TP / (TP + FN). Undefined when there were no positives. */\n\trecall?: number;\n\t/** Quiet when it should be: TN / (TN + FP). Undefined when there were no negatives. */\n\tspecificity?: number;\n\tresults: TriggerCaseResult[];\n}\n\nexport type TriggerEvalOutcome = { status: \"not-run\"; reason: string } | { status: \"ran\"; record: TriggerEvalRecord };\n\n/** Recall below this is an error: a skill that does not fire is not a capability. */\nconst MIN_RECALL = 0.8;\n/** Specificity below this is a warning: over-firing costs context but breaks nothing. */\nconst MIN_SPECIFICITY = 0.75;\n\n/** Read the plugin's declared gold set, or undefined when it has none. */\nexport function loadTriggerCases(root: string): TriggerCase[] | undefined {\n\tconst file = path.join(root, TRIGGER_GOLD_FILE);\n\tif (!existsSync(file)) return undefined;\n\ttry {\n\t\tconst raw = JSON.parse(readFileSync(file, \"utf8\")) as { cases?: unknown };\n\t\tif (!Array.isArray(raw.cases)) return undefined;\n\t\tconst cases = raw.cases.filter(\n\t\t\t(c): c is TriggerCase =>\n\t\t\t\t!!c &&\n\t\t\t\ttypeof c === \"object\" &&\n\t\t\t\ttypeof (c as TriggerCase).prompt === \"string\" &&\n\t\t\t\t(c as TriggerCase).prompt.trim().length > 0 &&\n\t\t\t\t((c as TriggerCase).expect === null || typeof (c as TriggerCase).expect === \"string\"),\n\t\t);\n\t\treturn cases.length > 0 ? cases : undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * The plugin's own passive capabilities, as judge candidates.\n *\n * Skills only. Commands are invoked by name (`/foo`) rather than selected from a\n * description, so \"does the description fire\" is not a question about them, and\n * scoring them would dilute the metric with cases that cannot fail.\n */\nexport function ownCandidates(plugin: NormalizedPlugin): TriggerCandidate[] {\n\tif (!plugin.skillsDir) return [];\n\tconst { skills } = loadSkillsFromDir({ dir: plugin.skillsDir, source: plugin.id });\n\treturn skills.map((s) => ({ name: s.name, description: s.description ?? \"\", own: true }));\n}\n\n/**\n * Sibling capabilities to put in front of the judge alongside the plugin's own.\n *\n * Without them the judge picks from a list of one and \"does it fire\" is nearly\n * free. The interesting failure is a description broad enough to win prompts\n * that belong to something else, and that only shows up against real competition.\n */\nexport function distractorCandidates(exclude: string, docs: readonly CapabilityDoc[]): TriggerCandidate[] {\n\treturn docs\n\t\t.filter((d) => (d.kind === \"skill\" || d.kind === \"mcp-tool\") && d.source !== exclude)\n\t\t.map((d) => ({ name: d.name, description: d.description, own: false }));\n}\n\n/**\n * Pin: a hash of exactly what was judged.\n *\n * Over candidates *and* cases, because both move the number. A rerun after a\n * description edit is measuring something else, and a record that cannot say so\n * is the failure mode the search harness was built to prevent.\n */\nexport function triggerCorpusHash(candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[]): string {\n\tconst h = createHash(\"sha256\");\n\tfor (const c of [...candidates].sort((a, b) => a.name.localeCompare(b.name))) {\n\t\th.update(`${c.name}\u0000${c.description}\u0000`);\n\t}\n\tfor (const c of cases) h.update(`${c.prompt}\u0000${c.expect ?? \"\"}\u0000`);\n\treturn h.digest(\"hex\").slice(0, 16);\n}\n\nfunction ratio(correct: number, total: number): number | undefined {\n\treturn total > 0 ? correct / total : undefined;\n}\n\n/**\n * Score a plugin's passive capabilities against its gold set.\n *\n * Never throws and never fabricates: no judge, no gold set, or no candidates all\n * return `not-run` with the reason. A judge that answers the wrong number of\n * prompts is also `not-run` — a partial alignment between prompts and verdicts\n * would silently score the wrong pairs.\n */\nexport async function runTriggerEval(\n\tpluginId: string,\n\tcandidates: readonly TriggerCandidate[],\n\tcases: readonly TriggerCase[] | undefined,\n\tjudge: TriggerJudge | undefined,\n): Promise<TriggerEvalOutcome> {\n\tif (!judge) return { status: \"not-run\", reason: \"no trigger judge configured (G4 needs a model)\" };\n\tif (!cases || cases.length === 0) {\n\t\treturn {\n\t\t\tstatus: \"not-run\",\n\t\t\treason: `no gold set: add ${TRIGGER_GOLD_FILE} with the prompts this plugin should and should not fire on`,\n\t\t};\n\t}\n\tif (candidates.length === 0) return { status: \"not-run\", reason: \"plugin has no passive capabilities to evaluate\" };\n\n\tconst prompts = cases.map((c) => c.prompt);\n\tlet verdicts: TriggerJudgeVerdict[];\n\ttry {\n\t\tverdicts = await judge({ candidates: [...candidates], prompts });\n\t} catch (error) {\n\t\treturn { status: \"not-run\", reason: `judge failed: ${(error as Error).message}` };\n\t}\n\tif (verdicts.length !== prompts.length) {\n\t\treturn {\n\t\t\tstatus: \"not-run\",\n\t\t\treason: `judge returned ${verdicts.length} verdict(s) for ${prompts.length} prompt(s)`,\n\t\t};\n\t}\n\n\tconst own = new Set(candidates.filter((c) => c.own).map((c) => c.name));\n\tlet truePos = 0;\n\tlet positives = 0;\n\tlet trueNeg = 0;\n\tlet negatives = 0;\n\n\tconst results: TriggerCaseResult[] = cases.map((c, i) => {\n\t\tconst actual = verdicts[i] ?? null;\n\t\tconst correct = actual === c.expect;\n\t\t// A \"positive\" is a case this plugin should win. A case expecting a\n\t\t// sibling's capability is a negative *for us* — which is the discriminative\n\t\t// half, and why distractors are in the candidate list at all.\n\t\tif (c.expect !== null && own.has(c.expect)) {\n\t\t\tpositives++;\n\t\t\tif (correct) truePos++;\n\t\t} else {\n\t\t\tnegatives++;\n\t\t\t// Quiet means: did not claim one of ours.\n\t\t\tif (actual === null || !own.has(actual)) trueNeg++;\n\t\t}\n\t\treturn { prompt: c.prompt, expected: c.expect, actual, correct };\n\t});\n\n\treturn {\n\t\tstatus: \"ran\",\n\t\trecord: {\n\t\t\tcorpusHash: triggerCorpusHash(candidates, cases),\n\t\t\ttimestampMs: Date.now(),\n\t\t\tpluginId,\n\t\t\tcandidateCount: candidates.length,\n\t\t\tcaseCount: cases.length,\n\t\t\trecall: ratio(truePos, positives),\n\t\t\tspecificity: ratio(trueNeg, negatives),\n\t\t\tresults,\n\t\t},\n\t};\n}\n\n/**\n * Map an outcome onto gate findings.\n *\n * `not-run` is `info`, never a pass and never a failure: G4 is opt-in and needs a\n * model, so a machine without one must not be blocked from publishing — but the\n * record has to say the check did not happen rather than leaving a silence that\n * reads like a green.\n */\nexport function triggerFindings(outcome: TriggerEvalOutcome): GateFinding[] {\n\tif (outcome.status === \"not-run\") {\n\t\treturn [{ gate: \"G4\", severity: \"info\", message: `G4 trigger eval did not run — ${outcome.reason}` }];\n\t}\n\tconst { recall, specificity, caseCount } = outcome.record;\n\tconst findings: GateFinding[] = [];\n\tconst pct = (v: number) => `${Math.round(v * 100)}%`;\n\n\tif (recall !== undefined && recall < MIN_RECALL) {\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"error\",\n\t\t\tmessage:\n\t\t\t\t`Skills fired on only ${pct(recall)} of the prompts they are for (need ${pct(MIN_RECALL)}). ` +\n\t\t\t\t\"Rewrite the description around the situation that should trigger it, not the mechanism.\",\n\t\t});\n\t}\n\tif (specificity !== undefined && specificity < MIN_SPECIFICITY) {\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"warning\",\n\t\t\tmessage:\n\t\t\t\t`Skills also fired on ${pct(1 - specificity)} of the prompts they should have stayed quiet on ` +\n\t\t\t\t`(need ${pct(MIN_SPECIFICITY)} specificity). An over-broad description costs context on every request.`,\n\t\t});\n\t}\n\tif (findings.length === 0) {\n\t\tconst parts = [\n\t\t\trecall !== undefined ? `recall ${pct(recall)}` : undefined,\n\t\t\tspecificity !== undefined ? `specificity ${pct(specificity)}` : undefined,\n\t\t].filter(Boolean);\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"info\",\n\t\t\tmessage: `G4 trigger eval passed over ${caseCount} case(s): ${parts.join(\", \")}.`,\n\t\t});\n\t}\n\treturn findings;\n}\n"]}

package/dist/core/extensions/plugins/trigger-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trigger-eval.js","sourceRoot":"","sources":["../../../../src/core/extensions/plugins/trigger-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAIpD,4FAA4F;AAC5F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;AAwDpE,qFAAqF;AACrF,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,yFAAyF;AACzF,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,0EAA0E;AAC1E,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAA6B;IACzE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IAChD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QAChD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAoB,EAAE,CACvB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,KAAK,QAAQ;YACrB,OAAQ,CAAiB,CAAC,MAAM,KAAK,QAAQ;YAC5C,CAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;YAC3C,CAAE,CAAiB,CAAC,MAAM,KAAK,IAAI,IAAI,OAAQ,CAAiB,CAAC,MAAM,KAAK,QAAQ,CAAC,CACtF,CAAC;QACF,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,MAAwB,EAAsB;IAC3E,IAAI,CAAC,MAAM,CAAC,SAAS;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IACnF,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAAA,CAC1F;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe,EAAE,IAA8B,EAAsB;IACzG,OAAO,IAAI;SACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC;SACpF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AAAA,CACzE;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,UAAuC,EAAE,KAA6B,EAAU;IACjH,MAAM,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9E,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,GAAG,CAAC,CAAC;IAClE,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACpC;AAED,SAAS,KAAK,CAAC,OAAe,EAAE,KAAa,EAAsB;IAClE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CAC/C;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,QAAgB,EAChB,UAAuC,EACvC,KAAyC,EACzC,KAA+B,EACD;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gDAAgD,EAAE,CAAC;IACnG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,OAAO;YACN,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,oBAAoB,iBAAiB,6DAA6D;SAC1G,CAAC;IACH,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gDAAgD,EAAE,CAAC;IAEpH,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,QAA+B,CAAC;IACpC,IAAI,CAAC;QACJ,QAAQ,GAAG,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,iBAAkB,KAAe,CAAC,OAAO,EAAE,EAAE,CAAC;IACnF,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;QACxC,OAAO;YACN,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,kBAAkB,QAAQ,CAAC,MAAM,mBAAmB,OAAO,CAAC,MAAM,YAAY;SACtF,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACxE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,OAAO,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC;QACpC,oEAAoE;QACpE,8EAA4E;QAC5E,8DAA8D;QAC9D,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,SAAS,EAAE,CAAC;YACZ,IAAI,OAAO;gBAAE,OAAO,EAAE,CAAC;QACxB,CAAC;aAAM,CAAC;YACP,SAAS,EAAE,CAAC;YACZ,0CAA0C;YAC1C,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,CAAC;QACpD,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAAA,CACjE,CAAC,CAAC;IAEH,OAAO;QACN,MAAM,EAAE,KAAK;QACb,MAAM,EAAE;YACP,UAAU,EAAE,iBAAiB,CAAC,UAAU,EAAE,KAAK,CAAC;YAChD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;YACvB,QAAQ;YACR,cAAc,EAAE,UAAU,CAAC,MAAM;YACjC,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC;YACjC,WAAW,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC;YACtC,OAAO;SACP;KACD,CAAC;AAAA,CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAiB;IAC3E,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,mCAAiC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAC1D,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;IAErD,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,GAAG,UAAU,EAAE,CAAC;QACjD,QAAQ,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,OAAO;YACjB,OAAO,EACN,wBAAwB,GAAG,CAAC,MAAM,CAAC,sCAAsC,GAAG,CAAC,UAAU,CAAC,KAAK;gBAC7F,yFAAyF;SAC1F,CAAC,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,eAAe,EAAE,CAAC;QAChE,QAAQ,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,SAAS;YACnB,OAAO,EACN,wBAAwB,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,mDAAmD;gBAC/F,SAAS,GAAG,CAAC,eAAe,CAAC,0EAA0E;SACxG,CAAC,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG;YACb,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;YAC1D,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;SACzE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,+BAA+B,SAAS,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;SACjF,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAAA,CAChB","sourcesContent":["/**\n * G4 — trigger eval: does an authored skill fire on the situations it is for,\n * and stay quiet otherwise?\n *\n * G1–G3 ask whether a plugin is well-formed, safe, and runnable. None of them\n * ask whether it is any *good*, and for a passive capability the whole of \"good\"\n * is the `description`: it is the only thing the model sees when deciding\n * whether to reach for the skill. A skill that parses, validates, and never\n * triggers is dead weight that costs context on every request.\n *\n * ## The circularity trap\n *\n * The tempting design is to generate test prompts from the description and then\n * check that the description matches them. That measures self-consistency and\n * always passes. So gold prompts are **supplied**, not generated (`eval/triggers.json`\n * in the plugin, or authored alongside it), and the negatives largely come for\n * free: a prompt whose right answer is a *different* capability is a negative for\n * this one. Discriminating against real siblings is the question that matters —\n * \"does it fire\" is easy, \"does it fire instead of the wrong thing\" is not.\n *\n * ## Reuse of the search eval harness\n *\n * §4.2 asks G4 to reuse `core/search/eval-harness.ts`. Its three *rules* are\n * adopted verbatim below and are the reason this module looks the way it does:\n * pin what you measured, record it machine-readably, and never let a degraded\n * run read like a real one. Its *machinery* could not be called: that harness\n * pins a corpus by checking out a git worktree and stamps provenance with\n * embsearch daemon state, and G4's corpus is a handful of description strings\n * with no repo and no embedder. Calling it would have meant a git worktree per\n * skill eval. The pinning here is therefore a content hash of exactly what was\n * judged — same guarantee, appropriate mechanism.\n *\n * See docs/plugin-system-architecture.md §4.2.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readFileSync } from \"node:fs\";\nimport * as path from \"node:path\";\nimport type { CapabilityDoc } from \"../../capabilities/registry.js\";\nimport { loadSkillsFromDir } from \"../../skills.js\";\nimport type { GateFinding } from \"./gates.js\";\nimport type { NormalizedPlugin } from \"./manifest.js\";\n\n/** Where a plugin declares the prompts its capabilities should (and should not) fire on. */\nexport const TRIGGER_GOLD_FILE = path.join(\"eval\", \"triggers.json\");\n\nexport interface TriggerCase {\n\tprompt: string;\n\t/** Capability name this prompt should select, or null when nothing should fire. */\n\texpect: string | null;\n}\n\n/** One capability offered to the judge as a candidate. */\nexport interface TriggerCandidate {\n\tname: string;\n\tdescription: string;\n\t/** False for the distractors drawn from other plugins. */\n\town: boolean;\n}\n\nexport interface TriggerJudgeRequest {\n\tcandidates: TriggerCandidate[];\n\tprompts: string[];\n}\n\n/** The judge's answer per prompt: the selected candidate name, or null for none. */\nexport type TriggerJudgeVerdict = string | null;\n\n/**\n * The single model call, injected.\n *\n * Injected rather than imported so the scoring, the gold-set handling and the\n * gate mapping are testable without a model — and so a caller with no model\n * configured gets an honest `not-run` instead of a gate that quietly passes.\n */\nexport type TriggerJudge = (request: TriggerJudgeRequest) => Promise<TriggerJudgeVerdict[]>;\n\nexport interface TriggerCaseResult {\n\tprompt: string;\n\texpected: string | null;\n\tactual: string | null;\n\tcorrect: boolean;\n}\n\nexport interface TriggerEvalRecord {\n\t/** Content hash of the candidates and cases judged — the pin. */\n\tcorpusHash: string;\n\ttimestampMs: number;\n\tpluginId: string;\n\tcandidateCount: number;\n\tcaseCount: number;\n\t/** Fires when it should: TP / (TP + FN). Undefined when there were no positives. */\n\trecall?: number;\n\t/** Quiet when it should be: TN / (TN + FP). Undefined when there were no negatives. */\n\tspecificity?: number;\n\tresults: TriggerCaseResult[];\n}\n\nexport type TriggerEvalOutcome = { status: \"not-run\"; reason: string } | { status: \"ran\"; record: TriggerEvalRecord };\n\n/** Recall below this is an error: a skill that does not fire is not a capability. */\nconst MIN_RECALL = 0.8;\n/** Specificity below this is a warning: over-firing costs context but breaks nothing. */\nconst MIN_SPECIFICITY = 0.75;\n\n/** Read the plugin's declared gold set, or undefined when it has none. */\nexport function loadTriggerCases(root: string): TriggerCase[] | undefined {\n\tconst file = path.join(root, TRIGGER_GOLD_FILE);\n\tif (!existsSync(file)) return undefined;\n\ttry {\n\t\tconst raw = JSON.parse(readFileSync(file, \"utf8\")) as { cases?: unknown };\n\t\tif (!Array.isArray(raw.cases)) return undefined;\n\t\tconst cases = raw.cases.filter(\n\t\t\t(c): c is TriggerCase =>\n\t\t\t\t!!c &&\n\t\t\t\ttypeof c === \"object\" &&\n\t\t\t\ttypeof (c as TriggerCase).prompt === \"string\" &&\n\t\t\t\t(c as TriggerCase).prompt.trim().length > 0 &&\n\t\t\t\t((c as TriggerCase).expect === null || typeof (c as TriggerCase).expect === \"string\"),\n\t\t);\n\t\treturn cases.length > 0 ? cases : undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * The plugin's own passive capabilities, as judge candidates.\n *\n * Skills only. Commands are invoked by name (`/foo`) rather than selected from a\n * description, so \"does the description fire\" is not a question about them, and\n * scoring them would dilute the metric with cases that cannot fail.\n */\nexport function ownCandidates(plugin: NormalizedPlugin): TriggerCandidate[] {\n\tif (!plugin.skillsDir) return [];\n\tconst { skills } = loadSkillsFromDir({ dir: plugin.skillsDir, source: plugin.id });\n\treturn skills.map((s) => ({ name: s.name, description: s.description ?? \"\", own: true }));\n}\n\n/**\n * Sibling capabilities to put in front of the judge alongside the plugin's own.\n *\n * Without them the judge picks from a list of one and \"does it fire\" is nearly\n * free. The interesting failure is a description broad enough to win prompts\n * that belong to something else, and that only shows up against real competition.\n */\nexport function distractorCandidates(exclude: string, docs: readonly CapabilityDoc[]): TriggerCandidate[] {\n\treturn docs\n\t\t.filter((d) => (d.kind === \"skill\" || d.kind === \"mcp-tool\") && d.source !== exclude)\n\t\t.map((d) => ({ name: d.name, description: d.description, own: false }));\n}\n\n/**\n * Pin: a hash of exactly what was judged.\n *\n * Over candidates *and* cases, because both move the number. A rerun after a\n * description edit is measuring something else, and a record that cannot say so\n * is the failure mode the search harness was built to prevent.\n */\nexport function triggerCorpusHash(candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[]): string {\n\tconst h = createHash(\"sha256\");\n\tfor (const c of [...candidates].sort((a, b) => a.name.localeCompare(b.name))) {\n\t\th.update(`${c.name}\u0000${c.description}\u0000`);\n\t}\n\tfor (const c of cases) h.update(`${c.prompt}\u0000${c.expect ?? \"\"}\u0000`);\n\treturn h.digest(\"hex\").slice(0, 16);\n}\n\nfunction ratio(correct: number, total: number): number | undefined {\n\treturn total > 0 ? correct / total : undefined;\n}\n\n/**\n * Score a plugin's passive capabilities against its gold set.\n *\n * Never throws and never fabricates: no judge, no gold set, or no candidates all\n * return `not-run` with the reason. A judge that answers the wrong number of\n * prompts is also `not-run` — a partial alignment between prompts and verdicts\n * would silently score the wrong pairs.\n */\nexport async function runTriggerEval(\n\tpluginId: string,\n\tcandidates: readonly TriggerCandidate[],\n\tcases: readonly TriggerCase[] | undefined,\n\tjudge: TriggerJudge | undefined,\n): Promise<TriggerEvalOutcome> {\n\tif (!judge) return { status: \"not-run\", reason: \"no trigger judge configured (G4 needs a model)\" };\n\tif (!cases || cases.length === 0) {\n\t\treturn {\n\t\t\tstatus: \"not-run\",\n\t\t\treason: `no gold set: add ${TRIGGER_GOLD_FILE} with the prompts this plugin should and should not fire on`,\n\t\t};\n\t}\n\tif (candidates.length === 0) return { status: \"not-run\", reason: \"plugin has no passive capabilities to evaluate\" };\n\n\tconst prompts = cases.map((c) => c.prompt);\n\tlet verdicts: TriggerJudgeVerdict[];\n\ttry {\n\t\tverdicts = await judge({ candidates: [...candidates], prompts });\n\t} catch (error) {\n\t\treturn { status: \"not-run\", reason: `judge failed: ${(error as Error).message}` };\n\t}\n\tif (verdicts.length !== prompts.length) {\n\t\treturn {\n\t\t\tstatus: \"not-run\",\n\t\t\treason: `judge returned ${verdicts.length} verdict(s) for ${prompts.length} prompt(s)`,\n\t\t};\n\t}\n\n\tconst own = new Set(candidates.filter((c) => c.own).map((c) => c.name));\n\tlet truePos = 0;\n\tlet positives = 0;\n\tlet trueNeg = 0;\n\tlet negatives = 0;\n\n\tconst results: TriggerCaseResult[] = cases.map((c, i) => {\n\t\tconst actual = verdicts[i] ?? null;\n\t\tconst correct = actual === c.expect;\n\t\t// A \"positive\" is a case this plugin should win. A case expecting a\n\t\t// sibling's capability is a negative *for us* — which is the discriminative\n\t\t// half, and why distractors are in the candidate list at all.\n\t\tif (c.expect !== null && own.has(c.expect)) {\n\t\t\tpositives++;\n\t\t\tif (correct) truePos++;\n\t\t} else {\n\t\t\tnegatives++;\n\t\t\t// Quiet means: did not claim one of ours.\n\t\t\tif (actual === null || !own.has(actual)) trueNeg++;\n\t\t}\n\t\treturn { prompt: c.prompt, expected: c.expect, actual, correct };\n\t});\n\n\treturn {\n\t\tstatus: \"ran\",\n\t\trecord: {\n\t\t\tcorpusHash: triggerCorpusHash(candidates, cases),\n\t\t\ttimestampMs: Date.now(),\n\t\t\tpluginId,\n\t\t\tcandidateCount: candidates.length,\n\t\t\tcaseCount: cases.length,\n\t\t\trecall: ratio(truePos, positives),\n\t\t\tspecificity: ratio(trueNeg, negatives),\n\t\t\tresults,\n\t\t},\n\t};\n}\n\n/**\n * Map an outcome onto gate findings.\n *\n * `not-run` is `info`, never a pass and never a failure: G4 is opt-in and needs a\n * model, so a machine without one must not be blocked from publishing — but the\n * record has to say the check did not happen rather than leaving a silence that\n * reads like a green.\n */\nexport function triggerFindings(outcome: TriggerEvalOutcome): GateFinding[] {\n\tif (outcome.status === \"not-run\") {\n\t\treturn [{ gate: \"G4\", severity: \"info\", message: `G4 trigger eval did not run — ${outcome.reason}` }];\n\t}\n\tconst { recall, specificity, caseCount } = outcome.record;\n\tconst findings: GateFinding[] = [];\n\tconst pct = (v: number) => `${Math.round(v * 100)}%`;\n\n\tif (recall !== undefined && recall < MIN_RECALL) {\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"error\",\n\t\t\tmessage:\n\t\t\t\t`Skills fired on only ${pct(recall)} of the prompts they are for (need ${pct(MIN_RECALL)}). ` +\n\t\t\t\t\"Rewrite the description around the situation that should trigger it, not the mechanism.\",\n\t\t});\n\t}\n\tif (specificity !== undefined && specificity < MIN_SPECIFICITY) {\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"warning\",\n\t\t\tmessage:\n\t\t\t\t`Skills also fired on ${pct(1 - specificity)} of the prompts they should have stayed quiet on ` +\n\t\t\t\t`(need ${pct(MIN_SPECIFICITY)} specificity). An over-broad description costs context on every request.`,\n\t\t});\n\t}\n\tif (findings.length === 0) {\n\t\tconst parts = [\n\t\t\trecall !== undefined ? `recall ${pct(recall)}` : undefined,\n\t\t\tspecificity !== undefined ? `specificity ${pct(specificity)}` : undefined,\n\t\t].filter(Boolean);\n\t\tfindings.push({\n\t\t\tgate: \"G4\",\n\t\t\tseverity: \"info\",\n\t\t\tmessage: `G4 trigger eval passed over ${caseCount} case(s): ${parts.join(\", \")}.`,\n\t\t});\n\t}\n\treturn findings;\n}\n"]}

package/dist/core/package-manager.d.ts

@@ -4,6 +4,12 @@ export interface PathMetadata {
     scope: SourceScope;
     origin: "package" | "top-level" | "claude-code";
     baseDir?: string;
+    /**
+     * Plugin id owning the resources under this path. Skills loaded from it are
+     * namespaced `<namespace>:<name>`, matching Claude Code, so two plugins
+     * shipping a `review` skill do not silently collide.
+     */
+    namespace?: string;
 }
 export interface ResolvedResource {
     path: string;