@provos/ironcurtain 0.1.0

package/dist/pipeline/llm-logger.d.ts

@@ -0,0 +1,42 @@
+/**
+ * LLM Interaction Logger -- AI SDK middleware that captures all LLM
+ * prompts and responses during the policy compilation pipeline.
+ *
+ * Uses `wrapLanguageModel()` with a custom middleware that intercepts
+ * every `doGenerate` call. The caller sets `context.stepName` before
+ * each pipeline phase so logs are labeled without changing module APIs.
+ *
+ * Writes a single JSONL file with one entry per LLM call.
+ */
+import type { LanguageModelMiddleware } from 'ai';
+export interface LlmLogEntry {
+    timestamp: string;
+    stepName: string;
+    modelId: string;
+    /**
+     * For the first entry (promptOffset === 0), this is the full prompt array.
+     * For subsequent entries, this contains only new messages since the last
+     * logged entry. Reconstruct the full prompt by concatenating all entries'
+     * prompt arrays in order.
+     */
+    prompt: unknown;
+    /** Index into the full prompt array where this entry's messages start. */
+    promptOffset: number;
+    responseText: string;
+    usage: {
+        inputTokens: number | undefined;
+        outputTokens: number | undefined;
+    };
+    durationMs: number;
+}
+export interface LlmLogContext {
+    stepName: string;
+}
+/**
+ * Creates an AI SDK middleware that logs every generate call to a JSONL file.
+ *
+ * @param logPath - Absolute path to the output JSONL file.
+ * @param context - Mutable context object. Set `context.stepName` before each
+ *   pipeline phase to label the log entries.
+ */
+export declare function createLlmLoggingMiddleware(logPath: string, context: LlmLogContext): LanguageModelMiddleware;

package/dist/pipeline/llm-logger.js

@@ -0,0 +1,78 @@
+/**
+ * LLM Interaction Logger -- AI SDK middleware that captures all LLM
+ * prompts and responses during the policy compilation pipeline.
+ *
+ * Uses `wrapLanguageModel()` with a custom middleware that intercepts
+ * every `doGenerate` call. The caller sets `context.stepName` before
+ * each pipeline phase so logs are labeled without changing module APIs.
+ *
+ * Writes a single JSONL file with one entry per LLM call.
+ */
+import { writeFileSync, appendFileSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+// ---------------------------------------------------------------------------
+// Middleware factory
+// ---------------------------------------------------------------------------
+/**
+ * Creates an AI SDK middleware that logs every generate call to a JSONL file.
+ *
+ * @param logPath - Absolute path to the output JSONL file.
+ * @param context - Mutable context object. Set `context.stepName` before each
+ *   pipeline phase to label the log entries.
+ */
+export function createLlmLoggingMiddleware(logPath, context) {
+    initLogFile(logPath);
+    // Track how many prompt items were logged so far to enable delta logging.
+    let previousPromptLength = 0;
+    return {
+        specificationVersion: 'v3',
+        wrapGenerate: async ({ doGenerate, params, model }) => {
+            const start = Date.now();
+            const result = await doGenerate();
+            const durationMs = Date.now() - start;
+            const responseText = extractTextFromContent(result.content);
+            // Only log new messages since the last entry (delta logging).
+            const fullPrompt = params.prompt;
+            const promptArray = Array.isArray(fullPrompt) ? fullPrompt : [fullPrompt];
+            const promptOffset = previousPromptLength;
+            const newMessages = promptArray.slice(promptOffset);
+            previousPromptLength = promptArray.length;
+            const entry = {
+                timestamp: new Date().toISOString(),
+                stepName: context.stepName,
+                modelId: model.modelId,
+                prompt: newMessages,
+                promptOffset,
+                responseText,
+                usage: {
+                    inputTokens: result.usage?.inputTokens?.total,
+                    outputTokens: result.usage?.outputTokens?.total,
+                },
+                durationMs,
+            };
+            appendLogEntry(logPath, entry);
+            return result;
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// File I/O helpers
+// ---------------------------------------------------------------------------
+function initLogFile(logPath) {
+    mkdirSync(dirname(logPath), { recursive: true });
+    // Truncate any existing file from a previous run
+    writeFileSync(logPath, '');
+}
+function appendLogEntry(logPath, entry) {
+    appendFileSync(logPath, JSON.stringify(entry) + '\n');
+}
+// ---------------------------------------------------------------------------
+// Content extraction
+// ---------------------------------------------------------------------------
+function extractTextFromContent(content) {
+    return content
+        .filter((part) => part.type === 'text' && typeof part.text === 'string')
+        .map(part => part.text)
+        .join('');
+}
+//# sourceMappingURL=llm-logger.js.map

package/dist/pipeline/llm-logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-logger.js","sourceRoot":"","sources":["../../src/pipeline/llm-logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgCpC,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,UAAU,0BAA0B,CACxC,OAAe,EACf,OAAsB;IAEtB,WAAW,CAAC,OAAO,CAAC,CAAC;IAErB,0EAA0E;IAC1E,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAE7B,OAAO;QACL,oBAAoB,EAAE,IAAI;QAC1B,YAAY,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAC;YAClC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YAEtC,MAAM,YAAY,GAAG,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAE5D,8DAA8D;YAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;YACjC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAC1E,MAAM,YAAY,GAAG,oBAAoB,CAAC;YAC1C,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACpD,oBAAoB,GAAG,WAAW,CAAC,MAAM,CAAC;YAE1C,MAAM,KAAK,GAAgB;gBACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,MAAM,EAAE,WAAW;gBACnB,YAAY;gBACZ,YAAY;gBACZ,KAAK,EAAE;oBACL,WAAW,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK;oBAC7C,YAAY,EAAE,MAAM,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK;iBAChD;gBACD,UAAU;aACX,CAAC;YAEF,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC/B,OAAO,MAAM,CAAC;QAChB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,WAAW,CAAC,OAAe;IAClC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,iDAAiD;IACjD,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,cAAc,CAAC,OAAe,EAAE,KAAkB;IACzD,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;AACxD,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,SAAS,sBAAsB,CAAC,OAA+C;IAC7E,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,IAAI,EAA0C,EAAE,CACvD,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CACtD;SACA,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;SACtB,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC"}

package/dist/pipeline/pipeline-shared.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Shared utilities for the policy compilation pipeline.
+ *
+ * Used by both `annotate.ts` (tool annotation) and `compile.ts`
+ * (policy compilation) to avoid duplication of config loading,
+ * content-hash caching, and spinner helpers.
+ */
+import type { LanguageModel } from 'ai';
+import { type Ora } from 'ora';
+import type { MCPServerConfig } from '../config/types.js';
+import { type LlmLogContext } from './llm-logger.js';
+export interface PipelineConfig {
+    constitutionPath: string;
+    constitutionText: string;
+    constitutionHash: string;
+    mcpServers: Record<string, MCPServerConfig>;
+    generatedDir: string;
+    packageGeneratedDir: string;
+    allowedDirectory: string;
+    protectedPaths: string[];
+}
+/**
+ * Loads the combined constitution text (base + optional user constitution).
+ * The user constitution file is at ~/.ironcurtain/constitution-user.md.
+ * When present, it is appended to the base constitution text.
+ */
+export declare function loadConstitutionText(basePath: string): string;
+export declare function loadPipelineConfig(): PipelineConfig;
+export declare function computeHash(...inputs: string[]): string;
+export declare function loadExistingArtifact<T>(generatedDir: string, filename: string, fallbackDir?: string): T | undefined;
+export declare function showCached(text: string): void;
+export declare function withSpinner<T>(text: string, fn: (spinner: Ora) => Promise<T>, successFn?: (result: T, elapsed: number) => string): Promise<{
+    result: T;
+    elapsed: number;
+}>;
+/** Writes a JSON artifact to the generated directory with consistent formatting. */
+export declare function writeArtifact(generatedDir: string, filename: string, data: unknown): void;
+export interface PipelineLlm {
+    model: LanguageModel;
+    logContext: LlmLogContext;
+    logPath: string;
+}
+/**
+ * Creates a language model wrapped with pipeline logging middleware.
+ * Both annotate.ts and compile.ts use this to set up their LLM instance.
+ */
+export declare function createPipelineLlm(generatedDir: string, initialStepName: string): Promise<PipelineLlm>;

package/dist/pipeline/pipeline-shared.js

@@ -0,0 +1,145 @@
+/**
+ * Shared utilities for the policy compilation pipeline.
+ *
+ * Used by both `annotate.ts` (tool annotation) and `compile.ts`
+ * (policy compilation) to avoid duplication of config loading,
+ * content-hash caching, and spinner helpers.
+ */
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { wrapLanguageModel } from 'ai';
+import chalk from 'chalk';
+import ora from 'ora';
+import { computeProtectedPaths } from '../config/index.js';
+import { createLanguageModel } from '../config/model-provider.js';
+import { getIronCurtainHome, getUserConstitutionPath, getUserGeneratedDir } from '../config/paths.js';
+import { loadUserConfig } from '../config/user-config.js';
+import { createLlmLoggingMiddleware } from './llm-logger.js';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+/**
+ * Loads the combined constitution text (base + optional user constitution).
+ * The user constitution file is at ~/.ironcurtain/constitution-user.md.
+ * When present, it is appended to the base constitution text.
+ */
+export function loadConstitutionText(basePath) {
+    const base = readFileSync(basePath, 'utf-8');
+    const userPath = getUserConstitutionPath();
+    if (existsSync(userPath)) {
+        const user = readFileSync(userPath, 'utf-8');
+        return `${base}\n\n${user}`;
+    }
+    return base;
+}
+export function loadPipelineConfig() {
+    const configDir = resolve(__dirname, '..', 'config');
+    const constitutionPath = resolve(configDir, 'constitution.md');
+    const constitutionText = loadConstitutionText(constitutionPath);
+    const constitutionHash = createHash('sha256').update(constitutionText).digest('hex');
+    const mcpServersPath = resolve(configDir, 'mcp-servers.json');
+    const mcpServers = JSON.parse(readFileSync(mcpServersPath, 'utf-8'));
+    const generatedDir = getUserGeneratedDir();
+    const packageGeneratedDir = resolve(configDir, 'generated');
+    const defaultAllowedDir = resolve(getIronCurtainHome(), 'sandbox');
+    const allowedDirectory = process.env.ALLOWED_DIRECTORY ?? defaultAllowedDir;
+    const auditLogPath = process.env.AUDIT_LOG_PATH ?? './audit.jsonl';
+    const protectedPaths = computeProtectedPaths({
+        constitutionPath,
+        generatedDir,
+        packageGeneratedDir,
+        mcpServersPath,
+        auditLogPath,
+    });
+    return {
+        constitutionPath,
+        constitutionText,
+        constitutionHash,
+        mcpServers,
+        generatedDir,
+        packageGeneratedDir,
+        allowedDirectory,
+        protectedPaths,
+    };
+}
+// ---------------------------------------------------------------------------
+// Content-Hash Caching
+// ---------------------------------------------------------------------------
+export function computeHash(...inputs) {
+    const hash = createHash('sha256');
+    for (const input of inputs) {
+        hash.update(input);
+    }
+    return hash.digest('hex');
+}
+export function loadExistingArtifact(generatedDir, filename, fallbackDir) {
+    const candidates = [resolve(generatedDir, filename)];
+    if (fallbackDir) {
+        candidates.push(resolve(fallbackDir, filename));
+    }
+    for (const path of candidates) {
+        if (existsSync(path)) {
+            try {
+                return JSON.parse(readFileSync(path, 'utf-8'));
+            }
+            catch {
+                // Corrupt file -- try next candidate
+            }
+        }
+    }
+    return undefined;
+}
+// ---------------------------------------------------------------------------
+// Spinner Helpers
+// ---------------------------------------------------------------------------
+export function showCached(text) {
+    const spinner = ora({ text, stream: process.stderr, discardStdin: false }).start();
+    spinner.succeed(`${text} ${chalk.dim('(cached)')}`);
+}
+export async function withSpinner(text, fn, successFn) {
+    const spinner = ora({ text, stream: process.stderr, discardStdin: false }).start();
+    const start = Date.now();
+    const timer = setInterval(() => {
+        const secs = ((Date.now() - start) / 1000).toFixed(0);
+        spinner.text = `${text} (${secs}s)`;
+    }, 1000);
+    try {
+        const result = await fn(spinner);
+        clearInterval(timer);
+        const elapsed = (Date.now() - start) / 1000;
+        const successText = successFn
+            ? successFn(result, elapsed)
+            : `${text} (${elapsed.toFixed(1)}s)`;
+        spinner.succeed(successText);
+        return { result, elapsed };
+    }
+    catch (err) {
+        clearInterval(timer);
+        spinner.fail(chalk.red(text));
+        throw err;
+    }
+}
+// ---------------------------------------------------------------------------
+// Artifact I/O
+// ---------------------------------------------------------------------------
+/** Writes a JSON artifact to the generated directory with consistent formatting. */
+export function writeArtifact(generatedDir, filename, data) {
+    mkdirSync(generatedDir, { recursive: true });
+    writeFileSync(resolve(generatedDir, filename), JSON.stringify(data, null, 2) + '\n');
+}
+/**
+ * Creates a language model wrapped with pipeline logging middleware.
+ * Both annotate.ts and compile.ts use this to set up their LLM instance.
+ */
+export async function createPipelineLlm(generatedDir, initialStepName) {
+    const userConfig = loadUserConfig();
+    const baseLlm = await createLanguageModel(userConfig.policyModelId, userConfig);
+    const logContext = { stepName: initialStepName };
+    const logPath = resolve(generatedDir, 'llm-interactions.jsonl');
+    const model = wrapLanguageModel({
+        model: baseLlm,
+        middleware: createLlmLoggingMiddleware(logPath, logContext),
+    });
+    return { model, logContext, logPath };
+}
+//# sourceMappingURL=pipeline-shared.js.map

package/dist/pipeline/pipeline-shared.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline-shared.js","sourceRoot":"","sources":["../../src/pipeline/pipeline-shared.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAiB,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEtG,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,0BAA0B,EAAsB,MAAM,iBAAiB,CAAC;AAEjF,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAiB1D;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAgB;IACnD,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;IAC3C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,GAAG,IAAI,OAAO,IAAI,EAAE,CAAC;IAC9B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IACrD,MAAM,gBAAgB,GAAG,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAC/D,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,gBAAgB,CAAC,CAAC;IAChE,MAAM,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAoC,IAAI,CAAC,KAAK,CAC5D,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CACtC,CAAC;IACF,MAAM,YAAY,GAAG,mBAAmB,EAAE,CAAC;IAC3C,MAAM,mBAAmB,GAAG,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,EAAE,EAAE,SAAS,CAAC,CAAC;IACnE,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,iBAAiB,CAAC;IAC5E,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,eAAe,CAAC;IAEnE,MAAM,cAAc,GAAG,qBAAqB,CAAC;QAC3C,gBAAgB;QAChB,YAAY;QACZ,mBAAmB;QACnB,cAAc;QACd,YAAY;KACb,CAAC,CAAC;IAEH,OAAO;QACL,gBAAgB;QAChB,gBAAgB;QAChB,gBAAgB;QAChB,UAAU;QACV,YAAY;QACZ,mBAAmB;QACnB,gBAAgB;QAChB,cAAc;KACf,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,MAAM,UAAU,WAAW,CAAC,GAAG,MAAgB;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,YAAoB,EACpB,QAAgB,EAChB,WAAoB;IAEpB,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC;IACrD,IAAI,WAAW,EAAE,CAAC;QAChB,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,qCAAqC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,OAAO,GAAG,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IACnF,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAY,EACZ,EAAgC,EAChC,SAAkD;IAElD,MAAM,OAAO,GAAG,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IACnF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtD,OAAO,CAAC,IAAI,GAAG,GAAG,IAAI,KAAK,IAAI,IAAI,CAAC;IACtC,CAAC,EAAE,IAAI,CAAC,CAAC;IACT,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC;QACjC,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC;QAC5C,MAAM,WAAW,GAAG,SAAS;YAC3B,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC;YAC5B,CAAC,CAAC,GAAG,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACvC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9B,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,oFAAoF;AACpF,MAAM,UAAU,aAAa,CAAC,YAAoB,EAAE,QAAgB,EAAE,IAAa;IACjF,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,aAAa,CACX,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CACrC,CAAC;AACJ,CAAC;AAYD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,YAAoB,EACpB,eAAuB;IAEvB,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,UAAU,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;IAChF,MAAM,UAAU,GAAkB,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC;IAChE,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,EAAE,wBAAwB,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAC9B,KAAK,EAAE,OAAO;QACd,UAAU,EAAE,0BAA0B,CAAC,OAAO,EAAE,UAAU,CAAC;KAC5D,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AACxC,CAAC"}

package/dist/pipeline/policy-verifier.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Policy Verifier -- Multi-round verification with real engine execution
+ * and LLM judge.
+ *
+ * The verifier instantiates a real PolicyEngine, runs test scenarios through
+ * it, and uses an LLM judge to analyze results, identify gaps, and generate
+ * additional probe scenarios. Up to maxRounds iterations.
+ */
+import type { LanguageModel } from 'ai';
+import { PolicyEngine } from '../trusted-process/policy-engine.js';
+import type { CompiledPolicyFile, ToolAnnotationsFile, TestScenario, ExecutionResult, VerificationResult, AttributedFailure, ScenarioCorrection } from './types.js';
+export declare function executeScenarios(engine: PolicyEngine, scenarios: TestScenario[]): ExecutionResult[];
+export declare function formatExecutionResults(results: ExecutionResult[]): string;
+export declare function verifyPolicy(constitutionText: string, compiledPolicy: CompiledPolicyFile, toolAnnotations: ToolAnnotationsFile, protectedPaths: string[], scenarios: TestScenario[], llm: LanguageModel, maxRounds?: number, allowedDirectory?: string, onProgress?: (message: string) => void): Promise<VerificationResult>;
+export interface DiscardedScenario {
+    scenario: TestScenario;
+    actual: ExecutionResult['actualDecision'];
+    rule: string;
+}
+/**
+ * Filters out scenarios whose expected decision conflicts with a structural
+ * invariant result. Structural invariants are ground truth (hardcoded in the
+ * engine), so a disagreeing scenario expectation is wrong by definition.
+ *
+ * Returns the valid scenarios and the discarded ones (with metadata for
+ * logging / feedback).
+ */
+export declare function filterStructuralConflicts(engine: PolicyEngine, scenarios: TestScenario[]): {
+    valid: TestScenario[];
+    discarded: DiscardedScenario[];
+};
+/**
+ * Extracts scenario corrections from attributed failures.
+ * Only failures blamed on 'scenario' or 'both' produce corrections.
+ * Handwritten scenarios are never auto-corrected — they are returned
+ * separately as warnings so the caller can reclassify them as rule issues.
+ */
+export declare function extractScenarioCorrections(attributedFailures: AttributedFailure[], scenarios: TestScenario[]): {
+    corrections: ScenarioCorrection[];
+    handwrittenWarnings: string[];
+};
+/**
+ * Applies corrections to a scenario list, returning a new array with
+ * updated expectedDecision and reasoning for corrected scenarios.
+ */
+export declare function applyScenarioCorrections(scenarios: TestScenario[], corrections: ScenarioCorrection[]): TestScenario[];

package/dist/pipeline/policy-verifier.js

@@ -0,0 +1,277 @@
+/**
+ * Policy Verifier -- Multi-round verification with real engine execution
+ * and LLM judge.
+ *
+ * The verifier instantiates a real PolicyEngine, runs test scenarios through
+ * it, and uses an LLM judge to analyze results, identify gaps, and generate
+ * additional probe scenarios. Up to maxRounds iterations.
+ */
+import { z } from 'zod';
+import { generateObjectWithRepair } from './generate-with-repair.js';
+import { PolicyEngine } from '../trusted-process/policy-engine.js';
+const DEFAULT_MAX_ROUNDS = 3;
+export function executeScenarios(engine, scenarios) {
+    return scenarios.map(scenario => {
+        const request = {
+            requestId: `verify-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+            serverName: scenario.request.serverName,
+            toolName: scenario.request.toolName,
+            arguments: scenario.request.arguments,
+            timestamp: new Date().toISOString(),
+        };
+        const result = engine.evaluate(request);
+        return {
+            scenario,
+            actualDecision: result.decision,
+            matchingRule: result.rule,
+            pass: result.decision === scenario.expectedDecision,
+        };
+    });
+}
+export function formatExecutionResults(results) {
+    return results.map((r, i) => {
+        const status = r.pass ? 'PASS' : 'FAIL';
+        return [
+            `[${i + 1}] ${status}: ${r.scenario.description}`,
+            `  Tool: ${r.scenario.request.serverName}/${r.scenario.request.toolName}`,
+            `  Args: ${JSON.stringify(r.scenario.request.arguments)}`,
+            `  Expected: ${r.scenario.expectedDecision}`,
+            `  Actual: ${r.actualDecision} (rule: ${r.matchingRule})`,
+            `  Source: ${r.scenario.source}`,
+        ].join('\n');
+    }).join('\n\n');
+}
+function buildJudgePrompt(constitutionText, compiledPolicy, toolAnnotations, protectedPaths, executionResults, roundNumber, previousAnalysis, availableTools) {
+    const rulesText = compiledPolicy.rules
+        .map((r, i) => `  ${i + 1}. [${r.name}] if: ${JSON.stringify(r.if)} then: ${r.then} -- ${r.reason}`)
+        .join('\n');
+    const resultsText = formatExecutionResults(executionResults);
+    const previousContext = previousAnalysis
+        ? `\n## Previous Round Analysis\n\n${previousAnalysis}\n`
+        : '';
+    return `You are a security policy verifier. Analyze whether the compiled policy correctly implements the constitution.
+
+## Constitution
+
+${constitutionText}
+
+## Compiled Policy Rules (evaluated in order, first match wins)
+
+${rulesText}
+
+## Structural Invariants (hardcoded, evaluated before compiled rules)
+
+- Unknown tools (no annotation): denied
+- Protected paths (any write/delete targeting these is automatically denied):
+${protectedPaths.map(p => `  - ${p}`).join('\n')}
+
+A \`deny\` result for write/delete operations on these paths is correct structural behavior, not a policy gap. Files with similar names inside the sandbox are NOT protected.
+
+## Execution Results (Round ${roundNumber})
+${previousContext}
+${resultsText}
+
+## Decision Types
+
+The three possible policy decisions are:
+- **allow** — the operation is explicitly permitted by the constitution
+- **deny** — the operation is categorically forbidden by the constitution (absolute prohibition)
+- **escalate** — the operation is not explicitly permitted but also not forbidden; it requires human approval
+
+When analyzing FAIL results, pay attention to whether the constitution implies "deny" vs "escalate". If the constitution does not explicitly forbid an operation, the correct decision is typically "escalate" (not "deny") so a human can make the judgment call.
+
+## Instructions
+
+1. Analyze any FAIL results. For each failure, determine the blame:
+   - **"rule"**: The compiled rule is wrong and needs to be fixed. The scenario expectation correctly reflects the constitution.
+   - **"scenario"**: The scenario expectation is wrong. The compiled rule is correct per the constitution. Provide the corrected expectedDecision and reasoning.
+   - **"both"**: The rule needs adjustment AND the scenario expectation is wrong. Provide the corrected expectedDecision and reasoning.
+2. Identify suspicious patterns (e.g., a broad allow rule shadowing a narrow deny, or "deny" used where "escalate" would be more appropriate).
+3. Identify missing coverage -- scenarios the constitution implies that were not tested.
+4. If you suspect gaps, generate additional test scenarios to probe them.
+5. Set "pass" to true ONLY if all results are correct and coverage is adequate.
+6. Return a "failureAttributions" entry for EVERY FAIL result. The scenarioDescription must exactly match the FAIL scenario's description.
+
+For additional scenarios, use concrete paths matching the directories in the compiled rules. Note: sandbox containment is handled by a structural invariant before compiled rules run — any tool call where all paths are within the sandbox directory is automatically allowed.
+
+## Available Tools
+
+IMPORTANT: Only use these exact server/tool combinations in additional scenarios. Do NOT invent tool names.
+
+${(availableTools ?? []).map(t => `- ${t.serverName}/${t.toolName}`).join('\n')}
+
+## Response Format
+
+Be concise. Keep the analysis to 2-3 sentences per issue found. Only generate additional scenarios that test genuinely untested gaps -- do not duplicate existing coverage. Limit additional scenarios to at most 5.`;
+}
+export async function verifyPolicy(constitutionText, compiledPolicy, toolAnnotations, protectedPaths, scenarios, llm, maxRounds = DEFAULT_MAX_ROUNDS, allowedDirectory, onProgress) {
+    const engine = new PolicyEngine(compiledPolicy, toolAnnotations, protectedPaths, allowedDirectory);
+    const allAnnotations = Object.values(toolAnnotations.servers).flatMap(s => s.tools);
+    const serverNamesList = [...new Set(allAnnotations.map(a => a.serverName))];
+    const toolNamesList = [...new Set(allAnnotations.map(a => a.toolName))];
+    const availableTools = allAnnotations.map(a => ({ serverName: a.serverName, toolName: a.toolName }));
+    const blameSchema = z.discriminatedUnion('kind', [
+        z.object({
+            kind: z.literal('rule'),
+            reasoning: z.string(),
+        }),
+        z.object({
+            kind: z.literal('scenario'),
+            reasoning: z.string(),
+            correctedDecision: z.enum(['allow', 'deny', 'escalate']),
+            correctedReasoning: z.string(),
+        }),
+        z.object({
+            kind: z.literal('both'),
+            reasoning: z.string(),
+            correctedDecision: z.enum(['allow', 'deny', 'escalate']),
+            correctedReasoning: z.string(),
+        }),
+    ]);
+    const responseSchema = z.object({
+        analysis: z.string(),
+        pass: z.boolean(),
+        failureAttributions: z.array(z.object({
+            scenarioDescription: z.string(),
+            blame: blameSchema,
+        })),
+        additionalScenarios: z.array(z.object({
+            description: z.string(),
+            request: z.object({
+                serverName: z.enum(serverNamesList),
+                toolName: z.enum(toolNamesList),
+                arguments: z.record(z.string(), z.unknown()),
+            }),
+            expectedDecision: z.enum(['allow', 'deny', 'escalate']),
+            reasoning: z.string(),
+        })),
+    });
+    const rounds = [];
+    const allFailedScenarios = [];
+    let currentScenarios = scenarios;
+    let previousAnalysis;
+    for (let round = 1; round <= maxRounds; round++) {
+        // Execute scenarios against the real engine
+        const executionResults = executeScenarios(engine, currentScenarios);
+        const failures = executionResults.filter(r => !r.pass);
+        allFailedScenarios.push(...failures);
+        // Send results to LLM judge
+        const prompt = buildJudgePrompt(constitutionText, compiledPolicy, toolAnnotations, protectedPaths, executionResults, round, previousAnalysis, availableTools);
+        const { output: judgment } = await generateObjectWithRepair({
+            model: llm,
+            schema: responseSchema,
+            prompt,
+            onProgress,
+        });
+        const newScenarios = judgment.additionalScenarios.map(s => ({
+            ...s,
+            source: 'generated',
+        }));
+        rounds.push({
+            round,
+            executionResults,
+            llmAnalysis: judgment.analysis,
+            newScenarios,
+            attributedFailures: judgment.failureAttributions,
+        });
+        previousAnalysis = judgment.analysis;
+        // If judge says pass and no new scenarios, we are done
+        if (judgment.pass && newScenarios.length === 0) {
+            return {
+                pass: true,
+                rounds,
+                summary: judgment.analysis,
+                failedScenarios: allFailedScenarios,
+            };
+        }
+        // If no new scenarios to probe, exit the loop
+        if (newScenarios.length === 0)
+            break;
+        // Run additional scenarios in next round
+        currentScenarios = newScenarios;
+    }
+    // Determine final result based on failed scenarios
+    const lastRound = rounds[rounds.length - 1];
+    const hasFailures = allFailedScenarios.length > 0;
+    return {
+        pass: !hasFailures && (lastRound?.llmAnalysis.toLowerCase().includes('pass') ?? false),
+        rounds,
+        summary: lastRound?.llmAnalysis ?? 'Verification did not complete',
+        failedScenarios: allFailedScenarios,
+    };
+}
+/**
+ * Filters out scenarios whose expected decision conflicts with a structural
+ * invariant result. Structural invariants are ground truth (hardcoded in the
+ * engine), so a disagreeing scenario expectation is wrong by definition.
+ *
+ * Returns the valid scenarios and the discarded ones (with metadata for
+ * logging / feedback).
+ */
+export function filterStructuralConflicts(engine, scenarios) {
+    const results = executeScenarios(engine, scenarios);
+    const valid = [];
+    const discarded = [];
+    for (const result of results) {
+        const isStructural = result.matchingRule.startsWith('structural-');
+        if (isStructural && !result.pass) {
+            discarded.push({
+                scenario: result.scenario,
+                actual: result.actualDecision,
+                rule: result.matchingRule,
+            });
+        }
+        else {
+            valid.push(result.scenario);
+        }
+    }
+    return { valid, discarded };
+}
+// ---------------------------------------------------------------------------
+// Scenario Corrections
+// ---------------------------------------------------------------------------
+/**
+ * Extracts scenario corrections from attributed failures.
+ * Only failures blamed on 'scenario' or 'both' produce corrections.
+ * Handwritten scenarios are never auto-corrected — they are returned
+ * separately as warnings so the caller can reclassify them as rule issues.
+ */
+export function extractScenarioCorrections(attributedFailures, scenarios) {
+    const corrections = [];
+    const handwrittenWarnings = [];
+    for (const af of attributedFailures) {
+        if (af.blame.kind === 'rule')
+            continue;
+        const scenario = scenarios.find(s => s.description === af.scenarioDescription);
+        if (!scenario)
+            continue;
+        if (scenario.source === 'handwritten') {
+            handwrittenWarnings.push(`Judge blamed scenario "${af.scenarioDescription}" but it is handwritten — treating as rule issue`);
+            continue;
+        }
+        corrections.push({
+            scenarioDescription: af.scenarioDescription,
+            correctedDecision: af.blame.correctedDecision,
+            correctedReasoning: af.blame.correctedReasoning,
+        });
+    }
+    return { corrections, handwrittenWarnings };
+}
+/**
+ * Applies corrections to a scenario list, returning a new array with
+ * updated expectedDecision and reasoning for corrected scenarios.
+ */
+export function applyScenarioCorrections(scenarios, corrections) {
+    const correctionMap = new Map(corrections.map(c => [c.scenarioDescription, c]));
+    return scenarios.map(s => {
+        const correction = correctionMap.get(s.description);
+        if (!correction)
+            return s;
+        return {
+            ...s,
+            expectedDecision: correction.correctedDecision,
+            reasoning: correction.correctedReasoning,
+        };
+    });
+}
+//# sourceMappingURL=policy-verifier.js.map