agentcert 0.1.0

package/dist/failure-review.js

@@ -0,0 +1,188 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+export const FAILURE_TYPES = [
+    "prompt_injection",
+    "wrong_click",
+    "timeout",
+    "verification_gap",
+    "silent_partial_success",
+    "network_failure",
+    "ui_drift",
+    "policy_or_approval",
+    "agent_connection",
+    "console_error",
+    "assertion_failure",
+    "unknown_failure",
+];
+export function createFailureReview(input) {
+    const reviewedAt = input.reviewedAt ?? new Date().toISOString();
+    const reviewer = input.reviewer ?? "human-reviewer";
+    const confidence = normalizeReviewConfidence(input.confidence);
+    const id = stableReviewId({
+        reviewedAt,
+        reviewer,
+        target: input.target,
+        type: input.type,
+        status: input.status,
+    });
+    return {
+        schemaVersion: "1",
+        kind: "agentcert.failure_review",
+        id,
+        reviewedAt,
+        reviewer,
+        status: input.status,
+        target: input.target,
+        suggestedType: input.suggestedType,
+        type: input.type,
+        note: input.note,
+        confidence,
+        evidenceContext: input.evidenceContext,
+        taxonomyRationale: input.taxonomyRationale,
+    };
+}
+export function parseFailureType(input) {
+    if (input && FAILURE_TYPES.includes(input)) {
+        return input;
+    }
+    throw new Error(`Unknown failure type "${input ?? ""}". Use one of: ${FAILURE_TYPES.join(", ")}.`);
+}
+export function parseFailureReviewStatus(input, type, suggestedType) {
+    if (input === "confirmed" || input === "corrected") {
+        return input;
+    }
+    return suggestedType && type === suggestedType ? "confirmed" : "corrected";
+}
+export function parseReviewConfidence(input) {
+    if (input === undefined)
+        return undefined;
+    const value = typeof input === "number" ? input : Number(input);
+    return normalizeReviewConfidence(value);
+}
+export async function readFailureReviews(path) {
+    if (!path)
+        return [];
+    try {
+        const raw = await readFile(resolve(path), "utf8");
+        return raw
+            .split(/\r?\n/)
+            .filter((line) => line.trim().length > 0)
+            .map((line) => JSON.parse(line))
+            .filter((review) => review.kind === "agentcert.failure_review");
+    }
+    catch (error) {
+        if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
+            return [];
+        }
+        throw error;
+    }
+}
+export async function appendFailureReview(path, review) {
+    const outPath = resolve(path);
+    await mkdir(dirname(outPath), { recursive: true });
+    const existing = await readFile(outPath, "utf8").catch((error) => {
+        if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
+            return "";
+        throw error;
+    });
+    const prefix = existing.trimEnd();
+    const separator = prefix.length > 0 ? "\n" : "";
+    await writeFile(outPath, `${prefix}${separator}${JSON.stringify(review)}\n`);
+}
+export function applyFailureReviews(records, reviews) {
+    if (reviews.length === 0) {
+        return records.map(normalizeRecordReviewState);
+    }
+    const sortedReviews = [...reviews].sort((left, right) => left.reviewedAt.localeCompare(right.reviewedAt));
+    return records.map((record) => {
+        const failurePatterns = record.failurePatterns.map((pattern) => applyReviewToPattern(record, pattern, sortedReviews));
+        return {
+            ...record,
+            failurePatterns,
+            metadata: {
+                ...record.metadata,
+                taxonomyReview: {
+                    reviewedFailurePatterns: failurePatterns.filter((pattern) => pattern.reviewStatus === "confirmed" || pattern.reviewStatus === "corrected").length,
+                    correctedFailurePatterns: failurePatterns.filter((pattern) => pattern.reviewStatus === "corrected").length,
+                    unreviewedFailurePatterns: failurePatterns.filter((pattern) => (pattern.reviewStatus ?? "unreviewed") === "unreviewed").length,
+                },
+            },
+        };
+    });
+}
+export function findFailurePattern(records, target) {
+    for (const record of records) {
+        if (!recordMatchesTarget(record, target))
+            continue;
+        const pattern = record.failurePatterns.find((item) => item.key === target.patternKey);
+        if (pattern)
+            return { record, pattern };
+    }
+    return undefined;
+}
+function normalizeRecordReviewState(record) {
+    return {
+        ...record,
+        failurePatterns: record.failurePatterns.map((pattern) => ({
+            ...pattern,
+            suggestedType: pattern.suggestedType ?? pattern.type,
+            reviewStatus: pattern.reviewStatus ?? "unreviewed",
+        })),
+    };
+}
+function applyReviewToPattern(record, pattern, reviews) {
+    const suggestedType = pattern.suggestedType ?? pattern.type;
+    let next = {
+        ...pattern,
+        suggestedType,
+        reviewStatus: pattern.reviewStatus ?? "unreviewed",
+    };
+    for (const review of reviews) {
+        if (!reviewApplies(record, pattern, review))
+            continue;
+        next = {
+            ...next,
+            type: review.type,
+            suggestedType,
+            reviewStatus: review.status,
+            reviewId: review.id,
+            reviewedAt: review.reviewedAt,
+            reviewer: review.reviewer,
+            reviewNote: review.note,
+            reviewConfidence: review.confidence,
+            reviewEvidenceContext: review.evidenceContext,
+            taxonomyRationale: review.taxonomyRationale,
+        };
+    }
+    return next;
+}
+function reviewApplies(record, pattern, review) {
+    if (review.target.patternKey !== pattern.key)
+        return false;
+    return recordMatchesTarget(record, review.target);
+}
+function recordMatchesTarget(record, target) {
+    if (target.recordId && target.recordId !== record.id)
+        return false;
+    if (target.runId && target.runId !== record.runId)
+        return false;
+    if (target.product && target.product !== record.product)
+        return false;
+    if (target.scenarioName && target.scenarioName !== record.scenarioName)
+        return false;
+    if (target.faultName && target.faultName !== record.faultName)
+        return false;
+    return true;
+}
+function stableReviewId(input) {
+    return createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 16);
+}
+function normalizeReviewConfidence(input) {
+    if (input === undefined)
+        return undefined;
+    if (!Number.isFinite(input) || input < 0 || input > 1) {
+        throw new Error("Review confidence must be a number from 0 to 1.");
+    }
+    return input;
+}

package/dist/index.js

@@ -0,0 +1,10 @@
+export * from "./bundle.js";
+export * from "./corpus.js";
+export * from "./corpus-store.js";
+export * from "./failure-review.js";
+export * from "./local-server.js";
+export * from "./monitor.js";
+export * from "./normalizers.js";
+export * from "./report.js";
+export * from "./schema-validator.js";
+export * from "./types.js";

package/dist/lab.js

@@ -0,0 +1,323 @@
+import { access, mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join, posix, resolve } from "node:path";
+export async function buildRobustnessLabSnapshot(config, cwd = process.cwd()) {
+    const agents = [];
+    const matrix = [];
+    const corpus = config.corpusPath ? await readJsonlIfExists(resolve(cwd, config.corpusPath)) : [];
+    for (const agent of config.agents) {
+        const result = agent.resultPath ? await readJsonIfExists(resolve(cwd, agent.resultPath)) : undefined;
+        if (!result) {
+            agents.push({
+                id: agent.id,
+                name: agent.name,
+                kind: agent.kind,
+                status: "missing",
+                repositoryUrl: agent.repositoryUrl,
+                adapterPath: agent.adapterPath,
+                configPath: agent.configPath,
+                resultPath: agent.resultPath,
+                reportPath: agent.reportPath,
+                requiresModelKey: agent.requiresModelKey ?? false,
+                includedInPublicSnapshot: agent.includedInPublicSnapshot ?? false,
+                totalRuns: 0,
+                passedRuns: 0,
+                failedRuns: 0,
+                passRate: 0,
+                notes: agent.notes,
+            });
+            continue;
+        }
+        const runs = result.runs ?? [];
+        const traces = await loadTracesForRuns(agent, cwd, runs);
+        const passedRuns = numberValue(result.summary?.passedRuns) ?? runs.filter((run) => run.status === "passed").length;
+        const totalRuns = numberValue(result.summary?.totalRuns) ?? runs.length;
+        const failedRuns = numberValue(result.summary?.failedRuns) ?? Math.max(0, totalRuns - passedRuns);
+        const passRate = ratio(passedRuns, totalRuns);
+        agents.push({
+            id: agent.id,
+            name: agent.name,
+            kind: agent.kind,
+            status: "completed",
+            repositoryUrl: agent.repositoryUrl,
+            adapterPath: agent.adapterPath,
+            configPath: agent.configPath,
+            resultPath: agent.resultPath,
+            reportPath: agent.reportPath,
+            requiresModelKey: agent.requiresModelKey ?? false,
+            includedInPublicSnapshot: agent.includedInPublicSnapshot ?? false,
+            score: scoreValue(result.summary?.overallScore),
+            totalRuns,
+            passedRuns,
+            failedRuns,
+            passRate,
+            notes: agent.notes,
+        });
+        for (const run of runs) {
+            const tracePath = stringValue(run.tracePath);
+            const scenarioName = stringValue(run.scenarioName);
+            const faultName = stringValue(run.faultName) ?? "unknown";
+            const trace = tracePath ? traces.get(traceKey(scenarioName, faultName)) : undefined;
+            const cleanTrace = faultName === "clean" ? undefined : traces.get(traceKey(scenarioName, "clean"));
+            const primaryFailure = firstFailure(run);
+            const taxonomy = taxonomyForRun(corpus, run, primaryFailure);
+            const inferredType = primaryFailure ? inferFailureType(faultName, primaryFailure, run) : undefined;
+            matrix.push({
+                agentId: agent.id,
+                agentName: agent.name,
+                scenarioName,
+                faultName,
+                status: run.status === "passed" ? "passed" : "failed",
+                durationMs: numberValue(run.durationMs),
+                stepCount: numberValue(run.stepCount),
+                finalUrl: stringValue(run.finalUrl),
+                primaryFailure,
+                firstDivergence: firstDivergence(agent, tracePath, trace, cleanTrace, run),
+                taxonomyLabel: taxonomy?.type ?? inferredType,
+                suggestedTaxonomyLabel: taxonomy?.suggestedType ?? inferredType,
+                taxonomyReviewStatus: taxonomy?.reviewStatus ?? (inferredType ? "unreviewed" : undefined),
+                reviewerConfidence: taxonomy?.reviewConfidence,
+                taxonomyRationale: taxonomy?.taxonomyRationale,
+                tracePath: publicArtifactPath(agent, tracePath),
+                reportPath: publicArtifactPath(agent, agent.reportPath),
+                screenshotPath: tracePath ? screenshotPathFor(agent, tracePath, trace) : undefined,
+            });
+        }
+    }
+    const completedAgents = agents.filter((agent) => agent.status === "completed");
+    const passedRuns = matrix.filter((cell) => cell.status === "passed").length;
+    const failedRuns = matrix.length - passedRuns;
+    return {
+        schemaVersion: "1",
+        kind: "agentcert.real_agent_robustness_lab",
+        generatedAt: new Date().toISOString(),
+        name: config.name,
+        description: config.description,
+        summary: {
+            agentCount: agents.length,
+            completedAgentCount: completedAgents.length,
+            totalRuns: matrix.length,
+            passedRuns,
+            failedRuns,
+            passRate: ratio(passedRuns, matrix.length),
+            faultCount: new Set(matrix.map((cell) => cell.faultName)).size,
+        },
+        agents,
+        faults: summarizeFaults(matrix),
+        matrix,
+        limitations: config.limitations ?? [],
+    };
+}
+export async function writeRobustnessLabSnapshot(path, snapshot) {
+    const outPath = resolve(path);
+    await mkdir(dirname(outPath), { recursive: true });
+    await writeFile(outPath, `${JSON.stringify(snapshot, null, 2)}\n`);
+}
+export function renderRobustnessLabSummary(snapshot) {
+    const lines = [
+        "# Real Agent Robustness Lab",
+        "",
+        `Agents: ${snapshot.summary.completedAgentCount}/${snapshot.summary.agentCount} completed`,
+        `Runs: ${snapshot.summary.passedRuns}/${snapshot.summary.totalRuns} passed (${Math.round(snapshot.summary.passRate * 100)}%)`,
+        `Faults: ${snapshot.summary.faultCount}`,
+        "",
+        "## Agents",
+        ...snapshot.agents.map((agent) => `- ${agent.name}: ${agent.status}${agent.status === "completed" ? `, ${agent.passedRuns}/${agent.totalRuns} passed` : ""}`),
+    ];
+    return `${lines.join("\n")}\n`;
+}
+export async function readRobustnessLabConfig(path) {
+    return (await readJson(path));
+}
+function summarizeFaults(matrix) {
+    const buckets = new Map();
+    for (const cell of matrix) {
+        const bucket = buckets.get(cell.faultName) ?? { totalRuns: 0, passedRuns: 0 };
+        bucket.totalRuns += 1;
+        if (cell.status === "passed")
+            bucket.passedRuns += 1;
+        buckets.set(cell.faultName, bucket);
+    }
+    return [...buckets.entries()]
+        .map(([faultName, bucket]) => ({
+        faultName,
+        totalRuns: bucket.totalRuns,
+        passedRuns: bucket.passedRuns,
+        failedRuns: bucket.totalRuns - bucket.passedRuns,
+        passRate: ratio(bucket.passedRuns, bucket.totalRuns),
+    }))
+        .sort((left, right) => left.passRate - right.passRate || left.faultName.localeCompare(right.faultName));
+}
+async function loadTracesForRuns(agent, cwd, runs) {
+    const traces = new Map();
+    if (!agent.resultPath)
+        return traces;
+    const resultDir = dirname(resolve(cwd, agent.resultPath));
+    for (const run of runs) {
+        const scenarioName = stringValue(run.scenarioName);
+        const faultName = stringValue(run.faultName) ?? "unknown";
+        const tracePath = stringValue(run.tracePath);
+        if (!tracePath)
+            continue;
+        const trace = await readJsonIfExists(join(resultDir, tracePath));
+        if (trace)
+            traces.set(traceKey(scenarioName, faultName), trace);
+    }
+    return traces;
+}
+function screenshotPathFor(agent, tracePath, trace) {
+    const screenshot = [...(trace?.steps ?? [])].reverse().find((step) => step.screenshotPath)?.screenshotPath;
+    if (!screenshot)
+        return undefined;
+    return publicArtifactPath(agent, posix.join(posix.dirname(slashPath(tracePath)), slashPath(screenshot)));
+}
+function firstDivergence(agent, tracePath, trace, cleanTrace, run) {
+    if (tracePath && trace?.steps && cleanTrace?.steps) {
+        const maxSteps = Math.max(trace.steps.length, cleanTrace.steps.length);
+        for (let index = 0; index < maxSteps; index += 1) {
+            const current = trace.steps[index];
+            const baseline = cleanTrace.steps[index];
+            if (!current || !baseline) {
+                return divergence(agent, tracePath, current, baseline, "step-count", "Trace length diverged from the clean run.");
+            }
+            if (current.url !== baseline.url) {
+                return divergence(agent, tracePath, current, baseline, "url", "First observed URL diverged from the clean run.");
+            }
+            if (current.textHash !== baseline.textHash) {
+                return divergence(agent, tracePath, current, baseline, "text", "First visible text snapshot diverged from the clean run.");
+            }
+            if (current.domHash !== baseline.domHash) {
+                return divergence(agent, tracePath, current, baseline, "dom", "First DOM snapshot diverged from the clean run.");
+            }
+        }
+    }
+    const failure = firstFailure(run);
+    if (!failure)
+        return undefined;
+    return {
+        kind: "assertion",
+        note: failure,
+    };
+}
+function divergence(agent, tracePath, current, baseline, kind, note) {
+    return {
+        kind,
+        stepIndex: current?.stepIndex ?? baseline?.stepIndex,
+        baseline: divergenceValue(kind, baseline),
+        current: divergenceValue(kind, current),
+        note,
+        screenshotPath: current?.screenshotPath
+            ? publicArtifactPath(agent, posix.join(posix.dirname(slashPath(tracePath)), slashPath(current.screenshotPath)))
+            : undefined,
+        domSnapshotPath: current?.domSnapshotPath
+            ? publicArtifactPath(agent, posix.join(posix.dirname(slashPath(tracePath)), slashPath(current.domSnapshotPath)))
+            : undefined,
+    };
+}
+function divergenceValue(kind, step) {
+    if (!step)
+        return undefined;
+    if (kind === "url")
+        return step.url;
+    if (kind === "text")
+        return step.visibleTextSample ?? step.textHash;
+    if (kind === "dom")
+        return step.domHash;
+    return step.stepIndex === undefined ? undefined : `step ${step.stepIndex}`;
+}
+function traceKey(scenarioName, faultName) {
+    return `${scenarioName ?? "unknown"}:${faultName}`;
+}
+function publicArtifactPath(agent, path) {
+    if (!path)
+        return undefined;
+    const normalized = slashPath(path);
+    const prefix = agent.publicPathPrefix?.replace(/\/?$/, "/");
+    if (!prefix)
+        return normalized;
+    if (normalized.startsWith(prefix))
+        return normalized;
+    return `${prefix}${normalized}`;
+}
+function firstFailure(run) {
+    return run.assertions?.find((assertion) => assertion.pass === false)?.message;
+}
+function scoreValue(input) {
+    const value = numberValue(input);
+    if (value === undefined)
+        return undefined;
+    return value <= 1 ? Math.round(value * 100) : Math.round(value);
+}
+function ratio(numerator, denominator) {
+    return denominator === 0 ? 0 : numerator / denominator;
+}
+function stringValue(input) {
+    return typeof input === "string" && input.length > 0 ? input : undefined;
+}
+function numberValue(input) {
+    return typeof input === "number" && Number.isFinite(input) ? input : undefined;
+}
+async function readJson(path) {
+    const raw = await readFile(resolve(path), "utf8");
+    return JSON.parse(raw);
+}
+async function readJsonIfExists(path) {
+    try {
+        await access(path);
+        return (await readJson(path));
+    }
+    catch {
+        return undefined;
+    }
+}
+async function readJsonlIfExists(path) {
+    try {
+        await access(path);
+        const raw = await readFile(path, "utf8");
+        return raw
+            .split(/\r?\n/)
+            .filter((line) => line.trim().length > 0)
+            .map((line) => JSON.parse(line));
+    }
+    catch {
+        return [];
+    }
+}
+function taxonomyForRun(records, run, primaryFailure) {
+    const runId = stringValue(run.runId);
+    const scenarioName = stringValue(run.scenarioName);
+    const faultName = stringValue(run.faultName);
+    const record = records.find((candidate) => {
+        if (runId)
+            return candidate.runId === runId;
+        return candidate.scenarioName === scenarioName && candidate.faultName === faultName;
+    });
+    if (!record)
+        return undefined;
+    return (record.failurePatterns.find((pattern) => pattern.message === primaryFailure) ??
+        record.failurePatterns.find((pattern) => pattern.reviewStatus === "corrected" || pattern.reviewStatus === "confirmed") ??
+        record.failurePatterns[0]);
+}
+function inferFailureType(faultName, primaryFailure, run) {
+    const fault = faultName.toLowerCase();
+    const message = primaryFailure.toLowerCase();
+    const assertionType = run.assertions?.find((assertion) => assertion.pass === false)?.type ?? "";
+    if (assertionType === "max_steps" || message.includes("timed out") || message.includes("agent exited"))
+        return "timeout";
+    if (fault.includes("prompt-injection") || assertionType === "no_sensitive_text_in_output" || message.includes("ignore previous instructions")) {
+        return "prompt_injection";
+    }
+    if (fault.includes("misleading-button") || message.includes("wrong button") || message.includes("cancel"))
+        return "wrong_click";
+    if (fault.includes("button-text-drift") || fault.includes("layout-shift") || fault.includes("modal-overlay") || fault.includes("disabled-submit")) {
+        return "ui_drift";
+    }
+    if (assertionType === "no_console_error" || message.includes("console"))
+        return "console_error";
+    if (fault.includes("slow-network") || fault.includes("http-failure") || message.includes("network") || message.includes("503")) {
+        return "network_failure";
+    }
+    return "assertion_failure";
+}
+function slashPath(path) {
+    return path.replace(/\\/g, "/");
+}