@allurereport/plugin-agent 3.10.0 → 3.11.0

package/dist/harness.js

@@ -10,20 +10,27 @@ const FALLBACK_ACTION = {
 };
 export const AGENT_ENRICHMENT_ACTIONS = Object.fromEntries(Object.entries(ENRICHMENT_ACTIONS_BY_CHECK_NAME).map(([checkName, action]) => [checkName, { checkName, ...action }]));
 export const SCOPE_REJECTING_CHECKS = [
-    "missing-expected-test",
-    "missing-expected-prefix",
-    "missing-expected-environment",
+    "expected-test-missing",
+    "expected-count-mismatch",
+    "expected-prefix-missing",
+    "expected-label-missing",
+    "expected-environment-missing",
+    "no-tests-observed",
     "unexpected-environment",
-    "forbidden-selector-match",
+    "forbidden-label-observed",
     "unexpected-test",
 ];
 export const ITERATION_REQUIRED_CHECKS = [
-    "invalid-expectations-file",
-    "no-visible-tests",
+    "expectations-invalid",
+    "expectations-empty",
+    "expectations-unsupported-control",
     "runner-failures-outside-logical-results",
-    "missing-expected-label-selector",
     "metadata-mismatch",
     "history-id-collision",
+    "expected-step-containing-missing",
+    "insufficient-expected-steps",
+    "insufficient-expected-attachments",
+    "missing-expected-attachment",
     "failed-without-useful-steps",
     "failed-without-attachments",
     "nontrivial-run-with-empty-trace",
@@ -42,6 +49,22 @@ const IMPACT_ORDER = {
     advisory: 2,
 };
 const uniqueValues = (values) => Array.from(new Set(values));
+const checkNameForFinding = (finding) => finding.check_id ?? finding.check_name;
+const subjectRefForFinding = (finding) => {
+    if (finding.subject_ref) {
+        return finding.subject_ref;
+    }
+    if (typeof finding.subject === "string") {
+        return finding.subject;
+    }
+    return finding.subject.path ?? finding.subject.id ?? finding.subject.type;
+};
+const subjectTypeForFinding = (finding) => finding.subject_type ??
+    (typeof finding.subject === "object" && finding.subject.type === "test"
+        ? "test"
+        : subjectRefForFinding(finding) === "run"
+            ? "run"
+            : "test");
 const normalizeStringArray = (value) => {
     if (typeof value === "string") {
         return value.length ? [value] : [];
@@ -112,14 +135,18 @@ const sortPlan = (items) => [...items].sort((left, right) => {
     return left.checkName.localeCompare(right.checkName);
 });
 const impactForFinding = (finding, antiDummyConfidenceThreshold) => {
-    if (SCOPE_REJECTING_CHECKS.includes(finding.check_name)) {
+    if (finding.impact === "reject" || finding.impact === "iterate" || finding.impact === "advisory") {
+        return finding.impact;
+    }
+    const checkName = checkNameForFinding(finding);
+    if (SCOPE_REJECTING_CHECKS.includes(checkName)) {
         return "reject";
     }
-    if (ANTI_DUMMY_CHECKS.includes(finding.check_name) &&
+    if (ANTI_DUMMY_CHECKS.includes(checkName) &&
         (finding.confidence ?? 0) >= antiDummyConfidenceThreshold) {
         return "reject";
     }
-    if (ITERATION_REQUIRED_CHECKS.includes(finding.check_name)) {
+    if (ITERATION_REQUIRED_CHECKS.includes(checkName)) {
         return "iterate";
     }
     if (finding.severity === "high") {
@@ -140,7 +167,7 @@ export const buildAgentExpectations = (input) => {
     };
 };
 export const mapFindingToEnrichmentAction = (finding) => {
-    const checkName = typeof finding === "string" ? finding : finding.check_name;
+    const checkName = typeof finding === "string" ? finding : checkNameForFinding(finding);
     const mapped = AGENT_ENRICHMENT_ACTIONS[checkName];
     return mapped ?? { ...FALLBACK_ACTION, checkName };
 };
@@ -166,16 +193,17 @@ export const planAgentEnrichmentReview = (output, options = {}) => {
     const testsByPath = new Map(output.tests.map((test) => [test.markdown_path, test]));
     const plan = sortPlan(output.findings.map((finding) => {
         const action = mapFindingToEnrichmentAction(finding);
-        const matchedTest = testsByPath.get(finding.subject);
+        const subject = subjectRefForFinding(finding);
+        const matchedTest = testsByPath.get(subject);
         return {
             ...action,
-            subject: finding.subject,
-            subjectType: finding.subject === "run" ? "run" : "test",
+            subject,
+            subjectType: subjectTypeForFinding(finding),
             severity: finding.severity,
             message: finding.message,
             explanation: finding.explanation,
-            remediationHint: finding.remediation_hint,
-            evidencePaths: finding.evidence_paths,
+            remediationHint: finding.action ?? finding.remediation_hint,
+            evidencePaths: finding.evidence?.paths ?? finding.evidence_paths,
             expectedReference: finding.expected_reference,
             confidence: finding.confidence,
             acceptanceImpact: impactForFinding(finding, antiDummyConfidenceThreshold),
@@ -189,7 +217,7 @@ export const planAgentEnrichmentReview = (output, options = {}) => {
     const advisory = plan.filter((item) => item.acceptanceImpact === "advisory");
     const notes = [];
     if (!output.run.expectations_present) {
-        notes.push("Generate ALLURE_AGENT_EXPECTATIONS before the next enrichment iteration so scope checks are comparable.");
+        notes.push("Declare inline expectations or provide an expectations file before the next enrichment iteration so scope checks are comparable.");
     }
     if (rejecting.some((item) => item.checkName === "noop-dominated-steps")) {
         notes.push("Reject noop-dominated enrichment: keep only steps tied to real actions or checks, and use real runtime attachments instead of placeholders.");

package/dist/index.d.ts

@@ -1,3 +1,11 @@
-export { type AgentPluginOptions } from "./model.js";
+export { type AgentAttachmentExpectationInput, type AgentEvidenceExpectationInput, type AgentExpectationSelectorInput, type AgentExpectationsInput, type AgentPluginOptions, parseAgentExpectations, } from "./model.js";
+export * from "./capabilities.js";
+export * from "./errors.js";
 export * from "./harness.js";
+export * from "./inline-expectations.js";
+export * from "./invalid-output.js";
+export * from "./paths.js";
+export * from "./query.js";
+export * from "./selection.js";
+export * from "./state.js";
 export { AgentPlugin as default } from "./plugin.js";

package/dist/index.js

@@ -1,2 +1,11 @@
+export { parseAgentExpectations, } from "./model.js";
+export * from "./capabilities.js";
+export * from "./errors.js";
 export * from "./harness.js";
+export * from "./inline-expectations.js";
+export * from "./invalid-output.js";
+export * from "./paths.js";
+export * from "./query.js";
+export * from "./selection.js";
+export * from "./state.js";
 export { AgentPlugin as default } from "./plugin.js";

package/dist/inline-expectations.d.ts

@@ -0,0 +1,23 @@
+import type { AgentExpectationsInput } from "./model.js";
+type SingleStringOptionValue = string | string[] | undefined;
+export type AgentInlineExpectationOptions = {
+    goal?: SingleStringOptionValue;
+    taskId?: SingleStringOptionValue;
+    expectTests?: SingleStringOptionValue;
+    expectLabels?: string[];
+    expectEnvironments?: string[];
+    expectFullNames?: string[];
+    expectPrefixes?: string[];
+    forbidLabels?: string[];
+    expectStepContains?: string[];
+    expectSteps?: SingleStringOptionValue;
+    expectAttachments?: SingleStringOptionValue;
+    expectAttachmentFilters?: string[];
+};
+export declare const buildAgentInlineExpectations: (options: AgentInlineExpectationOptions) => AgentExpectationsInput | undefined;
+export declare const validateAgentExpectationsFile: (params: {
+    cwd: string;
+    output?: string;
+    expectations?: string;
+}) => Promise<void>;
+export {};

package/dist/inline-expectations.js

@@ -0,0 +1,186 @@
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { AgentExpectationUsageError, AgentUsageError } from "./errors.js";
+import { parseAgentExpectations } from "./model.js";
+import { isPathInside } from "./paths.js";
+const readNonNegativeInteger = (value, optionName) => {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (!/^\d+$/.test(value)) {
+        throw new AgentExpectationUsageError(`${optionName} must be a non-negative integer`, optionName);
+    }
+    const parsed = Number(value);
+    if (!Number.isSafeInteger(parsed)) {
+        throw new AgentExpectationUsageError(`${optionName} must be a non-negative integer`, optionName);
+    }
+    return parsed;
+};
+const readPositiveInteger = (value, optionName) => {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (!/^[1-9]\d*$/.test(value)) {
+        throw new AgentExpectationUsageError(`${optionName} must be a positive integer`, optionName);
+    }
+    const parsed = Number(value);
+    if (!Number.isSafeInteger(parsed)) {
+        throw new AgentExpectationUsageError(`${optionName} must be a positive integer`, optionName);
+    }
+    return parsed;
+};
+const readSingleStringOption = (value, optionName) => {
+    const values = Array.isArray(value) ? value : typeof value === "string" ? [value] : [];
+    if (values.length > 1) {
+        throw new AgentExpectationUsageError(`Duplicate ${optionName} is not allowed`, optionName);
+    }
+    if (!values.length) {
+        return undefined;
+    }
+    const normalized = values[0].trim();
+    if (!normalized) {
+        throw new AgentExpectationUsageError(`${optionName} value must be non-empty`, optionName);
+    }
+    return normalized;
+};
+const parseNameValue = (value, optionName, example) => {
+    const parts = value.split("=");
+    if (parts.length !== 2) {
+        throw new AgentExpectationUsageError(`Invalid ${optionName} ${JSON.stringify(value)}. Expected ${example}`, optionName);
+    }
+    const name = parts[0].trim();
+    const filterValue = parts[1].trim();
+    if (!name || !filterValue) {
+        throw new AgentExpectationUsageError(`Invalid ${optionName} ${JSON.stringify(value)}. Expected ${example}`, optionName);
+    }
+    return {
+        name,
+        value: filterValue,
+    };
+};
+const addLabelValues = (target, values, optionName) => {
+    for (const rawValue of values ?? []) {
+        const { name, value } = parseNameValue(rawValue, optionName, "the form name=value, for example module=cli");
+        const current = target[name] ?? [];
+        if (!current.includes(value)) {
+            current.push(value);
+        }
+        target[name] = current;
+    }
+};
+const normalizeStringValues = (values, optionName) => {
+    const normalized = [];
+    for (const value of values ?? []) {
+        const trimmed = value.trim();
+        if (!trimmed) {
+            throw new AgentExpectationUsageError(`${optionName} value must be non-empty`, optionName);
+        }
+        normalized.push(trimmed);
+    }
+    return normalized;
+};
+export const buildAgentInlineExpectations = (options) => {
+    const expectedLabels = {};
+    const forbiddenLabels = {};
+    const expected = {};
+    const forbidden = {};
+    const evidence = {};
+    const attachmentFilters = [];
+    addLabelValues(expectedLabels, options.expectLabels, "--expect-label");
+    addLabelValues(forbiddenLabels, options.forbidLabels, "--forbid-label");
+    const expectTests = readNonNegativeInteger(readSingleStringOption(options.expectTests, "--expect-tests"), "--expect-tests");
+    const expectSteps = readPositiveInteger(readSingleStringOption(options.expectSteps, "--expect-steps"), "--expect-steps");
+    const expectAttachments = readPositiveInteger(readSingleStringOption(options.expectAttachments, "--expect-attachments"), "--expect-attachments");
+    const expectedEnvironments = normalizeStringValues(options.expectEnvironments, "--expect-env");
+    const expectedFullNames = normalizeStringValues(options.expectFullNames, "--expect-test");
+    const expectedPrefixes = normalizeStringValues(options.expectPrefixes, "--expect-prefix");
+    const expectedStepContains = normalizeStringValues(options.expectStepContains, "--expect-step-containing");
+    for (const rawValue of options.expectAttachmentFilters ?? []) {
+        const parsed = rawValue.includes("=")
+            ? parseNameValue(rawValue, "--expect-attachment", "a file name or a filter such as name=trace.zip or content-type=application/json")
+            : { name: "name", value: rawValue.trim() };
+        const normalizedName = parsed.name.toLowerCase().replace(/_/g, "-");
+        if (!parsed.value) {
+            throw new AgentExpectationUsageError("Invalid --expect-attachment value. Expected a non-empty file name or filter such as name=trace.zip", "--expect-attachment");
+        }
+        if (normalizedName === "name") {
+            attachmentFilters.push({ name: parsed.value });
+            continue;
+        }
+        if (normalizedName === "content-type" || normalizedName === "type") {
+            attachmentFilters.push({ content_type: parsed.value });
+            continue;
+        }
+        throw new AgentExpectationUsageError(`Invalid --expect-attachment key ${JSON.stringify(parsed.name)}. Expected name or content-type`, "--expect-attachment");
+    }
+    if (expectTests !== undefined) {
+        expected.test_count = expectTests;
+    }
+    if (expectedEnvironments.length) {
+        expected.environments = expectedEnvironments;
+    }
+    if (expectedFullNames.length) {
+        expected.full_names = expectedFullNames;
+    }
+    if (expectedPrefixes.length) {
+        expected.full_name_prefixes = expectedPrefixes;
+    }
+    if (Object.keys(expectedLabels).length) {
+        expected.label_values = expectedLabels;
+    }
+    if (Object.keys(forbiddenLabels).length) {
+        forbidden.label_values = forbiddenLabels;
+    }
+    if (expectSteps !== undefined) {
+        evidence.min_steps = expectSteps;
+    }
+    if (expectAttachments !== undefined) {
+        evidence.min_attachments = expectAttachments;
+    }
+    if (expectedStepContains.length) {
+        evidence.step_name_contains = expectedStepContains;
+    }
+    if (attachmentFilters.length) {
+        evidence.attachments = attachmentFilters;
+    }
+    if (expected.test_count === 0 &&
+        (expected.environments?.length ||
+            expected.full_names?.length ||
+            expected.full_name_prefixes?.length ||
+            Object.keys(expected.label_values ?? {}).length ||
+            evidence.step_name_contains?.length ||
+            evidence.min_steps !== undefined ||
+            evidence.min_attachments !== undefined ||
+            evidence.attachments?.length)) {
+        throw new AgentExpectationUsageError("--expect-tests 0 cannot be combined with positive scope or evidence expectations", "--expect-tests");
+    }
+    const inlineExpectations = {
+        ...(readSingleStringOption(options.goal, "--goal") ? { goal: readSingleStringOption(options.goal, "--goal") } : {}),
+        ...(readSingleStringOption(options.taskId, "--task-id")
+            ? { task_id: readSingleStringOption(options.taskId, "--task-id") }
+            : {}),
+        ...(Object.keys(expected).length ? { expected } : {}),
+        ...(Object.keys(forbidden).length ? { forbidden } : {}),
+        ...(Object.keys(evidence).length ? { evidence } : {}),
+    };
+    return Object.keys(inlineExpectations).length ? inlineExpectations : undefined;
+};
+export const validateAgentExpectationsFile = async (params) => {
+    const { cwd, output, expectations } = params;
+    if (!expectations) {
+        return;
+    }
+    const expectationsPath = resolve(cwd, expectations);
+    if (output) {
+        const outputDir = resolve(cwd, output);
+        if (isPathInside(outputDir, expectationsPath)) {
+            throw new AgentUsageError(`--expectations path ${JSON.stringify(expectationsPath)} must not be inside the agent output directory ${JSON.stringify(outputDir)}`);
+        }
+    }
+    try {
+        parseAgentExpectations(await readFile(expectationsPath, "utf-8"));
+    }
+    catch (error) {
+        throw new AgentExpectationUsageError(`Could not load expectations from ${expectationsPath}: ${error.message}`, "--expectations");
+    }
+};

package/dist/invalid-output.d.ts

@@ -0,0 +1,58 @@
+import { AgentExpectationUsageError } from "./errors.js";
+export declare const createInvalidExpectationFinding: (params: {
+    message: string;
+    sourceOption?: string;
+}) => {
+    schema_version: string;
+    check_id: string;
+    instance_id: string;
+    severity: string;
+    impact: string;
+    confidence: number;
+    category: string;
+    title: string;
+    message: string;
+    subject: {
+        type: string;
+    };
+    expected: {
+        option: string;
+        expectations?: undefined;
+    } | {
+        expectations: string;
+        option?: undefined;
+    };
+    observed: {
+        error: string;
+        execution_skipped: boolean;
+    };
+    evidence: {
+        paths: string[];
+    };
+    action: string;
+    source: {
+        kind: string;
+        option: string;
+    } | undefined;
+    legacy: {
+        finding_id: string;
+        check_name: string;
+        remediation_hint: string;
+    };
+    finding_id: string;
+    subject_ref: string;
+    subject_type: string;
+    check_name: string;
+    explanation: string;
+    evidence_paths: string[];
+    remediation_hint: string;
+    expected_reference: string | undefined;
+};
+export declare const writeInvalidAgentExpectationOutput: (params: {
+    outputDir: string;
+    command: string;
+    error: AgentExpectationUsageError;
+}) => Promise<{
+    outputDir: string;
+    generatedAt: string;
+}>;

package/dist/invalid-output.js

@@ -0,0 +1,238 @@
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+const isFileNotFoundError = (error) => typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
+const emptyAgentStats = () => ({
+    total: 0,
+    failed: 0,
+    broken: 0,
+    skipped: 0,
+    unknown: 0,
+    passed: 0,
+});
+const writeJson = async (path, value) => {
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, "utf-8");
+};
+const writeText = async (path, value) => {
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, value, "utf-8");
+};
+const writeJsonl = async (path, values) => {
+    await writeText(path, values.map((value) => JSON.stringify(value)).join("\n") + (values.length ? "\n" : ""));
+};
+export const createInvalidExpectationFinding = (params) => {
+    const action = "Fix expectation syntax before using the run as validation.";
+    return {
+        schema_version: "allure-agent-finding/v2",
+        check_id: "expectations-invalid",
+        instance_id: "F0001",
+        severity: "high",
+        impact: "reject",
+        confidence: 1,
+        category: "bootstrap",
+        title: "Expectation input is invalid",
+        message: params.message,
+        subject: {
+            type: "run",
+        },
+        expected: params.sourceOption ? { option: params.sourceOption } : { expectations: "valid M1 expectation input" },
+        observed: {
+            error: params.message,
+            execution_skipped: true,
+        },
+        evidence: {
+            paths: ["manifest/run.json"],
+        },
+        action,
+        source: params.sourceOption
+            ? {
+                kind: "inline-option",
+                option: params.sourceOption,
+            }
+            : undefined,
+        legacy: {
+            finding_id: "F0001",
+            check_name: "expectations-invalid",
+            remediation_hint: action,
+        },
+        finding_id: "F0001",
+        subject_ref: "run",
+        subject_type: "run",
+        check_name: "expectations-invalid",
+        explanation: "The agent expectation controls could not be parsed, so the test command was not executed.",
+        evidence_paths: ["manifest/run.json"],
+        remediation_hint: action,
+        expected_reference: params.sourceOption,
+    };
+};
+export const writeInvalidAgentExpectationOutput = async (params) => {
+    const { outputDir, command, error } = params;
+    const generatedAt = new Date().toISOString();
+    const finding = createInvalidExpectationFinding({
+        message: error.message,
+        sourceOption: error.sourceOption,
+    });
+    const stats = emptyAgentStats();
+    try {
+        await rm(outputDir, { recursive: true });
+    }
+    catch (rmError) {
+        if (!isFileNotFoundError(rmError)) {
+            console.error("could not clean output directory", rmError);
+        }
+    }
+    const runManifest = {
+        schema_version: "allure-agent-output/v1",
+        report_uuid: null,
+        generated_at: generatedAt,
+        phase: "done",
+        command,
+        actual_exit_code: null,
+        original_exit_code: null,
+        exit_code: null,
+        summary: {
+            stats,
+            modeled_stats: stats,
+            unmodeled_from_stats: stats,
+            compact: {
+                visible_results: 0,
+                logical_tests: 0,
+                unmodeled_visible_results: 0,
+                runner_failures_outside_logical_tests: 0,
+                completeness: "complete",
+                findings: 1,
+            },
+            duration_ms: {
+                total: 0,
+                average: 0,
+                max: 0,
+            },
+            environments: [],
+        },
+        modeling: {
+            completeness: "complete",
+            reasons: ["test command was skipped because agent expectations were invalid"],
+            modeledStats: stats,
+            unmodeledFromStats: stats,
+            runnerFailures: {
+                total: 0,
+                globalErrors: 0,
+                stderrActionable: 0,
+                samples: [],
+            },
+            stderr: {
+                actionableCount: 0,
+                actionableSamples: [],
+                noisyWarningCount: 0,
+                noisyWarningSamples: [],
+            },
+            compact: {
+                visible_results: 0,
+                logical_tests: 0,
+                unmodeled_visible_results: 0,
+                runner_failures_outside_logical_tests: 0,
+                completeness: "complete",
+            },
+        },
+        paths: {
+            index_md: "index.md",
+            agents_md: "AGENTS.md",
+            tests_manifest: "manifest/tests.jsonl",
+            findings_manifest: "manifest/findings.jsonl",
+            test_events_manifest: "manifest/test-events.jsonl",
+            expected_manifest: null,
+            process_logs: {
+                stdout: null,
+                stderr: null,
+            },
+        },
+        expectations_present: false,
+        expectations: null,
+        expectation_result: {
+            schema_version: "allure-agent-expectation-result/v1",
+            status: "unavailable",
+            impact: "reject",
+            source: {
+                kind: "none",
+                path: null,
+            },
+            recognized_control_count: 0,
+            unsupported_controls: [],
+            degraded_controls: [],
+            summary: {
+                expected_tests: 0,
+                observed_tests: 0,
+                missing_expected: 0,
+                forbidden_observed: 0,
+                unexpected_observed: 0,
+                evidence_mismatches: 0,
+            },
+            finding_ids: ["F0001"],
+        },
+        check_summary: {
+            total: 1,
+            countsBySeverity: {
+                high: 1,
+                warning: 0,
+                info: 0,
+            },
+            countsByCategory: {
+                bootstrap: 1,
+                scope: 0,
+                metadata: 0,
+                evidence: 0,
+                smells: 0,
+            },
+        },
+        agent_context: {
+            agent_name: null,
+            loop_id: null,
+            task_id: null,
+            conversation_id: null,
+        },
+    };
+    const index = [
+        "# Allure Agent Run",
+        "",
+        "- Phase: done",
+        `- Command: ${command || "(not executed)"}`,
+        "- Exit code: not available",
+        "",
+        "## Expectation Result",
+        "",
+        "- Status: unavailable",
+        "- Impact: reject",
+        "- Recognized controls: 0",
+        "- Summary: expectation input was invalid; test execution was skipped",
+        "",
+        "## Findings",
+        "",
+        `- [HIGH][reject][bootstrap] ${finding.title}`,
+        `  Expected: ${error.sourceOption ?? "valid M1 expectation input"}`,
+        `  Observed: ${error.message}`,
+        `  Action: ${finding.action}`,
+        "",
+        "## Machine-Readable Artifacts",
+        "",
+        "- Run Manifest: [manifest/run.json](manifest/run.json)",
+        "- Findings Manifest: [manifest/findings.jsonl](manifest/findings.jsonl)",
+        "",
+    ].join("\n");
+    await Promise.all([
+        writeJson(join(outputDir, "manifest", "run.json"), runManifest),
+        writeJsonl(join(outputDir, "manifest", "findings.jsonl"), [finding]),
+        writeJsonl(join(outputDir, "manifest", "tests.jsonl"), []),
+        writeJsonl(join(outputDir, "manifest", "test-events.jsonl"), []),
+        writeText(join(outputDir, "index.md"), index),
+        writeText(join(outputDir, "AGENTS.md"), [
+            "# Allure Agent Output",
+            "",
+            "Read `index.md`, `manifest/run.json`, and `manifest/findings.jsonl` before using this run.",
+            "",
+        ].join("\n")),
+    ]);
+    return {
+        outputDir,
+        generatedAt,
+    };
+};