agent-inspect 6.31.10 → 6.31.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +1 -1
- package/docs/NESTJS.md +56 -18
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-JCKC4ZRV.mjs → chunk-MQYHJL3W.mjs} +1035 -833
- package/packages/cli/dist/chunk-MQYHJL3W.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +2156 -1930
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +32 -9
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-4UD7JNDR.mjs → src-76R6E2LQ.mjs} +3 -3
- package/packages/cli/dist/{src-4UD7JNDR.mjs.map → src-76R6E2LQ.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +2060 -1884
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +56 -4
- package/packages/core/dist/advanced.d.ts +56 -4
- package/packages/core/dist/advanced.mjs +184 -20
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +48 -10
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-CPQAOJW4.mjs → chunk-CCYKNAOD.mjs} +50 -12
- package/packages/core/dist/chunk-CCYKNAOD.mjs.map +1 -0
- package/packages/core/dist/{chunk-7YYS742B.mjs → chunk-M3RJD3BP.mjs} +5 -5
- package/packages/core/dist/{chunk-7YYS742B.mjs.map → chunk-M3RJD3BP.mjs.map} +1 -1
- package/packages/core/dist/exporters.cjs +3 -3
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.mjs +1 -1
- package/packages/core/dist/{index-EQihlEei.d.cts → index-DACU4Dy1.d.cts} +12 -0
- package/packages/core/dist/{index-CgOAGuM5.d.ts → index-b4tIGrzq.d.ts} +12 -0
- package/packages/cli/dist/chunk-JCKC4ZRV.mjs.map +0 -1
- package/packages/core/dist/chunk-CPQAOJW4.mjs.map +0 -1
|
@@ -2483,7 +2483,7 @@ function stepMixLine(summary) {
|
|
|
2483
2483
|
if (summary.logicSteps > 0) parts.push(`${summary.logicSteps} logic`);
|
|
2484
2484
|
return parts.length > 0 ? parts.join(", ") : "none";
|
|
2485
2485
|
}
|
|
2486
|
-
function
|
|
2486
|
+
function executionLine(summary) {
|
|
2487
2487
|
if (summary.status === "success") {
|
|
2488
2488
|
return summary.errorSteps > 0 ? "Completed with step errors recorded." : "Completed successfully.";
|
|
2489
2489
|
}
|
|
@@ -2501,7 +2501,7 @@ function outcomeLine(summary) {
|
|
|
2501
2501
|
if (summary.status === "running") {
|
|
2502
2502
|
return "Run is still in progress (no run_completed).";
|
|
2503
2503
|
}
|
|
2504
|
-
return "
|
|
2504
|
+
return "Execution status unknown \u2014 inspect events may be incomplete.";
|
|
2505
2505
|
}
|
|
2506
2506
|
function buildRunWhatSummary(events) {
|
|
2507
2507
|
const base = buildRunSummary(events);
|
|
@@ -2592,7 +2592,7 @@ function renderRunWhat(summary, options = {}) {
|
|
|
2592
2592
|
lines.push(`Correlation: ${parts.join(", ")}`);
|
|
2593
2593
|
}
|
|
2594
2594
|
}
|
|
2595
|
-
lines.push(`
|
|
2595
|
+
lines.push(`Execution: ${executionLine(summary)}`);
|
|
2596
2596
|
if (summary.longestStep && summary.totalSteps > 0) {
|
|
2597
2597
|
lines.push(
|
|
2598
2598
|
`Slowest: ${summary.longestStep.name} (${formatDuration2(summary.longestStep.durationMs)}, ${summary.longestStep.type})`
|
|
@@ -2604,167 +2604,563 @@ function renderRunWhat(summary, options = {}) {
|
|
|
2604
2604
|
return lines.join("\n");
|
|
2605
2605
|
}
|
|
2606
2606
|
|
|
2607
|
-
// packages/core/src/
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2607
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
2608
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
2609
|
+
AI_TRACE_INPUT_INVALID: {
|
|
2610
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
2611
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
2612
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
2613
|
+
relatedCodes: ["invalid_input"]
|
|
2614
|
+
},
|
|
2615
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
2616
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
2617
|
+
summary: "No trace reader could detect the input format.",
|
|
2618
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
2619
|
+
relatedCodes: ["unsupported_format"]
|
|
2620
|
+
},
|
|
2621
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
2622
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
2623
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
2624
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
2625
|
+
relatedCodes: ["ambiguous_format"]
|
|
2626
|
+
},
|
|
2627
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
2628
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
2629
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
2630
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
2631
|
+
},
|
|
2632
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
2633
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2634
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
2635
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
2636
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
2637
|
+
},
|
|
2638
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
2639
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2640
|
+
summary: "A parentId equals its own event/step id.",
|
|
2641
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
2642
|
+
relatedCodes: [
|
|
2643
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
2644
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
2645
|
+
]
|
|
2646
|
+
},
|
|
2647
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
2648
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
2649
|
+
summary: "Trace contains a parentId cycle.",
|
|
2650
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
2651
|
+
relatedCodes: ["structure.cycle"]
|
|
2652
|
+
},
|
|
2653
|
+
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
2654
|
+
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
2655
|
+
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
2656
|
+
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
2657
|
+
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
2658
|
+
},
|
|
2659
|
+
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
2660
|
+
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
2661
|
+
summary: 'inspectRun does not accept a "writer" option.',
|
|
2662
|
+
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
2618
2663
|
}
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
}
|
|
2624
|
-
function fact(id, label, value, redactor) {
|
|
2625
|
-
return {
|
|
2626
|
-
id,
|
|
2627
|
-
label,
|
|
2628
|
-
value: redactValue(redactor, id.split(".").at(-1) ?? id, value),
|
|
2629
|
-
source: "trace",
|
|
2630
|
-
confidence: "observed"
|
|
2631
|
-
};
|
|
2632
|
-
}
|
|
2633
|
-
function topKinds(run) {
|
|
2634
|
-
return Object.entries(run.metadata.kinds).filter(([, count]) => count > 0).sort((a, b) => {
|
|
2635
|
-
if (b[1] !== a[1]) return b[1] - a[1];
|
|
2636
|
-
return a[0].localeCompare(b[0]);
|
|
2637
|
-
}).slice(0, 5).map(([kind, count]) => `${kind}:${count}`);
|
|
2664
|
+
});
|
|
2665
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
2666
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
2667
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
2668
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
2638
2669
|
}
|
|
2639
|
-
|
|
2640
|
-
|
|
2670
|
+
|
|
2671
|
+
// packages/core/src/checks/logical-events.ts
|
|
2672
|
+
function isRecord6(value) {
|
|
2673
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2641
2674
|
}
|
|
2642
|
-
function
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
// and shadows every real step. Rank actual work nodes only, matching the
|
|
2646
|
-
// stats slowest-step ranking, which never counts the run envelope.
|
|
2647
|
-
entry.node.event.kind !== "RUN"
|
|
2648
|
-
).sort((a, b) => {
|
|
2649
|
-
const delta = (b.node.event.durationMs ?? 0) - (a.node.event.durationMs ?? 0);
|
|
2650
|
-
return delta !== 0 ? delta : a.index - b.index;
|
|
2651
|
-
})[0];
|
|
2675
|
+
function legacyEvent(event) {
|
|
2676
|
+
const value = event.attributes?.legacyEvent;
|
|
2677
|
+
return typeof value === "string" ? value : void 0;
|
|
2652
2678
|
}
|
|
2653
|
-
function
|
|
2654
|
-
const
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
if (attrs === void 0) continue;
|
|
2658
|
-
for (const key of Object.keys(attrs).sort()) {
|
|
2659
|
-
facts.push({
|
|
2660
|
-
id: `node.${entry.index}.attributes.${key}`,
|
|
2661
|
-
label: `${entry.node.event.name} attribute ${key}`,
|
|
2662
|
-
value: redactValue(redactor, key, attrs[key]),
|
|
2663
|
-
source: "trace",
|
|
2664
|
-
confidence: "observed"
|
|
2665
|
-
});
|
|
2666
|
-
if (facts.length >= 8) return facts;
|
|
2667
|
-
}
|
|
2668
|
-
}
|
|
2669
|
-
return facts;
|
|
2679
|
+
function stepIdOf(event) {
|
|
2680
|
+
const value = event.attributes?.stepId;
|
|
2681
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2682
|
+
return void 0;
|
|
2670
2683
|
}
|
|
2671
|
-
function
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
fact("run.nodeCount", "Total node count", nodes.length, redactor),
|
|
2680
|
-
fact("run.errorNodeCount", "Error node count", countErrorNodes(nodes), redactor),
|
|
2681
|
-
fact("run.kinds", "Observed kind mix", topKinds(run), redactor)
|
|
2682
|
-
];
|
|
2683
|
-
if (run.durationMs !== void 0) {
|
|
2684
|
-
facts.push(fact("run.durationMs", "Run duration milliseconds", run.durationMs, redactor));
|
|
2685
|
-
}
|
|
2686
|
-
const slowest = slowestNode(nodes);
|
|
2687
|
-
if (slowest !== void 0) {
|
|
2688
|
-
facts.push(
|
|
2689
|
-
fact("run.slowestNode", "Slowest observed node", {
|
|
2690
|
-
name: slowest.node.event.name,
|
|
2691
|
-
kind: slowest.node.event.kind,
|
|
2692
|
-
durationMs: slowest.node.event.durationMs
|
|
2693
|
-
}, redactor)
|
|
2694
|
-
);
|
|
2695
|
-
}
|
|
2696
|
-
facts.push(...attributeFacts(nodes, redactor));
|
|
2697
|
-
return facts;
|
|
2684
|
+
function cloneEvent(event) {
|
|
2685
|
+
return {
|
|
2686
|
+
...event,
|
|
2687
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
2688
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
2689
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
2690
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
2691
|
+
};
|
|
2698
2692
|
}
|
|
2699
|
-
function
|
|
2700
|
-
const
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
label: "Outcome",
|
|
2709
|
-
text: "The run recorded an error status or at least one error node.",
|
|
2710
|
-
evidence: ["run.status", "run.errorNodeCount"],
|
|
2711
|
-
confidence: "deterministic"
|
|
2712
|
-
});
|
|
2713
|
-
} else if (run.status === "ok") {
|
|
2714
|
-
inferences.push({
|
|
2715
|
-
id: "outcome.success",
|
|
2716
|
-
label: "Outcome",
|
|
2717
|
-
text: "The run completed without observed error nodes.",
|
|
2718
|
-
evidence: ["run.status", "run.errorNodeCount"],
|
|
2719
|
-
confidence: "deterministic"
|
|
2720
|
-
});
|
|
2721
|
-
}
|
|
2722
|
-
if (kindFact !== void 0) {
|
|
2723
|
-
inferences.push({
|
|
2724
|
-
id: "shape.kind-mix",
|
|
2725
|
-
label: "Trace shape",
|
|
2726
|
-
text: "The explanation is based on the observed event kind mix, not generated content.",
|
|
2727
|
-
evidence: [kindFact.id],
|
|
2728
|
-
confidence: "deterministic"
|
|
2729
|
-
});
|
|
2730
|
-
}
|
|
2731
|
-
if (durationFact !== void 0) {
|
|
2732
|
-
inferences.push({
|
|
2733
|
-
id: "timing.duration",
|
|
2734
|
-
label: "Timing",
|
|
2735
|
-
text: "Timing claims are limited to persisted duration fields in the trace.",
|
|
2736
|
-
evidence: [durationFact.id],
|
|
2737
|
-
confidence: "deterministic"
|
|
2738
|
-
});
|
|
2693
|
+
function mergeAttributes(start, complete) {
|
|
2694
|
+
const merged = {
|
|
2695
|
+
...isRecord6(complete.attributes) ? complete.attributes : {},
|
|
2696
|
+
...isRecord6(start.attributes) ? start.attributes : {}
|
|
2697
|
+
};
|
|
2698
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
2699
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
2700
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
2701
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
2739
2702
|
}
|
|
2740
|
-
return
|
|
2703
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
2741
2704
|
}
|
|
2742
|
-
function
|
|
2743
|
-
const
|
|
2744
|
-
const
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2705
|
+
function pairStartComplete(start, complete) {
|
|
2706
|
+
const attributes = mergeAttributes(start, complete);
|
|
2707
|
+
const paired = {
|
|
2708
|
+
...cloneEvent(start),
|
|
2709
|
+
status: complete.status,
|
|
2710
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
2711
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
2712
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
2713
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
2714
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
2715
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
2716
|
+
};
|
|
2748
2717
|
return {
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
"Generated locally without provider or network calls.",
|
|
2758
|
-
"Facts are observed from normalized trace data; inferences are deterministic labels."
|
|
2759
|
-
]
|
|
2718
|
+
...paired,
|
|
2719
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
2720
|
+
projection: {
|
|
2721
|
+
paired: true,
|
|
2722
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
2723
|
+
parentNormalized: false,
|
|
2724
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
2725
|
+
}
|
|
2760
2726
|
};
|
|
2761
2727
|
}
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2728
|
+
function asLogical(event, extras) {
|
|
2729
|
+
return {
|
|
2730
|
+
...cloneEvent(event),
|
|
2731
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
2732
|
+
projection: {
|
|
2733
|
+
paired: false,
|
|
2734
|
+
absorbedEventIds: Object.freeze([]),
|
|
2735
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
2736
|
+
...{}
|
|
2737
|
+
}
|
|
2738
|
+
};
|
|
2739
|
+
}
|
|
2740
|
+
function projectLogicalEvents(events) {
|
|
2741
|
+
const diagnostics = [];
|
|
2742
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
2743
|
+
for (const event of events) {
|
|
2744
|
+
const list = byRun.get(event.runId) ?? [];
|
|
2745
|
+
list.push(event);
|
|
2746
|
+
byRun.set(event.runId, list);
|
|
2747
|
+
}
|
|
2748
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
2749
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
2750
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
2751
|
+
const logical = [];
|
|
2752
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
2753
|
+
for (const runId of orderedRuns) {
|
|
2754
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
2755
|
+
const starts = [];
|
|
2756
|
+
const completes = [];
|
|
2757
|
+
const others = [];
|
|
2758
|
+
for (const event of runEvents) {
|
|
2759
|
+
const legacy = legacyEvent(event);
|
|
2760
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
2761
|
+
starts.push(event);
|
|
2762
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
2763
|
+
completes.push(event);
|
|
2764
|
+
} else {
|
|
2765
|
+
others.push(event);
|
|
2766
|
+
}
|
|
2767
|
+
}
|
|
2768
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
2769
|
+
for (const start of starts) {
|
|
2770
|
+
const stepId = stepIdOf(start);
|
|
2771
|
+
let match;
|
|
2772
|
+
if (legacyEvent(start) === "run_started") {
|
|
2773
|
+
const candidates = completes.filter(
|
|
2774
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
2775
|
+
);
|
|
2776
|
+
if (candidates.length > 1) {
|
|
2777
|
+
diagnostics.push({
|
|
2778
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
2779
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
2780
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
2781
|
+
});
|
|
2782
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
2783
|
+
}
|
|
2784
|
+
match = candidates[0];
|
|
2785
|
+
} else if (stepId) {
|
|
2786
|
+
const candidates = completes.filter(
|
|
2787
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
2788
|
+
);
|
|
2789
|
+
if (candidates.length > 1) {
|
|
2790
|
+
diagnostics.push({
|
|
2791
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
2792
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
2793
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
2794
|
+
});
|
|
2795
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
2796
|
+
}
|
|
2797
|
+
match = candidates[0];
|
|
2798
|
+
}
|
|
2799
|
+
if (match) {
|
|
2800
|
+
usedCompletes.add(match.eventId);
|
|
2801
|
+
absorbedIds.add(match.eventId);
|
|
2802
|
+
const paired = pairStartComplete(start, match);
|
|
2803
|
+
logical.push(paired);
|
|
2804
|
+
logicalByRawId.set(start.eventId, paired);
|
|
2805
|
+
logicalByRawId.set(match.eventId, paired);
|
|
2806
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
2807
|
+
} else {
|
|
2808
|
+
diagnostics.push({
|
|
2809
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
2810
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
2811
|
+
eventIds: [start.eventId]
|
|
2812
|
+
});
|
|
2813
|
+
const alone = asLogical(start);
|
|
2814
|
+
logical.push(alone);
|
|
2815
|
+
logicalByRawId.set(start.eventId, alone);
|
|
2816
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
for (const complete of completes) {
|
|
2820
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
2821
|
+
diagnostics.push({
|
|
2822
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
2823
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
2824
|
+
eventIds: [complete.eventId]
|
|
2825
|
+
});
|
|
2826
|
+
const alone = asLogical(complete);
|
|
2827
|
+
logical.push(alone);
|
|
2828
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
2829
|
+
const stepId = stepIdOf(complete);
|
|
2830
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2831
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2834
|
+
for (const event of others) {
|
|
2835
|
+
const alone = asLogical(event);
|
|
2836
|
+
logical.push(alone);
|
|
2837
|
+
logicalByRawId.set(event.eventId, alone);
|
|
2838
|
+
const stepId = stepIdOf(event);
|
|
2839
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2840
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2844
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
2845
|
+
const normalized = [];
|
|
2846
|
+
for (const event of logical) {
|
|
2847
|
+
const originalParentId = event.parentId;
|
|
2848
|
+
if (!originalParentId) {
|
|
2849
|
+
normalized.push(event);
|
|
2850
|
+
continue;
|
|
2851
|
+
}
|
|
2852
|
+
let nextParent = originalParentId;
|
|
2853
|
+
let remapped = false;
|
|
2854
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
2855
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
2856
|
+
nextParent = viaAbsorbed.eventId;
|
|
2857
|
+
remapped = true;
|
|
2858
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
2859
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
2860
|
+
if (viaStep) {
|
|
2861
|
+
nextParent = viaStep;
|
|
2862
|
+
remapped = true;
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
if (!remapped) {
|
|
2866
|
+
if (originalParentId === event.eventId) {
|
|
2867
|
+
diagnostics.push({
|
|
2868
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2869
|
+
message: formatProgrammaticDiagnostic(
|
|
2870
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2871
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
2872
|
+
),
|
|
2873
|
+
eventIds: [event.eventId]
|
|
2874
|
+
});
|
|
2875
|
+
const { parentId: _drop, ...rest } = event;
|
|
2876
|
+
normalized.push({
|
|
2877
|
+
...rest,
|
|
2878
|
+
projection: {
|
|
2879
|
+
...event.projection,
|
|
2880
|
+
parentNormalized: true,
|
|
2881
|
+
originalParentId
|
|
2882
|
+
}
|
|
2883
|
+
});
|
|
2884
|
+
continue;
|
|
2885
|
+
}
|
|
2886
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
2887
|
+
const mapping = event.attributes?.parentMapping;
|
|
2888
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
2889
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
2890
|
+
if (!unresolved) {
|
|
2891
|
+
diagnostics.push({
|
|
2892
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
2893
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
2894
|
+
eventIds: [event.eventId]
|
|
2895
|
+
});
|
|
2896
|
+
}
|
|
2897
|
+
}
|
|
2898
|
+
normalized.push(event);
|
|
2899
|
+
continue;
|
|
2900
|
+
}
|
|
2901
|
+
if (nextParent === event.eventId) {
|
|
2902
|
+
diagnostics.push({
|
|
2903
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2904
|
+
message: formatProgrammaticDiagnostic(
|
|
2905
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2906
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
2907
|
+
),
|
|
2908
|
+
eventIds: [event.eventId]
|
|
2909
|
+
});
|
|
2910
|
+
const { parentId: _drop, ...rest } = event;
|
|
2911
|
+
normalized.push({
|
|
2912
|
+
...rest,
|
|
2913
|
+
projection: {
|
|
2914
|
+
...event.projection,
|
|
2915
|
+
parentNormalized: true,
|
|
2916
|
+
originalParentId
|
|
2917
|
+
}
|
|
2918
|
+
});
|
|
2919
|
+
continue;
|
|
2920
|
+
}
|
|
2921
|
+
diagnostics.push({
|
|
2922
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
2923
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
2924
|
+
eventIds: [event.eventId]
|
|
2925
|
+
});
|
|
2926
|
+
normalized.push({
|
|
2927
|
+
...event,
|
|
2928
|
+
parentId: nextParent,
|
|
2929
|
+
projection: {
|
|
2930
|
+
...event.projection,
|
|
2931
|
+
parentNormalized: true,
|
|
2932
|
+
originalParentId
|
|
2933
|
+
}
|
|
2934
|
+
});
|
|
2935
|
+
}
|
|
2936
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
2937
|
+
normalized.sort((a, b) => {
|
|
2938
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
2939
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
2940
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2941
|
+
});
|
|
2942
|
+
return {
|
|
2943
|
+
logicalEvents: Object.freeze(normalized),
|
|
2944
|
+
diagnostics: Object.freeze(diagnostics)
|
|
2945
|
+
};
|
|
2946
|
+
}
|
|
2947
|
+
function resolveCanonicalToolName(event) {
|
|
2948
|
+
const attrs = event.attributes;
|
|
2949
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
2950
|
+
if (direct) return direct;
|
|
2951
|
+
const metadata = attrs?.metadata;
|
|
2952
|
+
if (isRecord6(metadata)) {
|
|
2953
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2954
|
+
if (nested) return nested;
|
|
2955
|
+
}
|
|
2956
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2957
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
2958
|
+
}
|
|
2959
|
+
return event.name;
|
|
2960
|
+
}
|
|
2961
|
+
function pickString(record, keys) {
|
|
2962
|
+
if (!record) return void 0;
|
|
2963
|
+
for (const key of keys) {
|
|
2964
|
+
const value = record[key];
|
|
2965
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2966
|
+
}
|
|
2967
|
+
return void 0;
|
|
2968
|
+
}
|
|
2969
|
+
|
|
2970
|
+
// packages/core/src/explain.ts
|
|
2971
|
+
function toDisplayStatus(status) {
|
|
2972
|
+
if (status === "ok") return "success";
|
|
2973
|
+
if (status === "error") return "error";
|
|
2974
|
+
if (status === "running") return "running";
|
|
2975
|
+
return "unknown";
|
|
2976
|
+
}
|
|
2977
|
+
function flatten(nodes, out = []) {
|
|
2978
|
+
for (const node of nodes) {
|
|
2979
|
+
out.push({ node, index: out.length + 1 });
|
|
2980
|
+
flatten(node.children, out);
|
|
2981
|
+
}
|
|
2982
|
+
return out;
|
|
2983
|
+
}
|
|
2984
|
+
function redactValue(redactor, key, value) {
|
|
2985
|
+
return redactor.redactValue(key, value);
|
|
2986
|
+
}
|
|
2987
|
+
function fact(id, label, value, redactor) {
|
|
2988
|
+
return {
|
|
2989
|
+
id,
|
|
2990
|
+
label,
|
|
2991
|
+
value: redactValue(redactor, id.split(".").at(-1) ?? id, value),
|
|
2992
|
+
source: "trace",
|
|
2993
|
+
confidence: "observed"
|
|
2994
|
+
};
|
|
2995
|
+
}
|
|
2996
|
+
function topKinds(run) {
|
|
2997
|
+
return Object.entries(run.metadata.kinds).filter(([, count]) => count > 0).sort((a, b) => {
|
|
2998
|
+
if (b[1] !== a[1]) return b[1] - a[1];
|
|
2999
|
+
return a[0].localeCompare(b[0]);
|
|
3000
|
+
}).slice(0, 5).map(([kind, count]) => `${kind}:${count}`);
|
|
3001
|
+
}
|
|
3002
|
+
function countErrorNodes(nodes) {
|
|
3003
|
+
return nodes.filter((entry) => entry.node.event.status === "error").length;
|
|
3004
|
+
}
|
|
3005
|
+
function slowestNode(nodes) {
|
|
3006
|
+
return nodes.filter(
|
|
3007
|
+
(entry) => entry.node.event.durationMs !== void 0 && // The RUN boundary spans the whole run, so its duration always dominates
|
|
3008
|
+
// and shadows every real step. Rank actual work nodes only, matching the
|
|
3009
|
+
// stats slowest-step ranking, which never counts the run envelope.
|
|
3010
|
+
entry.node.event.kind !== "RUN"
|
|
3011
|
+
).sort((a, b) => {
|
|
3012
|
+
const delta = (b.node.event.durationMs ?? 0) - (a.node.event.durationMs ?? 0);
|
|
3013
|
+
return delta !== 0 ? delta : a.index - b.index;
|
|
3014
|
+
})[0];
|
|
3015
|
+
}
|
|
3016
|
+
function attributeFacts(nodes, redactor) {
|
|
3017
|
+
const facts = [];
|
|
3018
|
+
for (const entry of nodes) {
|
|
3019
|
+
const attrs = entry.node.event.attributes;
|
|
3020
|
+
if (attrs === void 0) continue;
|
|
3021
|
+
for (const key of Object.keys(attrs).sort()) {
|
|
3022
|
+
facts.push({
|
|
3023
|
+
id: `node.${entry.index}.attributes.${key}`,
|
|
3024
|
+
label: `${entry.node.event.name} attribute ${key}`,
|
|
3025
|
+
value: redactValue(redactor, key, attrs[key]),
|
|
3026
|
+
source: "trace",
|
|
3027
|
+
confidence: "observed"
|
|
3028
|
+
});
|
|
3029
|
+
if (facts.length >= 8) return facts;
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
return facts;
|
|
3033
|
+
}
|
|
3034
|
+
function buildFacts(run, redactor, events) {
|
|
3035
|
+
const nodes = flatten(run.children);
|
|
3036
|
+
const facts = [
|
|
3037
|
+
fact("run.id", "Run id", run.runId, redactor),
|
|
3038
|
+
fact("run.name", "Run name", run.name ?? run.runId, redactor),
|
|
3039
|
+
fact("run.status", "Run status", toDisplayStatus(run.status), redactor),
|
|
3040
|
+
fact("run.totalEvents", "Total events", run.metadata.totalEvents, redactor),
|
|
3041
|
+
fact("run.stepCount", "Top-level step count", run.children.length, redactor),
|
|
3042
|
+
fact("run.nodeCount", "Total node count", nodes.length, redactor),
|
|
3043
|
+
fact("run.errorNodeCount", "Error node count", countErrorNodes(nodes), redactor),
|
|
3044
|
+
fact("run.kinds", "Observed kind mix", topKinds(run), redactor)
|
|
3045
|
+
];
|
|
3046
|
+
if (run.durationMs !== void 0) {
|
|
3047
|
+
facts.push(fact("run.durationMs", "Run duration milliseconds", run.durationMs, redactor));
|
|
3048
|
+
}
|
|
3049
|
+
const slowest = slowestNode(nodes);
|
|
3050
|
+
if (slowest !== void 0) {
|
|
3051
|
+
facts.push(
|
|
3052
|
+
fact("run.slowestNode", "Slowest observed node", {
|
|
3053
|
+
name: slowest.node.event.name,
|
|
3054
|
+
kind: slowest.node.event.kind,
|
|
3055
|
+
durationMs: slowest.node.event.durationMs
|
|
3056
|
+
}, redactor)
|
|
3057
|
+
);
|
|
3058
|
+
}
|
|
3059
|
+
if (events !== void 0) {
|
|
3060
|
+
const scoped = events.filter((event) => event.runId === run.runId);
|
|
3061
|
+
const { logicalEvents: logical } = projectLogicalEvents(scoped);
|
|
3062
|
+
const logicalSteps = logical.filter((event) => event.kind !== "RUN");
|
|
3063
|
+
facts.push(
|
|
3064
|
+
fact("run.rawEventCount", "Raw persisted event count", scoped.length, redactor),
|
|
3065
|
+
fact(
|
|
3066
|
+
"run.logicalStepCount",
|
|
3067
|
+
"Logical lifecycle step count",
|
|
3068
|
+
logicalSteps.length,
|
|
3069
|
+
redactor
|
|
3070
|
+
),
|
|
3071
|
+
fact(
|
|
3072
|
+
"run.logicalEventCount",
|
|
3073
|
+
"Logical lifecycle event count (including RUN)",
|
|
3074
|
+
logical.length,
|
|
3075
|
+
redactor
|
|
3076
|
+
)
|
|
3077
|
+
);
|
|
3078
|
+
const slowestLogical = [...logicalSteps].filter((event) => typeof event.durationMs === "number").sort((a, b) => {
|
|
3079
|
+
const delta = (b.durationMs ?? 0) - (a.durationMs ?? 0);
|
|
3080
|
+
return delta !== 0 ? delta : a.eventId.localeCompare(b.eventId);
|
|
3081
|
+
})[0];
|
|
3082
|
+
if (slowestLogical !== void 0) {
|
|
3083
|
+
facts.push(
|
|
3084
|
+
fact("run.slowestLogicalStep", "Slowest logical lifecycle step", {
|
|
3085
|
+
name: slowestLogical.name,
|
|
3086
|
+
kind: slowestLogical.kind,
|
|
3087
|
+
durationMs: slowestLogical.durationMs
|
|
3088
|
+
}, redactor)
|
|
3089
|
+
);
|
|
3090
|
+
}
|
|
3091
|
+
}
|
|
3092
|
+
facts.push(...attributeFacts(nodes, redactor));
|
|
3093
|
+
return facts;
|
|
3094
|
+
}
|
|
3095
|
+
function buildInferences(run, facts) {
|
|
3096
|
+
const inferences = [];
|
|
3097
|
+
const errorFact = facts.find((item) => item.id === "run.errorNodeCount");
|
|
3098
|
+
const kindFact = facts.find((item) => item.id === "run.kinds");
|
|
3099
|
+
const durationFact = facts.find((item) => item.id === "run.durationMs");
|
|
3100
|
+
const errorNodeCount = typeof errorFact?.value === "number" ? errorFact.value : 0;
|
|
3101
|
+
if (run.status === "error" || errorNodeCount > 0) {
|
|
3102
|
+
inferences.push({
|
|
3103
|
+
id: "outcome.error",
|
|
3104
|
+
label: "Outcome",
|
|
3105
|
+
text: "The run recorded an error status or at least one error node.",
|
|
3106
|
+
evidence: ["run.status", "run.errorNodeCount"],
|
|
3107
|
+
confidence: "deterministic"
|
|
3108
|
+
});
|
|
3109
|
+
} else if (run.status === "ok") {
|
|
3110
|
+
inferences.push({
|
|
3111
|
+
id: "outcome.success",
|
|
3112
|
+
label: "Outcome",
|
|
3113
|
+
text: "The run completed without observed error nodes.",
|
|
3114
|
+
evidence: ["run.status", "run.errorNodeCount"],
|
|
3115
|
+
confidence: "deterministic"
|
|
3116
|
+
});
|
|
3117
|
+
}
|
|
3118
|
+
if (kindFact !== void 0) {
|
|
3119
|
+
inferences.push({
|
|
3120
|
+
id: "shape.kind-mix",
|
|
3121
|
+
label: "Trace shape",
|
|
3122
|
+
text: "The explanation is based on the observed event kind mix, not generated content.",
|
|
3123
|
+
evidence: [kindFact.id],
|
|
3124
|
+
confidence: "deterministic"
|
|
3125
|
+
});
|
|
3126
|
+
}
|
|
3127
|
+
if (durationFact !== void 0) {
|
|
3128
|
+
inferences.push({
|
|
3129
|
+
id: "timing.duration",
|
|
3130
|
+
label: "Timing",
|
|
3131
|
+
text: "Timing claims are limited to persisted duration fields in the trace.",
|
|
3132
|
+
evidence: [durationFact.id],
|
|
3133
|
+
confidence: "deterministic"
|
|
3134
|
+
});
|
|
3135
|
+
}
|
|
3136
|
+
return inferences;
|
|
3137
|
+
}
|
|
3138
|
+
function buildLocalExplanation(run, options = {}) {
|
|
3139
|
+
const redactionProfile = options.redactionProfile ?? "local";
|
|
3140
|
+
const resolved = resolveRedactionProfile(redactionProfile);
|
|
3141
|
+
const redactor = new Redactor({ extraKeys: resolved.extraKeys });
|
|
3142
|
+
const mode = options.mode ?? "local";
|
|
3143
|
+
const facts = buildFacts(run, redactor, options.events);
|
|
3144
|
+
return {
|
|
3145
|
+
mode,
|
|
3146
|
+
runId: String(redactValue(redactor, "runId", run.runId)),
|
|
3147
|
+
...run.name !== void 0 ? { name: String(redactValue(redactor, "name", run.name)) } : {},
|
|
3148
|
+
...run.status !== void 0 ? { status: toDisplayStatus(run.status) } : {},
|
|
3149
|
+
redactionProfile,
|
|
3150
|
+
facts,
|
|
3151
|
+
inferences: mode === "dry-run" ? [] : buildInferences(run, facts),
|
|
3152
|
+
notes: [
|
|
3153
|
+
"Generated locally without provider or network calls.",
|
|
3154
|
+
"Facts are observed from normalized trace data; inferences are deterministic labels."
|
|
3155
|
+
]
|
|
3156
|
+
};
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
// packages/core/src/outcomes/types.ts
|
|
3160
|
+
var OBSERVED_OUTCOME_STATUSES = [
|
|
3161
|
+
"passed",
|
|
3162
|
+
"failed",
|
|
3163
|
+
"unknown",
|
|
2768
3164
|
"skipped"
|
|
2769
3165
|
];
|
|
2770
3166
|
var OBSERVED_OUTCOME_METHODS = [
|
|
@@ -3479,7 +3875,17 @@ function enrichSessionSummary(summary, runs, options = {}) {
|
|
|
3479
3875
|
|
|
3480
3876
|
// packages/core/src/sessions/checks.ts
|
|
3481
3877
|
function emptySummary() {
|
|
3482
|
-
return {
|
|
3878
|
+
return {
|
|
3879
|
+
passed: 0,
|
|
3880
|
+
failed: 0,
|
|
3881
|
+
warnings: 0,
|
|
3882
|
+
errors: 0,
|
|
3883
|
+
rulesEvaluated: 0,
|
|
3884
|
+
rulesPassed: 0,
|
|
3885
|
+
rulesWarning: 0,
|
|
3886
|
+
rulesFailed: 0,
|
|
3887
|
+
rulesError: 0
|
|
3888
|
+
};
|
|
3483
3889
|
}
|
|
3484
3890
|
function mergeSummary(target, source) {
|
|
3485
3891
|
return {
|
|
@@ -3487,7 +3893,11 @@ function mergeSummary(target, source) {
|
|
|
3487
3893
|
failed: target.failed + source.failed,
|
|
3488
3894
|
warnings: target.warnings + source.warnings,
|
|
3489
3895
|
errors: target.errors + source.errors,
|
|
3490
|
-
rulesEvaluated: target.rulesEvaluated + source.rulesEvaluated
|
|
3896
|
+
rulesEvaluated: target.rulesEvaluated + source.rulesEvaluated,
|
|
3897
|
+
rulesPassed: target.rulesPassed + source.rulesPassed,
|
|
3898
|
+
rulesWarning: target.rulesWarning + source.rulesWarning,
|
|
3899
|
+
rulesFailed: target.rulesFailed + source.rulesFailed,
|
|
3900
|
+
rulesError: target.rulesError + source.rulesError
|
|
3491
3901
|
};
|
|
3492
3902
|
}
|
|
3493
3903
|
function sessionDiagnostic(code, message) {
|
|
@@ -3884,7 +4294,7 @@ var KNOWN_EVENTS = /* @__PURE__ */ new Set([
|
|
|
3884
4294
|
"step_started",
|
|
3885
4295
|
"step_completed"
|
|
3886
4296
|
]);
|
|
3887
|
-
function
|
|
4297
|
+
function isRecord7(value) {
|
|
3888
4298
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3889
4299
|
}
|
|
3890
4300
|
function safeParse(line) {
|
|
@@ -3906,7 +4316,7 @@ async function isAgentInspectTrace(filePath) {
|
|
|
3906
4316
|
if (trimmed === "") continue;
|
|
3907
4317
|
const parsed = safeParse(trimmed);
|
|
3908
4318
|
if (!parsed) continue;
|
|
3909
|
-
if (!
|
|
4319
|
+
if (!isRecord7(parsed)) continue;
|
|
3910
4320
|
checked += 1;
|
|
3911
4321
|
if (isTraceEvent(parsed)) return true;
|
|
3912
4322
|
const ev = parsed.event;
|
|
@@ -4034,691 +4444,329 @@ function buildPlaceholderArtifact() {
|
|
|
4034
4444
|
note: PLACEHOLDER_NOTE
|
|
4035
4445
|
};
|
|
4036
4446
|
}
|
|
4037
|
-
|
|
4038
|
-
// packages/core/src/bundle/summary.ts
|
|
4039
|
-
function markdownTable(rows) {
|
|
4040
|
-
const lines = ["| Field | Value |", "| --- | --- |"];
|
|
4041
|
-
for (const [key, value] of rows) {
|
|
4042
|
-
lines.push(`| ${key} | ${value ?? "unknown"} |`);
|
|
4043
|
-
}
|
|
4044
|
-
return lines.join("\n");
|
|
4045
|
-
}
|
|
4046
|
-
function buildBundleSummaryMarkdown(parts) {
|
|
4047
|
-
const { metadata, checks, redaction } = parts;
|
|
4048
|
-
const lines = [
|
|
4049
|
-
"# AgentInspect trace bundle",
|
|
4050
|
-
"",
|
|
4051
|
-
metadata.note,
|
|
4052
|
-
"",
|
|
4053
|
-
"## Overview",
|
|
4054
|
-
"",
|
|
4055
|
-
markdownTable([
|
|
4056
|
-
["Created", metadata.createdAt],
|
|
4057
|
-
["AgentInspect", metadata.agentInspectVersion],
|
|
4058
|
-
["Redaction profile", metadata.redactionProfile],
|
|
4059
|
-
["Safe status", metadata.safeStatus],
|
|
4060
|
-
["Source traces", metadata.sourceTraceCount],
|
|
4061
|
-
["Runs", metadata.runIds.join(", ")],
|
|
4062
|
-
...metadata.sessionId ? [["Session", metadata.sessionId]] : [],
|
|
4063
|
-
...metadata.since ? [["Since", metadata.since]] : []
|
|
4064
|
-
]),
|
|
4065
|
-
"",
|
|
4066
|
-
"## Safety checks",
|
|
4067
|
-
"",
|
|
4068
|
-
`Aggregate: **${checks.aggregateStatus}**`,
|
|
4069
|
-
""
|
|
4070
|
-
];
|
|
4071
|
-
for (const run of checks.runs) {
|
|
4072
|
-
const source = run.sourceStatus !== void 0 && run.sourceStatus !== run.status ? `; source ${run.sourceStatus}` : "";
|
|
4073
|
-
lines.push(
|
|
4074
|
-
`- \`${run.runId}\`: artifact ${run.status}${source} (${run.findings} finding(s), ${run.errors} error(s), ${run.warnings} warning(s))`
|
|
4075
|
-
);
|
|
4076
|
-
}
|
|
4077
|
-
lines.push("", "## Redaction", "", `Total findings: ${redaction.totalFindings}`, "");
|
|
4078
|
-
for (const run of redaction.runs) {
|
|
4079
|
-
const detectors = run.detectors.length > 0 ? run.detectors.join(", ") : "none";
|
|
4080
|
-
lines.push(`- \`${run.runId}\`: ${run.findings} finding(s); detectors: ${detectors}`);
|
|
4081
|
-
}
|
|
4082
|
-
lines.push(
|
|
4083
|
-
"",
|
|
4084
|
-
"## Files",
|
|
4085
|
-
"",
|
|
4086
|
-
...metadata.files.map((file) => `- \`${file}\``),
|
|
4087
|
-
"",
|
|
4088
|
-
"_Review every generated artifact before sharing outside your team._",
|
|
4089
|
-
""
|
|
4090
|
-
);
|
|
4091
|
-
return lines.join("\n");
|
|
4092
|
-
}
|
|
4093
|
-
function sanitizeBundleRunId(runId) {
|
|
4094
|
-
let safe = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
4095
|
-
safe = path7.basename(safe);
|
|
4096
|
-
safe = safe.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
|
4097
|
-
if (safe === "" || safe === "." || safe === "..") {
|
|
4098
|
-
safe = "run_unknown";
|
|
4099
|
-
}
|
|
4100
|
-
return safe;
|
|
4101
|
-
}
|
|
4102
|
-
function bundleRunAssetRelativePath(runId, extension) {
|
|
4103
|
-
const safe = sanitizeBundleRunId(runId);
|
|
4104
|
-
const ext = extension.startsWith(".") ? extension : `.${extension}`;
|
|
4105
|
-
return path7.posix.join("assets", "runs", `${safe}${ext}`);
|
|
4106
|
-
}
|
|
4107
|
-
function assertBundlePathContained(outputDir, relativePath) {
|
|
4108
|
-
const base = path7.resolve(outputDir);
|
|
4109
|
-
const resolved = path7.resolve(base, relativePath);
|
|
4110
|
-
if (resolved !== base && !resolved.startsWith(base + path7.sep)) {
|
|
4111
|
-
throw new Error(`Bundle path escapes output directory: ${relativePath}`);
|
|
4112
|
-
}
|
|
4113
|
-
return resolved;
|
|
4114
|
-
}
|
|
4115
|
-
function normalizeBundleOutputPath(out, options) {
|
|
4116
|
-
const trimmed = out.trim();
|
|
4117
|
-
if (trimmed === "") {
|
|
4118
|
-
throw new Error("--out requires a non-empty path.");
|
|
4119
|
-
}
|
|
4120
|
-
const resolved = path7.resolve(trimmed);
|
|
4121
|
-
if (options?.preserveZipExtension !== true && resolved.toLowerCase().endsWith(".zip")) {
|
|
4122
|
-
return resolved.slice(0, -4);
|
|
4123
|
-
}
|
|
4124
|
-
return resolved;
|
|
4125
|
-
}
|
|
4126
|
-
function defaultBundleOutputPath(runIds) {
|
|
4127
|
-
const label = runIds.length === 1 ? sanitizeBundleRunId(runIds[0]) : `multi-${runIds.length}`;
|
|
4128
|
-
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
4129
|
-
return path7.resolve(`agent-inspect-bundle-${label}-${stamp}`);
|
|
4130
|
-
}
|
|
4131
|
-
|
|
4132
|
-
// packages/core/src/evidence/types.ts
|
|
4133
|
-
var EVIDENCE_FORMAT_VERSION = "1.0";
|
|
4134
|
-
var EVIDENCE_ASSESSMENT_NOTE = "Best-effort local safety verification only; not a compliance certification.";
|
|
4135
|
-
var EVIDENCE_MANIFEST_FILENAME = "evidence.json";
|
|
4136
|
-
var EVIDENCE_RESOLVED_CONTRACT_FILENAME = "contract.resolved.json";
|
|
4137
|
-
var SHA256_RE = /^[a-f0-9]{64}$/i;
|
|
4138
|
-
function sha256Hex(data) {
|
|
4139
|
-
const hash = createHash("sha256");
|
|
4140
|
-
if (typeof data === "string") {
|
|
4141
|
-
hash.update(data, "utf8");
|
|
4142
|
-
} else {
|
|
4143
|
-
hash.update(data);
|
|
4144
|
-
}
|
|
4145
|
-
return hash.digest("hex");
|
|
4146
|
-
}
|
|
4147
|
-
function isSha256Hex(value) {
|
|
4148
|
-
return SHA256_RE.test(value);
|
|
4149
|
-
}
|
|
4150
|
-
function sha256Equals(expected, actual) {
|
|
4151
|
-
if (!isSha256Hex(expected) || !isSha256Hex(actual)) {
|
|
4152
|
-
return false;
|
|
4153
|
-
}
|
|
4154
|
-
const a = expected.toLowerCase();
|
|
4155
|
-
const b = actual.toLowerCase();
|
|
4156
|
-
if (a.length !== b.length) return false;
|
|
4157
|
-
let mismatch = 0;
|
|
4158
|
-
for (let i = 0; i < a.length; i += 1) {
|
|
4159
|
-
mismatch |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
4160
|
-
}
|
|
4161
|
-
return mismatch === 0;
|
|
4162
|
-
}
|
|
4163
|
-
function assertEvidenceRelativePath(relativePath) {
|
|
4164
|
-
if (typeof relativePath !== "string" || relativePath.trim() === "") {
|
|
4165
|
-
throw new Error("Evidence file path must be a non-empty relative path.");
|
|
4166
|
-
}
|
|
4167
|
-
const trimmed = relativePath.trim().replaceAll("\\", "/");
|
|
4168
|
-
if (path7.isAbsolute(trimmed) || trimmed.startsWith("/")) {
|
|
4169
|
-
throw new Error(`Evidence file path must be relative: ${relativePath}`);
|
|
4170
|
-
}
|
|
4171
|
-
const parts = trimmed.split("/").filter((part) => part !== "");
|
|
4172
|
-
if (parts.length === 0) {
|
|
4173
|
-
throw new Error(`Evidence file path must be relative: ${relativePath}`);
|
|
4174
|
-
}
|
|
4175
|
-
for (const part of parts) {
|
|
4176
|
-
if (part === "." || part === "..") {
|
|
4177
|
-
throw new Error(`Evidence file path must not contain "." or "..": ${relativePath}`);
|
|
4178
|
-
}
|
|
4179
|
-
}
|
|
4180
|
-
return parts.join("/");
|
|
4181
|
-
}
|
|
4182
|
-
|
|
4183
|
-
// packages/core/src/evidence/manifest.ts
|
|
4184
|
-
function stable(value) {
|
|
4185
|
-
if (Array.isArray(value)) return value.map(stable);
|
|
4186
|
-
if (value === null || typeof value !== "object") return value;
|
|
4187
|
-
const record = value;
|
|
4188
|
-
return Object.fromEntries(
|
|
4189
|
-
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable(record[key])])
|
|
4190
|
-
);
|
|
4191
|
-
}
|
|
4192
|
-
function serializeEvidenceManifest(manifest) {
|
|
4193
|
-
return `${JSON.stringify(stable(manifest), null, 2)}
|
|
4194
|
-
`;
|
|
4195
|
-
}
|
|
4196
|
-
function inferEvidenceFileRole(relativePath) {
|
|
4197
|
-
const normalized = assertEvidenceRelativePath(relativePath);
|
|
4198
|
-
const base = normalized.includes("/") ? normalized.slice(normalized.lastIndexOf("/") + 1) : normalized;
|
|
4199
|
-
if (base === "evidence.html" || base === "trace.html" || base.endsWith(".html")) {
|
|
4200
|
-
return "report";
|
|
4201
|
-
}
|
|
4202
|
-
if (base === "trace.jsonl" || base.endsWith(".jsonl")) {
|
|
4203
|
-
return "redacted-trace";
|
|
4204
|
-
}
|
|
4205
|
-
if (base === "check-results.json") {
|
|
4206
|
-
return "checks";
|
|
4207
|
-
}
|
|
4208
|
-
if (base === "contract.resolved.json") {
|
|
4209
|
-
return "contract";
|
|
4210
|
-
}
|
|
4211
|
-
if (base === "redaction-report.json") {
|
|
4212
|
-
return "redaction-report";
|
|
4213
|
-
}
|
|
4214
|
-
if (base === "summary.md") {
|
|
4215
|
-
return "summary";
|
|
4216
|
-
}
|
|
4217
|
-
return "other";
|
|
4218
|
-
}
|
|
4219
|
-
function buildEvidenceFileEntries(files) {
|
|
4220
|
-
const byPath = /* @__PURE__ */ new Map();
|
|
4221
|
-
for (const file of files) {
|
|
4222
|
-
const relativePath = assertEvidenceRelativePath(file.path);
|
|
4223
|
-
if (relativePath === EVIDENCE_MANIFEST_FILENAME) {
|
|
4224
|
-
throw new Error(
|
|
4225
|
-
`Do not include ${EVIDENCE_MANIFEST_FILENAME} in packaged file hashes (self-hash is undefined).`
|
|
4226
|
-
);
|
|
4227
|
-
}
|
|
4228
|
-
if (byPath.has(relativePath)) {
|
|
4229
|
-
throw new Error(`Duplicate evidence file path: ${relativePath}`);
|
|
4230
|
-
}
|
|
4231
|
-
byPath.set(relativePath, {
|
|
4232
|
-
path: relativePath,
|
|
4233
|
-
sha256: sha256Hex(file.content),
|
|
4234
|
-
role: file.role ?? inferEvidenceFileRole(relativePath)
|
|
4235
|
-
});
|
|
4236
|
-
}
|
|
4237
|
-
return [...byPath.values()].sort((a, b) => a.path.localeCompare(b.path));
|
|
4238
|
-
}
|
|
4239
|
-
function buildEvidenceManifest(parts) {
|
|
4240
|
-
const runIds = [...parts.runIds];
|
|
4241
|
-
if (runIds.length === 0) {
|
|
4242
|
-
throw new Error("Evidence manifest requires at least one run id.");
|
|
4243
|
-
}
|
|
4244
|
-
for (const item of parts.sourceHashes) {
|
|
4245
|
-
if (!runIds.includes(item.runId)) {
|
|
4246
|
-
throw new Error(`sourceHashes runId "${item.runId}" is not listed in source.runIds.`);
|
|
4247
|
-
}
|
|
4248
|
-
if (item.algorithm !== "sha256") {
|
|
4249
|
-
throw new Error(`Unsupported source hash algorithm: ${item.algorithm}`);
|
|
4250
|
-
}
|
|
4251
|
-
}
|
|
4252
|
-
const assessment = {
|
|
4253
|
-
status: parts.assessmentStatus,
|
|
4254
|
-
note: parts.note ?? EVIDENCE_ASSESSMENT_NOTE
|
|
4255
|
-
};
|
|
4256
|
-
if (parts.sourceStatus !== void 0) {
|
|
4257
|
-
assessment.sourceStatus = parts.sourceStatus;
|
|
4258
|
-
}
|
|
4259
|
-
return {
|
|
4260
|
-
evidenceFormatVersion: EVIDENCE_FORMAT_VERSION,
|
|
4261
|
-
generator: {
|
|
4262
|
-
name: parts.generatorName ?? "agent-inspect",
|
|
4263
|
-
version: parts.generatorVersion
|
|
4264
|
-
},
|
|
4265
|
-
createdAt: parts.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
4266
|
-
source: {
|
|
4267
|
-
runIds,
|
|
4268
|
-
traceSchemaVersions: [...parts.traceSchemaVersions].sort((a, b) => a.localeCompare(b)),
|
|
4269
|
-
sourceHashes: [...parts.sourceHashes].sort((a, b) => a.runId.localeCompare(b.runId))
|
|
4270
|
-
},
|
|
4271
|
-
policy: {
|
|
4272
|
-
redactionProfile: parts.redactionProfile,
|
|
4273
|
-
verificationPolicy: parts.verificationPolicy ?? parts.redactionProfile
|
|
4274
|
-
},
|
|
4275
|
-
assessment,
|
|
4276
|
-
...parts.semantics !== void 0 ? { semantics: { ...parts.semantics } } : {},
|
|
4277
|
-
...parts.contract !== void 0 ? { contract: { ...parts.contract } } : {},
|
|
4278
|
-
files: buildEvidenceFileEntries(parts.files)
|
|
4279
|
-
};
|
|
4280
|
-
}
|
|
4281
|
-
function validateEvidenceManifest(value) {
|
|
4282
|
-
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
4283
|
-
throw new Error("Evidence manifest must be a JSON object.");
|
|
4447
|
+
|
|
4448
|
+
// packages/core/src/bundle/summary.ts
|
|
4449
|
+
function markdownTable(rows) {
|
|
4450
|
+
const lines = ["| Field | Value |", "| --- | --- |"];
|
|
4451
|
+
for (const [key, value] of rows) {
|
|
4452
|
+
lines.push(`| ${key} | ${value ?? "unknown"} |`);
|
|
4284
4453
|
}
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4454
|
+
return lines.join("\n");
|
|
4455
|
+
}
|
|
4456
|
+
function buildBundleSummaryMarkdown(parts) {
|
|
4457
|
+
const { metadata, checks, redaction } = parts;
|
|
4458
|
+
const lines = [
|
|
4459
|
+
"# AgentInspect trace bundle",
|
|
4460
|
+
"",
|
|
4461
|
+
metadata.note,
|
|
4462
|
+
"",
|
|
4463
|
+
"## Overview",
|
|
4464
|
+
"",
|
|
4465
|
+
markdownTable([
|
|
4466
|
+
["Created", metadata.createdAt],
|
|
4467
|
+
["AgentInspect", metadata.agentInspectVersion],
|
|
4468
|
+
["Redaction profile", metadata.redactionProfile],
|
|
4469
|
+
["Safe status", metadata.safeStatus],
|
|
4470
|
+
["Source traces", metadata.sourceTraceCount],
|
|
4471
|
+
["Runs", metadata.runIds.join(", ")],
|
|
4472
|
+
...metadata.sessionId ? [["Session", metadata.sessionId]] : [],
|
|
4473
|
+
...metadata.since ? [["Since", metadata.since]] : []
|
|
4474
|
+
]),
|
|
4475
|
+
"",
|
|
4476
|
+
"## Safety checks",
|
|
4477
|
+
"",
|
|
4478
|
+
`Aggregate: **${checks.aggregateStatus}**`,
|
|
4479
|
+
""
|
|
4480
|
+
];
|
|
4481
|
+
for (const run of checks.runs) {
|
|
4482
|
+
const source = run.sourceStatus !== void 0 && run.sourceStatus !== run.status ? `; source ${run.sourceStatus}` : "";
|
|
4483
|
+
lines.push(
|
|
4484
|
+
`- \`${run.runId}\`: artifact ${run.status}${source} (${run.findings} finding(s), ${run.errors} error(s), ${run.warnings} warning(s))`
|
|
4289
4485
|
);
|
|
4290
4486
|
}
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
const source = record.source;
|
|
4296
|
-
if (source === null || typeof source !== "object" || Array.isArray(source)) {
|
|
4297
|
-
throw new Error("Evidence manifest requires source.");
|
|
4298
|
-
}
|
|
4299
|
-
const sourceRecord = source;
|
|
4300
|
-
if (!Array.isArray(sourceRecord.runIds) || sourceRecord.runIds.length === 0) {
|
|
4301
|
-
throw new Error("Evidence manifest source.runIds must be a non-empty array.");
|
|
4302
|
-
}
|
|
4303
|
-
if (!Array.isArray(sourceRecord.traceSchemaVersions)) {
|
|
4304
|
-
throw new Error("Evidence manifest source.traceSchemaVersions must be an array.");
|
|
4305
|
-
}
|
|
4306
|
-
if (!Array.isArray(sourceRecord.sourceHashes)) {
|
|
4307
|
-
throw new Error("Evidence manifest source.sourceHashes must be an array.");
|
|
4308
|
-
}
|
|
4309
|
-
const policy = record.policy;
|
|
4310
|
-
if (policy === null || typeof policy !== "object" || Array.isArray(policy)) {
|
|
4311
|
-
throw new Error("Evidence manifest requires policy.");
|
|
4312
|
-
}
|
|
4313
|
-
const assessment = record.assessment;
|
|
4314
|
-
if (assessment === null || typeof assessment !== "object" || Array.isArray(assessment) || typeof assessment.status !== "string") {
|
|
4315
|
-
throw new Error("Evidence manifest requires assessment.status.");
|
|
4316
|
-
}
|
|
4317
|
-
if (!Array.isArray(record.files) || record.files.length === 0) {
|
|
4318
|
-
throw new Error("Evidence manifest files must be a non-empty array.");
|
|
4319
|
-
}
|
|
4320
|
-
for (const file of record.files) {
|
|
4321
|
-
if (file === null || typeof file !== "object" || Array.isArray(file)) {
|
|
4322
|
-
throw new Error("Evidence manifest file entries must be objects.");
|
|
4323
|
-
}
|
|
4324
|
-
const entry = file;
|
|
4325
|
-
if (typeof entry.path !== "string") {
|
|
4326
|
-
throw new Error("Evidence file entry requires path.");
|
|
4327
|
-
}
|
|
4328
|
-
assertEvidenceRelativePath(entry.path);
|
|
4329
|
-
if (typeof entry.sha256 !== "string" || !/^[a-f0-9]{64}$/i.test(entry.sha256)) {
|
|
4330
|
-
throw new Error(`Evidence file entry requires sha256 hex for ${entry.path}.`);
|
|
4331
|
-
}
|
|
4487
|
+
lines.push("", "## Redaction", "", `Total findings: ${redaction.totalFindings}`, "");
|
|
4488
|
+
for (const run of redaction.runs) {
|
|
4489
|
+
const detectors = run.detectors.length > 0 ? run.detectors.join(", ") : "none";
|
|
4490
|
+
lines.push(`- \`${run.runId}\`: ${run.findings} finding(s); detectors: ${detectors}`);
|
|
4332
4491
|
}
|
|
4333
|
-
|
|
4492
|
+
lines.push(
|
|
4493
|
+
"",
|
|
4494
|
+
"## Files",
|
|
4495
|
+
"",
|
|
4496
|
+
...metadata.files.map((file) => `- \`${file}\``),
|
|
4497
|
+
"",
|
|
4498
|
+
"_Review every generated artifact before sharing outside your team._",
|
|
4499
|
+
""
|
|
4500
|
+
);
|
|
4501
|
+
return lines.join("\n");
|
|
4334
4502
|
}
|
|
4335
|
-
function
|
|
4336
|
-
let
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
throw new Error(`Evidence manifest is not valid JSON: ${message}`);
|
|
4503
|
+
function sanitizeBundleRunId(runId) {
|
|
4504
|
+
let safe = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
4505
|
+
safe = path7.basename(safe);
|
|
4506
|
+
safe = safe.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
|
4507
|
+
if (safe === "" || safe === "." || safe === "..") {
|
|
4508
|
+
safe = "run_unknown";
|
|
4342
4509
|
}
|
|
4343
|
-
return
|
|
4510
|
+
return safe;
|
|
4344
4511
|
}
|
|
4345
|
-
function
|
|
4346
|
-
const
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
if (trimmed === "") continue;
|
|
4350
|
-
try {
|
|
4351
|
-
const row = JSON.parse(trimmed);
|
|
4352
|
-
if (typeof row.schemaVersion === "string" && row.schemaVersion.trim() !== "") {
|
|
4353
|
-
versions.add(row.schemaVersion.trim());
|
|
4354
|
-
}
|
|
4355
|
-
} catch {
|
|
4356
|
-
}
|
|
4357
|
-
}
|
|
4358
|
-
return [...versions].sort((a, b) => a.localeCompare(b));
|
|
4512
|
+
function bundleRunAssetRelativePath(runId, extension) {
|
|
4513
|
+
const safe = sanitizeBundleRunId(runId);
|
|
4514
|
+
const ext = extension.startsWith(".") ? extension : `.${extension}`;
|
|
4515
|
+
return path7.posix.join("assets", "runs", `${safe}${ext}`);
|
|
4359
4516
|
}
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
4366
|
-
remediation: "For a file path, use openTraceFile(path).",
|
|
4367
|
-
relatedCodes: ["invalid_input"]
|
|
4368
|
-
},
|
|
4369
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
4370
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
4371
|
-
summary: "No trace reader could detect the input format.",
|
|
4372
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
4373
|
-
relatedCodes: ["unsupported_format"]
|
|
4374
|
-
},
|
|
4375
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
4376
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
4377
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
4378
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
4379
|
-
relatedCodes: ["ambiguous_format"]
|
|
4380
|
-
},
|
|
4381
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
4382
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
4383
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
4384
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
4385
|
-
},
|
|
4386
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
4387
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
4388
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
4389
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
4390
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
4391
|
-
},
|
|
4392
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
4393
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
4394
|
-
summary: "A parentId equals its own event/step id.",
|
|
4395
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
4396
|
-
relatedCodes: [
|
|
4397
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
4398
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
4399
|
-
]
|
|
4400
|
-
},
|
|
4401
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
4402
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
4403
|
-
summary: "Trace contains a parentId cycle.",
|
|
4404
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
4405
|
-
relatedCodes: ["structure.cycle"]
|
|
4406
|
-
},
|
|
4407
|
-
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
4408
|
-
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
4409
|
-
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
4410
|
-
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
4411
|
-
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
4412
|
-
},
|
|
4413
|
-
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
4414
|
-
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
4415
|
-
summary: 'inspectRun does not accept a "writer" option.',
|
|
4416
|
-
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
4517
|
+
function assertBundlePathContained(outputDir, relativePath) {
|
|
4518
|
+
const base = path7.resolve(outputDir);
|
|
4519
|
+
const resolved = path7.resolve(base, relativePath);
|
|
4520
|
+
if (resolved !== base && !resolved.startsWith(base + path7.sep)) {
|
|
4521
|
+
throw new Error(`Bundle path escapes output directory: ${relativePath}`);
|
|
4417
4522
|
}
|
|
4418
|
-
|
|
4419
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
4420
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
4421
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
4422
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
4523
|
+
return resolved;
|
|
4423
4524
|
}
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4525
|
+
function normalizeBundleOutputPath(out, options) {
|
|
4526
|
+
const trimmed = out.trim();
|
|
4527
|
+
if (trimmed === "") {
|
|
4528
|
+
throw new Error("--out requires a non-empty path.");
|
|
4529
|
+
}
|
|
4530
|
+
const resolved = path7.resolve(trimmed);
|
|
4531
|
+
if (options?.preserveZipExtension !== true && resolved.toLowerCase().endsWith(".zip")) {
|
|
4532
|
+
return resolved.slice(0, -4);
|
|
4533
|
+
}
|
|
4534
|
+
return resolved;
|
|
4428
4535
|
}
|
|
4429
|
-
function
|
|
4430
|
-
const
|
|
4431
|
-
|
|
4536
|
+
function defaultBundleOutputPath(runIds) {
|
|
4537
|
+
const label = runIds.length === 1 ? sanitizeBundleRunId(runIds[0]) : `multi-${runIds.length}`;
|
|
4538
|
+
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
4539
|
+
return path7.resolve(`agent-inspect-bundle-${label}-${stamp}`);
|
|
4432
4540
|
}
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4541
|
+
|
|
4542
|
+
// packages/core/src/evidence/types.ts
|
|
4543
|
+
var EVIDENCE_FORMAT_VERSION = "1.0";
|
|
4544
|
+
var EVIDENCE_ASSESSMENT_NOTE = "Best-effort local safety verification only; not a compliance certification.";
|
|
4545
|
+
var EVIDENCE_MANIFEST_FILENAME = "evidence.json";
|
|
4546
|
+
var EVIDENCE_RESOLVED_CONTRACT_FILENAME = "contract.resolved.json";
|
|
4547
|
+
var SHA256_RE = /^[a-f0-9]{64}$/i;
|
|
4548
|
+
function sha256Hex(data) {
|
|
4549
|
+
const hash = createHash("sha256");
|
|
4550
|
+
if (typeof data === "string") {
|
|
4551
|
+
hash.update(data, "utf8");
|
|
4552
|
+
} else {
|
|
4553
|
+
hash.update(data);
|
|
4554
|
+
}
|
|
4555
|
+
return hash.digest("hex");
|
|
4437
4556
|
}
|
|
4438
|
-
function
|
|
4439
|
-
return
|
|
4440
|
-
...event,
|
|
4441
|
-
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
4442
|
-
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
4443
|
-
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
4444
|
-
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
4445
|
-
};
|
|
4557
|
+
function isSha256Hex(value) {
|
|
4558
|
+
return SHA256_RE.test(value);
|
|
4446
4559
|
}
|
|
4447
|
-
function
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
...isRecord7(start.attributes) ? start.attributes : {}
|
|
4451
|
-
};
|
|
4452
|
-
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
4453
|
-
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
4454
|
-
if (complete.attributes?.errorStack !== void 0) {
|
|
4455
|
-
merged.errorStack = complete.attributes.errorStack;
|
|
4560
|
+
function sha256Equals(expected, actual) {
|
|
4561
|
+
if (!isSha256Hex(expected) || !isSha256Hex(actual)) {
|
|
4562
|
+
return false;
|
|
4456
4563
|
}
|
|
4457
|
-
|
|
4564
|
+
const a = expected.toLowerCase();
|
|
4565
|
+
const b = actual.toLowerCase();
|
|
4566
|
+
if (a.length !== b.length) return false;
|
|
4567
|
+
let mismatch = 0;
|
|
4568
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
4569
|
+
mismatch |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
4570
|
+
}
|
|
4571
|
+
return mismatch === 0;
|
|
4458
4572
|
}
|
|
4459
|
-
function
|
|
4460
|
-
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
}
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
projection: {
|
|
4475
|
-
paired: true,
|
|
4476
|
-
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
4477
|
-
parentNormalized: false,
|
|
4478
|
-
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
4573
|
+
function assertEvidenceRelativePath(relativePath) {
|
|
4574
|
+
if (typeof relativePath !== "string" || relativePath.trim() === "") {
|
|
4575
|
+
throw new Error("Evidence file path must be a non-empty relative path.");
|
|
4576
|
+
}
|
|
4577
|
+
const trimmed = relativePath.trim().replaceAll("\\", "/");
|
|
4578
|
+
if (path7.isAbsolute(trimmed) || trimmed.startsWith("/")) {
|
|
4579
|
+
throw new Error(`Evidence file path must be relative: ${relativePath}`);
|
|
4580
|
+
}
|
|
4581
|
+
const parts = trimmed.split("/").filter((part) => part !== "");
|
|
4582
|
+
if (parts.length === 0) {
|
|
4583
|
+
throw new Error(`Evidence file path must be relative: ${relativePath}`);
|
|
4584
|
+
}
|
|
4585
|
+
for (const part of parts) {
|
|
4586
|
+
if (part === "." || part === "..") {
|
|
4587
|
+
throw new Error(`Evidence file path must not contain "." or "..": ${relativePath}`);
|
|
4479
4588
|
}
|
|
4480
|
-
}
|
|
4589
|
+
}
|
|
4590
|
+
return parts.join("/");
|
|
4481
4591
|
}
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
}
|
|
4492
|
-
};
|
|
4592
|
+
|
|
4593
|
+
// packages/core/src/evidence/manifest.ts
|
|
4594
|
+
function stable(value) {
|
|
4595
|
+
if (Array.isArray(value)) return value.map(stable);
|
|
4596
|
+
if (value === null || typeof value !== "object") return value;
|
|
4597
|
+
const record = value;
|
|
4598
|
+
return Object.fromEntries(
|
|
4599
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable(record[key])])
|
|
4600
|
+
);
|
|
4493
4601
|
}
|
|
4494
|
-
function
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4602
|
+
function serializeEvidenceManifest(manifest) {
|
|
4603
|
+
return `${JSON.stringify(stable(manifest), null, 2)}
|
|
4604
|
+
`;
|
|
4605
|
+
}
|
|
4606
|
+
function inferEvidenceFileRole(relativePath) {
|
|
4607
|
+
const normalized = assertEvidenceRelativePath(relativePath);
|
|
4608
|
+
const base = normalized.includes("/") ? normalized.slice(normalized.lastIndexOf("/") + 1) : normalized;
|
|
4609
|
+
if (base === "evidence.html" || base === "trace.html" || base.endsWith(".html")) {
|
|
4610
|
+
return "report";
|
|
4501
4611
|
}
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
4505
|
-
const logical = [];
|
|
4506
|
-
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
4507
|
-
for (const runId of orderedRuns) {
|
|
4508
|
-
const runEvents = byRun.get(runId) ?? [];
|
|
4509
|
-
const starts = [];
|
|
4510
|
-
const completes = [];
|
|
4511
|
-
const others = [];
|
|
4512
|
-
for (const event of runEvents) {
|
|
4513
|
-
const legacy = legacyEvent(event);
|
|
4514
|
-
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
4515
|
-
starts.push(event);
|
|
4516
|
-
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
4517
|
-
completes.push(event);
|
|
4518
|
-
} else {
|
|
4519
|
-
others.push(event);
|
|
4520
|
-
}
|
|
4521
|
-
}
|
|
4522
|
-
const usedCompletes = /* @__PURE__ */ new Set();
|
|
4523
|
-
for (const start of starts) {
|
|
4524
|
-
const stepId = stepIdOf(start);
|
|
4525
|
-
let match;
|
|
4526
|
-
if (legacyEvent(start) === "run_started") {
|
|
4527
|
-
const candidates = completes.filter(
|
|
4528
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
4529
|
-
);
|
|
4530
|
-
if (candidates.length > 1) {
|
|
4531
|
-
diagnostics.push({
|
|
4532
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
4533
|
-
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
4534
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
4535
|
-
});
|
|
4536
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
4537
|
-
}
|
|
4538
|
-
match = candidates[0];
|
|
4539
|
-
} else if (stepId) {
|
|
4540
|
-
const candidates = completes.filter(
|
|
4541
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
4542
|
-
);
|
|
4543
|
-
if (candidates.length > 1) {
|
|
4544
|
-
diagnostics.push({
|
|
4545
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
4546
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
4547
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
4548
|
-
});
|
|
4549
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
4550
|
-
}
|
|
4551
|
-
match = candidates[0];
|
|
4552
|
-
}
|
|
4553
|
-
if (match) {
|
|
4554
|
-
usedCompletes.add(match.eventId);
|
|
4555
|
-
absorbedIds.add(match.eventId);
|
|
4556
|
-
const paired = pairStartComplete(start, match);
|
|
4557
|
-
logical.push(paired);
|
|
4558
|
-
logicalByRawId.set(start.eventId, paired);
|
|
4559
|
-
logicalByRawId.set(match.eventId, paired);
|
|
4560
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
4561
|
-
} else {
|
|
4562
|
-
diagnostics.push({
|
|
4563
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
4564
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
4565
|
-
eventIds: [start.eventId]
|
|
4566
|
-
});
|
|
4567
|
-
const alone = asLogical(start);
|
|
4568
|
-
logical.push(alone);
|
|
4569
|
-
logicalByRawId.set(start.eventId, alone);
|
|
4570
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
4571
|
-
}
|
|
4572
|
-
}
|
|
4573
|
-
for (const complete of completes) {
|
|
4574
|
-
if (usedCompletes.has(complete.eventId)) continue;
|
|
4575
|
-
diagnostics.push({
|
|
4576
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
4577
|
-
message: `No matching start for complete ${complete.eventId}.`,
|
|
4578
|
-
eventIds: [complete.eventId]
|
|
4579
|
-
});
|
|
4580
|
-
const alone = asLogical(complete);
|
|
4581
|
-
logical.push(alone);
|
|
4582
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
4583
|
-
const stepId = stepIdOf(complete);
|
|
4584
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
4585
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
4586
|
-
}
|
|
4587
|
-
}
|
|
4588
|
-
for (const event of others) {
|
|
4589
|
-
const alone = asLogical(event);
|
|
4590
|
-
logical.push(alone);
|
|
4591
|
-
logicalByRawId.set(event.eventId, alone);
|
|
4592
|
-
const stepId = stepIdOf(event);
|
|
4593
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
4594
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
4595
|
-
}
|
|
4596
|
-
}
|
|
4612
|
+
if (base === "trace.jsonl" || base.endsWith(".jsonl")) {
|
|
4613
|
+
return "redacted-trace";
|
|
4597
4614
|
}
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
if (originalParentId === event.eventId) {
|
|
4621
|
-
diagnostics.push({
|
|
4622
|
-
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
4623
|
-
message: formatProgrammaticDiagnostic(
|
|
4624
|
-
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
4625
|
-
`Removed self-parent edge on ${event.eventId}.`
|
|
4626
|
-
),
|
|
4627
|
-
eventIds: [event.eventId]
|
|
4628
|
-
});
|
|
4629
|
-
const { parentId: _drop, ...rest } = event;
|
|
4630
|
-
normalized.push({
|
|
4631
|
-
...rest,
|
|
4632
|
-
projection: {
|
|
4633
|
-
...event.projection,
|
|
4634
|
-
parentNormalized: true,
|
|
4635
|
-
originalParentId
|
|
4636
|
-
}
|
|
4637
|
-
});
|
|
4638
|
-
continue;
|
|
4639
|
-
}
|
|
4640
|
-
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
4641
|
-
const mapping = event.attributes?.parentMapping;
|
|
4642
|
-
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
4643
|
-
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
4644
|
-
if (!unresolved) {
|
|
4645
|
-
diagnostics.push({
|
|
4646
|
-
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
4647
|
-
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
4648
|
-
eventIds: [event.eventId]
|
|
4649
|
-
});
|
|
4650
|
-
}
|
|
4651
|
-
}
|
|
4652
|
-
normalized.push(event);
|
|
4653
|
-
continue;
|
|
4615
|
+
if (base === "check-results.json") {
|
|
4616
|
+
return "checks";
|
|
4617
|
+
}
|
|
4618
|
+
if (base === "contract.resolved.json") {
|
|
4619
|
+
return "contract";
|
|
4620
|
+
}
|
|
4621
|
+
if (base === "redaction-report.json") {
|
|
4622
|
+
return "redaction-report";
|
|
4623
|
+
}
|
|
4624
|
+
if (base === "summary.md") {
|
|
4625
|
+
return "summary";
|
|
4626
|
+
}
|
|
4627
|
+
return "other";
|
|
4628
|
+
}
|
|
4629
|
+
function buildEvidenceFileEntries(files) {
|
|
4630
|
+
const byPath = /* @__PURE__ */ new Map();
|
|
4631
|
+
for (const file of files) {
|
|
4632
|
+
const relativePath = assertEvidenceRelativePath(file.path);
|
|
4633
|
+
if (relativePath === EVIDENCE_MANIFEST_FILENAME) {
|
|
4634
|
+
throw new Error(
|
|
4635
|
+
`Do not include ${EVIDENCE_MANIFEST_FILENAME} in packaged file hashes (self-hash is undefined).`
|
|
4636
|
+
);
|
|
4654
4637
|
}
|
|
4655
|
-
if (
|
|
4656
|
-
|
|
4657
|
-
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
4658
|
-
message: formatProgrammaticDiagnostic(
|
|
4659
|
-
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
4660
|
-
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
4661
|
-
),
|
|
4662
|
-
eventIds: [event.eventId]
|
|
4663
|
-
});
|
|
4664
|
-
const { parentId: _drop, ...rest } = event;
|
|
4665
|
-
normalized.push({
|
|
4666
|
-
...rest,
|
|
4667
|
-
projection: {
|
|
4668
|
-
...event.projection,
|
|
4669
|
-
parentNormalized: true,
|
|
4670
|
-
originalParentId
|
|
4671
|
-
}
|
|
4672
|
-
});
|
|
4673
|
-
continue;
|
|
4638
|
+
if (byPath.has(relativePath)) {
|
|
4639
|
+
throw new Error(`Duplicate evidence file path: ${relativePath}`);
|
|
4674
4640
|
}
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
});
|
|
4680
|
-
normalized.push({
|
|
4681
|
-
...event,
|
|
4682
|
-
parentId: nextParent,
|
|
4683
|
-
projection: {
|
|
4684
|
-
...event.projection,
|
|
4685
|
-
parentNormalized: true,
|
|
4686
|
-
originalParentId
|
|
4687
|
-
}
|
|
4641
|
+
byPath.set(relativePath, {
|
|
4642
|
+
path: relativePath,
|
|
4643
|
+
sha256: sha256Hex(file.content),
|
|
4644
|
+
role: file.role ?? inferEvidenceFileRole(relativePath)
|
|
4688
4645
|
});
|
|
4689
4646
|
}
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4647
|
+
return [...byPath.values()].sort((a, b) => a.path.localeCompare(b.path));
|
|
4648
|
+
}
|
|
4649
|
+
function buildEvidenceManifest(parts) {
|
|
4650
|
+
const runIds = [...parts.runIds];
|
|
4651
|
+
if (runIds.length === 0) {
|
|
4652
|
+
throw new Error("Evidence manifest requires at least one run id.");
|
|
4653
|
+
}
|
|
4654
|
+
for (const item of parts.sourceHashes) {
|
|
4655
|
+
if (!runIds.includes(item.runId)) {
|
|
4656
|
+
throw new Error(`sourceHashes runId "${item.runId}" is not listed in source.runIds.`);
|
|
4657
|
+
}
|
|
4658
|
+
if (item.algorithm !== "sha256") {
|
|
4659
|
+
throw new Error(`Unsupported source hash algorithm: ${item.algorithm}`);
|
|
4660
|
+
}
|
|
4661
|
+
}
|
|
4662
|
+
const assessment = {
|
|
4663
|
+
status: parts.assessmentStatus,
|
|
4664
|
+
note: parts.note ?? EVIDENCE_ASSESSMENT_NOTE
|
|
4665
|
+
};
|
|
4666
|
+
if (parts.sourceStatus !== void 0) {
|
|
4667
|
+
assessment.sourceStatus = parts.sourceStatus;
|
|
4668
|
+
}
|
|
4696
4669
|
return {
|
|
4697
|
-
|
|
4698
|
-
|
|
4670
|
+
evidenceFormatVersion: EVIDENCE_FORMAT_VERSION,
|
|
4671
|
+
generator: {
|
|
4672
|
+
name: parts.generatorName ?? "agent-inspect",
|
|
4673
|
+
version: parts.generatorVersion
|
|
4674
|
+
},
|
|
4675
|
+
createdAt: parts.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
4676
|
+
source: {
|
|
4677
|
+
runIds,
|
|
4678
|
+
traceSchemaVersions: [...parts.traceSchemaVersions].sort((a, b) => a.localeCompare(b)),
|
|
4679
|
+
sourceHashes: [...parts.sourceHashes].sort((a, b) => a.runId.localeCompare(b.runId))
|
|
4680
|
+
},
|
|
4681
|
+
policy: {
|
|
4682
|
+
redactionProfile: parts.redactionProfile,
|
|
4683
|
+
verificationPolicy: parts.verificationPolicy ?? parts.redactionProfile
|
|
4684
|
+
},
|
|
4685
|
+
assessment,
|
|
4686
|
+
...parts.semantics !== void 0 ? { semantics: { ...parts.semantics } } : {},
|
|
4687
|
+
...parts.contract !== void 0 ? { contract: { ...parts.contract } } : {},
|
|
4688
|
+
...parts.inputs !== void 0 ? { inputs: { ...parts.inputs } } : {},
|
|
4689
|
+
files: buildEvidenceFileEntries(parts.files)
|
|
4699
4690
|
};
|
|
4700
4691
|
}
|
|
4701
|
-
function
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
if (direct) return direct;
|
|
4705
|
-
const metadata = attrs?.metadata;
|
|
4706
|
-
if (isRecord7(metadata)) {
|
|
4707
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
4708
|
-
if (nested) return nested;
|
|
4692
|
+
function validateEvidenceManifest(value) {
|
|
4693
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
4694
|
+
throw new Error("Evidence manifest must be a JSON object.");
|
|
4709
4695
|
}
|
|
4710
|
-
|
|
4711
|
-
|
|
4696
|
+
const record = value;
|
|
4697
|
+
if (record.evidenceFormatVersion !== EVIDENCE_FORMAT_VERSION) {
|
|
4698
|
+
throw new Error(
|
|
4699
|
+
`Unsupported evidenceFormatVersion: ${String(record.evidenceFormatVersion)}`
|
|
4700
|
+
);
|
|
4712
4701
|
}
|
|
4713
|
-
|
|
4702
|
+
const generator = record.generator;
|
|
4703
|
+
if (generator === null || typeof generator !== "object" || Array.isArray(generator) || typeof generator.name !== "string" || typeof generator.version !== "string") {
|
|
4704
|
+
throw new Error("Evidence manifest requires generator.name and generator.version.");
|
|
4705
|
+
}
|
|
4706
|
+
const source = record.source;
|
|
4707
|
+
if (source === null || typeof source !== "object" || Array.isArray(source)) {
|
|
4708
|
+
throw new Error("Evidence manifest requires source.");
|
|
4709
|
+
}
|
|
4710
|
+
const sourceRecord = source;
|
|
4711
|
+
if (!Array.isArray(sourceRecord.runIds) || sourceRecord.runIds.length === 0) {
|
|
4712
|
+
throw new Error("Evidence manifest source.runIds must be a non-empty array.");
|
|
4713
|
+
}
|
|
4714
|
+
if (!Array.isArray(sourceRecord.traceSchemaVersions)) {
|
|
4715
|
+
throw new Error("Evidence manifest source.traceSchemaVersions must be an array.");
|
|
4716
|
+
}
|
|
4717
|
+
if (!Array.isArray(sourceRecord.sourceHashes)) {
|
|
4718
|
+
throw new Error("Evidence manifest source.sourceHashes must be an array.");
|
|
4719
|
+
}
|
|
4720
|
+
const policy = record.policy;
|
|
4721
|
+
if (policy === null || typeof policy !== "object" || Array.isArray(policy)) {
|
|
4722
|
+
throw new Error("Evidence manifest requires policy.");
|
|
4723
|
+
}
|
|
4724
|
+
const assessment = record.assessment;
|
|
4725
|
+
if (assessment === null || typeof assessment !== "object" || Array.isArray(assessment) || typeof assessment.status !== "string") {
|
|
4726
|
+
throw new Error("Evidence manifest requires assessment.status.");
|
|
4727
|
+
}
|
|
4728
|
+
if (!Array.isArray(record.files) || record.files.length === 0) {
|
|
4729
|
+
throw new Error("Evidence manifest files must be a non-empty array.");
|
|
4730
|
+
}
|
|
4731
|
+
for (const file of record.files) {
|
|
4732
|
+
if (file === null || typeof file !== "object" || Array.isArray(file)) {
|
|
4733
|
+
throw new Error("Evidence manifest file entries must be objects.");
|
|
4734
|
+
}
|
|
4735
|
+
const entry = file;
|
|
4736
|
+
if (typeof entry.path !== "string") {
|
|
4737
|
+
throw new Error("Evidence file entry requires path.");
|
|
4738
|
+
}
|
|
4739
|
+
assertEvidenceRelativePath(entry.path);
|
|
4740
|
+
if (typeof entry.sha256 !== "string" || !/^[a-f0-9]{64}$/i.test(entry.sha256)) {
|
|
4741
|
+
throw new Error(`Evidence file entry requires sha256 hex for ${entry.path}.`);
|
|
4742
|
+
}
|
|
4743
|
+
}
|
|
4744
|
+
return value;
|
|
4714
4745
|
}
|
|
4715
|
-
function
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4746
|
+
function parseEvidenceManifestJson(text) {
|
|
4747
|
+
let parsed;
|
|
4748
|
+
try {
|
|
4749
|
+
parsed = JSON.parse(text);
|
|
4750
|
+
} catch (error) {
|
|
4751
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4752
|
+
throw new Error(`Evidence manifest is not valid JSON: ${message}`);
|
|
4720
4753
|
}
|
|
4721
|
-
return
|
|
4754
|
+
return validateEvidenceManifest(parsed);
|
|
4755
|
+
}
|
|
4756
|
+
function collectTraceSchemaVersions(jsonl) {
|
|
4757
|
+
const versions = /* @__PURE__ */ new Set();
|
|
4758
|
+
for (const line of jsonl.split(/\r?\n/)) {
|
|
4759
|
+
const trimmed = line.trim();
|
|
4760
|
+
if (trimmed === "") continue;
|
|
4761
|
+
try {
|
|
4762
|
+
const row = JSON.parse(trimmed);
|
|
4763
|
+
if (typeof row.schemaVersion === "string" && row.schemaVersion.trim() !== "") {
|
|
4764
|
+
versions.add(row.schemaVersion.trim());
|
|
4765
|
+
}
|
|
4766
|
+
} catch {
|
|
4767
|
+
}
|
|
4768
|
+
}
|
|
4769
|
+
return [...versions].sort((a, b) => a.localeCompare(b));
|
|
4722
4770
|
}
|
|
4723
4771
|
|
|
4724
4772
|
// packages/core/src/checks/derived-failure.ts
|
|
@@ -7069,7 +7117,17 @@ function emptyPassResult(input, runId) {
|
|
|
7069
7117
|
status: "pass",
|
|
7070
7118
|
format: input.read.format,
|
|
7071
7119
|
...runId !== void 0 ? { runId } : {},
|
|
7072
|
-
summary: {
|
|
7120
|
+
summary: {
|
|
7121
|
+
passed: 0,
|
|
7122
|
+
failed: 0,
|
|
7123
|
+
warnings: 0,
|
|
7124
|
+
errors: 0,
|
|
7125
|
+
rulesEvaluated: 0,
|
|
7126
|
+
rulesPassed: 0,
|
|
7127
|
+
rulesWarning: 0,
|
|
7128
|
+
rulesFailed: 0,
|
|
7129
|
+
rulesError: 0
|
|
7130
|
+
},
|
|
7073
7131
|
findings: [],
|
|
7074
7132
|
diagnostics: [],
|
|
7075
7133
|
ruleExecutions: []
|
|
@@ -7097,7 +7155,11 @@ function mergeContractResults(input, parts, extraFindings = []) {
|
|
|
7097
7155
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
7098
7156
|
).length,
|
|
7099
7157
|
errors,
|
|
7100
|
-
rulesEvaluated: ruleExecutions.length
|
|
7158
|
+
rulesEvaluated: ruleExecutions.length,
|
|
7159
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
7160
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
7161
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
7162
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
7101
7163
|
},
|
|
7102
7164
|
findings,
|
|
7103
7165
|
diagnostics,
|
|
@@ -7175,7 +7237,11 @@ function invalidContractInputResult() {
|
|
|
7175
7237
|
failed: 0,
|
|
7176
7238
|
warnings: 0,
|
|
7177
7239
|
errors: 1,
|
|
7178
|
-
rulesEvaluated: 0
|
|
7240
|
+
rulesEvaluated: 0,
|
|
7241
|
+
rulesPassed: 0,
|
|
7242
|
+
rulesWarning: 0,
|
|
7243
|
+
rulesFailed: 0,
|
|
7244
|
+
rulesError: 0
|
|
7179
7245
|
},
|
|
7180
7246
|
findings: [],
|
|
7181
7247
|
diagnostics: [
|
|
@@ -7228,7 +7294,11 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
7228
7294
|
failed: 0,
|
|
7229
7295
|
warnings: 0,
|
|
7230
7296
|
errors: shapeErrors.length,
|
|
7231
|
-
rulesEvaluated: 0
|
|
7297
|
+
rulesEvaluated: 0,
|
|
7298
|
+
rulesPassed: 0,
|
|
7299
|
+
rulesWarning: 0,
|
|
7300
|
+
rulesFailed: 0,
|
|
7301
|
+
rulesError: 0
|
|
7232
7302
|
},
|
|
7233
7303
|
findings: [],
|
|
7234
7304
|
diagnostics: shapeErrors.map((item) => ({
|
|
@@ -7253,7 +7323,11 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
7253
7323
|
failed: 0,
|
|
7254
7324
|
warnings: 0,
|
|
7255
7325
|
errors: scoped.diagnostics.length,
|
|
7256
|
-
rulesEvaluated: 0
|
|
7326
|
+
rulesEvaluated: 0,
|
|
7327
|
+
rulesPassed: 0,
|
|
7328
|
+
rulesWarning: 0,
|
|
7329
|
+
rulesFailed: 0,
|
|
7330
|
+
rulesError: 0
|
|
7257
7331
|
},
|
|
7258
7332
|
findings: [],
|
|
7259
7333
|
diagnostics: scoped.diagnostics,
|
|
@@ -7451,7 +7525,11 @@ function emptySummary2() {
|
|
|
7451
7525
|
failed: 0,
|
|
7452
7526
|
warnings: 0,
|
|
7453
7527
|
errors: 0,
|
|
7454
|
-
rulesEvaluated: 0
|
|
7528
|
+
rulesEvaluated: 0,
|
|
7529
|
+
rulesPassed: 0,
|
|
7530
|
+
rulesWarning: 0,
|
|
7531
|
+
rulesFailed: 0,
|
|
7532
|
+
rulesError: 0
|
|
7455
7533
|
};
|
|
7456
7534
|
}
|
|
7457
7535
|
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
@@ -7463,7 +7541,11 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
7463
7541
|
summary: {
|
|
7464
7542
|
...emptySummary2(),
|
|
7465
7543
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
7466
|
-
rulesEvaluated: ruleExecutions.length
|
|
7544
|
+
rulesEvaluated: ruleExecutions.length,
|
|
7545
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
7546
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
7547
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
7548
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
7467
7549
|
},
|
|
7468
7550
|
findings: [],
|
|
7469
7551
|
diagnostics: [...diagnostics],
|
|
@@ -7623,7 +7705,7 @@ function normalizeFinding(rule, finding) {
|
|
|
7623
7705
|
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
7624
7706
|
};
|
|
7625
7707
|
}
|
|
7626
|
-
function summarize(findings, diagnostics,
|
|
7708
|
+
function summarize(findings, diagnostics, ruleExecutions) {
|
|
7627
7709
|
return {
|
|
7628
7710
|
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
7629
7711
|
failed: findings.filter(
|
|
@@ -7633,7 +7715,11 @@ function summarize(findings, diagnostics, rulesEvaluated) {
|
|
|
7633
7715
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
7634
7716
|
).length,
|
|
7635
7717
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
7636
|
-
rulesEvaluated
|
|
7718
|
+
rulesEvaluated: ruleExecutions.length,
|
|
7719
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
7720
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
7721
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
7722
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
7637
7723
|
};
|
|
7638
7724
|
}
|
|
7639
7725
|
function classifyRuleExecution(findings, threw) {
|
|
@@ -9299,7 +9385,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
9299
9385
|
}
|
|
9300
9386
|
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
9301
9387
|
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
9302
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions
|
|
9388
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions);
|
|
9303
9389
|
const status = summary.failed > 0 ? "fail" : "pass";
|
|
9304
9390
|
return {
|
|
9305
9391
|
ok: status === "pass",
|
|
@@ -11242,6 +11328,28 @@ function asPositiveNumber(value, label) {
|
|
|
11242
11328
|
}
|
|
11243
11329
|
return value;
|
|
11244
11330
|
}
|
|
11331
|
+
function asExpectConfig(value, label) {
|
|
11332
|
+
if (value === void 0) return void 0;
|
|
11333
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
11334
|
+
throw new Error(`${label} must be an object.`);
|
|
11335
|
+
}
|
|
11336
|
+
const raw = value;
|
|
11337
|
+
const checkStatus = raw.checkStatus;
|
|
11338
|
+
if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
|
|
11339
|
+
throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
|
|
11340
|
+
}
|
|
11341
|
+
const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
|
|
11342
|
+
const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
|
|
11343
|
+
const rejectUnexpectedFindings = raw.rejectUnexpectedFindings === void 0 ? void 0 : typeof raw.rejectUnexpectedFindings === "boolean" ? raw.rejectUnexpectedFindings : (() => {
|
|
11344
|
+
throw new Error(`${label}.rejectUnexpectedFindings must be a boolean.`);
|
|
11345
|
+
})();
|
|
11346
|
+
return {
|
|
11347
|
+
checkStatus,
|
|
11348
|
+
...findingRuleIds !== void 0 ? { findingRuleIds } : {},
|
|
11349
|
+
...rejectUnexpectedFindings !== void 0 ? { rejectUnexpectedFindings } : {},
|
|
11350
|
+
...minAssertions !== void 0 ? { minAssertions } : {}
|
|
11351
|
+
};
|
|
11352
|
+
}
|
|
11245
11353
|
function validateCaseConfig(value, index) {
|
|
11246
11354
|
const diagnostics = [];
|
|
11247
11355
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
@@ -11284,7 +11392,8 @@ function validateCaseConfig(value, index) {
|
|
|
11284
11392
|
raw.expectedObservations,
|
|
11285
11393
|
`cases[${index}].expectedObservations`
|
|
11286
11394
|
)
|
|
11287
|
-
} : {}
|
|
11395
|
+
} : {},
|
|
11396
|
+
...asExpectConfig(raw.expect, `cases[${index}].expect`) !== void 0 ? { expect: asExpectConfig(raw.expect, `cases[${index}].expect`) } : {}
|
|
11288
11397
|
},
|
|
11289
11398
|
diagnostics
|
|
11290
11399
|
};
|
|
@@ -13919,6 +14028,78 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
13919
14028
|
}
|
|
13920
14029
|
return { ok: diagnostics.length === 0, diagnostics };
|
|
13921
14030
|
}
|
|
14031
|
+
function evaluateCaseExpect(suiteCase, checkResult, observationCount) {
|
|
14032
|
+
const expectation = suiteCase.expect;
|
|
14033
|
+
if (expectation === void 0) {
|
|
14034
|
+
return { ok: true, diagnostics: [] };
|
|
14035
|
+
}
|
|
14036
|
+
const diagnostics = [];
|
|
14037
|
+
if (checkResult.status !== expectation.checkStatus) {
|
|
14038
|
+
diagnostics.push(
|
|
14039
|
+
diagnostic5(
|
|
14040
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14041
|
+
`Expected check status "${expectation.checkStatus}", got "${checkResult.status}".`,
|
|
14042
|
+
"error",
|
|
14043
|
+
suiteCase.id
|
|
14044
|
+
)
|
|
14045
|
+
);
|
|
14046
|
+
}
|
|
14047
|
+
const failedFindings = checkResult.findings.filter(
|
|
14048
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
14049
|
+
);
|
|
14050
|
+
const requiredIds = expectation.findingRuleIds ?? [];
|
|
14051
|
+
for (const ruleId of requiredIds) {
|
|
14052
|
+
if (!failedFindings.some((finding) => finding.ruleId === ruleId)) {
|
|
14053
|
+
diagnostics.push(
|
|
14054
|
+
diagnostic5(
|
|
14055
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14056
|
+
`Expected failing finding for rule "${ruleId}" was not present.`,
|
|
14057
|
+
"error",
|
|
14058
|
+
suiteCase.id
|
|
14059
|
+
)
|
|
14060
|
+
);
|
|
14061
|
+
}
|
|
14062
|
+
}
|
|
14063
|
+
const rejectUnexpected = expectation.rejectUnexpectedFindings !== false;
|
|
14064
|
+
if (rejectUnexpected && requiredIds.length > 0) {
|
|
14065
|
+
for (const finding of failedFindings) {
|
|
14066
|
+
if (!requiredIds.includes(finding.ruleId)) {
|
|
14067
|
+
diagnostics.push(
|
|
14068
|
+
diagnostic5(
|
|
14069
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14070
|
+
`Unexpected failing finding for rule "${finding.ruleId}".`,
|
|
14071
|
+
"error",
|
|
14072
|
+
suiteCase.id
|
|
14073
|
+
)
|
|
14074
|
+
);
|
|
14075
|
+
}
|
|
14076
|
+
}
|
|
14077
|
+
}
|
|
14078
|
+
if (expectation.minAssertions !== void 0) {
|
|
14079
|
+
const assertionCount = checkResult.ruleExecutions.length + observationCount;
|
|
14080
|
+
if (assertionCount < expectation.minAssertions) {
|
|
14081
|
+
diagnostics.push(
|
|
14082
|
+
diagnostic5(
|
|
14083
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14084
|
+
`Expected at least ${expectation.minAssertions} assertion(s), got ${assertionCount}.`,
|
|
14085
|
+
"error",
|
|
14086
|
+
suiteCase.id
|
|
14087
|
+
)
|
|
14088
|
+
);
|
|
14089
|
+
}
|
|
14090
|
+
}
|
|
14091
|
+
if (expectation.checkStatus === "fail" && checkResult.diagnostics.some((item) => item.severity === "error") && checkResult.status === "error") {
|
|
14092
|
+
diagnostics.push(
|
|
14093
|
+
diagnostic5(
|
|
14094
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14095
|
+
'Check ended in evaluator/config error; semantic failure expectation requires status "fail".',
|
|
14096
|
+
"error",
|
|
14097
|
+
suiteCase.id
|
|
14098
|
+
)
|
|
14099
|
+
);
|
|
14100
|
+
}
|
|
14101
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
14102
|
+
}
|
|
13922
14103
|
async function runSuiteCase(suiteCase, config, options) {
|
|
13923
14104
|
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
13924
14105
|
if (resolved.missing || resolved.tracePath === void 0) {
|
|
@@ -13971,9 +14152,21 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
13971
14152
|
status: "pass",
|
|
13972
14153
|
format: read.format,
|
|
13973
14154
|
findings: [],
|
|
13974
|
-
diagnostics: []
|
|
14155
|
+
diagnostics: [],
|
|
14156
|
+
ruleExecutions: []
|
|
14157
|
+
};
|
|
13975
14158
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
13976
|
-
const
|
|
14159
|
+
const checkOk = checkResult.ok;
|
|
14160
|
+
const observationsOk = observationResult.ok;
|
|
14161
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
14162
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
14163
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
14164
|
+
const expectResult = evaluateCaseExpect(
|
|
14165
|
+
suiteCase,
|
|
14166
|
+
checkResult,
|
|
14167
|
+
compiled.observationCount
|
|
14168
|
+
);
|
|
14169
|
+
const checkFailureDiagnostics = [
|
|
13977
14170
|
...checkResult.diagnostics.map(
|
|
13978
14171
|
(item) => diagnostic5(
|
|
13979
14172
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
@@ -13989,15 +14182,24 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
13989
14182
|
finding.severity,
|
|
13990
14183
|
suiteCase.id
|
|
13991
14184
|
)
|
|
13992
|
-
)
|
|
13993
|
-
...observationResult.diagnostics
|
|
14185
|
+
)
|
|
13994
14186
|
];
|
|
13995
|
-
const
|
|
13996
|
-
const
|
|
13997
|
-
|
|
13998
|
-
|
|
13999
|
-
|
|
14000
|
-
|
|
14187
|
+
const expectMatched = suiteCase.expect !== void 0 && expectResult.ok && observationsOk;
|
|
14188
|
+
const diagnostics = expectMatched ? [
|
|
14189
|
+
...observationResult.diagnostics,
|
|
14190
|
+
...expectResult.diagnostics,
|
|
14191
|
+
diagnostic5(
|
|
14192
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
14193
|
+
`Expected semantic failure matched (check.status=${checkResult.status}).`,
|
|
14194
|
+
"info",
|
|
14195
|
+
suiteCase.id
|
|
14196
|
+
)
|
|
14197
|
+
] : [
|
|
14198
|
+
...checkFailureDiagnostics,
|
|
14199
|
+
...observationResult.diagnostics,
|
|
14200
|
+
...expectResult.diagnostics
|
|
14201
|
+
];
|
|
14202
|
+
const ok = suiteCase.expect !== void 0 ? expectResult.ok && observationsOk : checkOk && observationsOk;
|
|
14001
14203
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
14002
14204
|
return {
|
|
14003
14205
|
id: suiteCase.id,
|
|
@@ -15173,5 +15375,5 @@ function renderGateReport(result, options = {}) {
|
|
|
15173
15375
|
}
|
|
15174
15376
|
|
|
15175
15377
|
export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_REDACT_KEYS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isCredentialSensitiveKey, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToRunTrees, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, stringContainsHighConfidenceCredential, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
|
|
15176
|
-
//# sourceMappingURL=chunk-
|
|
15177
|
-
//# sourceMappingURL=chunk-
|
|
15378
|
+
//# sourceMappingURL=chunk-MQYHJL3W.mjs.map
|
|
15379
|
+
//# sourceMappingURL=chunk-MQYHJL3W.mjs.map
|