agent-inspect 6.31.10 → 6.31.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/NESTJS.md +56 -18
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-JCKC4ZRV.mjs → chunk-GBAGZVUX.mjs} +2268 -1963
- package/packages/cli/dist/chunk-GBAGZVUX.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +2277 -1947
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +32 -9
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-4UD7JNDR.mjs → src-F44EQY4S.mjs} +3 -3
- package/packages/cli/dist/{src-4UD7JNDR.mjs.map → src-F44EQY4S.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +2179 -1900
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +56 -4
- package/packages/core/dist/advanced.d.ts +56 -4
- package/packages/core/dist/advanced.mjs +303 -36
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +48 -10
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-CPQAOJW4.mjs → chunk-CCYKNAOD.mjs} +50 -12
- package/packages/core/dist/chunk-CCYKNAOD.mjs.map +1 -0
- package/packages/core/dist/{chunk-7YYS742B.mjs → chunk-M3RJD3BP.mjs} +5 -5
- package/packages/core/dist/{chunk-7YYS742B.mjs.map → chunk-M3RJD3BP.mjs.map} +1 -1
- package/packages/core/dist/exporters.cjs +3 -3
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.mjs +1 -1
- package/packages/core/dist/{index-EQihlEei.d.cts → index-DACU4Dy1.d.cts} +12 -0
- package/packages/core/dist/{index-CgOAGuM5.d.ts → index-b4tIGrzq.d.ts} +12 -0
- package/packages/cli/dist/chunk-JCKC4ZRV.mjs.map +0 -1
- package/packages/core/dist/chunk-CPQAOJW4.mjs.map +0 -1
|
@@ -2483,7 +2483,7 @@ function stepMixLine(summary) {
|
|
|
2483
2483
|
if (summary.logicSteps > 0) parts.push(`${summary.logicSteps} logic`);
|
|
2484
2484
|
return parts.length > 0 ? parts.join(", ") : "none";
|
|
2485
2485
|
}
|
|
2486
|
-
function
|
|
2486
|
+
function executionLine(summary) {
|
|
2487
2487
|
if (summary.status === "success") {
|
|
2488
2488
|
return summary.errorSteps > 0 ? "Completed with step errors recorded." : "Completed successfully.";
|
|
2489
2489
|
}
|
|
@@ -2501,7 +2501,7 @@ function outcomeLine(summary) {
|
|
|
2501
2501
|
if (summary.status === "running") {
|
|
2502
2502
|
return "Run is still in progress (no run_completed).";
|
|
2503
2503
|
}
|
|
2504
|
-
return "
|
|
2504
|
+
return "Execution status unknown \u2014 inspect events may be incomplete.";
|
|
2505
2505
|
}
|
|
2506
2506
|
function buildRunWhatSummary(events) {
|
|
2507
2507
|
const base = buildRunSummary(events);
|
|
@@ -2592,7 +2592,7 @@ function renderRunWhat(summary, options = {}) {
|
|
|
2592
2592
|
lines.push(`Correlation: ${parts.join(", ")}`);
|
|
2593
2593
|
}
|
|
2594
2594
|
}
|
|
2595
|
-
lines.push(`
|
|
2595
|
+
lines.push(`Execution: ${executionLine(summary)}`);
|
|
2596
2596
|
if (summary.longestStep && summary.totalSteps > 0) {
|
|
2597
2597
|
lines.push(
|
|
2598
2598
|
`Slowest: ${summary.longestStep.name} (${formatDuration2(summary.longestStep.durationMs)}, ${summary.longestStep.type})`
|
|
@@ -2604,2121 +2604,2169 @@ function renderRunWhat(summary, options = {}) {
|
|
|
2604
2604
|
return lines.join("\n");
|
|
2605
2605
|
}
|
|
2606
2606
|
|
|
2607
|
-
// packages/core/src/
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2607
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
2608
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
2609
|
+
AI_TRACE_INPUT_INVALID: {
|
|
2610
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
2611
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
2612
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
2613
|
+
relatedCodes: ["invalid_input"]
|
|
2614
|
+
},
|
|
2615
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
2616
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
2617
|
+
summary: "No trace reader could detect the input format.",
|
|
2618
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
2619
|
+
relatedCodes: ["unsupported_format"]
|
|
2620
|
+
},
|
|
2621
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
2622
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
2623
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
2624
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
2625
|
+
relatedCodes: ["ambiguous_format"]
|
|
2626
|
+
},
|
|
2627
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
2628
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
2629
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
2630
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
2631
|
+
},
|
|
2632
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
2633
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2634
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
2635
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
2636
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
2637
|
+
},
|
|
2638
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
2639
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2640
|
+
summary: "A parentId equals its own event/step id.",
|
|
2641
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
2642
|
+
relatedCodes: [
|
|
2643
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
2644
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
2645
|
+
]
|
|
2646
|
+
},
|
|
2647
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
2648
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
2649
|
+
summary: "Trace contains a parentId cycle.",
|
|
2650
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
2651
|
+
relatedCodes: ["structure.cycle"]
|
|
2652
|
+
},
|
|
2653
|
+
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
2654
|
+
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
2655
|
+
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
2656
|
+
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
2657
|
+
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
2658
|
+
},
|
|
2659
|
+
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
2660
|
+
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
2661
|
+
summary: 'inspectRun does not accept a "writer" option.',
|
|
2662
|
+
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
2618
2663
|
}
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
}
|
|
2624
|
-
function fact(id, label, value, redactor) {
|
|
2625
|
-
return {
|
|
2626
|
-
id,
|
|
2627
|
-
label,
|
|
2628
|
-
value: redactValue(redactor, id.split(".").at(-1) ?? id, value),
|
|
2629
|
-
source: "trace",
|
|
2630
|
-
confidence: "observed"
|
|
2631
|
-
};
|
|
2632
|
-
}
|
|
2633
|
-
function topKinds(run) {
|
|
2634
|
-
return Object.entries(run.metadata.kinds).filter(([, count]) => count > 0).sort((a, b) => {
|
|
2635
|
-
if (b[1] !== a[1]) return b[1] - a[1];
|
|
2636
|
-
return a[0].localeCompare(b[0]);
|
|
2637
|
-
}).slice(0, 5).map(([kind, count]) => `${kind}:${count}`);
|
|
2664
|
+
});
|
|
2665
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
2666
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
2667
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
2668
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
2638
2669
|
}
|
|
2639
|
-
|
|
2640
|
-
|
|
2670
|
+
|
|
2671
|
+
// packages/core/src/checks/logical-events.ts
|
|
2672
|
+
function isRecord6(value) {
|
|
2673
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2641
2674
|
}
|
|
2642
|
-
function
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
// and shadows every real step. Rank actual work nodes only, matching the
|
|
2646
|
-
// stats slowest-step ranking, which never counts the run envelope.
|
|
2647
|
-
entry.node.event.kind !== "RUN"
|
|
2648
|
-
).sort((a, b) => {
|
|
2649
|
-
const delta = (b.node.event.durationMs ?? 0) - (a.node.event.durationMs ?? 0);
|
|
2650
|
-
return delta !== 0 ? delta : a.index - b.index;
|
|
2651
|
-
})[0];
|
|
2675
|
+
function legacyEvent(event) {
|
|
2676
|
+
const value = event.attributes?.legacyEvent;
|
|
2677
|
+
return typeof value === "string" ? value : void 0;
|
|
2652
2678
|
}
|
|
2653
|
-
function
|
|
2654
|
-
const
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
if (attrs === void 0) continue;
|
|
2658
|
-
for (const key of Object.keys(attrs).sort()) {
|
|
2659
|
-
facts.push({
|
|
2660
|
-
id: `node.${entry.index}.attributes.${key}`,
|
|
2661
|
-
label: `${entry.node.event.name} attribute ${key}`,
|
|
2662
|
-
value: redactValue(redactor, key, attrs[key]),
|
|
2663
|
-
source: "trace",
|
|
2664
|
-
confidence: "observed"
|
|
2665
|
-
});
|
|
2666
|
-
if (facts.length >= 8) return facts;
|
|
2667
|
-
}
|
|
2668
|
-
}
|
|
2669
|
-
return facts;
|
|
2679
|
+
function stepIdOf(event) {
|
|
2680
|
+
const value = event.attributes?.stepId;
|
|
2681
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2682
|
+
return void 0;
|
|
2670
2683
|
}
|
|
2671
|
-
function
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
fact("run.nodeCount", "Total node count", nodes.length, redactor),
|
|
2680
|
-
fact("run.errorNodeCount", "Error node count", countErrorNodes(nodes), redactor),
|
|
2681
|
-
fact("run.kinds", "Observed kind mix", topKinds(run), redactor)
|
|
2682
|
-
];
|
|
2683
|
-
if (run.durationMs !== void 0) {
|
|
2684
|
-
facts.push(fact("run.durationMs", "Run duration milliseconds", run.durationMs, redactor));
|
|
2685
|
-
}
|
|
2686
|
-
const slowest = slowestNode(nodes);
|
|
2687
|
-
if (slowest !== void 0) {
|
|
2688
|
-
facts.push(
|
|
2689
|
-
fact("run.slowestNode", "Slowest observed node", {
|
|
2690
|
-
name: slowest.node.event.name,
|
|
2691
|
-
kind: slowest.node.event.kind,
|
|
2692
|
-
durationMs: slowest.node.event.durationMs
|
|
2693
|
-
}, redactor)
|
|
2694
|
-
);
|
|
2695
|
-
}
|
|
2696
|
-
facts.push(...attributeFacts(nodes, redactor));
|
|
2697
|
-
return facts;
|
|
2684
|
+
function cloneEvent(event) {
|
|
2685
|
+
return {
|
|
2686
|
+
...event,
|
|
2687
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
2688
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
2689
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
2690
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
2691
|
+
};
|
|
2698
2692
|
}
|
|
2699
|
-
function
|
|
2700
|
-
const
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
label: "Outcome",
|
|
2709
|
-
text: "The run recorded an error status or at least one error node.",
|
|
2710
|
-
evidence: ["run.status", "run.errorNodeCount"],
|
|
2711
|
-
confidence: "deterministic"
|
|
2712
|
-
});
|
|
2713
|
-
} else if (run.status === "ok") {
|
|
2714
|
-
inferences.push({
|
|
2715
|
-
id: "outcome.success",
|
|
2716
|
-
label: "Outcome",
|
|
2717
|
-
text: "The run completed without observed error nodes.",
|
|
2718
|
-
evidence: ["run.status", "run.errorNodeCount"],
|
|
2719
|
-
confidence: "deterministic"
|
|
2720
|
-
});
|
|
2721
|
-
}
|
|
2722
|
-
if (kindFact !== void 0) {
|
|
2723
|
-
inferences.push({
|
|
2724
|
-
id: "shape.kind-mix",
|
|
2725
|
-
label: "Trace shape",
|
|
2726
|
-
text: "The explanation is based on the observed event kind mix, not generated content.",
|
|
2727
|
-
evidence: [kindFact.id],
|
|
2728
|
-
confidence: "deterministic"
|
|
2729
|
-
});
|
|
2730
|
-
}
|
|
2731
|
-
if (durationFact !== void 0) {
|
|
2732
|
-
inferences.push({
|
|
2733
|
-
id: "timing.duration",
|
|
2734
|
-
label: "Timing",
|
|
2735
|
-
text: "Timing claims are limited to persisted duration fields in the trace.",
|
|
2736
|
-
evidence: [durationFact.id],
|
|
2737
|
-
confidence: "deterministic"
|
|
2738
|
-
});
|
|
2693
|
+
function mergeAttributes(start, complete) {
|
|
2694
|
+
const merged = {
|
|
2695
|
+
...isRecord6(complete.attributes) ? complete.attributes : {},
|
|
2696
|
+
...isRecord6(start.attributes) ? start.attributes : {}
|
|
2697
|
+
};
|
|
2698
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
2699
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
2700
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
2701
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
2739
2702
|
}
|
|
2740
|
-
return
|
|
2703
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
2741
2704
|
}
|
|
2742
|
-
function
|
|
2743
|
-
const
|
|
2744
|
-
const
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2705
|
+
function pairStartComplete(start, complete) {
|
|
2706
|
+
const attributes = mergeAttributes(start, complete);
|
|
2707
|
+
const paired = {
|
|
2708
|
+
...cloneEvent(start),
|
|
2709
|
+
status: complete.status,
|
|
2710
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
2711
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
2712
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
2713
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
2714
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
2715
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
2716
|
+
};
|
|
2748
2717
|
return {
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
"Generated locally without provider or network calls.",
|
|
2758
|
-
"Facts are observed from normalized trace data; inferences are deterministic labels."
|
|
2759
|
-
]
|
|
2718
|
+
...paired,
|
|
2719
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
2720
|
+
projection: {
|
|
2721
|
+
paired: true,
|
|
2722
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
2723
|
+
parentNormalized: false,
|
|
2724
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
2725
|
+
}
|
|
2760
2726
|
};
|
|
2761
2727
|
}
|
|
2762
|
-
|
|
2763
|
-
// packages/core/src/outcomes/types.ts
|
|
2764
|
-
var OBSERVED_OUTCOME_STATUSES = [
|
|
2765
|
-
"passed",
|
|
2766
|
-
"failed",
|
|
2767
|
-
"unknown",
|
|
2768
|
-
"skipped"
|
|
2769
|
-
];
|
|
2770
|
-
var OBSERVED_OUTCOME_METHODS = [
|
|
2771
|
-
"dom",
|
|
2772
|
-
"accessibility",
|
|
2773
|
-
"snapshot",
|
|
2774
|
-
"network",
|
|
2775
|
-
"storage",
|
|
2776
|
-
"filesystem",
|
|
2777
|
-
"database",
|
|
2778
|
-
"queue",
|
|
2779
|
-
"custom"
|
|
2780
|
-
];
|
|
2781
|
-
var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
|
|
2782
|
-
var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
|
|
2783
|
-
var OUTCOME_ATTRIBUTE_METHOD_KEY = "method";
|
|
2784
|
-
var OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY = "observedAt";
|
|
2785
|
-
var OUTCOME_LEGACY_EVENT = "outcome_observed";
|
|
2786
|
-
|
|
2787
|
-
// packages/core/src/outcomes/validate.ts
|
|
2788
|
-
function parseObservedOutcomeStatus(value) {
|
|
2789
|
-
const trimmed = value.trim().toLowerCase();
|
|
2790
|
-
if (OBSERVED_OUTCOME_STATUSES.includes(trimmed)) {
|
|
2791
|
-
return trimmed;
|
|
2792
|
-
}
|
|
2793
|
-
throw new Error(
|
|
2794
|
-
`Unsupported observation status "${value}". Use passed, failed, unknown, or skipped.`
|
|
2795
|
-
);
|
|
2796
|
-
}
|
|
2797
|
-
|
|
2798
|
-
// packages/core/src/outcomes/extract.ts
|
|
2799
|
-
function isOutcomeStatus(value) {
|
|
2800
|
-
return value === "passed" || value === "failed" || value === "unknown" || value === "skipped";
|
|
2801
|
-
}
|
|
2802
|
-
function parseMethod(value) {
|
|
2803
|
-
if (typeof value !== "string" || value.trim() === "") return void 0;
|
|
2804
|
-
return value;
|
|
2805
|
-
}
|
|
2806
|
-
function fromOutcomeObservedEvent(event) {
|
|
2807
|
-
return {
|
|
2808
|
-
outcomeId: event.outcomeId,
|
|
2809
|
-
runId: event.runId,
|
|
2810
|
-
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
2811
|
-
name: event.name,
|
|
2812
|
-
expectation: event.expectation,
|
|
2813
|
-
status: event.status,
|
|
2814
|
-
...event.method !== void 0 ? { method: event.method } : {},
|
|
2815
|
-
...event.actual !== void 0 ? { actual: event.actual } : {},
|
|
2816
|
-
...event.evidence !== void 0 ? { evidence: event.evidence } : {},
|
|
2817
|
-
observedAt: event.observedAt
|
|
2818
|
-
};
|
|
2819
|
-
}
|
|
2820
|
-
function fromPersistedOutcome(event) {
|
|
2821
|
-
if (event.kind !== "OUTCOME") return void 0;
|
|
2822
|
-
const attrs = event.attributes ?? {};
|
|
2823
|
-
const statusRaw = attrs[OUTCOME_ATTRIBUTE_STATUS_KEY];
|
|
2824
|
-
if (!isOutcomeStatus(statusRaw)) return void 0;
|
|
2825
|
-
const expectation = typeof attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
|
|
2826
|
-
const observedAtRaw = attrs[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
|
|
2827
|
-
const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
|
|
2728
|
+
function asLogical(event, extras) {
|
|
2828
2729
|
return {
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
|
|
2837
|
-
...attrs.evidence !== void 0 ? { evidence: attrs.evidence } : {},
|
|
2838
|
-
observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
|
|
2839
|
-
};
|
|
2840
|
-
}
|
|
2841
|
-
function extractOutcomesFromTraceEvents(events) {
|
|
2842
|
-
const out = [];
|
|
2843
|
-
for (const event of events) {
|
|
2844
|
-
if (event.event === OUTCOME_LEGACY_EVENT) {
|
|
2845
|
-
out.push(fromOutcomeObservedEvent(event));
|
|
2730
|
+
...cloneEvent(event),
|
|
2731
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
2732
|
+
projection: {
|
|
2733
|
+
paired: false,
|
|
2734
|
+
absorbedEventIds: Object.freeze([]),
|
|
2735
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
2736
|
+
...{}
|
|
2846
2737
|
}
|
|
2847
|
-
}
|
|
2848
|
-
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
2849
|
-
}
|
|
2850
|
-
function extractOutcomesFromPersistedEvents(events) {
|
|
2851
|
-
const out = [];
|
|
2852
|
-
for (const event of events) {
|
|
2853
|
-
const outcome = fromPersistedOutcome(event);
|
|
2854
|
-
if (outcome) out.push(outcome);
|
|
2855
|
-
}
|
|
2856
|
-
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
2857
|
-
}
|
|
2858
|
-
function summarizeObservedOutcomes(outcomes) {
|
|
2859
|
-
const summary = {
|
|
2860
|
-
total: outcomes.length,
|
|
2861
|
-
passed: 0,
|
|
2862
|
-
failed: 0,
|
|
2863
|
-
unknown: 0,
|
|
2864
|
-
skipped: 0,
|
|
2865
|
-
outcomes: [...outcomes]
|
|
2866
2738
|
};
|
|
2867
|
-
for (const outcome of outcomes) {
|
|
2868
|
-
summary[outcome.status] += 1;
|
|
2869
|
-
}
|
|
2870
|
-
return summary;
|
|
2871
|
-
}
|
|
2872
|
-
function outcomesMatchingStatus(outcomes, statuses) {
|
|
2873
|
-
const set = new Set(statuses);
|
|
2874
|
-
return outcomes.filter((outcome) => set.has(outcome.status));
|
|
2875
|
-
}
|
|
2876
|
-
function parseObservationFilter(value) {
|
|
2877
|
-
if (value === void 0 || value.trim() === "") return void 0;
|
|
2878
|
-
return parseObservedOutcomeStatus(value);
|
|
2879
|
-
}
|
|
2880
|
-
|
|
2881
|
-
// packages/core/src/outcomes/render.ts
|
|
2882
|
-
function renderObservedOutcomesMarkdown(summary) {
|
|
2883
|
-
if (summary.total === 0) {
|
|
2884
|
-
return "No observed outcomes recorded for this run.";
|
|
2885
|
-
}
|
|
2886
|
-
const lines = [
|
|
2887
|
-
`Total: ${summary.total} (passed ${summary.passed}, failed ${summary.failed}, unknown ${summary.unknown}, skipped ${summary.skipped})`,
|
|
2888
|
-
"",
|
|
2889
|
-
"| Name | Status | Expectation | Method |",
|
|
2890
|
-
"| --- | --- | --- | --- |"
|
|
2891
|
-
];
|
|
2892
|
-
for (const outcome of summary.outcomes) {
|
|
2893
|
-
lines.push(
|
|
2894
|
-
`| ${outcome.name} | ${outcome.status} | ${outcome.expectation.replace(/\|/g, "\\|")} | ${outcome.method ?? "-"} |`
|
|
2895
|
-
);
|
|
2896
|
-
}
|
|
2897
|
-
return lines.join("\n");
|
|
2898
|
-
}
|
|
2899
|
-
function renderObservedOutcomesHtml(summary) {
|
|
2900
|
-
if (summary.total === 0) {
|
|
2901
|
-
return "<p>No observed outcomes recorded for this run.</p>";
|
|
2902
|
-
}
|
|
2903
|
-
const rows = summary.outcomes.map((outcome) => {
|
|
2904
|
-
const esc = (v) => v.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
|
2905
|
-
return `<tr><td>${esc(outcome.name)}</td><td>${esc(outcome.status)}</td><td>${esc(outcome.expectation)}</td><td>${esc(outcome.method ?? "-")}</td></tr>`;
|
|
2906
|
-
}).join("");
|
|
2907
|
-
return `<p>Total: ${summary.total} (passed ${summary.passed}, failed ${summary.failed}, unknown ${summary.unknown}, skipped ${summary.skipped})</p><table><thead><tr><th>Name</th><th>Status</th><th>Expectation</th><th>Method</th></tr></thead><tbody>${rows}</tbody></table>`;
|
|
2908
|
-
}
|
|
2909
|
-
|
|
2910
|
-
// packages/core/src/search.ts
|
|
2911
|
-
function parseDurationFilter(expr) {
|
|
2912
|
-
const raw = expr.trim();
|
|
2913
|
-
const m = raw.match(/^(>=|<=|>|<)\s*(.+)$/);
|
|
2914
|
-
if (!m) {
|
|
2915
|
-
throw new Error(
|
|
2916
|
-
`Invalid --duration "${expr}". Use forms like >5s, >=500ms, <2m.`
|
|
2917
|
-
);
|
|
2918
|
-
}
|
|
2919
|
-
const op = m[1];
|
|
2920
|
-
const ms = parseDuration(m[2].trim());
|
|
2921
|
-
return { op, ms };
|
|
2922
|
-
}
|
|
2923
|
-
function durationMatches(valueMs, filter) {
|
|
2924
|
-
if (valueMs === void 0 || !Number.isFinite(valueMs)) return false;
|
|
2925
|
-
switch (filter.op) {
|
|
2926
|
-
case ">":
|
|
2927
|
-
return valueMs > filter.ms;
|
|
2928
|
-
case ">=":
|
|
2929
|
-
return valueMs >= filter.ms;
|
|
2930
|
-
case "<":
|
|
2931
|
-
return valueMs < filter.ms;
|
|
2932
|
-
case "<=":
|
|
2933
|
-
return valueMs <= filter.ms;
|
|
2934
|
-
default:
|
|
2935
|
-
return false;
|
|
2936
|
-
}
|
|
2937
|
-
}
|
|
2938
|
-
function normalizeStepTypeFilter(kind, type) {
|
|
2939
|
-
const v = (kind ?? type)?.trim().toLowerCase();
|
|
2940
|
-
return v && v !== "" ? v : void 0;
|
|
2941
|
-
}
|
|
2942
|
-
function nameMatches(hay, needle) {
|
|
2943
|
-
return hay.toLowerCase().includes(needle.toLowerCase());
|
|
2944
2739
|
}
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
const
|
|
2948
|
-
const
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
durationFilter = parseDurationFilter(options.duration);
|
|
2953
|
-
}
|
|
2954
|
-
const limit = options.limit ?? 50;
|
|
2955
|
-
const sessionId = options.session?.trim();
|
|
2956
|
-
const observationStatus2 = parseObservationFilter(options.observation);
|
|
2957
|
-
const hasContentFilter = Boolean(
|
|
2958
|
-
options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter || observationStatus2
|
|
2959
|
-
);
|
|
2960
|
-
const results = [];
|
|
2961
|
-
const sessionLabel = sessionId && sessionId !== "" ? sessionId : void 0;
|
|
2962
|
-
if (!hasContentFilter) {
|
|
2963
|
-
for (const m of filtered) {
|
|
2964
|
-
results.push({
|
|
2965
|
-
runId: m.runId,
|
|
2966
|
-
runName: m.name,
|
|
2967
|
-
runStatus: m.status,
|
|
2968
|
-
timestamp: m.startedAt,
|
|
2969
|
-
durationMs: m.durationMs,
|
|
2970
|
-
matchReason: sessionLabel ? `trace in session ${sessionLabel}` : "trace in directory",
|
|
2971
|
-
matchedFields: sessionLabel ? ["run", "session"] : ["run"],
|
|
2972
|
-
filePath: m.filePath,
|
|
2973
|
-
...sessionLabel ? { sessionId: sessionLabel } : {}
|
|
2974
|
-
});
|
|
2975
|
-
}
|
|
2976
|
-
results.sort((a, b) => {
|
|
2977
|
-
const ta = a.timestamp ?? 0;
|
|
2978
|
-
const tb = b.timestamp ?? 0;
|
|
2979
|
-
if (ta !== tb) return tb - ta;
|
|
2980
|
-
return a.runId.localeCompare(b.runId);
|
|
2981
|
-
});
|
|
2982
|
-
return results.slice(0, limit);
|
|
2740
|
+
function projectLogicalEvents(events) {
|
|
2741
|
+
const diagnostics = [];
|
|
2742
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
2743
|
+
for (const event of events) {
|
|
2744
|
+
const list = byRun.get(event.runId) ?? [];
|
|
2745
|
+
list.push(event);
|
|
2746
|
+
byRun.set(event.runId, list);
|
|
2983
2747
|
}
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2748
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
2749
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
2750
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
2751
|
+
const logical = [];
|
|
2752
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
2753
|
+
for (const runId of orderedRuns) {
|
|
2754
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
2755
|
+
const starts = [];
|
|
2756
|
+
const completes = [];
|
|
2757
|
+
const others = [];
|
|
2758
|
+
for (const event of runEvents) {
|
|
2759
|
+
const legacy = legacyEvent(event);
|
|
2760
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
2761
|
+
starts.push(event);
|
|
2762
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
2763
|
+
completes.push(event);
|
|
2764
|
+
} else {
|
|
2765
|
+
others.push(event);
|
|
2766
|
+
}
|
|
2991
2767
|
}
|
|
2992
|
-
|
|
2993
|
-
const
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
2768
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
2769
|
+
for (const start of starts) {
|
|
2770
|
+
const stepId = stepIdOf(start);
|
|
2771
|
+
let match;
|
|
2772
|
+
if (legacyEvent(start) === "run_started") {
|
|
2773
|
+
const candidates = completes.filter(
|
|
2774
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
2775
|
+
);
|
|
2776
|
+
if (candidates.length > 1) {
|
|
2777
|
+
diagnostics.push({
|
|
2778
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
2779
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
2780
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
2781
|
+
});
|
|
2782
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
2783
|
+
}
|
|
2784
|
+
match = candidates[0];
|
|
2785
|
+
} else if (stepId) {
|
|
2786
|
+
const candidates = completes.filter(
|
|
2787
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
2788
|
+
);
|
|
2789
|
+
if (candidates.length > 1) {
|
|
2790
|
+
diagnostics.push({
|
|
2791
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
2792
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
2793
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
2794
|
+
});
|
|
2795
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
2796
|
+
}
|
|
2797
|
+
match = candidates[0];
|
|
2798
|
+
}
|
|
2799
|
+
if (match) {
|
|
2800
|
+
usedCompletes.add(match.eventId);
|
|
2801
|
+
absorbedIds.add(match.eventId);
|
|
2802
|
+
const paired = pairStartComplete(start, match);
|
|
2803
|
+
logical.push(paired);
|
|
2804
|
+
logicalByRawId.set(start.eventId, paired);
|
|
2805
|
+
logicalByRawId.set(match.eventId, paired);
|
|
2806
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
2807
|
+
} else {
|
|
2808
|
+
diagnostics.push({
|
|
2809
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
2810
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
2811
|
+
eventIds: [start.eventId]
|
|
3022
2812
|
});
|
|
2813
|
+
const alone = asLogical(start);
|
|
2814
|
+
logical.push(alone);
|
|
2815
|
+
logicalByRawId.set(start.eventId, alone);
|
|
2816
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
3023
2817
|
}
|
|
3024
2818
|
}
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
if (opts.nameQuery && !nameMatches(m.name ?? m.runId, opts.nameQuery)) {
|
|
3040
|
-
return [];
|
|
3041
|
-
}
|
|
3042
|
-
if (opts.durationFilter && !durationMatches(m.durationMs, opts.durationFilter)) {
|
|
3043
|
-
return [];
|
|
3044
|
-
}
|
|
3045
|
-
const fields = [];
|
|
3046
|
-
if (opts.statusFilter) fields.push("run.status");
|
|
3047
|
-
if (opts.nameQuery) fields.push("run.name");
|
|
3048
|
-
if (opts.durationFilter) fields.push("run.durationMs");
|
|
3049
|
-
if (fields.length === 0) return [];
|
|
3050
|
-
return [
|
|
3051
|
-
{
|
|
3052
|
-
runId: m.runId,
|
|
3053
|
-
runName: m.name,
|
|
3054
|
-
runStatus: m.status,
|
|
3055
|
-
timestamp: m.startedAt,
|
|
3056
|
-
durationMs: m.durationMs,
|
|
3057
|
-
matchReason: `run match: ${fields.join(", ")}`,
|
|
3058
|
-
matchedFields: fields,
|
|
3059
|
-
filePath: m.filePath
|
|
2819
|
+
for (const complete of completes) {
|
|
2820
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
2821
|
+
diagnostics.push({
|
|
2822
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
2823
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
2824
|
+
eventIds: [complete.eventId]
|
|
2825
|
+
});
|
|
2826
|
+
const alone = asLogical(complete);
|
|
2827
|
+
logical.push(alone);
|
|
2828
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
2829
|
+
const stepId = stepIdOf(complete);
|
|
2830
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2831
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2832
|
+
}
|
|
3060
2833
|
}
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
2834
|
+
for (const event of others) {
|
|
2835
|
+
const alone = asLogical(event);
|
|
2836
|
+
logical.push(alone);
|
|
2837
|
+
logicalByRawId.set(event.eventId, alone);
|
|
2838
|
+
const stepId = stepIdOf(event);
|
|
2839
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2840
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2841
|
+
}
|
|
3069
2842
|
}
|
|
3070
2843
|
}
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
const
|
|
3075
|
-
if (!
|
|
3076
|
-
|
|
3077
|
-
const stepType = s.type;
|
|
3078
|
-
if (opts.stepTypeFilter && stepType !== opts.stepTypeFilter) {
|
|
3079
|
-
continue;
|
|
3080
|
-
}
|
|
3081
|
-
const hasStepFilters = opts.stepTypeFilter || opts.nameQuery || opts.toolQuery || opts.durationFilter || opts.statusFilter === "error" || opts.statusFilter === "success";
|
|
3082
|
-
if (!hasStepFilters) continue;
|
|
3083
|
-
if (opts.statusFilter === "error" && c.status === "error") {
|
|
3084
|
-
fields.push("step.status");
|
|
3085
|
-
} else if (opts.statusFilter === "success" && c.status === "success") {
|
|
3086
|
-
fields.push("step.status");
|
|
3087
|
-
} else if (opts.statusFilter === "error" || opts.statusFilter === "success") {
|
|
2844
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
2845
|
+
const normalized = [];
|
|
2846
|
+
for (const event of logical) {
|
|
2847
|
+
const originalParentId = event.parentId;
|
|
2848
|
+
if (!originalParentId) {
|
|
2849
|
+
normalized.push(event);
|
|
3088
2850
|
continue;
|
|
3089
2851
|
}
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
2852
|
+
let nextParent = originalParentId;
|
|
2853
|
+
let remapped = false;
|
|
2854
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
2855
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
2856
|
+
nextParent = viaAbsorbed.eventId;
|
|
2857
|
+
remapped = true;
|
|
2858
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
2859
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
2860
|
+
if (viaStep) {
|
|
2861
|
+
nextParent = viaStep;
|
|
2862
|
+
remapped = true;
|
|
2863
|
+
}
|
|
3098
2864
|
}
|
|
3099
|
-
if (
|
|
3100
|
-
if (
|
|
3101
|
-
|
|
2865
|
+
if (!remapped) {
|
|
2866
|
+
if (originalParentId === event.eventId) {
|
|
2867
|
+
diagnostics.push({
|
|
2868
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2869
|
+
message: formatProgrammaticDiagnostic(
|
|
2870
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2871
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
2872
|
+
),
|
|
2873
|
+
eventIds: [event.eventId]
|
|
2874
|
+
});
|
|
2875
|
+
const { parentId: _drop, ...rest } = event;
|
|
2876
|
+
normalized.push({
|
|
2877
|
+
...rest,
|
|
2878
|
+
projection: {
|
|
2879
|
+
...event.projection,
|
|
2880
|
+
parentNormalized: true,
|
|
2881
|
+
originalParentId
|
|
2882
|
+
}
|
|
2883
|
+
});
|
|
2884
|
+
continue;
|
|
2885
|
+
}
|
|
2886
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
2887
|
+
const mapping = event.attributes?.parentMapping;
|
|
2888
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
2889
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
2890
|
+
if (!unresolved) {
|
|
2891
|
+
diagnostics.push({
|
|
2892
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
2893
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
2894
|
+
eventIds: [event.eventId]
|
|
2895
|
+
});
|
|
2896
|
+
}
|
|
2897
|
+
}
|
|
2898
|
+
normalized.push(event);
|
|
2899
|
+
continue;
|
|
3102
2900
|
}
|
|
3103
|
-
if (
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
}
|
|
3123
|
-
async function loadTraceMetadataList(_traceDir, fileNames, getPath) {
|
|
3124
|
-
const metas = [];
|
|
3125
|
-
for (const fileName of fileNames) {
|
|
3126
|
-
try {
|
|
3127
|
-
const filePath = getPath(fileName);
|
|
3128
|
-
const meta = await extractMetadata(filePath);
|
|
3129
|
-
metas.push(meta);
|
|
3130
|
-
} catch {
|
|
2901
|
+
if (nextParent === event.eventId) {
|
|
2902
|
+
diagnostics.push({
|
|
2903
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2904
|
+
message: formatProgrammaticDiagnostic(
|
|
2905
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2906
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
2907
|
+
),
|
|
2908
|
+
eventIds: [event.eventId]
|
|
2909
|
+
});
|
|
2910
|
+
const { parentId: _drop, ...rest } = event;
|
|
2911
|
+
normalized.push({
|
|
2912
|
+
...rest,
|
|
2913
|
+
projection: {
|
|
2914
|
+
...event.projection,
|
|
2915
|
+
parentNormalized: true,
|
|
2916
|
+
originalParentId
|
|
2917
|
+
}
|
|
2918
|
+
});
|
|
2919
|
+
continue;
|
|
3131
2920
|
}
|
|
2921
|
+
diagnostics.push({
|
|
2922
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
2923
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
2924
|
+
eventIds: [event.eventId]
|
|
2925
|
+
});
|
|
2926
|
+
normalized.push({
|
|
2927
|
+
...event,
|
|
2928
|
+
parentId: nextParent,
|
|
2929
|
+
projection: {
|
|
2930
|
+
...event.projection,
|
|
2931
|
+
parentNormalized: true,
|
|
2932
|
+
originalParentId
|
|
2933
|
+
}
|
|
2934
|
+
});
|
|
3132
2935
|
}
|
|
3133
|
-
|
|
2936
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
2937
|
+
normalized.sort((a, b) => {
|
|
2938
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
2939
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
2940
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2941
|
+
});
|
|
2942
|
+
return {
|
|
2943
|
+
logicalEvents: Object.freeze(normalized),
|
|
2944
|
+
diagnostics: Object.freeze(diagnostics)
|
|
2945
|
+
};
|
|
3134
2946
|
}
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
const
|
|
3140
|
-
if (
|
|
3141
|
-
|
|
2947
|
+
function resolveCanonicalToolName(event) {
|
|
2948
|
+
const attrs = event.attributes;
|
|
2949
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
2950
|
+
if (direct) return direct;
|
|
2951
|
+
const metadata = attrs?.metadata;
|
|
2952
|
+
if (isRecord6(metadata)) {
|
|
2953
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2954
|
+
if (nested) return nested;
|
|
3142
2955
|
}
|
|
3143
|
-
|
|
3144
|
-
|
|
2956
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2957
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
3145
2958
|
}
|
|
3146
|
-
return
|
|
2959
|
+
return event.name;
|
|
3147
2960
|
}
|
|
3148
|
-
function
|
|
3149
|
-
if (!
|
|
3150
|
-
const
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
const mult = unit === "s" ? 1e3 : unit === "m" ? 6e4 : unit === "h" ? 36e5 : 864e5;
|
|
3156
|
-
return nowMs - amount * mult;
|
|
2961
|
+
function pickString(record, keys) {
|
|
2962
|
+
if (!record) return void 0;
|
|
2963
|
+
for (const key of keys) {
|
|
2964
|
+
const value = record[key];
|
|
2965
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2966
|
+
}
|
|
2967
|
+
return void 0;
|
|
3157
2968
|
}
|
|
3158
|
-
|
|
3159
|
-
|
|
2969
|
+
|
|
2970
|
+
// packages/core/src/explain.ts
|
|
2971
|
+
function toDisplayStatus(status) {
|
|
2972
|
+
if (status === "ok") return "success";
|
|
2973
|
+
if (status === "error") return "error";
|
|
2974
|
+
if (status === "running") return "running";
|
|
2975
|
+
return "unknown";
|
|
3160
2976
|
}
|
|
3161
|
-
function
|
|
3162
|
-
|
|
2977
|
+
function flatten(nodes, out = []) {
|
|
2978
|
+
for (const node of nodes) {
|
|
2979
|
+
out.push({ node, index: out.length + 1 });
|
|
2980
|
+
flatten(node.children, out);
|
|
2981
|
+
}
|
|
2982
|
+
return out;
|
|
3163
2983
|
}
|
|
3164
|
-
function
|
|
3165
|
-
|
|
3166
|
-
if (!summary) return 0;
|
|
3167
|
-
return summary.warn;
|
|
2984
|
+
function redactValue(redactor, key, value) {
|
|
2985
|
+
return redactor.redactValue(key, value);
|
|
3168
2986
|
}
|
|
3169
|
-
function
|
|
3170
|
-
const nowMs = options.nowMs ?? Date.now();
|
|
3171
|
-
const sinceMs = parseSinceMs(options.since, nowMs);
|
|
3172
|
-
const sinceIso = new Date(sinceMs).toISOString();
|
|
3173
|
-
const limit = Number.isInteger(options.limit) && options.limit > 0 ? options.limit : 20;
|
|
3174
|
-
const inWindow = index.sessions.filter((session) => {
|
|
3175
|
-
const activityMs2 = Date.parse(session.lastActivity);
|
|
3176
|
-
return Number.isFinite(activityMs2) && activityMs2 >= sinceMs;
|
|
3177
|
-
});
|
|
3178
|
-
const entries = [...inWindow].sort((a, b) => Date.parse(b.lastActivity) - Date.parse(a.lastActivity)).slice(0, limit).map((session) => ({
|
|
3179
|
-
sessionId: session.sessionId,
|
|
3180
|
-
status: session.status,
|
|
3181
|
-
summary: statusLine(session),
|
|
3182
|
-
lastActivity: session.lastActivity,
|
|
3183
|
-
runCount: session.runIds.length
|
|
3184
|
-
}));
|
|
3185
|
-
let failed = 0;
|
|
3186
|
-
let stale = 0;
|
|
3187
|
-
let guardrailWarningTotal = 0;
|
|
3188
|
-
for (const session of inWindow) {
|
|
3189
|
-
if (isFailed(session.status)) failed += 1;
|
|
3190
|
-
if (isStale(session.status)) stale += 1;
|
|
3191
|
-
guardrailWarningTotal += guardrailWarnings(session);
|
|
3192
|
-
}
|
|
2987
|
+
function fact(id, label, value, redactor) {
|
|
3193
2988
|
return {
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
entries
|
|
2989
|
+
id,
|
|
2990
|
+
label,
|
|
2991
|
+
value: redactValue(redactor, id.split(".").at(-1) ?? id, value),
|
|
2992
|
+
source: "trace",
|
|
2993
|
+
confidence: "observed"
|
|
3200
2994
|
};
|
|
3201
2995
|
}
|
|
3202
|
-
function
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
const todayMs = todayStart.getTime();
|
|
3208
|
-
const today = summary.entries.filter(
|
|
3209
|
-
(entry) => Date.parse(entry.lastActivity) >= todayMs
|
|
3210
|
-
);
|
|
3211
|
-
if (today.length > 0) {
|
|
3212
|
-
lines.push("Today");
|
|
3213
|
-
for (const entry of today) {
|
|
3214
|
-
lines.push(` ${entry.summary}`);
|
|
3215
|
-
}
|
|
3216
|
-
lines.push("");
|
|
3217
|
-
}
|
|
3218
|
-
lines.push(`Since ${summary.since}`);
|
|
3219
|
-
lines.push(` ${summary.sessions} sessions`);
|
|
3220
|
-
lines.push(` ${summary.failed} failed`);
|
|
3221
|
-
lines.push(` ${summary.stale} stale`);
|
|
3222
|
-
lines.push(` ${summary.guardrailWarnings} guardrail warnings`);
|
|
3223
|
-
return lines.join("\n");
|
|
2996
|
+
function topKinds(run) {
|
|
2997
|
+
return Object.entries(run.metadata.kinds).filter(([, count]) => count > 0).sort((a, b) => {
|
|
2998
|
+
if (b[1] !== a[1]) return b[1] - a[1];
|
|
2999
|
+
return a[0].localeCompare(b[0]);
|
|
3000
|
+
}).slice(0, 5).map(([kind, count]) => `${kind}:${count}`);
|
|
3224
3001
|
}
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3002
|
+
function countErrorNodes(nodes) {
|
|
3003
|
+
return nodes.filter((entry) => entry.node.event.status === "error").length;
|
|
3004
|
+
}
|
|
3005
|
+
function slowestNode(nodes) {
|
|
3006
|
+
return nodes.filter(
|
|
3007
|
+
(entry) => entry.node.event.durationMs !== void 0 && // The RUN boundary spans the whole run, so its duration always dominates
|
|
3008
|
+
// and shadows every real step. Rank actual work nodes only, matching the
|
|
3009
|
+
// stats slowest-step ranking, which never counts the run envelope.
|
|
3010
|
+
entry.node.event.kind !== "RUN"
|
|
3011
|
+
).sort((a, b) => {
|
|
3012
|
+
const delta = (b.node.event.durationMs ?? 0) - (a.node.event.durationMs ?? 0);
|
|
3013
|
+
return delta !== 0 ? delta : a.index - b.index;
|
|
3014
|
+
})[0];
|
|
3015
|
+
}
|
|
3016
|
+
function attributeFacts(nodes, redactor) {
|
|
3017
|
+
const facts = [];
|
|
3018
|
+
for (const entry of nodes) {
|
|
3019
|
+
const attrs = entry.node.event.attributes;
|
|
3020
|
+
if (attrs === void 0) continue;
|
|
3021
|
+
for (const key of Object.keys(attrs).sort()) {
|
|
3022
|
+
facts.push({
|
|
3023
|
+
id: `node.${entry.index}.attributes.${key}`,
|
|
3024
|
+
label: `${entry.node.event.name} attribute ${key}`,
|
|
3025
|
+
value: redactValue(redactor, key, attrs[key]),
|
|
3026
|
+
source: "trace",
|
|
3027
|
+
confidence: "observed"
|
|
3028
|
+
});
|
|
3029
|
+
if (facts.length >= 8) return facts;
|
|
3237
3030
|
}
|
|
3238
|
-
} catch {
|
|
3239
3031
|
}
|
|
3240
|
-
return
|
|
3241
|
-
runId: meta.runId,
|
|
3242
|
-
name: meta.name,
|
|
3243
|
-
status: meta.status,
|
|
3244
|
-
startedAt: meta.startedAt,
|
|
3245
|
-
endedAt: meta.endedAt,
|
|
3246
|
-
durationMs: meta.durationMs,
|
|
3247
|
-
filePath: meta.filePath,
|
|
3248
|
-
metadata
|
|
3249
|
-
};
|
|
3032
|
+
return facts;
|
|
3250
3033
|
}
|
|
3251
|
-
|
|
3252
|
-
const
|
|
3253
|
-
|
|
3254
|
-
|
|
3034
|
+
function buildFacts(run, redactor, events) {
|
|
3035
|
+
const nodes = flatten(run.children);
|
|
3036
|
+
const facts = [
|
|
3037
|
+
fact("run.id", "Run id", run.runId, redactor),
|
|
3038
|
+
fact("run.name", "Run name", run.name ?? run.runId, redactor),
|
|
3039
|
+
fact("run.status", "Run status", toDisplayStatus(run.status), redactor),
|
|
3040
|
+
fact("run.totalEvents", "Total events", run.metadata.totalEvents, redactor),
|
|
3041
|
+
fact("run.stepCount", "Top-level step count", run.children.length, redactor),
|
|
3042
|
+
fact("run.nodeCount", "Total node count", nodes.length, redactor),
|
|
3043
|
+
fact("run.errorNodeCount", "Error node count", countErrorNodes(nodes), redactor),
|
|
3044
|
+
fact("run.kinds", "Observed kind mix", topKinds(run), redactor)
|
|
3045
|
+
];
|
|
3046
|
+
if (run.durationMs !== void 0) {
|
|
3047
|
+
facts.push(fact("run.durationMs", "Run duration milliseconds", run.durationMs, redactor));
|
|
3255
3048
|
}
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
return void 0;
|
|
3049
|
+
const slowest = slowestNode(nodes);
|
|
3050
|
+
if (slowest !== void 0) {
|
|
3051
|
+
facts.push(
|
|
3052
|
+
fact("run.slowestNode", "Slowest observed node", {
|
|
3053
|
+
name: slowest.node.event.name,
|
|
3054
|
+
kind: slowest.node.event.kind,
|
|
3055
|
+
durationMs: slowest.node.event.durationMs
|
|
3056
|
+
}, redactor)
|
|
3057
|
+
);
|
|
3266
3058
|
}
|
|
3267
|
-
|
|
3059
|
+
if (events !== void 0) {
|
|
3060
|
+
const scoped = events.filter((event) => event.runId === run.runId);
|
|
3061
|
+
const { logicalEvents: logical } = projectLogicalEvents(scoped);
|
|
3062
|
+
const logicalSteps = logical.filter((event) => event.kind !== "RUN");
|
|
3063
|
+
facts.push(
|
|
3064
|
+
fact("run.rawEventCount", "Raw persisted event count", scoped.length, redactor),
|
|
3065
|
+
fact(
|
|
3066
|
+
"run.logicalStepCount",
|
|
3067
|
+
"Logical lifecycle step count",
|
|
3068
|
+
logicalSteps.length,
|
|
3069
|
+
redactor
|
|
3070
|
+
),
|
|
3071
|
+
fact(
|
|
3072
|
+
"run.logicalEventCount",
|
|
3073
|
+
"Logical lifecycle event count (including RUN)",
|
|
3074
|
+
logical.length,
|
|
3075
|
+
redactor
|
|
3076
|
+
)
|
|
3077
|
+
);
|
|
3078
|
+
const slowestLogical = [...logicalSteps].filter((event) => typeof event.durationMs === "number").sort((a, b) => {
|
|
3079
|
+
const delta = (b.durationMs ?? 0) - (a.durationMs ?? 0);
|
|
3080
|
+
return delta !== 0 ? delta : a.eventId.localeCompare(b.eventId);
|
|
3081
|
+
})[0];
|
|
3082
|
+
if (slowestLogical !== void 0) {
|
|
3083
|
+
facts.push(
|
|
3084
|
+
fact("run.slowestLogicalStep", "Slowest logical lifecycle step", {
|
|
3085
|
+
name: slowestLogical.name,
|
|
3086
|
+
kind: slowestLogical.kind,
|
|
3087
|
+
durationMs: slowestLogical.durationMs
|
|
3088
|
+
}, redactor)
|
|
3089
|
+
);
|
|
3090
|
+
}
|
|
3091
|
+
}
|
|
3092
|
+
facts.push(...attributeFacts(nodes, redactor));
|
|
3093
|
+
return facts;
|
|
3268
3094
|
}
|
|
3269
|
-
function
|
|
3270
|
-
|
|
3271
|
-
const
|
|
3272
|
-
|
|
3273
|
-
const
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
assignString("workflowName", record.workflowName);
|
|
3292
|
-
assignString("workflowStep", record.workflowStep);
|
|
3293
|
-
assignString("toolCallId", record.toolCallId);
|
|
3294
|
-
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
3295
|
-
assignString("linkedStepId", record.linkedStepId);
|
|
3296
|
-
assignString("operationId", record.operationId);
|
|
3297
|
-
assignString("attemptId", record.attemptId);
|
|
3298
|
-
assignString("fallbackOf", record.fallbackOf);
|
|
3299
|
-
assignString("idempotencyKey", record.idempotencyKey);
|
|
3300
|
-
assignString("correlationId", record.correlationId);
|
|
3301
|
-
assignString("requestId", record.requestId);
|
|
3302
|
-
assignString("decisionId", record.decisionId);
|
|
3303
|
-
const attempt = finitePositiveInt(record.attempt);
|
|
3304
|
-
if (attempt !== void 0) {
|
|
3305
|
-
out.attempt = attempt;
|
|
3306
|
-
found = true;
|
|
3095
|
+
function buildInferences(run, facts) {
|
|
3096
|
+
const inferences = [];
|
|
3097
|
+
const errorFact = facts.find((item) => item.id === "run.errorNodeCount");
|
|
3098
|
+
const kindFact = facts.find((item) => item.id === "run.kinds");
|
|
3099
|
+
const durationFact = facts.find((item) => item.id === "run.durationMs");
|
|
3100
|
+
const errorNodeCount = typeof errorFact?.value === "number" ? errorFact.value : 0;
|
|
3101
|
+
if (run.status === "error" || errorNodeCount > 0) {
|
|
3102
|
+
inferences.push({
|
|
3103
|
+
id: "outcome.error",
|
|
3104
|
+
label: "Outcome",
|
|
3105
|
+
text: "The run recorded an error status or at least one error node.",
|
|
3106
|
+
evidence: ["run.status", "run.errorNodeCount"],
|
|
3107
|
+
confidence: "deterministic"
|
|
3108
|
+
});
|
|
3109
|
+
} else if (run.status === "ok") {
|
|
3110
|
+
inferences.push({
|
|
3111
|
+
id: "outcome.success",
|
|
3112
|
+
label: "Outcome",
|
|
3113
|
+
text: "The run completed without observed error nodes.",
|
|
3114
|
+
evidence: ["run.status", "run.errorNodeCount"],
|
|
3115
|
+
confidence: "deterministic"
|
|
3116
|
+
});
|
|
3307
3117
|
}
|
|
3308
|
-
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
|
|
3118
|
+
if (kindFact !== void 0) {
|
|
3119
|
+
inferences.push({
|
|
3120
|
+
id: "shape.kind-mix",
|
|
3121
|
+
label: "Trace shape",
|
|
3122
|
+
text: "The explanation is based on the observed event kind mix, not generated content.",
|
|
3123
|
+
evidence: [kindFact.id],
|
|
3124
|
+
confidence: "deterministic"
|
|
3125
|
+
});
|
|
3312
3126
|
}
|
|
3313
|
-
|
|
3127
|
+
if (durationFact !== void 0) {
|
|
3128
|
+
inferences.push({
|
|
3129
|
+
id: "timing.duration",
|
|
3130
|
+
label: "Timing",
|
|
3131
|
+
text: "Timing claims are limited to persisted duration fields in the trace.",
|
|
3132
|
+
evidence: [durationFact.id],
|
|
3133
|
+
confidence: "deterministic"
|
|
3134
|
+
});
|
|
3135
|
+
}
|
|
3136
|
+
return inferences;
|
|
3314
3137
|
}
|
|
3315
|
-
function
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3138
|
+
function buildLocalExplanation(run, options = {}) {
|
|
3139
|
+
const redactionProfile = options.redactionProfile ?? "local";
|
|
3140
|
+
const resolved = resolveRedactionProfile(redactionProfile);
|
|
3141
|
+
const redactor = new Redactor({ extraKeys: resolved.extraKeys });
|
|
3142
|
+
const mode = options.mode ?? "local";
|
|
3143
|
+
const facts = buildFacts(run, redactor, options.events);
|
|
3144
|
+
return {
|
|
3145
|
+
mode,
|
|
3146
|
+
runId: String(redactValue(redactor, "runId", run.runId)),
|
|
3147
|
+
...run.name !== void 0 ? { name: String(redactValue(redactor, "name", run.name)) } : {},
|
|
3148
|
+
...run.status !== void 0 ? { status: toDisplayStatus(run.status) } : {},
|
|
3149
|
+
redactionProfile,
|
|
3150
|
+
facts,
|
|
3151
|
+
inferences: mode === "dry-run" ? [] : buildInferences(run, facts),
|
|
3152
|
+
notes: [
|
|
3153
|
+
"Generated locally without provider or network calls.",
|
|
3154
|
+
"Facts are observed from normalized trace data; inferences are deterministic labels."
|
|
3155
|
+
]
|
|
3156
|
+
};
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
// packages/core/src/outcomes/types.ts
|
|
3160
|
+
var OBSERVED_OUTCOME_STATUSES = [
|
|
3161
|
+
"passed",
|
|
3162
|
+
"failed",
|
|
3163
|
+
"unknown",
|
|
3164
|
+
"skipped"
|
|
3165
|
+
];
|
|
3166
|
+
var OBSERVED_OUTCOME_METHODS = [
|
|
3167
|
+
"dom",
|
|
3168
|
+
"accessibility",
|
|
3169
|
+
"snapshot",
|
|
3170
|
+
"network",
|
|
3171
|
+
"storage",
|
|
3172
|
+
"filesystem",
|
|
3173
|
+
"database",
|
|
3174
|
+
"queue",
|
|
3175
|
+
"custom"
|
|
3176
|
+
];
|
|
3177
|
+
var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
|
|
3178
|
+
var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
|
|
3179
|
+
var OUTCOME_ATTRIBUTE_METHOD_KEY = "method";
|
|
3180
|
+
var OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY = "observedAt";
|
|
3181
|
+
var OUTCOME_LEGACY_EVENT = "outcome_observed";
|
|
3182
|
+
|
|
3183
|
+
// packages/core/src/outcomes/validate.ts
|
|
3184
|
+
function parseObservedOutcomeStatus(value) {
|
|
3185
|
+
const trimmed = value.trim().toLowerCase();
|
|
3186
|
+
if (OBSERVED_OUTCOME_STATUSES.includes(trimmed)) {
|
|
3187
|
+
return trimmed;
|
|
3319
3188
|
}
|
|
3320
|
-
|
|
3189
|
+
throw new Error(
|
|
3190
|
+
`Unsupported observation status "${value}". Use passed, failed, unknown, or skipped.`
|
|
3191
|
+
);
|
|
3321
3192
|
}
|
|
3322
3193
|
|
|
3323
|
-
// packages/core/src/
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
error: 5,
|
|
3327
|
-
waiting_input: 4,
|
|
3328
|
-
idle: 3,
|
|
3329
|
-
stale: 2,
|
|
3330
|
-
completed: 1
|
|
3331
|
-
};
|
|
3332
|
-
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
3333
|
-
"running",
|
|
3334
|
-
"waiting_input",
|
|
3335
|
-
"idle",
|
|
3336
|
-
"completed",
|
|
3337
|
-
"error",
|
|
3338
|
-
"stale",
|
|
3339
|
-
"unknown"
|
|
3340
|
-
]);
|
|
3341
|
-
function isExplicitSessionStatus(value) {
|
|
3342
|
-
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
3194
|
+
// packages/core/src/outcomes/extract.ts
|
|
3195
|
+
function isOutcomeStatus(value) {
|
|
3196
|
+
return value === "passed" || value === "failed" || value === "unknown" || value === "skipped";
|
|
3343
3197
|
}
|
|
3344
|
-
function
|
|
3345
|
-
|
|
3198
|
+
function parseMethod(value) {
|
|
3199
|
+
if (typeof value !== "string" || value.trim() === "") return void 0;
|
|
3200
|
+
return value;
|
|
3346
3201
|
}
|
|
3347
|
-
function
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3202
|
+
function fromOutcomeObservedEvent(event) {
|
|
3203
|
+
return {
|
|
3204
|
+
outcomeId: event.outcomeId,
|
|
3205
|
+
runId: event.runId,
|
|
3206
|
+
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
3207
|
+
name: event.name,
|
|
3208
|
+
expectation: event.expectation,
|
|
3209
|
+
status: event.status,
|
|
3210
|
+
...event.method !== void 0 ? { method: event.method } : {},
|
|
3211
|
+
...event.actual !== void 0 ? { actual: event.actual } : {},
|
|
3212
|
+
...event.evidence !== void 0 ? { evidence: event.evidence } : {},
|
|
3213
|
+
observedAt: event.observedAt
|
|
3214
|
+
};
|
|
3354
3215
|
}
|
|
3355
|
-
function
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
return
|
|
3216
|
+
function fromPersistedOutcome(event) {
|
|
3217
|
+
if (event.kind !== "OUTCOME") return void 0;
|
|
3218
|
+
const attrs = event.attributes ?? {};
|
|
3219
|
+
const statusRaw = attrs[OUTCOME_ATTRIBUTE_STATUS_KEY];
|
|
3220
|
+
if (!isOutcomeStatus(statusRaw)) return void 0;
|
|
3221
|
+
const expectation = typeof attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
|
|
3222
|
+
const observedAtRaw = attrs[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
|
|
3223
|
+
const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
|
|
3224
|
+
return {
|
|
3225
|
+
outcomeId: event.eventId,
|
|
3226
|
+
runId: event.runId,
|
|
3227
|
+
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
3228
|
+
name: event.name,
|
|
3229
|
+
expectation,
|
|
3230
|
+
status: statusRaw,
|
|
3231
|
+
...parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) !== void 0 ? { method: parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) } : {},
|
|
3232
|
+
...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
|
|
3233
|
+
...attrs.evidence !== void 0 ? { evidence: attrs.evidence } : {},
|
|
3234
|
+
observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
|
|
3235
|
+
};
|
|
3364
3236
|
}
|
|
3365
|
-
function
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3237
|
+
function extractOutcomesFromTraceEvents(events) {
|
|
3238
|
+
const out = [];
|
|
3239
|
+
for (const event of events) {
|
|
3240
|
+
if (event.event === OUTCOME_LEGACY_EVENT) {
|
|
3241
|
+
out.push(fromOutcomeObservedEvent(event));
|
|
3242
|
+
}
|
|
3371
3243
|
}
|
|
3372
|
-
return
|
|
3244
|
+
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
3373
3245
|
}
|
|
3374
|
-
function
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
if (!isExplicitSessionStatus(raw)) continue;
|
|
3380
|
-
const priority = EXPLICIT_STATUS_PRIORITY[raw] ?? 0;
|
|
3381
|
-
if (priority > bestPriority) {
|
|
3382
|
-
bestPriority = priority;
|
|
3383
|
-
best = raw;
|
|
3384
|
-
}
|
|
3246
|
+
function extractOutcomesFromPersistedEvents(events) {
|
|
3247
|
+
const out = [];
|
|
3248
|
+
for (const event of events) {
|
|
3249
|
+
const outcome = fromPersistedOutcome(event);
|
|
3250
|
+
if (outcome) out.push(outcome);
|
|
3385
3251
|
}
|
|
3386
|
-
return
|
|
3252
|
+
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
3387
3253
|
}
|
|
3388
|
-
function
|
|
3389
|
-
const
|
|
3390
|
-
|
|
3391
|
-
|
|
3392
|
-
|
|
3393
|
-
|
|
3394
|
-
|
|
3395
|
-
|
|
3396
|
-
}
|
|
3397
|
-
|
|
3398
|
-
|
|
3399
|
-
let fail4 = 0;
|
|
3400
|
-
let warn2 = 0;
|
|
3401
|
-
let found = false;
|
|
3402
|
-
for (const run of runs) {
|
|
3403
|
-
const summary = run.metadata?.checkSummary;
|
|
3404
|
-
if (!summary || typeof summary !== "object") continue;
|
|
3405
|
-
const record = summary;
|
|
3406
|
-
if (typeof record.pass === "number") {
|
|
3407
|
-
pass += record.pass;
|
|
3408
|
-
found = true;
|
|
3409
|
-
}
|
|
3410
|
-
if (typeof record.fail === "number") {
|
|
3411
|
-
fail4 += record.fail;
|
|
3412
|
-
found = true;
|
|
3413
|
-
}
|
|
3414
|
-
if (typeof record.warn === "number") {
|
|
3415
|
-
warn2 += record.warn;
|
|
3416
|
-
found = true;
|
|
3417
|
-
}
|
|
3254
|
+
function summarizeObservedOutcomes(outcomes) {
|
|
3255
|
+
const summary = {
|
|
3256
|
+
total: outcomes.length,
|
|
3257
|
+
passed: 0,
|
|
3258
|
+
failed: 0,
|
|
3259
|
+
unknown: 0,
|
|
3260
|
+
skipped: 0,
|
|
3261
|
+
outcomes: [...outcomes]
|
|
3262
|
+
};
|
|
3263
|
+
for (const outcome of outcomes) {
|
|
3264
|
+
summary[outcome.status] += 1;
|
|
3418
3265
|
}
|
|
3419
|
-
return
|
|
3266
|
+
return summary;
|
|
3420
3267
|
}
|
|
3421
|
-
function
|
|
3422
|
-
|
|
3423
|
-
|
|
3424
|
-
if (typeof value === "string" && value.trim() !== "") {
|
|
3425
|
-
return value.trim();
|
|
3426
|
-
}
|
|
3427
|
-
}
|
|
3428
|
-
return void 0;
|
|
3268
|
+
function outcomesMatchingStatus(outcomes, statuses) {
|
|
3269
|
+
const set = new Set(statuses);
|
|
3270
|
+
return outcomes.filter((outcome) => set.has(outcome.status));
|
|
3429
3271
|
}
|
|
3430
|
-
function
|
|
3431
|
-
if (
|
|
3432
|
-
|
|
3433
|
-
const explicit = pickExplicitStatus(runs);
|
|
3434
|
-
if (explicit && explicit !== "running") return explicit;
|
|
3435
|
-
if (runs.some((run) => run.status === "error")) return "error";
|
|
3436
|
-
if (runs.every((run) => run.status === "success")) return "completed";
|
|
3437
|
-
const nowMs = options.nowMs ?? Date.now();
|
|
3438
|
-
const staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
3439
|
-
const lastMs = latestActivityMs(runs);
|
|
3440
|
-
if (lastMs > 0 && nowMs - lastMs > staleThresholdMs) return "stale";
|
|
3441
|
-
return "unknown";
|
|
3272
|
+
function parseObservationFilter(value) {
|
|
3273
|
+
if (value === void 0 || value.trim() === "") return void 0;
|
|
3274
|
+
return parseObservedOutcomeStatus(value);
|
|
3442
3275
|
}
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
let correlationId;
|
|
3449
|
-
let jobId;
|
|
3450
|
-
let workflowId;
|
|
3451
|
-
for (const run of sessionRuns) {
|
|
3452
|
-
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
3453
|
-
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
3454
|
-
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
3455
|
-
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
3456
|
-
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
3276
|
+
|
|
3277
|
+
// packages/core/src/outcomes/render.ts
|
|
3278
|
+
function renderObservedOutcomesMarkdown(summary) {
|
|
3279
|
+
if (summary.total === 0) {
|
|
3280
|
+
return "No observed outcomes recorded for this run.";
|
|
3457
3281
|
}
|
|
3458
|
-
const
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
|
|
3464
|
-
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
|
|
3472
|
-
|
|
3473
|
-
|
|
3474
|
-
|
|
3475
|
-
|
|
3476
|
-
|
|
3477
|
-
|
|
3282
|
+
const lines = [
|
|
3283
|
+
`Total: ${summary.total} (passed ${summary.passed}, failed ${summary.failed}, unknown ${summary.unknown}, skipped ${summary.skipped})`,
|
|
3284
|
+
"",
|
|
3285
|
+
"| Name | Status | Expectation | Method |",
|
|
3286
|
+
"| --- | --- | --- | --- |"
|
|
3287
|
+
];
|
|
3288
|
+
for (const outcome of summary.outcomes) {
|
|
3289
|
+
lines.push(
|
|
3290
|
+
`| ${outcome.name} | ${outcome.status} | ${outcome.expectation.replace(/\|/g, "\\|")} | ${outcome.method ?? "-"} |`
|
|
3291
|
+
);
|
|
3292
|
+
}
|
|
3293
|
+
return lines.join("\n");
|
|
3294
|
+
}
|
|
3295
|
+
function renderObservedOutcomesHtml(summary) {
|
|
3296
|
+
if (summary.total === 0) {
|
|
3297
|
+
return "<p>No observed outcomes recorded for this run.</p>";
|
|
3298
|
+
}
|
|
3299
|
+
const rows = summary.outcomes.map((outcome) => {
|
|
3300
|
+
const esc = (v) => v.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
|
3301
|
+
return `<tr><td>${esc(outcome.name)}</td><td>${esc(outcome.status)}</td><td>${esc(outcome.expectation)}</td><td>${esc(outcome.method ?? "-")}</td></tr>`;
|
|
3302
|
+
}).join("");
|
|
3303
|
+
return `<p>Total: ${summary.total} (passed ${summary.passed}, failed ${summary.failed}, unknown ${summary.unknown}, skipped ${summary.skipped})</p><table><thead><tr><th>Name</th><th>Status</th><th>Expectation</th><th>Method</th></tr></thead><tbody>${rows}</tbody></table>`;
|
|
3478
3304
|
}
|
|
3479
3305
|
|
|
3480
|
-
// packages/core/src/
|
|
3481
|
-
function
|
|
3482
|
-
|
|
3306
|
+
// packages/core/src/search.ts
|
|
3307
|
+
function parseDurationFilter(expr) {
|
|
3308
|
+
const raw = expr.trim();
|
|
3309
|
+
const m = raw.match(/^(>=|<=|>|<)\s*(.+)$/);
|
|
3310
|
+
if (!m) {
|
|
3311
|
+
throw new Error(
|
|
3312
|
+
`Invalid --duration "${expr}". Use forms like >5s, >=500ms, <2m.`
|
|
3313
|
+
);
|
|
3314
|
+
}
|
|
3315
|
+
const op = m[1];
|
|
3316
|
+
const ms = parseDuration(m[2].trim());
|
|
3317
|
+
return { op, ms };
|
|
3483
3318
|
}
|
|
3484
|
-
function
|
|
3485
|
-
return
|
|
3486
|
-
|
|
3487
|
-
|
|
3488
|
-
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3319
|
+
function durationMatches(valueMs, filter) {
|
|
3320
|
+
if (valueMs === void 0 || !Number.isFinite(valueMs)) return false;
|
|
3321
|
+
switch (filter.op) {
|
|
3322
|
+
case ">":
|
|
3323
|
+
return valueMs > filter.ms;
|
|
3324
|
+
case ">=":
|
|
3325
|
+
return valueMs >= filter.ms;
|
|
3326
|
+
case "<":
|
|
3327
|
+
return valueMs < filter.ms;
|
|
3328
|
+
case "<=":
|
|
3329
|
+
return valueMs <= filter.ms;
|
|
3330
|
+
default:
|
|
3331
|
+
return false;
|
|
3332
|
+
}
|
|
3492
3333
|
}
|
|
3493
|
-
function
|
|
3494
|
-
|
|
3334
|
+
function normalizeStepTypeFilter(kind, type) {
|
|
3335
|
+
const v = (kind ?? type)?.trim().toLowerCase();
|
|
3336
|
+
return v && v !== "" ? v : void 0;
|
|
3495
3337
|
}
|
|
3496
|
-
function
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
findings: [],
|
|
3508
|
-
diagnostics: [
|
|
3509
|
-
sessionDiagnostic(
|
|
3510
|
-
"AI_CHECK_INVALID_ARGUMENTS",
|
|
3511
|
-
`${scope.scopeKind} not found: ${scope.scopeLabel}`
|
|
3512
|
-
)
|
|
3513
|
-
],
|
|
3514
|
-
ruleExecutions: [],
|
|
3515
|
-
...scope.sessionWarnings?.length ? { sessionWarnings: [...scope.sessionWarnings] } : {}
|
|
3516
|
-
};
|
|
3517
|
-
}
|
|
3518
|
-
if (scope.empty || perRun.length === 0) {
|
|
3519
|
-
return {
|
|
3520
|
-
ok: false,
|
|
3521
|
-
status: "error",
|
|
3522
|
-
format: perRun[0]?.format ?? "agent-inspect-jsonl",
|
|
3523
|
-
scopeKind: scope.scopeKind,
|
|
3524
|
-
scopeLabel: scope.scopeLabel,
|
|
3525
|
-
runIds: scope.runIds,
|
|
3526
|
-
runResults: [],
|
|
3527
|
-
summary: { ...emptySummary(), errors: 1 },
|
|
3528
|
-
findings: [],
|
|
3529
|
-
diagnostics: [
|
|
3530
|
-
sessionDiagnostic(
|
|
3531
|
-
"AI_CHECK_TRACE_UNREADABLE",
|
|
3532
|
-
`No readable traces in ${scope.scopeKind}: ${scope.scopeLabel}`
|
|
3533
|
-
)
|
|
3534
|
-
],
|
|
3535
|
-
ruleExecutions: [],
|
|
3536
|
-
...scope.sessionWarnings?.length ? { sessionWarnings: [...scope.sessionWarnings] } : {}
|
|
3537
|
-
};
|
|
3338
|
+
function nameMatches(hay, needle) {
|
|
3339
|
+
return hay.toLowerCase().includes(needle.toLowerCase());
|
|
3340
|
+
}
|
|
3341
|
+
async function searchTraces(metas, options) {
|
|
3342
|
+
let filtered = filterTraces(metas, { since: options.since });
|
|
3343
|
+
const stepTypeFilter = normalizeStepTypeFilter(options.kind, options.type);
|
|
3344
|
+
const nameQuery = options.name?.trim();
|
|
3345
|
+
const toolQuery = options.tool?.trim();
|
|
3346
|
+
let durationFilter;
|
|
3347
|
+
if (options.duration) {
|
|
3348
|
+
durationFilter = parseDurationFilter(options.duration);
|
|
3538
3349
|
}
|
|
3539
|
-
|
|
3540
|
-
const
|
|
3541
|
-
const
|
|
3542
|
-
const
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3350
|
+
const limit = options.limit ?? 50;
|
|
3351
|
+
const sessionId = options.session?.trim();
|
|
3352
|
+
const observationStatus2 = parseObservationFilter(options.observation);
|
|
3353
|
+
const hasContentFilter = Boolean(
|
|
3354
|
+
options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter || observationStatus2
|
|
3355
|
+
);
|
|
3356
|
+
const results = [];
|
|
3357
|
+
const sessionLabel = sessionId && sessionId !== "" ? sessionId : void 0;
|
|
3358
|
+
if (!hasContentFilter) {
|
|
3359
|
+
for (const m of filtered) {
|
|
3360
|
+
results.push({
|
|
3361
|
+
runId: m.runId,
|
|
3362
|
+
runName: m.name,
|
|
3363
|
+
runStatus: m.status,
|
|
3364
|
+
timestamp: m.startedAt,
|
|
3365
|
+
durationMs: m.durationMs,
|
|
3366
|
+
matchReason: sessionLabel ? `trace in session ${sessionLabel}` : "trace in directory",
|
|
3367
|
+
matchedFields: sessionLabel ? ["run", "session"] : ["run"],
|
|
3368
|
+
filePath: m.filePath,
|
|
3369
|
+
...sessionLabel ? { sessionId: sessionLabel } : {}
|
|
3370
|
+
});
|
|
3551
3371
|
}
|
|
3372
|
+
results.sort((a, b) => {
|
|
3373
|
+
const ta = a.timestamp ?? 0;
|
|
3374
|
+
const tb = b.timestamp ?? 0;
|
|
3375
|
+
if (ta !== tb) return tb - ta;
|
|
3376
|
+
return a.runId.localeCompare(b.runId);
|
|
3377
|
+
});
|
|
3378
|
+
return results.slice(0, limit);
|
|
3552
3379
|
}
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
const
|
|
3380
|
+
for (const m of filtered) {
|
|
3381
|
+
if (options.status && m.status !== options.status) continue;
|
|
3382
|
+
let events = [];
|
|
3383
|
+
try {
|
|
3384
|
+
events = await readTraceEventsFromFile(m.filePath);
|
|
3385
|
+
} catch {
|
|
3386
|
+
continue;
|
|
3387
|
+
}
|
|
3388
|
+
if (events.length === 0) continue;
|
|
3389
|
+
const runMatches = matchRunLevel(m, {
|
|
3390
|
+
stepTypeFilter,
|
|
3391
|
+
nameQuery,
|
|
3392
|
+
toolQuery,
|
|
3393
|
+
durationFilter,
|
|
3394
|
+
statusFilter: options.status
|
|
3395
|
+
});
|
|
3396
|
+
results.push(...runMatches);
|
|
3397
|
+
const stepMatches = matchStepLevel(m, events, {
|
|
3398
|
+
stepTypeFilter,
|
|
3399
|
+
nameQuery,
|
|
3400
|
+
toolQuery,
|
|
3401
|
+
durationFilter,
|
|
3402
|
+
statusFilter: options.status
|
|
3403
|
+
});
|
|
3404
|
+
results.push(...stepMatches);
|
|
3405
|
+
if (observationStatus2) {
|
|
3406
|
+
const outcomes = extractOutcomesFromTraceEvents(events);
|
|
3407
|
+
const matched = outcomes.filter((outcome) => outcome.status === observationStatus2);
|
|
3408
|
+
for (const outcome of matched) {
|
|
3409
|
+
results.push({
|
|
3410
|
+
runId: m.runId,
|
|
3411
|
+
runName: m.name,
|
|
3412
|
+
runStatus: m.status,
|
|
3413
|
+
stepName: outcome.name,
|
|
3414
|
+
timestamp: outcome.observedAt,
|
|
3415
|
+
matchReason: `observation status=${outcome.status}`,
|
|
3416
|
+
matchedFields: ["outcome.status", "outcome.name"],
|
|
3417
|
+
filePath: m.filePath
|
|
3418
|
+
});
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
}
|
|
3422
|
+
results.sort((a, b) => {
|
|
3423
|
+
const ta = a.timestamp ?? 0;
|
|
3424
|
+
const tb = b.timestamp ?? 0;
|
|
3425
|
+
if (ta !== tb) return tb - ta;
|
|
3426
|
+
const runCmp = a.runId.localeCompare(b.runId);
|
|
3563
3427
|
if (runCmp !== 0) return runCmp;
|
|
3564
|
-
return a.
|
|
3428
|
+
return (a.stepName ?? "").localeCompare(b.stepName ?? "");
|
|
3565
3429
|
});
|
|
3566
|
-
|
|
3567
|
-
const status = hasErrors ? "error" : summary.failed > 0 ? "fail" : "pass";
|
|
3568
|
-
return {
|
|
3569
|
-
ok: status === "pass",
|
|
3570
|
-
status,
|
|
3571
|
-
format: perRun[0]?.format ?? "agent-inspect-jsonl",
|
|
3572
|
-
scopeKind: scope.scopeKind,
|
|
3573
|
-
scopeLabel: scope.scopeLabel,
|
|
3574
|
-
runIds: scope.runIds,
|
|
3575
|
-
runResults,
|
|
3576
|
-
summary,
|
|
3577
|
-
findings,
|
|
3578
|
-
diagnostics,
|
|
3579
|
-
ruleExecutions,
|
|
3580
|
-
...scope.sessionWarnings?.length ? { sessionWarnings: [...scope.sessionWarnings] } : {}
|
|
3581
|
-
};
|
|
3430
|
+
return results.slice(0, limit);
|
|
3582
3431
|
}
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3432
|
+
function matchRunLevel(m, opts) {
|
|
3433
|
+
if (opts.stepTypeFilter || opts.toolQuery) return [];
|
|
3434
|
+
if (opts.statusFilter && m.status !== opts.statusFilter) return [];
|
|
3435
|
+
if (opts.nameQuery && !nameMatches(m.name ?? m.runId, opts.nameQuery)) {
|
|
3436
|
+
return [];
|
|
3437
|
+
}
|
|
3438
|
+
if (opts.durationFilter && !durationMatches(m.durationMs, opts.durationFilter)) {
|
|
3439
|
+
return [];
|
|
3440
|
+
}
|
|
3441
|
+
const fields = [];
|
|
3442
|
+
if (opts.statusFilter) fields.push("run.status");
|
|
3443
|
+
if (opts.nameQuery) fields.push("run.name");
|
|
3444
|
+
if (opts.durationFilter) fields.push("run.durationMs");
|
|
3445
|
+
if (fields.length === 0) return [];
|
|
3446
|
+
return [
|
|
3447
|
+
{
|
|
3448
|
+
runId: m.runId,
|
|
3449
|
+
runName: m.name,
|
|
3450
|
+
runStatus: m.status,
|
|
3451
|
+
timestamp: m.startedAt,
|
|
3452
|
+
durationMs: m.durationMs,
|
|
3453
|
+
matchReason: `run match: ${fields.join(", ")}`,
|
|
3454
|
+
matchedFields: fields,
|
|
3455
|
+
filePath: m.filePath
|
|
3456
|
+
}
|
|
3457
|
+
];
|
|
3590
3458
|
}
|
|
3591
|
-
function
|
|
3592
|
-
const
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
if (
|
|
3596
|
-
|
|
3597
|
-
if (existing) {
|
|
3598
|
-
existing.runIds.push(runId);
|
|
3599
|
-
} else {
|
|
3600
|
-
byGroup.set(groupId, {
|
|
3601
|
-
groupId,
|
|
3602
|
-
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
3603
|
-
runIds: [runId]
|
|
3604
|
-
});
|
|
3459
|
+
function matchStepLevel(m, events, opts) {
|
|
3460
|
+
const out = [];
|
|
3461
|
+
const started = /* @__PURE__ */ new Map();
|
|
3462
|
+
for (const e of events) {
|
|
3463
|
+
if (e.event === "step_started") {
|
|
3464
|
+
started.set(e.stepId, e);
|
|
3605
3465
|
}
|
|
3606
3466
|
}
|
|
3607
|
-
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
const pushEdge = (edge) => {
|
|
3616
|
-
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
3617
|
-
if (seen.has(key)) return;
|
|
3618
|
-
seen.add(key);
|
|
3619
|
-
edges.push(edge);
|
|
3620
|
-
};
|
|
3621
|
-
for (const runId of runIds) {
|
|
3622
|
-
const meta = metaByRunId.get(runId);
|
|
3623
|
-
if (!meta) continue;
|
|
3624
|
-
if (meta.handoffFrom && meta.handoffTo) {
|
|
3625
|
-
pushEdge({
|
|
3626
|
-
from: meta.handoffFrom,
|
|
3627
|
-
to: meta.handoffTo,
|
|
3628
|
-
source: "manual",
|
|
3629
|
-
confidence: "explicit"
|
|
3630
|
-
});
|
|
3467
|
+
for (const e of events) {
|
|
3468
|
+
if (e.event !== "step_completed") continue;
|
|
3469
|
+
const c = e;
|
|
3470
|
+
const s = started.get(c.stepId);
|
|
3471
|
+
if (!s) continue;
|
|
3472
|
+
const fields = [];
|
|
3473
|
+
const stepType = s.type;
|
|
3474
|
+
if (opts.stepTypeFilter && stepType !== opts.stepTypeFilter) {
|
|
3631
3475
|
continue;
|
|
3632
3476
|
}
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3477
|
+
const hasStepFilters = opts.stepTypeFilter || opts.nameQuery || opts.toolQuery || opts.durationFilter || opts.statusFilter === "error" || opts.statusFilter === "success";
|
|
3478
|
+
if (!hasStepFilters) continue;
|
|
3479
|
+
if (opts.statusFilter === "error" && c.status === "error") {
|
|
3480
|
+
fields.push("step.status");
|
|
3481
|
+
} else if (opts.statusFilter === "success" && c.status === "success") {
|
|
3482
|
+
fields.push("step.status");
|
|
3483
|
+
} else if (opts.statusFilter === "error" || opts.statusFilter === "success") {
|
|
3484
|
+
continue;
|
|
3640
3485
|
}
|
|
3641
|
-
if (
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
to: meta.handoffTo,
|
|
3645
|
-
source: "manual",
|
|
3646
|
-
confidence: "explicit"
|
|
3647
|
-
});
|
|
3486
|
+
if (opts.nameQuery) {
|
|
3487
|
+
if (!nameMatches(s.name, opts.nameQuery)) continue;
|
|
3488
|
+
fields.push("step.name");
|
|
3648
3489
|
}
|
|
3649
|
-
if (
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
source: "inferred",
|
|
3654
|
-
confidence: "correlated"
|
|
3655
|
-
});
|
|
3656
|
-
warnings.push({
|
|
3657
|
-
code: "ambiguous-handoff-endpoints",
|
|
3658
|
-
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
3659
|
-
runId,
|
|
3660
|
-
sessionId
|
|
3661
|
-
});
|
|
3490
|
+
if (opts.toolQuery) {
|
|
3491
|
+
const toolName2 = typeof s.metadata?.toolName === "string" ? s.metadata.toolName : s.name;
|
|
3492
|
+
if (!nameMatches(toolName2, opts.toolQuery)) continue;
|
|
3493
|
+
fields.push("step.tool");
|
|
3662
3494
|
}
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
if (from !== 0) return from;
|
|
3667
|
-
return a.to.localeCompare(b.to);
|
|
3668
|
-
});
|
|
3669
|
-
}
|
|
3670
|
-
function buildRetries(runIds, metaByRunId, warnings, sessionId) {
|
|
3671
|
-
const retries = [];
|
|
3672
|
-
for (const runId of runIds) {
|
|
3673
|
-
const meta = metaByRunId.get(runId);
|
|
3674
|
-
if (!meta) continue;
|
|
3675
|
-
if (meta.retryOf) {
|
|
3676
|
-
retries.push({
|
|
3677
|
-
runId,
|
|
3678
|
-
retryOf: meta.retryOf,
|
|
3679
|
-
attempt: meta.attempt,
|
|
3680
|
-
source: "manual",
|
|
3681
|
-
confidence: "explicit"
|
|
3682
|
-
});
|
|
3683
|
-
continue;
|
|
3495
|
+
if (opts.durationFilter) {
|
|
3496
|
+
if (!durationMatches(c.durationMs, opts.durationFilter)) continue;
|
|
3497
|
+
fields.push("step.durationMs");
|
|
3684
3498
|
}
|
|
3685
|
-
if (
|
|
3686
|
-
|
|
3687
|
-
runId,
|
|
3688
|
-
attempt: meta.attempt,
|
|
3689
|
-
source: "inferred",
|
|
3690
|
-
confidence: "correlated"
|
|
3691
|
-
});
|
|
3692
|
-
warnings.push({
|
|
3693
|
-
code: "ambiguous-retry-link",
|
|
3694
|
-
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
3695
|
-
runId,
|
|
3696
|
-
sessionId
|
|
3697
|
-
});
|
|
3499
|
+
if (opts.stepTypeFilter) {
|
|
3500
|
+
fields.push("step.type");
|
|
3698
3501
|
}
|
|
3502
|
+
if (fields.length === 0) continue;
|
|
3503
|
+
out.push({
|
|
3504
|
+
runId: m.runId,
|
|
3505
|
+
runName: m.name,
|
|
3506
|
+
runStatus: m.status,
|
|
3507
|
+
stepId: c.stepId,
|
|
3508
|
+
stepName: s.name,
|
|
3509
|
+
stepType,
|
|
3510
|
+
timestamp: s.startTime ?? s.timestamp,
|
|
3511
|
+
durationMs: c.durationMs,
|
|
3512
|
+
matchReason: `step match: ${fields.join(", ")}`,
|
|
3513
|
+
matchedFields: fields,
|
|
3514
|
+
filePath: m.filePath
|
|
3515
|
+
});
|
|
3699
3516
|
}
|
|
3700
|
-
return
|
|
3517
|
+
return out;
|
|
3701
3518
|
}
|
|
3702
|
-
function
|
|
3703
|
-
const
|
|
3704
|
-
const
|
|
3705
|
-
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
const path14 = [];
|
|
3712
|
-
const visited = /* @__PURE__ */ new Set();
|
|
3713
|
-
const pushRun = (run, confidence, source) => {
|
|
3714
|
-
if (visited.has(run.runId)) return;
|
|
3715
|
-
visited.add(run.runId);
|
|
3716
|
-
path14.push({
|
|
3717
|
-
runId: run.runId,
|
|
3718
|
-
name: run.name,
|
|
3719
|
-
startedAt: run.startedAt,
|
|
3720
|
-
durationMs: run.durationMs,
|
|
3721
|
-
confidence,
|
|
3722
|
-
source
|
|
3723
|
-
});
|
|
3724
|
-
};
|
|
3725
|
-
for (const edge of handoffs) {
|
|
3726
|
-
if (edge.confidence !== "explicit") continue;
|
|
3727
|
-
const fromRun = [...runById.values()].find(
|
|
3728
|
-
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
3729
|
-
);
|
|
3730
|
-
const toRun = [...runById.values()].find(
|
|
3731
|
-
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
3732
|
-
);
|
|
3733
|
-
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
3734
|
-
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
3519
|
+
async function loadTraceMetadataList(_traceDir, fileNames, getPath) {
|
|
3520
|
+
const metas = [];
|
|
3521
|
+
for (const fileName of fileNames) {
|
|
3522
|
+
try {
|
|
3523
|
+
const filePath = getPath(fileName);
|
|
3524
|
+
const meta = await extractMetadata(filePath);
|
|
3525
|
+
metas.push(meta);
|
|
3526
|
+
} catch {
|
|
3527
|
+
}
|
|
3735
3528
|
}
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3529
|
+
return metas;
|
|
3530
|
+
}
|
|
3531
|
+
|
|
3532
|
+
// packages/core/src/sessions/activity.ts
|
|
3533
|
+
function statusLine(session) {
|
|
3534
|
+
const name = session.workflowId ?? session.correlationId ?? session.sessionId;
|
|
3535
|
+
const status = session.status;
|
|
3536
|
+
if (session.lastError) {
|
|
3537
|
+
return `${name} session ${session.sessionId} failed at ${session.lastError.message}`;
|
|
3740
3538
|
}
|
|
3741
|
-
|
|
3539
|
+
if (session.observationSummary) {
|
|
3540
|
+
return `${name} session ${session.sessionId} ${status} with observation warning`;
|
|
3541
|
+
}
|
|
3542
|
+
return `${name} session ${session.sessionId} ${status}`;
|
|
3742
3543
|
}
|
|
3743
|
-
function
|
|
3744
|
-
|
|
3745
|
-
|
|
3544
|
+
function parseSinceMs(since, nowMs) {
|
|
3545
|
+
if (!since || since.trim() === "") return nowMs - 7 * 864e5;
|
|
3546
|
+
const trimmed = since.trim().toLowerCase();
|
|
3547
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
3548
|
+
if (!match) return nowMs - 7 * 864e5;
|
|
3549
|
+
const amount = Number.parseInt(match[1], 10);
|
|
3550
|
+
const unit = match[2];
|
|
3551
|
+
const mult = unit === "s" ? 1e3 : unit === "m" ? 6e4 : unit === "h" ? 36e5 : 864e5;
|
|
3552
|
+
return nowMs - amount * mult;
|
|
3746
3553
|
}
|
|
3747
|
-
function
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
|
|
3751
|
-
|
|
3752
|
-
|
|
3554
|
+
function isFailed(status) {
|
|
3555
|
+
return status === "error";
|
|
3556
|
+
}
|
|
3557
|
+
function isStale(status) {
|
|
3558
|
+
return status === "stale";
|
|
3559
|
+
}
|
|
3560
|
+
function guardrailWarnings(session) {
|
|
3561
|
+
const summary = session.checkSummary;
|
|
3562
|
+
if (!summary) return 0;
|
|
3563
|
+
return summary.warn;
|
|
3564
|
+
}
|
|
3565
|
+
function buildActivitySummary(index, options = {}) {
|
|
3566
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
3567
|
+
const sinceMs = parseSinceMs(options.since, nowMs);
|
|
3568
|
+
const sinceIso = new Date(sinceMs).toISOString();
|
|
3569
|
+
const limit = Number.isInteger(options.limit) && options.limit > 0 ? options.limit : 20;
|
|
3570
|
+
const inWindow = index.sessions.filter((session) => {
|
|
3571
|
+
const activityMs2 = Date.parse(session.lastActivity);
|
|
3572
|
+
return Number.isFinite(activityMs2) && activityMs2 >= sinceMs;
|
|
3573
|
+
});
|
|
3574
|
+
const entries = [...inWindow].sort((a, b) => Date.parse(b.lastActivity) - Date.parse(a.lastActivity)).slice(0, limit).map((session) => ({
|
|
3575
|
+
sessionId: session.sessionId,
|
|
3576
|
+
status: session.status,
|
|
3577
|
+
summary: statusLine(session),
|
|
3578
|
+
lastActivity: session.lastActivity,
|
|
3579
|
+
runCount: session.runIds.length
|
|
3580
|
+
}));
|
|
3581
|
+
let failed = 0;
|
|
3582
|
+
let stale = 0;
|
|
3583
|
+
let guardrailWarningTotal = 0;
|
|
3584
|
+
for (const session of inWindow) {
|
|
3585
|
+
if (isFailed(session.status)) failed += 1;
|
|
3586
|
+
if (isStale(session.status)) stale += 1;
|
|
3587
|
+
guardrailWarningTotal += guardrailWarnings(session);
|
|
3753
3588
|
}
|
|
3754
|
-
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
|
|
3762
|
-
|
|
3763
|
-
|
|
3589
|
+
return {
|
|
3590
|
+
since: sinceIso,
|
|
3591
|
+
sessions: inWindow.length,
|
|
3592
|
+
failed,
|
|
3593
|
+
stale,
|
|
3594
|
+
guardrailWarnings: guardrailWarningTotal,
|
|
3595
|
+
entries
|
|
3596
|
+
};
|
|
3597
|
+
}
|
|
3598
|
+
function renderActivitySummaryHuman(summary, options) {
|
|
3599
|
+
const lines = [];
|
|
3600
|
+
const nowMs = options?.nowMs ?? Date.now();
|
|
3601
|
+
const todayStart = new Date(nowMs);
|
|
3602
|
+
todayStart.setHours(0, 0, 0, 0);
|
|
3603
|
+
const todayMs = todayStart.getTime();
|
|
3604
|
+
const today = summary.entries.filter(
|
|
3605
|
+
(entry) => Date.parse(entry.lastActivity) >= todayMs
|
|
3606
|
+
);
|
|
3607
|
+
if (today.length > 0) {
|
|
3608
|
+
lines.push("Today");
|
|
3609
|
+
for (const entry of today) {
|
|
3610
|
+
lines.push(` ${entry.summary}`);
|
|
3764
3611
|
}
|
|
3765
|
-
|
|
3766
|
-
bucket.push(run);
|
|
3767
|
-
sessionsByKey.set(key, bucket);
|
|
3612
|
+
lines.push("");
|
|
3768
3613
|
}
|
|
3769
|
-
|
|
3770
|
-
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3782
|
-
|
|
3783
|
-
|
|
3784
|
-
{
|
|
3785
|
-
|
|
3786
|
-
runIds,
|
|
3787
|
-
groups,
|
|
3788
|
-
handoffs,
|
|
3789
|
-
retries,
|
|
3790
|
-
criticalPath
|
|
3791
|
-
},
|
|
3792
|
-
runs,
|
|
3793
|
-
{
|
|
3794
|
-
nowMs: options.nowMs,
|
|
3795
|
-
staleThresholdMs: options.staleThresholdMs
|
|
3614
|
+
lines.push(`Since ${summary.since}`);
|
|
3615
|
+
lines.push(` ${summary.sessions} sessions`);
|
|
3616
|
+
lines.push(` ${summary.failed} failed`);
|
|
3617
|
+
lines.push(` ${summary.stale} stale`);
|
|
3618
|
+
lines.push(` ${summary.guardrailWarnings} guardrail warnings`);
|
|
3619
|
+
return lines.join("\n");
|
|
3620
|
+
}
|
|
3621
|
+
|
|
3622
|
+
// packages/core/src/sessions/load.ts
|
|
3623
|
+
async function enrichSessionRunRecord(meta) {
|
|
3624
|
+
let metadata;
|
|
3625
|
+
try {
|
|
3626
|
+
const events = await readTraceEventsFromFile(meta.filePath);
|
|
3627
|
+
for (const event of events) {
|
|
3628
|
+
if (event.event !== "run_started") continue;
|
|
3629
|
+
if (event.metadata && typeof event.metadata === "object") {
|
|
3630
|
+
metadata = event.metadata;
|
|
3796
3631
|
}
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
warnings.push({
|
|
3801
|
-
code: "missing-session-id",
|
|
3802
|
-
message: "No sessionId (or correlated groupId) found on input runs."
|
|
3803
|
-
});
|
|
3632
|
+
break;
|
|
3633
|
+
}
|
|
3634
|
+
} catch {
|
|
3804
3635
|
}
|
|
3805
|
-
warnings.sort((a, b) => {
|
|
3806
|
-
const code = a.code.localeCompare(b.code);
|
|
3807
|
-
if (code !== 0) return code;
|
|
3808
|
-
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
3809
|
-
});
|
|
3810
3636
|
return {
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3637
|
+
runId: meta.runId,
|
|
3638
|
+
name: meta.name,
|
|
3639
|
+
status: meta.status,
|
|
3640
|
+
startedAt: meta.startedAt,
|
|
3641
|
+
endedAt: meta.endedAt,
|
|
3642
|
+
durationMs: meta.durationMs,
|
|
3643
|
+
filePath: meta.filePath,
|
|
3644
|
+
metadata
|
|
3815
3645
|
};
|
|
3816
3646
|
}
|
|
3647
|
+
async function loadSessionRunRecords(metas) {
|
|
3648
|
+
const out = [];
|
|
3649
|
+
for (const meta of metas) {
|
|
3650
|
+
out.push(await enrichSessionRunRecord(meta));
|
|
3651
|
+
}
|
|
3652
|
+
return out;
|
|
3653
|
+
}
|
|
3817
3654
|
|
|
3818
|
-
// packages/core/src/sessions/
|
|
3819
|
-
function
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
if (
|
|
3824
|
-
|
|
3825
|
-
correlateByGroupId: options.correlateByGroupId === true
|
|
3826
|
-
});
|
|
3827
|
-
warnings.push(...index.warnings);
|
|
3828
|
-
const session = index.sessions.find((item) => item.sessionId === sessionId);
|
|
3829
|
-
if (!session) {
|
|
3830
|
-
return {
|
|
3831
|
-
metas: [],
|
|
3832
|
-
scopeLabel: sessionId,
|
|
3833
|
-
scopeKind: "session",
|
|
3834
|
-
runIds: [],
|
|
3835
|
-
warnings,
|
|
3836
|
-
notFound: true
|
|
3837
|
-
};
|
|
3838
|
-
}
|
|
3839
|
-
const runIdSet = new Set(session.runIds);
|
|
3840
|
-
const filtered = metas.filter((meta) => runIdSet.has(meta.runId));
|
|
3841
|
-
return {
|
|
3842
|
-
metas: filtered,
|
|
3843
|
-
scopeLabel: sessionId,
|
|
3844
|
-
scopeKind: "session",
|
|
3845
|
-
runIds: session.runIds,
|
|
3846
|
-
warnings,
|
|
3847
|
-
notFound: false
|
|
3848
|
-
};
|
|
3655
|
+
// packages/core/src/sessions/metadata.ts
|
|
3656
|
+
function isNonEmptyString3(value) {
|
|
3657
|
+
return typeof value === "string" && value.trim() !== "";
|
|
3658
|
+
}
|
|
3659
|
+
function finitePositiveInt(value) {
|
|
3660
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
3661
|
+
return void 0;
|
|
3849
3662
|
}
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
};
|
|
3663
|
+
return Math.trunc(value);
|
|
3664
|
+
}
|
|
3665
|
+
function extractSessionWorkflowMetadata(record) {
|
|
3666
|
+
if (!record) return void 0;
|
|
3667
|
+
const out = {};
|
|
3668
|
+
let found = false;
|
|
3669
|
+
const assignString = (key, value) => {
|
|
3670
|
+
if (isNonEmptyString3(value)) {
|
|
3671
|
+
out[key] = value.trim();
|
|
3672
|
+
found = true;
|
|
3861
3673
|
}
|
|
3862
|
-
const runIdSet = new Set(runIds);
|
|
3863
|
-
return {
|
|
3864
|
-
metas: metas.filter((meta) => runIdSet.has(meta.runId)),
|
|
3865
|
-
scopeLabel: groupId,
|
|
3866
|
-
scopeKind: "group",
|
|
3867
|
-
runIds,
|
|
3868
|
-
warnings,
|
|
3869
|
-
notFound: false
|
|
3870
|
-
};
|
|
3871
|
-
}
|
|
3872
|
-
return {
|
|
3873
|
-
metas: [...metas],
|
|
3874
|
-
scopeLabel: "",
|
|
3875
|
-
scopeKind: "session",
|
|
3876
|
-
runIds: [],
|
|
3877
|
-
warnings,
|
|
3878
|
-
notFound: false
|
|
3879
3674
|
};
|
|
3675
|
+
assignString("sessionId", record.sessionId);
|
|
3676
|
+
assignString("conversationId", record.conversationId);
|
|
3677
|
+
assignString("groupId", record.groupId);
|
|
3678
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
3679
|
+
assignString("retryOf", record.retryOf);
|
|
3680
|
+
assignString("retryReason", record.retryReason);
|
|
3681
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
3682
|
+
assignString("handoffTo", record.handoffTo);
|
|
3683
|
+
assignString("subAgentId", record.subAgentId);
|
|
3684
|
+
assignString("subAgentName", record.subAgentName);
|
|
3685
|
+
assignString("jobId", record.jobId);
|
|
3686
|
+
assignString("queueName", record.queueName);
|
|
3687
|
+
assignString("workflowName", record.workflowName);
|
|
3688
|
+
assignString("workflowStep", record.workflowStep);
|
|
3689
|
+
assignString("toolCallId", record.toolCallId);
|
|
3690
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
3691
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
3692
|
+
assignString("operationId", record.operationId);
|
|
3693
|
+
assignString("attemptId", record.attemptId);
|
|
3694
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
3695
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
3696
|
+
assignString("correlationId", record.correlationId);
|
|
3697
|
+
assignString("requestId", record.requestId);
|
|
3698
|
+
assignString("decisionId", record.decisionId);
|
|
3699
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
3700
|
+
if (attempt !== void 0) {
|
|
3701
|
+
out.attempt = attempt;
|
|
3702
|
+
found = true;
|
|
3703
|
+
}
|
|
3704
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
3705
|
+
if (attemptNumber !== void 0) {
|
|
3706
|
+
out.attemptNumber = attemptNumber;
|
|
3707
|
+
found = true;
|
|
3708
|
+
}
|
|
3709
|
+
return found ? out : void 0;
|
|
3880
3710
|
}
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3711
|
+
function sessionKeyForRun(meta, options) {
|
|
3712
|
+
if (meta?.sessionId) return meta.sessionId;
|
|
3713
|
+
if (options?.correlateByGroupId && meta?.groupId) {
|
|
3714
|
+
return `group:${meta.groupId}`;
|
|
3715
|
+
}
|
|
3716
|
+
return void 0;
|
|
3717
|
+
}
|
|
3718
|
+
|
|
3719
|
+
// packages/core/src/sessions/status.ts
|
|
3720
|
+
var DEFAULT_STALE_THRESHOLD_MS = 864e5;
|
|
3721
|
+
var EXPLICIT_STATUS_PRIORITY = {
|
|
3722
|
+
error: 5,
|
|
3723
|
+
waiting_input: 4,
|
|
3724
|
+
idle: 3,
|
|
3725
|
+
stale: 2,
|
|
3726
|
+
completed: 1
|
|
3727
|
+
};
|
|
3728
|
+
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
3729
|
+
"running",
|
|
3730
|
+
"waiting_input",
|
|
3731
|
+
"idle",
|
|
3732
|
+
"completed",
|
|
3733
|
+
"error",
|
|
3734
|
+
"stale",
|
|
3735
|
+
"unknown"
|
|
3886
3736
|
]);
|
|
3887
|
-
function
|
|
3888
|
-
return typeof value === "
|
|
3737
|
+
function isExplicitSessionStatus(value) {
|
|
3738
|
+
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
3889
3739
|
}
|
|
3890
|
-
function
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
3894
|
-
|
|
3740
|
+
function activityMs(run) {
|
|
3741
|
+
return run.endedAt ?? run.startedAt ?? 0;
|
|
3742
|
+
}
|
|
3743
|
+
function latestActivityMs(runs) {
|
|
3744
|
+
let latest = 0;
|
|
3745
|
+
for (const run of runs) {
|
|
3746
|
+
const ms = activityMs(run);
|
|
3747
|
+
if (ms > latest) latest = ms;
|
|
3895
3748
|
}
|
|
3749
|
+
return latest;
|
|
3896
3750
|
}
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
let checked = 0;
|
|
3904
|
-
for await (const line of rl) {
|
|
3905
|
-
const trimmed = line.trim();
|
|
3906
|
-
if (trimmed === "") continue;
|
|
3907
|
-
const parsed = safeParse(trimmed);
|
|
3908
|
-
if (!parsed) continue;
|
|
3909
|
-
if (!isRecord6(parsed)) continue;
|
|
3910
|
-
checked += 1;
|
|
3911
|
-
if (isTraceEvent(parsed)) return true;
|
|
3912
|
-
const ev = parsed.event;
|
|
3913
|
-
const runId = parsed.runId;
|
|
3914
|
-
if (typeof ev === "string" && KNOWN_EVENTS.has(ev) && typeof runId === "string") {
|
|
3915
|
-
return true;
|
|
3916
|
-
}
|
|
3917
|
-
if (checked >= 20) break;
|
|
3751
|
+
function earliestStart(runs) {
|
|
3752
|
+
let earliest;
|
|
3753
|
+
for (const run of runs) {
|
|
3754
|
+
if (run.startedAt === void 0) continue;
|
|
3755
|
+
if (earliest === void 0 || run.startedAt < earliest) {
|
|
3756
|
+
earliest = run.startedAt;
|
|
3918
3757
|
}
|
|
3919
|
-
return false;
|
|
3920
|
-
} catch {
|
|
3921
|
-
return false;
|
|
3922
3758
|
}
|
|
3759
|
+
return earliest;
|
|
3923
3760
|
}
|
|
3924
|
-
|
|
3925
|
-
|
|
3926
|
-
|
|
3927
|
-
const
|
|
3928
|
-
|
|
3929
|
-
|
|
3761
|
+
function latestEndWhenAllEnded(runs) {
|
|
3762
|
+
if (runs.length === 0) return void 0;
|
|
3763
|
+
let latest;
|
|
3764
|
+
for (const run of runs) {
|
|
3765
|
+
if (run.endedAt === void 0) return void 0;
|
|
3766
|
+
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
3930
3767
|
}
|
|
3931
|
-
return
|
|
3932
|
-
}
|
|
3933
|
-
function runActivityMs(run) {
|
|
3934
|
-
if (run.startedAt !== void 0 && Number.isFinite(run.startedAt)) return run.startedAt;
|
|
3935
|
-
if (run.endedAt !== void 0 && Number.isFinite(run.endedAt)) return run.endedAt;
|
|
3936
|
-
return void 0;
|
|
3768
|
+
return latest;
|
|
3937
3769
|
}
|
|
3938
|
-
function
|
|
3939
|
-
|
|
3940
|
-
|
|
3770
|
+
function pickExplicitStatus(runs) {
|
|
3771
|
+
let best;
|
|
3772
|
+
let bestPriority = 0;
|
|
3941
3773
|
for (const run of runs) {
|
|
3942
|
-
const
|
|
3943
|
-
if (
|
|
3944
|
-
|
|
3774
|
+
const raw = run.metadata?.sessionStatus;
|
|
3775
|
+
if (!isExplicitSessionStatus(raw)) continue;
|
|
3776
|
+
const priority = EXPLICIT_STATUS_PRIORITY[raw] ?? 0;
|
|
3777
|
+
if (priority > bestPriority) {
|
|
3778
|
+
bestPriority = priority;
|
|
3779
|
+
best = raw;
|
|
3945
3780
|
}
|
|
3946
3781
|
}
|
|
3947
|
-
return
|
|
3782
|
+
return best;
|
|
3948
3783
|
}
|
|
3949
|
-
function
|
|
3950
|
-
|
|
3784
|
+
function deriveLastError(runs) {
|
|
3785
|
+
const errorRuns = runs.filter((run) => run.status === "error").sort((a, b) => activityMs(b) - activityMs(a));
|
|
3786
|
+
const latest = errorRuns[0];
|
|
3787
|
+
if (!latest) return void 0;
|
|
3788
|
+
const meta = latest.metadata ?? {};
|
|
3789
|
+
const message = typeof meta.errorMessage === "string" && meta.errorMessage.trim() !== "" ? meta.errorMessage.trim() : latest.name ?? latest.runId;
|
|
3790
|
+
const code = typeof meta.errorCode === "string" && meta.errorCode.trim() !== "" ? meta.errorCode.trim() : void 0;
|
|
3791
|
+
return { runId: latest.runId, message, code };
|
|
3951
3792
|
}
|
|
3952
|
-
function
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
3964
|
-
"bundle accepts only one target: a run id, --session, or --since (not combined)."
|
|
3965
|
-
);
|
|
3966
|
-
}
|
|
3967
|
-
if (runId) {
|
|
3968
|
-
const known = runs.some((run) => run.runId === runId);
|
|
3969
|
-
if (!known) {
|
|
3970
|
-
throw new Error(`Run "${runId}" was not found in the trace directory.`);
|
|
3793
|
+
function deriveCheckSummary(runs) {
|
|
3794
|
+
let pass = 0;
|
|
3795
|
+
let fail4 = 0;
|
|
3796
|
+
let warn2 = 0;
|
|
3797
|
+
let found = false;
|
|
3798
|
+
for (const run of runs) {
|
|
3799
|
+
const summary = run.metadata?.checkSummary;
|
|
3800
|
+
if (!summary || typeof summary !== "object") continue;
|
|
3801
|
+
const record = summary;
|
|
3802
|
+
if (typeof record.pass === "number") {
|
|
3803
|
+
pass += record.pass;
|
|
3804
|
+
found = true;
|
|
3971
3805
|
}
|
|
3972
|
-
|
|
3973
|
-
|
|
3974
|
-
|
|
3975
|
-
const session = findSession(index, sessionId);
|
|
3976
|
-
if (!session) {
|
|
3977
|
-
throw new Error(`Session "${sessionId}" was not found.`);
|
|
3806
|
+
if (typeof record.fail === "number") {
|
|
3807
|
+
fail4 += record.fail;
|
|
3808
|
+
found = true;
|
|
3978
3809
|
}
|
|
3979
|
-
if (
|
|
3980
|
-
|
|
3810
|
+
if (typeof record.warn === "number") {
|
|
3811
|
+
warn2 += record.warn;
|
|
3812
|
+
found = true;
|
|
3981
3813
|
}
|
|
3982
|
-
return {
|
|
3983
|
-
runIds: [...session.runIds].sort((a, b) => a.localeCompare(b)),
|
|
3984
|
-
sessionId
|
|
3985
|
-
};
|
|
3986
3814
|
}
|
|
3987
|
-
|
|
3988
|
-
if (runIds.length === 0) {
|
|
3989
|
-
throw new Error(`No runs matched --since ${since}.`);
|
|
3990
|
-
}
|
|
3991
|
-
return { runIds, since };
|
|
3815
|
+
return found ? { pass, fail: fail4, warn: warn2 } : void 0;
|
|
3992
3816
|
}
|
|
3993
|
-
|
|
3994
|
-
|
|
3995
|
-
|
|
3996
|
-
|
|
3997
|
-
|
|
3998
|
-
|
|
3999
|
-
|
|
4000
|
-
return
|
|
3817
|
+
function deriveObservationSummary(runs) {
|
|
3818
|
+
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
3819
|
+
const value = run.metadata?.observationSummary;
|
|
3820
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
3821
|
+
return value.trim();
|
|
3822
|
+
}
|
|
3823
|
+
}
|
|
3824
|
+
return void 0;
|
|
4001
3825
|
}
|
|
4002
|
-
function
|
|
4003
|
-
if (
|
|
4004
|
-
return
|
|
3826
|
+
function deriveSessionStatus(runs, options = {}) {
|
|
3827
|
+
if (runs.length === 0) return "unknown";
|
|
3828
|
+
if (runs.some((run) => run.status === "running")) return "running";
|
|
3829
|
+
const explicit = pickExplicitStatus(runs);
|
|
3830
|
+
if (explicit && explicit !== "running") return explicit;
|
|
3831
|
+
if (runs.some((run) => run.status === "error")) return "error";
|
|
3832
|
+
if (runs.every((run) => run.status === "success")) return "completed";
|
|
3833
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
3834
|
+
const staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
3835
|
+
const lastMs = latestActivityMs(runs);
|
|
3836
|
+
if (lastMs > 0 && nowMs - lastMs > staleThresholdMs) return "stale";
|
|
3837
|
+
return "unknown";
|
|
4005
3838
|
}
|
|
4006
|
-
function
|
|
4007
|
-
|
|
4008
|
-
|
|
3839
|
+
function enrichSessionSummary(summary, runs, options = {}) {
|
|
3840
|
+
const sessionRuns = runs.filter((run) => summary.runIds.includes(run.runId)).sort((a, b) => a.runId.localeCompare(b.runId));
|
|
3841
|
+
const startedAt = earliestStart(sessionRuns);
|
|
3842
|
+
const endedAt = latestEndWhenAllEnded(sessionRuns);
|
|
3843
|
+
const durationMs = startedAt !== void 0 && endedAt !== void 0 ? endedAt - startedAt : void 0;
|
|
3844
|
+
let correlationId;
|
|
3845
|
+
let jobId;
|
|
3846
|
+
let workflowId;
|
|
3847
|
+
for (const run of sessionRuns) {
|
|
3848
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
3849
|
+
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
3850
|
+
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
3851
|
+
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
3852
|
+
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
3853
|
+
}
|
|
3854
|
+
const lastMs = latestActivityMs(sessionRuns);
|
|
3855
|
+
const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
3856
|
+
const retryCount2 = summary.retries.filter(
|
|
3857
|
+
(retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
|
|
3858
|
+
).length;
|
|
3859
|
+
return {
|
|
3860
|
+
...summary,
|
|
3861
|
+
status: deriveSessionStatus(sessionRuns, options),
|
|
3862
|
+
startedAt,
|
|
3863
|
+
endedAt,
|
|
3864
|
+
durationMs,
|
|
3865
|
+
correlationId,
|
|
3866
|
+
jobId,
|
|
3867
|
+
workflowId,
|
|
3868
|
+
lastError: deriveLastError(sessionRuns),
|
|
3869
|
+
lastActivity,
|
|
3870
|
+
retryCount: retryCount2,
|
|
3871
|
+
observationSummary: deriveObservationSummary(sessionRuns),
|
|
3872
|
+
checkSummary: deriveCheckSummary(sessionRuns)
|
|
3873
|
+
};
|
|
4009
3874
|
}
|
|
4010
3875
|
|
|
4011
|
-
// packages/core/src/
|
|
4012
|
-
|
|
4013
|
-
var PLACEHOLDER_NOTE = "No eval or performance artifacts were requested for this bundle.";
|
|
4014
|
-
function buildBundleMetadata(parts) {
|
|
4015
|
-
const aggregate = aggregateBundleSafeStatus(
|
|
4016
|
-
parts.checks.runs.map((run) => run.status)
|
|
4017
|
-
);
|
|
3876
|
+
// packages/core/src/sessions/checks.ts
|
|
3877
|
+
function emptySummary() {
|
|
4018
3878
|
return {
|
|
4019
|
-
|
|
4020
|
-
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
|
|
4024
|
-
|
|
4025
|
-
|
|
4026
|
-
|
|
4027
|
-
|
|
4028
|
-
...parts.resolve.since !== void 0 ? { since: parts.resolve.since } : {}
|
|
3879
|
+
passed: 0,
|
|
3880
|
+
failed: 0,
|
|
3881
|
+
warnings: 0,
|
|
3882
|
+
errors: 0,
|
|
3883
|
+
rulesEvaluated: 0,
|
|
3884
|
+
rulesPassed: 0,
|
|
3885
|
+
rulesWarning: 0,
|
|
3886
|
+
rulesFailed: 0,
|
|
3887
|
+
rulesError: 0
|
|
4029
3888
|
};
|
|
4030
3889
|
}
|
|
4031
|
-
function
|
|
3890
|
+
function mergeSummary(target, source) {
|
|
4032
3891
|
return {
|
|
4033
|
-
|
|
4034
|
-
|
|
3892
|
+
passed: target.passed + source.passed,
|
|
3893
|
+
failed: target.failed + source.failed,
|
|
3894
|
+
warnings: target.warnings + source.warnings,
|
|
3895
|
+
errors: target.errors + source.errors,
|
|
3896
|
+
rulesEvaluated: target.rulesEvaluated + source.rulesEvaluated,
|
|
3897
|
+
rulesPassed: target.rulesPassed + source.rulesPassed,
|
|
3898
|
+
rulesWarning: target.rulesWarning + source.rulesWarning,
|
|
3899
|
+
rulesFailed: target.rulesFailed + source.rulesFailed,
|
|
3900
|
+
rulesError: target.rulesError + source.rulesError
|
|
4035
3901
|
};
|
|
4036
3902
|
}
|
|
4037
|
-
|
|
4038
|
-
|
|
4039
|
-
function markdownTable(rows) {
|
|
4040
|
-
const lines = ["| Field | Value |", "| --- | --- |"];
|
|
4041
|
-
for (const [key, value] of rows) {
|
|
4042
|
-
lines.push(`| ${key} | ${value ?? "unknown"} |`);
|
|
4043
|
-
}
|
|
4044
|
-
return lines.join("\n");
|
|
3903
|
+
function sessionDiagnostic(code, message) {
|
|
3904
|
+
return { code, message, severity: "error" };
|
|
4045
3905
|
}
|
|
4046
|
-
function
|
|
4047
|
-
|
|
4048
|
-
|
|
4049
|
-
|
|
4050
|
-
|
|
4051
|
-
|
|
4052
|
-
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
[
|
|
4058
|
-
[
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
|
|
4067
|
-
"",
|
|
4068
|
-
`Aggregate: **${checks.aggregateStatus}**`,
|
|
4069
|
-
""
|
|
4070
|
-
];
|
|
4071
|
-
for (const run of checks.runs) {
|
|
4072
|
-
const source = run.sourceStatus !== void 0 && run.sourceStatus !== run.status ? `; source ${run.sourceStatus}` : "";
|
|
4073
|
-
lines.push(
|
|
4074
|
-
`- \`${run.runId}\`: artifact ${run.status}${source} (${run.findings} finding(s), ${run.errors} error(s), ${run.warnings} warning(s))`
|
|
4075
|
-
);
|
|
3906
|
+
function aggregateSessionCheckResults(perRun, scope) {
|
|
3907
|
+
if (scope.notFound) {
|
|
3908
|
+
return {
|
|
3909
|
+
ok: false,
|
|
3910
|
+
status: "error",
|
|
3911
|
+
format: perRun[0]?.format ?? "agent-inspect-jsonl",
|
|
3912
|
+
scopeKind: scope.scopeKind,
|
|
3913
|
+
scopeLabel: scope.scopeLabel,
|
|
3914
|
+
runIds: [],
|
|
3915
|
+
runResults: [],
|
|
3916
|
+
summary: { ...emptySummary(), errors: 1 },
|
|
3917
|
+
findings: [],
|
|
3918
|
+
diagnostics: [
|
|
3919
|
+
sessionDiagnostic(
|
|
3920
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
3921
|
+
`${scope.scopeKind} not found: ${scope.scopeLabel}`
|
|
3922
|
+
)
|
|
3923
|
+
],
|
|
3924
|
+
ruleExecutions: [],
|
|
3925
|
+
...scope.sessionWarnings?.length ? { sessionWarnings: [...scope.sessionWarnings] } : {}
|
|
3926
|
+
};
|
|
4076
3927
|
}
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
3928
|
+
if (scope.empty || perRun.length === 0) {
|
|
3929
|
+
return {
|
|
3930
|
+
ok: false,
|
|
3931
|
+
status: "error",
|
|
3932
|
+
format: perRun[0]?.format ?? "agent-inspect-jsonl",
|
|
3933
|
+
scopeKind: scope.scopeKind,
|
|
3934
|
+
scopeLabel: scope.scopeLabel,
|
|
3935
|
+
runIds: scope.runIds,
|
|
3936
|
+
runResults: [],
|
|
3937
|
+
summary: { ...emptySummary(), errors: 1 },
|
|
3938
|
+
findings: [],
|
|
3939
|
+
diagnostics: [
|
|
3940
|
+
sessionDiagnostic(
|
|
3941
|
+
"AI_CHECK_TRACE_UNREADABLE",
|
|
3942
|
+
`No readable traces in ${scope.scopeKind}: ${scope.scopeLabel}`
|
|
3943
|
+
)
|
|
3944
|
+
],
|
|
3945
|
+
ruleExecutions: [],
|
|
3946
|
+
...scope.sessionWarnings?.length ? { sessionWarnings: [...scope.sessionWarnings] } : {}
|
|
3947
|
+
};
|
|
4081
3948
|
}
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
safe = path7.basename(safe);
|
|
4096
|
-
safe = safe.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
|
4097
|
-
if (safe === "" || safe === "." || safe === "..") {
|
|
4098
|
-
safe = "run_unknown";
|
|
3949
|
+
let summary = emptySummary();
|
|
3950
|
+
const findings = [];
|
|
3951
|
+
const diagnostics = [];
|
|
3952
|
+
const ruleExecutions = [];
|
|
3953
|
+
const runResults = [];
|
|
3954
|
+
for (const result of perRun) {
|
|
3955
|
+
summary = mergeSummary(summary, result.summary);
|
|
3956
|
+
findings.push(...result.findings);
|
|
3957
|
+
diagnostics.push(...result.diagnostics);
|
|
3958
|
+
ruleExecutions.push(...result.ruleExecutions ?? []);
|
|
3959
|
+
if (result.runId) {
|
|
3960
|
+
runResults.push({ runId: result.runId, status: result.status });
|
|
3961
|
+
}
|
|
4099
3962
|
}
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4112
|
-
}
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
return path7.resolve(`agent-inspect-bundle-${label}-${stamp}`);
|
|
3963
|
+
runResults.sort((a, b) => a.runId.localeCompare(b.runId));
|
|
3964
|
+
findings.sort((a, b) => {
|
|
3965
|
+
const runCmp = (a.evidence[0]?.runId ?? "").localeCompare(
|
|
3966
|
+
b.evidence[0]?.runId ?? ""
|
|
3967
|
+
);
|
|
3968
|
+
if (runCmp !== 0) return runCmp;
|
|
3969
|
+
return a.ruleId.localeCompare(b.ruleId);
|
|
3970
|
+
});
|
|
3971
|
+
ruleExecutions.sort((a, b) => {
|
|
3972
|
+
const runCmp = (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
3973
|
+
if (runCmp !== 0) return runCmp;
|
|
3974
|
+
return a.ruleId.localeCompare(b.ruleId);
|
|
3975
|
+
});
|
|
3976
|
+
const hasErrors = diagnostics.some((item) => item.severity === "error");
|
|
3977
|
+
const status = hasErrors ? "error" : summary.failed > 0 ? "fail" : "pass";
|
|
3978
|
+
return {
|
|
3979
|
+
ok: status === "pass",
|
|
3980
|
+
status,
|
|
3981
|
+
format: perRun[0]?.format ?? "agent-inspect-jsonl",
|
|
3982
|
+
scopeKind: scope.scopeKind,
|
|
3983
|
+
scopeLabel: scope.scopeLabel,
|
|
3984
|
+
runIds: scope.runIds,
|
|
3985
|
+
runResults,
|
|
3986
|
+
summary,
|
|
3987
|
+
findings,
|
|
3988
|
+
diagnostics,
|
|
3989
|
+
ruleExecutions,
|
|
3990
|
+
...scope.sessionWarnings?.length ? { sessionWarnings: [...scope.sessionWarnings] } : {}
|
|
3991
|
+
};
|
|
4130
3992
|
}
|
|
4131
3993
|
|
|
4132
|
-
// packages/core/src/
|
|
4133
|
-
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
function sha256Hex(data) {
|
|
4139
|
-
const hash = createHash("sha256");
|
|
4140
|
-
if (typeof data === "string") {
|
|
4141
|
-
hash.update(data, "utf8");
|
|
4142
|
-
} else {
|
|
4143
|
-
hash.update(data);
|
|
4144
|
-
}
|
|
4145
|
-
return hash.digest("hex");
|
|
4146
|
-
}
|
|
4147
|
-
function isSha256Hex(value) {
|
|
4148
|
-
return SHA256_RE.test(value);
|
|
4149
|
-
}
|
|
4150
|
-
function sha256Equals(expected, actual) {
|
|
4151
|
-
if (!isSha256Hex(expected) || !isSha256Hex(actual)) {
|
|
4152
|
-
return false;
|
|
4153
|
-
}
|
|
4154
|
-
const a = expected.toLowerCase();
|
|
4155
|
-
const b = actual.toLowerCase();
|
|
4156
|
-
if (a.length !== b.length) return false;
|
|
4157
|
-
let mismatch = 0;
|
|
4158
|
-
for (let i = 0; i < a.length; i += 1) {
|
|
4159
|
-
mismatch |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
4160
|
-
}
|
|
4161
|
-
return mismatch === 0;
|
|
3994
|
+
// packages/core/src/sessions/index.ts
|
|
3995
|
+
function compareRuns(a, b) {
|
|
3996
|
+
const aStart = a.startedAt ?? 0;
|
|
3997
|
+
const bStart = b.startedAt ?? 0;
|
|
3998
|
+
if (aStart !== bStart) return aStart - bStart;
|
|
3999
|
+
return a.runId.localeCompare(b.runId);
|
|
4162
4000
|
}
|
|
4163
|
-
function
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
throw new Error(`Evidence file path must not contain "." or "..": ${relativePath}`);
|
|
4001
|
+
function buildGroups(runIds, metaByRunId) {
|
|
4002
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
4003
|
+
for (const runId of runIds) {
|
|
4004
|
+
const groupId = metaByRunId.get(runId)?.groupId;
|
|
4005
|
+
if (!groupId) continue;
|
|
4006
|
+
const existing = byGroup.get(groupId);
|
|
4007
|
+
if (existing) {
|
|
4008
|
+
existing.runIds.push(runId);
|
|
4009
|
+
} else {
|
|
4010
|
+
byGroup.set(groupId, {
|
|
4011
|
+
groupId,
|
|
4012
|
+
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
4013
|
+
runIds: [runId]
|
|
4014
|
+
});
|
|
4178
4015
|
}
|
|
4179
4016
|
}
|
|
4180
|
-
return
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
function stable(value) {
|
|
4185
|
-
if (Array.isArray(value)) return value.map(stable);
|
|
4186
|
-
if (value === null || typeof value !== "object") return value;
|
|
4187
|
-
const record = value;
|
|
4188
|
-
return Object.fromEntries(
|
|
4189
|
-
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable(record[key])])
|
|
4190
|
-
);
|
|
4191
|
-
}
|
|
4192
|
-
function serializeEvidenceManifest(manifest) {
|
|
4193
|
-
return `${JSON.stringify(stable(manifest), null, 2)}
|
|
4194
|
-
`;
|
|
4195
|
-
}
|
|
4196
|
-
function inferEvidenceFileRole(relativePath) {
|
|
4197
|
-
const normalized = assertEvidenceRelativePath(relativePath);
|
|
4198
|
-
const base = normalized.includes("/") ? normalized.slice(normalized.lastIndexOf("/") + 1) : normalized;
|
|
4199
|
-
if (base === "evidence.html" || base === "trace.html" || base.endsWith(".html")) {
|
|
4200
|
-
return "report";
|
|
4201
|
-
}
|
|
4202
|
-
if (base === "trace.jsonl" || base.endsWith(".jsonl")) {
|
|
4203
|
-
return "redacted-trace";
|
|
4204
|
-
}
|
|
4205
|
-
if (base === "check-results.json") {
|
|
4206
|
-
return "checks";
|
|
4207
|
-
}
|
|
4208
|
-
if (base === "contract.resolved.json") {
|
|
4209
|
-
return "contract";
|
|
4210
|
-
}
|
|
4211
|
-
if (base === "redaction-report.json") {
|
|
4212
|
-
return "redaction-report";
|
|
4213
|
-
}
|
|
4214
|
-
if (base === "summary.md") {
|
|
4215
|
-
return "summary";
|
|
4216
|
-
}
|
|
4217
|
-
return "other";
|
|
4017
|
+
return [...byGroup.values()].map((group) => ({
|
|
4018
|
+
...group,
|
|
4019
|
+
runIds: [...group.runIds].sort((a, b) => a.localeCompare(b))
|
|
4020
|
+
}));
|
|
4218
4021
|
}
|
|
4219
|
-
function
|
|
4220
|
-
const
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4022
|
+
function buildHandoffs(runIds, metaByRunId, warnings, sessionId) {
|
|
4023
|
+
const edges = [];
|
|
4024
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4025
|
+
const pushEdge = (edge) => {
|
|
4026
|
+
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
4027
|
+
if (seen.has(key)) return;
|
|
4028
|
+
seen.add(key);
|
|
4029
|
+
edges.push(edge);
|
|
4030
|
+
};
|
|
4031
|
+
for (const runId of runIds) {
|
|
4032
|
+
const meta = metaByRunId.get(runId);
|
|
4033
|
+
if (!meta) continue;
|
|
4034
|
+
if (meta.handoffFrom && meta.handoffTo) {
|
|
4035
|
+
pushEdge({
|
|
4036
|
+
from: meta.handoffFrom,
|
|
4037
|
+
to: meta.handoffTo,
|
|
4038
|
+
source: "manual",
|
|
4039
|
+
confidence: "explicit"
|
|
4040
|
+
});
|
|
4041
|
+
continue;
|
|
4227
4042
|
}
|
|
4228
|
-
if (
|
|
4229
|
-
|
|
4043
|
+
if (meta.handoffFrom) {
|
|
4044
|
+
pushEdge({
|
|
4045
|
+
from: meta.handoffFrom,
|
|
4046
|
+
to: runId,
|
|
4047
|
+
source: "manual",
|
|
4048
|
+
confidence: "explicit"
|
|
4049
|
+
});
|
|
4050
|
+
}
|
|
4051
|
+
if (meta.handoffTo) {
|
|
4052
|
+
pushEdge({
|
|
4053
|
+
from: runId,
|
|
4054
|
+
to: meta.handoffTo,
|
|
4055
|
+
source: "manual",
|
|
4056
|
+
confidence: "explicit"
|
|
4057
|
+
});
|
|
4058
|
+
}
|
|
4059
|
+
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
4060
|
+
pushEdge({
|
|
4061
|
+
from: meta.parentGroupId,
|
|
4062
|
+
to: meta.subAgentId,
|
|
4063
|
+
source: "inferred",
|
|
4064
|
+
confidence: "correlated"
|
|
4065
|
+
});
|
|
4066
|
+
warnings.push({
|
|
4067
|
+
code: "ambiguous-handoff-endpoints",
|
|
4068
|
+
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
4069
|
+
runId,
|
|
4070
|
+
sessionId
|
|
4071
|
+
});
|
|
4230
4072
|
}
|
|
4231
|
-
byPath.set(relativePath, {
|
|
4232
|
-
path: relativePath,
|
|
4233
|
-
sha256: sha256Hex(file.content),
|
|
4234
|
-
role: file.role ?? inferEvidenceFileRole(relativePath)
|
|
4235
|
-
});
|
|
4236
4073
|
}
|
|
4237
|
-
return
|
|
4074
|
+
return edges.sort((a, b) => {
|
|
4075
|
+
const from = a.from.localeCompare(b.from);
|
|
4076
|
+
if (from !== 0) return from;
|
|
4077
|
+
return a.to.localeCompare(b.to);
|
|
4078
|
+
});
|
|
4238
4079
|
}
|
|
4239
|
-
function
|
|
4240
|
-
const
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4080
|
+
function buildRetries(runIds, metaByRunId, warnings, sessionId) {
|
|
4081
|
+
const retries = [];
|
|
4082
|
+
for (const runId of runIds) {
|
|
4083
|
+
const meta = metaByRunId.get(runId);
|
|
4084
|
+
if (!meta) continue;
|
|
4085
|
+
if (meta.retryOf) {
|
|
4086
|
+
retries.push({
|
|
4087
|
+
runId,
|
|
4088
|
+
retryOf: meta.retryOf,
|
|
4089
|
+
attempt: meta.attempt,
|
|
4090
|
+
source: "manual",
|
|
4091
|
+
confidence: "explicit"
|
|
4092
|
+
});
|
|
4093
|
+
continue;
|
|
4247
4094
|
}
|
|
4248
|
-
if (
|
|
4249
|
-
|
|
4095
|
+
if (meta.attempt !== void 0 && meta.attempt > 1) {
|
|
4096
|
+
retries.push({
|
|
4097
|
+
runId,
|
|
4098
|
+
attempt: meta.attempt,
|
|
4099
|
+
source: "inferred",
|
|
4100
|
+
confidence: "correlated"
|
|
4101
|
+
});
|
|
4102
|
+
warnings.push({
|
|
4103
|
+
code: "ambiguous-retry-link",
|
|
4104
|
+
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
4105
|
+
runId,
|
|
4106
|
+
sessionId
|
|
4107
|
+
});
|
|
4250
4108
|
}
|
|
4251
4109
|
}
|
|
4252
|
-
|
|
4253
|
-
status: parts.assessmentStatus,
|
|
4254
|
-
note: parts.note ?? EVIDENCE_ASSESSMENT_NOTE
|
|
4255
|
-
};
|
|
4256
|
-
if (parts.sourceStatus !== void 0) {
|
|
4257
|
-
assessment.sourceStatus = parts.sourceStatus;
|
|
4258
|
-
}
|
|
4259
|
-
return {
|
|
4260
|
-
evidenceFormatVersion: EVIDENCE_FORMAT_VERSION,
|
|
4261
|
-
generator: {
|
|
4262
|
-
name: parts.generatorName ?? "agent-inspect",
|
|
4263
|
-
version: parts.generatorVersion
|
|
4264
|
-
},
|
|
4265
|
-
createdAt: parts.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
4266
|
-
source: {
|
|
4267
|
-
runIds,
|
|
4268
|
-
traceSchemaVersions: [...parts.traceSchemaVersions].sort((a, b) => a.localeCompare(b)),
|
|
4269
|
-
sourceHashes: [...parts.sourceHashes].sort((a, b) => a.runId.localeCompare(b.runId))
|
|
4270
|
-
},
|
|
4271
|
-
policy: {
|
|
4272
|
-
redactionProfile: parts.redactionProfile,
|
|
4273
|
-
verificationPolicy: parts.verificationPolicy ?? parts.redactionProfile
|
|
4274
|
-
},
|
|
4275
|
-
assessment,
|
|
4276
|
-
...parts.semantics !== void 0 ? { semantics: { ...parts.semantics } } : {},
|
|
4277
|
-
...parts.contract !== void 0 ? { contract: { ...parts.contract } } : {},
|
|
4278
|
-
files: buildEvidenceFileEntries(parts.files)
|
|
4279
|
-
};
|
|
4110
|
+
return retries.sort((a, b) => a.runId.localeCompare(b.runId));
|
|
4280
4111
|
}
|
|
4281
|
-
function
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4112
|
+
function buildCriticalPath(runs, handoffs) {
|
|
4113
|
+
const runById = new Map(runs.map((run) => [run.runId, run]));
|
|
4114
|
+
const explicitTargets = new Set(
|
|
4115
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.to)
|
|
4116
|
+
);
|
|
4117
|
+
const explicitSources = new Set(
|
|
4118
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
4119
|
+
);
|
|
4120
|
+
const ordered = [...runs].sort(compareRuns);
|
|
4121
|
+
const path14 = [];
|
|
4122
|
+
const visited = /* @__PURE__ */ new Set();
|
|
4123
|
+
const pushRun = (run, confidence, source) => {
|
|
4124
|
+
if (visited.has(run.runId)) return;
|
|
4125
|
+
visited.add(run.runId);
|
|
4126
|
+
path14.push({
|
|
4127
|
+
runId: run.runId,
|
|
4128
|
+
name: run.name,
|
|
4129
|
+
startedAt: run.startedAt,
|
|
4130
|
+
durationMs: run.durationMs,
|
|
4131
|
+
confidence,
|
|
4132
|
+
source
|
|
4133
|
+
});
|
|
4134
|
+
};
|
|
4135
|
+
for (const edge of handoffs) {
|
|
4136
|
+
if (edge.confidence !== "explicit") continue;
|
|
4137
|
+
const fromRun = [...runById.values()].find(
|
|
4138
|
+
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
4289
4139
|
);
|
|
4140
|
+
const toRun = [...runById.values()].find(
|
|
4141
|
+
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
4142
|
+
);
|
|
4143
|
+
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
4144
|
+
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
4290
4145
|
}
|
|
4291
|
-
const
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
const source = record.source;
|
|
4296
|
-
if (source === null || typeof source !== "object" || Array.isArray(source)) {
|
|
4297
|
-
throw new Error("Evidence manifest requires source.");
|
|
4298
|
-
}
|
|
4299
|
-
const sourceRecord = source;
|
|
4300
|
-
if (!Array.isArray(sourceRecord.runIds) || sourceRecord.runIds.length === 0) {
|
|
4301
|
-
throw new Error("Evidence manifest source.runIds must be a non-empty array.");
|
|
4302
|
-
}
|
|
4303
|
-
if (!Array.isArray(sourceRecord.traceSchemaVersions)) {
|
|
4304
|
-
throw new Error("Evidence manifest source.traceSchemaVersions must be an array.");
|
|
4305
|
-
}
|
|
4306
|
-
if (!Array.isArray(sourceRecord.sourceHashes)) {
|
|
4307
|
-
throw new Error("Evidence manifest source.sourceHashes must be an array.");
|
|
4308
|
-
}
|
|
4309
|
-
const policy = record.policy;
|
|
4310
|
-
if (policy === null || typeof policy !== "object" || Array.isArray(policy)) {
|
|
4311
|
-
throw new Error("Evidence manifest requires policy.");
|
|
4146
|
+
for (const run of ordered) {
|
|
4147
|
+
if (visited.has(run.runId)) continue;
|
|
4148
|
+
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
4149
|
+
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
4312
4150
|
}
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4151
|
+
return path14;
|
|
4152
|
+
}
|
|
4153
|
+
function metaRunIdMatches(run, token, runById) {
|
|
4154
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
4155
|
+
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
4156
|
+
}
|
|
4157
|
+
function buildSessionIndex(inputRuns, options = {}) {
|
|
4158
|
+
const warnings = [];
|
|
4159
|
+
const runs = [...inputRuns].sort(compareRuns);
|
|
4160
|
+
const metaByRunId = /* @__PURE__ */ new Map();
|
|
4161
|
+
for (const run of runs) {
|
|
4162
|
+
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
4316
4163
|
}
|
|
4317
|
-
|
|
4318
|
-
|
|
4164
|
+
const sessionsByKey = /* @__PURE__ */ new Map();
|
|
4165
|
+
const unscopedRunIds = [];
|
|
4166
|
+
for (const run of runs) {
|
|
4167
|
+
const meta = metaByRunId.get(run.runId);
|
|
4168
|
+
const key = sessionKeyForRun(meta, {
|
|
4169
|
+
correlateByGroupId: options.correlateByGroupId === true
|
|
4170
|
+
});
|
|
4171
|
+
if (!key) {
|
|
4172
|
+
unscopedRunIds.push(run.runId);
|
|
4173
|
+
continue;
|
|
4174
|
+
}
|
|
4175
|
+
const bucket = sessionsByKey.get(key) ?? [];
|
|
4176
|
+
bucket.push(run);
|
|
4177
|
+
sessionsByKey.set(key, bucket);
|
|
4319
4178
|
}
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4179
|
+
const sessions = [...sessionsByKey.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([sessionId, sessionRuns]) => {
|
|
4180
|
+
const runIds = sessionRuns.map((run) => run.runId).sort();
|
|
4181
|
+
const handoffs = buildHandoffs(runIds, metaByRunId, warnings, sessionId);
|
|
4182
|
+
const retries = buildRetries(runIds, metaByRunId, warnings, sessionId);
|
|
4183
|
+
const groups = buildGroups(runIds, metaByRunId);
|
|
4184
|
+
const criticalPath = buildCriticalPath(sessionRuns, handoffs);
|
|
4185
|
+
const confidences = new Set(handoffs.map((edge) => edge.confidence));
|
|
4186
|
+
if (confidences.has("explicit") && confidences.has("correlated")) {
|
|
4187
|
+
warnings.push({
|
|
4188
|
+
code: "mixed-confidence-group",
|
|
4189
|
+
message: "Session aggregates explicit and correlated handoff edges.",
|
|
4190
|
+
sessionId
|
|
4191
|
+
});
|
|
4323
4192
|
}
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4193
|
+
return enrichSessionSummary(
|
|
4194
|
+
{
|
|
4195
|
+
sessionId,
|
|
4196
|
+
runIds,
|
|
4197
|
+
groups,
|
|
4198
|
+
handoffs,
|
|
4199
|
+
retries,
|
|
4200
|
+
criticalPath
|
|
4201
|
+
},
|
|
4202
|
+
runs,
|
|
4203
|
+
{
|
|
4204
|
+
nowMs: options.nowMs,
|
|
4205
|
+
staleThresholdMs: options.staleThresholdMs
|
|
4206
|
+
}
|
|
4207
|
+
);
|
|
4208
|
+
});
|
|
4209
|
+
if (sessions.length === 0 && runs.length > 0) {
|
|
4210
|
+
warnings.push({
|
|
4211
|
+
code: "missing-session-id",
|
|
4212
|
+
message: "No sessionId (or correlated groupId) found on input runs."
|
|
4213
|
+
});
|
|
4214
|
+
}
|
|
4215
|
+
warnings.sort((a, b) => {
|
|
4216
|
+
const code = a.code.localeCompare(b.code);
|
|
4217
|
+
if (code !== 0) return code;
|
|
4218
|
+
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
4219
|
+
});
|
|
4220
|
+
return {
|
|
4221
|
+
runs,
|
|
4222
|
+
sessions,
|
|
4223
|
+
unscopedRunIds: unscopedRunIds.sort(),
|
|
4224
|
+
warnings
|
|
4225
|
+
};
|
|
4226
|
+
}
|
|
4227
|
+
|
|
4228
|
+
// packages/core/src/sessions/scope.ts
|
|
4229
|
+
function filterMetasBySessionScope(metas, records, options) {
|
|
4230
|
+
const sessionId = options.sessionId?.trim();
|
|
4231
|
+
const groupId = options.groupId?.trim();
|
|
4232
|
+
const warnings = [];
|
|
4233
|
+
if (sessionId) {
|
|
4234
|
+
const index = buildSessionIndex(records, {
|
|
4235
|
+
correlateByGroupId: options.correlateByGroupId === true
|
|
4236
|
+
});
|
|
4237
|
+
warnings.push(...index.warnings);
|
|
4238
|
+
const session = index.sessions.find((item) => item.sessionId === sessionId);
|
|
4239
|
+
if (!session) {
|
|
4240
|
+
return {
|
|
4241
|
+
metas: [],
|
|
4242
|
+
scopeLabel: sessionId,
|
|
4243
|
+
scopeKind: "session",
|
|
4244
|
+
runIds: [],
|
|
4245
|
+
warnings,
|
|
4246
|
+
notFound: true
|
|
4247
|
+
};
|
|
4327
4248
|
}
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4249
|
+
const runIdSet = new Set(session.runIds);
|
|
4250
|
+
const filtered = metas.filter((meta) => runIdSet.has(meta.runId));
|
|
4251
|
+
return {
|
|
4252
|
+
metas: filtered,
|
|
4253
|
+
scopeLabel: sessionId,
|
|
4254
|
+
scopeKind: "session",
|
|
4255
|
+
runIds: session.runIds,
|
|
4256
|
+
warnings,
|
|
4257
|
+
notFound: false
|
|
4258
|
+
};
|
|
4259
|
+
}
|
|
4260
|
+
if (groupId) {
|
|
4261
|
+
const runIds = records.filter((run) => extractSessionWorkflowMetadata(run.metadata)?.groupId === groupId).map((run) => run.runId).sort();
|
|
4262
|
+
if (runIds.length === 0) {
|
|
4263
|
+
return {
|
|
4264
|
+
metas: [],
|
|
4265
|
+
scopeLabel: groupId,
|
|
4266
|
+
scopeKind: "group",
|
|
4267
|
+
runIds: [],
|
|
4268
|
+
warnings,
|
|
4269
|
+
notFound: true
|
|
4270
|
+
};
|
|
4331
4271
|
}
|
|
4272
|
+
const runIdSet = new Set(runIds);
|
|
4273
|
+
return {
|
|
4274
|
+
metas: metas.filter((meta) => runIdSet.has(meta.runId)),
|
|
4275
|
+
scopeLabel: groupId,
|
|
4276
|
+
scopeKind: "group",
|
|
4277
|
+
runIds,
|
|
4278
|
+
warnings,
|
|
4279
|
+
notFound: false
|
|
4280
|
+
};
|
|
4332
4281
|
}
|
|
4333
|
-
return
|
|
4282
|
+
return {
|
|
4283
|
+
metas: [...metas],
|
|
4284
|
+
scopeLabel: "",
|
|
4285
|
+
scopeKind: "session",
|
|
4286
|
+
runIds: [],
|
|
4287
|
+
warnings,
|
|
4288
|
+
notFound: false
|
|
4289
|
+
};
|
|
4334
4290
|
}
|
|
4335
|
-
|
|
4336
|
-
|
|
4291
|
+
var KNOWN_EVENTS = /* @__PURE__ */ new Set([
|
|
4292
|
+
"run_started",
|
|
4293
|
+
"run_completed",
|
|
4294
|
+
"step_started",
|
|
4295
|
+
"step_completed"
|
|
4296
|
+
]);
|
|
4297
|
+
function isRecord7(value) {
|
|
4298
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4299
|
+
}
|
|
4300
|
+
function safeParse(line) {
|
|
4337
4301
|
try {
|
|
4338
|
-
|
|
4339
|
-
} catch
|
|
4340
|
-
|
|
4341
|
-
throw new Error(`Evidence manifest is not valid JSON: ${message}`);
|
|
4302
|
+
return JSON.parse(line);
|
|
4303
|
+
} catch {
|
|
4304
|
+
return void 0;
|
|
4342
4305
|
}
|
|
4343
|
-
return validateEvidenceManifest(parsed);
|
|
4344
4306
|
}
|
|
4345
|
-
function
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4307
|
+
async function isAgentInspectTrace(filePath) {
|
|
4308
|
+
try {
|
|
4309
|
+
const rl = createInterface({
|
|
4310
|
+
input: createReadStream(filePath, { encoding: "utf8" }),
|
|
4311
|
+
crlfDelay: Infinity
|
|
4312
|
+
});
|
|
4313
|
+
let checked = 0;
|
|
4314
|
+
for await (const line of rl) {
|
|
4315
|
+
const trimmed = line.trim();
|
|
4316
|
+
if (trimmed === "") continue;
|
|
4317
|
+
const parsed = safeParse(trimmed);
|
|
4318
|
+
if (!parsed) continue;
|
|
4319
|
+
if (!isRecord7(parsed)) continue;
|
|
4320
|
+
checked += 1;
|
|
4321
|
+
if (isTraceEvent(parsed)) return true;
|
|
4322
|
+
const ev = parsed.event;
|
|
4323
|
+
const runId = parsed.runId;
|
|
4324
|
+
if (typeof ev === "string" && KNOWN_EVENTS.has(ev) && typeof runId === "string") {
|
|
4325
|
+
return true;
|
|
4326
|
+
}
|
|
4327
|
+
if (checked >= 20) break;
|
|
4328
|
+
}
|
|
4329
|
+
return false;
|
|
4330
|
+
} catch {
|
|
4331
|
+
return false;
|
|
4357
4332
|
}
|
|
4358
|
-
return [...versions].sort((a, b) => a.localeCompare(b));
|
|
4359
4333
|
}
|
|
4360
4334
|
|
|
4361
|
-
// packages/core/src/
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
remediation: "For a file path, use openTraceFile(path).",
|
|
4367
|
-
relatedCodes: ["invalid_input"]
|
|
4368
|
-
},
|
|
4369
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
4370
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
4371
|
-
summary: "No trace reader could detect the input format.",
|
|
4372
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
4373
|
-
relatedCodes: ["unsupported_format"]
|
|
4374
|
-
},
|
|
4375
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
4376
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
4377
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
4378
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
4379
|
-
relatedCodes: ["ambiguous_format"]
|
|
4380
|
-
},
|
|
4381
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
4382
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
4383
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
4384
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
4385
|
-
},
|
|
4386
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
4387
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
4388
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
4389
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
4390
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
4391
|
-
},
|
|
4392
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
4393
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
4394
|
-
summary: "A parentId equals its own event/step id.",
|
|
4395
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
4396
|
-
relatedCodes: [
|
|
4397
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
4398
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
4399
|
-
]
|
|
4400
|
-
},
|
|
4401
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
4402
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
4403
|
-
summary: "Trace contains a parentId cycle.",
|
|
4404
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
4405
|
-
relatedCodes: ["structure.cycle"]
|
|
4406
|
-
},
|
|
4407
|
-
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
4408
|
-
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
4409
|
-
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
4410
|
-
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
4411
|
-
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
4412
|
-
},
|
|
4413
|
-
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
4414
|
-
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
4415
|
-
summary: 'inspectRun does not accept a "writer" option.',
|
|
4416
|
-
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
4335
|
+
// packages/core/src/bundle/resolve.ts
|
|
4336
|
+
function parseSinceCutoff(since) {
|
|
4337
|
+
const trimmed = since.trim();
|
|
4338
|
+
if (trimmed === "") {
|
|
4339
|
+
throw new Error("--since requires a non-empty duration (e.g. 24h, 7d).");
|
|
4417
4340
|
}
|
|
4418
|
-
|
|
4419
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
4420
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
4421
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
4422
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
4423
|
-
}
|
|
4424
|
-
|
|
4425
|
-
// packages/core/src/checks/logical-events.ts
|
|
4426
|
-
function isRecord7(value) {
|
|
4427
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4428
|
-
}
|
|
4429
|
-
function legacyEvent(event) {
|
|
4430
|
-
const value = event.attributes?.legacyEvent;
|
|
4431
|
-
return typeof value === "string" ? value : void 0;
|
|
4341
|
+
return Date.now() - parseDuration(trimmed);
|
|
4432
4342
|
}
|
|
4433
|
-
function
|
|
4434
|
-
|
|
4435
|
-
if (
|
|
4343
|
+
function runActivityMs(run) {
|
|
4344
|
+
if (run.startedAt !== void 0 && Number.isFinite(run.startedAt)) return run.startedAt;
|
|
4345
|
+
if (run.endedAt !== void 0 && Number.isFinite(run.endedAt)) return run.endedAt;
|
|
4436
4346
|
return void 0;
|
|
4437
4347
|
}
|
|
4438
|
-
function
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4348
|
+
function runsInSinceWindow(runs, since) {
|
|
4349
|
+
const cutoff = parseSinceCutoff(since);
|
|
4350
|
+
const ids = [];
|
|
4351
|
+
for (const run of runs) {
|
|
4352
|
+
const activity = runActivityMs(run);
|
|
4353
|
+
if (activity !== void 0 && activity >= cutoff) {
|
|
4354
|
+
ids.push(run.runId);
|
|
4355
|
+
}
|
|
4356
|
+
}
|
|
4357
|
+
return ids.sort((a, b) => a.localeCompare(b));
|
|
4446
4358
|
}
|
|
4447
|
-
function
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4359
|
+
function findSession(index, sessionId) {
|
|
4360
|
+
return index.sessions.find((session) => session.sessionId === sessionId);
|
|
4361
|
+
}
|
|
4362
|
+
function resolveBundleRunIds(index, runs, options) {
|
|
4363
|
+
const runId = options.runId?.trim();
|
|
4364
|
+
const sessionId = options.sessionId?.trim();
|
|
4365
|
+
const since = options.since?.trim();
|
|
4366
|
+
const modes = [runId ? 1 : 0, sessionId ? 1 : 0, since ? 1 : 0].reduce((a, b) => a + b, 0);
|
|
4367
|
+
if (modes === 0) {
|
|
4368
|
+
throw new Error(
|
|
4369
|
+
"bundle requires a run id, --session <sessionId>, or --since <duration>."
|
|
4370
|
+
);
|
|
4456
4371
|
}
|
|
4457
|
-
|
|
4372
|
+
if (modes > 1) {
|
|
4373
|
+
throw new Error(
|
|
4374
|
+
"bundle accepts only one target: a run id, --session, or --since (not combined)."
|
|
4375
|
+
);
|
|
4376
|
+
}
|
|
4377
|
+
if (runId) {
|
|
4378
|
+
const known = runs.some((run) => run.runId === runId);
|
|
4379
|
+
if (!known) {
|
|
4380
|
+
throw new Error(`Run "${runId}" was not found in the trace directory.`);
|
|
4381
|
+
}
|
|
4382
|
+
return { runIds: [runId] };
|
|
4383
|
+
}
|
|
4384
|
+
if (sessionId) {
|
|
4385
|
+
const session = findSession(index, sessionId);
|
|
4386
|
+
if (!session) {
|
|
4387
|
+
throw new Error(`Session "${sessionId}" was not found.`);
|
|
4388
|
+
}
|
|
4389
|
+
if (session.runIds.length === 0) {
|
|
4390
|
+
throw new Error(`Session "${sessionId}" has no runs to bundle.`);
|
|
4391
|
+
}
|
|
4392
|
+
return {
|
|
4393
|
+
runIds: [...session.runIds].sort((a, b) => a.localeCompare(b)),
|
|
4394
|
+
sessionId
|
|
4395
|
+
};
|
|
4396
|
+
}
|
|
4397
|
+
const runIds = runsInSinceWindow(runs, since);
|
|
4398
|
+
if (runIds.length === 0) {
|
|
4399
|
+
throw new Error(`No runs matched --since ${since}.`);
|
|
4400
|
+
}
|
|
4401
|
+
return { runIds, since };
|
|
4458
4402
|
}
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4403
|
+
|
|
4404
|
+
// packages/core/src/bundle/safety-status.ts
|
|
4405
|
+
function aggregateBundleSafeStatus(statuses) {
|
|
4406
|
+
if (statuses.length === 0) return "UNKNOWN";
|
|
4407
|
+
if (statuses.some((status) => status === "UNSAFE")) return "UNSAFE";
|
|
4408
|
+
if (statuses.some((status) => status === "UNKNOWN")) return "UNKNOWN";
|
|
4409
|
+
if (statuses.some((status) => status === "SAFE WITH WARNINGS")) return "SAFE WITH WARNINGS";
|
|
4410
|
+
return "SAFE";
|
|
4411
|
+
}
|
|
4412
|
+
function toMetadataSafeStatus(status) {
|
|
4413
|
+
if (status === "SAFE WITH WARNINGS") return "SAFE_WITH_WARNINGS";
|
|
4414
|
+
return status;
|
|
4415
|
+
}
|
|
4416
|
+
function bundleFailsOnSafety(status, allowUnsafe) {
|
|
4417
|
+
if (allowUnsafe) return false;
|
|
4418
|
+
return status === "UNSAFE" || status === "UNKNOWN";
|
|
4419
|
+
}
|
|
4420
|
+
|
|
4421
|
+
// packages/core/src/bundle/manifest.ts
|
|
4422
|
+
var BUNDLE_NOTE = "Generated locally by AgentInspect. Bundles are derived copies for review \u2014 not compliance or security certification. Review before sharing.";
|
|
4423
|
+
var PLACEHOLDER_NOTE = "No eval or performance artifacts were requested for this bundle.";
|
|
4424
|
+
function buildBundleMetadata(parts) {
|
|
4425
|
+
const aggregate = aggregateBundleSafeStatus(
|
|
4426
|
+
parts.checks.runs.map((run) => run.status)
|
|
4427
|
+
);
|
|
4471
4428
|
return {
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4429
|
+
createdAt: parts.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
4430
|
+
agentInspectVersion: parts.agentInspectVersion,
|
|
4431
|
+
redactionProfile: parts.profile,
|
|
4432
|
+
sourceTraceCount: parts.resolve.runIds.length,
|
|
4433
|
+
runIds: [...parts.resolve.runIds],
|
|
4434
|
+
safeStatus: toMetadataSafeStatus(aggregate),
|
|
4435
|
+
files: [...parts.files].sort((a, b) => a.localeCompare(b)),
|
|
4436
|
+
note: BUNDLE_NOTE,
|
|
4437
|
+
...parts.resolve.sessionId !== void 0 ? { sessionId: parts.resolve.sessionId } : {},
|
|
4438
|
+
...parts.resolve.since !== void 0 ? { since: parts.resolve.since } : {}
|
|
4480
4439
|
};
|
|
4481
4440
|
}
|
|
4482
|
-
function
|
|
4441
|
+
function buildPlaceholderArtifact() {
|
|
4483
4442
|
return {
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
projection: {
|
|
4487
|
-
paired: false,
|
|
4488
|
-
absorbedEventIds: Object.freeze([]),
|
|
4489
|
-
parentNormalized: extras?.parentNormalized === true,
|
|
4490
|
-
...{}
|
|
4491
|
-
}
|
|
4443
|
+
status: "not_requested",
|
|
4444
|
+
note: PLACEHOLDER_NOTE
|
|
4492
4445
|
};
|
|
4493
4446
|
}
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
byRun.set(event.runId, list);
|
|
4447
|
+
|
|
4448
|
+
// packages/core/src/bundle/summary.ts
|
|
4449
|
+
function markdownTable(rows) {
|
|
4450
|
+
const lines = ["| Field | Value |", "| --- | --- |"];
|
|
4451
|
+
for (const [key, value] of rows) {
|
|
4452
|
+
lines.push(`| ${key} | ${value ?? "unknown"} |`);
|
|
4501
4453
|
}
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
const
|
|
4506
|
-
const
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
|
|
4529
|
-
|
|
4530
|
-
|
|
4531
|
-
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
4535
|
-
});
|
|
4536
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
4537
|
-
}
|
|
4538
|
-
match = candidates[0];
|
|
4539
|
-
} else if (stepId) {
|
|
4540
|
-
const candidates = completes.filter(
|
|
4541
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
4542
|
-
);
|
|
4543
|
-
if (candidates.length > 1) {
|
|
4544
|
-
diagnostics.push({
|
|
4545
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
4546
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
4547
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
4548
|
-
});
|
|
4549
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
4550
|
-
}
|
|
4551
|
-
match = candidates[0];
|
|
4552
|
-
}
|
|
4553
|
-
if (match) {
|
|
4554
|
-
usedCompletes.add(match.eventId);
|
|
4555
|
-
absorbedIds.add(match.eventId);
|
|
4556
|
-
const paired = pairStartComplete(start, match);
|
|
4557
|
-
logical.push(paired);
|
|
4558
|
-
logicalByRawId.set(start.eventId, paired);
|
|
4559
|
-
logicalByRawId.set(match.eventId, paired);
|
|
4560
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
4561
|
-
} else {
|
|
4562
|
-
diagnostics.push({
|
|
4563
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
4564
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
4565
|
-
eventIds: [start.eventId]
|
|
4566
|
-
});
|
|
4567
|
-
const alone = asLogical(start);
|
|
4568
|
-
logical.push(alone);
|
|
4569
|
-
logicalByRawId.set(start.eventId, alone);
|
|
4570
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
4571
|
-
}
|
|
4572
|
-
}
|
|
4573
|
-
for (const complete of completes) {
|
|
4574
|
-
if (usedCompletes.has(complete.eventId)) continue;
|
|
4575
|
-
diagnostics.push({
|
|
4576
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
4577
|
-
message: `No matching start for complete ${complete.eventId}.`,
|
|
4578
|
-
eventIds: [complete.eventId]
|
|
4579
|
-
});
|
|
4580
|
-
const alone = asLogical(complete);
|
|
4581
|
-
logical.push(alone);
|
|
4582
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
4583
|
-
const stepId = stepIdOf(complete);
|
|
4584
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
4585
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
4586
|
-
}
|
|
4587
|
-
}
|
|
4588
|
-
for (const event of others) {
|
|
4589
|
-
const alone = asLogical(event);
|
|
4590
|
-
logical.push(alone);
|
|
4591
|
-
logicalByRawId.set(event.eventId, alone);
|
|
4592
|
-
const stepId = stepIdOf(event);
|
|
4593
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
4594
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
4595
|
-
}
|
|
4596
|
-
}
|
|
4454
|
+
return lines.join("\n");
|
|
4455
|
+
}
|
|
4456
|
+
function buildBundleSummaryMarkdown(parts) {
|
|
4457
|
+
const { metadata, checks, redaction } = parts;
|
|
4458
|
+
const lines = [
|
|
4459
|
+
"# AgentInspect trace bundle",
|
|
4460
|
+
"",
|
|
4461
|
+
metadata.note,
|
|
4462
|
+
"",
|
|
4463
|
+
"## Overview",
|
|
4464
|
+
"",
|
|
4465
|
+
markdownTable([
|
|
4466
|
+
["Created", metadata.createdAt],
|
|
4467
|
+
["AgentInspect", metadata.agentInspectVersion],
|
|
4468
|
+
["Redaction profile", metadata.redactionProfile],
|
|
4469
|
+
["Safe status", metadata.safeStatus],
|
|
4470
|
+
["Source traces", metadata.sourceTraceCount],
|
|
4471
|
+
["Runs", metadata.runIds.join(", ")],
|
|
4472
|
+
...metadata.sessionId ? [["Session", metadata.sessionId]] : [],
|
|
4473
|
+
...metadata.since ? [["Since", metadata.since]] : []
|
|
4474
|
+
]),
|
|
4475
|
+
"",
|
|
4476
|
+
"## Safety checks",
|
|
4477
|
+
"",
|
|
4478
|
+
`Aggregate: **${checks.aggregateStatus}**`,
|
|
4479
|
+
""
|
|
4480
|
+
];
|
|
4481
|
+
for (const run of checks.runs) {
|
|
4482
|
+
const source = run.sourceStatus !== void 0 && run.sourceStatus !== run.status ? `; source ${run.sourceStatus}` : "";
|
|
4483
|
+
lines.push(
|
|
4484
|
+
`- \`${run.runId}\`: artifact ${run.status}${source} (${run.findings} finding(s), ${run.errors} error(s), ${run.warnings} warning(s))`
|
|
4485
|
+
);
|
|
4597
4486
|
}
|
|
4598
|
-
|
|
4599
|
-
const
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4656
|
-
|
|
4657
|
-
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
|
|
4663
|
-
|
|
4664
|
-
|
|
4665
|
-
|
|
4666
|
-
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
|
|
4487
|
+
lines.push("", "## Redaction", "", `Total findings: ${redaction.totalFindings}`, "");
|
|
4488
|
+
for (const run of redaction.runs) {
|
|
4489
|
+
const detectors = run.detectors.length > 0 ? run.detectors.join(", ") : "none";
|
|
4490
|
+
lines.push(`- \`${run.runId}\`: ${run.findings} finding(s); detectors: ${detectors}`);
|
|
4491
|
+
}
|
|
4492
|
+
lines.push(
|
|
4493
|
+
"",
|
|
4494
|
+
"## Files",
|
|
4495
|
+
"",
|
|
4496
|
+
...metadata.files.map((file) => `- \`${file}\``),
|
|
4497
|
+
"",
|
|
4498
|
+
"_Review every generated artifact before sharing outside your team._",
|
|
4499
|
+
""
|
|
4500
|
+
);
|
|
4501
|
+
return lines.join("\n");
|
|
4502
|
+
}
|
|
4503
|
+
function sanitizeBundleRunId(runId) {
|
|
4504
|
+
let safe = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
4505
|
+
safe = path7.basename(safe);
|
|
4506
|
+
safe = safe.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
|
4507
|
+
if (safe === "" || safe === "." || safe === "..") {
|
|
4508
|
+
safe = "run_unknown";
|
|
4509
|
+
}
|
|
4510
|
+
return safe;
|
|
4511
|
+
}
|
|
4512
|
+
function bundleRunAssetRelativePath(runId, extension) {
|
|
4513
|
+
const safe = sanitizeBundleRunId(runId);
|
|
4514
|
+
const ext = extension.startsWith(".") ? extension : `.${extension}`;
|
|
4515
|
+
return path7.posix.join("assets", "runs", `${safe}${ext}`);
|
|
4516
|
+
}
|
|
4517
|
+
function assertBundlePathContained(outputDir, relativePath) {
|
|
4518
|
+
const base = path7.resolve(outputDir);
|
|
4519
|
+
const resolved = path7.resolve(base, relativePath);
|
|
4520
|
+
if (resolved !== base && !resolved.startsWith(base + path7.sep)) {
|
|
4521
|
+
throw new Error(`Bundle path escapes output directory: ${relativePath}`);
|
|
4522
|
+
}
|
|
4523
|
+
return resolved;
|
|
4524
|
+
}
|
|
4525
|
+
function normalizeBundleOutputPath(out, options) {
|
|
4526
|
+
const trimmed = out.trim();
|
|
4527
|
+
if (trimmed === "") {
|
|
4528
|
+
throw new Error("--out requires a non-empty path.");
|
|
4529
|
+
}
|
|
4530
|
+
const resolved = path7.resolve(trimmed);
|
|
4531
|
+
if (options?.preserveZipExtension !== true && resolved.toLowerCase().endsWith(".zip")) {
|
|
4532
|
+
return resolved.slice(0, -4);
|
|
4533
|
+
}
|
|
4534
|
+
return resolved;
|
|
4535
|
+
}
|
|
4536
|
+
function defaultBundleOutputPath(runIds) {
|
|
4537
|
+
const label = runIds.length === 1 ? sanitizeBundleRunId(runIds[0]) : `multi-${runIds.length}`;
|
|
4538
|
+
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
4539
|
+
return path7.resolve(`agent-inspect-bundle-${label}-${stamp}`);
|
|
4540
|
+
}
|
|
4541
|
+
|
|
4542
|
+
// packages/core/src/evidence/types.ts
|
|
4543
|
+
var EVIDENCE_FORMAT_VERSION = "1.0";
|
|
4544
|
+
var EVIDENCE_ASSESSMENT_NOTE = "Best-effort local safety verification only; not a compliance certification.";
|
|
4545
|
+
var EVIDENCE_MANIFEST_FILENAME = "evidence.json";
|
|
4546
|
+
var EVIDENCE_RESOLVED_CONTRACT_FILENAME = "contract.resolved.json";
|
|
4547
|
+
var SHA256_RE = /^[a-f0-9]{64}$/i;
|
|
4548
|
+
function sha256Hex(data) {
|
|
4549
|
+
const hash = createHash("sha256");
|
|
4550
|
+
if (typeof data === "string") {
|
|
4551
|
+
hash.update(data, "utf8");
|
|
4552
|
+
} else {
|
|
4553
|
+
hash.update(data);
|
|
4554
|
+
}
|
|
4555
|
+
return hash.digest("hex");
|
|
4556
|
+
}
|
|
4557
|
+
function isSha256Hex(value) {
|
|
4558
|
+
return SHA256_RE.test(value);
|
|
4559
|
+
}
|
|
4560
|
+
function sha256Equals(expected, actual) {
|
|
4561
|
+
if (!isSha256Hex(expected) || !isSha256Hex(actual)) {
|
|
4562
|
+
return false;
|
|
4563
|
+
}
|
|
4564
|
+
const a = expected.toLowerCase();
|
|
4565
|
+
const b = actual.toLowerCase();
|
|
4566
|
+
if (a.length !== b.length) return false;
|
|
4567
|
+
let mismatch = 0;
|
|
4568
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
4569
|
+
mismatch |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
4570
|
+
}
|
|
4571
|
+
return mismatch === 0;
|
|
4572
|
+
}
|
|
4573
|
+
function assertEvidenceRelativePath(relativePath) {
|
|
4574
|
+
if (typeof relativePath !== "string" || relativePath.trim() === "") {
|
|
4575
|
+
throw new Error("Evidence file path must be a non-empty relative path.");
|
|
4576
|
+
}
|
|
4577
|
+
const trimmed = relativePath.trim().replaceAll("\\", "/");
|
|
4578
|
+
if (path7.isAbsolute(trimmed) || trimmed.startsWith("/")) {
|
|
4579
|
+
throw new Error(`Evidence file path must be relative: ${relativePath}`);
|
|
4580
|
+
}
|
|
4581
|
+
const parts = trimmed.split("/").filter((part) => part !== "");
|
|
4582
|
+
if (parts.length === 0) {
|
|
4583
|
+
throw new Error(`Evidence file path must be relative: ${relativePath}`);
|
|
4584
|
+
}
|
|
4585
|
+
for (const part of parts) {
|
|
4586
|
+
if (part === "." || part === "..") {
|
|
4587
|
+
throw new Error(`Evidence file path must not contain "." or "..": ${relativePath}`);
|
|
4674
4588
|
}
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4589
|
+
}
|
|
4590
|
+
return parts.join("/");
|
|
4591
|
+
}
|
|
4592
|
+
|
|
4593
|
+
// packages/core/src/evidence/manifest.ts
|
|
4594
|
+
function stable(value) {
|
|
4595
|
+
if (Array.isArray(value)) return value.map(stable);
|
|
4596
|
+
if (value === null || typeof value !== "object") return value;
|
|
4597
|
+
const record = value;
|
|
4598
|
+
return Object.fromEntries(
|
|
4599
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable(record[key])])
|
|
4600
|
+
);
|
|
4601
|
+
}
|
|
4602
|
+
function serializeEvidenceManifest(manifest) {
|
|
4603
|
+
return `${JSON.stringify(stable(manifest), null, 2)}
|
|
4604
|
+
`;
|
|
4605
|
+
}
|
|
4606
|
+
function inferEvidenceFileRole(relativePath) {
|
|
4607
|
+
const normalized = assertEvidenceRelativePath(relativePath);
|
|
4608
|
+
const base = normalized.includes("/") ? normalized.slice(normalized.lastIndexOf("/") + 1) : normalized;
|
|
4609
|
+
if (base === "evidence.html" || base === "trace.html" || base.endsWith(".html")) {
|
|
4610
|
+
return "report";
|
|
4611
|
+
}
|
|
4612
|
+
if (base === "trace.jsonl" || base.endsWith(".jsonl")) {
|
|
4613
|
+
return "redacted-trace";
|
|
4614
|
+
}
|
|
4615
|
+
if (base === "check-results.json") {
|
|
4616
|
+
return "checks";
|
|
4617
|
+
}
|
|
4618
|
+
if (base === "contract.resolved.json") {
|
|
4619
|
+
return "contract";
|
|
4620
|
+
}
|
|
4621
|
+
if (base === "redaction-report.json") {
|
|
4622
|
+
return "redaction-report";
|
|
4623
|
+
}
|
|
4624
|
+
if (base === "summary.md") {
|
|
4625
|
+
return "summary";
|
|
4626
|
+
}
|
|
4627
|
+
return "other";
|
|
4628
|
+
}
|
|
4629
|
+
function buildEvidenceFileEntries(files) {
|
|
4630
|
+
const byPath = /* @__PURE__ */ new Map();
|
|
4631
|
+
for (const file of files) {
|
|
4632
|
+
const relativePath = assertEvidenceRelativePath(file.path);
|
|
4633
|
+
if (relativePath === EVIDENCE_MANIFEST_FILENAME) {
|
|
4634
|
+
throw new Error(
|
|
4635
|
+
`Do not include ${EVIDENCE_MANIFEST_FILENAME} in packaged file hashes (self-hash is undefined).`
|
|
4636
|
+
);
|
|
4637
|
+
}
|
|
4638
|
+
if (byPath.has(relativePath)) {
|
|
4639
|
+
throw new Error(`Duplicate evidence file path: ${relativePath}`);
|
|
4640
|
+
}
|
|
4641
|
+
byPath.set(relativePath, {
|
|
4642
|
+
path: relativePath,
|
|
4643
|
+
sha256: sha256Hex(file.content),
|
|
4644
|
+
role: file.role ?? inferEvidenceFileRole(relativePath)
|
|
4688
4645
|
});
|
|
4689
4646
|
}
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4647
|
+
return [...byPath.values()].sort((a, b) => a.path.localeCompare(b.path));
|
|
4648
|
+
}
|
|
4649
|
+
function buildEvidenceManifest(parts) {
|
|
4650
|
+
const runIds = [...parts.runIds];
|
|
4651
|
+
if (runIds.length === 0) {
|
|
4652
|
+
throw new Error("Evidence manifest requires at least one run id.");
|
|
4653
|
+
}
|
|
4654
|
+
for (const item of parts.sourceHashes) {
|
|
4655
|
+
if (!runIds.includes(item.runId)) {
|
|
4656
|
+
throw new Error(`sourceHashes runId "${item.runId}" is not listed in source.runIds.`);
|
|
4657
|
+
}
|
|
4658
|
+
if (item.algorithm !== "sha256") {
|
|
4659
|
+
throw new Error(`Unsupported source hash algorithm: ${item.algorithm}`);
|
|
4660
|
+
}
|
|
4661
|
+
}
|
|
4662
|
+
const assessment = {
|
|
4663
|
+
status: parts.assessmentStatus,
|
|
4664
|
+
note: parts.note ?? EVIDENCE_ASSESSMENT_NOTE
|
|
4665
|
+
};
|
|
4666
|
+
if (parts.sourceStatus !== void 0) {
|
|
4667
|
+
assessment.sourceStatus = parts.sourceStatus;
|
|
4668
|
+
}
|
|
4696
4669
|
return {
|
|
4697
|
-
|
|
4698
|
-
|
|
4670
|
+
evidenceFormatVersion: EVIDENCE_FORMAT_VERSION,
|
|
4671
|
+
generator: {
|
|
4672
|
+
name: parts.generatorName ?? "agent-inspect",
|
|
4673
|
+
version: parts.generatorVersion
|
|
4674
|
+
},
|
|
4675
|
+
createdAt: parts.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
4676
|
+
source: {
|
|
4677
|
+
runIds,
|
|
4678
|
+
traceSchemaVersions: [...parts.traceSchemaVersions].sort((a, b) => a.localeCompare(b)),
|
|
4679
|
+
sourceHashes: [...parts.sourceHashes].sort((a, b) => a.runId.localeCompare(b.runId))
|
|
4680
|
+
},
|
|
4681
|
+
policy: {
|
|
4682
|
+
redactionProfile: parts.redactionProfile,
|
|
4683
|
+
verificationPolicy: parts.verificationPolicy ?? parts.redactionProfile
|
|
4684
|
+
},
|
|
4685
|
+
assessment,
|
|
4686
|
+
...parts.semantics !== void 0 ? { semantics: { ...parts.semantics } } : {},
|
|
4687
|
+
...parts.contract !== void 0 ? { contract: { ...parts.contract } } : {},
|
|
4688
|
+
...parts.inputs !== void 0 ? { inputs: { ...parts.inputs } } : {},
|
|
4689
|
+
files: buildEvidenceFileEntries(parts.files)
|
|
4699
4690
|
};
|
|
4700
4691
|
}
|
|
4701
|
-
function
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
if (direct) return direct;
|
|
4705
|
-
const metadata = attrs?.metadata;
|
|
4706
|
-
if (isRecord7(metadata)) {
|
|
4707
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
4708
|
-
if (nested) return nested;
|
|
4692
|
+
function validateEvidenceManifest(value) {
|
|
4693
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
4694
|
+
throw new Error("Evidence manifest must be a JSON object.");
|
|
4709
4695
|
}
|
|
4710
|
-
|
|
4711
|
-
|
|
4696
|
+
const record = value;
|
|
4697
|
+
if (record.evidenceFormatVersion !== EVIDENCE_FORMAT_VERSION) {
|
|
4698
|
+
throw new Error(
|
|
4699
|
+
`Unsupported evidenceFormatVersion: ${String(record.evidenceFormatVersion)}`
|
|
4700
|
+
);
|
|
4712
4701
|
}
|
|
4713
|
-
|
|
4702
|
+
const generator = record.generator;
|
|
4703
|
+
if (generator === null || typeof generator !== "object" || Array.isArray(generator) || typeof generator.name !== "string" || typeof generator.version !== "string") {
|
|
4704
|
+
throw new Error("Evidence manifest requires generator.name and generator.version.");
|
|
4705
|
+
}
|
|
4706
|
+
const source = record.source;
|
|
4707
|
+
if (source === null || typeof source !== "object" || Array.isArray(source)) {
|
|
4708
|
+
throw new Error("Evidence manifest requires source.");
|
|
4709
|
+
}
|
|
4710
|
+
const sourceRecord = source;
|
|
4711
|
+
if (!Array.isArray(sourceRecord.runIds) || sourceRecord.runIds.length === 0) {
|
|
4712
|
+
throw new Error("Evidence manifest source.runIds must be a non-empty array.");
|
|
4713
|
+
}
|
|
4714
|
+
if (!Array.isArray(sourceRecord.traceSchemaVersions)) {
|
|
4715
|
+
throw new Error("Evidence manifest source.traceSchemaVersions must be an array.");
|
|
4716
|
+
}
|
|
4717
|
+
if (!Array.isArray(sourceRecord.sourceHashes)) {
|
|
4718
|
+
throw new Error("Evidence manifest source.sourceHashes must be an array.");
|
|
4719
|
+
}
|
|
4720
|
+
const policy = record.policy;
|
|
4721
|
+
if (policy === null || typeof policy !== "object" || Array.isArray(policy)) {
|
|
4722
|
+
throw new Error("Evidence manifest requires policy.");
|
|
4723
|
+
}
|
|
4724
|
+
const assessment = record.assessment;
|
|
4725
|
+
if (assessment === null || typeof assessment !== "object" || Array.isArray(assessment) || typeof assessment.status !== "string") {
|
|
4726
|
+
throw new Error("Evidence manifest requires assessment.status.");
|
|
4727
|
+
}
|
|
4728
|
+
if (!Array.isArray(record.files) || record.files.length === 0) {
|
|
4729
|
+
throw new Error("Evidence manifest files must be a non-empty array.");
|
|
4730
|
+
}
|
|
4731
|
+
for (const file of record.files) {
|
|
4732
|
+
if (file === null || typeof file !== "object" || Array.isArray(file)) {
|
|
4733
|
+
throw new Error("Evidence manifest file entries must be objects.");
|
|
4734
|
+
}
|
|
4735
|
+
const entry = file;
|
|
4736
|
+
if (typeof entry.path !== "string") {
|
|
4737
|
+
throw new Error("Evidence file entry requires path.");
|
|
4738
|
+
}
|
|
4739
|
+
assertEvidenceRelativePath(entry.path);
|
|
4740
|
+
if (typeof entry.sha256 !== "string" || !/^[a-f0-9]{64}$/i.test(entry.sha256)) {
|
|
4741
|
+
throw new Error(`Evidence file entry requires sha256 hex for ${entry.path}.`);
|
|
4742
|
+
}
|
|
4743
|
+
}
|
|
4744
|
+
return value;
|
|
4714
4745
|
}
|
|
4715
|
-
function
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4746
|
+
function parseEvidenceManifestJson(text) {
|
|
4747
|
+
let parsed;
|
|
4748
|
+
try {
|
|
4749
|
+
parsed = JSON.parse(text);
|
|
4750
|
+
} catch (error) {
|
|
4751
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4752
|
+
throw new Error(`Evidence manifest is not valid JSON: ${message}`);
|
|
4720
4753
|
}
|
|
4721
|
-
return
|
|
4754
|
+
return validateEvidenceManifest(parsed);
|
|
4755
|
+
}
|
|
4756
|
+
function collectTraceSchemaVersions(jsonl) {
|
|
4757
|
+
const versions = /* @__PURE__ */ new Set();
|
|
4758
|
+
for (const line of jsonl.split(/\r?\n/)) {
|
|
4759
|
+
const trimmed = line.trim();
|
|
4760
|
+
if (trimmed === "") continue;
|
|
4761
|
+
try {
|
|
4762
|
+
const row = JSON.parse(trimmed);
|
|
4763
|
+
if (typeof row.schemaVersion === "string" && row.schemaVersion.trim() !== "") {
|
|
4764
|
+
versions.add(row.schemaVersion.trim());
|
|
4765
|
+
}
|
|
4766
|
+
} catch {
|
|
4767
|
+
}
|
|
4768
|
+
}
|
|
4769
|
+
return [...versions].sort((a, b) => a.localeCompare(b));
|
|
4722
4770
|
}
|
|
4723
4771
|
|
|
4724
4772
|
// packages/core/src/checks/derived-failure.ts
|
|
@@ -7069,7 +7117,17 @@ function emptyPassResult(input, runId) {
|
|
|
7069
7117
|
status: "pass",
|
|
7070
7118
|
format: input.read.format,
|
|
7071
7119
|
...runId !== void 0 ? { runId } : {},
|
|
7072
|
-
summary: {
|
|
7120
|
+
summary: {
|
|
7121
|
+
passed: 0,
|
|
7122
|
+
failed: 0,
|
|
7123
|
+
warnings: 0,
|
|
7124
|
+
errors: 0,
|
|
7125
|
+
rulesEvaluated: 0,
|
|
7126
|
+
rulesPassed: 0,
|
|
7127
|
+
rulesWarning: 0,
|
|
7128
|
+
rulesFailed: 0,
|
|
7129
|
+
rulesError: 0
|
|
7130
|
+
},
|
|
7073
7131
|
findings: [],
|
|
7074
7132
|
diagnostics: [],
|
|
7075
7133
|
ruleExecutions: []
|
|
@@ -7097,7 +7155,11 @@ function mergeContractResults(input, parts, extraFindings = []) {
|
|
|
7097
7155
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
7098
7156
|
).length,
|
|
7099
7157
|
errors,
|
|
7100
|
-
rulesEvaluated: ruleExecutions.length
|
|
7158
|
+
rulesEvaluated: ruleExecutions.length,
|
|
7159
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
7160
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
7161
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
7162
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
7101
7163
|
},
|
|
7102
7164
|
findings,
|
|
7103
7165
|
diagnostics,
|
|
@@ -7175,7 +7237,11 @@ function invalidContractInputResult() {
|
|
|
7175
7237
|
failed: 0,
|
|
7176
7238
|
warnings: 0,
|
|
7177
7239
|
errors: 1,
|
|
7178
|
-
rulesEvaluated: 0
|
|
7240
|
+
rulesEvaluated: 0,
|
|
7241
|
+
rulesPassed: 0,
|
|
7242
|
+
rulesWarning: 0,
|
|
7243
|
+
rulesFailed: 0,
|
|
7244
|
+
rulesError: 0
|
|
7179
7245
|
},
|
|
7180
7246
|
findings: [],
|
|
7181
7247
|
diagnostics: [
|
|
@@ -7228,7 +7294,11 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
7228
7294
|
failed: 0,
|
|
7229
7295
|
warnings: 0,
|
|
7230
7296
|
errors: shapeErrors.length,
|
|
7231
|
-
rulesEvaluated: 0
|
|
7297
|
+
rulesEvaluated: 0,
|
|
7298
|
+
rulesPassed: 0,
|
|
7299
|
+
rulesWarning: 0,
|
|
7300
|
+
rulesFailed: 0,
|
|
7301
|
+
rulesError: 0
|
|
7232
7302
|
},
|
|
7233
7303
|
findings: [],
|
|
7234
7304
|
diagnostics: shapeErrors.map((item) => ({
|
|
@@ -7253,7 +7323,11 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
7253
7323
|
failed: 0,
|
|
7254
7324
|
warnings: 0,
|
|
7255
7325
|
errors: scoped.diagnostics.length,
|
|
7256
|
-
rulesEvaluated: 0
|
|
7326
|
+
rulesEvaluated: 0,
|
|
7327
|
+
rulesPassed: 0,
|
|
7328
|
+
rulesWarning: 0,
|
|
7329
|
+
rulesFailed: 0,
|
|
7330
|
+
rulesError: 0
|
|
7257
7331
|
},
|
|
7258
7332
|
findings: [],
|
|
7259
7333
|
diagnostics: scoped.diagnostics,
|
|
@@ -7451,7 +7525,11 @@ function emptySummary2() {
|
|
|
7451
7525
|
failed: 0,
|
|
7452
7526
|
warnings: 0,
|
|
7453
7527
|
errors: 0,
|
|
7454
|
-
rulesEvaluated: 0
|
|
7528
|
+
rulesEvaluated: 0,
|
|
7529
|
+
rulesPassed: 0,
|
|
7530
|
+
rulesWarning: 0,
|
|
7531
|
+
rulesFailed: 0,
|
|
7532
|
+
rulesError: 0
|
|
7455
7533
|
};
|
|
7456
7534
|
}
|
|
7457
7535
|
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
@@ -7463,7 +7541,11 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
7463
7541
|
summary: {
|
|
7464
7542
|
...emptySummary2(),
|
|
7465
7543
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
7466
|
-
rulesEvaluated: ruleExecutions.length
|
|
7544
|
+
rulesEvaluated: ruleExecutions.length,
|
|
7545
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
7546
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
7547
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
7548
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
7467
7549
|
},
|
|
7468
7550
|
findings: [],
|
|
7469
7551
|
diagnostics: [...diagnostics],
|
|
@@ -7623,7 +7705,7 @@ function normalizeFinding(rule, finding) {
|
|
|
7623
7705
|
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
7624
7706
|
};
|
|
7625
7707
|
}
|
|
7626
|
-
function summarize(findings, diagnostics,
|
|
7708
|
+
function summarize(findings, diagnostics, ruleExecutions) {
|
|
7627
7709
|
return {
|
|
7628
7710
|
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
7629
7711
|
failed: findings.filter(
|
|
@@ -7633,7 +7715,11 @@ function summarize(findings, diagnostics, rulesEvaluated) {
|
|
|
7633
7715
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
7634
7716
|
).length,
|
|
7635
7717
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
7636
|
-
rulesEvaluated
|
|
7718
|
+
rulesEvaluated: ruleExecutions.length,
|
|
7719
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
7720
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
7721
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
7722
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
7637
7723
|
};
|
|
7638
7724
|
}
|
|
7639
7725
|
function classifyRuleExecution(findings, threw) {
|
|
@@ -9299,7 +9385,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
9299
9385
|
}
|
|
9300
9386
|
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
9301
9387
|
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
9302
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions
|
|
9388
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions);
|
|
9303
9389
|
const status = summary.failed > 0 ? "fail" : "pass";
|
|
9304
9390
|
return {
|
|
9305
9391
|
ok: status === "pass",
|
|
@@ -11238,10 +11324,143 @@ function asStringArray(value, label) {
|
|
|
11238
11324
|
function asPositiveNumber(value, label) {
|
|
11239
11325
|
if (value === void 0) return void 0;
|
|
11240
11326
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
11241
|
-
throw new Error(`${label} must be a non-negative number.`);
|
|
11327
|
+
throw new Error(`${label} must be a non-negative finite number.`);
|
|
11328
|
+
}
|
|
11329
|
+
return value;
|
|
11330
|
+
}
|
|
11331
|
+
function asBoolean(value, label) {
|
|
11332
|
+
if (value === void 0) return void 0;
|
|
11333
|
+
if (typeof value !== "boolean") {
|
|
11334
|
+
throw new Error(`${label} must be a boolean (not a string or other type).`);
|
|
11242
11335
|
}
|
|
11243
11336
|
return value;
|
|
11244
11337
|
}
|
|
11338
|
+
function assertObject(value, label) {
|
|
11339
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
11340
|
+
throw new Error(`${label} must be an object.`);
|
|
11341
|
+
}
|
|
11342
|
+
return value;
|
|
11343
|
+
}
|
|
11344
|
+
function assertKnownKeys(raw, allowed, label) {
|
|
11345
|
+
const allowedSet = new Set(allowed);
|
|
11346
|
+
for (const key of Object.keys(raw)) {
|
|
11347
|
+
if (!allowedSet.has(key)) {
|
|
11348
|
+
throw new Error(`${label} has unknown key "${key}".`);
|
|
11349
|
+
}
|
|
11350
|
+
}
|
|
11351
|
+
}
|
|
11352
|
+
var CASE_KEYS = [
|
|
11353
|
+
"id",
|
|
11354
|
+
"trace",
|
|
11355
|
+
"runId",
|
|
11356
|
+
"input",
|
|
11357
|
+
"requireTools",
|
|
11358
|
+
"forbidTools",
|
|
11359
|
+
"maxDurationMs",
|
|
11360
|
+
"expectedObservations",
|
|
11361
|
+
"expect"
|
|
11362
|
+
];
|
|
11363
|
+
var EXPECT_KEYS = [
|
|
11364
|
+
"checkStatus",
|
|
11365
|
+
"findingRuleIds",
|
|
11366
|
+
"rejectUnexpectedFindings",
|
|
11367
|
+
"minAssertions"
|
|
11368
|
+
];
|
|
11369
|
+
var CHECKS_KEYS = ["select", "run", "tool", "llm"];
|
|
11370
|
+
var CHECKS_RUN_KEYS = ["maxDurationMs", "maxDepth"];
|
|
11371
|
+
var CHECKS_TOOL_KEYS = ["required", "forbidden"];
|
|
11372
|
+
var CHECKS_LLM_KEYS = ["allowedModels", "maxTotalTokens"];
|
|
11373
|
+
var EVAL_KEYS = [
|
|
11374
|
+
"requireSuccess",
|
|
11375
|
+
"requiredTools",
|
|
11376
|
+
"forbiddenTools",
|
|
11377
|
+
"maxDurationMs",
|
|
11378
|
+
"maxDepth",
|
|
11379
|
+
"maxRetries",
|
|
11380
|
+
"maxTotalTokens"
|
|
11381
|
+
];
|
|
11382
|
+
var ROOT_KEYS = [
|
|
11383
|
+
"name",
|
|
11384
|
+
"traces",
|
|
11385
|
+
"cases",
|
|
11386
|
+
"checks",
|
|
11387
|
+
"eval",
|
|
11388
|
+
"redactionProfile",
|
|
11389
|
+
"artifacts",
|
|
11390
|
+
"baseline",
|
|
11391
|
+
"candidate"
|
|
11392
|
+
];
|
|
11393
|
+
function asExpectConfig(value, label) {
|
|
11394
|
+
if (value === void 0) return void 0;
|
|
11395
|
+
const raw = assertObject(value, label);
|
|
11396
|
+
assertKnownKeys(raw, EXPECT_KEYS, label);
|
|
11397
|
+
const checkStatus = raw.checkStatus;
|
|
11398
|
+
if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
|
|
11399
|
+
throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
|
|
11400
|
+
}
|
|
11401
|
+
const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
|
|
11402
|
+
const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
|
|
11403
|
+
const rejectUnexpectedFindings = asBoolean(
|
|
11404
|
+
raw.rejectUnexpectedFindings,
|
|
11405
|
+
`${label}.rejectUnexpectedFindings`
|
|
11406
|
+
);
|
|
11407
|
+
return {
|
|
11408
|
+
checkStatus,
|
|
11409
|
+
...findingRuleIds !== void 0 ? { findingRuleIds } : {},
|
|
11410
|
+
...rejectUnexpectedFindings !== void 0 ? { rejectUnexpectedFindings } : {},
|
|
11411
|
+
...minAssertions !== void 0 ? { minAssertions } : {}
|
|
11412
|
+
};
|
|
11413
|
+
}
|
|
11414
|
+
function asChecksConfig(value, label) {
|
|
11415
|
+
const raw = assertObject(value, label);
|
|
11416
|
+
assertKnownKeys(raw, CHECKS_KEYS, label);
|
|
11417
|
+
const checks = {};
|
|
11418
|
+
const select = asStringArray(raw.select, `${label}.select`);
|
|
11419
|
+
if (select !== void 0) checks.select = select;
|
|
11420
|
+
if (raw.run !== void 0) {
|
|
11421
|
+
const run = assertObject(raw.run, `${label}.run`);
|
|
11422
|
+
assertKnownKeys(run, CHECKS_RUN_KEYS, `${label}.run`);
|
|
11423
|
+
checks.run = {
|
|
11424
|
+
...asPositiveNumber(run.maxDurationMs, `${label}.run.maxDurationMs`) !== void 0 ? { maxDurationMs: asPositiveNumber(run.maxDurationMs, `${label}.run.maxDurationMs`) } : {},
|
|
11425
|
+
...asPositiveNumber(run.maxDepth, `${label}.run.maxDepth`) !== void 0 ? { maxDepth: asPositiveNumber(run.maxDepth, `${label}.run.maxDepth`) } : {}
|
|
11426
|
+
};
|
|
11427
|
+
}
|
|
11428
|
+
if (raw.tool !== void 0) {
|
|
11429
|
+
const tool = assertObject(raw.tool, `${label}.tool`);
|
|
11430
|
+
assertKnownKeys(tool, CHECKS_TOOL_KEYS, `${label}.tool`);
|
|
11431
|
+
checks.tool = {
|
|
11432
|
+
...asStringArray(tool.required, `${label}.tool.required`) !== void 0 ? { required: asStringArray(tool.required, `${label}.tool.required`) } : {},
|
|
11433
|
+
...asStringArray(tool.forbidden, `${label}.tool.forbidden`) !== void 0 ? { forbidden: asStringArray(tool.forbidden, `${label}.tool.forbidden`) } : {}
|
|
11434
|
+
};
|
|
11435
|
+
}
|
|
11436
|
+
if (raw.llm !== void 0) {
|
|
11437
|
+
const llm = assertObject(raw.llm, `${label}.llm`);
|
|
11438
|
+
assertKnownKeys(llm, CHECKS_LLM_KEYS, `${label}.llm`);
|
|
11439
|
+
checks.llm = {
|
|
11440
|
+
...asStringArray(llm.allowedModels, `${label}.llm.allowedModels`) !== void 0 ? { allowedModels: asStringArray(llm.allowedModels, `${label}.llm.allowedModels`) } : {},
|
|
11441
|
+
...asPositiveNumber(llm.maxTotalTokens, `${label}.llm.maxTotalTokens`) !== void 0 ? {
|
|
11442
|
+
maxTotalTokens: asPositiveNumber(
|
|
11443
|
+
llm.maxTotalTokens,
|
|
11444
|
+
`${label}.llm.maxTotalTokens`
|
|
11445
|
+
)
|
|
11446
|
+
} : {}
|
|
11447
|
+
};
|
|
11448
|
+
}
|
|
11449
|
+
return checks;
|
|
11450
|
+
}
|
|
11451
|
+
function asEvalConfig(value, label) {
|
|
11452
|
+
const raw = assertObject(value, label);
|
|
11453
|
+
assertKnownKeys(raw, EVAL_KEYS, label);
|
|
11454
|
+
return {
|
|
11455
|
+
...asBoolean(raw.requireSuccess, `${label}.requireSuccess`) !== void 0 ? { requireSuccess: asBoolean(raw.requireSuccess, `${label}.requireSuccess`) } : {},
|
|
11456
|
+
...asStringArray(raw.requiredTools, `${label}.requiredTools`) !== void 0 ? { requiredTools: asStringArray(raw.requiredTools, `${label}.requiredTools`) } : {},
|
|
11457
|
+
...asStringArray(raw.forbiddenTools, `${label}.forbiddenTools`) !== void 0 ? { forbiddenTools: asStringArray(raw.forbiddenTools, `${label}.forbiddenTools`) } : {},
|
|
11458
|
+
...asPositiveNumber(raw.maxDurationMs, `${label}.maxDurationMs`) !== void 0 ? { maxDurationMs: asPositiveNumber(raw.maxDurationMs, `${label}.maxDurationMs`) } : {},
|
|
11459
|
+
...asPositiveNumber(raw.maxDepth, `${label}.maxDepth`) !== void 0 ? { maxDepth: asPositiveNumber(raw.maxDepth, `${label}.maxDepth`) } : {},
|
|
11460
|
+
...asPositiveNumber(raw.maxRetries, `${label}.maxRetries`) !== void 0 ? { maxRetries: asPositiveNumber(raw.maxRetries, `${label}.maxRetries`) } : {},
|
|
11461
|
+
...asPositiveNumber(raw.maxTotalTokens, `${label}.maxTotalTokens`) !== void 0 ? { maxTotalTokens: asPositiveNumber(raw.maxTotalTokens, `${label}.maxTotalTokens`) } : {}
|
|
11462
|
+
};
|
|
11463
|
+
}
|
|
11245
11464
|
function validateCaseConfig(value, index) {
|
|
11246
11465
|
const diagnostics = [];
|
|
11247
11466
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
@@ -11252,6 +11471,7 @@ function validateCaseConfig(value, index) {
|
|
|
11252
11471
|
}
|
|
11253
11472
|
const raw = value;
|
|
11254
11473
|
try {
|
|
11474
|
+
assertKnownKeys(raw, CASE_KEYS, `cases[${index}]`);
|
|
11255
11475
|
const id = asString(raw.id, `cases[${index}].id`);
|
|
11256
11476
|
if (id === void 0) {
|
|
11257
11477
|
diagnostics.push(
|
|
@@ -11284,7 +11504,8 @@ function validateCaseConfig(value, index) {
|
|
|
11284
11504
|
raw.expectedObservations,
|
|
11285
11505
|
`cases[${index}].expectedObservations`
|
|
11286
11506
|
)
|
|
11287
|
-
} : {}
|
|
11507
|
+
} : {},
|
|
11508
|
+
...asExpectConfig(raw.expect, `cases[${index}].expect`) !== void 0 ? { expect: asExpectConfig(raw.expect, `cases[${index}].expect`) } : {}
|
|
11288
11509
|
},
|
|
11289
11510
|
diagnostics
|
|
11290
11511
|
};
|
|
@@ -11299,6 +11520,7 @@ function normalizeSuiteConfig(value) {
|
|
|
11299
11520
|
throw new Error("Suite config must export an object.");
|
|
11300
11521
|
}
|
|
11301
11522
|
const raw = value;
|
|
11523
|
+
assertKnownKeys(raw, ROOT_KEYS, "suite config");
|
|
11302
11524
|
const name = asString(raw.name, "name");
|
|
11303
11525
|
const traces = asString(raw.traces, "traces");
|
|
11304
11526
|
if (name === void 0) throw new Error("name is required.");
|
|
@@ -11327,25 +11549,15 @@ function normalizeSuiteConfig(value) {
|
|
|
11327
11549
|
}
|
|
11328
11550
|
const config = { name, traces, cases };
|
|
11329
11551
|
if (raw.checks !== void 0) {
|
|
11330
|
-
|
|
11331
|
-
throw new Error("checks must be an object.");
|
|
11332
|
-
}
|
|
11333
|
-
config.checks = raw.checks;
|
|
11552
|
+
config.checks = asChecksConfig(raw.checks, "checks");
|
|
11334
11553
|
}
|
|
11335
11554
|
if (raw.eval !== void 0) {
|
|
11336
|
-
|
|
11337
|
-
throw new Error("eval must be an object.");
|
|
11338
|
-
}
|
|
11339
|
-
config.eval = raw.eval;
|
|
11555
|
+
config.eval = asEvalConfig(raw.eval, "eval");
|
|
11340
11556
|
}
|
|
11341
11557
|
if (raw.artifacts !== void 0) {
|
|
11342
|
-
|
|
11343
|
-
|
|
11344
|
-
|
|
11345
|
-
const outputDir = asString(
|
|
11346
|
-
raw.artifacts.outputDir,
|
|
11347
|
-
"artifacts.outputDir"
|
|
11348
|
-
);
|
|
11558
|
+
const artifacts = assertObject(raw.artifacts, "artifacts");
|
|
11559
|
+
assertKnownKeys(artifacts, ["outputDir"], "artifacts");
|
|
11560
|
+
const outputDir = asString(artifacts.outputDir, "artifacts.outputDir");
|
|
11349
11561
|
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
11350
11562
|
}
|
|
11351
11563
|
if (raw.baseline !== void 0) {
|
|
@@ -13919,6 +14131,78 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
13919
14131
|
}
|
|
13920
14132
|
return { ok: diagnostics.length === 0, diagnostics };
|
|
13921
14133
|
}
|
|
14134
|
+
function evaluateCaseExpect(suiteCase, checkResult, observationCount) {
|
|
14135
|
+
const expectation = suiteCase.expect;
|
|
14136
|
+
if (expectation === void 0) {
|
|
14137
|
+
return { ok: true, diagnostics: [] };
|
|
14138
|
+
}
|
|
14139
|
+
const diagnostics = [];
|
|
14140
|
+
if (checkResult.status !== expectation.checkStatus) {
|
|
14141
|
+
diagnostics.push(
|
|
14142
|
+
diagnostic5(
|
|
14143
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14144
|
+
`Expected check status "${expectation.checkStatus}", got "${checkResult.status}".`,
|
|
14145
|
+
"error",
|
|
14146
|
+
suiteCase.id
|
|
14147
|
+
)
|
|
14148
|
+
);
|
|
14149
|
+
}
|
|
14150
|
+
const failedFindings = checkResult.findings.filter(
|
|
14151
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
14152
|
+
);
|
|
14153
|
+
const requiredIds = expectation.findingRuleIds ?? [];
|
|
14154
|
+
for (const ruleId of requiredIds) {
|
|
14155
|
+
if (!failedFindings.some((finding) => finding.ruleId === ruleId)) {
|
|
14156
|
+
diagnostics.push(
|
|
14157
|
+
diagnostic5(
|
|
14158
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14159
|
+
`Expected failing finding for rule "${ruleId}" was not present.`,
|
|
14160
|
+
"error",
|
|
14161
|
+
suiteCase.id
|
|
14162
|
+
)
|
|
14163
|
+
);
|
|
14164
|
+
}
|
|
14165
|
+
}
|
|
14166
|
+
const rejectUnexpected = expectation.rejectUnexpectedFindings !== false;
|
|
14167
|
+
if (rejectUnexpected && requiredIds.length > 0) {
|
|
14168
|
+
for (const finding of failedFindings) {
|
|
14169
|
+
if (!requiredIds.includes(finding.ruleId)) {
|
|
14170
|
+
diagnostics.push(
|
|
14171
|
+
diagnostic5(
|
|
14172
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14173
|
+
`Unexpected failing finding for rule "${finding.ruleId}".`,
|
|
14174
|
+
"error",
|
|
14175
|
+
suiteCase.id
|
|
14176
|
+
)
|
|
14177
|
+
);
|
|
14178
|
+
}
|
|
14179
|
+
}
|
|
14180
|
+
}
|
|
14181
|
+
if (expectation.minAssertions !== void 0) {
|
|
14182
|
+
const assertionCount = checkResult.ruleExecutions.length + observationCount;
|
|
14183
|
+
if (assertionCount < expectation.minAssertions) {
|
|
14184
|
+
diagnostics.push(
|
|
14185
|
+
diagnostic5(
|
|
14186
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14187
|
+
`Expected at least ${expectation.minAssertions} assertion(s), got ${assertionCount}.`,
|
|
14188
|
+
"error",
|
|
14189
|
+
suiteCase.id
|
|
14190
|
+
)
|
|
14191
|
+
);
|
|
14192
|
+
}
|
|
14193
|
+
}
|
|
14194
|
+
if (expectation.checkStatus === "fail" && checkResult.diagnostics.some((item) => item.severity === "error") && checkResult.status === "error") {
|
|
14195
|
+
diagnostics.push(
|
|
14196
|
+
diagnostic5(
|
|
14197
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
14198
|
+
'Check ended in evaluator/config error; semantic failure expectation requires status "fail".',
|
|
14199
|
+
"error",
|
|
14200
|
+
suiteCase.id
|
|
14201
|
+
)
|
|
14202
|
+
);
|
|
14203
|
+
}
|
|
14204
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
14205
|
+
}
|
|
13922
14206
|
async function runSuiteCase(suiteCase, config, options) {
|
|
13923
14207
|
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
13924
14208
|
if (resolved.missing || resolved.tracePath === void 0) {
|
|
@@ -13971,9 +14255,21 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
13971
14255
|
status: "pass",
|
|
13972
14256
|
format: read.format,
|
|
13973
14257
|
findings: [],
|
|
13974
|
-
diagnostics: []
|
|
14258
|
+
diagnostics: [],
|
|
14259
|
+
ruleExecutions: []
|
|
14260
|
+
};
|
|
13975
14261
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
13976
|
-
const
|
|
14262
|
+
const checkOk = checkResult.ok;
|
|
14263
|
+
const observationsOk = observationResult.ok;
|
|
14264
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
14265
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
14266
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
14267
|
+
const expectResult = evaluateCaseExpect(
|
|
14268
|
+
suiteCase,
|
|
14269
|
+
checkResult,
|
|
14270
|
+
compiled.observationCount
|
|
14271
|
+
);
|
|
14272
|
+
const checkFailureDiagnostics = [
|
|
13977
14273
|
...checkResult.diagnostics.map(
|
|
13978
14274
|
(item) => diagnostic5(
|
|
13979
14275
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
@@ -13989,15 +14285,24 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
13989
14285
|
finding.severity,
|
|
13990
14286
|
suiteCase.id
|
|
13991
14287
|
)
|
|
13992
|
-
)
|
|
13993
|
-
...observationResult.diagnostics
|
|
14288
|
+
)
|
|
13994
14289
|
];
|
|
13995
|
-
const
|
|
13996
|
-
const
|
|
13997
|
-
|
|
13998
|
-
|
|
13999
|
-
|
|
14000
|
-
|
|
14290
|
+
const expectMatched = suiteCase.expect !== void 0 && expectResult.ok && observationsOk;
|
|
14291
|
+
const diagnostics = expectMatched ? [
|
|
14292
|
+
...observationResult.diagnostics,
|
|
14293
|
+
...expectResult.diagnostics,
|
|
14294
|
+
diagnostic5(
|
|
14295
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
14296
|
+
`Expected semantic failure matched (check.status=${checkResult.status}).`,
|
|
14297
|
+
"info",
|
|
14298
|
+
suiteCase.id
|
|
14299
|
+
)
|
|
14300
|
+
] : [
|
|
14301
|
+
...checkFailureDiagnostics,
|
|
14302
|
+
...observationResult.diagnostics,
|
|
14303
|
+
...expectResult.diagnostics
|
|
14304
|
+
];
|
|
14305
|
+
const ok = suiteCase.expect !== void 0 ? expectResult.ok && observationsOk : checkOk && observationsOk;
|
|
14001
14306
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
14002
14307
|
return {
|
|
14003
14308
|
id: suiteCase.id,
|
|
@@ -15173,5 +15478,5 @@ function renderGateReport(result, options = {}) {
|
|
|
15173
15478
|
}
|
|
15174
15479
|
|
|
15175
15480
|
export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_REDACT_KEYS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isCredentialSensitiveKey, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToRunTrees, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, stringContainsHighConfidenceCredential, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
|
|
15176
|
-
//# sourceMappingURL=chunk-
|
|
15177
|
-
//# sourceMappingURL=chunk-
|
|
15481
|
+
//# sourceMappingURL=chunk-GBAGZVUX.mjs.map
|
|
15482
|
+
//# sourceMappingURL=chunk-GBAGZVUX.mjs.map
|