@evalops/maestro 0.10.17 → 0.10.20

package/dist/server/agent-trajectory-inspection.js

@@ -0,0 +1,168 @@
+export const AGENT_TRAJECTORY_INSPECTION_SCHEMA = "evalops.maestro.agent-trajectory-inspection.v1";
+function redactedEvidence(evidence) {
+    return evidence
+        .map((anchor) => ({
+        ...anchor,
+        redacted: true,
+        label: `${anchor.kind}:${anchor.id}`,
+    }))
+        .sort((left, right) => {
+        const kindDelta = left.kind.localeCompare(right.kind);
+        return kindDelta === 0 ? left.id.localeCompare(right.id) : kindDelta;
+    });
+}
+function timelineItemIdsForEvidence(evidence) {
+    return evidence
+        .filter((anchor) => anchor.kind === "timeline_item")
+        .map((anchor) => anchor.id)
+        .sort();
+}
+function timelineItemIdsForEvents(eventIds, eventsById) {
+    const ids = new Set();
+    for (const eventId of eventIds) {
+        const event = eventsById.get(eventId);
+        for (const timelineItemId of event?.timelineItemIds ?? []) {
+            ids.add(timelineItemId);
+        }
+    }
+    return [...ids].sort();
+}
+function metadataKeys(item) {
+    const metadata = item.metadata;
+    if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
+        return [];
+    }
+    return Object.keys(metadata).sort();
+}
+function inspectTimelineItem(item) {
+    return {
+        id: item.id,
+        timestamp: item.timestamp,
+        type: item.type,
+        status: item.status,
+        visibility: item.visibility,
+        source: item.source,
+        title: item.title,
+        ...(item.summary ? { summary: item.summary } : {}),
+        ...(item.role ? { role: item.role } : {}),
+        ...(item.toolName ? { toolName: item.toolName } : {}),
+        ...(item.pendingRequestKind
+            ? { pendingRequestKind: item.pendingRequestKind }
+            : {}),
+        ...(item.platformOperation
+            ? { platformOperation: item.platformOperation }
+            : {}),
+        metadataKeys: metadataKeys(item),
+        redacted: true,
+    };
+}
+function inspectEvent(event) {
+    return {
+        id: event.id,
+        sequence: event.sequence,
+        timestamp: event.timestamp,
+        kind: event.kind,
+        phase: event.phase,
+        actor: event.actor,
+        type: event.type,
+        status: event.status,
+        visibility: event.visibility,
+        source: event.source,
+        title: event.title,
+        ...(event.summary ? { summary: event.summary } : {}),
+        ...(event.toolName ? { toolName: event.toolName } : {}),
+        relatedIds: event.relatedIds ?? [],
+        timelineItemIds: timelineItemIdsForEvidence(event.evidence),
+        evidence: redactedEvidence(event.evidence),
+    };
+}
+function finalAnswerFromEvents(events) {
+    const reversedEvents = [...events].reverse();
+    const event = reversedEvents.find((candidate) => candidate.actor === "assistant" &&
+        candidate.type === "message.assistant") ??
+        reversedEvents.find((candidate) => candidate.phase === "finish" || candidate.phase === "recover");
+    if (!event)
+        return undefined;
+    return {
+        eventId: event.id,
+        timelineItemIds: event.timelineItemIds,
+        title: event.title,
+        ...(event.summary ? { summary: event.summary } : {}),
+        redacted: true,
+    };
+}
+export function buildAgentTrajectoryInspectionReport({ timelineItems, trajectory, replay, score, }) {
+    const events = trajectory.events.map(inspectEvent);
+    const finalAnswer = finalAnswerFromEvents(events);
+    const eventsById = new Map(events.map((event) => [event.id, event]));
+    const replayDeltas = replay.deltas.map((delta) => ({
+        id: delta.id,
+        severity: delta.severity,
+        ruleId: delta.ruleId,
+        message: delta.message,
+        ...(delta.eventId ? { eventId: delta.eventId } : {}),
+        timelineItemIds: delta.eventId
+            ? timelineItemIdsForEvents([delta.eventId], eventsById)
+            : timelineItemIdsForEvidence(delta.evidence),
+        evidence: redactedEvidence(delta.evidence),
+    }));
+    const scoreFindings = score.findings.map((finding) => ({
+        ruleId: finding.ruleId,
+        status: finding.status,
+        severity: finding.severity,
+        message: finding.message,
+        eventIds: finding.eventIds,
+        timelineItemIds: timelineItemIdsForEvents(finding.eventIds, eventsById),
+        evidence: redactedEvidence(finding.evidence),
+        remediation: finding.remediation,
+    }));
+    const jumpTargets = new Set();
+    for (const event of events) {
+        for (const timelineItemId of event.timelineItemIds) {
+            jumpTargets.add(`${event.id}->${timelineItemId}`);
+        }
+    }
+    for (const delta of replayDeltas) {
+        for (const timelineItemId of delta.timelineItemIds) {
+            jumpTargets.add(`${delta.id}->${timelineItemId}`);
+        }
+    }
+    for (const finding of scoreFindings) {
+        for (const timelineItemId of finding.timelineItemIds) {
+            jumpTargets.add(`${finding.ruleId}->${timelineItemId}`);
+        }
+    }
+    return {
+        schemaVersion: AGENT_TRAJECTORY_INSPECTION_SCHEMA,
+        trajectorySchemaVersion: trajectory.schemaVersion,
+        replaySchemaVersion: replay.schemaVersion,
+        scoreSchemaVersion: score.schemaVersion,
+        run: trajectory.run,
+        redaction: {
+            default: "redacted",
+            omitted: [
+                "raw prompts",
+                "raw tool arguments",
+                "raw tool outputs",
+                "full file diffs",
+                "timeline metadata values",
+                "secrets",
+            ],
+        },
+        counts: {
+            timelineItems: timelineItems.length,
+            events: events.length,
+            replayDeltas: replayDeltas.length,
+            scoreFindings: scoreFindings.length,
+            scoreFailures: score.counts.failed,
+            scoreWarnings: score.counts.warnings,
+            jumpTargets: jumpTargets.size,
+        },
+        ...(finalAnswer ? { finalAnswer } : {}),
+        timelineItems: timelineItems.map(inspectTimelineItem),
+        events,
+        replayDeltas,
+        scoreFindings,
+    };
+}
+//# sourceMappingURL=agent-trajectory-inspection.js.map

package/dist/server/agent-trajectory-inspection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-trajectory-inspection.js","sourceRoot":"","sources":["../../src/server/agent-trajectory-inspection.ts"],"names":[],"mappings":"AAcA,MAAM,CAAC,MAAM,kCAAkC,GAC9C,gDAAgD,CAAC;AA4GlD,SAAS,gBAAgB,CACxB,QAA0B;IAE1B,OAAO,QAAQ;SACb,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACjB,GAAG,MAAM;QACT,QAAQ,EAAE,IAAa;QACvB,KAAK,EAAE,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,EAAE;KACpC,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACrB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtD,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,0BAA0B,CAAC,QAA0B;IAC7D,OAAO,QAAQ;SACb,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,eAAe,CAAC;SACnD,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;SAC1B,IAAI,EAAE,CAAC;AACV,CAAC;AAED,SAAS,wBAAwB,CAChC,QAAkB,EAClB,UAAuD;IAEvD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACtC,KAAK,MAAM,cAAc,IAAI,KAAK,EAAE,eAAe,IAAI,EAAE,EAAE,CAAC;YAC3D,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACzB,CAAC;IACF,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,SAAS,YAAY,CAAC,IAA6B;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC/B,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1E,OAAO,EAAE,CAAC;IACX,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,mBAAmB,CAC3B,IAA6B;IAE7B,OAAO;QACN,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,IAAI,CAAC,kBAAkB;YAC1B,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,EAAE;YACjD,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,IAAI,CAAC,iBAAiB;YACzB,CAAC,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,EAAE;YAC/C,CAAC,CAAC,EAAE,CAAC;QACN,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC;QAChC,QAAQ,EAAE,IAAI;KACd,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CACpB,KAA2B;IAE3B,OAAO;QACN,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,EAAE;QAClC,eAAe,EAAE,0BAA0B,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC3D,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;KAC1C,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC7B,MAAwC;IAExC,MAAM,cAAc,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7C,MAAM,KAAK,GACV,cAAc,CAAC,IAAI,CAClB,CAAC,SAAS,EAAE,EAAE,CACb,SAAS,CAAC,KAAK,KAAK,WAAW;QAC/B,SAAS,CAAC,IAAI,KAAK,mBAAmB,CACvC;QACD,cAAc,CAAC,IAAI,CAClB,CAAC,SAAS,EAAE,EAAE,CACb,SAAS,CAAC,KAAK,KAAK,QAAQ,IAAI,SAAS,CAAC,KAAK,KAAK,SAAS,CAC9D,CAAC;IACH,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,OAAO;QACN,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,QAAQ,EAAE,IAAI;KACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oCAAoC,CAAC,EACpD,aAAa,EACb,UAAU,EACV,MAAM,EACN,KAAK,GACwC;IAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACrE,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAClD,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,eAAe,EAAE,KAAK,CAAC,OAAO;YAC7B,CAAC,CAAC,wBAAwB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC;YACvD,CAAC,CAAC,0BAA0B,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC7C,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;KAC1C,CAAC,CAAC,CAAC;IACJ,MAAM,aAAa,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACtD,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,eAAe,EAAE,wBAAwB,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC;QACvE,QAAQ,EAAE,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC5C,WAAW,EAAE,OAAO,CAAC,WAAW;KAChC,CAAC,CAAC,CAAC;IACJ,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,MAAM,cAAc,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;YACpD,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,cAAc,EAAE,CAAC,CAAC;QACnD,CAAC;IACF,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QAClC,KAAK,MAAM,cAAc,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;YACpD,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,cAAc,EAAE,CAAC,CAAC;QACnD,CAAC;IACF,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QACrC,KAAK,MAAM,cAAc,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;YACtD,WAAW,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC,CAAC;QACzD,CAAC;IACF,CAAC;IAED,OAAO;QACN,aAAa,EAAE,kCAAkC;QACjD,uBAAuB,EAAE,UAAU,CAAC,aAAa;QACjD,mBAAmB,EAAE,MAAM,CAAC,aAAa;QACzC,kBAAkB,EAAE,KAAK,CAAC,aAAa;QACvC,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,SAAS,EAAE;YACV,OAAO,EAAE,UAAU;YACnB,OAAO,EAAE;gBACR,aAAa;gBACb,oBAAoB;gBACpB,kBAAkB;gBAClB,iBAAiB;gBACjB,0BAA0B;gBAC1B,SAAS;aACT;SACD;QACD,MAAM,EAAE;YACP,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,YAAY,EAAE,YAAY,CAAC,MAAM;YACjC,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;YAClC,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ;YACpC,WAAW,EAAE,WAAW,CAAC,IAAI;SAC7B;QACD,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,aAAa,EAAE,aAAa,CAAC,GAAG,CAAC,mBAAmB,CAAC;QACrD,MAAM;QACN,YAAY;QACZ,aAAa;KACb,CAAC;AACH,CAAC","sourcesContent":["import type { ComposerRunTimelineItem } from \"@evalops/contracts\";\nimport type {\n\tAgentTrajectoryReplayDelta,\n\tAgentTrajectoryReplayReport,\n} from \"./agent-trajectory-replay.js\";\nimport type {\n\tAgentTrajectoryScoreFinding,\n\tAgentTrajectoryScoreReport,\n} from \"./agent-trajectory-scorers.js\";\nimport type {\n\tAgentTrajectoryEvent,\n\tAgentTrajectoryReport,\n} from \"./agent-trajectory.js\";\n\nexport const AGENT_TRAJECTORY_INSPECTION_SCHEMA =\n\t\"evalops.maestro.agent-trajectory-inspection.v1\";\n\ntype EvidenceAnchor = AgentTrajectoryEvent[\"evidence\"][number];\n\nexport interface AgentTrajectoryInspectionEvidenceAnchor\n\textends EvidenceAnchor {\n\tredacted: true;\n\tlabel: string;\n}\n\nexport interface AgentTrajectoryInspectionTimelineItem {\n\tid: string;\n\ttimestamp: string;\n\ttype: string;\n\tstatus: ComposerRunTimelineItem[\"status\"];\n\tvisibility: ComposerRunTimelineItem[\"visibility\"];\n\tsource: ComposerRunTimelineItem[\"source\"];\n\ttitle: string;\n\tsummary?: string;\n\trole?: ComposerRunTimelineItem[\"role\"];\n\ttoolName?: string;\n\tpendingRequestKind?: ComposerRunTimelineItem[\"pendingRequestKind\"];\n\tplatformOperation?: ComposerRunTimelineItem[\"platformOperation\"];\n\tmetadataKeys: string[];\n\tredacted: true;\n}\n\nexport interface AgentTrajectoryInspectionEvent {\n\tid: string;\n\tsequence: number;\n\ttimestamp: string;\n\tkind: AgentTrajectoryEvent[\"kind\"];\n\tphase: AgentTrajectoryEvent[\"phase\"];\n\tactor: AgentTrajectoryEvent[\"actor\"];\n\ttype: string;\n\tstatus: AgentTrajectoryEvent[\"status\"];\n\tvisibility: AgentTrajectoryEvent[\"visibility\"];\n\tsource: AgentTrajectoryEvent[\"source\"];\n\ttitle: string;\n\tsummary?: string;\n\ttoolName?: string;\n\trelatedIds: string[];\n\ttimelineItemIds: string[];\n\tevidence: AgentTrajectoryInspectionEvidenceAnchor[];\n}\n\nexport interface AgentTrajectoryInspectionReplayDelta {\n\tid: string;\n\tseverity: AgentTrajectoryReplayDelta[\"severity\"];\n\truleId: string;\n\tmessage: string;\n\teventId?: string;\n\ttimelineItemIds: string[];\n\tevidence: AgentTrajectoryInspectionEvidenceAnchor[];\n}\n\nexport interface AgentTrajectoryInspectionScoreFinding {\n\truleId: string;\n\tstatus: AgentTrajectoryScoreFinding[\"status\"];\n\tseverity: AgentTrajectoryScoreFinding[\"severity\"];\n\tmessage: string;\n\teventIds: string[];\n\ttimelineItemIds: string[];\n\tevidence: AgentTrajectoryInspectionEvidenceAnchor[];\n\tremediation: string;\n}\n\nexport interface AgentTrajectoryInspectionFinalAnswer {\n\teventId: string;\n\ttimelineItemIds: string[];\n\ttitle: string;\n\tsummary?: string;\n\tredacted: true;\n}\n\nexport interface AgentTrajectoryInspectionReport {\n\tschemaVersion: typeof AGENT_TRAJECTORY_INSPECTION_SCHEMA;\n\ttrajectorySchemaVersion: AgentTrajectoryReport[\"schemaVersion\"];\n\treplaySchemaVersion: AgentTrajectoryReplayReport[\"schemaVersion\"];\n\tscoreSchemaVersion: AgentTrajectoryScoreReport[\"schemaVersion\"];\n\trun: AgentTrajectoryReport[\"run\"];\n\tredaction: {\n\t\tdefault: \"redacted\";\n\t\tomitted: string[];\n\t};\n\tcounts: {\n\t\ttimelineItems: number;\n\t\tevents: number;\n\t\treplayDeltas: number;\n\t\tscoreFindings: number;\n\t\tscoreFailures: number;\n\t\tscoreWarnings: number;\n\t\tjumpTargets: number;\n\t};\n\tfinalAnswer?: AgentTrajectoryInspectionFinalAnswer;\n\ttimelineItems: AgentTrajectoryInspectionTimelineItem[];\n\tevents: AgentTrajectoryInspectionEvent[];\n\treplayDeltas: AgentTrajectoryInspectionReplayDelta[];\n\tscoreFindings: AgentTrajectoryInspectionScoreFinding[];\n}\n\nexport interface BuildAgentTrajectoryInspectionReportOptions {\n\ttimelineItems: ComposerRunTimelineItem[];\n\ttrajectory: AgentTrajectoryReport;\n\treplay: AgentTrajectoryReplayReport;\n\tscore: AgentTrajectoryScoreReport;\n}\n\nfunction redactedEvidence(\n\tevidence: EvidenceAnchor[],\n): AgentTrajectoryInspectionEvidenceAnchor[] {\n\treturn evidence\n\t\t.map((anchor) => ({\n\t\t\t...anchor,\n\t\t\tredacted: true as const,\n\t\t\tlabel: `${anchor.kind}:${anchor.id}`,\n\t\t}))\n\t\t.sort((left, right) => {\n\t\t\tconst kindDelta = left.kind.localeCompare(right.kind);\n\t\t\treturn kindDelta === 0 ? left.id.localeCompare(right.id) : kindDelta;\n\t\t});\n}\n\nfunction timelineItemIdsForEvidence(evidence: EvidenceAnchor[]): string[] {\n\treturn evidence\n\t\t.filter((anchor) => anchor.kind === \"timeline_item\")\n\t\t.map((anchor) => anchor.id)\n\t\t.sort();\n}\n\nfunction timelineItemIdsForEvents(\n\teventIds: string[],\n\teventsById: Map<string, AgentTrajectoryInspectionEvent>,\n): string[] {\n\tconst ids = new Set<string>();\n\tfor (const eventId of eventIds) {\n\t\tconst event = eventsById.get(eventId);\n\t\tfor (const timelineItemId of event?.timelineItemIds ?? []) {\n\t\t\tids.add(timelineItemId);\n\t\t}\n\t}\n\treturn [...ids].sort();\n}\n\nfunction metadataKeys(item: ComposerRunTimelineItem): string[] {\n\tconst metadata = item.metadata;\n\tif (!metadata || typeof metadata !== \"object\" || Array.isArray(metadata)) {\n\t\treturn [];\n\t}\n\treturn Object.keys(metadata).sort();\n}\n\nfunction inspectTimelineItem(\n\titem: ComposerRunTimelineItem,\n): AgentTrajectoryInspectionTimelineItem {\n\treturn {\n\t\tid: item.id,\n\t\ttimestamp: item.timestamp,\n\t\ttype: item.type,\n\t\tstatus: item.status,\n\t\tvisibility: item.visibility,\n\t\tsource: item.source,\n\t\ttitle: item.title,\n\t\t...(item.summary ? { summary: item.summary } : {}),\n\t\t...(item.role ? { role: item.role } : {}),\n\t\t...(item.toolName ? { toolName: item.toolName } : {}),\n\t\t...(item.pendingRequestKind\n\t\t\t? { pendingRequestKind: item.pendingRequestKind }\n\t\t\t: {}),\n\t\t...(item.platformOperation\n\t\t\t? { platformOperation: item.platformOperation }\n\t\t\t: {}),\n\t\tmetadataKeys: metadataKeys(item),\n\t\tredacted: true,\n\t};\n}\n\nfunction inspectEvent(\n\tevent: AgentTrajectoryEvent,\n): AgentTrajectoryInspectionEvent {\n\treturn {\n\t\tid: event.id,\n\t\tsequence: event.sequence,\n\t\ttimestamp: event.timestamp,\n\t\tkind: event.kind,\n\t\tphase: event.phase,\n\t\tactor: event.actor,\n\t\ttype: event.type,\n\t\tstatus: event.status,\n\t\tvisibility: event.visibility,\n\t\tsource: event.source,\n\t\ttitle: event.title,\n\t\t...(event.summary ? { summary: event.summary } : {}),\n\t\t...(event.toolName ? { toolName: event.toolName } : {}),\n\t\trelatedIds: event.relatedIds ?? [],\n\t\ttimelineItemIds: timelineItemIdsForEvidence(event.evidence),\n\t\tevidence: redactedEvidence(event.evidence),\n\t};\n}\n\nfunction finalAnswerFromEvents(\n\tevents: AgentTrajectoryInspectionEvent[],\n): AgentTrajectoryInspectionFinalAnswer | undefined {\n\tconst reversedEvents = [...events].reverse();\n\tconst event =\n\t\treversedEvents.find(\n\t\t\t(candidate) =>\n\t\t\t\tcandidate.actor === \"assistant\" &&\n\t\t\t\tcandidate.type === \"message.assistant\",\n\t\t) ??\n\t\treversedEvents.find(\n\t\t\t(candidate) =>\n\t\t\t\tcandidate.phase === \"finish\" || candidate.phase === \"recover\",\n\t\t);\n\tif (!event) return undefined;\n\treturn {\n\t\teventId: event.id,\n\t\ttimelineItemIds: event.timelineItemIds,\n\t\ttitle: event.title,\n\t\t...(event.summary ? { summary: event.summary } : {}),\n\t\tredacted: true,\n\t};\n}\n\nexport function buildAgentTrajectoryInspectionReport({\n\ttimelineItems,\n\ttrajectory,\n\treplay,\n\tscore,\n}: BuildAgentTrajectoryInspectionReportOptions): AgentTrajectoryInspectionReport {\n\tconst events = trajectory.events.map(inspectEvent);\n\tconst finalAnswer = finalAnswerFromEvents(events);\n\tconst eventsById = new Map(events.map((event) => [event.id, event]));\n\tconst replayDeltas = replay.deltas.map((delta) => ({\n\t\tid: delta.id,\n\t\tseverity: delta.severity,\n\t\truleId: delta.ruleId,\n\t\tmessage: delta.message,\n\t\t...(delta.eventId ? { eventId: delta.eventId } : {}),\n\t\ttimelineItemIds: delta.eventId\n\t\t\t? timelineItemIdsForEvents([delta.eventId], eventsById)\n\t\t\t: timelineItemIdsForEvidence(delta.evidence),\n\t\tevidence: redactedEvidence(delta.evidence),\n\t}));\n\tconst scoreFindings = score.findings.map((finding) => ({\n\t\truleId: finding.ruleId,\n\t\tstatus: finding.status,\n\t\tseverity: finding.severity,\n\t\tmessage: finding.message,\n\t\teventIds: finding.eventIds,\n\t\ttimelineItemIds: timelineItemIdsForEvents(finding.eventIds, eventsById),\n\t\tevidence: redactedEvidence(finding.evidence),\n\t\tremediation: finding.remediation,\n\t}));\n\tconst jumpTargets = new Set<string>();\n\tfor (const event of events) {\n\t\tfor (const timelineItemId of event.timelineItemIds) {\n\t\t\tjumpTargets.add(`${event.id}->${timelineItemId}`);\n\t\t}\n\t}\n\tfor (const delta of replayDeltas) {\n\t\tfor (const timelineItemId of delta.timelineItemIds) {\n\t\t\tjumpTargets.add(`${delta.id}->${timelineItemId}`);\n\t\t}\n\t}\n\tfor (const finding of scoreFindings) {\n\t\tfor (const timelineItemId of finding.timelineItemIds) {\n\t\t\tjumpTargets.add(`${finding.ruleId}->${timelineItemId}`);\n\t\t}\n\t}\n\n\treturn {\n\t\tschemaVersion: AGENT_TRAJECTORY_INSPECTION_SCHEMA,\n\t\ttrajectorySchemaVersion: trajectory.schemaVersion,\n\t\treplaySchemaVersion: replay.schemaVersion,\n\t\tscoreSchemaVersion: score.schemaVersion,\n\t\trun: trajectory.run,\n\t\tredaction: {\n\t\t\tdefault: \"redacted\",\n\t\t\tomitted: [\n\t\t\t\t\"raw prompts\",\n\t\t\t\t\"raw tool arguments\",\n\t\t\t\t\"raw tool outputs\",\n\t\t\t\t\"full file diffs\",\n\t\t\t\t\"timeline metadata values\",\n\t\t\t\t\"secrets\",\n\t\t\t],\n\t\t},\n\t\tcounts: {\n\t\t\ttimelineItems: timelineItems.length,\n\t\t\tevents: events.length,\n\t\t\treplayDeltas: replayDeltas.length,\n\t\t\tscoreFindings: scoreFindings.length,\n\t\t\tscoreFailures: score.counts.failed,\n\t\t\tscoreWarnings: score.counts.warnings,\n\t\t\tjumpTargets: jumpTargets.size,\n\t\t},\n\t\t...(finalAnswer ? { finalAnswer } : {}),\n\t\ttimelineItems: timelineItems.map(inspectTimelineItem),\n\t\tevents,\n\t\treplayDeltas,\n\t\tscoreFindings,\n\t};\n}\n"]}

package/dist/server/agent-trajectory-replay.d.ts

@@ -0,0 +1,56 @@
+import type { AgentTrajectoryEvent, AgentTrajectoryEventKind, AgentTrajectoryPhase, AgentTrajectoryReport } from "./agent-trajectory.js";
+export declare const AGENT_TRAJECTORY_REPLAY_SCHEMA = "evalops.maestro.agent-trajectory-replay.v1";
+export type AgentTrajectoryReplaySeverity = "error" | "warning";
+export interface AgentTrajectoryReplayDelta {
+    id: string;
+    severity: AgentTrajectoryReplaySeverity;
+    ruleId: string;
+    sequence?: number;
+    eventId?: string;
+    phase?: AgentTrajectoryPhase;
+    kind?: AgentTrajectoryEventKind;
+    expected?: string;
+    observed?: string;
+    message: string;
+    evidence: AgentTrajectoryEvent["evidence"];
+}
+export interface AgentTrajectoryReplayPhaseSummary {
+    phase: AgentTrajectoryPhase;
+    events: number;
+    firstSequence: number;
+    lastSequence: number;
+}
+export interface AgentTrajectoryReplayToolSummary {
+    toolCallId: string;
+    toolName?: string;
+    requestedSequence?: number;
+    resultSequences: number[];
+    terminalStatus?: "completed" | "failed";
+    evidence: AgentTrajectoryEvent["evidence"];
+}
+export interface AgentTrajectoryReplayReport {
+    schemaVersion: typeof AGENT_TRAJECTORY_REPLAY_SCHEMA;
+    trajectorySchemaVersion: AgentTrajectoryReport["schemaVersion"];
+    run: AgentTrajectoryReport["run"];
+    deterministic: true;
+    counts: {
+        events: number;
+        deltas: number;
+        errors: number;
+        warnings: number;
+        toolCalls: number;
+        phases: number;
+    };
+    phases: AgentTrajectoryReplayPhaseSummary[];
+    toolCalls: AgentTrajectoryReplayToolSummary[];
+    deltas: AgentTrajectoryReplayDelta[];
+}
+export interface AgentTrajectoryReplayToolExpectation {
+    terminalStatus?: "completed" | "failed";
+    requiredArtifactIds?: string[];
+}
+export interface ReplayAgentTrajectoryOptions {
+    expectedTools?: Record<string, AgentTrajectoryReplayToolExpectation>;
+}
+export declare function replayAgentTrajectoryReport(report: AgentTrajectoryReport, options?: ReplayAgentTrajectoryOptions): AgentTrajectoryReplayReport;
+//# sourceMappingURL=agent-trajectory-replay.d.ts.map

package/dist/server/agent-trajectory-replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-trajectory-replay.d.ts","sourceRoot":"","sources":["../../src/server/agent-trajectory-replay.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACX,oBAAoB,EACpB,wBAAwB,EACxB,oBAAoB,EACpB,qBAAqB,EACrB,MAAM,uBAAuB,CAAC;AAE/B,eAAO,MAAM,8BAA8B,+CACE,CAAC;AAE9C,MAAM,MAAM,6BAA6B,GAAG,OAAO,GAAG,SAAS,CAAC;AAEhE,MAAM,WAAW,0BAA0B;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,6BAA6B,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,IAAI,CAAC,EAAE,wBAAwB,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,oBAAoB,CAAC,UAAU,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,iCAAiC;IACjD,KAAK,EAAE,oBAAoB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gCAAgC;IAChD,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,cAAc,CAAC,EAAE,WAAW,GAAG,QAAQ,CAAC;IACxC,QAAQ,EAAE,oBAAoB,CAAC,UAAU,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,2BAA2B;IAC3C,aAAa,EAAE,OAAO,8BAA8B,CAAC;IACrD,uBAAuB,EAAE,qBAAqB,CAAC,eAAe,CAAC,CAAC;IAChE,GAAG,EAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAClC,aAAa,EAAE,IAAI,CAAC;IACpB,MAAM,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,iCAAiC,EAAE,CAAC;IAC5C,SAAS,EAAE,gCAAgC,EAAE,CAAC;IAC9C,MAAM,EAAE,0BAA0B,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,oCAAoC;IACpD,cAAc,CAAC,EAAE,WAAW,GAAG,QAAQ,CAAC;IACxC,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,4BAA4B;IAC5C,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,oCAAoC,CAAC,CAAC;CACrE;AA+HD,wBAAgB,2BAA2B,CAC1C,MAAM,EAAE,qBAAqB,EAC7B,OAAO,GAAE,4BAAiC,GACxC,2BAA2B,CAsI7B"}

package/dist/server/agent-trajectory-replay.js

@@ -0,0 +1,214 @@
+import { validateAgentTrajectoryReport } from "./agent-trajectory-validation.js";
+export const AGENT_TRAJECTORY_REPLAY_SCHEMA = "evalops.maestro.agent-trajectory-replay.v1";
+function evidenceIds(event, kind) {
+    return event.evidence
+        .filter((anchor) => anchor.kind === kind)
+        .map((anchor) => anchor.id)
+        .sort();
+}
+function firstEvidenceId(event, kind) {
+    return evidenceIds(event, kind)[0];
+}
+function mergeEvidence(target, source) {
+    const seen = new Set(target.map((anchor) => `${anchor.kind}:${anchor.id}`));
+    for (const anchor of source) {
+        const key = `${anchor.kind}:${anchor.id}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        target.push(anchor);
+    }
+    return target.sort((a, b) => {
+        const kindDelta = a.kind.localeCompare(b.kind);
+        return kindDelta === 0 ? a.id.localeCompare(b.id) : kindDelta;
+    });
+}
+function delta(deltas, params) {
+    deltas.push({
+        id: `delta:${String(deltas.length + 1).padStart(3, "0")}`,
+        ...params,
+    });
+}
+function ensureToolState(tools, toolCallId, event) {
+    let state = tools.get(toolCallId);
+    if (!state) {
+        state = {
+            toolCallId,
+            toolName: event.toolName,
+            resultSequences: [],
+            evidence: [],
+            artifactIds: new Set(),
+        };
+        tools.set(toolCallId, state);
+    }
+    if (!state.toolName && event.toolName) {
+        state.toolName = event.toolName;
+    }
+    mergeEvidence(state.evidence, event.evidence);
+    return state;
+}
+function summarizePhases(events) {
+    const byPhase = new Map();
+    for (const event of events) {
+        const existing = byPhase.get(event.phase);
+        if (existing) {
+            existing.events += 1;
+            existing.lastSequence = event.sequence;
+            continue;
+        }
+        byPhase.set(event.phase, {
+            phase: event.phase,
+            events: 1,
+            firstSequence: event.sequence,
+            lastSequence: event.sequence,
+        });
+    }
+    return [...byPhase.values()].sort((a, b) => a.firstSequence - b.firstSequence);
+}
+function summarizeTools(tools) {
+    return [...tools.values()]
+        .map((tool) => ({
+        toolCallId: tool.toolCallId,
+        ...(tool.toolName ? { toolName: tool.toolName } : {}),
+        ...(tool.requestedSequence !== undefined
+            ? { requestedSequence: tool.requestedSequence }
+            : {}),
+        resultSequences: [...tool.resultSequences].sort((a, b) => a - b),
+        ...(tool.terminalStatus ? { terminalStatus: tool.terminalStatus } : {}),
+        evidence: [...tool.evidence],
+    }))
+        .sort((a, b) => {
+        const sequenceA = a.requestedSequence ?? Number.MAX_SAFE_INTEGER;
+        const sequenceB = b.requestedSequence ?? Number.MAX_SAFE_INTEGER;
+        if (sequenceA !== sequenceB)
+            return sequenceA - sequenceB;
+        return a.toolCallId.localeCompare(b.toolCallId);
+    });
+}
+export function replayAgentTrajectoryReport(report, options = {}) {
+    const deltas = [];
+    const validation = validateAgentTrajectoryReport(report);
+    for (const failure of validation.failures) {
+        delta(deltas, {
+            severity: "error",
+            ruleId: "trajectory.validation",
+            message: failure,
+            evidence: [],
+        });
+    }
+    const tools = new Map();
+    for (const event of report.events) {
+        const toolCallIds = evidenceIds(event, "tool_call");
+        const artifactIds = evidenceIds(event, "artifact");
+        if (event.type === "tool.requested") {
+            const toolCallId = firstEvidenceId(event, "tool_call");
+            if (!toolCallId)
+                continue;
+            const state = ensureToolState(tools, toolCallId, event);
+            if (state.requestedSequence !== undefined) {
+                delta(deltas, {
+                    severity: "error",
+                    ruleId: "tool.duplicate_request",
+                    sequence: event.sequence,
+                    eventId: event.id,
+                    phase: event.phase,
+                    kind: event.kind,
+                    expected: "one tool.requested per tool call",
+                    observed: `previous request at sequence ${state.requestedSequence}`,
+                    message: `Tool call ${toolCallId} was requested more than once.`,
+                    evidence: event.evidence,
+                });
+            }
+            state.requestedSequence = event.sequence;
+        }
+        if (event.type === "tool.completed" || event.type === "tool.failed") {
+            const toolCallId = firstEvidenceId(event, "tool_call");
+            if (!toolCallId)
+                continue;
+            const state = ensureToolState(tools, toolCallId, event);
+            const terminalStatus = event.type === "tool.completed" ? "completed" : "failed";
+            if (state.terminalStatus !== undefined &&
+                state.terminalStatus !== terminalStatus) {
+                delta(deltas, {
+                    severity: "error",
+                    ruleId: "tool.conflicting_terminal_status",
+                    sequence: event.sequence,
+                    eventId: event.id,
+                    phase: event.phase,
+                    kind: event.kind,
+                    expected: state.terminalStatus,
+                    observed: terminalStatus,
+                    message: `Tool call ${toolCallId} has conflicting terminal statuses.`,
+                    evidence: event.evidence,
+                });
+            }
+            state.terminalStatus = terminalStatus;
+            state.resultSequences.push(event.sequence);
+        }
+        for (const toolCallId of toolCallIds) {
+            const state = ensureToolState(tools, toolCallId, event);
+            for (const artifactId of artifactIds) {
+                state.artifactIds.add(artifactId);
+            }
+        }
+    }
+    for (const [toolCallId, expectation] of Object.entries(options.expectedTools ?? {}).sort(([left], [right]) => left.localeCompare(right))) {
+        const state = tools.get(toolCallId);
+        if (!state) {
+            delta(deltas, {
+                severity: "error",
+                ruleId: "tool.expected_missing",
+                expected: toolCallId,
+                observed: "missing",
+                message: `Expected tool call ${toolCallId} was not present in the trajectory.`,
+                evidence: [],
+            });
+            continue;
+        }
+        if (expectation.terminalStatus &&
+            state.terminalStatus !== expectation.terminalStatus) {
+            delta(deltas, {
+                severity: "error",
+                ruleId: "tool.terminal_status_mismatch",
+                expected: expectation.terminalStatus,
+                observed: state.terminalStatus ?? "missing",
+                message: `Tool call ${toolCallId} terminal status did not match replay expectation.`,
+                evidence: state.evidence,
+            });
+        }
+        for (const artifactId of expectation.requiredArtifactIds ?? []) {
+            if (state.artifactIds.has(artifactId))
+                continue;
+            delta(deltas, {
+                severity: "error",
+                ruleId: "tool.required_artifact_missing",
+                expected: artifactId,
+                observed: "missing",
+                message: `Tool call ${toolCallId} did not produce required artifact ${artifactId}.`,
+                evidence: state.evidence,
+            });
+        }
+    }
+    const errors = deltas.filter((item) => item.severity === "error").length;
+    const warnings = deltas.filter((item) => item.severity === "warning").length;
+    const phases = summarizePhases(report.events);
+    const toolCalls = summarizeTools(tools);
+    return {
+        schemaVersion: AGENT_TRAJECTORY_REPLAY_SCHEMA,
+        trajectorySchemaVersion: report.schemaVersion,
+        run: report.run,
+        deterministic: true,
+        counts: {
+            events: report.events.length,
+            deltas: deltas.length,
+            errors,
+            warnings,
+            toolCalls: toolCalls.length,
+            phases: phases.length,
+        },
+        phases,
+        toolCalls,
+        deltas,
+    };
+}
+//# sourceMappingURL=agent-trajectory-replay.js.map

package/dist/server/agent-trajectory-replay.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-trajectory-replay.js","sourceRoot":"","sources":["../../src/server/agent-trajectory-replay.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,kCAAkC,CAAC;AAQjF,MAAM,CAAC,MAAM,8BAA8B,GAC1C,4CAA4C,CAAC;AAuE9C,SAAS,WAAW,CACnB,KAA2B,EAC3B,IAAsD;IAEtD,OAAO,KAAK,CAAC,QAAQ;SACnB,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC;SACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;SAC1B,IAAI,EAAE,CAAC;AACV,CAAC;AAED,SAAS,eAAe,CACvB,KAA2B,EAC3B,IAAsD;IAEtD,OAAO,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,aAAa,CACrB,MAAwC,EACxC,MAAwC;IAExC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC5E,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QAC1C,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC3B,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/C,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/D,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CACb,MAAoC,EACpC,MAA8C;IAE9C,MAAM,CAAC,IAAI,CAAC;QACX,EAAE,EAAE,SAAS,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;QACzD,GAAG,MAAM;KACT,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACvB,KAA6B,EAC7B,UAAkB,EAClB,KAA2B;IAE3B,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAClC,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,KAAK,GAAG;YACP,UAAU;YACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,eAAe,EAAE,EAAE;YACnB,QAAQ,EAAE,EAAE;YACZ,WAAW,EAAE,IAAI,GAAG,EAAU;SAC9B,CAAC;QACF,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IACjC,CAAC;IACD,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9C,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CACvB,MAA8B;IAE9B,MAAM,OAAO,GAAG,IAAI,GAAG,EAGpB,CAAC;IACJ,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,QAAQ,EAAE,CAAC;YACd,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;YACrB,QAAQ,CAAC,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC;YACvC,SAAS;QACV,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,CAAC;YACT,aAAa,EAAE,KAAK,CAAC,QAAQ;YAC7B,YAAY,EAAE,KAAK,CAAC,QAAQ;SAC5B,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAChC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAC3C,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACtB,KAA6B;IAE7B,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;SACxB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACf,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,IAAI,CAAC,iBAAiB,KAAK,SAAS;YACvC,CAAC,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,EAAE;YAC/C,CAAC,CAAC,EAAE,CAAC;QACN,eAAe,EAAE,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QAChE,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;KAC5B,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,SAAS,GAAG,CAAC,CAAC,iBAAiB,IAAI,MAAM,CAAC,gBAAgB,CAAC;QACjE,MAAM,SAAS,GAAG,CAAC,CAAC,iBAAiB,IAAI,MAAM,CAAC,gBAAgB,CAAC;QACjE,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,SAAS,GAAG,SAAS,CAAC;QAC1D,OAAO,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,2BAA2B,CAC1C,MAA6B,EAC7B,UAAwC,EAAE;IAE1C,MAAM,MAAM,GAAiC,EAAE,CAAC;IAChD,MAAM,UAAU,GAAG,6BAA6B,CAAC,MAAM,CAAC,CAAC;IACzD,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,CAAC,MAAM,EAAE;YACb,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,uBAAuB;YAC/B,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,EAAE;SACZ,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YACvD,IAAI,CAAC,UAAU;gBAAE,SAAS;YAC1B,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACxD,IAAI,KAAK,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;gBAC3C,KAAK,CAAC,MAAM,EAAE;oBACb,QAAQ,EAAE,OAAO;oBACjB,MAAM,EAAE,wBAAwB;oBAChC,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,QAAQ,EAAE,kCAAkC;oBAC5C,QAAQ,EAAE,gCAAgC,KAAK,CAAC,iBAAiB,EAAE;oBACnE,OAAO,EAAE,aAAa,UAAU,gCAAgC;oBAChE,QAAQ,EAAE,KAAK,CAAC,QAAQ;iBACxB,CAAC,CAAC;YACJ,CAAC;YACD,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC1C,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACrE,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YACvD,IAAI,CAAC,UAAU;gBAAE,SAAS;YAC1B,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACxD,MAAM,cAAc,GACnB,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;YAC1D,IACC,KAAK,CAAC,cAAc,KAAK,SAAS;gBAClC,KAAK,CAAC,cAAc,KAAK,cAAc,EACtC,CAAC;gBACF,KAAK,CAAC,MAAM,EAAE;oBACb,QAAQ,EAAE,OAAO;oBACjB,MAAM,EAAE,kCAAkC;oBAC1C,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,QAAQ,EAAE,KAAK,CAAC,cAAc;oBAC9B,QAAQ,EAAE,cAAc;oBACxB,OAAO,EAAE,aAAa,UAAU,qCAAqC;oBACrE,QAAQ,EAAE,KAAK,CAAC,QAAQ;iBACxB,CAAC,CAAC;YACJ,CAAC;YACD,KAAK,CAAC,cAAc,GAAG,cAAc,CAAC;YACtC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACxD,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACtC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACnC,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CACrD,OAAO,CAAC,aAAa,IAAI,EAAE,CAC3B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,CAAC,MAAM,EAAE;gBACb,QAAQ,EAAE,OAAO;gBACjB,MAAM,EAAE,uBAAuB;gBAC/B,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,SAAS;gBACnB,OAAO,EAAE,sBAAsB,UAAU,qCAAqC;gBAC9E,QAAQ,EAAE,EAAE;aACZ,CAAC,CAAC;YACH,SAAS;QACV,CAAC;QACD,IACC,WAAW,CAAC,cAAc;YAC1B,KAAK,CAAC,cAAc,KAAK,WAAW,CAAC,cAAc,EAClD,CAAC;YACF,KAAK,CAAC,MAAM,EAAE;gBACb,QAAQ,EAAE,OAAO;gBACjB,MAAM,EAAE,+BAA+B;gBACvC,QAAQ,EAAE,WAAW,CAAC,cAAc;gBACpC,QAAQ,EAAE,KAAK,CAAC,cAAc,IAAI,SAAS;gBAC3C,OAAO,EAAE,aAAa,UAAU,oDAAoD;gBACpF,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACxB,CAAC,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,WAAW,CAAC,mBAAmB,IAAI,EAAE,EAAE,CAAC;YAChE,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,SAAS;YAChD,KAAK,CAAC,MAAM,EAAE;gBACb,QAAQ,EAAE,OAAO;gBACjB,MAAM,EAAE,gCAAgC;gBACxC,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,SAAS;gBACnB,OAAO,EAAE,aAAa,UAAU,sCAAsC,UAAU,GAAG;gBACnF,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACxB,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAC7E,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACxC,OAAO;QACN,aAAa,EAAE,8BAA8B;QAC7C,uBAAuB,EAAE,MAAM,CAAC,aAAa;QAC7C,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,aAAa,EAAE,IAAI;QACnB,MAAM,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM;YACN,QAAQ;YACR,SAAS,EAAE,SAAS,CAAC,MAAM;YAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;SACrB;QACD,MAAM;QACN,SAAS;QACT,MAAM;KACN,CAAC;AACH,CAAC","sourcesContent":["import { validateAgentTrajectoryReport } from \"./agent-trajectory-validation.js\";\nimport type {\n\tAgentTrajectoryEvent,\n\tAgentTrajectoryEventKind,\n\tAgentTrajectoryPhase,\n\tAgentTrajectoryReport,\n} from \"./agent-trajectory.js\";\n\nexport const AGENT_TRAJECTORY_REPLAY_SCHEMA =\n\t\"evalops.maestro.agent-trajectory-replay.v1\";\n\nexport type AgentTrajectoryReplaySeverity = \"error\" | \"warning\";\n\nexport interface AgentTrajectoryReplayDelta {\n\tid: string;\n\tseverity: AgentTrajectoryReplaySeverity;\n\truleId: string;\n\tsequence?: number;\n\teventId?: string;\n\tphase?: AgentTrajectoryPhase;\n\tkind?: AgentTrajectoryEventKind;\n\texpected?: string;\n\tobserved?: string;\n\tmessage: string;\n\tevidence: AgentTrajectoryEvent[\"evidence\"];\n}\n\nexport interface AgentTrajectoryReplayPhaseSummary {\n\tphase: AgentTrajectoryPhase;\n\tevents: number;\n\tfirstSequence: number;\n\tlastSequence: number;\n}\n\nexport interface AgentTrajectoryReplayToolSummary {\n\ttoolCallId: string;\n\ttoolName?: string;\n\trequestedSequence?: number;\n\tresultSequences: number[];\n\tterminalStatus?: \"completed\" | \"failed\";\n\tevidence: AgentTrajectoryEvent[\"evidence\"];\n}\n\nexport interface AgentTrajectoryReplayReport {\n\tschemaVersion: typeof AGENT_TRAJECTORY_REPLAY_SCHEMA;\n\ttrajectorySchemaVersion: AgentTrajectoryReport[\"schemaVersion\"];\n\trun: AgentTrajectoryReport[\"run\"];\n\tdeterministic: true;\n\tcounts: {\n\t\tevents: number;\n\t\tdeltas: number;\n\t\terrors: number;\n\t\twarnings: number;\n\t\ttoolCalls: number;\n\t\tphases: number;\n\t};\n\tphases: AgentTrajectoryReplayPhaseSummary[];\n\ttoolCalls: AgentTrajectoryReplayToolSummary[];\n\tdeltas: AgentTrajectoryReplayDelta[];\n}\n\nexport interface AgentTrajectoryReplayToolExpectation {\n\tterminalStatus?: \"completed\" | \"failed\";\n\trequiredArtifactIds?: string[];\n}\n\nexport interface ReplayAgentTrajectoryOptions {\n\texpectedTools?: Record<string, AgentTrajectoryReplayToolExpectation>;\n}\n\ninterface ToolState {\n\ttoolCallId: string;\n\ttoolName?: string;\n\trequestedSequence?: number;\n\tresultSequences: number[];\n\tterminalStatus?: \"completed\" | \"failed\";\n\tevidence: AgentTrajectoryEvent[\"evidence\"];\n\tartifactIds: Set<string>;\n}\n\nfunction evidenceIds(\n\tevent: AgentTrajectoryEvent,\n\tkind: AgentTrajectoryEvent[\"evidence\"][number][\"kind\"],\n): string[] {\n\treturn event.evidence\n\t\t.filter((anchor) => anchor.kind === kind)\n\t\t.map((anchor) => anchor.id)\n\t\t.sort();\n}\n\nfunction firstEvidenceId(\n\tevent: AgentTrajectoryEvent,\n\tkind: AgentTrajectoryEvent[\"evidence\"][number][\"kind\"],\n): string | undefined {\n\treturn evidenceIds(event, kind)[0];\n}\n\nfunction mergeEvidence(\n\ttarget: AgentTrajectoryEvent[\"evidence\"],\n\tsource: AgentTrajectoryEvent[\"evidence\"],\n): AgentTrajectoryEvent[\"evidence\"] {\n\tconst seen = new Set(target.map((anchor) => `${anchor.kind}:${anchor.id}`));\n\tfor (const anchor of source) {\n\t\tconst key = `${anchor.kind}:${anchor.id}`;\n\t\tif (seen.has(key)) continue;\n\t\tseen.add(key);\n\t\ttarget.push(anchor);\n\t}\n\treturn target.sort((a, b) => {\n\t\tconst kindDelta = a.kind.localeCompare(b.kind);\n\t\treturn kindDelta === 0 ? a.id.localeCompare(b.id) : kindDelta;\n\t});\n}\n\nfunction delta(\n\tdeltas: AgentTrajectoryReplayDelta[],\n\tparams: Omit<AgentTrajectoryReplayDelta, \"id\">,\n): void {\n\tdeltas.push({\n\t\tid: `delta:${String(deltas.length + 1).padStart(3, \"0\")}`,\n\t\t...params,\n\t});\n}\n\nfunction ensureToolState(\n\ttools: Map<string, ToolState>,\n\ttoolCallId: string,\n\tevent: AgentTrajectoryEvent,\n): ToolState {\n\tlet state = tools.get(toolCallId);\n\tif (!state) {\n\t\tstate = {\n\t\t\ttoolCallId,\n\t\t\ttoolName: event.toolName,\n\t\t\tresultSequences: [],\n\t\t\tevidence: [],\n\t\t\tartifactIds: new Set<string>(),\n\t\t};\n\t\ttools.set(toolCallId, state);\n\t}\n\tif (!state.toolName && event.toolName) {\n\t\tstate.toolName = event.toolName;\n\t}\n\tmergeEvidence(state.evidence, event.evidence);\n\treturn state;\n}\n\nfunction summarizePhases(\n\tevents: AgentTrajectoryEvent[],\n): AgentTrajectoryReplayPhaseSummary[] {\n\tconst byPhase = new Map<\n\t\tAgentTrajectoryPhase,\n\t\tAgentTrajectoryReplayPhaseSummary\n\t>();\n\tfor (const event of events) {\n\t\tconst existing = byPhase.get(event.phase);\n\t\tif (existing) {\n\t\t\texisting.events += 1;\n\t\t\texisting.lastSequence = event.sequence;\n\t\t\tcontinue;\n\t\t}\n\t\tbyPhase.set(event.phase, {\n\t\t\tphase: event.phase,\n\t\t\tevents: 1,\n\t\t\tfirstSequence: event.sequence,\n\t\t\tlastSequence: event.sequence,\n\t\t});\n\t}\n\treturn [...byPhase.values()].sort(\n\t\t(a, b) => a.firstSequence - b.firstSequence,\n\t);\n}\n\nfunction summarizeTools(\n\ttools: Map<string, ToolState>,\n): AgentTrajectoryReplayToolSummary[] {\n\treturn [...tools.values()]\n\t\t.map((tool) => ({\n\t\t\ttoolCallId: tool.toolCallId,\n\t\t\t...(tool.toolName ? { toolName: tool.toolName } : {}),\n\t\t\t...(tool.requestedSequence !== undefined\n\t\t\t\t? { requestedSequence: tool.requestedSequence }\n\t\t\t\t: {}),\n\t\t\tresultSequences: [...tool.resultSequences].sort((a, b) => a - b),\n\t\t\t...(tool.terminalStatus ? { terminalStatus: tool.terminalStatus } : {}),\n\t\t\tevidence: [...tool.evidence],\n\t\t}))\n\t\t.sort((a, b) => {\n\t\t\tconst sequenceA = a.requestedSequence ?? Number.MAX_SAFE_INTEGER;\n\t\t\tconst sequenceB = b.requestedSequence ?? Number.MAX_SAFE_INTEGER;\n\t\t\tif (sequenceA !== sequenceB) return sequenceA - sequenceB;\n\t\t\treturn a.toolCallId.localeCompare(b.toolCallId);\n\t\t});\n}\n\nexport function replayAgentTrajectoryReport(\n\treport: AgentTrajectoryReport,\n\toptions: ReplayAgentTrajectoryOptions = {},\n): AgentTrajectoryReplayReport {\n\tconst deltas: AgentTrajectoryReplayDelta[] = [];\n\tconst validation = validateAgentTrajectoryReport(report);\n\tfor (const failure of validation.failures) {\n\t\tdelta(deltas, {\n\t\t\tseverity: \"error\",\n\t\t\truleId: \"trajectory.validation\",\n\t\t\tmessage: failure,\n\t\t\tevidence: [],\n\t\t});\n\t}\n\n\tconst tools = new Map<string, ToolState>();\n\tfor (const event of report.events) {\n\t\tconst toolCallIds = evidenceIds(event, \"tool_call\");\n\t\tconst artifactIds = evidenceIds(event, \"artifact\");\n\t\tif (event.type === \"tool.requested\") {\n\t\t\tconst toolCallId = firstEvidenceId(event, \"tool_call\");\n\t\t\tif (!toolCallId) continue;\n\t\t\tconst state = ensureToolState(tools, toolCallId, event);\n\t\t\tif (state.requestedSequence !== undefined) {\n\t\t\t\tdelta(deltas, {\n\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\truleId: \"tool.duplicate_request\",\n\t\t\t\t\tsequence: event.sequence,\n\t\t\t\t\teventId: event.id,\n\t\t\t\t\tphase: event.phase,\n\t\t\t\t\tkind: event.kind,\n\t\t\t\t\texpected: \"one tool.requested per tool call\",\n\t\t\t\t\tobserved: `previous request at sequence ${state.requestedSequence}`,\n\t\t\t\t\tmessage: `Tool call ${toolCallId} was requested more than once.`,\n\t\t\t\t\tevidence: event.evidence,\n\t\t\t\t});\n\t\t\t}\n\t\t\tstate.requestedSequence = event.sequence;\n\t\t}\n\n\t\tif (event.type === \"tool.completed\" || event.type === \"tool.failed\") {\n\t\t\tconst toolCallId = firstEvidenceId(event, \"tool_call\");\n\t\t\tif (!toolCallId) continue;\n\t\t\tconst state = ensureToolState(tools, toolCallId, event);\n\t\t\tconst terminalStatus =\n\t\t\t\tevent.type === \"tool.completed\" ? \"completed\" : \"failed\";\n\t\t\tif (\n\t\t\t\tstate.terminalStatus !== undefined &&\n\t\t\t\tstate.terminalStatus !== terminalStatus\n\t\t\t) {\n\t\t\t\tdelta(deltas, {\n\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\truleId: \"tool.conflicting_terminal_status\",\n\t\t\t\t\tsequence: event.sequence,\n\t\t\t\t\teventId: event.id,\n\t\t\t\t\tphase: event.phase,\n\t\t\t\t\tkind: event.kind,\n\t\t\t\t\texpected: state.terminalStatus,\n\t\t\t\t\tobserved: terminalStatus,\n\t\t\t\t\tmessage: `Tool call ${toolCallId} has conflicting terminal statuses.`,\n\t\t\t\t\tevidence: event.evidence,\n\t\t\t\t});\n\t\t\t}\n\t\t\tstate.terminalStatus = terminalStatus;\n\t\t\tstate.resultSequences.push(event.sequence);\n\t\t}\n\n\t\tfor (const toolCallId of toolCallIds) {\n\t\t\tconst state = ensureToolState(tools, toolCallId, event);\n\t\t\tfor (const artifactId of artifactIds) {\n\t\t\t\tstate.artifactIds.add(artifactId);\n\t\t\t}\n\t\t}\n\t}\n\n\tfor (const [toolCallId, expectation] of Object.entries(\n\t\toptions.expectedTools ?? {},\n\t).sort(([left], [right]) => left.localeCompare(right))) {\n\t\tconst state = tools.get(toolCallId);\n\t\tif (!state) {\n\t\t\tdelta(deltas, {\n\t\t\t\tseverity: \"error\",\n\t\t\t\truleId: \"tool.expected_missing\",\n\t\t\t\texpected: toolCallId,\n\t\t\t\tobserved: \"missing\",\n\t\t\t\tmessage: `Expected tool call ${toolCallId} was not present in the trajectory.`,\n\t\t\t\tevidence: [],\n\t\t\t});\n\t\t\tcontinue;\n\t\t}\n\t\tif (\n\t\t\texpectation.terminalStatus &&\n\t\t\tstate.terminalStatus !== expectation.terminalStatus\n\t\t) {\n\t\t\tdelta(deltas, {\n\t\t\t\tseverity: \"error\",\n\t\t\t\truleId: \"tool.terminal_status_mismatch\",\n\t\t\t\texpected: expectation.terminalStatus,\n\t\t\t\tobserved: state.terminalStatus ?? \"missing\",\n\t\t\t\tmessage: `Tool call ${toolCallId} terminal status did not match replay expectation.`,\n\t\t\t\tevidence: state.evidence,\n\t\t\t});\n\t\t}\n\t\tfor (const artifactId of expectation.requiredArtifactIds ?? []) {\n\t\t\tif (state.artifactIds.has(artifactId)) continue;\n\t\t\tdelta(deltas, {\n\t\t\t\tseverity: \"error\",\n\t\t\t\truleId: \"tool.required_artifact_missing\",\n\t\t\t\texpected: artifactId,\n\t\t\t\tobserved: \"missing\",\n\t\t\t\tmessage: `Tool call ${toolCallId} did not produce required artifact ${artifactId}.`,\n\t\t\t\tevidence: state.evidence,\n\t\t\t});\n\t\t}\n\t}\n\n\tconst errors = deltas.filter((item) => item.severity === \"error\").length;\n\tconst warnings = deltas.filter((item) => item.severity === \"warning\").length;\n\tconst phases = summarizePhases(report.events);\n\tconst toolCalls = summarizeTools(tools);\n\treturn {\n\t\tschemaVersion: AGENT_TRAJECTORY_REPLAY_SCHEMA,\n\t\ttrajectorySchemaVersion: report.schemaVersion,\n\t\trun: report.run,\n\t\tdeterministic: true,\n\t\tcounts: {\n\t\t\tevents: report.events.length,\n\t\t\tdeltas: deltas.length,\n\t\t\terrors,\n\t\t\twarnings,\n\t\t\ttoolCalls: toolCalls.length,\n\t\t\tphases: phases.length,\n\t\t},\n\t\tphases,\n\t\ttoolCalls,\n\t\tdeltas,\n\t};\n}\n"]}

package/dist/server/agent-trajectory-scenarios.d.ts

@@ -0,0 +1,109 @@
+import { MAESTRO_SCENARIO_SCHEMA, type MaestroScenario, type MaestroScenarioAssertion, type MaestroScenarioExternalRefs, type MaestroScenarioOutcome, type MaestroScenarioPlatformLink, type MaestroScenarioReleaseGate, type MaestroScenarioRequiredArtifact, type MaestroScenarioReviewLabel, type MaestroScenarioSeverity, type MaestroScenarioWorkspaceHydrationMode, type MaestroScenarioWorkspaceManifest } from "@evalops/contracts";
+import type { AgentTrajectoryInspectionReport } from "./agent-trajectory-inspection.js";
+import type { AgentTrajectoryReplayReport } from "./agent-trajectory-replay.js";
+import type { AgentTrajectoryScoreReport } from "./agent-trajectory-scorers.js";
+import type { AgentTrajectoryEvent, AgentTrajectoryReport } from "./agent-trajectory.js";
+export declare const AGENT_TRAJECTORY_SCENARIO_RESULT_SCHEMA = "evalops.maestro.agent-trajectory-scenario-result.v1";
+export { MAESTRO_SCENARIO_WORKSPACE_MANIFEST_SCHEMA, MAESTRO_SCENARIO_SCHEMA, type MaestroScenario, type MaestroScenarioAssertion, type MaestroScenarioExternalRefs, type MaestroScenarioGateTier, type MaestroScenarioOutcome, type MaestroScenarioPlatformLink, type MaestroScenarioReleaseGate, type MaestroScenarioReviewLabel, type MaestroScenarioRequiredArtifact, type MaestroScenarioSeverity, type MaestroScenarioWorkspaceManifest, type MaestroScenarioWorkspaceHydrationMode, } from "@evalops/contracts";
+export type AgentTrajectoryScenarioStatus = "pass" | "fail" | "warn";
+export interface AgentTrajectoryScenarioEvidence {
+    kind: string;
+    id: string;
+    source: "trajectory" | "replay" | "score" | "inspection" | "scenario";
+    label: string;
+}
+export interface AgentTrajectoryScenarioAssertionResult {
+    id: string;
+    kind: MaestroScenarioAssertion["kind"];
+    status: AgentTrajectoryScenarioStatus;
+    severity: MaestroScenarioSeverity;
+    message: string;
+    evidence: AgentTrajectoryScenarioEvidence[];
+}
+export interface AgentTrajectoryScenarioProvenanceStep {
+    eventId: string;
+    eventType: string;
+    phase: AgentTrajectoryEvent["phase"];
+    actor: AgentTrajectoryEvent["actor"];
+    evidence: AgentTrajectoryScenarioEvidence[];
+}
+export interface AgentTrajectoryScenarioDiff {
+    baselineRunId: string;
+    candidateRunId: string;
+    eventsDelta: number;
+    toolCallsDelta: number;
+    scoreFailuresDelta: number;
+}
+export interface AgentTrajectoryScenarioWorkspaceSummary {
+    manifestId: string;
+    source: MaestroScenarioWorkspaceManifest["source"];
+    recordedAt: string;
+    hydrationMode: MaestroScenarioWorkspaceHydrationMode;
+    files: number;
+    toolAdapters: number;
+}
+export interface AgentTrajectoryScenarioReleaseGateSummary extends MaestroScenarioReleaseGate {
+    satisfied: boolean;
+    missingArtifacts: MaestroScenarioRequiredArtifact[];
+    budgetViolations: string[];
+    policyViolations: string[];
+}
+export interface AgentTrajectoryScenarioResult {
+    schemaVersion: typeof AGENT_TRAJECTORY_SCENARIO_RESULT_SCHEMA;
+    scenarioSchemaVersion: typeof MAESTRO_SCENARIO_SCHEMA;
+    scenario: {
+        id: string;
+        title: string;
+        expectedOutcome: MaestroScenarioOutcome;
+        observedOutcome: MaestroScenarioOutcome;
+        reviewLabels: MaestroScenarioReviewLabel[];
+    };
+    externalRefs?: MaestroScenarioExternalRefs;
+    run: AgentTrajectoryReport["run"] & {
+        scenarioId: string;
+        replay: true;
+    };
+    counts: {
+        assertions: number;
+        passed: number;
+        failed: number;
+        warnings: number;
+        events: number;
+        toolCalls: number;
+        replayDeltas: number;
+        scoreFailures: number;
+        scoreWarnings: number;
+        workspaceFiles: number;
+        toolAdapters: number;
+    };
+    platform: MaestroScenarioPlatformLink & {
+        evidenceEventType: "maestro.events.eval.scored";
+    };
+    releaseGate?: AgentTrajectoryScenarioReleaseGateSummary;
+    workspace?: AgentTrajectoryScenarioWorkspaceSummary;
+    assumptions: MaestroScenario["assumptions"];
+    assertions: AgentTrajectoryScenarioAssertionResult[];
+    provenance: AgentTrajectoryScenarioProvenanceStep[];
+    diff?: AgentTrajectoryScenarioDiff;
+}
+interface ScenarioInputs {
+    trajectory: AgentTrajectoryReport;
+    replay: AgentTrajectoryReplayReport;
+    score: AgentTrajectoryScoreReport;
+    inspection?: AgentTrajectoryInspectionReport;
+    workspaceManifest?: MaestroScenarioWorkspaceManifest;
+    baselineTrajectory?: AgentTrajectoryReport;
+    candidateTrajectory?: AgentTrajectoryReport;
+    baselineScore?: AgentTrajectoryScoreReport;
+    candidateScore?: AgentTrajectoryScoreReport;
+}
+export declare function parseAgentTrajectoryScenario(value: unknown, label: string): MaestroScenario;
+export declare function loadAgentTrajectoryScenario(path: string): MaestroScenario;
+export declare function loadAgentTrajectoryScenarioInputs(scenario: MaestroScenario, baseDir: string): ScenarioInputs;
+export declare function validateAgentTrajectoryScenario(scenario: MaestroScenario, label?: string): void;
+export declare function evaluateAgentTrajectoryScenario(scenario: MaestroScenario, inputs: ScenarioInputs): AgentTrajectoryScenarioResult;
+export declare function scenarioResultToJunit(result: AgentTrajectoryScenarioResult): string;
+export declare function runAgentTrajectoryScenarioFile(path: string, options?: {
+    baseDir?: string;
+}): AgentTrajectoryScenarioResult;
+//# sourceMappingURL=agent-trajectory-scenarios.d.ts.map

package/dist/server/agent-trajectory-scenarios.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-trajectory-scenarios.d.ts","sourceRoot":"","sources":["../../src/server/agent-trajectory-scenarios.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,uBAAuB,EAEvB,KAAK,eAAe,EACpB,KAAK,wBAAwB,EAC7B,KAAK,2BAA2B,EAEhC,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,0BAA0B,EAC/B,KAAK,+BAA+B,EACpC,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAE5B,KAAK,qCAAqC,EAC1C,KAAK,gCAAgC,EAErC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,kCAAkC,CAAC;AACxF,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,8BAA8B,CAAC;AAChF,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,KAAK,EACX,oBAAoB,EACpB,qBAAqB,EACrB,MAAM,uBAAuB,CAAC;AAG/B,eAAO,MAAM,uCAAuC,wDACE,CAAC;AACvD,OAAO,EACN,0CAA0C,EAC1C,uBAAuB,EACvB,KAAK,eAAe,EACpB,KAAK,wBAAwB,EAC7B,KAAK,2BAA2B,EAChC,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,0BAA0B,EAC/B,KAAK,0BAA0B,EAC/B,KAAK,+BAA+B,EACpC,KAAK,uBAAuB,EAC5B,KAAK,gCAAgC,EACrC,KAAK,qCAAqC,GAC1C,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,6BAA6B,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAErE,MAAM,WAAW,+BAA+B;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,GAAG,QAAQ,GAAG,OAAO,GAAG,YAAY,GAAG,UAAU,CAAC;IACtE,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,sCAAsC;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,wBAAwB,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,EAAE,6BAA6B,CAAC;IACtC,QAAQ,EAAE,uBAAuB,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,+BAA+B,EAAE,CAAC;CAC5C;AAED,MAAM,WAAW,qCAAqC;IACrD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACrC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACrC,QAAQ,EAAE,+BAA+B,EAAE,CAAC;CAC5C;AAED,MAAM,WAAW,2BAA2B;IAC3C,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,uCAAuC;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,gCAAgC,CAAC,QAAQ,CAAC,CAAC;IACnD,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,qCAAqC,CAAC;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,yCAChB,SAAQ,0BAA0B;IAClC,SAAS,EAAE,OAAO,CAAC;IACnB,gBAAgB,EAAE,+BAA+B,EAAE,CAAC;IACpD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,6BAA6B;IAC7C,aAAa,EAAE,OAAO,uCAAuC,CAAC;IAC9D,qBAAqB,EAAE,OAAO,uBAAuB,CAAC;IACtD,QAAQ,EAAE;QACT,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,eAAe,EAAE,sBAAsB,CAAC;QACxC,eAAe,EAAE,sBAAsB,CAAC;QACxC,YAAY,EAAE,0BAA0B,EAAE,CAAC;KAC3C,CAAC;IACF,YAAY,CAAC,EAAE,2BAA2B,CAAC;IAC3C,GAAG,EAAE,qBAAqB,CAAC,KAAK,CAAC,GAAG;QACnC,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,IAAI,CAAC;KACb,CAAC;IACF,MAAM,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,QAAQ,EAAE,2BAA2B,GAAG;QACvC,iBAAiB,EAAE,4BAA4B,CAAC;KAChD,CAAC;IACF,WAAW,CAAC,EAAE,yCAAyC,CAAC;IACxD,SAAS,CAAC,EAAE,uCAAuC,CAAC;IACpD,WAAW,EAAE,eAAe,CAAC,aAAa,CAAC,CAAC;IAC5C,UAAU,EAAE,sCAAsC,EAAE,CAAC;IACrD,UAAU,EAAE,qCAAqC,EAAE,CAAC;IACpD,IAAI,CAAC,EAAE,2BAA2B,CAAC;CACnC;AAED,UAAU,cAAc;IACvB,UAAU,EAAE,qBAAqB,CAAC;IAClC,MAAM,EAAE,2BAA2B,CAAC;IACpC,KAAK,EAAE,0BAA0B,CAAC;IAClC,UAAU,CAAC,EAAE,+BAA+B,CAAC;IAC7C,iBAAiB,CAAC,EAAE,gCAAgC,CAAC;IACrD,kBAAkB,CAAC,EAAE,qBAAqB,CAAC;IAC3C,mBAAmB,CAAC,EAAE,qBAAqB,CAAC;IAC5C,aAAa,CAAC,EAAE,0BAA0B,CAAC;IAC3C,cAAc,CAAC,EAAE,0BAA0B,CAAC;CAC5C;AAoMD,wBAAgB,4BAA4B,CAC3C,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,GACX,eAAe,CAajB;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAEzE;AAED,wBAAgB,iCAAiC,CAChD,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,MAAM,GACb,cAAc,CAqEhB;AAED,wBAAgB,+BAA+B,CAC9C,QAAQ,EAAE,eAAe,EACzB,KAAK,SAAc,GACjB,IAAI,CAmSN;AAqjBD,wBAAgB,+BAA+B,CAC9C,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,cAAc,GACpB,6BAA6B,CAsD/B;AAED,wBAAgB,qBAAqB,CACpC,MAAM,EAAE,6BAA6B,GACnC,MAAM,CAqCR;AAED,wBAAgB,8BAA8B,CAC7C,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GAChC,6BAA6B,CAO/B"}