@wix/pathgrade 1.0.36 → 1.0.38

package/README.md

@@ -164,6 +164,8 @@ npx pathgrade run
 - **Scorer**: a function or judge that evaluates output or behavior
 - **Evaluation**: the aggregated result of one or more scorers, returned as a score from `0.0` to `1.0`
 
+For protocol-neutral multi-agent graphs, see [Evaluating agent flows](docs/agent-flow-evaluation.md).
+
 ## Scorers
 
 Scorers evaluate the agent's output and behavior. `evaluate()` runs all scorers and computes a weighted average between `0.0` and `1.0`.

package/dist/adapter-kit/index.d.ts

@@ -13,5 +13,5 @@ export { getPathgradeDir } from '../reporters/results-path.js';
 export { printReportSummary } from '../reporters/report-summary.js';
 export { fmt } from '../utils/cli.js';
 export { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, type CaseContext, type CaseContextProvider, type CaseContextProviderHandle, type CaseContextScope, type CurrentCaseContext, } from '../sdk/case-context.js';
-export { subscribeToEvalResults, type EvalResultObserver, type ResultObserverHandle, type ResultObserverOptions, type ResultObserverOwner, } from '../sdk/result-capture.js';
-export type { Agent, PathgradeTestMeta, RecordedEvalResult, } from '../sdk/types.js';
+export { subscribeToEvalResults, subscribeToEvaluationResults, type EvalResultObserver, type ResultObserverHandle, type ResultObserverOptions, type ResultObserverOwner, type EvaluationResultEvent, type EvaluationResultObserver, } from '../sdk/result-capture.js';
+export type { Agent, AgentFlowTrace, PathgradeTestMeta, RecordedEvalResult, } from '../sdk/types.js';

package/dist/adapter-kit/index.js

@@ -11,4 +11,4 @@ export { getPathgradeDir } from '../reporters/results-path.js';
 export { printReportSummary } from '../reporters/report-summary.js';
 export { fmt } from '../utils/cli.js';
 export { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, } from '../sdk/case-context.js';
-export { subscribeToEvalResults, } from '../sdk/result-capture.js';
+export { subscribeToEvalResults, subscribeToEvaluationResults, } from '../sdk/result-capture.js';

package/dist/adapters/jest/lifecycle.js

@@ -1,4 +1,4 @@
-import { installCaseContextProvider, subscribeToEvalResults, createRunnerLifecycleHooks, } from '@wix/pathgrade/adapter-kit';
+import { installCaseContextProvider, subscribeToEvaluationResults, createRunnerLifecycleHooks, } from '@wix/pathgrade/adapter-kit';
 import { appendJestMetadata } from './metadata.js';
 import { jestCaseId } from './results.js';
 const metadataByCaseId = new Map();
@@ -58,5 +58,7 @@ function resetJestLifecycleMetadata() {
     metadataByCaseId.clear();
 }
 function defaultSubscribeToResults(callback) {
-    return subscribeToEvalResults(({ result, agent }) => callback({ result, agent }), { owner: 'adapter', key: 'jest-lifecycle' });
+    return subscribeToEvaluationResults(event => callback(event.sourceKind === 'flow'
+        ? { result: event.result, flow: event.flow, case: event.case }
+        : { result: event.result, agent: event.agent, case: event.case }), { owner: 'adapter', key: 'jest-lifecycle' });
 }

package/dist/adapters/node-test/index.js

@@ -2,7 +2,7 @@ import nodeTest from 'node:test';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { subscribeToEvalResults } from '../../sdk/result-capture.js';
+import { subscribeToEvaluationResults } from '../../sdk/result-capture.js';
 import { createRunnerLifecycleHooks } from '../../runners/lifecycle-hooks.js';
 const lifecycle = createRunnerLifecycleHooks();
 let subscribed = false;
@@ -69,7 +69,9 @@ function installResultCapture() {
     if (subscribed)
         return;
     subscribed = true;
-    subscribeToEvalResults(event => lifecycle.onResult(event), {
+    subscribeToEvaluationResults(event => lifecycle.onResult(event.sourceKind === 'flow'
+        ? { result: event.result, flow: event.flow, case: event.case }
+        : { result: event.result, agent: event.agent, case: event.case }), {
         owner: 'adapter',
         key: 'node-test-lifecycle',
     });

package/dist/agents/codex-app-server/usage-accounting.d.ts

@@ -1,6 +1,5 @@
 import type { AgentTurnResult } from '../../types.js';
 import type { AppServerTransport } from './transport.js';
-export declare const WIX_CODEX_USD_PER_CREDIT = 0.034;
 export declare class CodexTurnUsageAccounting {
     private baseline?;
     private latest?;

package/dist/agents/codex-app-server/usage-accounting.js

@@ -2,7 +2,7 @@ const TOKEN_KEYS = [
     'totalTokens', 'inputTokens', 'cachedInputTokens',
     'cacheWriteInputTokens', 'outputTokens', 'reasoningOutputTokens',
 ];
-export const WIX_CODEX_USD_PER_CREDIT = 0.034;
+const WIX_CODEX_USD_PER_CREDIT = 0.034;
 const USAGE_READ_FALLBACK_TIMEOUT_MS = 5_000;
 export class CodexTurnUsageAccounting {
     baseline;

package/dist/reporting/comparison-contract.js

@@ -12,7 +12,7 @@ export function buildComparisonContract(input) {
             case: caseIdentity,
             definition: evaluation.evaluationDefinitionKey ?? `position:${index}`,
             scorer: evaluation.scorerRevision,
-            runtime: runtimeIdentity(evaluation.agent ?? evaluation.trial?.agent),
+            runtime: runtimeIdentity(evaluation),
         }));
     });
     const definitionRevision = input.group.sourceRevision
@@ -83,12 +83,26 @@ export function createScorerRevision(scorers) {
         return undefined;
     }
 }
-function runtimeIdentity(agent) {
+function runtimeIdentity(evaluation) {
+    const agent = evaluation.agent ?? evaluation.trial?.agent;
+    if (agent) {
+        return {
+            name: agent.name,
+            model: agent.resolvedModel,
+            transport: agent.transport,
+            interaction: agent.interactionMode,
+        };
+    }
+    const flow = evaluation.trial?.flow_trace;
     return {
-        name: agent?.name,
-        model: agent?.resolvedModel,
-        transport: agent?.transport,
-        interaction: agent?.interactionMode,
+        flow: flow?.completeness.runtimeIdentity === 'complete'
+            && flow.participants.every(participant => participant.runtime) ? {
+            ...(flow.protocol ? { protocol: flow.protocol } : {}),
+            rootParticipantId: flow.rootParticipantId,
+            participants: flow.participants
+                .map(participant => ({ id: participant.id, runtime: participant.runtime }))
+                .toSorted((left, right) => left.id.localeCompare(right.id)),
+        } : undefined,
     };
 }
 function scorerDeclaration(scorer) {

package/dist/reporting/core.js

@@ -30,7 +30,7 @@ export function buildPathgradeReport(input) {
         const traceFile = `traces/${slug(group.groupName)}.json`;
         traces.push({ traceFile, trials: report.trials });
         const strippedTrials = report.trials.map(trial => {
-            const { session_log, conversation, ...rest } = trial;
+            const { session_log, conversation, flow_trace, ...rest } = trial;
             return rest;
         });
         const { trials: _trials, ...rest } = report;

package/dist/runners/adapter.d.ts

@@ -1,5 +1,5 @@
 import type { PathgradeSelectionReport } from '../types.js';
-import type { Agent, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
+import type { Agent, AgentFlowTrace, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
 import type { CaseContextScope } from '../sdk/case-context.js';
 import type { NormalizedRunSnapshot } from './model.js';
 export declare const runnerAdapterContractVersion = 1;
@@ -61,5 +61,7 @@ export interface AdapterCaseContext {
 }
 export interface EvalResultEvent {
     result: RecordedEvalResult;
-    agent: Agent;
+    agent?: Agent;
+    flow?: AgentFlowTrace;
+    case?: import('../sdk/case-context.js').CaseContext;
 }

package/dist/runners/lifecycle-hooks.js

@@ -2,7 +2,12 @@ import { lifecycleCore } from '../sdk/lifecycle.js';
 import { runWithCaseContext } from '../sdk/case-context.js';
 export function createRunnerLifecycleHooks() {
     return {
-        onResult: event => lifecycleCore.recordResult(event.result, event.agent),
+        onResult: event => {
+            if (event.flow)
+                lifecycleCore.recordFlowResult(event.result, event.flow, event.case?.caseId);
+            else if (event.agent)
+                lifecycleCore.recordResult(event.result, event.agent);
+        },
         withCaseContext: (context, run) => runWithCaseContext(toCaseContext(context), run),
         flushCase: caseId => lifecycleCore.flushCase({ caseId }),
         cleanupRun: () => lifecycleCore.cleanupAll(),

package/dist/runners/vitest-lifecycle.d.ts

@@ -6,6 +6,7 @@ export type VitestAfterEachFn = (fn: (ctx: {
     task: {
         id: string;
         meta: Record<string, unknown>;
+        suite?: unknown;
     };
 }) => Promise<void>) => void;
 export type VitestAfterAllFn = (fn: () => Promise<void>) => void;

package/dist/runners/vitest-lifecycle.js

@@ -1,10 +1,15 @@
 import path from 'node:path';
-import { subscribeToEvalResults } from '../sdk/result-capture.js';
+import { subscribeToEvaluationResults } from '../sdk/result-capture.js';
 import { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, } from '../sdk/case-context.js';
 import { lifecycleCore } from '../sdk/lifecycle.js';
 export function createVitestLifecycleHooks() {
     return {
-        onResult: event => recordVitestResult(event.result, event.agent),
+        onResult: event => {
+            if (event.flow)
+                recordVitestFlowResult(event.result, event.flow, event.case);
+            else if (event.agent)
+                recordVitestResult(event.result, event.agent);
+        },
         withCaseContext: (context, run) => runWithCaseContext(toCaseContext(context), run),
         flushCase: caseId => lifecycleCore.flushCase({ caseId }),
         cleanupRun: () => lifecycleCore.cleanupAll(),
@@ -18,7 +23,11 @@ export function installVitestLifecycle(input) {
     let resultsUnsubscribed = false;
     input.aroundEach?.(async (runTest, { task }) => lifecycle.withCaseContext(caseContextForTask(task), runTest));
     input.afterEach(async ({ task }) => {
-        const results = await lifecycle.flushCase(task.id);
+        const filePath = filePathForTask(task);
+        const results = [
+            ...(filePath ? await lifecycle.flushCase(`file:${filePath}`) : []),
+            ...await lifecycle.flushCase(task.id),
+        ];
         if (results.length > 0) {
             task.meta.pathgrade = results;
         }
@@ -69,7 +78,9 @@ export function resetVitestLifecycle(handle) {
     handle?.restore();
 }
 function defaultSubscribeToResults(callback) {
-    return subscribeToEvalResults(({ result, agent }) => callback({ result, agent }), { owner: 'adapter', key: 'vitest-lifecycle' });
+    return subscribeToEvaluationResults(event => callback(event.sourceKind === 'flow'
+        ? { result: event.result, flow: event.flow, case: event.case }
+        : { result: event.result, agent: event.agent, case: event.case }), { owner: 'adapter', key: 'vitest-lifecycle' });
 }
 function defaultInstallFileContextProvider() {
     return installCaseContextProvider(currentFileContext);
@@ -148,3 +159,9 @@ function recordVitestResult(result, agent) {
     }
     lifecycleCore.recordResult(result, agent);
 }
+function recordVitestFlowResult(result, flow, capturedCase) {
+    const caseId = capturedCase?.scope === 'runner-case'
+        ? capturedCase.caseId
+        : currentTaskId() || capturedCase?.caseId;
+    lifecycleCore.recordFlowResult(result, flow, caseId || undefined);
+}

package/dist/sdk/agent-flow.d.ts

@@ -0,0 +1,51 @@
+export type AgentFlowEvidenceAvailability = 'complete' | 'partial' | 'unavailable';
+export interface AgentFlowEvidenceCompleteness {
+    topology: AgentFlowEvidenceAvailability;
+    outcomes: AgentFlowEvidenceAvailability;
+    timing: AgentFlowEvidenceAvailability;
+    usage: AgentFlowEvidenceAvailability;
+    runtimeIdentity: AgentFlowEvidenceAvailability;
+}
+export interface AgentFlowParticipant {
+    id: string;
+    name?: string;
+    role?: string;
+    runtime?: string;
+}
+export interface AgentFlowInteraction {
+    id: string;
+    sequence: number;
+    sourceParticipantId: string;
+    targetParticipantId: string;
+    operation: string;
+    parentInteractionId?: string;
+    taskId?: string;
+    contextId?: string;
+    state?: string;
+    inputText?: string;
+    outputText?: string;
+    error?: string;
+    startedAt?: string;
+    endedAt?: string;
+}
+export interface AgentFlowTrace {
+    version: 1;
+    protocol?: string;
+    rootParticipantId: string;
+    participants: readonly AgentFlowParticipant[];
+    interactions: readonly AgentFlowInteraction[];
+    outcome: {
+        state: string;
+        outputText?: string;
+        error?: string;
+    };
+    completeness: AgentFlowEvidenceCompleteness;
+}
+export interface AgentFlowSummary {
+    participant_count: number;
+    interaction_count: number;
+    max_observed_depth: number;
+    root_state: string;
+    completeness: AgentFlowEvidenceCompleteness;
+}
+export declare function summarizeFlow(flow: AgentFlowTrace): AgentFlowSummary;

package/dist/sdk/agent-flow.js

@@ -0,0 +1,23 @@
+export function summarizeFlow(flow) {
+    const parents = new Map(flow.interactions.map(interaction => [interaction.id, interaction.parentInteractionId]));
+    const depth = (id) => {
+        let current = parents.get(id);
+        let value = 1;
+        const seen = new Set([id]);
+        while (current && !seen.has(current)) {
+            seen.add(current);
+            value += 1;
+            current = parents.get(current);
+        }
+        return value;
+    };
+    return {
+        participant_count: flow.participants.length,
+        interaction_count: flow.interactions.length,
+        max_observed_depth: flow.interactions.length > 0
+            ? Math.max(...flow.interactions.map(interaction => depth(interaction.id)))
+            : 0,
+        root_state: flow.outcome.state,
+        completeness: structuredClone(flow.completeness),
+    };
+}

package/dist/sdk/evaluate.d.ts

@@ -1,4 +1,4 @@
-import type { Agent, Scorer, RecordedEvalResult, ScorerResultEntry, EvaluateOptions } from './types.js';
+import type { Agent, Scorer, RecordedEvalResult, ScorerResultEntry, EvaluateOptions, AgentFlowTrace } from './types.js';
 export type OnScorerErrorMode = NonNullable<EvaluateOptions['onScorerError']>;
 export declare class EvalScorerError extends Error {
     readonly scorerErrors: ScorerResultEntry[];
@@ -9,6 +9,7 @@ type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: Eva
 type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>) & {
     fromSnapshot: EvaluateFromSnapshot;
 };
+export declare function evaluateFlow(flow: AgentFlowTrace, scorers: Scorer[], opts?: EvaluateOptions): Promise<RecordedEvalResult>;
 /** Internal runConversation hook: step scorers must not consume final-run attribution. */
 export declare function evaluateStepScorers(agent: Agent, scorers: Scorer[], opts?: EvaluateOptions): Promise<RecordedEvalResult>;
 export declare const evaluate: EvaluateFn;

package/dist/sdk/evaluate.js

@@ -1,7 +1,7 @@
 import { countShellCommandsFromLog, extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
 import { createScorerRevision } from '../reporting/comparison-contract.js';
 import { getRuntime } from './eval-runtime.js';
-import { emitEvalResult } from './result-capture.js';
+import { emitEvalResult, emitFlowEvalResult } from './result-capture.js';
 import { runJudgePipeline } from './judge-pipeline.js';
 import { runScorer } from './run-scorer.js';
 import { createLLMClient } from '../utils/llm.js';
@@ -9,6 +9,7 @@ import { sandboxExec } from '../providers/sandbox-exec.js';
 import { buildTranscript, loadRunSnapshot, WorkspaceMissingError } from './snapshots.js';
 import fs from 'fs-extra';
 import path from 'path';
+import { summarizeFlow } from './agent-flow.js';
 export class EvalScorerError extends Error {
     scorerErrors;
     result;
@@ -19,6 +20,60 @@ export class EvalScorerError extends Error {
         this.scorerErrors = scorerErrors;
     }
 }
+export async function evaluateFlow(flow, scorers, opts) {
+    if (scorers.some(scorer => scorer.type === 'tool_usage')) {
+        throw new TypeError('tool_usage scorers require an Agent and cannot evaluate an AgentFlowTrace');
+    }
+    if (scorers.some(scorer => scorer.type === 'judge' && scorer.tools && scorer.tools.length > 0)) {
+        throw new TypeError('tool-enabled judge scorers require an Agent workspace and cannot evaluate an AgentFlowTrace');
+    }
+    const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
+                name: 'runtime', isAvailable: async () => true,
+                call: (prompt, callOpts) => getRuntime().llm.call(prompt, callOpts),
+            }] });
+    const unavailable = async () => {
+        throw new WorkspaceMissingError('AgentFlowTrace does not include a workspace');
+    };
+    const ctx = {
+        workspace: '',
+        log: [],
+        transcript: JSON.stringify(flow),
+        toolEvents: [],
+        runCommand: unavailable,
+        artifacts: { list: () => [], read: unavailable, latest: async () => null },
+        flow,
+    };
+    const scoringStartedAt = performance.now();
+    const beforeTokenUsage = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
+    const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
+        ? await trackedLLM.measure(() => evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM }))
+        : await (async () => {
+            const result = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
+            const afterTokenUsage = trackedLLM.tokenUsage ?? beforeTokenUsage;
+            return { result, tokens: {
+                    inputTokens: Math.max(0, afterTokenUsage.inputTokens - beforeTokenUsage.inputTokens),
+                    outputTokens: Math.max(0, afterTokenUsage.outputTokens - beforeTokenUsage.outputTokens),
+                } };
+        })();
+    const scorerRevision = createScorerRevision(scorers);
+    const recordedResult = {
+        ...evalResult,
+        tokenUsage: deltaTokenUsage,
+        resultKind: 'evaluated',
+        scoringDurationMs: Math.max(0, performance.now() - scoringStartedAt),
+        recordedAt: new Date().toISOString(),
+        ...(opts?.evaluationDefinitionKey ? { evaluationDefinitionKey: opts.evaluationDefinitionKey } : {}),
+        ...(scorerRevision ? { scorerRevision } : {}),
+        trial: {
+            ...buildTrialResult([], { ...evalResult, tokenUsage: deltaTokenUsage }),
+            flow_summary: summarizeFlow(flow),
+            flow_trace: structuredClone(flow),
+        },
+    };
+    emitFlowEvalResult({ result: recordedResult, flow });
+    maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
+    return recordedResult;
+}
 /**
  * Run scorers against a trial and compute a weighted average score.
  *

package/dist/sdk/index.d.ts

@@ -9,7 +9,7 @@ export type { McpMockApprovalRule, McpMockJsonValue } from './mcp-mock-approvals
 export type { CompileScenarioResult, EvidenceEnvelope, JsonObject, JsonPrimitive, JsonValue, MatchExpr, MatchPredicate, McpContent, ScenarioArtifact, ScenarioCase, ScenarioDiagnostic, ScenarioEvidence, ScenarioMachineV2, ScenarioRejection, ScenarioSuccess, ScenarioToolError, } from './scenario-machine-v2.js';
 export type { ExpectedScenarioCaseMatch, ScenarioCaseMatchEvidence, ScenarioStateTransitionEvidence, } from './scenario-evidence.js';
 export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
-export { evaluate, EvalScorerError } from './evaluate.js';
+export { evaluate, evaluateFlow, EvalScorerError } from './evaluate.js';
 export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
 export { createPersona } from './persona.js';
 export { createConversationWindow } from './conversation-window.js';
@@ -26,11 +26,13 @@ export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from '.
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export type { AskUserToolEvent, AskUserToolEventArguments, AskUserToolEventQuestionArgument, } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';
-export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, } from './result-capture.js';
+export { emitEvalResult, emitFlowEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, subscribeToEvaluationResults, } from './result-capture.js';
 export { getAgentCapabilities } from './types.js';
 export type { AgentTransport, AgentCapabilities, AgentName, McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './types.js';
 export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
 export type { Agent, AgentOptions, DebugOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentExecutionTransport, AgentInteractionMode, } from './types.js';
+export type { AgentFlowTrace, AgentFlowParticipant, AgentFlowInteraction, AgentFlowEvidenceAvailability, AgentFlowEvidenceCompleteness, AgentFlowSummary, } from './agent-flow.js';
+export type { EvaluationResultEvent, EvaluationResultObserver } from './result-capture.js';
 export type { ConversationWindow, ConversationWindowOptions } from './conversation-window.js';
 export type { JudgePipelineOptions } from './judge-pipeline.js';
 export type { RunScorerOptions } from './run-scorer.js';

package/dist/sdk/index.js

@@ -7,7 +7,7 @@ export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
 export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
 export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
 export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
-export { evaluate, EvalScorerError } from './evaluate.js';
+export { evaluate, evaluateFlow, EvalScorerError } from './evaluate.js';
 export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
 export { createPersona } from './persona.js';
 export { createConversationWindow } from './conversation-window.js';
@@ -23,7 +23,7 @@ export { parsePathgradeReport } from '../reporting/report-parser.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';
-export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, } from './result-capture.js';
+export { emitEvalResult, emitFlowEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, subscribeToEvaluationResults, } from './result-capture.js';
 export { getAgentCapabilities } from './types.js';
 export { COMPARISON_CONTRACT_VERSION, NORMALIZED_RUN_MODEL_VERSION, PATHGRADE_REPORT_VERSION, TASK_INVENTORY_VERSION, } from '../reporting/reliability-contract.js';
 export { createAgentLLM, createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';

package/dist/sdk/lifecycle.d.ts

@@ -1,4 +1,4 @@
-import type { Agent, PathgradeTestMeta, RecordedEvalResult } from './types.js';
+import type { Agent, AgentFlowTrace, PathgradeTestMeta, RecordedEvalResult } from './types.js';
 export type LifecycleAgentOwner = {
     type: 'runner-case';
     caseId: string;
@@ -24,6 +24,7 @@ declare function registerAgent(agent: Agent, owner?: LifecycleAgentOwner | null)
 declare function untrackAgent(agent: Agent): void;
 declare function releaseAgent(agent: Agent): void;
 declare function recordResult(result: RecordedEvalResult, agent: Agent, attribution?: ResultAttribution): void;
+declare function recordFlowResult(result: RecordedEvalResult, flow: AgentFlowTrace, caseId?: string): void;
 declare function flushCase(input: FlushCaseInput): Promise<PathgradeTestMeta[]>;
 declare function cleanupAll(): Promise<void>;
 declare function reset(): void;
@@ -33,6 +34,7 @@ export declare const lifecycleCore: {
     releaseAgent: typeof releaseAgent;
     getAgentOwner: typeof getAgentOwner;
     recordResult: typeof recordResult;
+    recordFlowResult: typeof recordFlowResult;
     flushCase: typeof flushCase;
     cleanupAll: typeof cleanupAll;
     reset: typeof reset;

package/dist/sdk/lifecycle.js

@@ -1,14 +1,19 @@
 import { getCurrentCaseContext } from './case-context.js';
 import { buildDiagnosticsReport } from './diagnostics.js';
 import { countShellCommandsFromLog } from '../tool-events.js';
+import { summarizeFlow } from './agent-flow.js';
 const lifecycleStateKey = Symbol.for('@wix/pathgrade/lifecycle-state');
 const sharedLifecycle = globalThis;
 const lifecycleState = sharedLifecycle[lifecycleStateKey] ?? (sharedLifecycle[lifecycleStateKey] = {
     pendingAgents: new Set(),
     agentOwners: new WeakMap(),
     agentResults: new WeakMap(),
+    flowResults: new Map(),
+    nextResultSequence: 0,
 });
-const { pendingAgents, agentOwners, agentResults } = lifecycleState;
+lifecycleState.nextResultSequence ??= 0;
+lifecycleState.flowResults ??= new Map();
+const { pendingAgents, agentOwners, agentResults, flowResults } = lifecycleState;
 function currentAgentOwner() {
     const current = getCurrentCaseContext();
     if (current.status !== 'active')
@@ -74,6 +79,7 @@ function recordResult(result, agent, attribution) {
     const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
     const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
     agentResults.get(agent).push({
+        sequence: lifecycleState.nextResultSequence++,
         attribution: attribution ?? currentResultAttribution(owner),
         meta: {
             score: result.score,
@@ -101,8 +107,60 @@ function recordResult(result, agent, attribution) {
         },
     });
 }
+function recordFlowResult(result, flow, caseId) {
+    const current = getCurrentCaseContext();
+    const resolvedCaseId = caseId ?? (current.status === 'active' && current.context.scope === 'runner-case'
+        ? current.context.caseId
+        : undefined);
+    if (!resolvedCaseId)
+        return;
+    const entries = flowResults.get(resolvedCaseId) ?? [];
+    entries.push({
+        sequence: lifecycleState.nextResultSequence++,
+        meta: {
+            score: result.score,
+            scorers: result.scorers,
+            ...(result.evaluationDefinitionKey ? { evaluationDefinitionKey: result.evaluationDefinitionKey } : {}),
+            ...(result.scorerRevision ? { scorerRevision: result.scorerRevision } : {}),
+            trial: {
+                ...(result.trial ?? {
+                    trial_id: 0,
+                    ...(result.score !== undefined ? { reward: result.score } : {}),
+                    scorer_results: result.scorers.map(scorer => ({
+                        scorer_type: scorer.type === 'check' || scorer.type === 'score' ? 'deterministic' : scorer.type === 'judge' ? 'llm_rubric' : 'tool_usage',
+                        score: scorer.score,
+                        weight: scorer.weight,
+                        details: scorer.details ?? '',
+                        status: scorer.status,
+                    })),
+                    duration_ms: 0,
+                    n_commands: 0,
+                    input_tokens: result.tokenUsage?.inputTokens ?? 0,
+                    output_tokens: result.tokenUsage?.outputTokens ?? 0,
+                    session_log: [],
+                }),
+                flow_summary: summarizeFlow(flow),
+                flow_trace: structuredClone(flow),
+            },
+            resultKind: result.resultKind ?? 'evaluated',
+            ...(result.scoringDurationMs !== undefined ? { scoringDurationMs: result.scoringDurationMs } : {}),
+            ...(result.recordedAt ? { recordedAt: result.recordedAt } : {}),
+            diagnostics: buildDiagnosticsReport({
+                completionReason: ['completed', 'failed', 'rejected', 'canceled', 'auth-required'].includes(flow.outcome.state)
+                    ? flow.outcome.state
+                    : undefined,
+                completionDetail: flow.outcome.error,
+                score: result.score,
+                scorers: result.scorers,
+                log: [],
+            }),
+        },
+    });
+    flowResults.set(resolvedCaseId, entries);
+}
 async function flushCase(input) {
-    const results = [];
+    const results = flowResults.get(input.caseId) ?? [];
+    flowResults.delete(input.caseId);
     const toDispose = [];
     for (const agent of pendingAgents) {
         const owner = agentOwners.get(agent);
@@ -110,7 +168,7 @@ async function flushCase(input) {
         const meta = agentResults.get(agent);
         const matchingMeta = meta?.filter((entry) => resultMatchesCase(entry, input.caseId));
         if (matchingMeta && matchingMeta.length > 0 && canFlushResultsToCase(owner, input.caseId)) {
-            results.push(...matchingMeta.map((entry) => entry.meta));
+            results.push(...matchingMeta);
             const remainingMeta = meta?.filter((entry) => !resultMatchesCase(entry, input.caseId)) ?? [];
             if (remainingMeta.length > 0) {
                 agentResults.set(agent, remainingMeta);
@@ -124,7 +182,7 @@ async function flushCase(input) {
             if (!hasResultsForCase) {
                 const synthTrial = synthesizeTrialFromAgent(agent);
                 if (synthTrial) {
-                    results.push(synthTrial);
+                    results.push({ sequence: lifecycleState.nextResultSequence++, meta: synthTrial });
                 }
             }
             pendingAgents.delete(agent);
@@ -133,7 +191,7 @@ async function flushCase(input) {
         }
     }
     await Promise.all(toDispose.map((agent) => agent.dispose().catch(() => { })));
-    return results;
+    return results.toSorted((left, right) => left.sequence - right.sequence).map(entry => entry.meta);
 }
 function synthesizeTrialFromAgent(agent) {
     if (agent.log.length === 0)
@@ -181,10 +239,14 @@ function synthesizeTrialFromAgent(agent) {
 async function cleanupAll() {
     const toDispose = [...pendingAgents];
     pendingAgents.clear();
+    flowResults.clear();
+    lifecycleState.nextResultSequence = 0;
     await Promise.all(toDispose.map((agent) => agent.dispose().catch(() => { })));
 }
 function reset() {
     pendingAgents.clear();
+    flowResults.clear();
+    lifecycleState.nextResultSequence = 0;
 }
 export const lifecycleCore = {
     registerAgent,
@@ -192,6 +254,7 @@ export const lifecycleCore = {
     releaseAgent,
     getAgentOwner,
     recordResult,
+    recordFlowResult,
     flushCase,
     cleanupAll,
     reset,

package/dist/sdk/result-capture.d.ts

@@ -1,4 +1,4 @@
-import type { Agent, RecordedEvalResult } from './types.js';
+import type { Agent, AgentFlowTrace, RecordedEvalResult } from './types.js';
 import { type CaseContext } from './case-context.js';
 export interface EvalResultEvent {
     readonly result: RecordedEvalResult;
@@ -6,6 +6,15 @@ export interface EvalResultEvent {
     readonly case?: CaseContext;
 }
 export type EvalResultObserver = (event: EvalResultEvent) => void;
+export type EvaluationResultEvent = (EvalResultEvent & {
+    readonly sourceKind: 'agent';
+}) | {
+    readonly sourceKind: 'flow';
+    readonly result: RecordedEvalResult;
+    readonly flow: AgentFlowTrace;
+    readonly case?: CaseContext;
+};
+export type EvaluationResultObserver = (event: EvaluationResultEvent) => void;
 export type ResultObserverOwner = 'user' | 'adapter' | 'test';
 export interface ResultObserverOptions {
     readonly owner?: ResultObserverOwner;
@@ -15,6 +24,10 @@ export interface ResultObserverHandle {
     unsubscribe(): void;
 }
 export declare function subscribeToEvalResults(observer: EvalResultObserver, options?: ResultObserverOptions): ResultObserverHandle;
+export declare function subscribeToEvaluationResults(observer: EvaluationResultObserver, options?: ResultObserverOptions): ResultObserverHandle;
 export declare function emitEvalResult(event: EvalResultEvent): void;
+export declare function emitFlowEvalResult(event: Omit<Extract<EvaluationResultEvent, {
+    sourceKind: 'flow';
+}>, 'sourceKind'>): void;
 export declare function resetUserResultObservers(): void;
 export declare function resetAllResultObserversForTests(): void;

package/dist/sdk/result-capture.js

@@ -3,6 +3,10 @@ const observerRegistryKey = Symbol.for('@wix/pathgrade/eval-result-observers');
 const globalRegistry = globalThis;
 const observers = globalRegistry[observerRegistryKey]
     ?? (globalRegistry[observerRegistryKey] = new Set());
+const evaluationObserverRegistryKey = Symbol.for('@wix/pathgrade/evaluation-result-observers');
+const evaluationGlobalRegistry = globalThis;
+const evaluationObservers = evaluationGlobalRegistry[evaluationObserverRegistryKey]
+    ?? (evaluationGlobalRegistry[evaluationObserverRegistryKey] = new Set());
 export function subscribeToEvalResults(observer, options = {}) {
     if (options.owner === 'adapter' && options.key) {
         removeObserverByOwnerAndKey(options.owner, options.key);
@@ -23,6 +27,22 @@ export function subscribeToEvalResults(observer, options = {}) {
         },
     };
 }
+export function subscribeToEvaluationResults(observer, options = {}) {
+    if (options.owner === 'adapter' && options.key) {
+        removeEvaluationObserverByOwnerAndKey(options.owner, options.key);
+    }
+    const subscription = { observer, owner: options.owner ?? 'user', key: options.key };
+    evaluationObservers.add(subscription);
+    let active = true;
+    return {
+        unsubscribe() {
+            if (!active)
+                return;
+            active = false;
+            evaluationObservers.delete(subscription);
+        },
+    };
+}
 export function emitEvalResult(event) {
     const currentCase = getCurrentCaseContext();
     const deliveredEvent = event.case || currentCase.status !== 'active'
@@ -40,6 +60,14 @@ export function emitEvalResult(event) {
             // broken hook must not block adapter-owned reporting.
         }
     }
+    deliverEvaluationEvent({ ...deliveredEvent, sourceKind: 'agent' });
+}
+export function emitFlowEvalResult(event) {
+    const currentCase = getCurrentCaseContext();
+    const deliveredEvent = event.case || currentCase.status !== 'active'
+        ? event
+        : { ...event, case: currentCase.context };
+    deliverEvaluationEvent({ ...deliveredEvent, sourceKind: 'flow' });
 }
 export function resetUserResultObservers() {
     for (const subscription of observers) {
@@ -47,9 +75,14 @@ export function resetUserResultObservers() {
             observers.delete(subscription);
         }
     }
+    for (const subscription of evaluationObservers) {
+        if (subscription.owner === 'user')
+            evaluationObservers.delete(subscription);
+    }
 }
 export function resetAllResultObserversForTests() {
     observers.clear();
+    evaluationObservers.clear();
 }
 function removeObserverByOwnerAndKey(owner, key) {
     for (const subscription of observers) {
@@ -58,3 +91,19 @@ function removeObserverByOwnerAndKey(owner, key) {
         }
     }
 }
+function removeEvaluationObserverByOwnerAndKey(owner, key) {
+    for (const subscription of evaluationObservers) {
+        if (subscription.owner === owner && subscription.key === key)
+            evaluationObservers.delete(subscription);
+    }
+}
+function deliverEvaluationEvent(event) {
+    for (const subscription of [...evaluationObservers]) {
+        try {
+            subscription.observer(event);
+        }
+        catch {
+            // Generic result capture has the same best-effort isolation as the legacy stream.
+        }
+    }
+}

package/dist/sdk/types.d.ts

@@ -8,6 +8,8 @@ import type { McpSafetyOptions } from './mcp-safety.js';
 import type { McpMockApprovalRule } from './mcp-mock-approvals.js';
 import type { EvidenceEnvelope, JsonValue } from '../internal/direct-mcp-v2/types.js';
 import type { ScenarioMachineV2 } from './scenario-machine-v2.js';
+import type { AgentFlowTrace } from './agent-flow.js';
+export type { AgentFlowTrace, AgentFlowParticipant, AgentFlowInteraction, AgentFlowEvidenceAvailability, AgentFlowEvidenceCompleteness, AgentFlowSummary } from './agent-flow.js';
 export type AgentName = 'claude' | 'codex' | 'cursor' | 'opencode';
 export type AgentInteractionMode = 'prompt' | 'start_chat' | 'conversation';
 /** Runtime channel that actually executed the agent. */
@@ -351,6 +353,8 @@ export interface ScorerContext {
     artifacts: SessionArtifacts;
     /** Canonical host evidence for ScenarioMachineV2 scorers. */
     scenarioEvidence?: readonly EvidenceEnvelope[];
+    /** Protocol-neutral multi-agent execution evidence, when evaluating a flow. */
+    flow?: AgentFlowTrace;
 }
 export interface EvaluateOptions {
     failFast?: boolean;

package/dist/types.d.ts

@@ -172,6 +172,8 @@ export interface TrialResult {
     scoring_duration_ms?: number;
     recorded_at?: string;
     agent?: import('./sdk/types.js').AgentExecutionMetadata;
+    flow_summary?: import('./sdk/types.js').AgentFlowSummary;
+    flow_trace?: import('./sdk/types.js').AgentFlowTrace;
     conversation?: {
         turns: ConversationTurn[];
         total_turns: number;
@@ -211,12 +213,12 @@ export interface ComparisonContract {
     unavailable_reasons?: ComparisonUnavailableReason[];
 }
 /**
- * TrialResult with `session_log` and `conversation` stripped. These fields
+ * TrialResult with large trace fields stripped. These fields
  * live only in the per-group trace files; the consolidated results.json keeps
  * the rest so consumers (preview, `pathgrade report`) can compute summaries
  * without loading trace data.
  */
-export type StrippedTrialResult = Omit<TrialResult, 'session_log' | 'conversation'>;
+export type StrippedTrialResult = Omit<TrialResult, 'session_log' | 'conversation' | 'flow_trace'>;
 /**
  * Per-group entry in the consolidated `.pathgrade/results.json` report.
  */

package/docs/agent-flow-evaluation.md

@@ -0,0 +1,31 @@
+# Evaluating agent flows
+
+Use `evaluateFlow()` when the subject is an already-observed interaction graph rather than a Pathgrade-managed coding agent. The input is protocol-neutral, so an adapter can project A2A or another orchestration protocol into the same evidence contract.
+
+```typescript
+import { check, evaluateFlow, type AgentFlowTrace } from '@wix/pathgrade';
+
+const flow: AgentFlowTrace = {
+    version: 1,
+    protocol: 'a2a',
+    rootParticipantId: 'planner',
+    participants: [{ id: 'planner' }, { id: 'flights' }],
+    interactions: [{
+        id: 'delegate-1', sequence: 1,
+        sourceParticipantId: 'planner', targetParticipantId: 'flights',
+        operation: 'message/send', state: 'completed',
+    }],
+    outcome: { state: 'completed' },
+    completeness: {
+        topology: 'complete', outcomes: 'partial', timing: 'unavailable',
+        usage: 'unavailable', runtimeIdentity: 'partial',
+    },
+};
+
+await evaluateFlow(flow, [
+    check('delegated flight search', ({ flow }) =>
+        flow?.interactions.some(item => item.targetParticipantId === 'flights') === true),
+]);
+```
+
+`check()`, `score()`, and `judge()` work with flow evidence through `ScorerContext.flow`. `toolUsage()` is agent-only and is rejected explicitly for flows. Consolidated reports retain a compact `flow_summary`; the full `flow_trace` stays in the trace artifact. Completeness fields describe evidence availability and must not be read as inferred downstream success or timing.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.36",
+  "version": "1.0.38",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -64,6 +64,7 @@
     "!dist/**/*.d.ts.map",
     "bin/",
     "docs/OPENAI_OAUTH_JUDGE.md",
+    "docs/agent-flow-evaluation.md",
     "templates/",
     "README.md"
   ],
@@ -141,5 +142,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "598b0b5b60df2e9caafd91fa570ae1a473f6f61f29d14bdc8204ea46"
+  "falconPackageHash": "199b36204420a97fad83bac6e53d8b614d2b6592cb701e7db41d3bad"
 }