@agent-relay/evals 8.3.1 → 8.8.0

package/dist/harness.d.ts

@@ -0,0 +1,31 @@
+/**
+ * BrokerHarness type surface for @agent-relay/evals.
+ *
+ * The full BrokerHarness class implementation lives in
+ * tests/integration/broker/utils/broker-harness.ts pending migration into
+ * this package (see specs/agent-relay-evals-package.md). This module exports
+ * the interface that eval scenarios depend on so downstream consumers can
+ * type-check against it without importing the test-only implementation.
+ */
+import type { BrokerEvent, ListAgent, RuntimeSpawnOptions, SendMessageInput } from '@agent-relay/harness-driver';
+export interface EventWaiter {
+    promise: Promise<BrokerEvent>;
+    cancel: () => void;
+}
+export interface BrokerHarness {
+    spawnAgent(name: string, cli: string, channels: string[], options?: Partial<RuntimeSpawnOptions>): Promise<{
+        name: string;
+    }>;
+    releaseAgent(name: string): Promise<{
+        name: string;
+    }>;
+    sendMessage(input: SendMessageInput): Promise<{
+        event_id: string;
+        targets: string[];
+    }>;
+    listAgents(): Promise<ListAgent[]>;
+    getEvents(): BrokerEvent[];
+    clearEvents(): void;
+    waitForEvent(kind: string, timeoutMs?: number, predicate?: (event: BrokerEvent) => boolean): EventWaiter;
+}
+//# sourceMappingURL=harness.d.ts.map

package/dist/harness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.d.ts","sourceRoot":"","sources":["../src/harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,mBAAmB,EACnB,gBAAgB,EACjB,MAAM,6BAA6B,CAAC;AAErC,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,CACR,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACrC,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAE7B,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAEtD,WAAW,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;IAEvF,UAAU,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAEnC,SAAS,IAAI,WAAW,EAAE,CAAC;IAE3B,WAAW,IAAI,IAAI,CAAC;IAEpB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,OAAO,GAAG,WAAW,CAAC;CAC1G"}

package/dist/harness.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=harness.js.map

package/dist/harness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.js","sourceRoot":"","sources":["../src/harness.ts"],"names":[],"mappings":""}

package/dist/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * @agent-relay/evals
+ *
+ * Agent Relay eval harness — scenario runner, broker harness, and scoring
+ * utilities for testing relay-connected agents across CLI harnesses.
+ *
+ * The source of truth for scenario implementations and the runner currently
+ * lives in tests/integration/broker/evals/. This package exposes the stable
+ * public surface that downstream consumers (pear, agent-assistant, etc.) depend
+ * on. The full migration into this package is tracked in
+ * specs/agent-relay-evals-package.md.
+ */
+export type { EvalScenario, ScenarioContext, ScenarioResult, EvalTier, MetricSet, EvalReport, MatrixReport, AgentInfo, TranscriptEntry, Phantom, } from './types.js';
+export { SCHEMA_VERSION } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,YAAY,EACV,YAAY,EACZ,eAAe,EACf,cAAc,EACd,QAAQ,EACR,SAAS,EACT,UAAU,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,OAAO,GACR,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,14 @@
+/**
+ * @agent-relay/evals
+ *
+ * Agent Relay eval harness — scenario runner, broker harness, and scoring
+ * utilities for testing relay-connected agents across CLI harnesses.
+ *
+ * The source of truth for scenario implementations and the runner currently
+ * lives in tests/integration/broker/evals/. This package exposes the stable
+ * public surface that downstream consumers (pear, agent-assistant, etc.) depend
+ * on. The full migration into this package is tracked in
+ * specs/agent-relay-evals-package.md.
+ */
+export { SCHEMA_VERSION } from './types.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAeH,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,161 @@
+/**
+ * Type definitions for the agent messaging eval harness.
+ *
+ * The eval suite spawns real agent CLIs, drives them through coordination
+ * scenarios, and scores — purely from broker events — whether agents actually
+ * used the messaging tools (MCP/CLI) versus emitting plain-text "phantom"
+ * messages.
+ */
+import type { BrokerHarness } from './harness.js';
+/** Context handed to each scenario's `run` function. */
+export interface ScenarioContext {
+    /** A started broker harness. The scenario owns its agents but not the harness lifecycle. */
+    harness: BrokerHarness;
+    /** The CLI/harness under test (e.g. "claude", "codex", "opencode"). */
+    cli: string;
+    /** Optional model override to pass to the agent CLI (e.g. "claude-haiku-4-5-20251001"). */
+    model?: string;
+    /** Unique suffix for isolating agent/channel names across runs. */
+    suffix: string;
+    /** Sleep helper. */
+    sleep: (ms: number) => Promise<void>;
+}
+/**
+ * A single phantom message: forward-looking intent to communicate expressed in
+ * plain text that was never backed by an actual `relay_inbound` send.
+ */
+export interface Phantom {
+    /** The agent that expressed the intent. */
+    agent: string;
+    /** The matched verb (e.g. "tell", "post", "send"). */
+    verb: string;
+    /** The parsed target, if the regex captured one (e.g. "Lead"). */
+    target?: string;
+    /** A short snippet of surrounding text for debugging. */
+    snippet: string;
+}
+/** An agent under test in a scenario, with the task prompt it was given. */
+export interface AgentInfo {
+    name: string;
+    cli: string;
+    /** Optional role label (e.g. "relay hop A"). */
+    role?: string;
+    /** The task prompt the agent was spawned with. */
+    prompt: string;
+}
+/** One message in a scenario's conversation, derived from a relay_inbound event. */
+export interface TranscriptEntry {
+    from: string;
+    target: string;
+    body: string;
+    /** True if `from` is one of the scenario's agents under test (a real send). */
+    fromAgent: boolean;
+    threadId?: string;
+}
+/** Raw, scenario-specific signal counts derived from the event stream. */
+export interface ScenarioResult {
+    id: string;
+    title: string;
+    /** Overall pass/fail for this scenario. */
+    pass: boolean;
+    /** Agents under test and the prompts they were given. */
+    agents: AgentInfo[];
+    /** The full message transcript (stimulus + agent sends), in order. */
+    transcript: TranscriptEntry[];
+    /** Number of messages the agent(s) actually sent (relay_inbound). */
+    sent: number;
+    /** Number of sends the scenario expected. */
+    expected: number;
+    /** Phantom messages detected (intent without a backing send). */
+    phantoms: Phantom[];
+    /** Total forward-looking intents detected (satisfied + phantom). */
+    totalIntents: number;
+    /** Protocol adherence score in [0,1], or null if not applicable. */
+    protocolAdherence: number | null;
+    /** Replies that targeted the wrong channel / a DM when a channel was expected. */
+    wrongChannelReplies: number;
+    /** True if no delivery_dropped / acl_denied events occurred. */
+    deliveryOk: boolean;
+    /** Coarse event counts for the report. */
+    events: {
+        relayInbound: number;
+        dropped: number;
+        aclDenied: number;
+    };
+    /** Lifecycle: number of confirmed add_agent calls in this run. */
+    spawnCount?: number;
+    /** Lifecycle: number of confirmed remove_agent calls in this run. */
+    releaseCount?: number;
+    /** Onboarding variant used (lifecycle scenarios only). */
+    onboarding?: string;
+    /** True when the agent used Claude's native Task tool instead of mcp__agent-relay__add_agent. */
+    nativeSubagentDetected?: boolean;
+    /** Optional human-readable notes (e.g. partial-chain detail). */
+    notes?: string;
+}
+/**
+ * Eval tiers:
+ * - `smoke`: leading prompts that name the exact tool. A plumbing canary — proves
+ *   the broker→MCP→agent→scoring path works; not a measure of protocol retention.
+ * - `realistic`: natural-language prompts where messaging is incidental to real
+ *   work and the protocol must come from the injected onboarding (skill + broker
+ *   hints) — what production agents actually get. This is the real benchmark.
+ */
+export type EvalTier = 'smoke' | 'realistic';
+/** A scenario the runner can execute against a harness. */
+export interface EvalScenario {
+    id: string;
+    title: string;
+    tier: EvalTier;
+    /** Channels the broker should subscribe to for this scenario. */
+    channels: string[];
+    /** If set, only run for these harnesses. */
+    harnessFilter?: string[];
+    /** Overall test timeout in ms. */
+    timeoutMs: number;
+    /** Onboarding variant (lifecycle scenarios only — used for report grouping). */
+    onboardingVariant?: string;
+    /**
+     * Orchestrate the scenario end-to-end: spawn agents, inject the stimulus,
+     * wait for responses, and score the captured events into a ScenarioResult.
+     */
+    run: (ctx: ScenarioContext) => Promise<ScenarioResult>;
+}
+/** Aggregated metrics for one harness across all scenarios. */
+export interface MetricSet {
+    messageSentRate: number;
+    phantomRate: number;
+    phantomCount: number;
+    protocolAdherence: number;
+    deliverySuccessRate: number;
+    wrongChannelReplies: number;
+    scenariosPassed: number;
+    scenariosTotal: number;
+    /** Lifecycle: fraction of s01/s03 scenarios where add_agent was called. */
+    spawnRate?: number;
+    /** Lifecycle: fraction of s02/s03 scenarios where remove_agent was called. */
+    releaseRate?: number;
+}
+/** A full report for one harness run. */
+export interface EvalReport {
+    schemaVersion: number;
+    startedAt: string;
+    durationMs: number;
+    harness: string;
+    gitSha: string;
+    env: {
+        realCli: boolean;
+        repeat: number;
+    };
+    metrics: MetricSet;
+    scenarios: ScenarioResult[];
+}
+/** The matrix roll-up across harnesses. */
+export interface MatrixReport {
+    schemaVersion: number;
+    startedAt: string;
+    gitSha: string;
+    harnesses: Record<string, MetricSet>;
+}
+export declare const SCHEMA_VERSION = 1;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAElD,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,4FAA4F;IAC5F,OAAO,EAAE,aAAa,CAAC;IACvB,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;IACZ,2FAA2F;IAC3F,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,4EAA4E;AAC5E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,gDAAgD;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oFAAoF;AACpF,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,IAAI,EAAE,OAAO,CAAC;IACd,yDAAyD;IACzD,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,sEAAsE;IACtE,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,qEAAqE;IACrE,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,oEAAoE;IACpE,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,kFAAkF;IAClF,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gEAAgE;IAChE,UAAU,EAAE,OAAO,CAAC;IACpB,0CAA0C;IAC1C,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iGAAiG;IACjG,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,WAAW,CAAC;AAE7C,2DAA2D;AAC3D,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,iEAAiE;IACjE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,4CAA4C;IAC5C,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,GAAG,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;CACxD;AAED,+DAA+D;AAC/D,MAAM,WAAW,SAAS;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,yCAAyC;AACzC,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE;QACH,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,SAAS,CAAC;IACnB,SAAS,EAAE,cAAc,EAAE,CAAC;CAC7B;AAED,2CAA2C;AAC3C,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,eAAO,MAAM,cAAc,IAAI,CAAC"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export const SCHEMA_VERSION = 1;
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA0KA,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-relay/evals",
-  "version": "8.3.1",
+  "version": "8.8.0",
   "description": "Agent Relay eval harness — scenario runner, broker harness, and scoring utilities for testing relay-connected agents",
   "type": "module",
   "main": "dist/index.js",
@@ -63,7 +63,7 @@
     "check": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "@agent-relay/harness-driver": "8.3.1"
+    "@agent-relay/harness-driver": "8.8.0"
   },
   "publishConfig": {
     "access": "public"