@agent-relay/evals 8.8.1 → 8.8.2

package/dist/index.d.ts

@@ -10,6 +10,6 @@
  * on. The full migration into this package is tracked in
  * specs/agent-relay-evals-package.md.
  */
-export type { EvalScenario, ScenarioContext, ScenarioResult, EvalTier, MetricSet, EvalReport, MatrixReport, AgentInfo, TranscriptEntry, Phantom, } from './types.js';
+export type { EvalScenario, ScenarioContext, ScenarioResult, EvalTier, MetricSet, EvalReport, MatrixReport, AgentInfo, TranscriptEntry, Phantom, MountScenario, MountScenarioResult, MountCellMetrics, } from './types.js';
 export { SCHEMA_VERSION } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,YAAY,EACV,YAAY,EACZ,eAAe,EACf,cAAc,EACd,QAAQ,EACR,SAAS,EACT,UAAU,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,OAAO,GACR,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,YAAY,EACV,YAAY,EACZ,eAAe,EACf,cAAc,EACd,QAAQ,EACR,SAAS,EACT,UAAU,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,OAAO,EAEP,aAAa,EACb,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAeH,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAmBH,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/scenarios/onboarding.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Onboarding variants for spawn/release reliability evals.
+ *
+ * Ordered from lightest to heaviest. The eval runner tests each variant and
+ * the goal is to find the MINIMUM text that achieves 10/10 reliability, so
+ * variants at the top of the list are preferred if they prove sufficient.
+ *
+ * bare        → zero spawn/release guidance (measures the baseline failure rate)
+ * one-liner   → single sentence naming both tools (minimum viable hint)
+ * brief       → tool names + parameters + when-to-use (compact but complete)
+ * skill       → full reference with examples (maximum clarity, maximum tokens)
+ */
+export type OnboardingVariant = 'bare' | 'one-liner' | 'brief' | 'skill';
+export declare const ONBOARDING_VARIANTS: readonly ["bare", "one-liner", "brief", "skill"];
+/**
+ * Return the onboarding text suffix for a given variant.
+ * Appended to the scenario-specific role description.
+ */
+export declare function onboardingText(variant: OnboardingVariant): string;
+//# sourceMappingURL=onboarding.d.ts.map

package/dist/scenarios/onboarding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"onboarding.d.ts","sourceRoot":"","sources":["../../src/scenarios/onboarding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC;AAEzE,eAAO,MAAM,mBAAmB,kDAKiB,CAAC;AAElD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CA8CjE"}

package/dist/scenarios/onboarding.js

@@ -0,0 +1,66 @@
+/**
+ * Onboarding variants for spawn/release reliability evals.
+ *
+ * Ordered from lightest to heaviest. The eval runner tests each variant and
+ * the goal is to find the MINIMUM text that achieves 10/10 reliability, so
+ * variants at the top of the list are preferred if they prove sufficient.
+ *
+ * bare        → zero spawn/release guidance (measures the baseline failure rate)
+ * one-liner   → single sentence naming both tools (minimum viable hint)
+ * brief       → tool names + parameters + when-to-use (compact but complete)
+ * skill       → full reference with examples (maximum clarity, maximum tokens)
+ */
+export const ONBOARDING_VARIANTS = [
+    'bare',
+    'one-liner',
+    'brief',
+    'skill',
+];
+/**
+ * Return the onboarding text suffix for a given variant.
+ * Appended to the scenario-specific role description.
+ */
+export function onboardingText(variant) {
+    switch (variant) {
+        case 'bare':
+            return '';
+        case 'one-liner':
+            return '\n\nCall mcp__agent-relay__add_agent to spawn a worker agent for a task, and mcp__agent-relay__remove_agent to release workers when they are done.';
+        case 'brief':
+            return `
+
+## Agent management
+- Spawn a relay worker: mcp__agent-relay__add_agent({ name, cli: "claude", task })
+  name = unique identifier, task = full instructions for the worker.
+- Release a relay worker: mcp__agent-relay__remove_agent({ name })
+When the task says to delegate or assign work, call add_agent. Release with remove_agent when the worker reports done.`;
+        case 'skill':
+            return `
+
+## Managing Worker Agents
+
+### Spawn a relay worker
+To delegate work, call:
+  mcp__agent-relay__add_agent({ name: "WorkerName", cli: "claude", task: "detailed instructions" })
+
+Required fields: name (unique string), cli ("claude"), task (full instructions for the worker).
+The relay worker will DM you "ACK: <understanding>" when it starts and "DONE: <result>" when complete.
+
+**Important**: When your task asks you to "assign to a worker", "delegate to an agent", or "spawn a relay worker",
+this means calling mcp__agent-relay__add_agent — never your built-in Task capability.
+
+### Release a relay worker
+As soon as a relay worker reports done, call:
+  mcp__agent-relay__remove_agent({ name: "WorkerName" })
+
+Always release relay workers when done — unreleased agents waste resources.
+
+### When to spawn vs do the work yourself
+If the task explicitly asks you to delegate or assign work to a worker, always spawn — do not do it yourself.
+Spawn for anything large, parallel, or that needs specialised focus.
+Only handle it yourself when the task is trivial AND you were not asked to delegate.`;
+        default:
+            throw new Error(`Unknown OnboardingVariant: ${variant}`);
+    }
+}
+//# sourceMappingURL=onboarding.js.map

package/dist/scenarios/onboarding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"onboarding.js","sourceRoot":"","sources":["../../src/scenarios/onboarding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,MAAM;IACN,WAAW;IACX,OAAO;IACP,OAAO;CACwC,CAAC;AAElD;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,OAA0B;IACvD,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,MAAM;YACT,OAAO,EAAE,CAAC;QAEZ,KAAK,WAAW;YACd,OAAO,oJAAoJ,CAAC;QAE9J,KAAK,OAAO;YACV,OAAO;;;;;;uHAM0G,CAAC;QAEpH,KAAK,OAAO;YACV,OAAO;;;;;;;;;;;;;;;;;;;;;;;qFAuBwE,CAAC;QAElF;YACE,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAiB,EAAE,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}

package/dist/scoring/mount.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Filesystem scoring utilities for agents that use a relayfile writeback mount.
+ *
+ * The writeback pattern: an agent writes a JSON file to a path under the
+ * .integrations/ mount (e.g. .integrations/slack/channels/<id>/messages/msg.json)
+ * and the relayfile writeback consumer picks it up and dispatches to the provider.
+ *
+ * Usage:
+ *   const snapshot = snapshotMount(fixtureDir)
+ *   // ... spawn agent, wait for exit ...
+ *   const newFiles = newMountFiles(fixtureDir, snapshot)
+ *   const score = scoreMountRun({ mountDir: fixtureDir, newFiles, expectedPathPrefix, events })
+ */
+import type { BrokerEvent } from '@agent-relay/harness-driver';
+/** Per-dimension scoring for one mount writeback eval run. */
+export interface MountScore {
+    /** Agent created at least one file under the mount root. */
+    wroteSomething: boolean;
+    /** File(s) landed under the expected path prefix (correct provider + resource). */
+    correctPath: boolean;
+    /** All files at the expected path are syntactically valid JSON. */
+    jsonValid: boolean;
+    /** Agent wrote under discovery/ — schema-only, must never happen. */
+    discoveryViolation: boolean;
+    /** Agent sent at least one relay_inbound message instead of/in addition to writing a file. */
+    usedRelayMessaging: boolean;
+    /** Agent exited cleanly rather than timing out. */
+    cleanExit: boolean;
+    /** Overall pass: correctPath && jsonValid && !discoveryViolation. */
+    pass: boolean;
+    /** All files created by the agent, relative to mountDir. */
+    filesWritten: string[];
+    /** Subset of filesWritten that match expectedPathPrefix. */
+    filesAtCorrectPath: string[];
+}
+/** Options for scoreMountRun(). */
+export interface ScoreMountRunOptions {
+    /** Root of the .integrations/ mount directory (absolute path). */
+    mountDir: string;
+    /** Files created by the agent since spawning (from newMountFiles). */
+    newFiles: string[];
+    /**
+     * Expected path prefix the writeback file must be under (relative to mountDir).
+     * Example: ".integrations/slack/channels/C12345__general/messages/"
+     */
+    expectedPathPrefix: string;
+    /** Broker events captured during the run. */
+    events: BrokerEvent[];
+    /** Whether the agent exited without timing out. Defaults to true. */
+    cleanExit?: boolean;
+}
+/** Summary stats across repeated runs of one scenario×variant cell. */
+export interface MountCellStats {
+    runs: number;
+    passed: number;
+    passRate: number;
+    wroteSomethingRate: number;
+    correctPathRate: number;
+    discoveryViolationRate: number;
+    usedRelayMessagingRate: number;
+}
+/**
+ * Snapshot all file paths currently under mountDir.
+ * Call before spawning the agent; diff with newMountFiles after.
+ */
+export declare function snapshotMount(mountDir: string): Set<string>;
+/**
+ * Return absolute paths of files under mountDir that weren't in the snapshot.
+ * These are the files the agent created during the run.
+ */
+export declare function newMountFiles(mountDir: string, snapshot: Set<string>): string[];
+/**
+ * Score an agent run against a mount writeback scenario.
+ *
+ * Pass = agent wrote to the correct path, the file is valid JSON, and it did
+ * NOT write under discovery/.
+ */
+export declare function scoreMountRun(opts: ScoreMountRunOptions): MountScore;
+/**
+ * Roll up repeated MountScore results into pass-rate statistics for one cell
+ * (scenario × variant) in a report matrix.
+ */
+export declare function mountCellStats(scores: MountScore[]): MountCellStats;
+//# sourceMappingURL=mount.d.ts.map

package/dist/scoring/mount.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mount.d.ts","sourceRoot":"","sources":["../../src/scoring/mount.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAI/D,8DAA8D;AAC9D,MAAM,WAAW,UAAU;IACzB,4DAA4D;IAC5D,cAAc,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,WAAW,EAAE,OAAO,CAAC;IACrB,mEAAmE;IACnE,SAAS,EAAE,OAAO,CAAC;IACnB,qEAAqE;IACrE,kBAAkB,EAAE,OAAO,CAAC;IAC5B,8FAA8F;IAC9F,kBAAkB,EAAE,OAAO,CAAC;IAC5B,mDAAmD;IACnD,SAAS,EAAE,OAAO,CAAC;IACnB,qEAAqE;IACrE,IAAI,EAAE,OAAO,CAAC;IACd,4DAA4D;IAC5D,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,4DAA4D;IAC5D,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,mCAAmC;AACnC,MAAM,WAAW,oBAAoB;IACnC,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;OAGG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,6CAA6C;IAC7C,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,qEAAqE;IACrE,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,uEAAuE;AACvE,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAYD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAI3D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAI/E;AAID;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,oBAAoB,GAAG,UAAU,CA+BpE;AAID;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAsBnE"}

package/dist/scoring/mount.js

@@ -0,0 +1,106 @@
+/**
+ * Filesystem scoring utilities for agents that use a relayfile writeback mount.
+ *
+ * The writeback pattern: an agent writes a JSON file to a path under the
+ * .integrations/ mount (e.g. .integrations/slack/channels/<id>/messages/msg.json)
+ * and the relayfile writeback consumer picks it up and dispatches to the provider.
+ *
+ * Usage:
+ *   const snapshot = snapshotMount(fixtureDir)
+ *   // ... spawn agent, wait for exit ...
+ *   const newFiles = newMountFiles(fixtureDir, snapshot)
+ *   const score = scoreMountRun({ mountDir: fixtureDir, newFiles, expectedPathPrefix, events })
+ */
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+// ── Mount file tracking ───────────────────────────────────────────────────────
+function walkDir(dir, out) {
+    if (!existsSync(dir))
+        return;
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const full = join(dir, entry.name);
+        entry.isDirectory() ? walkDir(full, out) : out.add(full);
+    }
+}
+/**
+ * Snapshot all file paths currently under mountDir.
+ * Call before spawning the agent; diff with newMountFiles after.
+ */
+export function snapshotMount(mountDir) {
+    const existing = new Set();
+    walkDir(mountDir, existing);
+    return existing;
+}
+/**
+ * Return absolute paths of files under mountDir that weren't in the snapshot.
+ * These are the files the agent created during the run.
+ */
+export function newMountFiles(mountDir, snapshot) {
+    const current = new Set();
+    walkDir(mountDir, current);
+    return [...current].filter((f) => !snapshot.has(f));
+}
+// ── Scoring ───────────────────────────────────────────────────────────────────
+/**
+ * Score an agent run against a mount writeback scenario.
+ *
+ * Pass = agent wrote to the correct path, the file is valid JSON, and it did
+ * NOT write under discovery/.
+ */
+export function scoreMountRun(opts) {
+    const { mountDir, newFiles, expectedPathPrefix, events, cleanExit = true } = opts;
+    const relFiles = newFiles.map((f) => relative(mountDir, f));
+    const inExpectedPath = relFiles.filter((p) => p.startsWith(expectedPathPrefix));
+    const inDiscovery = relFiles.filter((p) => p.includes('/discovery/') || p.startsWith('discovery/'));
+    const jsonValid = inExpectedPath.length > 0 &&
+        inExpectedPath.every((p) => {
+            try {
+                JSON.parse(readFileSync(join(mountDir, p), 'utf8'));
+                return true;
+            }
+            catch {
+                return false;
+            }
+        });
+    const usedRelayMessaging = events.some((e) => e.kind === 'relay_inbound');
+    return {
+        wroteSomething: relFiles.length > 0,
+        correctPath: inExpectedPath.length > 0,
+        jsonValid,
+        discoveryViolation: inDiscovery.length > 0,
+        usedRelayMessaging,
+        cleanExit,
+        pass: inExpectedPath.length > 0 && jsonValid && inDiscovery.length === 0,
+        filesWritten: relFiles,
+        filesAtCorrectPath: inExpectedPath,
+    };
+}
+// ── Aggregate stats ───────────────────────────────────────────────────────────
+/**
+ * Roll up repeated MountScore results into pass-rate statistics for one cell
+ * (scenario × variant) in a report matrix.
+ */
+export function mountCellStats(scores) {
+    const n = scores.length;
+    if (n === 0)
+        return {
+            runs: 0,
+            passed: 0,
+            passRate: 0,
+            wroteSomethingRate: 0,
+            correctPathRate: 0,
+            discoveryViolationRate: 0,
+            usedRelayMessagingRate: 0,
+        };
+    const rate = (pred) => scores.filter(pred).length / n;
+    return {
+        runs: n,
+        passed: scores.filter((s) => s.pass).length,
+        passRate: rate((s) => s.pass),
+        wroteSomethingRate: rate((s) => s.wroteSomething),
+        correctPathRate: rate((s) => s.correctPath),
+        discoveryViolationRate: rate((s) => s.discoveryViolation),
+        usedRelayMessagingRate: rate((s) => s.usedRelayMessaging),
+    };
+}
+//# sourceMappingURL=mount.js.map

package/dist/scoring/mount.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mount.js","sourceRoot":"","sources":["../../src/scoring/mount.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAwD3C,iFAAiF;AAEjF,SAAS,OAAO,CAAC,GAAW,EAAE,GAAgB;IAC5C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO;IAC7B,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC5B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB,EAAE,QAAqB;IACnE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC3B,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,iFAAiF;AAEjF;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAA0B;IACtD,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;IAElF,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAChF,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC;IAEpG,MAAM,SAAS,GACb,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;gBACpD,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;IAEL,MAAM,kBAAkB,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,eAAe,CAAC,CAAC;IAE1E,OAAO;QACL,cAAc,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC;QACnC,WAAW,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC;QACtC,SAAS;QACT,kBAAkB,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC;QAC1C,kBAAkB;QAClB,SAAS;QACT,IAAI,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxE,YAAY,EAAE,QAAQ;QACtB,kBAAkB,EAAE,cAAc;KACnC,CAAC;AACJ,CAAC;AAED,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAoB;IACjD,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,IAAI,CAAC,KAAK,CAAC;QACT,OAAO;YACL,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,CAAC;YACT,QAAQ,EAAE,CAAC;YACX,kBAAkB,EAAE,CAAC;YACrB,eAAe,EAAE,CAAC;YAClB,sBAAsB,EAAE,CAAC;YACzB,sBAAsB,EAAE,CAAC;SAC1B,CAAC;IACJ,MAAM,IAAI,GAAG,CAAC,IAAgC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClF,OAAO;QACL,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM;QAC3C,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7B,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;QACjD,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3C,sBAAsB,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC;QACzD,sBAAsB,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC;KAC1D,CAAC;AACJ,CAAC"}

package/dist/types.d.ts

@@ -158,4 +158,48 @@ export interface MatrixReport {
     harnesses: Record<string, MetricSet>;
 }
 export declare const SCHEMA_VERSION = 1;
+/**
+ * A scenario that tests whether an agent correctly writes a writeback file
+ * to the relayfile .integrations/ mount instead of (or alongside) relay messaging.
+ */
+export interface MountScenario {
+    id: string;
+    title: string;
+    /**
+     * Path prefix (relative to the fixture/mount root) the agent must write
+     * under to pass. Example: ".integrations/slack/channels/C12345__general/messages/"
+     */
+    expectedPathPrefix: string;
+    /**
+     * Task text to inject into the agent at spawn.
+     * The eval runner may prepend a variant-specific prefix before this.
+     */
+    task: string;
+}
+/** One scored run of a MountScenario. */
+export interface MountScenarioResult {
+    scenarioId: string;
+    scenarioTitle: string;
+    /** Onboarding variant under test (e.g. "bare", "claude-md", "slim-inject", "full-inject"). */
+    variant: string;
+    /** 1-based repetition index within this scenario×variant cell. */
+    run: number;
+    pass: boolean;
+    wroteSomething: boolean;
+    correctPath: boolean;
+    jsonValid: boolean;
+    discoveryViolation: boolean;
+    usedRelayMessaging: boolean;
+    filesWritten: string[];
+    durationMs: number;
+    error?: string;
+}
+/** Aggregated metrics for one scenario×variant cell across repeated runs. */
+export interface MountCellMetrics {
+    scenarioId: string;
+    variant: string;
+    runs: number;
+    passed: number;
+    passRate: number;
+}
 //# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAElD,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,4FAA4F;IAC5F,OAAO,EAAE,aAAa,CAAC;IACvB,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;IACZ,2FAA2F;IAC3F,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,4EAA4E;AAC5E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,gDAAgD;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oFAAoF;AACpF,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,IAAI,EAAE,OAAO,CAAC;IACd,yDAAyD;IACzD,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,sEAAsE;IACtE,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,qEAAqE;IACrE,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,oEAAoE;IACpE,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,kFAAkF;IAClF,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gEAAgE;IAChE,UAAU,EAAE,OAAO,CAAC;IACpB,0CAA0C;IAC1C,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iGAAiG;IACjG,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,WAAW,CAAC;AAE7C,2DAA2D;AAC3D,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,iEAAiE;IACjE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,4CAA4C;IAC5C,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,GAAG,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;CACxD;AAED,+DAA+D;AAC/D,MAAM,WAAW,SAAS;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,yCAAyC;AACzC,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE;QACH,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,SAAS,CAAC;IACnB,SAAS,EAAE,cAAc,EAAE,CAAC;CAC7B;AAED,2CAA2C;AAC3C,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,eAAO,MAAM,cAAc,IAAI,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAElD,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,4FAA4F;IAC5F,OAAO,EAAE,aAAa,CAAC;IACvB,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;IACZ,2FAA2F;IAC3F,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,4EAA4E;AAC5E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,gDAAgD;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oFAAoF;AACpF,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,IAAI,EAAE,OAAO,CAAC;IACd,yDAAyD;IACzD,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,sEAAsE;IACtE,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,qEAAqE;IACrE,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,oEAAoE;IACpE,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,kFAAkF;IAClF,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gEAAgE;IAChE,UAAU,EAAE,OAAO,CAAC;IACpB,0CAA0C;IAC1C,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iGAAiG;IACjG,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,WAAW,CAAC;AAE7C,2DAA2D;AAC3D,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,iEAAiE;IACjE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,4CAA4C;IAC5C,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,GAAG,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;CACxD;AAED,+DAA+D;AAC/D,MAAM,WAAW,SAAS;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,yCAAyC;AACzC,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE;QACH,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,SAAS,CAAC;IACnB,SAAS,EAAE,cAAc,EAAE,CAAC;CAC7B;AAED,2CAA2C;AAC3C,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,eAAO,MAAM,cAAc,IAAI,CAAC;AAOhC;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,8FAA8F;IAC9F,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,OAAO,CAAC;IACd,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,6EAA6E;AAC7E,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-relay/evals",
-  "version": "8.8.1",
+  "version": "8.8.2",
   "description": "Agent Relay eval harness — scenario runner, broker harness, and scoring utilities for testing relay-connected agents",
   "type": "module",
   "main": "dist/index.js",
@@ -46,6 +46,11 @@
       "import": "./dist/scenarios/onboarding.js",
       "default": "./dist/scenarios/onboarding.js"
     },
+    "./scoring/mount": {
+      "types": "./dist/scoring/mount.d.ts",
+      "import": "./dist/scoring/mount.js",
+      "default": "./dist/scoring/mount.js"
+    },
     "./scenarios/core": {
       "types": "./dist/scenarios/index.d.ts",
       "import": "./dist/scenarios/index.js",
@@ -63,7 +68,7 @@
     "check": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "@agent-relay/harness-driver": "8.8.1"
+    "@agent-relay/harness-driver": "8.8.2"
   },
   "publishConfig": {
     "access": "public"