@alumnium/test 0.24.2 → 0.25.0-alpha.1

package/README.md

@@ -1,3 +1,3 @@
 # @alumnium/test
 
-Alumnium test record and replay: agent-agnostic recordings, masking and the test cache. See [the main `alumnium` package page](https://www.npmjs.com/package/alumnium) for more details.
+Alumnium test runner. See [the main `alumnium` package page](https://www.npmjs.com/package/alumnium) for more details.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@alumnium/test",
-  "version": "0.24.2",
-  "description": "Alumnium test record and replay: agent-agnostic recordings, masking and the test cache",
+  "version": "0.25.0-alpha.1",
+  "description": "Alumnium test runner",
   "license": "MIT",
   "author": "Alex Rodionov <p0deje@gmail.com>",
   "contributors": [
@@ -32,8 +32,9 @@
     }
   },
   "dependencies": {
-    "@alumnium/core": "0.24.1",
+    "@alumnium/core": "0.25.0-alpha.1",
     "@js-fns/xxhash": "^0.1.0",
+    "@modelcontextprotocol/sdk": "^1.29.0",
     "zod": "^4.4.1"
   }
 }

package/src/index.cjs

@@ -68,11 +68,15 @@ var __export = (target, all) => {
 // test/src/index.ts
 var exports_src = {};
 __export(exports_src, {
+  BashExecutor: () => BashExecutor,
   FileTestCache: () => FileTestCache,
   Masker: () => Masker,
+  McpExecutor: () => McpExecutor,
+  Player: () => Player,
   Recorder: () => Recorder,
   Recording: () => Recording,
   RecordingLeakError: () => RecordingLeakError,
+  composeExecutors: () => composeExecutors,
 });
 module.exports = __toCommonJS(exports_src);
 
@@ -585,6 +589,12 @@ class ToolCallMasker {
       return [];
     }
     const json = parseJsonStructure(text);
+    if (!json && options.secretsOnly) {
+      logger4.debug(
+        `Failed tool call ${callId} output is not JSON, ignoring it`,
+      );
+      return [];
+    }
     if (!json) {
       logger4.info(
         `Tool call ${callId} output is not a JSON object or array, ignoring it for masking. ` +
@@ -805,8 +815,421 @@ class Masker {
     );
   }
 }
-// test/src/Recorder.ts
+// test/src/player/BashExecutor.ts
+var import_node_child_process = require("node:child_process");
+var import_zod2 = __toESM(require("zod"), 1);
 var import_internal5 = require("@alumnium/core/internal");
+var { logger: logger5 } = import_internal5.Telemetry.get(
+  "packages/typescript/test/src/player/BashExecutor.ts",
+);
+var BASH_TOOL = "Bash";
+var DEFAULT_TIMEOUT_MS = 120000;
+var BashInput = import_zod2.default.object({
+  command: import_zod2.default.string(),
+  timeout: import_zod2.default.number().positive().optional(),
+});
+
+class BashExecutor {
+  #props;
+  constructor(props) {
+    this.#props = props;
+  }
+  handles(call) {
+    return !call.server && call.tool === BASH_TOOL;
+  }
+  async execute(call) {
+    const { command, timeout } = BashInput.parse(call.input);
+    if (!this.#allows(command))
+      throw new Error(`Command is not allowed to replay: ${command}`);
+    logger5.debug(`Running command: ${command}`);
+    const { code, stdout, stderr } = await this.#run(command, {
+      cwd: this.#props.cwd,
+      env: { ...process.env, ...this.#props.env },
+      timeoutMs: timeout ?? this.#props.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    });
+    if (code === 0) return { content: stdout, isError: false };
+    return {
+      content: [stdout, stderr, `Exit code ${code}`].filter(Boolean).join(`
+`),
+      isError: true,
+    };
+  }
+  #allows(command) {
+    return this.#props.allow.some((allowed) =>
+      typeof allowed === "string"
+        ? command.startsWith(allowed)
+        : allowed.test(command),
+    );
+  }
+  #run(command, options) {
+    return new Promise((resolve, reject) => {
+      const child = import_node_child_process.spawn("bash", ["-c", command], {
+        cwd: options.cwd,
+        env: options.env,
+        timeout: options.timeoutMs,
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+      let stdout = "";
+      let stderr = "";
+      child.stdout.setEncoding("utf8").on("data", (data) => (stdout += data));
+      child.stderr.setEncoding("utf8").on("data", (data) => (stderr += data));
+      child.on("error", reject);
+      child.on("close", (code) => resolve({ code, stdout, stderr }));
+    });
+  }
+}
+// test/src/player/McpExecutor.ts
+var import_client = require("@modelcontextprotocol/sdk/client/index.js");
+var import_stdio = require("@modelcontextprotocol/sdk/client/stdio.js");
+var import_streamableHttp = require("@modelcontextprotocol/sdk/client/streamableHttp.js");
+var import_internal6 = require("@alumnium/core/internal");
+var { logger: logger6 } = import_internal6.Telemetry.get(
+  "packages/typescript/test/src/player/McpExecutor.ts",
+);
+var DEFAULT_RETRIES = 2;
+var DEFAULT_RETRY_DELAY_MS = 1000;
+
+class McpExecutor {
+  #servers;
+  #retries;
+  #retryDelayMs;
+  #clients = new Map();
+  constructor(props) {
+    this.#servers = props.servers;
+    this.#retries = props.retries ?? DEFAULT_RETRIES;
+    this.#retryDelayMs = props.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
+  }
+  handles(call) {
+    return Boolean(call.server && call.server in this.#servers);
+  }
+  async execute(call, options = {}) {
+    const toolKey = Recording.toolKey(call);
+    const client = await this.#client(call.server);
+    for (let attempt = 0; ; attempt++) {
+      const canRetry = attempt < this.#retries;
+      const delayMs = this.#retryDelayMs * 2 ** attempt;
+      try {
+        const result = await client.callTool({
+          name: call.tool,
+          arguments: call.input,
+          ...(options.meta ? { _meta: options.meta } : {}),
+        });
+        const toolResult = {
+          content: result.content ?? [],
+          isError: Boolean(result.isError),
+        };
+        if (!toolResult.isError || !canRetry) return toolResult;
+        logger6.warn(`Tool '${toolKey}' reported an error, retrying`);
+      } catch (error) {
+        if (!canRetry) throw error;
+        logger6.warn(`Tool '${toolKey}' failed, retrying: {error}`, { error });
+      }
+      await new Promise((resolve) => setTimeout(resolve, delayMs));
+    }
+  }
+  async close() {
+    const clients = [...this.#clients];
+    this.#clients.clear();
+    await Promise.all(
+      clients.map(async ([name, client]) => {
+        if (this.#servers[name]?.type === "client") return;
+        try {
+          await (await client).close();
+        } catch (error) {
+          logger6.warn(`Failed to close MCP server '${name}': {error}`, {
+            error,
+          });
+        }
+      }),
+    );
+  }
+  #client(name) {
+    let client = this.#clients.get(name);
+    if (!client) {
+      client = this.#connect(name);
+      client.catch(() => this.#clients.delete(name));
+      this.#clients.set(name, client);
+    }
+    return client;
+  }
+  async #connect(name) {
+    const server = this.#servers[name];
+    if (!server) throw new Error(`Unknown MCP server '${name}'`);
+    if (server.type === "client") return server.client;
+    logger6.debug(`Connecting to MCP server '${name}'`);
+    const transport =
+      server.type === "http"
+        ? new import_streamableHttp.StreamableHTTPClientTransport(
+            new URL(server.url),
+            {
+              requestInit: { headers: server.headers ?? {} },
+            },
+          )
+        : new import_stdio.StdioClientTransport({
+            command: server.command,
+            args: server.args ?? [],
+            env: { ...this.#definedEnv(), ...server.env },
+            ...(server.cwd ? { cwd: server.cwd } : {}),
+            stderr: "inherit",
+          });
+    const client = new import_client.Client({
+      name: "alumnium-test-player",
+      version: import_internal6.ALUMNIUM_CORE_VERSION,
+    });
+    await client.connect(transport);
+    return client;
+  }
+  #definedEnv() {
+    return Object.fromEntries(
+      Object.entries(process.env).filter((entry) => entry[1] !== undefined),
+    );
+  }
+}
+// test/src/player/Player.ts
+var import_zod3 = __toESM(require("zod"), 1);
+var import_internal7 = require("@alumnium/core/internal");
+
+// test/src/masker/alumniumMaskerProps.ts
+var DEFAULT_ALUMNIUM_SERVER = "alumnium";
+var ALUMNIUM_START_TOOL = "start";
+function alumniumMaskerProps(props, server = DEFAULT_ALUMNIUM_SERVER) {
+  return {
+    ...props,
+    registerResponsePaths: {
+      ...alumniumRegisterResponsePaths(server),
+      ...props.registerResponsePaths,
+    },
+    wholeValueOnlyTools: [`${server}/*`, ...(props.wholeValueOnlyTools ?? [])],
+  };
+}
+function alumniumRegisterResponsePaths(server) {
+  return {
+    [`${server}/*`]: [],
+    [`${server}/${ALUMNIUM_START_TOOL}`]: ["id"],
+  };
+}
+
+// test/src/player/Player.ts
+var { logger: logger7 } = import_internal7.Telemetry.get(
+  "packages/typescript/test/src/player/Player.ts",
+);
+var START_TOOL = "start";
+var STOP_TOOL = "stop";
+var CHECK_TOOL = "check";
+var NO_CACHE_META = { "alumnium.ai/cache": false };
+var NO_CHANGE_ANALYSIS_META = { "alumnium.ai/change-analysis": false };
+var DEFAULT_SKIP_TOOLS = ["ToolSearch", "TodoWrite"];
+var DriverOutput = import_zod3.default.object({
+  id: import_zod3.default.string(),
+});
+var CheckOutput = import_zod3.default.object({
+  result: import_zod3.default.enum(["success", "failure"]),
+});
+
+class Player {
+  #options;
+  #maskerProps;
+  constructor(props) {
+    const {
+      executor,
+      alumniumServer = DEFAULT_ALUMNIUM_SERVER,
+      skipTools = DEFAULT_SKIP_TOOLS,
+      checkConfirmations,
+      observer = {},
+      ...maskerProps
+    } = props;
+    this.#options = {
+      executor,
+      server: alumniumServer,
+      skipTools: new Set(skipTools),
+      checkConfirmations,
+      observer,
+    };
+    this.#maskerProps = alumniumMaskerProps(maskerProps, alumniumServer);
+  }
+  async play(recording) {
+    const run = new PlayerRun(this.#options, new Masker(this.#maskerProps));
+    try {
+      return await run.play(recording);
+    } finally {
+      await run.stopDrivers();
+    }
+  }
+}
+
+class Diverged extends Error {}
+
+class PlayerRun {
+  #options;
+  #masker;
+  #driverIds = new Set();
+  constructor(options, masker) {
+    this.#options = options;
+    this.#masker = masker;
+  }
+  get #observer() {
+    return this.#options.observer;
+  }
+  async play(recording) {
+    const stepsCount = Recording.executableStepsCount(recording);
+    let stepNumber = 0;
+    for (const [stepIndex, step] of recording.steps.entries()) {
+      if (step.kind === "narration") {
+        this.#observer.onNarration?.(step);
+        continue;
+      }
+      logger7.info(`Replaying step ${++stepNumber}/${stepsCount}`);
+      try {
+        await this.#playToolStep(step);
+      } catch (error) {
+        const message = error instanceof Diverged ? error.message : `${error}`;
+        logger7.error(`Replay diverged at step ${stepNumber}: ${message}`);
+        return { status: "diverged", error: message, stepIndex };
+      }
+    }
+    logger7.info(`Replayed all ${stepsCount} steps`);
+    return { status: "replayed", verdict: recording.verdict };
+  }
+  async stopDrivers() {
+    for (const id of this.#driverIds) {
+      try {
+        logger7.info(`Stopping driver ${id} the replay did not stop`);
+        await this.#options.executor.execute({
+          server: this.#options.server,
+          tool: STOP_TOOL,
+          input: { id, save_cache: false },
+        });
+      } catch (error) {
+        logger7.warn(`Failed to stop driver ${id}: {error}`, { error });
+      }
+    }
+    this.#driverIds.clear();
+  }
+  async #playToolStep(step) {
+    const { call: recordedCall, result: recordedResult } = step;
+    const { id, ...callProps } = recordedCall;
+    const toolKey = Recording.toolKey(recordedCall);
+    if (recordedResult.isError)
+      return this.#skip(recordedCall, "it failed when recorded");
+    if (!this.#options.executor.handles(callProps)) {
+      if (this.#options.skipTools.has(toolKey))
+        return this.#skip(recordedCall, "it only exists inside the agent");
+      throw new Diverged(`No executor for tool '${toolKey}'`);
+    }
+    const call = {
+      ...callProps,
+      input: this.#masker.unmaskInput(toolKey, recordedCall.input),
+    };
+    const unresolved = Masker.findUnresolvedMasks(call.input);
+    if (unresolved.length)
+      throw new Diverged(
+        `Tool '${toolKey}' input has unresolved masks: ${unresolved.join(", ")}`,
+      );
+    const result = await this.#execute(
+      call,
+      call.server === this.#options.server && call.tool === START_TOOL
+        ? { meta: NO_CHANGE_ANALYSIS_META }
+        : undefined,
+    );
+    if (result.isError)
+      throw new Diverged(
+        `Tool '${toolKey}' failed: ${contentTexts(result.content).join(`
+`)}`,
+      );
+    this.#masker.registerOutput(id, result.content, toolKey);
+    if (call.server !== this.#options.server) return;
+    switch (call.tool) {
+      case START_TOOL: {
+        const driverId = this.#parseOutput(result.content, DriverOutput)?.id;
+        if (driverId) this.#driverIds.add(driverId);
+        break;
+      }
+      case STOP_TOOL: {
+        const driverId = DriverOutput.safeParse(call.input).data?.id;
+        if (driverId) this.#driverIds.delete(driverId);
+        break;
+      }
+      case CHECK_TOOL:
+        await this.#compareCheck(call, recordedResult, result);
+        break;
+    }
+  }
+  async #execute(call, options) {
+    this.#observer.onToolCall?.(call);
+    const result = await this.#options.executor.execute(call, options);
+    this.#observer.onToolResult?.(call, result);
+    return result;
+  }
+  #skip(call, reason) {
+    logger7.info(`Skipping tool '${Recording.toolKey(call)}', ${reason}`);
+    this.#observer.onToolSkipped?.(call, reason);
+  }
+  async #compareCheck(call, recorded, replayed) {
+    const confirmations =
+      this.#options.checkConfirmations ??
+      import_internal7.Env.ALUMNIUM_CHECK_CONFIRMATIONS;
+    const expected = this.#checkVerdict(recorded);
+    let actual = this.#checkVerdict(replayed);
+    let outcome;
+    if (!expected || !actual) {
+      throw new Diverged(
+        `Cannot read '${Recording.toolKey(call)}' verdict to compare`,
+      );
+    } else if (expected === actual) {
+      outcome = "agreed";
+    } else if (expected === "failure") {
+      outcome = "improved";
+    } else {
+      let attempt = 0;
+      while (actual !== expected && attempt++ < confirmations) {
+        logger7.info(`Re-asking check ${attempt}/${confirmations}`);
+        const result = await this.#execute(call, { meta: NO_CACHE_META });
+        actual = result.isError ? null : this.#checkVerdict(result);
+      }
+      outcome = actual === expected ? "unstable" : "disagreed";
+    }
+    this.#observer.onCheck?.(call, outcome);
+    if (outcome === "disagreed")
+      throw new Diverged(
+        `Check '${call.input["statement"]}' failed where it passed when recorded` +
+          (confirmations
+            ? `, re-asked ${confirmations} times with the cache bypassed`
+            : ""),
+      );
+  }
+  #checkVerdict(result) {
+    return this.#parseOutput(result.content, CheckOutput)?.result ?? null;
+  }
+  #parseOutput(content, schema) {
+    for (const text of contentTexts(content)) {
+      const parsed = schema.safeParse(parseJson(text));
+      if (parsed.success) return parsed.data;
+    }
+    return null;
+  }
+}
+// test/src/player/ToolExecutor.ts
+function composeExecutors(...executors) {
+  const find = (call) => executors.find((executor) => executor.handles(call));
+  return {
+    handles: (call) => Boolean(find(call)),
+    async execute(call, options) {
+      const executor = find(call);
+      if (!executor)
+        throw new Error(
+          `No executor handles tool '${Recording.toolKey(call)}'`,
+        );
+      return executor.execute(call, options);
+    },
+    async close() {
+      await Promise.all(
+        executors.map(async (executor) => await executor.close?.()),
+      );
+    },
+  };
+}
+// test/src/Recorder.ts
+var import_internal8 = require("@alumnium/core/internal");
 
 // test/src/RecordingLeakError.ts
 class RecordingLeakError extends Error {
@@ -822,11 +1245,9 @@ class RecordingLeakError extends Error {
 }
 
 // test/src/Recorder.ts
-var { logger: logger5 } = import_internal5.Telemetry.get(
+var { logger: logger8 } = import_internal8.Telemetry.get(
   "packages/typescript/test/src/Recorder.ts",
 );
-var DEFAULT_ALUMNIUM_SERVER = "alumnium";
-var ALUMNIUM_START_TOOL = "start";
 
 class Recorder {
   #text;
@@ -841,24 +1262,13 @@ class Recorder {
     this.#version = version ?? Recording.DEFAULT_VERSION;
     this.#text = text;
     this.#agent = agent;
-    const server = alumniumServer ?? DEFAULT_ALUMNIUM_SERVER;
-    this.#masker = new Masker({
-      ...maskerProps,
-      registerResponsePaths: {
-        ...this.#alumniumRegisterResponsePaths(server),
-        ...maskerProps.registerResponsePaths,
-      },
-      wholeValueOnlyTools: [
-        `${server}/*`,
-        ...(maskerProps.wholeValueOnlyTools ?? []),
-      ],
-    });
+    this.#masker = new Masker(alumniumMaskerProps(maskerProps, alumniumServer));
   }
   onToolUse(use) {
     this.#assertNotFinished();
     const { id, server, tool, input } = use;
     const toolKey = Recording.toolKey({ server, tool });
-    logger5.debug(`Recording tool use '${toolKey}' ({id})`, { id });
+    logger8.debug(`Recording tool use '${toolKey}' ({id})`, { id });
     this.#pendingCalls.set(id, {
       id,
       ...(server ? { server } : {}),
@@ -872,11 +1282,11 @@ class Recorder {
     const isError = Boolean(toolResult.isError);
     const call = this.#pendingCalls.get(id);
     if (!call) {
-      logger5.debug(`No recorded tool use for result '${id}', ignoring it`);
+      logger8.debug(`No recorded tool use for result '${id}', ignoring it`);
       return;
     }
     this.#pendingCalls.delete(id);
-    logger5.debug(`Recording '${call.tool}' tool result`);
+    logger8.debug(`Recording '${call.tool}' tool result`);
     this.#masker.registerOutput(id, content, Recording.toolKey(call), {
       secretsOnly: isError,
     });
@@ -898,7 +1308,7 @@ class Recorder {
   finish(verdict) {
     this.#assertNotFinished();
     if (this.#pendingCalls.size)
-      logger5.warn(
+      logger8.warn(
         `Dropping ${this.#pendingCalls.size} tool calls with no result: ${[
           ...this.#pendingCalls.values(),
         ]
@@ -911,7 +1321,7 @@ class Recorder {
       version: this.#version,
       text: this.#text,
       agent: this.#agent,
-      alumnium: { version: import_internal5.ALUMNIUM_VERSION },
+      alumnium: { version: import_internal8.ALUMNIUM_CORE_VERSION },
       hostVars: [],
       steps: this.#steps.map((step) => this.#maskStep(step)),
       verdict: {
@@ -946,16 +1356,10 @@ class Recorder {
       };
     }
   }
-  #alumniumRegisterResponsePaths(server = DEFAULT_ALUMNIUM_SERVER) {
-    return {
-      [`${server}/*`]: [],
-      [`${server}/${ALUMNIUM_START_TOOL}`]: ["id"],
-    };
-  }
   #assertNotFinished() {
     if (this.#recording) throw new Error("Recording is already finished");
   }
 }
 
-//# debugId=1012AF1D16D4DEA864756E2164756E21
+//# debugId=18EDAFFEFAD0500964756E2164756E21
 //# sourceMappingURL=index.cjs.map