@alumnium/test 0.0.0-stage → 0.24.1

package/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Alexey Rodionov, Tatiana Shepeleva
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -1,3 +1,3 @@
-# Temporary Holding Version
+# @alumnium/test
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Alumnium test record and replay: agent-agnostic recordings, masking and the test cache. See [the main `alumnium` package page](https://www.npmjs.com/package/alumnium) for more details.

package/package.json

@@ -1,6 +1,39 @@
 {
   "name": "@alumnium/test",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.24.1",
+  "description": "Alumnium test record and replay: agent-agnostic recordings, masking and the test cache",
+  "license": "MIT",
+  "author": "Alex Rodionov <p0deje@gmail.com>",
+  "contributors": [
+    "Alex Rodionov <p0deje@gmail.com>",
+    "Sasha Koss <koss@nocorp.me>",
+    "Tatiana Shepeleva <tati.shep@gmail.com>"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/alumnium-hq/alumnium.git"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE.md"
+  ],
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./src/index.d.ts",
+        "default": "./src/index.js"
+      },
+      "require": {
+        "types": "./src/index.d.ts",
+        "default": "./src/index.cjs"
+      }
+    }
+  },
+  "dependencies": {
+    "@alumnium/core": "0.24.0",
+    "@js-fns/xxhash": "^0.1.0",
+    "zod": "^4.4.1"
+  }
+}

package/src/FileTestCache.d.ts

@@ -0,0 +1,26 @@
+import { Recording } from "./Recording.ts";
+import type { TestCache } from "./TestCache.ts";
+export declare namespace FileTestCache {
+  interface Props {
+    /** Directory to store recordings in, `<ALUMNIUM_CACHE_PATH>/tests` by default. */
+    dir?: string | undefined;
+  }
+}
+/**
+ * Test cache storing each recording as `<xxh64(text)>.json`, in the `tests`
+ * directory of the Alumnium cache.
+ */
+export declare class FileTestCache implements TestCache {
+  #private;
+  constructor(props?: FileTestCache.Props);
+  static defaultDir(): string;
+  get dir(): string;
+  get lookups(): TestCache.Lookups;
+  lookup(
+    text: string,
+    options?: TestCache.LookupOptions,
+  ): Promise<Recording.Type | null>;
+  update(recording: Recording.Type): Promise<void>;
+  save(): Promise<void>;
+  discard(): Promise<void>;
+}

package/src/Recorder.d.ts

@@ -0,0 +1,85 @@
+import { Masker } from "./masker/Masker.ts";
+import { Recording } from "./Recording.ts";
+export declare namespace Recorder {
+  interface Props extends Masker.Props {
+    text: string;
+    agent: string;
+    /** See `Recording.Schema` `version`, `Recording.DEFAULT_VERSION` by default. */
+    version?: number | undefined;
+    /** Name the host gave Alumnium's MCP server, `alumnium` by default. */
+    alumniumServer?: string | undefined;
+  }
+  interface ToolUse {
+    id: string;
+    /** MCP server the tool belongs to, absent for a host tool (e.g. `Bash`). */
+    server?: string | undefined;
+    tool: string;
+    input: unknown;
+  }
+  interface ToolResult {
+    id: string;
+    content: Recording.Content;
+    isError?: boolean | undefined;
+  }
+  interface Narration {
+    narration: Recording.NarrationStep["narration"];
+    text: string;
+  }
+}
+/**
+ * Agent-agnostic sink turning the host agent's tool calls and narration into a
+ * replayable recording. The host owns the agent loop and reports what happens
+ * in it. The recorder masks the values a replay has to refresh or must never
+ * see in clear (see `Masker`) and pairs each call with its result.
+ */
+export declare class Recorder {
+  #private;
+  constructor(props: Recorder.Props);
+  /**
+   * Records a tool call, held until its result arrives.
+   *
+   * Masked here, on the call, and not once the step is pushed in `onToolResult`.
+   * A tool call's output is only registered once its result arrives,
+   * so a call can neither mask its own input, nor - when the
+   * agent runs tools in parallel - be masked with a value that did not exist
+   * yet when it was issued.
+   *
+   * A tool replay cannot execute is masked too. Its input is
+   * never re-run, so nothing gets substituted back into it, but masking still
+   * keeps the value a recording happened to produce - often a credential - out
+   * of the stored recording.
+   *
+   * @param use - Tool call as the host agent issued it.
+   */
+  onToolUse(use: Recorder.ToolUse): void;
+  /**
+   * Records the result of a tool call, pushing the call as a step.
+   *
+   * Steps land in the order results arrive. The values a call produced are
+   * registered under its id, which later masks refer to.
+   *
+   * @param toolResult - Result of a call reported with `onToolUse`.
+   */
+  onToolResult(toolResult: Recorder.ToolResult): void;
+  /**
+   * Records what the agent said, so that a replay of this recording reads the
+   * way the recording did. Nothing is executed from it.
+   *
+   * Pushed as it arrives, while a tool call is pushed once its result
+   * does. Narration therefore lands ahead of the call it introduced, which is
+   * the order the agent produced them in.
+   *
+   * @param narration - Prose as the agent wrote it.
+   */
+  onNarration(narration: Recorder.Narration): void;
+  /**
+   * Finishes the recording with the run's verdict. Calls still waiting for a
+   * result are dropped, since replay has nothing to compare them against.
+   *
+   * @param verdict - Verdict the host agent reached.
+   * @returns Finished recording.
+   * @throws {RecordingLeakError} when a host variable or a secret survived
+   *   masking, so that the recording is never stored with it.
+   */
+  finish(verdict: Recording.Verdict): Recording.Type;
+}

package/src/Recording.d.ts

@@ -0,0 +1,288 @@
+import z from "zod";
+export declare namespace Recording {
+  type Id = z.infer<typeof Recording.Id>;
+  type Input = z.infer<typeof Recording.Input>;
+  type Content = z.infer<typeof Recording.Content>;
+  type ToolCall = z.infer<typeof Recording.ToolCall>;
+  type ToolResult = z.infer<typeof Recording.ToolResult>;
+  type ToolStep = z.infer<typeof Recording.ToolStep>;
+  type NarrationStep = z.infer<typeof Recording.NarrationStep>;
+  type Step = z.infer<typeof Recording.Step>;
+  type Verdict = z.infer<typeof Recording.Verdict>;
+  type Type = z.infer<typeof Recording.Schema>;
+}
+/**
+ * A replayable recording of a test run: every tool call the host agent made,
+ * paired with its result, and the agent's narration in between.
+ *
+ * Agent-neutral on purpose. The host (e.g. a test runner or the CLI)
+ * owns the agent loop and translates its own messages into the calls and
+ * results recorded here, so nothing in a recording is specific to the agent
+ * that made it but its name.
+ */
+export declare abstract class Recording {
+  static Id: z.core.$ZodBranded<z.ZodString, "Recording.Id", "out">;
+  static Input: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+  /**
+   * MCP content blocks a tool returned, or a bare string for a host tool that
+   * returns plain output. Only text blocks are ever read, every other block is
+   * kept as it was.
+   */
+  static Content: z.ZodUnion<
+    readonly [
+      z.ZodString,
+      z.ZodArray<
+        z.ZodObject<
+          {
+            type: z.ZodString;
+          },
+          z.core.$loose
+        >
+      >,
+    ]
+  >;
+  static ToolCall: z.ZodObject<
+    {
+      id: z.ZodString;
+      server: z.ZodOptional<z.ZodString>;
+      tool: z.ZodString;
+      input: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    },
+    z.core.$strip
+  >;
+  static ToolResult: z.ZodObject<
+    {
+      content: z.ZodUnion<
+        readonly [
+          z.ZodString,
+          z.ZodArray<
+            z.ZodObject<
+              {
+                type: z.ZodString;
+              },
+              z.core.$loose
+            >
+          >,
+        ]
+      >;
+      isError: z.ZodBoolean;
+    },
+    z.core.$strip
+  >;
+  /**
+   * A tool call and its result, whether it went to Alumnium's own MCP server,
+   * another MCP server or a host tool - `call.server` tells them apart. Replay
+   * re-issues every one, since a tool call can produce values later calls
+   * consume.
+   */
+  static ToolStep: z.ZodObject<
+    {
+      kind: z.ZodLiteral<"tool-call">;
+      call: z.ZodObject<
+        {
+          id: z.ZodString;
+          server: z.ZodOptional<z.ZodString>;
+          tool: z.ZodString;
+          input: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        },
+        z.core.$strip
+      >;
+      result: z.ZodObject<
+        {
+          content: z.ZodUnion<
+            readonly [
+              z.ZodString,
+              z.ZodArray<
+                z.ZodObject<
+                  {
+                    type: z.ZodString;
+                  },
+                  z.core.$loose
+                >
+              >,
+            ]
+          >;
+          isError: z.ZodBoolean;
+        },
+        z.core.$strip
+      >;
+    },
+    z.core.$strip
+  >;
+  /**
+   * Something the agent said rather than did: a piece of its thinking, or a
+   * message meant for the person running the test. Recorded so that a replay
+   * reads like the recording it came from, and never executed.
+   */
+  static NarrationStep: z.ZodObject<
+    {
+      kind: z.ZodLiteral<"narration">;
+      narration: z.ZodEnum<{
+        assistant: "assistant";
+        thinking: "thinking";
+      }>;
+      text: z.ZodString;
+    },
+    z.core.$strip
+  >;
+  static Step: z.ZodUnion<
+    readonly [
+      z.ZodObject<
+        {
+          kind: z.ZodLiteral<"tool-call">;
+          call: z.ZodObject<
+            {
+              id: z.ZodString;
+              server: z.ZodOptional<z.ZodString>;
+              tool: z.ZodString;
+              input: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            },
+            z.core.$strip
+          >;
+          result: z.ZodObject<
+            {
+              content: z.ZodUnion<
+                readonly [
+                  z.ZodString,
+                  z.ZodArray<
+                    z.ZodObject<
+                      {
+                        type: z.ZodString;
+                      },
+                      z.core.$loose
+                    >
+                  >,
+                ]
+              >;
+              isError: z.ZodBoolean;
+            },
+            z.core.$strip
+          >;
+        },
+        z.core.$strip
+      >,
+      z.ZodObject<
+        {
+          kind: z.ZodLiteral<"narration">;
+          narration: z.ZodEnum<{
+            assistant: "assistant";
+            thinking: "thinking";
+          }>;
+          text: z.ZodString;
+        },
+        z.core.$strip
+      >,
+    ]
+  >;
+  static Verdict: z.ZodObject<
+    {
+      status: z.ZodEnum<{
+        failure: "failure";
+        success: "success";
+      }>;
+      details: z.ZodString;
+    },
+    z.core.$strip
+  >;
+  static DEFAULT_VERSION: number;
+  static Schema: z.ZodObject<
+    {
+      id: z.core.$ZodBranded<z.ZodString, "Recording.Id", "out">;
+      version: z.ZodNumber;
+      text: z.ZodString;
+      agent: z.ZodString;
+      alumnium: z.ZodObject<
+        {
+          version: z.ZodString;
+        },
+        z.core.$strip
+      >;
+      hostVars: z.ZodArray<z.ZodString>;
+      steps: z.ZodArray<
+        z.ZodUnion<
+          readonly [
+            z.ZodObject<
+              {
+                kind: z.ZodLiteral<"tool-call">;
+                call: z.ZodObject<
+                  {
+                    id: z.ZodString;
+                    server: z.ZodOptional<z.ZodString>;
+                    tool: z.ZodString;
+                    input: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+                  },
+                  z.core.$strip
+                >;
+                result: z.ZodObject<
+                  {
+                    content: z.ZodUnion<
+                      readonly [
+                        z.ZodString,
+                        z.ZodArray<
+                          z.ZodObject<
+                            {
+                              type: z.ZodString;
+                            },
+                            z.core.$loose
+                          >
+                        >,
+                      ]
+                    >;
+                    isError: z.ZodBoolean;
+                  },
+                  z.core.$strip
+                >;
+              },
+              z.core.$strip
+            >,
+            z.ZodObject<
+              {
+                kind: z.ZodLiteral<"narration">;
+                narration: z.ZodEnum<{
+                  assistant: "assistant";
+                  thinking: "thinking";
+                }>;
+                text: z.ZodString;
+              },
+              z.core.$strip
+            >,
+          ]
+        >
+      >;
+      verdict: z.ZodObject<
+        {
+          status: z.ZodEnum<{
+            failure: "failure";
+            success: "success";
+          }>;
+          details: z.ZodString;
+        },
+        z.core.$strip
+      >;
+    },
+    z.core.$strip
+  >;
+  /**
+   * Converts test text to a recording ID using a hash function.
+   *
+   * @param text - Test text to convert to ID.
+   * @returns Recording ID generated from the trimmed text.
+   */
+  static textToId(text: string): Recording.Id;
+  /**
+   * Key a tool call is configured by, e.g. in `maskResponsePaths`:
+   * `<server>/<tool>` for an MCP tool, the bare tool name for a host tool.
+   *
+   * @param call - Tool call to get the key of.
+   * @returns Tool key.
+   */
+  static toolKey(call: Pick<Recording.ToolCall, "server" | "tool">): string;
+  /**
+   * Number of steps a replay executes, which is everything the agent did as
+   * opposed to said.
+   *
+   * @param recording - Recording to count the steps of.
+   * @returns Number of executable steps.
+   */
+  static executableStepsCount(recording: Recording.Type): number;
+}

package/src/RecordingLeakError.d.ts

@@ -0,0 +1,4 @@
+export declare class RecordingLeakError extends Error {
+  readonly masks: string[];
+  constructor(masks: string[]);
+}

package/src/TestCache.d.ts

@@ -0,0 +1,38 @@
+import type { Recording } from "./Recording.ts";
+export declare namespace TestCache {
+  interface LookupOptions {
+    /**
+     * Recording version to accept, `Recording.DEFAULT_VERSION` by default. A
+     * recording of another version is a miss, see `Recording.Schema` `version`.
+     */
+    version?: number | undefined;
+  }
+  interface Lookups {
+    hits: number;
+    misses: number;
+  }
+}
+export interface TestCache {
+  readonly lookups: TestCache.Lookups;
+  /**
+   * Looks up the recording of a test, staged or saved.
+   *
+   * @param text - Test text.
+   * @param options - Lookup options.
+   * @returns Recording if found in the accepted version, `null` otherwise.
+   */
+  lookup(
+    text: string,
+    options?: TestCache.LookupOptions,
+  ): Promise<Recording.Type | null>;
+  /**
+   * Stages a recording, replacing any other one of the same test.
+   *
+   * @param recording - Recording to stage.
+   */
+  update(recording: Recording.Type): Promise<void>;
+  /** Persists the staged recordings. */
+  save(): Promise<void>;
+  /** Drops the staged recordings. */
+  discard(): Promise<void>;
+}