@langchain/sandbox-standard-tests 0.0.3

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,246 @@
+# @langchain/sandbox-standard-tests
+
+Shared integration test suites for [deepagents](https://github.com/langchain-ai/deepagentsjs) sandbox providers. Run a single function call and get comprehensive coverage of the `SandboxBackendProtocol` — lifecycle management, command execution, file I/O, search, and more.
+
+The package is **framework-agnostic** — it works with any test runner that provides `describe`, `it`, `expect`, `beforeAll`, and `afterAll`. A first-class Vitest sub-export is included for convenience.
+
+## Installation
+
+```bash
+npm install @langchain/sandbox-standard-tests
+```
+
+## Quick start
+
+### With Vitest (recommended)
+
+Import from `@langchain/sandbox-standard-tests/vitest` and the Vitest primitives are injected automatically:
+
+```ts
+import { sandboxStandardTests } from "@langchain/sandbox-standard-tests/vitest";
+import { MySandbox } from "./sandbox.js";
+
+sandboxStandardTests({
+  name: "MySandbox",
+  skip: !process.env.MY_SANDBOX_TOKEN,
+  timeout: 120_000,
+  createSandbox: (opts) => MySandbox.create({ ...opts }),
+  closeSandbox: (sb) => sb.close(),
+  resolvePath: (name) => `/tmp/${name}`,
+});
+```
+
+### With any test runner
+
+Import from the root entry point and pass your runner's primitives via the `runner` config property:
+
+```ts
+import { sandboxStandardTests } from "@langchain/sandbox-standard-tests";
+import { describe, it, expect, beforeAll, afterAll } from "bun:test";
+import { MySandbox } from "./sandbox.js";
+
+sandboxStandardTests({
+  name: "MySandbox",
+  runner: { describe, it, expect, beforeAll, afterAll },
+  createSandbox: (opts) => MySandbox.create({ ...opts }),
+  closeSandbox: (sb) => sb.close(),
+  resolvePath: (name) => `/tmp/${name}`,
+});
+```
+
+Run with your test runner of choice:
+
+```bash
+npx vitest run sandbox.int.test.ts
+```
+
+That single `sandboxStandardTests()` call registers **11 describe blocks** covering every method on the sandbox protocol.
+
+## Configuration
+
+`sandboxStandardTests` accepts a `StandardTestsConfig<T>` object:
+
+| Option                       | Type                               | Required | Description                                                                                                                                                |
+| ---------------------------- | ---------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `name`                       | `string`                           | yes      | Display name shown in the test runner (e.g. `"ModalSandbox"`).                                                                                             |
+| `runner`                     | `TestRunner`                       | yes\*    | Test-runner primitives (`describe`, `it`, `expect`, `beforeAll`, `afterAll`). \*Optional when importing from `/vitest`.                                    |
+| `createSandbox`              | `(opts?) => Promise<T>`            | yes      | Factory that creates and returns a running sandbox. Receives an optional `{ initialFiles }` map.                                                           |
+| `resolvePath`                | `(relativePath: string) => string` | yes      | Converts a relative filename (e.g. `"test-file.txt"`) to the provider-specific absolute path (e.g. `"/tmp/test-file.txt"` or `"/home/app/test-file.txt"`). |
+| `closeSandbox`               | `(sandbox: T) => Promise<void>`    | no       | Teardown function. If omitted the "close" lifecycle test is skipped.                                                                                       |
+| `createUninitializedSandbox` | `() => T`                          | no       | Factory for a sandbox that has **not** been started yet. Enables the two-step initialization test.                                                         |
+| `skip`                       | `boolean`                          | no       | Skip the entire suite (useful when credentials are missing).                                                                                               |
+| `sequential`                 | `boolean`                          | no       | Run tests sequentially instead of in parallel (useful to avoid provider concurrency limits).                                                               |
+| `timeout`                    | `number`                           | no       | Per-test timeout in ms. Defaults to `120_000` (2 min).                                                                                                     |
+
+### `TestRunner`
+
+The `runner` object must provide these five primitives from your test framework:
+
+```ts
+interface TestRunner {
+  describe: SuiteFn;
+  it: TestFn;
+  expect: ExpectFn;
+  beforeAll: HookFn;
+  afterAll: HookFn;
+}
+```
+
+The `describe` and `it` functions may optionally expose `.skip`, `.skipIf(condition)`, and `.sequential` modifiers. When a modifier is not available the suite gracefully degrades (e.g. `describe.sequential` falls back to `describe`).
+
+### `SandboxInstance`
+
+Your sandbox class must implement the `SandboxInstance` interface, which extends `SandboxBackendProtocol` from `deepagents`:
+
+```ts
+interface SandboxInstance extends SandboxBackendProtocol {
+  readonly isRunning: boolean;
+  uploadFiles(
+    files: Array<[string, Uint8Array]>,
+  ): MaybePromise<FileUploadResponse[]>;
+  downloadFiles(paths: string[]): MaybePromise<FileDownloadResponse[]>;
+  initialize?(): Promise<void>;
+}
+```
+
+The key difference from the base protocol is that `uploadFiles` and `downloadFiles` are **required** (they are optional in `SandboxBackendProtocol`).
+
+## What gets tested
+
+| Suite                 | What it covers                                                                               |
+| --------------------- | -------------------------------------------------------------------------------------------- |
+| **Lifecycle**         | `create`, `isRunning`, `close`, two-step `initialize`                                        |
+| **Command execution** | `echo`, exit codes, multiline output, stderr, env vars, non-existent commands                |
+| **File operations**   | `uploadFiles`, `downloadFiles`, round-trip integrity                                         |
+| **write()**           | New files, parent directory creation, overwrite, special characters, unicode, long content   |
+| **read()**            | Basic read, non-existent path, `offset`, `limit`, `offset + limit`, unicode, chunked reads   |
+| **edit()**            | Single/multi occurrence, `replaceAll`, not-found handling, special chars, multiline, unicode |
+| **lsInfo()**          | Directory listing, empty dirs, hidden files, large directories, absolute paths               |
+| **grepRaw()**         | Pattern search, glob filters, case sensitivity, nested directories, unicode                  |
+| **globInfo()**        | Wildcards, recursive patterns, extension filters, character classes, deeply nested           |
+| **Initial files**     | Basic seeding, nested paths, empty files                                                     |
+| **Integration**       | End-to-end write → read → edit workflows, complex directory operations, error handling       |
+
+## Sandbox reuse strategy
+
+To avoid spinning up too many sandbox instances (which can hit provider concurrency limits), the test suite uses a **single shared sandbox** for the majority of tests. Only two kinds of tests create temporary instances:
+
+- **Lifecycle** tests that verify `close` and two-step initialization
+- **Initial files** tests that require a fresh sandbox with pre-seeded content
+
+These temporary sandboxes are torn down immediately, so the concurrent sandbox count never exceeds **2**.
+
+## Retry helper
+
+The package exports a `withRetry` utility for working around transient sandbox creation failures (e.g. provider concurrency limits):
+
+```ts
+import { withRetry } from "@langchain/sandbox-standard-tests/vitest";
+
+const sandbox = await withRetry(
+  () => MySandbox.create({ memoryMb: 512 }),
+  5, // max attempts (default: 5)
+  15_000, // delay between attempts in ms (default: 15 000)
+);
+```
+
+## Real-world examples
+
+### Remote provider (Modal)
+
+```ts
+import {
+  sandboxStandardTests,
+  withRetry,
+} from "@langchain/sandbox-standard-tests/vitest";
+import { ModalSandbox } from "./sandbox.js";
+
+const hasCredentials = !!(
+  process.env.MODAL_TOKEN_ID && process.env.MODAL_TOKEN_SECRET
+);
+
+sandboxStandardTests({
+  name: "ModalSandbox",
+  skip: !hasCredentials,
+  timeout: 180_000,
+  createSandbox: (opts) =>
+    ModalSandbox.create({ imageName: "alpine:3.21", ...opts }),
+  createUninitializedSandbox: () =>
+    new ModalSandbox({ imageName: "alpine:3.21" }),
+  closeSandbox: (sb) => sb.close(),
+  resolvePath: (name) => `/tmp/${name}`,
+});
+```
+
+### Sequential execution (Deno Deploy)
+
+```ts
+import { sandboxStandardTests } from "@langchain/sandbox-standard-tests/vitest";
+import { DenoSandbox } from "./sandbox.js";
+
+sandboxStandardTests({
+  name: "DenoSandbox",
+  skip: !process.env.DENO_DEPLOY_TOKEN,
+  sequential: true,
+  timeout: 120_000,
+  createSandbox: (opts) => DenoSandbox.create({ memoryMb: 768, ...opts }),
+  createUninitializedSandbox: () => new DenoSandbox({ memoryMb: 768 }),
+  closeSandbox: (sb) => sb.close(),
+  resolvePath: (name) => `/home/app/${name}`,
+});
+```
+
+### Local provider (Node VFS)
+
+```ts
+import { sandboxStandardTests } from "@langchain/sandbox-standard-tests/vitest";
+import { VfsSandbox } from "./sandbox.js";
+
+sandboxStandardTests({
+  name: "VfsSandbox",
+  skip: process.platform === "win32",
+  timeout: 30_000,
+  createSandbox: (opts) => VfsSandbox.create(opts),
+  closeSandbox: (sb) => sb.stop(),
+  resolvePath: (name) => name,
+});
+```
+
+### Custom runner (Bun)
+
+```ts
+import { sandboxStandardTests } from "@langchain/sandbox-standard-tests";
+import { describe, it, expect, beforeAll, afterAll } from "bun:test";
+import { MySandbox } from "./sandbox.js";
+
+sandboxStandardTests({
+  name: "MySandbox",
+  runner: { describe, it, expect, beforeAll, afterAll },
+  createSandbox: (opts) => MySandbox.create(opts),
+  closeSandbox: (sb) => sb.close(),
+  resolvePath: (name) => `/tmp/${name}`,
+});
+```
+
+## Adding provider-specific tests
+
+After calling `sandboxStandardTests`, you can add provider-specific tests in the same file using standard Vitest `describe` / `it` blocks:
+
+```ts
+sandboxStandardTests({
+  /* ... */
+});
+
+describe("MySandbox Provider-Specific Tests", () => {
+  it("should support custom image types", async () => {
+    const sb = await MySandbox.create({ image: "python:3.12" });
+    const result = await sb.execute("python --version");
+    expect(result.exitCode).toBe(0);
+    await sb.close();
+  });
+});
+```
+
+## License
+
+MIT

package/dist/index.cjs

@@ -0,0 +1,5 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
+const require_sandbox = require('./sandbox-C22TOwhI.cjs');
+
+exports.sandboxStandardTests = require_sandbox.sandboxStandardTests;
+exports.withRetry = require_sandbox.withRetry;

package/dist/index.d.cts

@@ -0,0 +1,2 @@
+import { a as SandboxInstance, c as TestFn, i as HookFn, l as TestRunner, n as withRetry, o as StandardTestsConfig, r as ExpectFn, s as SuiteFn, t as sandboxStandardTests } from "./sandbox-BX7bEJLz.cjs";
+export { type ExpectFn, type HookFn, type SandboxInstance, type StandardTestsConfig, type SuiteFn, type TestFn, type TestRunner, sandboxStandardTests, withRetry };

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+import { a as SandboxInstance, c as TestFn, i as HookFn, l as TestRunner, n as withRetry, o as StandardTestsConfig, r as ExpectFn, s as SuiteFn, t as sandboxStandardTests } from "./sandbox-CWDp3zl_.js";
+export { type ExpectFn, type HookFn, type SandboxInstance, type StandardTestsConfig, type SuiteFn, type TestFn, type TestRunner, sandboxStandardTests, withRetry };

package/dist/index.js

@@ -0,0 +1,3 @@
+import { n as withRetry, t as sandboxStandardTests } from "./sandbox-C1ApHibz.js";
+
+export { sandboxStandardTests, withRetry };

package/dist/sandbox-BX7bEJLz.d.cts

@@ -0,0 +1,168 @@
+import { FileDownloadResponse, FileUploadResponse, MaybePromise, SandboxBackendProtocol } from "deepagents";
+
+//#region src/types.d.ts
+/** A `describe` / suite function accepted by the standard tests. */
+interface SuiteFn {
+  (name: string, fn: () => void): void;
+  skip?: SuiteFn;
+  sequential?: SuiteFn;
+  skipIf?: (condition: boolean) => SuiteFn;
+}
+/** An `it` / test function accepted by the standard tests. */
+interface TestFn {
+  (name: string, fn: () => void | Promise<void>, timeout?: number): void;
+  skipIf?: (condition: boolean) => TestFn;
+}
+/** A `beforeAll` / `afterAll` hook function. */
+type HookFn = (fn: () => void | Promise<void>, timeout?: number) => void;
+/**
+ * An `expect` function.
+ *
+ * The return type is intentionally `any` — every test framework exposes
+ * its own matcher API and fully typing it would couple the package to a
+ * specific runner.
+ */
+type ExpectFn = (value: unknown) => any;
+/**
+ * Test-runner primitives required by the standard test suite.
+ *
+ * Pass the primitives from your test framework (Vitest, Jest, …) when
+ * importing from the root entry point.  The `@langchain/sandbox-standard-tests/vitest`
+ * sub-export fills these in automatically.
+ */
+interface TestRunner {
+  describe: SuiteFn;
+  it: TestFn;
+  expect: ExpectFn;
+  beforeAll: HookFn;
+  afterAll: HookFn;
+}
+/**
+ * Interface for sandbox instances used in standard tests.
+ *
+ * Extends the canonical `SandboxBackendProtocol` from deepagents with
+ * test-specific properties (`isRunning`, `initialize`) and makes
+ * `uploadFiles`/`downloadFiles` required (they are optional in the
+ * base protocol).
+ */
+interface SandboxInstance extends SandboxBackendProtocol {
+  /** Whether the sandbox is currently running */
+  readonly isRunning: boolean;
+  /** Upload multiple files (required for standard tests) */
+  uploadFiles(files: Array<[string, Uint8Array]>): MaybePromise<FileUploadResponse[]>;
+  /** Download multiple files (required for standard tests) */
+  downloadFiles(paths: string[]): MaybePromise<FileDownloadResponse[]>;
+  /** Optional two-step initialization */
+  initialize?(): Promise<void>;
+}
+/**
+ * Configuration for the standard sandbox test suite.
+ *
+ * @typeParam T - The concrete sandbox type (e.g., ModalSandbox, DenoSandbox)
+ */
+interface StandardTestsConfig<T extends SandboxInstance = SandboxInstance> {
+  /**
+   * Display name for the test suite (e.g., "ModalSandbox", "DenoSandbox").
+   */
+  name: string;
+  /**
+   * Test-runner primitives (`describe`, `it`, `expect`, `beforeAll`, `afterAll`).
+   *
+   * Required when importing from the root entry point.  Pre-filled when
+   * importing from `@langchain/sandbox-standard-tests/vitest`.
+   */
+  runner: TestRunner;
+  /**
+   * Skip all tests when true (e.g., when credentials are missing).
+   */
+  skip?: boolean;
+  /**
+   * Run tests sequentially to avoid concurrency limits.
+   */
+  sequential?: boolean;
+  /**
+   * Timeout for each test in milliseconds.
+   * @default 120_000
+   */
+  timeout?: number;
+  /**
+   * Factory function to create a new sandbox instance.
+   *
+   * The test suite passes `initialFiles` with paths already resolved via
+   * `resolvePath`. The implementation should pass them through to the
+   * provider's create method.
+   *
+   * `initialFiles` values are always strings (not Uint8Array) in the
+   * standard tests.
+   */
+  createSandbox: (options?: {
+    initialFiles?: Record<string, string>;
+  }) => Promise<T>;
+  /**
+   * Optional factory for creating an uninitialized sandbox for the
+   * two-step initialization test. If omitted, the test is skipped.
+   */
+  createUninitializedSandbox?: () => T;
+  /**
+   * Close / cleanup a sandbox instance.
+   */
+  closeSandbox?: (sandbox: T) => Promise<void>;
+  /**
+   * Convert a relative file path (e.g., `"test-file.txt"`) to the
+   * provider-specific absolute or working-directory path
+   * (e.g., `"/tmp/test-file.txt"` or just `"test-file.txt"`).
+   */
+  resolvePath: (relativePath: string) => string;
+}
+//#endregion
+//#region src/sandbox.d.ts
+/**
+ * Retry an async operation with a fixed delay between attempts.
+ *
+ * Useful for working around transient sandbox concurrency limits:
+ * when a provider rejects creation because the organisation has too
+ * many running sandboxes, waiting a short while and retrying usually
+ * succeeds once a previous sandbox finishes shutting down.
+ *
+ * @param fn - The async operation to attempt
+ * @param maxRetries - Maximum number of attempts (default: 3)
+ * @param delayMs - Milliseconds to wait between attempts (default: 10 000)
+ * @returns The result of the first successful attempt
+ *
+ * @example
+ * ```ts
+ * const sandbox = await withRetry(() => DenoSandbox.create({ memoryMb: 768 }));
+ * ```
+ */
+declare function withRetry<T>(fn: () => Promise<T>, maxRetries?: number, delayMs?: number): Promise<T>;
+/**
+ * Run the standard sandbox integration tests against a provider.
+ *
+ * A single shared sandbox is created in `beforeAll` and reused for the
+ * majority of tests (command execution, file operations). Tests that
+ * inherently need their own sandbox (lifecycle close/init, initialFiles)
+ * create a temporary one and destroy it immediately, so the concurrent
+ * sandbox count never exceeds **2** (shared + 1 temporary).
+ *
+ * @example
+ * ```ts
+ * import { sandboxStandardTests } from "@langchain/sandbox-standard-tests/vitest";
+ * import { ModalSandbox } from "./sandbox.js";
+ *
+ * sandboxStandardTests({
+ *   name: "ModalSandbox",
+ *   skip: !process.env.MODAL_TOKEN_ID,
+ *   timeout: 180_000,
+ *   createSandbox: (opts) =>
+ *     ModalSandbox.create({ imageName: "alpine:3.21", ...opts }),
+ *   createUninitializedSandbox: () =>
+ *     new ModalSandbox({ imageName: "alpine:3.21" }),
+ *   closeSandbox: (sb) => sb.close(),
+ *   resolvePath: (name) => `/tmp/${name}`,
+ * });
+ * ```
+ */
+declare function sandboxStandardTests<T extends SandboxInstance>(config: StandardTestsConfig<T>): void;
+//#endregion
+export { SandboxInstance as a, TestFn as c, HookFn as i, TestRunner as l, withRetry as n, StandardTestsConfig as o, ExpectFn as r, SuiteFn as s, sandboxStandardTests as t };
+//# sourceMappingURL=sandbox-BX7bEJLz.d.cts.map