@ontrails/testing 0.2.0

package/README.md

@@ -0,0 +1,157 @@
+# @ontrails/testing
+
+Contract-driven testing for Trails. Add examples to your trails, then `testAll(graph)` runs them as assertions, validates output schemas, checks composition graphs, and verifies structural integrity. One line of test code, full contract coverage.
+
+## Usage
+
+```typescript
+import { testAll } from '@ontrails/testing';
+import { graph } from '../app';
+
+testAll(graph);
+```
+
+That single call covers example execution, contract validation, detour checks, and topo validation. For most apps, this is all you need.
+
+If you want finer control:
+
+```typescript
+import { testExamples, testContracts, testDetours } from '@ontrails/testing';
+
+testExamples(graph);   // Run every trail's examples as tests
+testContracts(graph);  // Validate outputs against declared schemas
+testDetours(graph);    // Validate detour constructor, recover, and ordering semantics
+```
+
+## API
+
+| Export | What it does |
+| --- | --- |
+| `testAll(topo, ctx?)` | Single-line contract suite: validation + examples + contracts + detours |
+| `testExamples(topo, ctx?)` | Run trail examples as `describe`/`test` blocks |
+| `testTrail(trail, scenarios)` | Custom scenarios for edge cases, error paths, and composition chains |
+| `testContracts(topo, ctx?)` | Validate output against declared schemas |
+| `testDetours(topo)` | Validate detour constructor, recover, and shadowing semantics |
+| `createComposeContext(options?)` | Mock `ComposeFn` for testing composite trails; returns preconfigured `Result` values keyed by trail ID |
+| `createTestContext(options?)` | `TrailContext` with sensible test defaults |
+| `createTestLogger()` | Logger that captures entries in memory for assertions |
+
+See the [API Reference](../../docs/api-reference.md) for the full list.
+
+## testTrail
+
+For edge cases that do not belong in agent-facing examples:
+
+```typescript
+import { testTrail } from '@ontrails/testing';
+import { ValidationError, NotFoundError } from '@ontrails/core';
+
+testTrail(showTrail, [
+  { description: 'empty name', input: { name: '' }, expectOk: true },
+  { description: 'missing name', input: {}, expectErr: ValidationError },
+  { description: 'not found', input: { name: 'missing' }, expectErr: NotFoundError },
+]);
+```
+
+## Testing composition (trails with composes)
+
+`testTrail` works the same for trails with `composes` -- it exercises the composition graph:
+
+```typescript
+import { testTrail } from '@ontrails/testing';
+
+testTrail(onboardTrail, [
+  { description: 'happy path', input: { name: 'Delta', type: 'tool' }, expectOk: true },
+  { description: 'add fails', input: { name: 'Alpha' }, expectErr: AlreadyExistsError },
+]);
+```
+
+When you need to isolate a composite trail and stub out its dependencies, use `createComposeContext`:
+
+```typescript
+import { createComposeContext, testTrail } from '@ontrails/testing';
+import { Result } from '@ontrails/core';
+
+const compose = createComposeContext({
+  responses: {
+    'entity.add': Result.ok({ id: '1', name: 'Delta', type: 'tool' }),
+    'search': Result.ok({ results: [] }),
+  },
+});
+testTrail(onboardTrail, [
+  {
+    description: 'uses mocked composes',
+    input: { name: 'Delta', type: 'tool' },
+    expectOk: true,
+  },
+], { compose });
+```
+
+Calls to unregistered trail IDs return `Result.err` with a descriptive message, so missing stubs fail loudly.
+
+## Surface Harnesses
+
+Surface harnesses live on explicit subpaths so renders that only import contract helpers from `@ontrails/testing` do not need CLI, MCP, or HTTP peers. Install the peer package for the subpath you use:
+
+- `@ontrails/testing/cli` requires `@ontrails/cli`
+- `@ontrails/testing/mcp` requires `@ontrails/mcp`
+- `@ontrails/testing/http` requires `@ontrails/http`
+- `@ontrails/testing/established` and `@ontrails/testing/surface-parity`
+  require all three shipped surface peers
+
+```typescript
+import { createCliHarness } from '@ontrails/testing/cli';
+import { createHttpHarness } from '@ontrails/testing/http';
+import { createMcpHarness } from '@ontrails/testing/mcp';
+
+// CLI
+const cli = createCliHarness({ graph });
+const result = await cli.run('entity show --name Alpha --output json');
+expect(result.exitCode).toBe(0);
+
+// MCP
+const mcp = createMcpHarness({ graph });
+const tool = await mcp.callTool('myapp_entity_show', { name: 'Alpha' });
+expect(tool.isError).toBe(false);
+
+// HTTP
+const http = createHttpHarness({ graph });
+const response = await http.get('/entity/show', { name: 'Alpha' });
+expect(response.status).toBe(200);
+```
+
+Use `testAllEstablished()` from `@ontrails/testing/established` when an established app should run the root contract suite and validate CLI, MCP, and HTTP renderings in one call.
+
+## Surface Parity
+
+`testSurfaceParity()` is a focused gate for established apps that want to prove their examples behave the same through every shipped surface.
+
+```typescript
+import { testSurfaceParity } from '@ontrails/testing/surface-parity';
+import { graph } from '../app';
+
+const createDeterministicTestDb = () => ({});
+
+testSurfaceParity(graph, {
+  createResources: () => ({
+    'db.main': createDeterministicTestDb(),
+  }),
+  exclusions: [
+    {
+      example: 'Creates generated data',
+      reason: 'output includes generated IDs that intentionally differ per surface run',
+      trailId: 'entity.add',
+    },
+  ],
+});
+```
+
+The helper compares normalized success payloads and normalized TrailsError category/code pairs. CLI command names, MCP tool names, HTTP method/path values, transport envelopes, activation consumers, internal trails, and planned WebSocket work stay outside the equality check. Use `createResources` when examples need deterministic fixtures for each surface invocation, and use exclusions for intentional semantic differences that should not be silently skipped.
+
+## Installation
+
+These commands target stable `0.2.0`. Run them after that version is published to npm.
+
+```bash
+bun add --exact -d @ontrails/testing@0.2.0
+```

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@ontrails/testing",
+  "version": "0.2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/outfitter-dev/trails.git",
+    "directory": "packages/testing"
+  },
+  "files": [
+    "src/**/*.ts",
+    "!src/**/__tests__/**",
+    "!src/**/*.test.ts",
+    "!src/**/*.test-d.ts",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts",
+    "./cli": "./src/cli.ts",
+    "./established": "./src/all-established.ts",
+    "./http": "./src/http.ts",
+    "./mcp": "./src/mcp.ts",
+    "./surface-parity": "./src/surface-parity.ts",
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit",
+    "lint": "oxlint ./src",
+    "clean": "rm -rf dist *.tsbuildinfo"
+  },
+  "devDependencies": {
+    "@ontrails/drizzle": "^0.2.0",
+    "@ontrails/store": "^0.2.0"
+  },
+  "peerDependencies": {
+    "@ontrails/cli": "^0.2.0",
+    "@ontrails/core": "^0.2.0",
+    "@ontrails/http": "^0.2.0",
+    "@ontrails/mcp": "^0.2.0",
+    "@ontrails/observability": "^0.2.0",
+    "zod": "^4.3.5"
+  },
+  "peerDependenciesMeta": {
+    "@ontrails/cli": {
+      "optional": true
+    },
+    "@ontrails/http": {
+      "optional": true
+    },
+    "@ontrails/mcp": {
+      "optional": true
+    }
+  }
+}

package/src/all-established.ts

@@ -0,0 +1,168 @@
+/**
+ * testAllEstablished - contract suite plus shipped surface rendering checks.
+ *
+ * This helper intentionally lives behind a surface subpath so root
+ * `@ontrails/testing` imports do not pull CLI, MCP, or HTTP peers into
+ * contract-only consumers.
+ */
+
+import { describe, expect, test } from 'bun:test';
+
+import type { Topo, TrailContext } from '@ontrails/core';
+import { validateEstablishedTopo } from '@ontrails/core';
+
+import { registerContractSuite } from './all.js';
+import type { TestExecutionOptions } from './context.js';
+import { createCliHarness } from './harness-cli.js';
+import type { CliHarnessOptions } from './harness-cli.js';
+import { createHttpHarness } from './harness-http.js';
+import type { HttpHarnessOptions } from './harness-http.js';
+import { createMcpHarness } from './harness-mcp.js';
+import type { McpHarnessOptions } from './harness-mcp.js';
+
+export interface TestAllEstablishedOptions {
+  readonly cli?: Omit<CliHarnessOptions, 'graph'> | undefined;
+  readonly createPermit?:
+    | ((trail: {
+        readonly permit?:
+          | { readonly scopes: readonly string[] }
+          | 'public'
+          | undefined;
+      }) =>
+        | {
+            readonly id: string;
+            readonly scopes: readonly string[];
+          }
+        | undefined)
+    | undefined;
+  readonly ctx?: Partial<TrailContext> | undefined;
+  readonly http?: Omit<HttpHarnessOptions, 'graph'> | undefined;
+  readonly mcp?: Omit<McpHarnessOptions, 'graph'> | undefined;
+  readonly resources?: Record<string, unknown> | undefined;
+  readonly strictPermits?: boolean | undefined;
+}
+
+type EstablishedInput =
+  | Partial<TrailContext>
+  | TestAllEstablishedOptions
+  | (() => Partial<TrailContext> | TestAllEstablishedOptions);
+
+const isEstablishedOptions = (
+  input: Partial<TrailContext> | TestAllEstablishedOptions | undefined
+): input is TestAllEstablishedOptions =>
+  input !== undefined &&
+  (Object.hasOwn(input, 'cli') ||
+    Object.hasOwn(input, 'createPermit') ||
+    Object.hasOwn(input, 'ctx') ||
+    Object.hasOwn(input, 'http') ||
+    Object.hasOwn(input, 'mcp') ||
+    Object.hasOwn(input, 'resources') ||
+    Object.hasOwn(input, 'strictPermits'));
+
+const normalizeEstablishedOptions = (
+  input?: Partial<TrailContext> | TestAllEstablishedOptions
+): TestAllEstablishedOptions =>
+  isEstablishedOptions(input) ? input : { ctx: input };
+
+const toExecutionOptions = (
+  options: TestAllEstablishedOptions
+): TestExecutionOptions => ({
+  ...(options.createPermit === undefined
+    ? {}
+    : { createPermit: options.createPermit }),
+  ...(options.ctx === undefined ? {} : { ctx: options.ctx }),
+  ...(options.resources === undefined ? {} : { resources: options.resources }),
+  ...(options.strictPermits === undefined
+    ? {}
+    : { strictPermits: options.strictPermits }),
+});
+
+const toCliHarnessOptions = (
+  topo: Topo,
+  options: TestAllEstablishedOptions
+) => {
+  const cliOptions = {
+    graph: topo,
+    ...options.cli,
+  };
+
+  if (options.ctx !== undefined) {
+    cliOptions.ctx = options.ctx;
+  }
+
+  return cliOptions;
+};
+
+const toMcpHarnessOptions = (
+  topo: Topo,
+  options: TestAllEstablishedOptions
+) => ({
+  graph: topo,
+  ...options.mcp,
+});
+
+const toHttpHarnessOptions = (
+  topo: Topo,
+  options: TestAllEstablishedOptions
+) => {
+  const httpOptions = {
+    graph: topo,
+    ...options.http,
+  };
+
+  if (options.ctx !== undefined) {
+    httpOptions.ctx = options.ctx;
+  }
+
+  return httpOptions;
+};
+
+const registerEstablishedSurfaceSuite = (
+  topo: Topo,
+  resolveInput: () =>
+    | Partial<TrailContext>
+    | TestAllEstablishedOptions
+    | undefined
+): void => {
+  describe('surfaces', () => {
+    test('CLI rendering validates established topo', () => {
+      const options = normalizeEstablishedOptions(resolveInput());
+      expect(() =>
+        createCliHarness(toCliHarnessOptions(topo, options))
+      ).not.toThrow();
+    });
+
+    test('MCP rendering validates established topo', () => {
+      const options = normalizeEstablishedOptions(resolveInput());
+      expect(() =>
+        createMcpHarness(toMcpHarnessOptions(topo, options))
+      ).not.toThrow();
+    });
+
+    test('HTTP rendering validates established topo', () => {
+      const options = normalizeEstablishedOptions(resolveInput());
+      expect(() =>
+        createHttpHarness(toHttpHarnessOptions(topo, options))
+      ).not.toThrow();
+    });
+  });
+};
+
+export const testAllEstablished = (
+  topo: Topo,
+  optionsOrFactory?: EstablishedInput
+): void => {
+  const resolveInput =
+    typeof optionsOrFactory === 'function'
+      ? optionsOrFactory
+      : () => optionsOrFactory;
+
+  registerContractSuite(
+    topo,
+    () => toExecutionOptions(normalizeEstablishedOptions(resolveInput())),
+    validateEstablishedTopo
+  );
+  registerEstablishedSurfaceSuite(topo, resolveInput);
+};
+
+export type { TestAllInput } from './all.js';

package/src/all.ts

@@ -0,0 +1,94 @@
+/**
+ * testAll — single-line contract suite for any Topo.
+ *
+ * Wraps topo validation, example execution, contract checks, and detour
+ * contract validation into one describe block.
+ */
+
+import { describe, expect, test } from 'bun:test';
+
+import type { Topo, TrailContext } from '@ontrails/core';
+import { validateTopo } from '@ontrails/core';
+
+import { testContracts } from './contracts.js';
+import type { TestExecutionOptions } from './context.js';
+import { testDetours } from './detours.js';
+import { testExamples } from './examples.js';
+
+/**
+ * Run the full contract test suite for a Topo.
+ *
+ * Generates a `contract` describe block containing:
+ * - Structural validation via `validateTopo`
+ * - Example execution via `testExamples`
+ * - Output contract checks via `testContracts`
+ * - Detour contract validation via `testDetours`
+ *
+ * Accepts either a static context or a factory function that produces a
+ * fresh context per test (useful when the context contains mutable state
+ * like an in-memory store).
+ *
+ * @example
+ * ```ts
+ * import { testAll } from '@ontrails/testing';
+ * import { graph } from '../src/app.js';
+ *
+ * testAll(graph);
+ * ```
+ */
+export type TestAllInput =
+  | Partial<TrailContext>
+  | TestExecutionOptions
+  | (() => Partial<TrailContext> | TestExecutionOptions);
+
+const formatValidationFailure = (error: Error): string => {
+  const issues = (
+    error as {
+      context?: { issues?: readonly Record<string, unknown>[] };
+    }
+  ).context?.issues;
+
+  if (issues === undefined || issues.length === 0) {
+    return error.message;
+  }
+
+  const details = issues.map((issue) => {
+    const id = typeof issue['id'] === 'string' ? issue['id'] : undefined;
+    const message =
+      typeof issue['message'] === 'string' ? issue['message'] : undefined;
+    const rule = typeof issue['rule'] === 'string' ? issue['rule'] : undefined;
+
+    return [rule, id, message].filter(Boolean).join(': ');
+  });
+
+  return [error.message, ...details].join('\n');
+};
+
+const assertValidTopo = (result: ReturnType<typeof validateTopo>): void => {
+  if (result.isErr()) {
+    throw new Error(formatValidationFailure(result.error));
+  }
+};
+
+export const registerContractSuite = (
+  topo: Topo,
+  ctxOrFactory: TestAllInput | undefined,
+  validate: (topo: Topo) => ReturnType<typeof validateTopo>
+): void => {
+  describe('contract', () => {
+    test('topo validates', () => {
+      expect(() => assertValidTopo(validate(topo))).not.toThrow();
+    });
+
+    // oxlint-disable-next-line jest/require-hook -- these generate describe/test blocks, not setup code
+    testExamples(topo, ctxOrFactory);
+    // oxlint-disable-next-line jest/require-hook -- these generate describe/test blocks, not setup code
+    testContracts(topo, ctxOrFactory);
+    // oxlint-disable-next-line jest/require-hook -- these generate describe/test blocks, not setup code
+    testDetours(topo);
+  });
+};
+
+export const testAll = (topo: Topo, ctxOrFactory?: TestAllInput): void => {
+  registerContractSuite(topo, ctxOrFactory, validateTopo);
+};