@ontrails/testing 1.0.0-beta.0

package/src/harness-cli.ts

@@ -0,0 +1,310 @@
+/**
+ * CLI integration test harness.
+ *
+ * Builds CLI commands from an App, executes them in-process,
+ * and captures stdout/stderr.
+ */
+
+import { buildCliCommands } from '@ontrails/cli';
+import type { CliCommand } from '@ontrails/cli';
+
+import { createTestContext } from './context.js';
+import type {
+  CliHarness,
+  CliHarnessOptions,
+  CliHarnessResult,
+} from './types.js';
+
+// ---------------------------------------------------------------------------
+// Tokenizer
+// ---------------------------------------------------------------------------
+
+/** Parse a command string into tokens (simple split, no quoting support). */
+const parseCommandString = (input: string): string[] =>
+  input
+    .trim()
+    .split(/\s+/)
+    .filter((s) => s.length > 0);
+
+// ---------------------------------------------------------------------------
+// Command resolution
+// ---------------------------------------------------------------------------
+
+/** Try to match group + name from tokens. */
+const tryGroupMatch = (
+  commands: CliCommand[],
+  firstToken: string,
+  secondToken: string | undefined,
+  tokens: string[]
+): { command: CliCommand; flagTokens: string[] } | undefined => {
+  if (secondToken === undefined || secondToken.startsWith('-')) {
+    return undefined;
+  }
+  const match = commands.find(
+    (c) => c.group === firstToken && c.name === secondToken
+  );
+  if (match === undefined) {
+    return undefined;
+  }
+  return { command: match, flagTokens: tokens.slice(2) };
+};
+
+/** Try to match a direct name from the first token. */
+const tryDirectMatch = (
+  commands: CliCommand[],
+  firstToken: string,
+  tokens: string[]
+): { command: CliCommand; flagTokens: string[] } | undefined => {
+  const match = commands.find(
+    (c) => c.name === firstToken && c.group === undefined
+  );
+  if (match === undefined) {
+    return undefined;
+  }
+  return { command: match, flagTokens: tokens.slice(1) };
+};
+
+/** Resolve a command from tokens, handling group.name patterns. */
+const resolveCommand = (
+  commands: CliCommand[],
+  tokens: string[]
+): { command: CliCommand; flagTokens: string[] } | undefined => {
+  const [firstToken, secondToken] = tokens;
+
+  if (firstToken === undefined) {
+    return undefined;
+  }
+
+  return (
+    tryGroupMatch(commands, firstToken, secondToken, tokens) ??
+    tryDirectMatch(commands, firstToken, tokens)
+  );
+};
+
+// ---------------------------------------------------------------------------
+// Flag parsing
+// ---------------------------------------------------------------------------
+
+/** Parse a value flag (--key value) and return new index. */
+const parseValueFlag = (
+  key: string,
+  next: string,
+  flags: Record<string, unknown>
+): void => {
+  const num = Number(next);
+  flags[key] = Number.isNaN(num) ? next : num;
+};
+
+/** Parse a single flag token and advance the index. */
+const parseSingleFlag = (
+  tokens: string[],
+  i: number,
+  flags: Record<string, unknown>
+): number => {
+  const token = tokens[i];
+  if (token === undefined || !token.startsWith('--')) {
+    return i + 1;
+  }
+
+  const key = token.slice(2);
+  const next = tokens[i + 1];
+
+  if (next !== undefined && !next.startsWith('-')) {
+    parseValueFlag(key, next, flags);
+    return i + 2;
+  }
+
+  flags[key] = true;
+  return i + 1;
+};
+
+/** Parse flag tokens into a record. */
+const parseFlagTokens = (tokens: string[]): Record<string, unknown> => {
+  const flags: Record<string, unknown> = {};
+  let i = 0;
+
+  while (i < tokens.length) {
+    i = parseSingleFlag(tokens, i, flags);
+  }
+
+  return flags;
+};
+
+// ---------------------------------------------------------------------------
+// Stream capture
+// ---------------------------------------------------------------------------
+
+interface CapturedStreams {
+  readonly getStderr: () => string;
+  readonly getStdout: () => string;
+  readonly restore: () => void;
+  readonly writeStdout: (text: string) => void;
+}
+
+/** Create interceptors for stdout/stderr capture. */
+const captureStreams = (): CapturedStreams => {
+  let stdout = '';
+  let stderr = '';
+  const origStdoutWrite = process.stdout.write;
+  const origStderrWrite = process.stderr.write;
+
+  process.stdout.write = ((chunk: string | Uint8Array): boolean => {
+    stdout +=
+      typeof chunk === 'string' ? chunk : new TextDecoder().decode(chunk);
+    return true;
+  }) as typeof process.stdout.write;
+
+  process.stderr.write = ((chunk: string | Uint8Array): boolean => {
+    stderr +=
+      typeof chunk === 'string' ? chunk : new TextDecoder().decode(chunk);
+    return true;
+  }) as typeof process.stderr.write;
+
+  return {
+    getStderr: () => stderr,
+    getStdout: () => stdout,
+    restore: () => {
+      process.stdout.write = origStdoutWrite;
+      process.stderr.write = origStderrWrite;
+    },
+    writeStdout: (text: string) => {
+      stdout += text;
+    },
+  };
+};
+
+// ---------------------------------------------------------------------------
+// Output formatting
+// ---------------------------------------------------------------------------
+
+/** Try to parse a string as JSON, returning undefined on failure. */
+const tryParseJson = (text: string): unknown => {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+};
+
+/** Format result value into stdout and build the CLI result. */
+const formatSuccessResult = (
+  value: unknown,
+  flags: Record<string, unknown>,
+  streams: CapturedStreams
+): CliHarnessResult => {
+  const outputMode =
+    flags['output'] ?? (flags['json'] === true ? 'json' : 'text');
+
+  if (outputMode === 'json') {
+    const jsonStr = `${JSON.stringify(value, null, 2)}\n`;
+    streams.writeStdout(jsonStr);
+    return {
+      exitCode: 0,
+      json: tryParseJson(jsonStr),
+      stderr: streams.getStderr(),
+      stdout: streams.getStdout(),
+    };
+  }
+
+  const formatted =
+    typeof value === 'string'
+      ? `${value}\n`
+      : `${JSON.stringify(value, null, 2)}\n`;
+  streams.writeStdout(formatted);
+
+  return {
+    exitCode: 0,
+    json: tryParseJson(streams.getStdout().trim()),
+    stderr: streams.getStderr(),
+    stdout: streams.getStdout(),
+  };
+};
+
+// ---------------------------------------------------------------------------
+// Execute command
+// ---------------------------------------------------------------------------
+
+/** Build an error result from a caught exception. */
+const buildErrorResult = (
+  error: unknown,
+  streams: CapturedStreams
+): CliHarnessResult => {
+  streams.restore();
+  const message = error instanceof Error ? error.message : String(error);
+  return {
+    exitCode: 1,
+    stderr: streams.getStderr() || message,
+    stdout: streams.getStdout(),
+  };
+};
+
+/** Execute a resolved command and return the result. */
+const executeCommand = async (
+  command: CliCommand,
+  flags: Record<string, unknown>,
+  streams: CapturedStreams
+): Promise<CliHarnessResult> => {
+  const ctx = createTestContext();
+  const result = await command.execute({}, flags, ctx);
+  streams.restore();
+
+  if (result.isErr()) {
+    return {
+      exitCode: 1,
+      stderr: streams.getStderr() || result.error.message,
+      stdout: streams.getStdout(),
+    };
+  }
+
+  return formatSuccessResult(result.value, flags, streams);
+};
+
+/** Run the full command pipeline: resolve, parse, execute. */
+const runCommand = async (
+  commands: CliCommand[],
+  commandString: string
+): Promise<CliHarnessResult> => {
+  const parts = parseCommandString(commandString);
+  const resolved = resolveCommand(commands, parts);
+  if (resolved === undefined) {
+    return {
+      exitCode: 1,
+      stderr: `Unknown command: ${commandString}`,
+      stdout: '',
+    };
+  }
+
+  const { command, flagTokens } = resolved;
+  const flags = parseFlagTokens(flagTokens);
+  const streams = captureStreams();
+
+  try {
+    return await executeCommand(command, flags, streams);
+  } catch (error: unknown) {
+    return buildErrorResult(error, streams);
+  }
+};
+
+// ---------------------------------------------------------------------------
+// createCliHarness
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a CLI harness for integration testing.
+ *
+ * Builds commands from the app's topo and provides a `run()` method
+ * that parses command strings and executes them in-process.
+ *
+ * ```ts
+ * const harness = createCliHarness({ app });
+ * const result = await harness.run("entity show --name Alpha --output json");
+ * expect(result.exitCode).toBe(0);
+ * ```
+ */
+export const createCliHarness = (options: CliHarnessOptions): CliHarness => {
+  const commands = buildCliCommands(options.app);
+
+  return {
+    run: (commandString: string) => runCommand(commands, commandString),
+  };
+};

package/src/harness-mcp.ts

@@ -0,0 +1,65 @@
+/**
+ * MCP integration test harness.
+ *
+ * Builds MCP tools from an App, invokes them directly (no transport),
+ * and returns the MCP tool response.
+ */
+
+import { buildMcpTools } from '@ontrails/mcp';
+import type { McpToolDefinition } from '@ontrails/mcp';
+
+import type {
+  McpHarness,
+  McpHarnessOptions,
+  McpHarnessResult,
+} from './types.js';
+
+// ---------------------------------------------------------------------------
+// createMcpHarness
+// ---------------------------------------------------------------------------
+
+/**
+ * Create an MCP harness for integration testing.
+ *
+ * Builds MCP tools from the app's topo and provides a `callTool()` method
+ * that invokes tools directly without any transport layer.
+ *
+ * ```ts
+ * const harness = createMcpHarness({ app });
+ * const result = await harness.callTool("myapp_entity_show", { name: "Alpha" });
+ * expect(result.isError).toBe(false);
+ * ```
+ */
+export const createMcpHarness = (options: McpHarnessOptions): McpHarness => {
+  const tools = buildMcpTools(options.app);
+  const toolMap = new Map<string, McpToolDefinition>();
+  for (const tool of tools) {
+    toolMap.set(tool.name, tool);
+  }
+
+  return {
+    async callTool(
+      name: string,
+      args: Record<string, unknown>
+    ): Promise<McpHarnessResult> {
+      const tool = toolMap.get(name);
+      if (tool === undefined) {
+        return {
+          content: [{ text: `Unknown tool: ${name}`, type: 'text' }],
+          isError: true,
+        };
+      }
+
+      const result = await tool.handler(args, {
+        progressToken: undefined,
+        sendProgress: undefined,
+        signal: undefined,
+      });
+
+      return {
+        content: result.content,
+        isError: result.isError ?? false,
+      };
+    },
+  };
+};

package/src/hike.ts

@@ -0,0 +1,283 @@
+/**
+ * testHike — composition-aware scenario testing for hikes.
+ *
+ * Tests the composition graph: which trails were followed, in what order,
+ * and supports failure injection from followed trail examples.
+ */
+
+import { describe, expect, test } from 'bun:test';
+
+import type { AnyHike, AnyTrail, FollowFn, TrailContext } from '@ontrails/core';
+import {
+  InternalError,
+  Result,
+  ValidationError,
+  validateInput,
+} from '@ontrails/core';
+
+import {
+  assertErrorMatch,
+  assertFullMatch,
+  assertSchemaMatch,
+  expectOk,
+} from './assertions.js';
+import { mergeTestContext } from './context.js';
+import type { HikeScenario } from './types.js';
+
+// ---------------------------------------------------------------------------
+// Follow trace
+// ---------------------------------------------------------------------------
+
+interface FollowRecord {
+  readonly id: string;
+  readonly input: unknown;
+}
+
+// ---------------------------------------------------------------------------
+// Injection helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Find an error example on a trail by name or description substring.
+ */
+const findErrorExample = (
+  trailDef: AnyTrail,
+  description: string
+): string | undefined => {
+  const example = trailDef.examples?.find(
+    (ex) =>
+      ex.error !== undefined &&
+      (ex.description?.includes(description) || ex.name.includes(description))
+  );
+  return example?.error;
+};
+
+/**
+ * Try to inject an error from a followed trail's example.
+ * Returns undefined when no injection is configured for this trail ID.
+ */
+const tryInjectError = (
+  id: string,
+  scenario: HikeScenario,
+  trailsMap: ReadonlyMap<string, AnyTrail> | undefined
+): Result<unknown, Error> | undefined => {
+  const injection = scenario.injectFromExample?.[id];
+  if (injection === undefined) {
+    return undefined;
+  }
+
+  const trailDef = trailsMap?.get(id);
+  if (trailDef === undefined) {
+    return Result.err(
+      new InternalError(`Cannot inject: trail "${id}" not in topo`)
+    );
+  }
+  const errorName = findErrorExample(trailDef, injection);
+  if (errorName === undefined) {
+    return Result.err(
+      new InternalError(
+        `No error example matching "${injection}" on trail "${id}"`
+      )
+    );
+  }
+  return Result.err(new Error(errorName));
+};
+
+/**
+ * Execute a trail from the map, validating input first.
+ */
+const executeFromMap = (
+  id: string,
+  input: unknown,
+  trailsMap: ReadonlyMap<string, AnyTrail> | undefined,
+  ctx: TrailContext
+): Result<unknown, Error> | Promise<Result<unknown, Error>> | undefined => {
+  const trailDef = trailsMap?.get(id);
+  if (trailDef === undefined) {
+    return undefined;
+  }
+
+  const validated = validateInput(trailDef.input, input);
+  if (validated.isErr()) {
+    return validated;
+  }
+  return trailDef.implementation(validated.value, ctx);
+};
+
+// ---------------------------------------------------------------------------
+// Follow factory
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a recording follow function that optionally injects errors.
+ */
+const createRecordingFollow = (
+  trace: FollowRecord[],
+  scenario: HikeScenario,
+  trailsMap: ReadonlyMap<string, AnyTrail> | undefined,
+  baseFollow: FollowFn | undefined,
+  ctx: TrailContext
+): FollowFn => {
+  // The generic O on FollowFn is erased at runtime; the cast is safe
+  // because callers narrow via isOk/isErr before accessing the value.
+  const follow = (id: string, input: unknown) => {
+    trace.push({ id, input });
+
+    const injected = tryInjectError(id, scenario, trailsMap);
+    if (injected !== undefined) {
+      return Promise.resolve(injected);
+    }
+
+    if (baseFollow !== undefined) {
+      return baseFollow(id, input);
+    }
+
+    const executed = executeFromMap(id, input, trailsMap, ctx);
+    if (executed !== undefined) {
+      return Promise.resolve(executed);
+    }
+
+    return Promise.resolve(Result.ok());
+  };
+  return follow as FollowFn;
+};
+
+// ---------------------------------------------------------------------------
+// Scenario assertions
+// ---------------------------------------------------------------------------
+
+const assertScenarioResult = (
+  result: Result<unknown, Error>,
+  scenario: HikeScenario,
+  hikeDef: AnyHike
+): void => {
+  if (scenario.expectValue !== undefined) {
+    assertFullMatch(result, scenario.expectValue);
+  } else if (scenario.expectErr !== undefined) {
+    assertErrorMatch(result, scenario.expectErr, scenario.expectErrMessage);
+  } else if (scenario.expectErrMessage !== undefined) {
+    expect(result.isErr()).toBe(true);
+    if (result.isErr()) {
+      expect(result.error.message).toContain(scenario.expectErrMessage);
+    }
+  } else if (scenario.expectOk === true) {
+    expect(result.isOk()).toBe(true);
+    assertSchemaMatch(result, hikeDef.output);
+  }
+};
+
+const assertFollowTrace = (
+  trace: readonly FollowRecord[],
+  scenario: HikeScenario
+): void => {
+  if (scenario.expectFollowed !== undefined) {
+    const followedIds = trace.map((r) => r.id);
+    expect(followedIds).toEqual([...scenario.expectFollowed]);
+  }
+  if (scenario.expectFollowedCount !== undefined) {
+    const counts: Record<string, number> = {};
+    for (const record of trace) {
+      counts[record.id] = (counts[record.id] ?? 0) + 1;
+    }
+    expect(counts).toEqual({ ...scenario.expectFollowedCount });
+  }
+};
+
+const handleValidationError = (
+  validated: Result<unknown, Error>,
+  scenario: HikeScenario
+): boolean => {
+  if (!validated.isErr()) {
+    return false;
+  }
+  if (scenario.expectErr === ValidationError) {
+    expect(validated.error).toBeInstanceOf(ValidationError);
+    if (scenario.expectErrMessage !== undefined) {
+      expect(validated.error.message).toContain(scenario.expectErrMessage);
+    }
+    return true;
+  }
+  throw new Error(
+    `Input validation failed unexpectedly: ${validated.error.message}`
+  );
+};
+
+// ---------------------------------------------------------------------------
+// Scenario runner
+// ---------------------------------------------------------------------------
+
+const buildTestContext = (
+  scenario: HikeScenario,
+  ctx: Partial<TrailContext> | undefined,
+  trailsMap: ReadonlyMap<string, AnyTrail> | undefined
+): { trace: FollowRecord[]; testCtx: TrailContext } => {
+  const trace: FollowRecord[] = [];
+  const baseCtx = mergeTestContext(ctx);
+  const follow = createRecordingFollow(
+    trace,
+    scenario,
+    trailsMap,
+    baseCtx.follow,
+    baseCtx
+  );
+  return { testCtx: { ...baseCtx, follow }, trace };
+};
+
+const runScenario = async (
+  hikeDef: AnyHike,
+  scenario: HikeScenario,
+  ctx: Partial<TrailContext> | undefined,
+  trailsMap: ReadonlyMap<string, AnyTrail> | undefined
+): Promise<void> => {
+  const validated = validateInput(hikeDef.input, scenario.input);
+  if (handleValidationError(validated, scenario)) {
+    return;
+  }
+
+  const { trace, testCtx } = buildTestContext(scenario, ctx, trailsMap);
+  const result = await hikeDef.implementation(expectOk(validated), testCtx);
+  assertFollowTrace(trace, scenario);
+  assertScenarioResult(result, scenario, hikeDef);
+};
+
+// ---------------------------------------------------------------------------
+// testHike
+// ---------------------------------------------------------------------------
+
+/** Options for testHike that provide trail definitions for injection. */
+export interface TestHikeOptions {
+  /** Partial context overrides. */
+  readonly ctx?: Partial<TrailContext> | undefined;
+  /** Map of trail ID to trail definition, used for injectFromExample. */
+  readonly trails?: ReadonlyMap<string, AnyTrail> | undefined;
+}
+
+/**
+ * Generate a describe block for a hike with one test per scenario.
+ *
+ * @example
+ * ```ts
+ * testHike(onboardHike, [
+ *   {
+ *     description: "follows add then relate",
+ *     input: { name: "Alpha" },
+ *     expectOk: true,
+ *     expectFollowed: ["entity.add", "entity.relate"],
+ *   },
+ * ]);
+ * ```
+ */
+export const testHike = (
+  hikeDef: AnyHike,
+  scenarios: readonly HikeScenario[],
+  options?: TestHikeOptions
+): void => {
+  describe(hikeDef.id, () => {
+    test.each([...scenarios])(
+      '$description',
+      async (scenario: HikeScenario) => {
+        await runScenario(hikeDef, scenario, options?.ctx, options?.trails);
+      }
+    );
+  });
+};

package/src/index.ts

@@ -0,0 +1,40 @@
+// Contract-driven testing
+export { testAll } from './all.js';
+export { testExamples } from './examples.js';
+export { testHike } from './hike.js';
+export { testTrail } from './trail.js';
+export { testContracts } from './contracts.js';
+export { testDetours } from './detours.js';
+
+// Assertions
+export {
+  assertErrorMatch,
+  assertFullMatch,
+  assertSchemaMatch,
+  expectErr,
+  expectOk,
+} from './assertions.js';
+
+// Mock factories
+export { createTestContext } from './context.js';
+export { createTestLogger } from './logger.js';
+
+// Surface harnesses
+export { createCliHarness } from './harness-cli.js';
+export { createMcpHarness } from './harness-mcp.js';
+
+// Types
+export type { TestHikeOptions } from './hike.js';
+
+export type {
+  HikeScenario,
+  TestScenario,
+  TestLogger,
+  TestTrailContextOptions,
+  CliHarness,
+  CliHarnessOptions,
+  CliHarnessResult,
+  McpHarness,
+  McpHarnessOptions,
+  McpHarnessResult,
+} from './types.js';