@ontrails/testing 1.0.0-beta.13 → 1.0.0-beta.15

package/src/examples.ts

@@ -11,7 +11,7 @@ import { describe, expect, test } from 'bun:test';
 
 import type {
   CrossFn,
-  ProvisionOverrideMap,
+  ResourceOverrideMap,
   Topo,
   TrailExample,
   Trail,
@@ -23,13 +23,16 @@ import {
   AmbiguousError,
   AssertionError,
   AuthError,
+  buildCrossValidationSchema,
   CancelledError,
   ConflictError,
+  DerivationError,
   InternalError,
   NetworkError,
   NotFoundError,
   PermissionError,
   RateLimitError,
+  RetryExhaustedError,
   executeTrail,
   Result,
   TimeoutError,
@@ -42,16 +45,18 @@ import type { z } from 'zod';
 import {
   assertErrorMatch,
   assertFullMatch,
+  assertPartialMatch,
   assertSchemaMatch,
 } from './assertions.js';
 import {
-  defaultMintPermit,
-  mergeProvisionOverrides,
+  defaultCreatePermit,
+  mergeResourceOverrides,
   mergeTestContext,
   normalizeTestExecutionOptions,
-  resolveMockProvisions,
+  createMockResources,
 } from './context.js';
-import type { MintableTrail, TestExecutionOptions } from './context.js';
+import type { PermittedTrail, TestExecutionOptions } from './context.js';
+import { isDerivedExample, deriveTrailExamples } from './effective-examples.js';
 
 // ---------------------------------------------------------------------------
 // Error class name -> constructor map
@@ -64,11 +69,15 @@ const ERROR_MAP: Record<string, new (...args: never[]) => Error> = {
   AuthError: AuthError as new (...args: never[]) => Error,
   CancelledError: CancelledError as new (...args: never[]) => Error,
   ConflictError: ConflictError as new (...args: never[]) => Error,
+  DerivationError: DerivationError as new (...args: never[]) => Error,
   InternalError: InternalError as new (...args: never[]) => Error,
   NetworkError: NetworkError as new (...args: never[]) => Error,
   NotFoundError: NotFoundError as new (...args: never[]) => Error,
   PermissionError: PermissionError as new (...args: never[]) => Error,
   RateLimitError: RateLimitError as new (...args: never[]) => Error,
+  RetryExhaustedError: RetryExhaustedError as unknown as new (
+    ...args: never[]
+  ) => Error,
   TimeoutError: TimeoutError as new (...args: never[]) => Error,
   TrailsError: TrailsError as unknown as new (...args: never[]) => Error,
   ValidationError: ValidationError as new (...args: never[]) => Error,
@@ -91,16 +100,14 @@ const assertProgressiveMatch = (
   output: z.ZodType | undefined
 ): void => {
   if (example.expected !== undefined) {
-    assertFullMatch(result, example.expected);
-    return;
+    return assertFullMatch(result, example.expected);
+  }
+  if (example.expectedMatch !== undefined) {
+    return assertPartialMatch(result, example.expectedMatch);
   }
-
   if (example.error !== undefined) {
-    const errorClass = resolveErrorClass(example.error);
-    assertErrorMatch(result, errorClass);
-    return;
+    return assertErrorMatch(result, resolveErrorClass(example.error));
   }
-
   assertSchemaMatch(result, output);
 };
 
@@ -129,12 +136,12 @@ const handleValidationError = (
 };
 
 /**
- * Apply auto-minting: if the trail declares scoped permits and the context
- * doesn't already have a permit, mint one and merge it into the context.
+ * Apply auto-permit: if the trail declares scoped permits and the context
+ * doesn't already have a permit, create one and merge it into the context.
  */
-const applyAutoMint = (
+const applyAutoPermit = (
   ctx: TrailContext,
-  trailDef: MintableTrail,
+  trailDef: PermittedTrail,
   opts: TestExecutionOptions
 ): TrailContext => {
   if (opts.strictPermits) {
@@ -143,8 +150,8 @@ const applyAutoMint = (
   if (ctx.permit !== undefined) {
     return ctx;
   }
-  const mint = opts.mintPermit ?? defaultMintPermit;
-  const permit = mint(trailDef);
+  const create = opts.createPermit ?? defaultCreatePermit;
+  const permit = create(trailDef);
   if (!permit) {
     return ctx;
   }
@@ -156,11 +163,11 @@ const applyAutoMint = (
  * Handles validation, execution, and assertions.
  */
 const runExample = async (
-  t: Trail<unknown, unknown>,
+  t: Trail<unknown, unknown, unknown>,
   example: TrailExample<unknown, unknown>,
   output: z.ZodType | undefined,
   testCtx: TrailContext,
-  provisions?: ProvisionOverrideMap,
+  resources?: ResourceOverrideMap,
   opts?: TestExecutionOptions
 ): Promise<void> => {
   const validated = validateInput(t.input, example.input);
@@ -169,11 +176,11 @@ const runExample = async (
     return;
   }
 
-  const ctx = opts ? applyAutoMint(testCtx, t, opts) : testCtx;
+  const ctx = opts ? applyAutoPermit(testCtx, t, opts) : testCtx;
 
   const result = await executeTrail(t, example.input, {
     ctx,
-    provisions: provisions ?? opts?.provisions,
+    resources: resources ?? opts?.resources,
   });
   assertProgressiveMatch(result, example, output);
 };
@@ -194,39 +201,70 @@ const createCoverageCross = (
   baseCross: CrossFn | undefined,
   topo: Topo,
   ctx: TrailContext,
-  provisions?: ProvisionOverrideMap
+  resources?: ResourceOverrideMap
 ): CrossFn => {
-  const cross = (id: string, input: unknown) => {
+  const invokeCross = async (
+    idOrTrail: string | { readonly id: string },
+    input: unknown,
+    self: CrossFn
+  ) => {
+    const id = typeof idOrTrail === 'string' ? idOrTrail : idOrTrail.id;
     called.add(id);
 
     if (baseCross !== undefined) {
-      return baseCross(id, input);
+      return await baseCross(id, input);
     }
 
     const trailDef = topo.get(id);
     if (trailDef !== undefined) {
-      return executeTrail(trailDef, input, {
-        ctx: { ...ctx, cross },
-        provisions,
+      return await executeTrail(trailDef, input, {
+        ctx: { ...ctx, cross: self },
+        resources,
+        validationSchema: buildCrossValidationSchema(trailDef),
       });
     }
 
-    return Promise.resolve(Result.ok());
+    return Result.ok();
   };
-  return cross as CrossFn;
+
+  // Accepts either a trail object (typed cross), a string id (untyped),
+  // or a batch of `[target, input]` tuples.
+  const cross = async function cross(
+    idOrTrail:
+      | string
+      | { readonly id: string }
+      | readonly (readonly [string | { readonly id: string }, unknown])[],
+    input?: unknown
+  ) {
+    if (Array.isArray(idOrTrail)) {
+      return await Promise.all(
+        idOrTrail.map(([target, batchInput]) =>
+          invokeCross(target, batchInput, cross as CrossFn)
+        )
+      );
+    }
+
+    return await invokeCross(
+      idOrTrail as string | { readonly id: string },
+      input,
+      cross as CrossFn
+    );
+  } as CrossFn;
+
+  return cross;
 };
 
 /**
  * Run a single example against a trail with crossings, recording cross calls.
  */
 const runCompositionExample = async (
-  trailDef: Trail<unknown, unknown>,
+  trailDef: Trail<unknown, unknown, unknown>,
   example: TrailExample<unknown, unknown>,
   output: z.ZodType | undefined,
   baseCtx: TrailContext,
   called: Set<string>,
   topo: Topo,
-  provisions?: ProvisionOverrideMap,
+  resources?: ResourceOverrideMap,
   opts?: TestExecutionOptions
 ): Promise<void> => {
   const validated = validateInput(trailDef.input, example.input);
@@ -235,19 +273,23 @@ const runCompositionExample = async (
     return;
   }
 
-  const mintedCtx = opts ? applyAutoMint(baseCtx, trailDef, opts) : baseCtx;
+  const permittedCtx = opts
+    ? applyAutoPermit(baseCtx, trailDef, opts)
+    : baseCtx;
   const cross = createCoverageCross(
     called,
-    mintedCtx.cross,
+    permittedCtx.cross,
     topo,
-    mintedCtx,
-    provisions
+    permittedCtx,
+    resources
   );
-  const testCtx: TrailContext = { ...mintedCtx, cross };
+  const testCtx: TrailContext = { ...permittedCtx, cross };
 
+  // Top-level trail validates against trail.input (not merged crossInput).
+  // Merged validation only applies to cross targets in executeFromMap/createCoverageCross.
   const result = await executeTrail(trailDef, example.input, {
     ctx: testCtx,
-    provisions: provisions ?? opts?.provisions,
+    resources: resources ?? opts?.resources,
   });
   assertProgressiveMatch(result, example, output);
 };
@@ -264,7 +306,7 @@ const runCompositionExample = async (
  *
  * One line in your test file:
  * ```ts
- * testExamples(app);
+ * testExamples(graph);
  * ```
  */
 export const testExamples = (
@@ -276,11 +318,12 @@ export const testExamples = (
 ): void => {
   const resolveInput =
     typeof ctxOrFactory === 'function' ? ctxOrFactory : () => ctxOrFactory;
-  const allTrails = app.list() as Trail<unknown, unknown>[];
-
-  const withExamples = allTrails.filter(
-    (t) => t.examples !== undefined && t.examples.length > 0
-  );
+  const withExamples = (app.list() as Trail<unknown, unknown, unknown>[])
+    .map((trailDef) => ({
+      ...trailDef,
+      examples: deriveTrailExamples(trailDef),
+    }))
+    .filter((trailDef) => trailDef.examples.length > 0);
   const simpleTrails = withExamples.filter((t) => t.crosses.length === 0);
   const compositionTrails = withExamples.filter((t) => t.crosses.length > 0);
 
@@ -296,19 +339,27 @@ export const testExamples = (
         'example: $name',
         async (example: TrailExample<unknown, unknown>) => {
           const resolved = normalizeTestExecutionOptions(resolveInput());
-          const provisions = mergeProvisionOverrides(
-            await resolveMockProvisions(app),
+          const resources = mergeResourceOverrides(
+            await createMockResources(app),
             resolved.ctx,
-            resolved.provisions
+            resolved.resources
           );
           const testCtx = mergeTestContext(resolved.ctx);
-          await runExample(t, example, output, testCtx, provisions, resolved);
+          await runExample(t, example, output, testCtx, resources, resolved);
         }
       );
     });
   }
 
-  // Composition trails: use recording cross and check coverage
+  // Composition trails: use recording cross and check coverage.
+  //
+  // Crossing coverage only runs against AUTHORED examples. Contour-derived
+  // fixtures are opportunistic coverage that may not exercise every
+  // `ctx.cross()` branch in the trail, so asserting coverage against them
+  // would produce false failures for trails whose authored intent was a
+  // single path. When a trail has zero authored examples the coverage
+  // assertion is skipped entirely — the derived-example runs still
+  // execute, but they are not required to cover declared crossings.
   if (compositionTrails.length > 0) {
     describe.each(compositionTrails)('$id', (t) => {
       const { examples, output } = t;
@@ -316,16 +367,29 @@ export const testExamples = (
         return;
       }
 
-      const called = new Set<string>();
+      const calledFromAuthored = new Set<string>();
+      const hasAuthoredExamples = examples.some(
+        (example) => !isDerivedExample(example)
+      );
+
+      // Only record cross calls from authored examples. Derived fixtures
+      // execute normally but do not contribute to coverage — the sink map
+      // routes each example to the right bucket without an inline
+      // conditional inside the test body.
+      const discardSink = new Set<string>();
+      const pickCoverageSink = (
+        example: TrailExample<unknown, unknown>
+      ): Set<string> =>
+        isDerivedExample(example) ? discardSink : calledFromAuthored;
 
       test.each([...examples])(
         'example: $name',
         async (example: TrailExample<unknown, unknown>) => {
           const resolved = normalizeTestExecutionOptions(resolveInput());
-          const provisions = mergeProvisionOverrides(
-            await resolveMockProvisions(app),
+          const resources = mergeResourceOverrides(
+            await createMockResources(app),
             resolved.ctx,
-            resolved.provisions
+            resolved.resources
           );
           const baseCtx = mergeTestContext(resolved.ctx);
           await runCompositionExample(
@@ -333,18 +397,22 @@ export const testExamples = (
             example,
             output,
             baseCtx,
-            called,
+            pickCoverageSink(example),
             app,
-            provisions,
+            resources,
             resolved
           );
         }
       );
 
-      test('crossing coverage', () => {
-        const uncovered = t.crosses.filter((id) => !called.has(id));
-        expect(uncovered).toEqual([]);
-      });
+      if (hasAuthoredExamples) {
+        test('crossing coverage', () => {
+          const uncovered = t.crosses.filter(
+            (id) => !calledFromAuthored.has(id)
+          );
+          expect(uncovered).toEqual([]);
+        });
+      }
     });
   }
 };

package/src/harness-cli.ts

@@ -1,14 +1,15 @@
 /**
  * CLI integration test harness.
  *
- * Builds CLI commands from an App, executes them in-process,
+ * Builds CLI commands from a graph, executes them in-process,
  * and captures stdout/stderr.
  */
 
-import { buildCliCommands } from '@ontrails/cli';
+import { deriveCliCommands } from '@ontrails/cli';
 import type { CliCommand } from '@ontrails/cli';
+import type { TrailContext } from '@ontrails/core';
 
-import { createTestContext } from './context.js';
+import { mergeTestContext } from './context.js';
 import type {
   CliHarness,
   CliHarnessOptions,
@@ -30,55 +31,31 @@ const parseCommandString = (input: string): string[] =>
 // Command resolution
 // ---------------------------------------------------------------------------
 
-/** Try to match group + name from tokens. */
-const tryGroupMatch = (
-  commands: CliCommand[],
-  firstToken: string,
-  secondToken: string | undefined,
-  tokens: string[]
-): { command: CliCommand; flagTokens: string[] } | undefined => {
-  if (secondToken === undefined || secondToken.startsWith('-')) {
-    return undefined;
-  }
-  const match = commands.find(
-    (c) => c.group === firstToken && c.name === secondToken
-  );
-  if (match === undefined) {
-    return undefined;
-  }
-  return { command: match, flagTokens: tokens.slice(2) };
-};
+const matchesPath = (
+  path: readonly string[],
+  tokens: readonly string[]
+): boolean =>
+  path.length <= tokens.length &&
+  path.every((segment, index) => tokens[index] === segment);
 
-/** Try to match a direct name from the first token. */
-const tryDirectMatch = (
+/** Resolve a command from tokens using the longest matching command path. */
+const resolveCommand = (
   commands: CliCommand[],
-  firstToken: string,
   tokens: string[]
 ): { command: CliCommand; flagTokens: string[] } | undefined => {
-  const match = commands.find(
-    (c) => c.name === firstToken && c.group === undefined
-  );
-  if (match === undefined) {
+  if (tokens.length === 0) {
     return undefined;
   }
-  return { command: match, flagTokens: tokens.slice(1) };
-};
 
-/** Resolve a command from tokens, handling group.name patterns. */
-const resolveCommand = (
-  commands: CliCommand[],
-  tokens: string[]
-): { command: CliCommand; flagTokens: string[] } | undefined => {
-  const [firstToken, secondToken] = tokens;
+  const [match] = commands
+    .filter((command) => matchesPath(command.path, tokens))
+    .toSorted((a, b) => b.path.length - a.path.length);
 
-  if (firstToken === undefined) {
+  if (match === undefined) {
     return undefined;
   }
 
-  return (
-    tryGroupMatch(commands, firstToken, secondToken, tokens) ??
-    tryDirectMatch(commands, firstToken, tokens)
-  );
+  return { command: match, flagTokens: tokens.slice(match.path.length) };
 };
 
 // ---------------------------------------------------------------------------
@@ -242,9 +219,10 @@ const buildErrorResult = (
 const executeCommand = async (
   command: CliCommand,
   flags: Record<string, unknown>,
-  streams: CapturedStreams
+  streams: CapturedStreams,
+  ctxOverrides?: Partial<TrailContext>
 ): Promise<CliHarnessResult> => {
-  const ctx = createTestContext();
+  const ctx = mergeTestContext(ctxOverrides);
   const result = await command.execute({}, flags, ctx);
   streams.restore();
 
@@ -262,7 +240,8 @@ const executeCommand = async (
 /** Run the full command pipeline: resolve, parse, execute. */
 const runCommand = async (
   commands: CliCommand[],
-  commandString: string
+  commandString: string,
+  ctxOverrides?: Partial<TrailContext>
 ): Promise<CliHarnessResult> => {
   const parts = parseCommandString(commandString);
   const resolved = resolveCommand(commands, parts);
@@ -279,7 +258,7 @@ const runCommand = async (
   const streams = captureStreams();
 
   try {
-    return await executeCommand(command, flags, streams);
+    return await executeCommand(command, flags, streams, ctxOverrides);
   } catch (error: unknown) {
     return buildErrorResult(error, streams);
   }
@@ -292,19 +271,24 @@ const runCommand = async (
 /**
  * Create a CLI harness for integration testing.
  *
- * Builds commands from the app's topo and provides a `run()` method
+ * Builds commands from the graph's topo and provides a `run()` method
  * that parses command strings and executes them in-process.
  *
  * ```ts
- * const harness = createCliHarness({ app });
+ * const harness = createCliHarness({ graph });
  * const result = await harness.run("entity show --name Alpha --output json");
  * expect(result.exitCode).toBe(0);
  * ```
  */
 export const createCliHarness = (options: CliHarnessOptions): CliHarness => {
-  const commands = buildCliCommands(options.app);
+  const { ctx, graph, ...deriveOptions } = options;
+  const commandsResult = deriveCliCommands(graph, deriveOptions);
+  if (commandsResult.isErr()) {
+    throw commandsResult.error;
+  }
+  const commands = commandsResult.value;
 
   return {
-    run: (commandString: string) => runCommand(commands, commandString),
+    run: (commandString: string) => runCommand(commands, commandString, ctx),
   };
 };

package/src/harness-mcp.ts

@@ -1,11 +1,11 @@
 /**
  * MCP integration test harness.
  *
- * Builds MCP tools from an App, invokes them directly (no transport),
+ * Builds MCP tools from a graph, invokes them directly (no transport),
  * and returns the MCP tool response.
  */
 
-import { buildMcpTools } from '@ontrails/mcp';
+import { deriveMcpTools } from '@ontrails/mcp';
 import type { McpToolDefinition } from '@ontrails/mcp';
 
 import type {
@@ -21,17 +21,18 @@ import type {
 /**
  * Create an MCP harness for integration testing.
  *
- * Builds MCP tools from the app's topo and provides a `callTool()` method
+ * Builds MCP tools from the graph's topo and provides a `callTool()` method
  * that invokes tools directly without any transport boundary.
  *
  * ```ts
- * const harness = createMcpHarness({ app });
+ * const harness = createMcpHarness({ graph });
  * const result = await harness.callTool("myapp_entity_show", { name: "Alpha" });
  * expect(result.isError).toBe(false);
  * ```
  */
 export const createMcpHarness = (options: McpHarnessOptions): McpHarness => {
-  const toolsResult = buildMcpTools(options.app);
+  const { extra, graph, ...deriveOptions } = options;
+  const toolsResult = deriveMcpTools(graph, deriveOptions);
   if (toolsResult.isErr()) {
     throw toolsResult.error;
   }
@@ -54,9 +55,9 @@ export const createMcpHarness = (options: McpHarnessOptions): McpHarness => {
       }
 
       const result = await tool.handler(args, {
-        abortSignal: undefined,
-        progressToken: undefined,
-        sendProgress: undefined,
+        abortSignal: extra?.abortSignal,
+        progressToken: extra?.progressToken,
+        sendProgress: extra?.sendProgress,
       });
 
       return {

package/src/index.ts

@@ -1,5 +1,6 @@
 // Contract-driven testing
 export { testAll } from './all.js';
+export { testAllEstablished } from './all.js';
 export { testExamples } from './examples.js';
 export { testCrosses } from './crosses.js';
 export { testTrail } from './trail.js';
@@ -10,16 +11,22 @@ export { testDetours } from './detours.js';
 export {
   assertErrorMatch,
   assertFullMatch,
+  assertPartialMatch,
   assertSchemaMatch,
+  errResultMatch,
   expectErr,
   expectOk,
+  okResultMatch,
 } from './assertions.js';
 
+// Scenario testing
+export { executeScenarioSteps, ref, scenario } from './scenario.js';
+
 // Mock factories
 export {
   createCrossContext,
   createTestContext,
-  defaultMintPermit,
+  defaultCreatePermit,
 } from './context.js';
 export { createTestLogger } from './logger.js';
 
@@ -30,14 +37,17 @@ export { createMcpHarness } from './harness-mcp.js';
 // Types
 export type { CreateCrossContextOptions } from './context.js';
 export type {
-  MintableTrail,
-  MintedPermit,
+  PermittedTrail,
+  MinimalPermit,
   TestExecutionOptions,
 } from './context.js';
 export type { TestCrossOptions } from './crosses.js';
 
 export type {
   CrossScenario,
+  RefToken,
+  ScenarioStep,
+  TestAllEstablishedOptions,
   TestScenario,
   TestLogger,
   TestTrailContextOptions,