@ontrails/testing 1.0.0-beta.14 → 1.0.0-beta.16

package/src/scenario.ts

@@ -0,0 +1,368 @@
+/**
+ * Multi-step scenario runner for composition testing.
+ *
+ * Scenarios express multi-trail flows as structured data — arrays of steps
+ * with cross-step references via `ref()`. Each step invokes a trail through
+ * the normal execution pipeline (validation, layers, blaze, Result).
+ */
+
+import { describe, test } from 'bun:test';
+
+import type {
+  AnyTrail,
+  CrossBatchOptions,
+  CrossFn,
+  ResourceOverrideMap,
+  Result,
+  Topo,
+} from '@ontrails/core';
+import {
+  buildCrossValidationSchema,
+  claimNextCrossBatchIndex,
+  createCrossBatchValidationResults,
+  executeTrail,
+  InternalError,
+  normalizeCrossBatchConcurrency,
+  Result as R,
+} from '@ontrails/core';
+
+import { assertPartialMatch, expectOk } from './assertions.js';
+import { createTestContext, createMockResources } from './context.js';
+import type { RefToken, ScenarioStep } from './types.js';
+
+type ScenarioCrossTarget = string | { readonly id: string };
+type ScenarioCrossCall = readonly [ScenarioCrossTarget, unknown];
+
+// ---------------------------------------------------------------------------
+// ref() — cross-step reference marker
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a reference marker for cross-step data in scenario inputs.
+ *
+ * `ref('create.id')` resolves to the `id` field of the step aliased as
+ * `create`. Dot-paths are supported for nested access.
+ *
+ * @example
+ * ```typescript
+ * scenario('Fork flow', app, [
+ *   { cross: createGist, input: { name: 'Hello' }, as: 'original' },
+ *   { cross: forkGist, input: { id: ref('original.id') } },
+ * ]);
+ * ```
+ */
+export const ref = (path: string): RefToken => ({ __ref: true, path });
+
+// ---------------------------------------------------------------------------
+// Internals
+// ---------------------------------------------------------------------------
+
+/** Type guard for RefToken. */
+const isRef = (value: unknown): value is RefToken =>
+  typeof value === 'object' &&
+  value !== null &&
+  '__ref' in value &&
+  (value as Record<string, unknown>)['__ref'] === true &&
+  'path' in value;
+
+/**
+ * Resolve a dot-path against the outputs map.
+ *
+ * `ref('create.id')` splits into step name `create` and field path `id`.
+ * The first segment is the step alias; remaining segments are property lookups.
+ */
+/**
+ * Walk remaining segments of a dot-path, drilling into the step output.
+ */
+const drillPath = (
+  path: string,
+  segments: readonly string[],
+  start: unknown
+): unknown => {
+  let current: unknown = start;
+  for (let i = 1; i < segments.length; i += 1) {
+    const segment = segments[i];
+    if (segment === undefined) {
+      break;
+    }
+    if (typeof current !== 'object' || current === null) {
+      throw new Error(
+        `ref('${path}'): cannot access '${segment}' on ${typeof current}`
+      );
+    }
+    current = (current as Record<string, unknown>)[segment];
+  }
+  return current;
+};
+
+const resolvePath = (path: string, outputs: Map<string, unknown>): unknown => {
+  const segments = path.split('.');
+  const [stepName] = segments;
+  if (stepName === undefined) {
+    throw new Error(`ref(): empty path`);
+  }
+
+  const stepOutput = outputs.get(stepName);
+  if (stepOutput === undefined) {
+    throw new Error(
+      `ref('${path}'): no step output found for alias '${stepName}'`
+    );
+  }
+
+  return drillPath(path, segments, stepOutput);
+};
+
+/**
+ * Recursively walk a value, replacing RefToken instances with resolved values.
+ */
+export const deriveRefs = (
+  value: unknown,
+  outputs: Map<string, unknown>
+): unknown => {
+  if (isRef(value)) {
+    return resolvePath(value.path, outputs);
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((item) => deriveRefs(item, outputs));
+  }
+
+  if (typeof value === 'object' && value !== null) {
+    const result: Record<string, unknown> = {};
+    for (const [key, val] of Object.entries(value)) {
+      result[key] = deriveRefs(val, outputs);
+    }
+    return result;
+  }
+
+  return value;
+};
+
+// ---------------------------------------------------------------------------
+// scenario() — the public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Define a multi-step scenario test for composition flows.
+ *
+ * Each step invokes a trail through the normal execution pipeline.
+ * Steps can reference prior step outputs via `ref()`. If any step
+ * fails, the scenario stops and reports which step failed.
+ *
+ * @example
+ * ```typescript
+ * scenario('Create and show', app, [
+ *   { cross: createItem, input: { name: 'Test' }, as: 'created' },
+ *   { cross: showItem, input: { id: ref('created.id') },
+ *     expectedMatch: { found: true } },
+ * ]);
+ * ```
+ */
+/** Assert the result of a step against its expectations and record output. */
+const assertStepExpectations = async (
+  step: ScenarioStep,
+  result: Result<unknown, Error>,
+  outputs: Map<string, unknown>
+): Promise<void> => {
+  const value = expectOk(result);
+  if (step.expected !== undefined) {
+    const { expect } = await import('bun:test');
+    expect(value).toEqual(deriveRefs(step.expected, outputs));
+  } else if (step.expectedMatch !== undefined) {
+    assertPartialMatch(result, deriveRefs(step.expectedMatch, outputs));
+  }
+  if (step.as !== undefined) {
+    outputs.set(step.as, value);
+  }
+};
+
+const executeUnlimitedCrossBatch = async (
+  calls: readonly ScenarioCrossCall[],
+  runCall: (
+    call: ScenarioCrossCall,
+    branchIndex: number
+  ) => Promise<Result<unknown, Error>>
+): Promise<Result<unknown, Error>[]> =>
+  await Promise.all(
+    calls.map((call, branchIndex) => runCall(call, branchIndex))
+  );
+
+const executeLimitedCrossBatch = async (
+  calls: readonly ScenarioCrossCall[],
+  runCall: (
+    call: ScenarioCrossCall,
+    branchIndex: number
+  ) => Promise<Result<unknown, Error>>,
+  limit: number
+): Promise<Result<unknown, Error>[]> => {
+  const results = Array.from<Result<unknown, Error>>({ length: calls.length });
+  const nextIndex = { value: 0 };
+
+  const runWorker = async () => {
+    while (true) {
+      const branchIndex = claimNextCrossBatchIndex(nextIndex, calls);
+      if (branchIndex === undefined) {
+        return;
+      }
+
+      const call = calls[branchIndex];
+      if (call === undefined) {
+        // Defensive: `claimNextCrossBatchIndex` only returns indices within
+        // bounds, so this slot should always be populated. If it ever isn't,
+        // surface a clear InternalError in place of the missing slot and keep
+        // the worker loop running so sibling branches still get processed.
+        results[branchIndex] = R.err(
+          new InternalError(
+            `unreachable: concurrent cross batch call missing at index ${branchIndex}`
+          )
+        );
+        continue;
+      }
+
+      results[branchIndex] = await runCall(call, branchIndex);
+    }
+  };
+
+  await Promise.all(Array.from({ length: limit }, runWorker));
+  return results;
+};
+
+/**
+ * Build a cross function that resolves trails from the topo and executes
+ * them through the standard pipeline. Mirrors the pattern in crosses.ts
+ * `executeFromMap` but without recording or injection.
+ */
+const createScenarioCross = (
+  app: Topo,
+  resources?: ResourceOverrideMap
+): CrossFn => {
+  const invokeCross = async (
+    idOrTrail: ScenarioCrossTarget,
+    input: unknown,
+    self: CrossFn
+  ) => {
+    const id = typeof idOrTrail === 'string' ? idOrTrail : idOrTrail.id;
+    const trailDef: AnyTrail | undefined = app.get(id);
+    if (trailDef === undefined) {
+      return R.err(new InternalError(`cross: trail "${id}" not found in topo`));
+    }
+    const baseCtx = createTestContext();
+    return await executeTrail(trailDef, input, {
+      ctx: { ...baseCtx, cross: self },
+      resources,
+      topo: app,
+      validationSchema: buildCrossValidationSchema(trailDef),
+    });
+  };
+
+  const executeCrossBatch = async (
+    calls: readonly ScenarioCrossCall[],
+    self: CrossFn,
+    options?: CrossBatchOptions
+  ): Promise<Result<unknown, Error>[]> => {
+    if (calls.length === 0) {
+      return [];
+    }
+
+    const concurrency = normalizeCrossBatchConcurrency(options);
+    if (concurrency.isErr()) {
+      return createCrossBatchValidationResults(calls, concurrency.error);
+    }
+
+    const runCall = async (
+      [target, batchInput]: ScenarioCrossCall,
+      _branchIndex: number
+    ) => await invokeCross(target, batchInput, self);
+
+    const limit = concurrency.value ?? calls.length;
+    return limit >= calls.length
+      ? await executeUnlimitedCrossBatch(calls, runCall)
+      : await executeLimitedCrossBatch(calls, runCall, limit);
+  };
+
+  const cross = async function cross(
+    idOrTrail: ScenarioCrossTarget | readonly ScenarioCrossCall[],
+    inputOrOptions?: unknown
+  ) {
+    if (Array.isArray(idOrTrail)) {
+      return await executeCrossBatch(
+        idOrTrail,
+        cross as CrossFn,
+        inputOrOptions as CrossBatchOptions | undefined
+      );
+    }
+
+    return await invokeCross(
+      idOrTrail as ScenarioCrossTarget,
+      inputOrOptions,
+      cross as CrossFn
+    );
+  } as CrossFn;
+
+  return cross;
+};
+
+/**
+ * Execute a single scenario step: run the trail, assert expectations,
+ * and record outputs.
+ */
+const executeStep = async (
+  step: ScenarioStep,
+  index: number,
+  app: Topo,
+  outputs: Map<string, unknown>,
+  resources?: ResourceOverrideMap
+): Promise<void> => {
+  if (step.as !== undefined && outputs.has(step.as)) {
+    throw new Error(
+      `scenario: duplicate step alias "${step.as}" — each alias must be unique`
+    );
+  }
+
+  const scenarioCross = createScenarioCross(app, resources);
+  const baseCtx = createTestContext();
+  const resolvedInput = deriveRefs(step.input, outputs);
+  const result = await executeTrail(step.cross, resolvedInput, {
+    ctx: { ...baseCtx, cross: scenarioCross },
+    resources,
+    topo: app,
+  });
+
+  if (result.isErr()) {
+    throw new Error(
+      `Step ${String(index + 1)} ("${step.as ?? step.cross.id}") failed: ${result.error.message}`
+    );
+  }
+
+  await assertStepExpectations(step, result, outputs);
+};
+
+/**
+ * Execute scenario steps sequentially, resolving mock resources once upfront.
+ *
+ * Exported for direct use in tests that need to assert on step execution
+ * without the describe/test wrapper that `scenario()` provides.
+ */
+export const executeScenarioSteps = async (
+  app: Topo,
+  steps: readonly ScenarioStep[]
+): Promise<void> => {
+  const outputs = new Map<string, unknown>();
+  const resources = await createMockResources(app);
+
+  for (const [index, step] of steps.entries()) {
+    await executeStep(step, index, app, outputs, resources);
+  }
+};
+
+export const scenario = (
+  name: string,
+  app: Topo,
+  steps: readonly ScenarioStep[]
+): void => {
+  describe(name, () => {
+    test('executes all steps', async () => {
+      await executeScenarioSteps(app, steps);
+    });
+  });
+};

package/src/signals.ts

@@ -0,0 +1,221 @@
+import { isDeepStrictEqual } from 'node:util';
+
+import type {
+  FireFn,
+  TrailContext,
+  TrailExample,
+  TrailExampleSignalAssertion,
+} from '@ontrails/core';
+import { summarizeSignalPayload } from '@ontrails/core';
+
+export interface RecordedSignal {
+  readonly payload: unknown;
+  readonly signalId: string;
+}
+
+export interface SignalAssertionHarness {
+  readonly assert: () => void;
+  readonly ctx: TrailContext;
+}
+
+const noop = (): void => undefined;
+
+const resolveSignalId = (signal: unknown): string => {
+  if (typeof signal === 'string') {
+    return signal;
+  }
+  if (
+    typeof signal === 'object' &&
+    signal !== null &&
+    'id' in signal &&
+    typeof (signal as { readonly id: unknown }).id === 'string'
+  ) {
+    return (signal as { readonly id: string }).id;
+  }
+  return '<unknown signal>';
+};
+
+const assertionSignalId = (assertion: TrailExampleSignalAssertion): string =>
+  resolveSignalId(assertion.signal);
+
+const formatPayloadSummary = (payload: unknown): string => {
+  const summary = summarizeSignalPayload(payload);
+  const parts = [
+    `redacted=${summary.redacted}`,
+    `shape=${summary.shape}`,
+    `digest=${summary.digest}`,
+  ];
+  if (summary.topLevelEntryCount !== undefined) {
+    parts.push(`topLevelEntryCount=${summary.topLevelEntryCount}`);
+  }
+  return `{${parts.join(' ')}}`;
+};
+
+const formatAssertion = (assertion: TrailExampleSignalAssertion): string => {
+  const parts = [`signal=${assertionSignalId(assertion)}`];
+  if (assertion.payload !== undefined) {
+    parts.push(`payloadSummary=${formatPayloadSummary(assertion.payload)}`);
+  }
+  if (assertion.payloadMatch !== undefined) {
+    parts.push(
+      `payloadMatchSummary=${formatPayloadSummary(assertion.payloadMatch)}`
+    );
+  }
+  if (assertion.times !== undefined) {
+    parts.push(`times=${assertion.times}`);
+  }
+  return parts.join(' ');
+};
+
+const formatObserved = (observed: readonly RecordedSignal[]): string => {
+  if (observed.length === 0) {
+    return '<none>';
+  }
+  return observed
+    .map(
+      (record) =>
+        `${record.signalId} payloadSummary=${formatPayloadSummary(record.payload)}`
+    )
+    .join('; ');
+};
+
+const subsetArrayMatches = (
+  actual: readonly unknown[],
+  expected: readonly unknown[]
+): boolean => {
+  const consumed = new Set<number>();
+  for (const expectedItem of expected) {
+    const matchIndex = actual.findIndex(
+      (actualItem, index) =>
+        !consumed.has(index) &&
+        // oxlint-disable-next-line no-use-before-define -- mutual recursion with subsetMatches
+        subsetMatches(actualItem, expectedItem)
+    );
+    if (matchIndex === -1) {
+      return false;
+    }
+    consumed.add(matchIndex);
+  }
+  return true;
+};
+
+const subsetObjectMatches = (
+  actual: Record<string, unknown>,
+  expected: Record<string, unknown>
+): boolean =>
+  Object.keys(expected).every(
+    (key) =>
+      key in actual &&
+      // oxlint-disable-next-line no-use-before-define -- mutual recursion with subsetMatches
+      subsetMatches(actual[key], expected[key])
+  );
+
+const subsetMatches = (actual: unknown, expected: unknown): boolean => {
+  if (Array.isArray(expected)) {
+    return Array.isArray(actual) && subsetArrayMatches(actual, expected);
+  }
+  if (expected !== null && typeof expected === 'object') {
+    return (
+      actual !== null &&
+      typeof actual === 'object' &&
+      !Array.isArray(actual) &&
+      subsetObjectMatches(
+        actual as Record<string, unknown>,
+        expected as Record<string, unknown>
+      )
+    );
+  }
+  return isDeepStrictEqual(actual, expected);
+};
+
+const payloadMatches = (
+  payload: unknown,
+  assertion: TrailExampleSignalAssertion
+): boolean => {
+  if (
+    assertion.payload !== undefined &&
+    !isDeepStrictEqual(payload, assertion.payload)
+  ) {
+    return false;
+  }
+  if (
+    assertion.payloadMatch !== undefined &&
+    !subsetMatches(payload, assertion.payloadMatch)
+  ) {
+    return false;
+  }
+  return true;
+};
+
+const signalMatches = (
+  record: RecordedSignal,
+  assertion: TrailExampleSignalAssertion
+): boolean =>
+  record.signalId === assertionSignalId(assertion) &&
+  payloadMatches(record.payload, assertion);
+
+const assertValidTimes = (assertion: TrailExampleSignalAssertion): number => {
+  const times = assertion.times ?? 1;
+  if (!Number.isInteger(times) || times < 1) {
+    throw new Error(
+      `Signal assertion has invalid times value: ${formatAssertion(assertion)}`
+    );
+  }
+  return times;
+};
+
+const assertSignalAssertion = (
+  example: TrailExample<unknown, unknown>,
+  assertion: TrailExampleSignalAssertion,
+  observed: readonly RecordedSignal[],
+  consumed: Set<number>
+): void => {
+  const times = assertValidTimes(assertion);
+  for (let count = 0; count < times; count += 1) {
+    const matchIndex = observed.findIndex(
+      (record, index) =>
+        !consumed.has(index) && signalMatches(record, assertion)
+    );
+    if (matchIndex === -1) {
+      throw new Error(
+        `Example "${example.name}" expected signal ${formatAssertion(
+          assertion
+        )}; observed ${formatObserved(observed)}`
+      );
+    }
+    consumed.add(matchIndex);
+  }
+};
+
+export const assertSignalAssertions = (
+  example: TrailExample<unknown, unknown>,
+  observed: readonly RecordedSignal[]
+): void => {
+  const consumed = new Set<number>();
+  for (const assertion of example.signals ?? []) {
+    assertSignalAssertion(example, assertion, observed, consumed);
+  }
+};
+
+export const withSignalAssertions = (
+  ctx: TrailContext,
+  example: TrailExample<unknown, unknown>
+): SignalAssertionHarness => {
+  if (example.signals === undefined || example.signals.length === 0) {
+    return { assert: noop, ctx };
+  }
+
+  const observed: RecordedSignal[] = [];
+  const baseFire = ctx.fire as
+    | ((signal: unknown, payload: unknown) => Promise<void>)
+    | undefined;
+  const fire = (async (signal: unknown, payload: unknown): Promise<void> => {
+    observed.push({ payload, signalId: resolveSignalId(signal) });
+    await baseFire?.(signal, payload);
+  }) as FireFn;
+
+  return {
+    assert: () => assertSignalAssertions(example, observed),
+    ctx: { ...ctx, fire },
+  };
+};

package/src/types.ts

@@ -2,8 +2,16 @@
  * Shared types for @ontrails/testing.
  */
 
-import type { Logger, Topo } from '@ontrails/core';
-import type { LogLevel, LogRecord } from '@ontrails/logging';
+import type { DeriveCliCommandsOptions } from '@ontrails/cli';
+import type { McpExtra, DeriveMcpToolsOptions } from '@ontrails/mcp';
+import type {
+  AnyTrail,
+  Logger,
+  Topo,
+  TraceFn,
+  TrailContext,
+} from '@ontrails/core';
+import type { LogLevel, LogRecord } from '@ontrails/observe';
 
 // ---------------------------------------------------------------------------
 // Test Scenario (for testTrail)
@@ -66,6 +74,7 @@ export interface TestTrailContextOptions {
   readonly logger?: Logger | undefined;
   readonly requestId?: string | undefined;
   readonly abortSignal?: AbortSignal | undefined;
+  readonly trace?: TraceFn | undefined;
 }
 
 // ---------------------------------------------------------------------------
@@ -73,8 +82,12 @@ export interface TestTrailContextOptions {
 // ---------------------------------------------------------------------------
 
 /** Options for creating a CLI harness. */
-export interface CliHarnessOptions {
-  readonly app: Topo;
+export interface CliHarnessOptions extends Omit<
+  DeriveCliCommandsOptions,
+  'onResult' | 'presets' | 'resolveInput'
+> {
+  readonly ctx?: Partial<TrailContext> | undefined;
+  readonly graph: Topo;
 }
 
 /** A test harness for CLI commands. */
@@ -97,8 +110,9 @@ export interface CliHarnessResult {
 // ---------------------------------------------------------------------------
 
 /** Options for creating an MCP harness. */
-export interface McpHarnessOptions {
-  readonly app: Topo;
+export interface McpHarnessOptions extends DeriveMcpToolsOptions {
+  readonly extra?: Partial<McpExtra> | undefined;
+  readonly graph: Topo;
 }
 
 /** A test harness for MCP tools. */
@@ -115,3 +129,47 @@ export interface McpHarnessResult {
   readonly content: unknown;
   readonly isError: boolean;
 }
+
+// ---------------------------------------------------------------------------
+// Established verification
+// ---------------------------------------------------------------------------
+
+export interface TestAllEstablishedOptions {
+  readonly cli?: Omit<CliHarnessOptions, 'graph'> | undefined;
+  readonly createPermit?:
+    | ((trail: {
+        readonly permit?:
+          | { readonly scopes: readonly string[] }
+          | 'public'
+          | undefined;
+      }) =>
+        | {
+            readonly id: string;
+            readonly scopes: readonly string[];
+          }
+        | undefined)
+    | undefined;
+  readonly ctx?: Partial<TrailContext> | undefined;
+  readonly mcp?: Omit<McpHarnessOptions, 'graph'> | undefined;
+  readonly resources?: Record<string, unknown> | undefined;
+  readonly strictPermits?: boolean | undefined;
+}
+
+// ---------------------------------------------------------------------------
+// Scenario (for composition testing)
+// ---------------------------------------------------------------------------
+
+/** Marker for cross-step references in scenario inputs. */
+export interface RefToken {
+  readonly __ref: true;
+  readonly path: string;
+}
+
+/** A single step in a scenario. */
+export interface ScenarioStep {
+  readonly cross: AnyTrail;
+  readonly input: Record<string, unknown>;
+  readonly as?: string | undefined;
+  readonly expected?: unknown | undefined;
+  readonly expectedMatch?: unknown | undefined;
+}

package/.turbo/turbo-build.log

@@ -1 +0,0 @@
-$ tsc -b

package/.turbo/turbo-lint.log

@@ -1,3 +0,0 @@
-$ oxlint ./src
-Found 0 warnings and 0 errors.
-Finished in 29ms on 22 files with 93 rules using 24 threads.

package/.turbo/turbo-typecheck.log

@@ -1 +0,0 @@
-$ tsc --noEmit