@directive-run/ai 0.2.0 → 0.4.0

package/dist/testing.d.cts

@@ -1,10 +1,10 @@
-import { AgentOrchestrator, OrchestratorOptions } from './index.cjs';
-import { A as ApprovalRequest, I as InputGuardrailData, O as OutputGuardrailData, M as Message, T as ToolCallGuardrailData, a as ToolCall, G as GuardrailResult, b as AgentLike, c as AgentRunner, R as RunOptions, d as GuardrailFn, e as GuardrailContext } from './types-Bbar7yKz.cjs';
+import { w as MultiAgentOrchestrator, x as MultiAgentOrchestratorOptions, c as AgentOrchestrator, O as OrchestratorOptions, H as HealthMonitor, C as MultiplexedStreamChunk, D as DebugTimeline, ah as TaskRegistration, X as ReflectionEvaluator } from './multi-agent-orchestrator-YFs28JsF.cjs';
+import { o as ApprovalRequest, u as BreakpointModifications, I as InputGuardrailData, O as OutputGuardrailData, M as Message, T as ToolCallGuardrailData, b2 as ToolCall, ad as GuardrailResult, A as AgentLike, b as AgentRunner, R as RunOptions, v as BreakpointRequest, C as Checkpoint, V as DagExecutionContext, X as DagNodeStatus, aN as RerouteEvent, aS as Scratchpad, a1 as DebugEventType, L as CheckpointStore, aj as InMemoryCheckpointStore, W as DagNode, Z as DagPattern, D as DebugEvent, G as GuardrailFn, aa as GuardrailContext } from './types-D5veI9su.cjs';
 import '@directive-run/core';
 import '@directive-run/core/plugins';
 
 /**
- * OpenAI Agents Testing Utilities
+ * Directive AI Testing Utilities
  *
  * Provides testing helpers for:
  * - Mock agent runners with configurable responses
@@ -14,7 +14,7 @@ import '@directive-run/core/plugins';
  *
  * @example
  * ```typescript
- * import { createMockAgentRunner, testGuardrail, createApprovalSimulator } from '@directive-run/ai';
+ * import { createMockAgentRunner, testGuardrail, createApprovalSimulator } from '@directive-run/ai/testing';
  *
  * describe('MyOrchestrator', () => {
  *   it('should block PII in input', async () => {
@@ -341,5 +341,390 @@ declare function createTimeController(startTime?: number): {
     set(time: number): void;
     reset(): void;
 };
+/** Options for test multi-agent orchestrator */
+interface TestMultiAgentOrchestratorOptions extends Omit<MultiAgentOrchestratorOptions, "runner"> {
+    /** Mock responses keyed by agent ID — internally mapped to agent names for the mock runner */
+    mockResponses?: Record<string, MockAgentConfig>;
+    /** Default mock response for unmatched agents */
+    defaultMockResponse?: MockAgentConfig;
+    /** Mock tasks keyed by task ID — auto-generates TaskRegistration wrappers */
+    mockTasks?: Record<string, {
+        output: unknown;
+        delay?: number;
+        shouldError?: boolean;
+    }>;
+}
+/** Test multi-agent orchestrator with additional testing utilities */
+interface TestMultiAgentOrchestrator extends MultiAgentOrchestrator {
+    /** The mock runner */
+    mockRunner: MockAgentRunner;
+    /** Approval simulator */
+    approvalSimulator: ApprovalSimulator;
+    /** Get recorded agent calls */
+    getCalls(): RecordedCall[];
+    /** Get approval requests */
+    getApprovalRequests(): ApprovalRequest[];
+    /** Reset all state */
+    resetAll(): void;
+}
+/**
+ * Create a test multi-agent orchestrator with mocking and simulation built in.
+ *
+ * @example
+ * ```typescript
+ * const test = createTestMultiAgentOrchestrator({
+ *   agents: {
+ *     researcher: { agent: { name: 'researcher' } },
+ *     writer: { agent: { name: 'writer' } },
+ *   },
+ *   mockResponses: {
+ *     researcher: { output: 'Research results', totalTokens: 100 },
+ *     writer: { output: 'Written article', totalTokens: 200 },
+ *   },
+ * });
+ *
+ * const result = await test.runAgent('researcher', 'What is AI?');
+ * expect(result.output).toBe('Research results');
+ * expect(test.getCalls()).toHaveLength(1);
+ * ```
+ */
+declare function createTestMultiAgentOrchestrator(options: TestMultiAgentOrchestratorOptions): TestMultiAgentOrchestrator;
+/**
+ * Assert that a multi-agent orchestrator has specific state.
+ *
+ * @example
+ * ```typescript
+ * assertMultiAgentState(orchestrator, {
+ *   agentStatus: { researcher: 'completed', writer: 'idle' },
+ *   globalTokens: { min: 0, max: 1000 },
+ *   pendingHandoffs: 0,
+ * });
+ * ```
+ */
+declare function assertMultiAgentState(orchestrator: MultiAgentOrchestrator, expected: {
+    agentStatus?: Record<string, "idle" | "running" | "completed" | "error">;
+    taskStatus?: Record<string, string>;
+    totalTokens?: {
+        agentId?: string;
+        min?: number;
+        max?: number;
+    };
+    globalTokens?: {
+        min?: number;
+        max?: number;
+    };
+    pendingHandoffs?: number;
+}): void;
+/**
+ * Create a mock TaskRegistration for testing.
+ *
+ * @example
+ * ```typescript
+ * const task = createMockTask({ result: "processed" }, { delay: 100 });
+ * const orchestrator = createTestMultiAgentOrchestrator({
+ *   agents: { writer: { agent: { name: "writer" } } },
+ *   tasks: { process: task },
+ * });
+ * ```
+ */
+declare function createMockTask(output: unknown, options?: {
+    delay?: number;
+    shouldError?: boolean;
+    label?: string;
+    description?: string;
+}): TaskRegistration;
+
+/**
+ * Create a test DAG pattern from a simplified node spec.
+ *
+ * @example
+ * ```typescript
+ * const pattern = createTestDag({
+ *   A: { handler: "researcher" },
+ *   B: { handler: "writer", deps: ["A"] },
+ *   C: { handler: "reviewer", deps: ["B"] },
+ * });
+ * ```
+ */
+declare function createTestDag<T = unknown>(nodes: Record<string, Pick<DagNode, "handler" | "deps" | "when" | "transform" | "timeout" | "priority">>, merge?: (context: DagExecutionContext) => T | Promise<T>, options?: {
+    timeout?: number;
+    maxConcurrent?: number;
+    onNodeError?: "fail" | "skip-downstream" | "continue";
+}): DagPattern<T>;
+/**
+ * Assert that a DAG execution produced the expected node statuses.
+ *
+ * @example
+ * ```typescript
+ * assertDagExecution(context, {
+ *   nodeStatuses: { A: "completed", B: "completed", C: "skipped" },
+ *   completedNodes: ["A", "B"],
+ *   skippedNodes: ["C"],
+ * });
+ * ```
+ */
+declare function assertDagExecution(context: DagExecutionContext, expected: {
+    nodeStatuses?: Record<string, DagNodeStatus>;
+    completedNodes?: string[];
+    skippedNodes?: string[];
+    errorNodes?: string[];
+    outputContains?: Record<string, unknown>;
+}): void;
+
+/**
+ * Create a test debug timeline pre-populated with events.
+ *
+ * @example
+ * ```typescript
+ * const timeline = createTestTimeline([
+ *   { type: "agent_start", agentId: "researcher", inputLength: 42 },
+ *   { type: "agent_complete", agentId: "researcher", outputLength: 100, durationMs: 500, totalTokens: 200 },
+ * ]);
+ *
+ * expect(timeline.getEventsForAgent("researcher")).toHaveLength(2);
+ * ```
+ */
+declare function createTestTimeline(events?: Array<Partial<DebugEvent> & {
+    type: DebugEventType;
+}>, options?: {
+    maxEvents?: number;
+}): DebugTimeline;
+/**
+ * Assert that a debug timeline contains expected events.
+ *
+ * @example
+ * ```typescript
+ * assertTimelineEvents(timeline, {
+ *   totalEvents: 5,
+ *   eventTypes: ["agent_start", "guardrail_check", "agent_complete"],
+ *   agentEvents: { researcher: 3, writer: 2 },
+ *   hasType: "guardrail_check",
+ * });
+ * ```
+ */
+declare function assertTimelineEvents(timeline: DebugTimeline, expected: {
+    totalEvents?: number;
+    minEvents?: number;
+    maxEvents?: number;
+    eventTypes?: DebugEventType[];
+    agentEvents?: Record<string, number>;
+    hasType?: DebugEventType;
+    doesNotHaveType?: DebugEventType;
+}): void;
+
+/**
+ * Create a runner that always fails, useful for testing self-healing.
+ *
+ * @example
+ * ```typescript
+ * const failing = createFailingRunner(new Error("Provider down"));
+ * const orchestrator = createAgentOrchestrator({
+ *   runner: failing,
+ *   selfHealing: { fallbackRunners: [backupRunner] },
+ * });
+ * ```
+ */
+declare function createFailingRunner(error?: Error, options?: {
+    delay?: number;
+    failAfter?: number;
+}): AgentRunner;
+/**
+ * Assert that an agent was rerouted during execution.
+ *
+ * @example
+ * ```typescript
+ * const events: RerouteEvent[] = [];
+ * const orchestrator = createMultiAgentOrchestrator({
+ *   selfHealing: {
+ *     onReroute: (event) => events.push(event),
+ *   },
+ * });
+ *
+ * // ... trigger reroute ...
+ * assertRerouted(events, {
+ *   fromAgent: "primary",
+ *   toAgent: "backup",
+ *   reason: /circuit breaker/i,
+ * });
+ * ```
+ */
+declare function assertRerouted(events: RerouteEvent[], expected: {
+    fromAgent?: string;
+    toAgent?: string;
+    reason?: string | RegExp;
+    minReroutes?: number;
+}): void;
+/**
+ * Assert the health state of an agent in the health monitor.
+ *
+ * @example
+ * ```typescript
+ * assertAgentHealth(monitor, "researcher", {
+ *   minScore: 70,
+ *   circuitState: "CLOSED",
+ * });
+ * ```
+ */
+declare function assertAgentHealth(monitor: HealthMonitor, agentId: string, expected: {
+    minScore?: number;
+    maxScore?: number;
+    circuitState?: "CLOSED" | "OPEN" | "HALF_OPEN";
+    minSuccessRate?: number;
+}): void;
+/**
+ * Create a test checkpoint store (wraps InMemoryCheckpointStore with assertion helpers).
+ *
+ * @example
+ * ```typescript
+ * const store = createTestCheckpointStore();
+ * const orchestrator = createAgentOrchestrator({ runner, checkpointStore: store });
+ * const cp = await orchestrator.checkpoint();
+ * expect(store.saved).toHaveLength(1);
+ * ```
+ */
+declare function createTestCheckpointStore(maxCheckpoints?: number): CheckpointStore & {
+    saved: Checkpoint[];
+    inner: InMemoryCheckpointStore;
+    /** Get the most recently saved checkpoint */
+    getLatest: () => Checkpoint | undefined;
+};
+/**
+ * Assert that a checkpoint has expected properties.
+ */
+declare function assertCheckpoint(checkpoint: Checkpoint, expected: {
+    orchestratorType?: "single" | "multi";
+    hasTimeline?: boolean;
+    hasMemory?: boolean;
+    hasSystemExport?: boolean;
+    label?: string;
+}): void;
+/** Options for the breakpoint simulator */
+interface BreakpointSimulatorOptions {
+    /** Auto-resume after this delay (ms). Default: 0 (immediate) */
+    autoResumeDelay?: number;
+    /** Modifications to apply on resume */
+    modifications?: BreakpointModifications;
+    /** If true, cancel instead of resume */
+    cancel?: boolean;
+    /** Cancel reason */
+    cancelReason?: string;
+}
+/**
+ * Create a breakpoint simulator that auto-resolves breakpoints.
+ *
+ * @example
+ * ```typescript
+ * const simulator = createBreakpointSimulator({ autoResumeDelay: 10 });
+ * const orchestrator = createAgentOrchestrator({
+ *   runner,
+ *   breakpoints: [{ type: "pre_agent_run" }],
+ *   onBreakpoint: simulator.handler,
+ * });
+ * // Run agent — breakpoint fires and auto-resumes
+ * await orchestrator.run("test input");
+ * expect(simulator.hits).toHaveLength(1);
+ * ```
+ */
+/** Minimal interface for breakpoint-capable orchestrators */
+interface BreakpointCapable {
+    resumeBreakpoint(id: string, modifications?: BreakpointModifications): void;
+    cancelBreakpoint(id: string, reason?: string): void;
+}
+declare function createBreakpointSimulator(options?: BreakpointSimulatorOptions): {
+    handler: (request: BreakpointRequest) => void;
+    hits: BreakpointRequest[];
+    attachTo: (orchestrator: BreakpointCapable) => void;
+};
+/**
+ * Assert that a breakpoint was hit with expected properties.
+ */
+declare function assertBreakpointHit(hits: BreakpointRequest[], expected: {
+    type?: string;
+    agentId?: string;
+    count?: number;
+}): void;
+/**
+ * Create a mock SafeParseable schema for testing.
+ *
+ * @example
+ * ```typescript
+ * const schema = createMockSchema((data) => typeof data === "object" && data !== null);
+ * const orchestrator = createAgentOrchestrator({
+ *   runner,
+ *   outputSchema: schema,
+ * });
+ * ```
+ */
+declare function createMockSchema<T>(validate: (data: unknown) => boolean, description?: string): {
+    safeParse: (data: unknown) => {
+        success: boolean;
+        data?: T;
+        error?: {
+            message: string;
+        };
+    };
+    description?: string;
+};
+/**
+ * Collect all chunks from a multiplexed stream into an array.
+ *
+ * @example
+ * ```typescript
+ * const { stream } = orchestrator.runParallelStream(["a", "b"], "input", merge);
+ * const chunks = await collectMultiplexedStream(stream);
+ * expect(chunks.length).toBeGreaterThan(0);
+ * ```
+ */
+declare function collectMultiplexedStream(stream: AsyncIterable<MultiplexedStreamChunk>): Promise<MultiplexedStreamChunk[]>;
+/**
+ * Assert properties of collected multiplexed stream chunks.
+ */
+declare function assertMultiplexedStream(chunks: MultiplexedStreamChunk[], expected: {
+    agentIds?: string[];
+    minChunks?: number;
+    hasDone?: boolean;
+    hasErrors?: boolean;
+}): void;
+/**
+ * Create a test reflection evaluator that passes after N iterations.
+ *
+ * @example
+ * ```typescript
+ * const evaluator = createTestReflectionEvaluator({ passAfter: 2 });
+ * const reflective = withReflection(runner, { evaluate: evaluator });
+ * ```
+ */
+declare function createTestReflectionEvaluator(options?: {
+    /** Pass after this many evaluations (1 = pass on first try). Default: 1 */
+    passAfter?: number;
+    /** Feedback to provide on failure */
+    feedback?: string;
+    /** Score to assign (0-1) */
+    score?: number;
+}): ReflectionEvaluator;
+/**
+ * Assert that a scratchpad contains expected values.
+ *
+ * @example
+ * ```typescript
+ * assertScratchpadState(orchestrator.scratchpad!, {
+ *   "plan.status": "complete",
+ *   "research.results": expect.any(Array),
+ * });
+ * ```
+ */
+declare function assertScratchpadState(scratchpad: Scratchpad, expected: Record<string, unknown>): void;
+/**
+ * Assert that derived values match expected values.
+ *
+ * @example
+ * ```typescript
+ * assertDerivedValues(orchestrator, {
+ *   totalRuns: 3,
+ *   allComplete: true,
+ * });
+ * ```
+ */
+declare function assertDerivedValues(orchestrator: MultiAgentOrchestrator, expected: Record<string, unknown>): void;
 
-export { type ApprovalSimulator, type ApprovalSimulatorOptions, type ConstraintSnapshot, type GuardrailTestInput, type GuardrailTestResult, type MockAgentConfig, type MockAgentRunner, type MockAgentRunnerOptions, type RecordedCall, type TestOrchestrator, type TestOrchestratorOptions, assertOrchestratorState, createApprovalSimulator, createConstraintRecorder, createMockAgentRunner, createTestOrchestrator, createTimeController, testGuardrail, testGuardrailBatch };
+export { type ApprovalSimulator, type ApprovalSimulatorOptions, type BreakpointCapable, type BreakpointSimulatorOptions, type ConstraintSnapshot, type GuardrailTestInput, type GuardrailTestResult, type MockAgentConfig, type MockAgentRunner, type MockAgentRunnerOptions, type RecordedCall, type TestMultiAgentOrchestrator, type TestMultiAgentOrchestratorOptions, type TestOrchestrator, type TestOrchestratorOptions, assertAgentHealth, assertBreakpointHit, assertCheckpoint, assertDagExecution, assertDerivedValues, assertMultiAgentState, assertMultiplexedStream, assertOrchestratorState, assertRerouted, assertScratchpadState, assertTimelineEvents, collectMultiplexedStream, createApprovalSimulator, createBreakpointSimulator, createConstraintRecorder, createFailingRunner, createMockAgentRunner, createMockSchema, createMockTask, createTestCheckpointStore, createTestDag, createTestMultiAgentOrchestrator, createTestOrchestrator, createTestReflectionEvaluator, createTestTimeline, createTimeController, testGuardrail, testGuardrailBatch };