@ddse/acm-runtime 0.5.0

package/src/resumable-executor.ts

@@ -0,0 +1,471 @@
+// Resumable executor with checkpoint and resume support
+import {
+  InternalContextScopeImpl,
+  type Goal,
+  type Context,
+  type Plan,
+  type CapabilityRegistry,
+  type ToolRegistry,
+  type PolicyEngine,
+  type StreamSink,
+  type RunContext,
+  ExternalContextProviderAdapter,
+} from '@ddse/acm-sdk';
+import { evaluateGuard } from './guards.js';
+import { MemoryLedger } from './ledger.js';
+import { withRetry } from './retry.js';
+import {
+  createCheckpoint,
+  validateCheckpoint,
+  MemoryCheckpointStore,
+  type Checkpoint,
+  type CheckpointStore,
+  type CheckpointState,
+} from './checkpoint.js';
+import type { ExecutePlanOptions, ExecutePlanResult, TaskExecutionRecord } from './executor.js';
+import { buildTaskNarrative, synthesizeGoalSummary } from './executor.js';
+import { createInstrumentedToolGetter } from './tool-envelope.js';
+
+/**
+ * Extended options for resumable execution
+ */
+export interface ResumableExecutePlanOptions extends ExecutePlanOptions {
+  resumeFrom?: string;           // Checkpoint ID to resume from
+  checkpointInterval?: number;   // Checkpoint after N tasks (default: 1)
+  checkpointStore?: CheckpointStore;  // Storage backend
+  runId?: string;                // Execution run identifier
+}
+
+/**
+ * Execute a plan with checkpoint and resume support
+ */
+export async function executeResumablePlan(
+  options: ResumableExecutePlanOptions
+): Promise<ExecutePlanResult> {
+  const {
+    goal,
+    context,
+    plan,
+    capabilityRegistry,
+    toolRegistry,
+    policy,
+    verify,
+    stream,
+    nucleusFactory,
+    nucleusConfig,
+    contextProvider,
+    resumeFrom,
+    checkpointInterval = 1,
+    checkpointStore = new MemoryCheckpointStore(),
+    runId = `run-${Date.now()}`,
+  } = options;
+
+  const ledger = options.ledger ?? new MemoryLedger();
+  let outputs: Record<string, any> = {};
+  let executionRecords: Record<string, TaskExecutionRecord> = {};
+  const policyContext: Record<string, any> = {};
+  const metrics = { costUsd: 0, elapsedSec: 0 };
+  let startTime = Date.now();
+  let executed = new Set<string>();
+  let tasksExecutedSinceCheckpoint = 0;
+
+  // Restore from checkpoint if resuming
+  if (resumeFrom) {
+    console.log(`Resuming from checkpoint: ${resumeFrom}`);
+    const checkpoint = await checkpointStore.get(runId, resumeFrom);
+
+    if (!checkpoint) {
+      throw new Error(`Checkpoint not found: ${resumeFrom}`);
+    }
+
+    if (!validateCheckpoint(checkpoint)) {
+      throw new Error(`Invalid checkpoint: ${resumeFrom}`);
+    }
+
+    // Restore state
+  outputs = checkpoint.state.outputs;
+  executionRecords = checkpoint.state.executionRecords ?? {};
+    executed = new Set(checkpoint.state.executed);
+    metrics.costUsd = checkpoint.state.metrics.costUsd;
+    metrics.elapsedSec = checkpoint.state.metrics.elapsedSec;
+
+    // Restore ledger
+    for (const entry of checkpoint.state.ledger) {
+      ledger.append(entry.type, entry.details);
+    }
+
+    console.log(`Restored ${executed.size} completed tasks`);
+    startTime = Date.now() - metrics.elapsedSec * 1000;
+  } else {
+    // Log plan selection for new execution
+    ledger.append('PLAN_SELECTED', {
+      planId: plan.id,
+      contextRef: plan.contextRef,
+      capabilityMapVersion: plan.capabilityMapVersion,
+    });
+  }
+
+  // Helper to create checkpoint
+  const saveCheckpoint = async () => {
+    const state: CheckpointState = {
+      goal,
+      context,
+      plan,
+  outputs,
+  executionRecords,
+      executed: Array.from(executed),
+      ledger: ledger.getEntries() as any[],
+      metrics: { ...metrics },
+    };
+
+    const checkpoint = createCheckpoint(runId, state);
+    await checkpointStore.put(runId, checkpoint);
+    
+    if (stream) {
+      stream.emit('checkpoint', {
+        checkpointId: checkpoint.id,
+        tasksCompleted: executed.size,
+      });
+    }
+
+    return checkpoint;
+  };
+
+  // Build execution order based on edges
+  const pending = plan.tasks.filter(t => !executed.has(t.id));
+
+  while (pending.length > 0) {
+    const readyTasks = pending.filter(taskSpec => {
+      // Skip already executed tasks
+      if (executed.has(taskSpec.id)) {
+        return false;
+      }
+
+      // Check if all dependencies are satisfied
+      const incomingEdges = plan.edges.filter(e => e.to === taskSpec.id);
+      if (incomingEdges.length === 0) {
+        return true; // No dependencies
+      }
+
+      return incomingEdges.every(edge => {
+        if (!executed.has(edge.from)) {
+          return false;
+        }
+
+        // Evaluate guard if present
+        if (edge.guard) {
+          const guardResult = evaluateGuard(edge.guard, {
+            context,
+            outputs,
+            policy: policyContext,
+          });
+
+          ledger.append('GUARD_EVAL', {
+            edge: `${edge.from}->${edge.to}`,
+            guard: edge.guard,
+            result: guardResult,
+          });
+
+          return guardResult;
+        }
+
+        return true;
+      });
+    });
+
+    if (readyTasks.length === 0) {
+      break; // No more tasks can be executed
+    }
+
+    // Execute ready tasks
+    for (const taskSpec of readyTasks) {
+      pending.splice(pending.indexOf(taskSpec), 1);
+
+      const capabilityName = taskSpec.capabilityRef || taskSpec.capability;
+      if (!capabilityName) {
+        throw new Error(`Task ${taskSpec.id} missing capability reference`);
+      }
+
+      const task = capabilityRegistry.resolve(capabilityName);
+      if (!task) {
+        throw new Error(`Task not found for capability: ${capabilityName}`);
+      }
+
+      const nucleusAllowedTools = new Set<string>(nucleusConfig.allowedTools ?? []);
+      for (const tool of taskSpec.tools ?? []) {
+        nucleusAllowedTools.add(tool.name);
+      }
+
+      const nucleus = nucleusFactory({
+        goalId: goal.id,
+        goalIntent: goal.intent,
+        planId: plan.id,
+        taskId: taskSpec.id,
+        contextRef: plan.contextRef,
+        context,
+        llmCall: nucleusConfig.llmCall,
+        hooks: nucleusConfig.hooks,
+        allowedTools: Array.from(nucleusAllowedTools),
+      });
+
+      const internalScope = new InternalContextScopeImpl(entry => {
+        ledger.append(entry.type, entry.details);
+      });
+      nucleus.setInternalContext(internalScope);
+
+      const getTool = createInstrumentedToolGetter({
+        taskId: taskSpec.id,
+        capability: capabilityName,
+        toolRegistry,
+        ledger,
+      });
+
+      // Build run context
+      const runContext: RunContext = {
+        goal,
+        context,
+        outputs,
+        metrics,
+        getTool,
+        getCapabilityRegistry: () => capabilityRegistry,
+        stream,
+        nucleus,
+        internalContext: internalScope,
+      };
+
+      let preflight = await nucleus.preflight();
+      if (preflight.status === 'NEEDS_CONTEXT') {
+        const requestedDirectives = preflight.retrievalDirectives;
+
+        ledger.append('CONTEXT_INTERNALIZED', {
+          taskId: taskSpec.id,
+          directives: requestedDirectives,
+          status: contextProvider ? 'requested' : 'unhandled',
+        });
+
+        if (!contextProvider) {
+          throw new Error(
+            `Task ${taskSpec.id} requires additional context retrieval: ${requestedDirectives.join(', ')}`
+          );
+        }
+
+        await contextProvider.fulfill({
+          directives: requestedDirectives,
+          scope: internalScope,
+          runContext,
+          nucleus,
+        });
+
+        preflight = await nucleus.preflight();
+        if (preflight.status === 'NEEDS_CONTEXT') {
+          throw new Error(
+            `Task ${taskSpec.id} still requires additional context after adapter execution: ${preflight.retrievalDirectives.join(', ')}`
+          );
+        }
+
+        ledger.append('CONTEXT_INTERNALIZED', {
+          taskId: taskSpec.id,
+          directives: requestedDirectives,
+          status: 'resolved',
+        });
+      }
+
+      // Policy pre-check
+      if (policy) {
+        const policyInput = task.policyInput?.(runContext, taskSpec.input) ?? {};
+        const decision = await policy.evaluate('task.pre', {
+          taskId: taskSpec.id,
+          capability: taskSpec.capability,
+          ...policyInput,
+        });
+
+        ledger.append('POLICY_PRE', {
+          taskId: taskSpec.id,
+          decision,
+        });
+
+        if (!decision.allow) {
+          throw new Error(`Policy denied task ${taskSpec.id}: ${decision.reason}`);
+        }
+
+        policyContext[taskSpec.id] = decision;
+      }
+
+      // Execute task with retry
+      const ledgerBaseline = ledger.getEntries().length;
+
+      ledger.append('TASK_START', {
+        taskId: taskSpec.id,
+        capability: capabilityName,
+        input: taskSpec.input,
+      });
+
+      stream?.emit('task', { taskId: taskSpec.id, status: 'running' });
+
+      try {
+        const executeTask = async () => task.execute(runContext, taskSpec.input);
+
+        const retryConfig = taskSpec.retry || (taskSpec.retryPolicy
+          ? {
+              attempts: taskSpec.retryPolicy.maxAttempts || 3,
+              backoff: 'exp' as const,
+            }
+          : undefined);
+
+        const output = retryConfig
+          ? await withRetry(executeTask, retryConfig)
+          : await executeTask();
+
+        outputs[taskSpec.id] = output;
+
+        // Policy post-check
+        if (policy) {
+          const decision = await policy.evaluate('task.post', {
+            taskId: taskSpec.id,
+            output,
+          });
+
+          ledger.append('POLICY_POST', {
+            taskId: taskSpec.id,
+            decision,
+          });
+        }
+
+        // Verification
+        if (verify && taskSpec.verification && taskSpec.verification.length > 0) {
+          const verified = await verify(taskSpec.id, output, taskSpec.verification);
+
+          ledger.append('VERIFICATION', {
+            taskId: taskSpec.id,
+            expressions: taskSpec.verification,
+            result: verified,
+          });
+
+          if (!verified) {
+            throw new Error(`Verification failed for task ${taskSpec.id}`);
+          }
+        }
+
+        const postcheck = await nucleus.postcheck(output);
+        if (postcheck.status === 'NEEDS_COMPENSATION') {
+          ledger.append('ERROR', {
+            taskId: taskSpec.id,
+            stage: 'NUCLEUS_POSTCHECK',
+            message: postcheck.reason,
+          });
+          throw new Error(`Task ${taskSpec.id} requires compensation: ${postcheck.reason}`);
+        }
+        if (postcheck.status === 'ESCALATE') {
+          ledger.append('ERROR', {
+            taskId: taskSpec.id,
+            stage: 'NUCLEUS_POSTCHECK',
+            message: postcheck.reason,
+          });
+          throw new Error(`Task ${taskSpec.id} escalated: ${postcheck.reason}`);
+        }
+
+        const narrative = buildTaskNarrative(ledger, ledgerBaseline, taskSpec.id, postcheck);
+
+        executionRecords[taskSpec.id] = {
+          output,
+          narrative,
+        };
+
+        ledger.append('TASK_END', {
+          taskId: taskSpec.id,
+          output,
+          narrative,
+        });
+
+        stream?.emit('task', { taskId: taskSpec.id, status: 'completed', output, narrative });
+
+        executed.add(taskSpec.id);
+        tasksExecutedSinceCheckpoint++;
+
+        // Create checkpoint if interval reached
+        if (tasksExecutedSinceCheckpoint >= checkpointInterval) {
+          await saveCheckpoint();
+          tasksExecutedSinceCheckpoint = 0;
+        }
+      } catch (err) {
+        const error = err as Error;
+        ledger.append('ERROR', {
+          taskId: taskSpec.id,
+          error: error.message,
+          stack: error.stack,
+        });
+
+        stream?.emit('task', { taskId: taskSpec.id, status: 'failed', error: error.message });
+
+        // Save checkpoint before throwing to allow resume
+        await saveCheckpoint();
+
+        throw error;
+      }
+    }
+  }
+
+  // Final checkpoint
+  if (tasksExecutedSinceCheckpoint > 0) {
+    await saveCheckpoint();
+  }
+
+  const goalSummary = await synthesizeGoalSummary({
+    goal,
+    plan,
+    executionRecords,
+    context,
+    nucleusFactory,
+    nucleusConfig,
+    ledger,
+    stream,
+  });
+
+  metrics.elapsedSec = (Date.now() - startTime) / 1000;
+
+  return {
+    outputsByTask: executionRecords,
+    ledger: ledger.getEntries(),
+    goalSummary,
+  };
+}
+
+/**
+ * ResumableExecutor class for managing resumable executions
+ */
+export class ResumableExecutor {
+  constructor(
+    private checkpointStore: CheckpointStore = new MemoryCheckpointStore()
+  ) {}
+
+  /**
+   * Execute a plan with checkpointing enabled
+   */
+  async execute(options: ResumableExecutePlanOptions): Promise<ExecutePlanResult> {
+    return executeResumablePlan({
+      ...options,
+      checkpointStore: options.checkpointStore ?? this.checkpointStore,
+    });
+  }
+
+  /**
+   * List available checkpoints for a run
+   */
+  async listCheckpoints(runId: string) {
+    return this.checkpointStore.list(runId);
+  }
+
+  /**
+   * Get a specific checkpoint
+   */
+  async getCheckpoint(runId: string, checkpointId?: string): Promise<Checkpoint | null> {
+    return this.checkpointStore.get(runId, checkpointId);
+  }
+
+  /**
+   * Prune old checkpoints
+   */
+  async pruneCheckpoints(runId: string, keepLast: number = 5): Promise<void> {
+    return this.checkpointStore.prune(runId, keepLast);
+  }
+}

package/src/retry.ts

@@ -0,0 +1,37 @@
+// Retry logic with backoff
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  config: {
+    attempts: number;
+    backoff: 'fixed' | 'exp';
+    baseMs?: number;
+    jitter?: boolean;
+  }
+): Promise<T> {
+  const baseMs = config.baseMs ?? 1000;
+  let lastError: Error | undefined;
+
+  for (let attempt = 0; attempt < config.attempts; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err as Error;
+      
+      if (attempt < config.attempts - 1) {
+        let delayMs = baseMs;
+        
+        if (config.backoff === 'exp') {
+          delayMs = baseMs * Math.pow(2, attempt);
+        }
+        
+        if (config.jitter) {
+          delayMs = delayMs * (0.5 + Math.random() * 0.5);
+        }
+        
+        await new Promise(resolve => setTimeout(resolve, delayMs));
+      }
+    }
+  }
+
+  throw lastError;
+}

package/src/tool-envelope.ts

@@ -0,0 +1,113 @@
+import { createHash } from 'crypto';
+import type { Tool, ToolCallEnvelope, ToolRegistry } from '@ddse/acm-sdk';
+import type { MemoryLedger } from './ledger.js';
+
+type ToolGetterOptions = {
+  taskId: string;
+  capability: string;
+  toolRegistry: ToolRegistry;
+  ledger?: MemoryLedger;
+};
+
+type InstrumentedTool = Tool<any, any> & {
+  call(input: any, idemKey?: string): Promise<any>;
+};
+
+function computeDigest(payload: unknown): string {
+  const normalized = typeof payload === 'string' ? payload : JSON.stringify(payload ?? {});
+  const hash = createHash('sha256');
+  hash.update(normalized);
+  return hash.digest('hex').substring(0, 32);
+}
+
+function cloneWithInstrumentedCall(
+  toolName: string,
+  tool: Tool<any, any>,
+  options: ToolGetterOptions
+): InstrumentedTool {
+  const instrumented = Object.create(tool) as InstrumentedTool;
+
+  instrumented.call = async (input: any, idemKey?: string) => {
+    const start = Date.now();
+    const envelopeBase: ToolCallEnvelope = {
+      id: idemKey ?? `${options.taskId}-${toolName}-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`,
+      name: toolName,
+      input: input ?? {},
+      metadata: {
+        timestamp: start,
+        digest: computeDigest(input ?? {}),
+      },
+    };
+
+    options.ledger?.append('TOOL_CALL', {
+      stage: 'start',
+      taskId: options.taskId,
+      capability: options.capability,
+      tool: toolName,
+      envelope: envelopeBase,
+    });
+
+    try {
+      const result = await tool.call.call(tool, input, idemKey);
+      const completed: ToolCallEnvelope = {
+        ...envelopeBase,
+        output: result,
+        metadata: {
+          ...envelopeBase.metadata,
+          duration_ms: Date.now() - start,
+        },
+      };
+
+      options.ledger?.append('TOOL_CALL', {
+        stage: 'complete',
+        taskId: options.taskId,
+        capability: options.capability,
+        tool: toolName,
+        envelope: completed,
+      });
+
+      return result;
+    } catch (error: any) {
+      const errEnvelope: ToolCallEnvelope = {
+        ...envelopeBase,
+        error: {
+          code: 'ERROR',
+          message: error?.message ?? String(error),
+        },
+        metadata: {
+          ...envelopeBase.metadata,
+          duration_ms: Date.now() - start,
+        },
+      };
+
+      options.ledger?.append('TOOL_CALL', {
+        stage: 'error',
+        taskId: options.taskId,
+        capability: options.capability,
+        tool: toolName,
+        envelope: errEnvelope,
+      });
+
+      throw error;
+    }
+  };
+
+  return instrumented;
+}
+
+export function createInstrumentedToolGetter(options: ToolGetterOptions) {
+  const cache = new Map<Tool<any, any>, InstrumentedTool>();
+
+  return (toolName: string) => {
+    const tool = options.toolRegistry.get(toolName);
+    if (!tool) {
+      return undefined;
+    }
+
+    if (!cache.has(tool)) {
+      cache.set(tool, cloneWithInstrumentedCall(toolName, tool, options));
+    }
+
+    return cache.get(tool);
+  };
+}