@od-oneapp/ai-platform 0.1.0

package/src/pipelines/adapters/trace-storage-memory.ts

@@ -0,0 +1,319 @@
+/**
+ * @fileoverview In-Memory Trace Storage Adapter
+ * In-Memory Trace Storage Adapter
+ * For testing and development (no persistence)
+ */
+
+import { generateId } from '../../shared';
+
+import type { Artifact, TraceStorage } from '../trace-storage-interface';
+import type { PipelineConfig, PipelineRun, ResumptionPoint, StepResult } from '../types';
+
+/**
+ * In-memory storage adapter
+ * - Fast for testing and development
+ * - No external dependencies
+ * - Data lost on process restart
+ */
+export class MemoryTraceStorage implements TraceStorage {
+  private runs: Map<string, PipelineRun> = new Map();
+  private steps: Map<string, StepResult[]> = new Map();
+  private artifacts: Map<string, Buffer | string | object> = new Map();
+  private resumptionPoints: Map<string, ResumptionPoint> = new Map();
+  private actorIndex: Map<string, Set<string>> = new Map();
+  private pipelineIndex: Map<string, Set<string>> = new Map();
+
+  async initialize(): Promise<void> {
+    // No initialization needed for memory storage
+  }
+
+  async createRun(config: PipelineConfig): Promise<PipelineRun> {
+    const runId = `${config.id}-${generateId()}`;
+
+    const run: PipelineRun = {
+      runId,
+      pipelineId: config.id,
+      actorId: config.actorId,
+      status: 'running',
+      steps: [],
+      currentStepIndex: 0,
+      startedAt: new Date(),
+      totalPHIDetections: 0,
+      totalTokens: 0,
+      complianceMetrics: {
+        phiRedacted: 0,
+        abacChecks: 0,
+        auditEvents: 0,
+        approvalRequests: 0,
+      },
+      metadata: config.metadata,
+    };
+
+    // Store run
+    this.runs.set(runId, run);
+
+    // Index by actor
+    let actorRuns = this.actorIndex.get(config.actorId);
+    if (!actorRuns) {
+      actorRuns = new Set();
+      this.actorIndex.set(config.actorId, actorRuns);
+    }
+    actorRuns.add(runId);
+
+    // Index by pipeline
+    let pipelineRuns = this.pipelineIndex.get(config.id);
+    if (!pipelineRuns) {
+      pipelineRuns = new Set();
+      this.pipelineIndex.set(config.id, pipelineRuns);
+    }
+    pipelineRuns.add(runId);
+
+    return run;
+  }
+
+  async getRun(runId: string): Promise<PipelineRun | null> {
+    return this.runs.get(runId) ?? null;
+  }
+
+  async updateRun(
+    runId: string,
+    updates: Partial<
+      Pick<
+        PipelineRun,
+        | 'status'
+        | 'currentStepIndex'
+        | 'completedAt'
+        | 'totalDurationMs'
+        | 'error'
+        | 'resumeToken'
+        | 'complianceMetrics'
+        | 'totalPHIDetections'
+        | 'totalTokens'
+      >
+    >,
+  ): Promise<void> {
+    const run = this.runs.get(runId);
+    if (!run) {
+      throw new Error(`Run ${runId} not found`);
+    }
+
+    Object.assign(run, updates);
+    this.runs.set(runId, run);
+  }
+
+  async addStepResult(runId: string, result: StepResult): Promise<void> {
+    let runSteps = this.steps.get(runId);
+    if (!runSteps) {
+      runSteps = [];
+      this.steps.set(runId, runSteps);
+    }
+    runSteps.push(result);
+  }
+
+  async getStepResults(runId: string): Promise<StepResult[]> {
+    return this.steps.get(runId) ?? [];
+  }
+
+  async storeArtifact(runId: string, stepId: string, artifact: Artifact): Promise<string> {
+    const artifactId = `${runId}/${stepId}/${artifact.type}`;
+    this.artifacts.set(artifactId, artifact.content);
+    return `memory://${artifactId}`;
+  }
+
+  async getArtifact(url: string): Promise<Buffer | string | object> {
+    const artifactId = url.replace('memory://', '');
+    const artifact = this.artifacts.get(artifactId);
+    if (!artifact) {
+      throw new Error(`Artifact not found: ${url}`);
+    }
+    return artifact;
+  }
+
+  async createResumptionPoint(
+    runId: string,
+    stepIndex: number,
+    previousResults: StepResult[],
+  ): Promise<ResumptionPoint> {
+    const token = generateId();
+    const expiresAt = new Date(Date.now() + 3600 * 1000); // 1 hour
+
+    const resumption: ResumptionPoint = {
+      runId,
+      stepIndex,
+      previousResults,
+      token,
+      expiresAt,
+    };
+
+    this.resumptionPoints.set(token, resumption);
+    return resumption;
+  }
+
+  async getResumptionPoint(token: string): Promise<ResumptionPoint | null> {
+    const resumption = this.resumptionPoints.get(token);
+    if (!resumption) {
+      return null;
+    }
+
+    // Check if expired
+    if (resumption.expiresAt < new Date()) {
+      this.resumptionPoints.delete(token);
+      return null;
+    }
+
+    return resumption;
+  }
+
+  async cleanupExpiredResumptionPoints(): Promise<number> {
+    const now = new Date();
+    let count = 0;
+
+    for (const [token, resumption] of this.resumptionPoints.entries()) {
+      if (resumption.expiresAt < now) {
+        this.resumptionPoints.delete(token);
+        count++;
+      }
+    }
+
+    return count;
+  }
+
+  async queryRunsByActor(
+    actorId: string,
+    options?: {
+      limit?: number;
+      offset?: number;
+      status?: PipelineRun['status'];
+    },
+  ): Promise<PipelineRun[]> {
+    const runIds = this.actorIndex.get(actorId) ?? new Set();
+    let runs = [...runIds].flatMap(id => {
+      const run = this.runs.get(id);
+      return run ? [run] : [];
+    });
+
+    // Filter by status
+    if (options?.status) {
+      runs = runs.filter(r => r.status === options.status);
+    }
+
+    // Sort by start time (most recent first)
+    runs.sort((a, b) => b.startedAt.getTime() - a.startedAt.getTime());
+
+    // Pagination
+    const offset = options?.offset ?? 0;
+    const limit = options?.limit || 10;
+    return runs.slice(offset, offset + limit);
+  }
+
+  async queryRunsByPipeline(
+    pipelineId: string,
+    options?: {
+      limit?: number;
+      offset?: number;
+      status?: PipelineRun['status'];
+    },
+  ): Promise<PipelineRun[]> {
+    const runIds = this.pipelineIndex.get(pipelineId) ?? new Set();
+    let runs = [...runIds].flatMap(id => {
+      const run = this.runs.get(id);
+      return run ? [run] : [];
+    });
+
+    // Filter by status
+    if (options?.status) {
+      runs = runs.filter(r => r.status === options.status);
+    }
+
+    // Sort by start time (most recent first)
+    runs.sort((a, b) => b.startedAt.getTime() - a.startedAt.getTime());
+
+    // Pagination
+    const offset = options?.offset ?? 0;
+    const limit = options?.limit || 10;
+    return runs.slice(offset, offset + limit);
+  }
+
+  async getComplianceMetrics(
+    startDate: Date,
+    endDate: Date,
+    options?: {
+      actorId?: string;
+      pipelineId?: string;
+    },
+  ): Promise<{
+    totalRuns: number;
+    totalPHIDetections: number;
+    totalAuditEvents: number;
+    totalApprovals: number;
+    phiByType: Record<string, number>;
+    runsByStatus: Record<PipelineRun['status'], number>;
+  }> {
+    let runs = [...this.runs.values()];
+
+    // Filter by date range
+    runs = runs.filter(r => r.startedAt >= startDate && r.startedAt <= endDate);
+
+    // Filter by actor
+    if (options?.actorId) {
+      runs = runs.filter(r => r.actorId === options.actorId);
+    }
+
+    // Filter by pipeline
+    if (options?.pipelineId) {
+      runs = runs.filter(r => r.pipelineId === options.pipelineId);
+    }
+
+    // Aggregate metrics
+    const metrics = {
+      totalRuns: runs.length,
+      totalPHIDetections: runs.reduce((sum, r) => sum + r.totalPHIDetections, 0),
+      totalAuditEvents: runs.reduce((sum, r) => sum + (r.complianceMetrics.auditEvents ?? 0), 0),
+      totalApprovals: runs.reduce((sum, r) => sum + (r.complianceMetrics.approvalRequests ?? 0), 0),
+      phiByType: {} as Record<string, number>,
+      runsByStatus: {} as Record<PipelineRun['status'], number>,
+    };
+
+    // Count by status
+    for (const run of runs) {
+      metrics.runsByStatus[run.status] = (metrics.runsByStatus[run.status] || 0) + 1;
+    }
+
+    return metrics;
+  }
+
+  async close(): Promise<void> {
+    // No cleanup needed for memory storage
+  }
+
+  /**
+   * Clear all data (useful for testing)
+   */
+  clear(): void {
+    this.runs.clear();
+    this.steps.clear();
+    this.artifacts.clear();
+    this.resumptionPoints.clear();
+    this.actorIndex.clear();
+    this.pipelineIndex.clear();
+  }
+
+  /**
+   * Get storage statistics
+   */
+  getStats() {
+    return {
+      runs: this.runs.size,
+      steps: [...this.steps.values()].reduce((sum, steps) => sum + steps.length, 0),
+      artifacts: this.artifacts.size,
+      resumptionPoints: this.resumptionPoints.size,
+    };
+  }
+}
+
+/**
+ * Create in-memory storage adapter
+ */
+export function createMemoryTraceStorage(): MemoryTraceStorage {
+  return new MemoryTraceStorage();
+}

package/src/pipelines/defaults.ts

@@ -0,0 +1,109 @@
+/**
+ * @fileoverview Pipeline Defaults
+ * Pipeline Defaults
+ * DRY default configurations for common pipeline scenarios
+ */
+
+import type { ComplianceOptions } from '../governance/compliance/types';
+
+/**
+ * Default compliance options for HIPAA-compliant pipelines
+ */
+export const HIPAA_COMPLIANCE: ComplianceOptions = {
+  enablePHIRedaction: true,
+  enableABAC: true,
+  policyVersion: 'v2.0',
+};
+
+/**
+ * Default compliance options for general use (PHI redaction only)
+ */
+export const BASIC_COMPLIANCE: ComplianceOptions = {
+  enablePHIRedaction: true,
+  enableABAC: false,
+  policyVersion: 'v2.0',
+};
+
+/**
+ * No compliance (for non-sensitive data)
+ */
+export const NO_COMPLIANCE: ComplianceOptions = {
+  enablePHIRedaction: false,
+  enableABAC: false,
+  policyVersion: 'v2.0',
+};
+
+/**
+ * Common temperature settings
+ */
+export const TEMPERATURES = {
+  /** Deterministic, factual (0.0-0.3) */
+  factual: 0.1,
+  /** Balanced, research (0.3-0.5) */
+  balanced: 0.4,
+  /** Creative, synthesis (0.6-0.8) */
+  creative: 0.7,
+  /** Very creative (0.8-1.0) */
+  experimental: 0.9,
+} as const;
+
+/**
+ * Common system prompts
+ */
+export const SYSTEM_PROMPTS = {
+  research: `You are a research assistant. Provide comprehensive, well-researched information.
+Include key facts, concepts, and context. Be thorough and cite sources when possible.`,
+
+  analysis: `You are an expert analyst. Provide deep analysis with:
+- Key insights and patterns
+- Critical evaluation
+- Implications and recommendations
+- Knowledge gaps
+
+Be thorough and analytical.`,
+
+  synthesis: `You are a synthesis expert. Create clear, actionable summaries with:
+- Executive summary (3-5 sentences)
+- Key findings
+- Actionable recommendations
+- Next steps
+
+Write clearly for a general audience.`,
+
+  editor: `You are an expert editor. Improve clarity, coherence, and impact.
+Focus on structure, flow, and readability. Preserve the author's voice.`,
+
+  critic: `You are a constructive critic. Identify areas for improvement while acknowledging strengths.
+Be specific, actionable, and balanced in your feedback.`,
+} as const;
+
+/**
+ * Common timeouts (milliseconds)
+ */
+export const TIMEOUTS = {
+  /** Quick operations (30s) */
+  quick: 30000,
+  /** Standard operations (2min) */
+  standard: 120000,
+  /** Long operations (5min) */
+  long: 300000,
+  /** Very long operations (10min) */
+  veryLong: 600000,
+} as const;
+
+/**
+ * Common retry configurations
+ */
+export const RETRY_CONFIGS = {
+  /** No retry */
+  none: { maxAttempts: 1, delayMs: 0 },
+
+  /** Standard retry: 3 attempts with exponential backoff */
+  standard: { maxAttempts: 3, delayMs: 1000, backoff: 'exponential' as const },
+
+  /** Aggressive retry: 5 attempts with exponential backoff */
+  aggressive: { maxAttempts: 5, delayMs: 2000, backoff: 'exponential' as const },
+
+  /** Quick retry: 2 attempts with linear backoff */
+  quick: { maxAttempts: 2, delayMs: 500, backoff: 'linear' as const },
+} as const;

package/src/pipelines/index.ts

@@ -0,0 +1,24 @@
+/**
+ * Pipeline orchestration module
+ * Multi-step execution with resumption, compliance, and tracing
+ * Ported from oneapp-onstage production patterns
+ */
+
+// Core pipeline execution
+export * from './multi-step-wrapper';
+export * from './step-executor';
+export * from './validation';
+export * from './types';
+export * from './defaults';
+
+// Message transforms
+export * from './message-transforms';
+
+// Pipeline presets
+export * from './pipeline-presets';
+
+// Trace storage
+export * from './trace-storage-interface';
+export * from './storage-factory';
+export * from './adapters/trace-storage-blob';
+export * from './adapters/trace-storage-memory';

package/src/pipelines/message-transforms.ts

@@ -0,0 +1,107 @@
+/**
+ * @fileoverview Message Transformation Helpers
+ * Message Transformation Helpers
+ * DRY helpers for common message transformation patterns
+ */
+
+import type { CoreMessage } from '../shared';
+import type { StepResult } from './types';
+
+/**
+ * Message type for pipeline steps.
+ * Aligned with SDK's CoreMessage type.
+ */
+type PipelineMessage = CoreMessage;
+
+/**
+ * Use previous step's output as next input
+ * Most common pattern in pipelines
+ */
+export function usePreviousOutput(previousResults: StepResult[]): PipelineMessage[] {
+  const lastOutput = previousResults[previousResults.length - 1]?.output;
+
+  if (typeof lastOutput === 'string') {
+    return [{ role: 'user', content: lastOutput }];
+  }
+
+  return [{ role: 'user', content: 'Continue with the previous result.' }];
+}
+
+/**
+ * Combine all previous outputs into one message
+ */
+export function combineAllOutputs(
+  previousResults: StepResult[],
+  separator: string = '\n\n---\n\n',
+): PipelineMessage[] {
+  const allOutputs = previousResults
+    .map(r => r.output)
+    .filter((o): o is string => typeof o === 'string')
+    .join(separator);
+
+  return [{ role: 'user', content: allOutputs }];
+}
+
+/**
+ * Create a message with context from previous step
+ */
+export function withContext(prompt: string, previousResults: StepResult[]): PipelineMessage[] {
+  const lastOutput = previousResults[previousResults.length - 1]?.output;
+
+  if (typeof lastOutput === 'string') {
+    return [
+      {
+        role: 'user',
+        content: `${prompt}\n\nContext:\n${lastOutput}`,
+      },
+    ];
+  }
+
+  return [{ role: 'user', content: prompt }];
+}
+
+/**
+ * Create analysis prompt from research output
+ */
+export function analyzeResearch(previousResults: StepResult[]): PipelineMessage[] {
+  const research = previousResults[0]?.output as string;
+  return [
+    {
+      role: 'user',
+      content: `Research findings:\n\n${research}\n\nProvide your analysis.`,
+    },
+  ];
+}
+
+/**
+ * Create synthesis prompt from research and analysis
+ */
+export function synthesizeResults(previousResults: StepResult[]): PipelineMessage[] {
+  const research = previousResults[0]?.output as string;
+  const analysis = previousResults[1]?.output as string;
+
+  return [
+    {
+      role: 'user',
+      content: `Research:\n${research}\n\nAnalysis:\n${analysis}\n\nProvide your synthesis.`,
+    },
+  ];
+}
+
+/**
+ * Create refinement prompt
+ */
+export function refine(instruction: string, previousResults: StepResult[]): PipelineMessage[] {
+  const lastOutput = previousResults[previousResults.length - 1]?.output;
+
+  if (typeof lastOutput === 'string') {
+    return [
+      {
+        role: 'user',
+        content: `Previous version:\n\n${lastOutput}\n\n${instruction}`,
+      },
+    ];
+  }
+
+  return [{ role: 'user', content: instruction }];
+}