@od-oneapp/ai-platform 0.1.0

package/src/pipelines/trace-storage-interface.ts

@@ -0,0 +1,216 @@
+/**
+ * @fileoverview Trace Storage Interface
+ * Trace Storage Interface
+ * Abstract interface for pluggable pipeline storage backends
+ */
+
+import type { PipelineConfig, PipelineRun, ResumptionPoint, StepResult } from './types';
+
+/**
+ * Artifact to store in blob storage
+ */
+export interface Artifact {
+  /** Artifact type */
+  type: 'input' | 'output' | 'redacted' | 'metadata';
+  /** Content to store */
+  content: string | Buffer | object;
+  /** Content type (MIME type) */
+  contentType: string;
+  /** Custom metadata */
+  metadata?: Record<string, string>;
+}
+
+/**
+ * Abstract interface for trace storage
+ *
+ * Implementations:
+ * - BlobTraceStorage (Vercel Blob + Upstash Redis)
+ * - PostgresTraceStorage (Neon Postgres + Vercel Blob)
+ */
+export interface TraceStorage {
+  /**
+   * Initialize storage (setup connections, tables, etc.)
+   */
+  initialize(): Promise<void>;
+
+  /**
+   * Create a new pipeline run
+   */
+  createRun(config: PipelineConfig): Promise<PipelineRun>;
+
+  /**
+   * Get pipeline run by ID
+   */
+  getRun(runId: string): Promise<PipelineRun | null>;
+
+  /**
+   * Update pipeline run status
+   */
+  updateRun(
+    runId: string,
+    updates: Partial<
+      Pick<
+        PipelineRun,
+        | 'status'
+        | 'currentStepIndex'
+        | 'completedAt'
+        | 'totalDurationMs'
+        | 'error'
+        | 'resumeToken'
+        | 'complianceMetrics'
+        | 'totalPHIDetections'
+        | 'totalTokens'
+      >
+    >,
+  ): Promise<void>;
+
+  /**
+   * Add step result to run
+   */
+  addStepResult(runId: string, result: StepResult): Promise<void>;
+
+  /**
+   * Get step results for a run
+   */
+  getStepResults(runId: string): Promise<StepResult[]>;
+
+  /**
+   * Store artifact in blob storage
+   * Returns URL to the stored artifact
+   */
+  storeArtifact(runId: string, stepId: string, artifact: Artifact): Promise<string>;
+
+  /**
+   * Retrieve artifact from blob storage
+   */
+  getArtifact(url: string): Promise<Buffer | string | object>;
+
+  /**
+   * Create resumption point
+   */
+  createResumptionPoint(
+    runId: string,
+    stepIndex: number,
+    previousResults: StepResult[],
+  ): Promise<ResumptionPoint>;
+
+  /**
+   * Get resumption point by token
+   */
+  getResumptionPoint(token: string): Promise<ResumptionPoint | null>;
+
+  /**
+   * Delete expired resumption points
+   */
+  cleanupExpiredResumptionPoints(): Promise<number>;
+
+  /**
+   * Query runs by actor
+   */
+  queryRunsByActor(
+    actorId: string,
+    options?: {
+      limit?: number;
+      offset?: number;
+      status?: PipelineRun['status'];
+    },
+  ): Promise<PipelineRun[]>;
+
+  /**
+   * Query runs by pipeline ID
+   */
+  queryRunsByPipeline(
+    pipelineId: string,
+    options?: {
+      limit?: number;
+      offset?: number;
+      status?: PipelineRun['status'];
+    },
+  ): Promise<PipelineRun[]>;
+
+  /**
+   * Get compliance metrics for a time range
+   */
+  getComplianceMetrics(
+    startDate: Date,
+    endDate: Date,
+    options?: {
+      actorId?: string;
+      pipelineId?: string;
+    },
+  ): Promise<{
+    totalRuns: number;
+    totalPHIDetections: number;
+    totalAuditEvents: number;
+    totalApprovals: number;
+    phiByType: Record<string, number>;
+    runsByStatus: Record<PipelineRun['status'], number>;
+  }>;
+
+  /**
+   * Close storage connections
+   */
+  close(): Promise<void>;
+}
+
+/**
+ * Storage adapter configuration
+ */
+export interface StorageConfig {
+  /** Storage type */
+  type: 'blob' | 'postgres';
+
+  /** Vercel Blob configuration (for both adapters) */
+  blob?: {
+    token: string;
+    baseUrl?: string;
+  };
+
+  /** Upstash Redis configuration (for blob adapter) */
+  redis?: {
+    url: string;
+    token: string;
+  };
+
+  /** Postgres configuration (for postgres adapter) */
+  postgres?: {
+    connectionString: string;
+    schema?: string;
+  };
+
+  /** Cache TTL in seconds */
+  cacheTTL?: number;
+
+  /** Artifact retention in days */
+  artifactRetentionDays?: number;
+
+  /** Maximum artifact size in bytes (default: 5 MB) */
+  artifactMaxBytes?: number;
+
+  /** Resumption point TTL in seconds (default: 1 hour) */
+  resumptionTTL?: number;
+}
+
+/**
+ * Helper to create storage adapter
+ *
+ * For ESM compatibility, import adapters directly:
+ * ```typescript
+ * import { createBlobTraceStorage } from '#/lib/ai-onedigital/adapters/trace-storage-blob';
+ * const storage = createBlobTraceStorage(config);
+ * ```
+ */
+export function createTraceStorage(config: StorageConfig): TraceStorage {
+  switch (config.type) {
+    case 'blob':
+      throw new Error(
+        'Use createBlobTraceStorage() directly from #/lib/ai-onedigital/adapters/trace-storage-blob',
+      );
+    case 'postgres':
+      throw new Error(
+        'PostgresTraceStorage not yet implemented. See adapters/trace-storage-postgres/README.md',
+      );
+    default:
+      throw new Error(`Unknown storage type: ${config.type}. Use 'blob' or 'postgres'`);
+  }
+}

package/src/pipelines/types.ts

@@ -0,0 +1,255 @@
+/**
+ * @fileoverview Multi-Step Pipeline Types
+ * Multi-Step Pipeline Types
+ * Type definitions for compliant AI pipelines with trace storage
+ */
+
+import type { ComplianceOptions, PHIDetection } from '../governance/compliance/types';
+import type { CoreMessage, ModelRef, SDKToolSet } from '../shared';
+
+/**
+ * Message type for pipeline steps.
+ * Aligned with SDK's CoreMessage type.
+ */
+type PipelineMessage = CoreMessage;
+
+/**
+ * Pipeline step status
+ */
+export type StepStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
+
+/**
+ * Pipeline step result
+ */
+export interface StepResult {
+  /** Step identifier */
+  stepId: string;
+  /** Step name */
+  stepName: string;
+  /** Step index in pipeline */
+  stepIndex: number;
+  /** Step status */
+  status: StepStatus;
+  /** Step output (text, tool result, etc.) */
+  output: unknown;
+  /** PHI detections from this step */
+  phiDetections: PHIDetection[];
+  /** Start timestamp */
+  startedAt: Date;
+  /** End timestamp */
+  completedAt?: Date;
+  /** Duration in milliseconds */
+  durationMs?: number;
+  /** Error if failed */
+  error?: {
+    message: string;
+    code?: string;
+    stack?: string;
+  };
+  /** Model usage statistics */
+  usage?: {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+  };
+  /** Artifact URLs (stored in blob storage) */
+  artifacts?: {
+    input?: string; // URL to input artifact
+    output?: string; // URL to output artifact
+    redacted?: string; // URL to redacted version
+  };
+  /** Custom metadata */
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Pipeline step configuration
+ */
+export interface PipelineStep {
+  /** Unique step identifier */
+  id: string;
+  /** Human-readable step name */
+  name: string;
+  /** Step description */
+  description?: string;
+  /** Model to use for this step */
+  model: ModelRef; // AI SDK model object or string
+  /** System prompt for this step */
+  systemPrompt?: string;
+  /** Temperature (0-1) */
+  temperature?: number;
+  /** Max tokens to generate */
+  maxTokens?: number;
+  /** Whether this step streams output */
+  streaming?: boolean;
+  /** Tools available for this step */
+  tools?: SDKToolSet;
+  /** Whether to skip this step based on previous results */
+  skipIf?: (previousResults: StepResult[]) => boolean | Promise<boolean>;
+  /** Transform input from previous step */
+  transformInput?: (
+    previousResults: StepResult[],
+  ) => PipelineMessage[] | Promise<PipelineMessage[]>;
+  /** Transform output before storing */
+  transformOutput?: (output: unknown) => unknown | Promise<unknown>;
+  /** Custom compliance options for this step */
+  compliance?: Partial<ComplianceOptions>;
+  /** Timeout in milliseconds */
+  timeoutMs?: number;
+  /** Retry configuration */
+  retry?: {
+    maxAttempts: number;
+    delayMs: number;
+    backoff?: 'linear' | 'exponential';
+  };
+}
+
+/**
+ * Pipeline configuration
+ */
+export interface PipelineConfig {
+  /** Unique pipeline identifier */
+  id: string;
+  /** Pipeline name */
+  name: string;
+  /** Pipeline description */
+  description?: string;
+  /** Pipeline steps */
+  steps: PipelineStep[];
+  /** Global compliance options (can be overridden per step) */
+  compliance: ComplianceOptions;
+  /** Actor executing the pipeline */
+  actorId: string;
+  /** Initial input messages */
+  initialMessages: PipelineMessage[];
+  /** Storage backend for traces and artifacts */
+  storage: 'blob' | 'postgres' | 'custom';
+  /** Custom storage implementation (if storage='custom') */
+  customStorage?: unknown; // TraceStorage interface
+  /** Whether to enable stream resumption */
+  enableResumption?: boolean;
+  /** Resume from existing pipeline run */
+  resumeFromRunId?: string;
+  /** Pipeline-level metadata */
+  metadata?: Record<string, unknown>;
+  /** Maximum parallel steps (default: 1 for sequential) */
+  maxParallel?: number;
+  /** Global timeout for entire pipeline (milliseconds) */
+  timeoutMs?: number;
+  /** Callback hooks */
+  hooks?: {
+    onStepStart?: (step: PipelineStep, runId: string) => void | Promise<void>;
+    onStepComplete?: (result: StepResult, runId: string) => void | Promise<void>;
+    onStepError?: (step: PipelineStep, error: Error, runId: string) => void | Promise<void>;
+    onPipelineComplete?: (results: StepResult[], runId: string) => void | Promise<void>;
+    onPipelineError?: (error: Error, runId: string) => void | Promise<void>;
+  };
+}
+
+/**
+ * Pipeline run metadata
+ */
+export interface PipelineRun {
+  /** Unique run identifier */
+  runId: string;
+  /** Pipeline configuration ID */
+  pipelineId: string;
+  /** Actor who initiated the run */
+  actorId: string;
+  /** Run status */
+  status: 'pending' | 'running' | 'completed' | 'failed' | 'paused';
+  /** Step results */
+  steps: StepResult[];
+  /** Current step index */
+  currentStepIndex: number;
+  /** Start timestamp */
+  startedAt: Date;
+  /** Completion timestamp */
+  completedAt?: Date;
+  /** Total duration in milliseconds */
+  totalDurationMs?: number;
+  /** Total PHI detections across all steps */
+  totalPHIDetections: number;
+  /** Total tokens used */
+  totalTokens: number;
+  /** Aggregated compliance metrics */
+  complianceMetrics: {
+    phiRedacted: number;
+    abacChecks: number;
+    auditEvents: number;
+    approvalRequests: number;
+  };
+  /** Error if pipeline failed */
+  error?: {
+    message: string;
+    code?: string;
+    stack?: string;
+    failedStep?: string;
+  };
+  /** Resume token for stream resumption */
+  resumeToken?: string;
+  /** Custom metadata */
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Pipeline execution result
+ */
+export interface PipelineResult {
+  /** Run metadata */
+  run: PipelineRun;
+  /** Final output (from last successful step) */
+  output: unknown;
+  /** All step results */
+  steps: StepResult[];
+  /** Whether pipeline completed successfully */
+  success: boolean;
+  /** Aggregated compliance summary */
+  compliance: {
+    totalPHIRedacted: number;
+    phiByType: Record<string, number>;
+    totalAuditEvents: number;
+    policyViolations: number;
+  };
+}
+
+/**
+ * Stream chunk for real-time updates
+ */
+export interface PipelineStreamChunk {
+  /** Chunk type */
+  type:
+    | 'step_start'
+    | 'step_progress'
+    | 'step_complete'
+    | 'step_error'
+    | 'pipeline_complete'
+    | 'pipeline_error'
+    | 'phi_detection'
+    | 'abac_check'
+    | 'approval_request';
+  /** Run ID */
+  runId: string;
+  /** Step ID (if applicable) */
+  stepId?: string;
+  /** Timestamp */
+  timestamp: Date;
+  /** Chunk data */
+  data: unknown;
+}
+
+/**
+ * Resumption point for paused pipelines
+ */
+export interface ResumptionPoint {
+  /** Run ID to resume */
+  runId: string;
+  /** Step index to resume from */
+  stepIndex: number;
+  /** Accumulated results up to this point */
+  previousResults: StepResult[];
+  /** Resume token */
+  token: string;
+  /** Expiry timestamp */
+  expiresAt: Date;
+}