@od-oneapp/ai-platform 0.1.0

package/src/pipelines/adapters/trace-storage-blob.ts

@@ -0,0 +1,458 @@
+/**
+ * @fileoverview Blob Trace Storage Adapter
+ * Blob Trace Storage Adapter
+ * Uses Vercel Blob for artifacts + Upstash Redis for metadata
+ */
+
+import { createVercelBlobClient } from '../../integrations/blob-storage';
+import { createUpstashRedis } from '../../integrations/redis';
+import { generateId } from '../../shared';
+
+import type { BlobStorageClient } from '../../integrations/blob-storage';
+import type { RedisClientExtended } from '../../integrations/redis';
+import type { Artifact, StorageConfig, TraceStorage } from '../trace-storage-interface';
+import type { PipelineConfig, PipelineRun, ResumptionPoint, StepResult } from '../types';
+
+const DEFAULT_MAX_ARTIFACT_BYTES = 5 * 1024 * 1024;
+
+const stringifyArtifact = (value: unknown): string => {
+  const seen = new WeakSet<object>();
+
+  return JSON.stringify(value, (key, current) => {
+    void key;
+    if (typeof current === 'bigint') {
+      return current.toString();
+    }
+
+    if (typeof current === 'object' && current !== null) {
+      if (seen.has(current)) {
+        throw new Error('Artifact content contains a circular reference');
+      }
+      seen.add(current);
+    }
+
+    return current;
+  });
+};
+
+const serializeArtifactContent = (artifact: Artifact, maxBytes: number): Buffer => {
+  try {
+    const { content: artifactContent } = artifact;
+    let content: Buffer;
+    if (typeof artifactContent === 'string') {
+      content = Buffer.from(artifactContent);
+    } else if (Buffer.isBuffer(artifactContent)) {
+      content = artifactContent;
+    } else {
+      content = Buffer.from(stringifyArtifact(artifactContent));
+    }
+
+    if (content.byteLength > maxBytes) {
+      throw new Error(`Artifact content exceeds max size (${maxBytes} bytes)`);
+    }
+
+    return content;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : 'Unknown error';
+    throw new Error(`Failed to serialize artifact content: ${message}`);
+  }
+};
+
+/**
+ * Blob storage adapter
+ * - Metadata: Upstash Redis (fast queries)
+ * - Artifacts: Vercel Blob (large files)
+ */
+export class BlobTraceStorage implements TraceStorage {
+  private redis: RedisClientExtended;
+  private blob: BlobStorageClient;
+  private cacheTTL: number;
+  private artifactRetentionDays: number;
+  private artifactMaxBytes: number;
+  private resumptionTTL: number;
+
+  constructor(config: StorageConfig) {
+    if (!config.redis) {
+      throw new Error('Redis configuration required for BlobTraceStorage');
+    }
+    if (!config.blob) {
+      throw new Error('Blob configuration required for BlobTraceStorage');
+    }
+
+    this.redis = createUpstashRedis({
+      url: config.redis.url,
+      token: config.redis.token,
+    });
+
+    this.blob = createVercelBlobClient({ token: config.blob.token, baseUrl: config.blob.baseUrl });
+    this.cacheTTL = config.cacheTTL ?? 3600; // 1 hour
+    this.artifactRetentionDays = config.artifactRetentionDays ?? 90;
+    this.artifactMaxBytes = config.artifactMaxBytes ?? DEFAULT_MAX_ARTIFACT_BYTES;
+    this.resumptionTTL = config.resumptionTTL || 3600; // 1 hour
+  }
+
+  async initialize(): Promise<void> {
+    // Test Redis connection
+    try {
+      await this.redis.ping();
+    } catch (error) {
+      throw new Error(
+        `Failed to connect to Redis: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      );
+    }
+  }
+
+  async createRun(config: PipelineConfig): Promise<PipelineRun> {
+    const runId = `${config.id}-${generateId()}`;
+
+    const run: PipelineRun = {
+      runId,
+      pipelineId: config.id,
+      actorId: config.actorId,
+      status: 'running',
+      steps: [],
+      currentStepIndex: 0,
+      startedAt: new Date(),
+      totalPHIDetections: 0,
+      totalTokens: 0,
+      complianceMetrics: {
+        phiRedacted: 0,
+        abacChecks: 0,
+        auditEvents: 0,
+        approvalRequests: 0,
+      },
+      metadata: config.metadata,
+    };
+
+    // Store in Redis
+    await this.redis.set(`run:${runId}`, JSON.stringify(run), {
+      ex: this.cacheTTL,
+    });
+
+    // Add to actor index
+    await this.redis.zadd(`actor:${config.actorId}:runs`, {
+      score: Date.now(),
+      member: runId,
+    });
+
+    // Add to pipeline index
+    await this.redis.zadd(`pipeline:${config.id}:runs`, {
+      score: Date.now(),
+      member: runId,
+    });
+
+    // Add to global index for cross-actor/pipeline queries
+    await this.redis.zadd('global:runs', {
+      score: Date.now(),
+      member: runId,
+    });
+
+    return run;
+  }
+
+  async getRun(runId: string): Promise<PipelineRun | null> {
+    const data = await this.redis.get(`run:${runId}`);
+    if (!data) {
+      return null;
+    }
+
+    const run = JSON.parse(data) as PipelineRun;
+
+    // Convert date strings back to Date objects
+    run.startedAt = new Date(run.startedAt);
+    if (run.completedAt) {
+      run.completedAt = new Date(run.completedAt);
+    }
+
+    return run;
+  }
+
+  async updateRun(
+    runId: string,
+    updates: Partial<
+      Pick<
+        PipelineRun,
+        | 'status'
+        | 'currentStepIndex'
+        | 'completedAt'
+        | 'totalDurationMs'
+        | 'error'
+        | 'resumeToken'
+        | 'complianceMetrics'
+        | 'totalPHIDetections'
+        | 'totalTokens'
+      >
+    >,
+  ): Promise<void> {
+    const run = await this.getRun(runId);
+    if (!run) {
+      throw new Error(`Run ${runId} not found`);
+    }
+
+    const updated = { ...run, ...updates };
+
+    await this.redis.set(`run:${runId}`, JSON.stringify(updated), {
+      ex: this.cacheTTL,
+    });
+  }
+
+  async addStepResult(runId: string, result: StepResult): Promise<void> {
+    // Store step result
+    await this.redis.set(`run:${runId}:step:${result.stepId}`, JSON.stringify(result), {
+      ex: this.cacheTTL,
+    });
+
+    // Add to step index
+    await this.redis.zadd(`run:${runId}:steps`, {
+      score: result.stepIndex,
+      member: result.stepId,
+    });
+  }
+
+  async getStepResults(runId: string): Promise<StepResult[]> {
+    // Get step IDs in order
+    const stepIds = await this.redis.zrange<string[]>(`run:${runId}:steps`, 0, -1);
+
+    // Fetch all steps in parallel
+    const results = await Promise.all(
+      stepIds.map(async (stepId: string) => {
+        const data = await this.redis.get(`run:${runId}:step:${stepId}`);
+        if (!data) {
+          return null;
+        }
+        const step = JSON.parse(data) as StepResult;
+        // Convert dates
+        step.startedAt = new Date(step.startedAt);
+        if (step.completedAt) {
+          step.completedAt = new Date(step.completedAt);
+        }
+        return step;
+      }),
+    );
+
+    return results.filter((r): r is StepResult => r !== null);
+  }
+
+  async storeArtifact(runId: string, stepId: string, artifact: Artifact): Promise<string> {
+    const pathname = `pipelines/${runId}/${stepId}/${artifact.type}`;
+
+    const content = serializeArtifactContent(artifact, this.artifactMaxBytes);
+
+    // Upload to blob storage
+    const result = await this.blob.put(pathname, content, {
+      access: 'public',
+      contentType: artifact.contentType,
+    });
+
+    return result.url;
+  }
+
+  async getArtifact(url: string): Promise<Buffer | string | object> {
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch artifact: ${response.statusText}`);
+    }
+
+    const contentType = response.headers.get('content-type') ?? '';
+
+    if (contentType.includes('application/json')) {
+      return (await response.json()) as object;
+    } else if (contentType.includes('text/')) {
+      return await response.text();
+    } else {
+      return Buffer.from(await response.arrayBuffer());
+    }
+  }
+
+  async createResumptionPoint(
+    runId: string,
+    stepIndex: number,
+    previousResults: StepResult[],
+  ): Promise<ResumptionPoint> {
+    const token = generateId();
+    const expiresAt = new Date(Date.now() + this.resumptionTTL * 1000);
+
+    const resumption: ResumptionPoint = {
+      runId,
+      stepIndex,
+      previousResults,
+      token,
+      expiresAt,
+    };
+
+    // Store in Redis with TTL
+    await this.redis.set(`resumption:${token}`, JSON.stringify(resumption), {
+      ex: this.resumptionTTL,
+    });
+
+    return resumption;
+  }
+
+  async getResumptionPoint(token: string): Promise<ResumptionPoint | null> {
+    const data = await this.redis.get(`resumption:${token}`);
+    if (!data) {
+      return null;
+    }
+
+    const resumption = JSON.parse(data) as ResumptionPoint;
+
+    // Convert dates
+    resumption.expiresAt = new Date(resumption.expiresAt);
+    resumption.previousResults = resumption.previousResults.map(r => ({
+      ...r,
+      startedAt: new Date(r.startedAt),
+      completedAt: r.completedAt ? new Date(r.completedAt) : undefined,
+    }));
+
+    return resumption;
+  }
+
+  async cleanupExpiredResumptionPoints(): Promise<number> {
+    // Redis automatically expires keys with TTL
+    // This is a no-op for this implementation
+    return 0;
+  }
+
+  async queryRunsByActor(
+    actorId: string,
+    options?: {
+      limit?: number;
+      offset?: number;
+      status?: PipelineRun['status'];
+    },
+  ): Promise<PipelineRun[]> {
+    const limit = options?.limit ?? 10;
+    const offset = options?.offset || 0;
+
+    // Get run IDs from sorted set (most recent first)
+    const runIds = await this.redis.zrange<string[]>(
+      `actor:${actorId}:runs`,
+      offset,
+      offset + limit - 1,
+      {
+        rev: true,
+      },
+    );
+
+    // Fetch runs in parallel
+    const runs = await Promise.all(runIds.map(id => this.getRun(id)));
+
+    // Filter by status if provided
+    let filtered = runs.filter((r): r is PipelineRun => r !== null);
+    if (options?.status) {
+      filtered = filtered.filter(r => r.status === options.status);
+    }
+
+    return filtered;
+  }
+
+  async queryRunsByPipeline(
+    pipelineId: string,
+    options?: {
+      limit?: number;
+      offset?: number;
+      status?: PipelineRun['status'];
+    },
+  ): Promise<PipelineRun[]> {
+    const limit = options?.limit ?? 10;
+    const offset = options?.offset || 0;
+
+    // Get run IDs from sorted set (most recent first)
+    const runIds = await this.redis.zrange<string[]>(
+      `pipeline:${pipelineId}:runs`,
+      offset,
+      offset + limit - 1,
+      {
+        rev: true,
+      },
+    );
+
+    // Fetch runs in parallel
+    const runs = await Promise.all(runIds.map(id => this.getRun(id)));
+
+    // Filter by status if provided
+    let filtered = runs.filter((r): r is PipelineRun => r !== null);
+    if (options?.status) {
+      filtered = filtered.filter(r => r.status === options.status);
+    }
+
+    return filtered;
+  }
+
+  async getComplianceMetrics(
+    startDate: Date,
+    endDate: Date,
+    options?: {
+      actorId?: string;
+      pipelineId?: string;
+    },
+  ): Promise<{
+    totalRuns: number;
+    totalPHIDetections: number;
+    totalAuditEvents: number;
+    totalApprovals: number;
+    phiByType: Record<string, number>;
+    runsByStatus: Record<PipelineRun['status'], number>;
+  }> {
+    // Get runs in date range
+    let runIds: string[];
+    if (options?.actorId) {
+      runIds = await this.redis.zrange<string[]>(
+        `actor:${options.actorId}:runs`,
+        startDate.getTime(),
+        endDate.getTime(),
+        { byScore: true },
+      );
+    } else if (options?.pipelineId) {
+      runIds = await this.redis.zrange<string[]>(
+        `pipeline:${options.pipelineId}:runs`,
+        startDate.getTime(),
+        endDate.getTime(),
+        { byScore: true },
+      );
+    } else {
+      // Global metrics: query all runs by date range
+      runIds = await this.redis.zrange<string[]>(
+        'global:runs',
+        startDate.getTime(),
+        endDate.getTime(),
+        {
+          byScore: true,
+        },
+      );
+    }
+
+    // Fetch all runs
+    const runs = await Promise.all(runIds.map(id => this.getRun(id)));
+
+    const validRuns = runs.filter((r): r is PipelineRun => r !== null);
+
+    // Aggregate metrics
+    const metrics = {
+      totalRuns: validRuns.length,
+      totalPHIDetections: validRuns.reduce((sum, r) => sum + r.totalPHIDetections, 0),
+      totalAuditEvents: validRuns.reduce((sum, r) => sum + r.complianceMetrics.auditEvents, 0),
+      totalApprovals: validRuns.reduce((sum, r) => sum + r.complianceMetrics.approvalRequests, 0),
+      phiByType: {} as Record<string, number>,
+      runsByStatus: {} as Record<PipelineRun['status'], number>,
+    };
+
+    // Count by status
+    for (const run of validRuns) {
+      metrics.runsByStatus[run.status] = (metrics.runsByStatus[run.status] || 0) + 1;
+    }
+
+    return metrics;
+  }
+
+  async close(): Promise<void> {
+    // Redis client doesn't need explicit closing
+  }
+}
+
+/**
+ * Create blob storage adapter
+ */
+export function createBlobTraceStorage(config: StorageConfig): BlobTraceStorage {
+  return new BlobTraceStorage(config);
+}