claude-memory-layer 1.0.0

package/src/core/vector-outbox.ts

@@ -0,0 +1,295 @@
+/**
+ * Vector Outbox V2 - Transactional Outbox Pattern
+ * AXIOMMIND Principle 6: DuckDB → outbox → LanceDB unidirectional flow
+ */
+
+import { Database } from 'duckdb';
+import { randomUUID } from 'crypto';
+import type {
+  OutboxJob,
+  OutboxStatus,
+  OutboxItemKind,
+  VALID_OUTBOX_TRANSITIONS
+} from './types.js';
+
+export interface OutboxConfig {
+  embeddingVersion: string;
+  maxRetries: number;
+  stuckThresholdMs: number;
+  cleanupDays: number;
+}
+
+const DEFAULT_CONFIG: OutboxConfig = {
+  embeddingVersion: 'v1',
+  maxRetries: 3,
+  stuckThresholdMs: 5 * 60 * 1000,  // 5 minutes
+  cleanupDays: 7
+};
+
+export interface OutboxMetrics {
+  pendingCount: number;
+  processingCount: number;
+  doneCount: number;
+  failedCount: number;
+  oldestPendingAge: number | null;
+}
+
+export class VectorOutbox {
+  private config: OutboxConfig;
+
+  constructor(
+    private db: Database,
+    config?: Partial<OutboxConfig>
+  ) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Enqueue item for vectorization (idempotent)
+   */
+  async enqueue(
+    itemKind: OutboxItemKind,
+    itemId: string,
+    embeddingVersion?: string
+  ): Promise<string> {
+    const version = embeddingVersion ?? this.config.embeddingVersion;
+    const jobId = randomUUID();
+    const now = new Date().toISOString();
+
+    await this.db.run(
+      `INSERT INTO vector_outbox (
+        job_id, item_kind, item_id, embedding_version, status, retry_count, created_at, updated_at
+      ) VALUES (?, ?, ?, ?, 'pending', 0, ?, ?)
+      ON CONFLICT (item_kind, item_id, embedding_version) DO NOTHING`,
+      [jobId, itemKind, itemId, version, now, now]
+    );
+
+    return jobId;
+  }
+
+  /**
+   * Claim pending jobs for processing
+   */
+  async claimJobs(limit: number = 32): Promise<OutboxJob[]> {
+    const now = new Date().toISOString();
+
+    // Atomic claim using UPDATE RETURNING
+    const rows = await this.db.all<Array<Record<string, unknown>>>(
+      `UPDATE vector_outbox
+       SET status = 'processing', updated_at = ?
+       WHERE job_id IN (
+         SELECT job_id FROM vector_outbox
+         WHERE status = 'pending'
+         ORDER BY created_at ASC
+         LIMIT ?
+       )
+       RETURNING *`,
+      [now, limit]
+    );
+
+    return rows.map(row => this.rowToJob(row));
+  }
+
+  /**
+   * Mark job as done
+   */
+  async markDone(jobId: string): Promise<void> {
+    await this.db.run(
+      `UPDATE vector_outbox
+       SET status = 'done', updated_at = ?
+       WHERE job_id = ?`,
+      [new Date().toISOString(), jobId]
+    );
+  }
+
+  /**
+   * Mark job as failed
+   */
+  async markFailed(jobId: string, error: string): Promise<void> {
+    const now = new Date().toISOString();
+
+    // Check retry count
+    const rows = await this.db.all<Array<{ retry_count: number }>>(
+      `SELECT retry_count FROM vector_outbox WHERE job_id = ?`,
+      [jobId]
+    );
+
+    if (rows.length === 0) return;
+
+    const retryCount = rows[0].retry_count;
+    const newStatus: OutboxStatus = retryCount >= this.config.maxRetries - 1
+      ? 'failed'
+      : 'pending';  // Will retry
+
+    await this.db.run(
+      `UPDATE vector_outbox
+       SET status = ?, error = ?, retry_count = retry_count + 1, updated_at = ?
+       WHERE job_id = ?`,
+      [newStatus, error, now, jobId]
+    );
+  }
+
+  /**
+   * Get job by ID
+   */
+  async getJob(jobId: string): Promise<OutboxJob | null> {
+    const rows = await this.db.all<Array<Record<string, unknown>>>(
+      `SELECT * FROM vector_outbox WHERE job_id = ?`,
+      [jobId]
+    );
+
+    if (rows.length === 0) return null;
+    return this.rowToJob(rows[0]);
+  }
+
+  /**
+   * Get jobs by status
+   */
+  async getJobsByStatus(status: OutboxStatus, limit: number = 100): Promise<OutboxJob[]> {
+    const rows = await this.db.all<Array<Record<string, unknown>>>(
+      `SELECT * FROM vector_outbox
+       WHERE status = ?
+       ORDER BY created_at ASC
+       LIMIT ?`,
+      [status, limit]
+    );
+
+    return rows.map(row => this.rowToJob(row));
+  }
+
+  /**
+   * Reconcile: recover stuck and retry failed jobs
+   */
+  async reconcile(): Promise<{ recovered: number; retried: number }> {
+    const now = new Date();
+    const stuckThreshold = new Date(now.getTime() - this.config.stuckThresholdMs);
+
+    // Recover stuck processing jobs
+    const recoveredResult = await this.db.run(
+      `UPDATE vector_outbox
+       SET status = 'pending', updated_at = ?
+       WHERE status = 'processing'
+       AND updated_at < ?`,
+      [now.toISOString(), stuckThreshold.toISOString()]
+    );
+
+    // Retry failed jobs that haven't exceeded max retries
+    const retriedResult = await this.db.run(
+      `UPDATE vector_outbox
+       SET status = 'pending', updated_at = ?
+       WHERE status = 'failed'
+       AND retry_count < ?`,
+      [now.toISOString(), this.config.maxRetries]
+    );
+
+    // Get counts (DuckDB doesn't return affected rows easily)
+    const recoveredRows = await this.db.all<Array<{ count: number }>>(
+      `SELECT COUNT(*) as count FROM vector_outbox
+       WHERE status = 'pending' AND updated_at = ?`,
+      [now.toISOString()]
+    );
+
+    return {
+      recovered: 0,  // Approximate
+      retried: 0     // Approximate
+    };
+  }
+
+  /**
+   * Cleanup old done jobs
+   */
+  async cleanup(): Promise<number> {
+    const threshold = new Date();
+    threshold.setDate(threshold.getDate() - this.config.cleanupDays);
+
+    await this.db.run(
+      `DELETE FROM vector_outbox
+       WHERE status = 'done'
+       AND updated_at < ?`,
+      [threshold.toISOString()]
+    );
+
+    return 0;  // DuckDB doesn't return affected rows easily
+  }
+
+  /**
+   * Get metrics
+   */
+  async getMetrics(): Promise<OutboxMetrics> {
+    const statusCounts = await this.db.all<Array<{ status: string; count: number }>>(
+      `SELECT status, COUNT(*) as count
+       FROM vector_outbox
+       GROUP BY status`
+    );
+
+    const oldestPending = await this.db.all<Array<{ created_at: string }>>(
+      `SELECT created_at FROM vector_outbox
+       WHERE status = 'pending'
+       ORDER BY created_at ASC
+       LIMIT 1`
+    );
+
+    const metrics: OutboxMetrics = {
+      pendingCount: 0,
+      processingCount: 0,
+      doneCount: 0,
+      failedCount: 0,
+      oldestPendingAge: null
+    };
+
+    for (const row of statusCounts) {
+      switch (row.status) {
+        case 'pending':
+          metrics.pendingCount = Number(row.count);
+          break;
+        case 'processing':
+          metrics.processingCount = Number(row.count);
+          break;
+        case 'done':
+          metrics.doneCount = Number(row.count);
+          break;
+        case 'failed':
+          metrics.failedCount = Number(row.count);
+          break;
+      }
+    }
+
+    if (oldestPending.length > 0) {
+      const oldestDate = new Date(oldestPending[0].created_at);
+      metrics.oldestPendingAge = Date.now() - oldestDate.getTime();
+    }
+
+    return metrics;
+  }
+
+  /**
+   * Validate state transition
+   */
+  isValidTransition(from: OutboxStatus, to: OutboxStatus): boolean {
+    const validTransitions = [
+      { from: 'pending', to: 'processing' },
+      { from: 'processing', to: 'done' },
+      { from: 'processing', to: 'failed' },
+      { from: 'failed', to: 'pending' }
+    ];
+
+    return validTransitions.some(t => t.from === from && t.to === to);
+  }
+
+  /**
+   * Convert database row to OutboxJob
+   */
+  private rowToJob(row: Record<string, unknown>): OutboxJob {
+    return {
+      jobId: row.job_id as string,
+      itemKind: row.item_kind as OutboxItemKind,
+      itemId: row.item_id as string,
+      embeddingVersion: row.embedding_version as string,
+      status: row.status as OutboxStatus,
+      retryCount: row.retry_count as number,
+      error: row.error as string | undefined,
+      createdAt: new Date(row.created_at as string),
+      updatedAt: new Date(row.updated_at as string)
+    };
+  }
+}

package/src/core/vector-store.ts

@@ -0,0 +1,182 @@
+/**
+ * LanceDB Vector Store for semantic search
+ * AXIOMMIND Principle 6: Vector store consistency (DuckDB → outbox → LanceDB unidirectional)
+ */
+
+import * as lancedb from '@lancedb/lancedb';
+import type { VectorRecord } from './types.js';
+
+export interface SearchResult {
+  id: string;
+  eventId: string;
+  content: string;
+  score: number;
+  sessionId: string;
+  eventType: string;
+  timestamp: string;
+}
+
+export class VectorStore {
+  private db: lancedb.Connection | null = null;
+  private table: lancedb.Table | null = null;
+  private readonly tableName = 'conversations';
+
+  constructor(private dbPath: string) {}
+
+  /**
+   * Initialize LanceDB connection
+   */
+  async initialize(): Promise<void> {
+    if (this.db) return;
+
+    this.db = await lancedb.connect(this.dbPath);
+
+    // Try to open existing table
+    try {
+      const tables = await this.db.tableNames();
+      if (tables.includes(this.tableName)) {
+        this.table = await this.db.openTable(this.tableName);
+      }
+    } catch {
+      // Table doesn't exist yet, will be created on first insert
+      this.table = null;
+    }
+  }
+
+  /**
+   * Add or update vector record
+   */
+  async upsert(record: VectorRecord): Promise<void> {
+    await this.initialize();
+
+    if (!this.db) {
+      throw new Error('Database not initialized');
+    }
+
+    const data = {
+      id: record.id,
+      eventId: record.eventId,
+      sessionId: record.sessionId,
+      eventType: record.eventType,
+      content: record.content,
+      vector: record.vector,
+      timestamp: record.timestamp,
+      metadata: JSON.stringify(record.metadata || {})
+    };
+
+    if (!this.table) {
+      // Create table with first record
+      this.table = await this.db.createTable(this.tableName, [data]);
+    } else {
+      await this.table.add([data]);
+    }
+  }
+
+  /**
+   * Add multiple vector records in batch
+   */
+  async upsertBatch(records: VectorRecord[]): Promise<void> {
+    if (records.length === 0) return;
+
+    await this.initialize();
+
+    if (!this.db) {
+      throw new Error('Database not initialized');
+    }
+
+    const data = records.map(record => ({
+      id: record.id,
+      eventId: record.eventId,
+      sessionId: record.sessionId,
+      eventType: record.eventType,
+      content: record.content,
+      vector: record.vector,
+      timestamp: record.timestamp,
+      metadata: JSON.stringify(record.metadata || {})
+    }));
+
+    if (!this.table) {
+      this.table = await this.db.createTable(this.tableName, data);
+    } else {
+      await this.table.add(data);
+    }
+  }
+
+  /**
+   * Search for similar vectors
+   */
+  async search(
+    queryVector: number[],
+    options: {
+      limit?: number;
+      minScore?: number;
+      sessionId?: string;
+    } = {}
+  ): Promise<SearchResult[]> {
+    await this.initialize();
+
+    if (!this.table) {
+      return [];
+    }
+
+    const { limit = 5, minScore = 0.7, sessionId } = options;
+
+    let query = this.table.search(queryVector).limit(limit * 2); // Get more for filtering
+
+    // Apply session filter if specified
+    if (sessionId) {
+      query = query.where(`sessionId = '${sessionId}'`);
+    }
+
+    const results = await query.toArray();
+
+    return results
+      .filter(r => {
+        // Convert distance to score (assuming cosine distance)
+        const score = 1 - (r._distance || 0);
+        return score >= minScore;
+      })
+      .slice(0, limit)
+      .map(r => ({
+        id: r.id as string,
+        eventId: r.eventId as string,
+        content: r.content as string,
+        score: 1 - (r._distance || 0),
+        sessionId: r.sessionId as string,
+        eventType: r.eventType as string,
+        timestamp: r.timestamp as string
+      }));
+  }
+
+  /**
+   * Delete vector by event ID
+   */
+  async delete(eventId: string): Promise<void> {
+    if (!this.table) return;
+    await this.table.delete(`eventId = '${eventId}'`);
+  }
+
+  /**
+   * Get total count of vectors
+   */
+  async count(): Promise<number> {
+    if (!this.table) return 0;
+    const result = await this.table.countRows();
+    return result;
+  }
+
+  /**
+   * Check if vector exists for event
+   */
+  async exists(eventId: string): Promise<boolean> {
+    if (!this.table) return false;
+
+    const results = await this.table
+      .search([])
+      .where(`eventId = '${eventId}'`)
+      .limit(1)
+      .toArray();
+
+    return results.length > 0;
+  }
+}