claude-memory-layer 1.0.31 → 1.0.32

package/src/core/vector-outbox.ts

@@ -1,299 +0,0 @@
-/**
- * Vector Outbox V2 - Transactional Outbox Pattern
- * AXIOMMIND Principle 6: DuckDB → outbox → LanceDB unidirectional flow
- */
-
-import { dbRun, dbAll, toDate, type Database } from './db-wrapper.js';
-import { randomUUID } from 'crypto';
-import type {
-  OutboxJob,
-  OutboxStatus,
-  OutboxItemKind
-} from './types.js';
-
-export interface OutboxConfig {
-  embeddingVersion: string;
-  maxRetries: number;
-  stuckThresholdMs: number;
-  cleanupDays: number;
-}
-
-const DEFAULT_CONFIG: OutboxConfig = {
-  embeddingVersion: 'v1',
-  maxRetries: 3,
-  stuckThresholdMs: 5 * 60 * 1000,  // 5 minutes
-  cleanupDays: 7
-};
-
-export interface OutboxMetrics {
-  pendingCount: number;
-  processingCount: number;
-  doneCount: number;
-  failedCount: number;
-  oldestPendingAge: number | null;
-}
-
-export class VectorOutbox {
-  private config: OutboxConfig;
-
-  constructor(
-    private db: Database,
-    config?: Partial<OutboxConfig>
-  ) {
-    this.config = { ...DEFAULT_CONFIG, ...config };
-  }
-
-  /**
-   * Enqueue item for vectorization (idempotent)
-   */
-  async enqueue(
-    itemKind: OutboxItemKind,
-    itemId: string,
-    embeddingVersion?: string
-  ): Promise<string> {
-    const version = embeddingVersion ?? this.config.embeddingVersion;
-    const jobId = randomUUID();
-    const now = new Date().toISOString();
-
-    await dbRun(
-      this.db,
-      `INSERT INTO vector_outbox (
-        job_id, item_kind, item_id, embedding_version, status, retry_count, created_at, updated_at
-      ) VALUES (?, ?, ?, ?, 'pending', 0, ?, ?)
-      ON CONFLICT (item_kind, item_id, embedding_version) DO NOTHING`,
-      [jobId, itemKind, itemId, version, now, now]
-    );
-
-    return jobId;
-  }
-
-  /**
-   * Claim pending jobs for processing
-   */
-  async claimJobs(limit: number = 32): Promise<OutboxJob[]> {
-    const now = new Date().toISOString();
-
-    // Atomic claim using UPDATE RETURNING
-    const rows = await dbAll<Record<string, unknown>>(
-      this.db,
-      `UPDATE vector_outbox
-       SET status = 'processing', updated_at = ?
-       WHERE job_id IN (
-         SELECT job_id FROM vector_outbox
-         WHERE status = 'pending'
-         ORDER BY created_at ASC
-         LIMIT ?
-       )
-       RETURNING *`,
-      [now, limit]
-    );
-
-    return rows.map(row => this.rowToJob(row));
-  }
-
-  /**
-   * Mark job as done
-   */
-  async markDone(jobId: string): Promise<void> {
-    await dbRun(
-      this.db,
-      `UPDATE vector_outbox
-       SET status = 'done', updated_at = ?
-       WHERE job_id = ?`,
-      [new Date().toISOString(), jobId]
-    );
-  }
-
-  /**
-   * Mark job as failed
-   */
-  async markFailed(jobId: string, error: string): Promise<void> {
-    const now = new Date().toISOString();
-
-    // Check retry count
-    const rows = await dbAll<{ retry_count: number }>(
-      this.db,
-      `SELECT retry_count FROM vector_outbox WHERE job_id = ?`,
-      [jobId]
-    );
-
-    if (rows.length === 0) return;
-
-    const retryCount = rows[0].retry_count;
-    const newStatus: OutboxStatus = retryCount >= this.config.maxRetries - 1
-      ? 'failed'
-      : 'pending';  // Will retry
-
-    await dbRun(
-      this.db,
-      `UPDATE vector_outbox
-       SET status = ?, error = ?, retry_count = retry_count + 1, updated_at = ?
-       WHERE job_id = ?`,
-      [newStatus, error, now, jobId]
-    );
-  }
-
-  /**
-   * Get job by ID
-   */
-  async getJob(jobId: string): Promise<OutboxJob | null> {
-    const rows = await dbAll<Record<string, unknown>>(
-      this.db,
-      `SELECT * FROM vector_outbox WHERE job_id = ?`,
-      [jobId]
-    );
-
-    if (rows.length === 0) return null;
-    return this.rowToJob(rows[0]);
-  }
-
-  /**
-   * Get jobs by status
-   */
-  async getJobsByStatus(status: OutboxStatus, limit: number = 100): Promise<OutboxJob[]> {
-    const rows = await dbAll<Record<string, unknown>>(
-      this.db,
-      `SELECT * FROM vector_outbox
-       WHERE status = ?
-       ORDER BY created_at ASC
-       LIMIT ?`,
-      [status, limit]
-    );
-
-    return rows.map(row => this.rowToJob(row));
-  }
-
-  /**
-   * Reconcile: recover stuck and retry failed jobs
-   */
-  async reconcile(): Promise<{ recovered: number; retried: number }> {
-    const now = new Date();
-    const stuckThreshold = new Date(now.getTime() - this.config.stuckThresholdMs);
-
-    // Recover stuck processing jobs
-    await dbRun(
-      this.db,
-      `UPDATE vector_outbox
-       SET status = 'pending', updated_at = ?
-       WHERE status = 'processing'
-       AND updated_at < ?`,
-      [now.toISOString(), stuckThreshold.toISOString()]
-    );
-
-    // Retry failed jobs that haven't exceeded max retries
-    await dbRun(
-      this.db,
-      `UPDATE vector_outbox
-       SET status = 'pending', updated_at = ?
-       WHERE status = 'failed'
-       AND retry_count < ?`,
-      [now.toISOString(), this.config.maxRetries]
-    );
-
-    return {
-      recovered: 0,  // Approximate
-      retried: 0     // Approximate
-    };
-  }
-
-  /**
-   * Cleanup old done jobs
-   */
-  async cleanup(): Promise<number> {
-    const threshold = new Date();
-    threshold.setDate(threshold.getDate() - this.config.cleanupDays);
-
-    await dbRun(
-      this.db,
-      `DELETE FROM vector_outbox
-       WHERE status = 'done'
-       AND updated_at < ?`,
-      [threshold.toISOString()]
-    );
-
-    return 0;  // DuckDB doesn't return affected rows easily
-  }
-
-  /**
-   * Get metrics
-   */
-  async getMetrics(): Promise<OutboxMetrics> {
-    const statusCounts = await dbAll<{ status: string; count: number }>(
-      this.db,
-      `SELECT status, COUNT(*) as count
-       FROM vector_outbox
-       GROUP BY status`
-    );
-
-    const oldestPending = await dbAll<{ created_at: string }>(
-      this.db,
-      `SELECT created_at FROM vector_outbox
-       WHERE status = 'pending'
-       ORDER BY created_at ASC
-       LIMIT 1`
-    );
-
-    const metrics: OutboxMetrics = {
-      pendingCount: 0,
-      processingCount: 0,
-      doneCount: 0,
-      failedCount: 0,
-      oldestPendingAge: null
-    };
-
-    for (const row of statusCounts) {
-      switch (row.status) {
-        case 'pending':
-          metrics.pendingCount = Number(row.count);
-          break;
-        case 'processing':
-          metrics.processingCount = Number(row.count);
-          break;
-        case 'done':
-          metrics.doneCount = Number(row.count);
-          break;
-        case 'failed':
-          metrics.failedCount = Number(row.count);
-          break;
-      }
-    }
-
-    if (oldestPending.length > 0) {
-      const oldestDate = new Date(oldestPending[0].created_at);
-      metrics.oldestPendingAge = Date.now() - oldestDate.getTime();
-    }
-
-    return metrics;
-  }
-
-  /**
-   * Validate state transition
-   */
-  isValidTransition(from: OutboxStatus, to: OutboxStatus): boolean {
-    const validTransitions = [
-      { from: 'pending', to: 'processing' },
-      { from: 'processing', to: 'done' },
-      { from: 'processing', to: 'failed' },
-      { from: 'failed', to: 'pending' }
-    ];
-
-    return validTransitions.some(t => t.from === from && t.to === to);
-  }
-
-  /**
-   * Convert database row to OutboxJob
-   */
-  private rowToJob(row: Record<string, unknown>): OutboxJob {
-    return {
-      jobId: row.job_id as string,
-      itemKind: row.item_kind as OutboxItemKind,
-      itemId: row.item_id as string,
-      embeddingVersion: row.embedding_version as string,
-      status: row.status as OutboxStatus,
-      retryCount: row.retry_count as number,
-      error: row.error as string | undefined,
-      createdAt: toDate(row.created_at),
-      updatedAt: toDate(row.updated_at)
-    };
-  }
-}

package/src/core/vector-store.ts

@@ -1,231 +0,0 @@
-/**
- * LanceDB Vector Store for semantic search
- * AXIOMMIND Principle 6: Vector store consistency (DuckDB → outbox → LanceDB unidirectional)
- */
-
-import * as lancedb from '@lancedb/lancedb';
-import type { VectorRecord } from './types.js';
-
-export interface SearchResult {
-  id: string;
-  eventId: string;
-  content: string;
-  score: number;
-  sessionId: string;
-  eventType: string;
-  timestamp: string;
-}
-
-export class VectorStore {
-  private db: lancedb.Connection | null = null;
-  private table: lancedb.Table | null = null;
-  private readonly tableName = 'conversations';
-
-  constructor(private dbPath: string) {}
-
-  /**
-   * Initialize LanceDB connection
-   */
-  async initialize(): Promise<void> {
-    if (this.db) return;
-
-    this.db = await lancedb.connect(this.dbPath);
-
-    // Try to open existing table
-    try {
-      const tables = await this.db.tableNames();
-      if (tables.includes(this.tableName)) {
-        this.table = await this.db.openTable(this.tableName);
-      }
-    } catch {
-      // Table doesn't exist yet, will be created on first insert
-      this.table = null;
-    }
-  }
-
-  /**
-   * Add or update vector record
-   */
-  async upsert(record: VectorRecord): Promise<void> {
-    await this.initialize();
-
-    if (!this.db) {
-      throw new Error('Database not initialized');
-    }
-
-    const data = {
-      id: record.id,
-      eventId: record.eventId,
-      sessionId: record.sessionId,
-      eventType: record.eventType,
-      content: record.content,
-      vector: record.vector,
-      timestamp: record.timestamp,
-      metadata: JSON.stringify(record.metadata || {})
-    };
-
-    if (!this.table) {
-      // Create table with first record (handle race condition)
-      try {
-        this.table = await this.db.createTable(this.tableName, [data]);
-      } catch (e: any) {
-        if (e?.message?.includes('already exists')) {
-          this.table = await this.db.openTable(this.tableName);
-          await this.table.add([data]);
-        } else {
-          throw e;
-        }
-      }
-    } else {
-      await this.table.add([data]);
-    }
-  }
-
-  /**
-   * Add multiple vector records in batch
-   */
-  async upsertBatch(records: VectorRecord[]): Promise<void> {
-    if (records.length === 0) return;
-
-    await this.initialize();
-
-    if (!this.db) {
-      throw new Error('Database not initialized');
-    }
-
-    const data = records.map(record => ({
-      id: record.id,
-      eventId: record.eventId,
-      sessionId: record.sessionId,
-      eventType: record.eventType,
-      content: record.content,
-      vector: record.vector,
-      timestamp: record.timestamp,
-      metadata: JSON.stringify(record.metadata || {})
-    }));
-
-    if (!this.table) {
-      try {
-        this.table = await this.db.createTable(this.tableName, data);
-      } catch (e: any) {
-        if (e?.message?.includes('already exists')) {
-          this.table = await this.db.openTable(this.tableName);
-          await this.table.add(data);
-        } else {
-          throw e;
-        }
-      }
-    } else {
-      await this.table.add(data);
-    }
-  }
-
-  /**
-   * Search for similar vectors
-   */
-  async search(
-    queryVector: number[],
-    options: {
-      limit?: number;
-      minScore?: number;
-      sessionId?: string;
-    } = {}
-  ): Promise<SearchResult[]> {
-    await this.initialize();
-
-    if (!this.table) {
-      return [];
-    }
-
-    const { limit = 5, minScore = 0.7, sessionId } = options;
-
-    // Use cosine distance for semantic similarity
-    let query = this.table
-      .search(queryVector)
-      .distanceType('cosine')
-      .limit(limit * 2); // Get more for filtering
-
-    // Apply session filter if specified
-    if (sessionId) {
-      query = query.where(`sessionId = '${sessionId}'`);
-    }
-
-    const results = await query.toArray();
-
-    return results
-      .filter(r => {
-        // Convert cosine distance to similarity score
-        // Cosine distance ranges from 0 (identical) to 2 (opposite)
-        // Score = 1 - (distance / 2) gives range [0, 1]
-        const distance = r._distance || 0;
-        const score = 1 - (distance / 2);
-        return score >= minScore;
-      })
-      .slice(0, limit)
-      .map(r => {
-        const distance = r._distance || 0;
-        const score = 1 - (distance / 2);
-        return {
-          id: r.id as string,
-          eventId: r.eventId as string,
-          content: r.content as string,
-          score,
-          sessionId: r.sessionId as string,
-          eventType: r.eventType as string,
-          timestamp: r.timestamp as string
-        };
-      });
-  }
-
-  /**
-   * Delete vector by event ID
-   */
-  async delete(eventId: string): Promise<void> {
-    if (!this.table) return;
-    await this.table.delete(`eventId = '${eventId}'`);
-  }
-
-  /**
-   * Get total count of vectors
-   */
-  async count(): Promise<number> {
-    if (!this.table) return 0;
-    const result = await this.table.countRows();
-    return result;
-  }
-
-  /**
-   * Clear all vectors (used for embedding model migration)
-   */
-  async clearAll(): Promise<void> {
-    await this.initialize();
-    if (!this.db) return;
-
-    try {
-      if (typeof (this.db as any).dropTable === 'function') {
-        await (this.db as any).dropTable(this.tableName);
-      } else if (typeof (this.db as any).drop_table === 'function') {
-        await (this.db as any).drop_table(this.tableName);
-      }
-    } catch {
-      // Ignore if table does not exist
-    }
-
-    this.table = null;
-  }
-
-  /**
-   * Check if vector exists for event
-   */
-  async exists(eventId: string): Promise<boolean> {
-    if (!this.table) return false;
-
-    const results = await this.table
-      .search([])
-      .where(`eventId = '${eventId}'`)
-      .limit(1)
-      .toArray();
-
-    return results.length > 0;
-  }
-}