claude-memory-layer 1.0.0

package/src/core/vector-worker.ts

@@ -0,0 +1,488 @@
+/**
+ * Vector Worker - Single-Writer Pattern Implementation
+ * AXIOMMIND Principle 6: DuckDB → outbox → LanceDB unidirectional flow
+ */
+
+import { EventStore } from './event-store.js';
+import { VectorStore } from './vector-store.js';
+import { Embedder } from './embedder.js';
+import type { OutboxItem, VectorRecord } from './types.js';
+
+export interface WorkerConfig {
+  batchSize: number;
+  pollIntervalMs: number;
+  maxRetries: number;
+}
+
+const DEFAULT_CONFIG: WorkerConfig = {
+  batchSize: 32,
+  pollIntervalMs: 1000,
+  maxRetries: 3
+};
+
+export class VectorWorker {
+  private readonly eventStore: EventStore;
+  private readonly vectorStore: VectorStore;
+  private readonly embedder: Embedder;
+  private readonly config: WorkerConfig;
+  private running = false;
+  private pollTimeout: NodeJS.Timeout | null = null;
+
+  constructor(
+    eventStore: EventStore,
+    vectorStore: VectorStore,
+    embedder: Embedder,
+    config: Partial<WorkerConfig> = {}
+  ) {
+    this.eventStore = eventStore;
+    this.vectorStore = vectorStore;
+    this.embedder = embedder;
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Start the worker polling loop
+   */
+  start(): void {
+    if (this.running) return;
+    this.running = true;
+    this.poll();
+  }
+
+  /**
+   * Stop the worker
+   */
+  stop(): void {
+    this.running = false;
+    if (this.pollTimeout) {
+      clearTimeout(this.pollTimeout);
+      this.pollTimeout = null;
+    }
+  }
+
+  /**
+   * Process a single batch of outbox items
+   */
+  async processBatch(): Promise<number> {
+    const items = await this.eventStore.getPendingOutboxItems(this.config.batchSize);
+
+    if (items.length === 0) {
+      return 0;
+    }
+
+    const successful: string[] = [];
+    const failed: string[] = [];
+
+    try {
+      // Generate embeddings for all items
+      const embeddings = await this.embedder.embedBatch(items.map(i => i.content));
+
+      // Prepare vector records
+      const records: VectorRecord[] = [];
+
+      for (let i = 0; i < items.length; i++) {
+        const item = items[i];
+        const embedding = embeddings[i];
+
+        // Get event details
+        const event = await this.eventStore.getEvent(item.eventId);
+        if (!event) {
+          failed.push(item.id);
+          continue;
+        }
+
+        records.push({
+          id: `vec_${item.id}`,
+          eventId: item.eventId,
+          sessionId: event.sessionId,
+          eventType: event.eventType,
+          content: item.content,
+          vector: embedding.vector,
+          timestamp: event.timestamp.toISOString(),
+          metadata: event.metadata
+        });
+
+        successful.push(item.id);
+      }
+
+      // Batch insert to vector store
+      if (records.length > 0) {
+        await this.vectorStore.upsertBatch(records);
+      }
+
+      // Mark successful items as done
+      if (successful.length > 0) {
+        await this.eventStore.completeOutboxItems(successful);
+      }
+
+      // Mark failed items
+      if (failed.length > 0) {
+        await this.eventStore.failOutboxItems(failed, 'Event not found');
+      }
+
+      return successful.length;
+    } catch (error) {
+      // Mark all items as failed
+      const allIds = items.map(i => i.id);
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      await this.eventStore.failOutboxItems(allIds, errorMessage);
+      throw error;
+    }
+  }
+
+  /**
+   * Poll for new items
+   */
+  private async poll(): Promise<void> {
+    if (!this.running) return;
+
+    try {
+      await this.processBatch();
+    } catch (error) {
+      console.error('Vector worker error:', error);
+    }
+
+    // Schedule next poll
+    this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+  }
+
+  /**
+   * Process all pending items (blocking)
+   */
+  async processAll(): Promise<number> {
+    let totalProcessed = 0;
+    let processed: number;
+
+    do {
+      processed = await this.processBatch();
+      totalProcessed += processed;
+    } while (processed > 0);
+
+    return totalProcessed;
+  }
+
+  /**
+   * Check if worker is running
+   */
+  isRunning(): boolean {
+    return this.running;
+  }
+}
+
+/**
+ * Create and start a vector worker
+ */
+export function createVectorWorker(
+  eventStore: EventStore,
+  vectorStore: VectorStore,
+  embedder: Embedder,
+  config?: Partial<WorkerConfig>
+): VectorWorker {
+  const worker = new VectorWorker(eventStore, vectorStore, embedder, config);
+  return worker;
+}
+
+// ============================================================
+// Vector Worker V2 - Extended for Task Entity System
+// ============================================================
+
+import { Database } from 'duckdb';
+import { VectorOutbox } from './vector-outbox.js';
+import type { OutboxJob, OutboxItemKind } from './types.js';
+
+export interface WorkerConfigV2 {
+  batchSize: number;
+  pollIntervalMs: number;
+  maxRetries: number;
+  embeddingVersion: string;
+}
+
+const DEFAULT_CONFIG_V2: WorkerConfigV2 = {
+  batchSize: 32,
+  pollIntervalMs: 1000,
+  maxRetries: 3,
+  embeddingVersion: 'v1'
+};
+
+/**
+ * Content provider interface for different item kinds
+ */
+export interface ContentProvider {
+  getContent(itemKind: OutboxItemKind, itemId: string): Promise<{
+    content: string;
+    metadata: Record<string, unknown>;
+  } | null>;
+}
+
+/**
+ * Default content provider using database
+ */
+export class DefaultContentProvider implements ContentProvider {
+  constructor(private db: Database) {}
+
+  async getContent(itemKind: OutboxItemKind, itemId: string): Promise<{
+    content: string;
+    metadata: Record<string, unknown>;
+  } | null> {
+    switch (itemKind) {
+      case 'entry':
+        return this.getEntryContent(itemId);
+      case 'task_title':
+        return this.getTaskTitleContent(itemId);
+      case 'event':
+        return this.getEventContent(itemId);
+      default:
+        return null;
+    }
+  }
+
+  private async getEntryContent(entryId: string): Promise<{
+    content: string;
+    metadata: Record<string, unknown>;
+  } | null> {
+    const rows = await this.db.all<Array<Record<string, unknown>>>(
+      `SELECT title, content_json, entry_type FROM entries WHERE entry_id = ?`,
+      [entryId]
+    );
+
+    if (rows.length === 0) return null;
+
+    const row = rows[0];
+    const contentJson = typeof row.content_json === 'string'
+      ? JSON.parse(row.content_json)
+      : row.content_json;
+
+    return {
+      content: `${row.title}\n${JSON.stringify(contentJson)}`,
+      metadata: {
+        itemKind: 'entry',
+        entryType: row.entry_type
+      }
+    };
+  }
+
+  private async getTaskTitleContent(taskId: string): Promise<{
+    content: string;
+    metadata: Record<string, unknown>;
+  } | null> {
+    const rows = await this.db.all<Array<Record<string, unknown>>>(
+      `SELECT title, search_text, current_json FROM entities
+       WHERE entity_id = ? AND entity_type = 'task'`,
+      [taskId]
+    );
+
+    if (rows.length === 0) return null;
+
+    const row = rows[0];
+    return {
+      content: row.search_text as string || row.title as string,
+      metadata: {
+        itemKind: 'task_title',
+        entityType: 'task'
+      }
+    };
+  }
+
+  private async getEventContent(eventId: string): Promise<{
+    content: string;
+    metadata: Record<string, unknown>;
+  } | null> {
+    const rows = await this.db.all<Array<Record<string, unknown>>>(
+      `SELECT content, event_type, session_id FROM events WHERE id = ?`,
+      [eventId]
+    );
+
+    if (rows.length === 0) return null;
+
+    const row = rows[0];
+    return {
+      content: row.content as string,
+      metadata: {
+        itemKind: 'event',
+        eventType: row.event_type,
+        sessionId: row.session_id
+      }
+    };
+  }
+}
+
+/**
+ * Vector Worker V2 - Supports multiple item kinds
+ */
+export class VectorWorkerV2 {
+  private readonly outbox: VectorOutbox;
+  private readonly vectorStore: VectorStore;
+  private readonly embedder: Embedder;
+  private readonly contentProvider: ContentProvider;
+  private readonly config: WorkerConfigV2;
+  private running = false;
+  private pollTimeout: NodeJS.Timeout | null = null;
+
+  constructor(
+    db: Database,
+    vectorStore: VectorStore,
+    embedder: Embedder,
+    config: Partial<WorkerConfigV2> = {},
+    contentProvider?: ContentProvider
+  ) {
+    this.outbox = new VectorOutbox(db, {
+      embeddingVersion: config.embeddingVersion ?? DEFAULT_CONFIG_V2.embeddingVersion,
+      maxRetries: config.maxRetries ?? DEFAULT_CONFIG_V2.maxRetries
+    });
+    this.vectorStore = vectorStore;
+    this.embedder = embedder;
+    this.config = { ...DEFAULT_CONFIG_V2, ...config };
+    this.contentProvider = contentProvider ?? new DefaultContentProvider(db);
+  }
+
+  /**
+   * Start the worker polling loop
+   */
+  start(): void {
+    if (this.running) return;
+    this.running = true;
+    this.poll();
+  }
+
+  /**
+   * Stop the worker
+   */
+  stop(): void {
+    this.running = false;
+    if (this.pollTimeout) {
+      clearTimeout(this.pollTimeout);
+      this.pollTimeout = null;
+    }
+  }
+
+  /**
+   * Process a single batch of outbox jobs
+   */
+  async processBatch(): Promise<number> {
+    const jobs = await this.outbox.claimJobs(this.config.batchSize);
+
+    if (jobs.length === 0) {
+      return 0;
+    }
+
+    let successCount = 0;
+
+    for (const job of jobs) {
+      try {
+        await this.processJob(job);
+        await this.outbox.markDone(job.jobId);
+        successCount++;
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        await this.outbox.markFailed(job.jobId, errorMessage);
+      }
+    }
+
+    return successCount;
+  }
+
+  /**
+   * Process a single job
+   */
+  private async processJob(job: OutboxJob): Promise<void> {
+    // Get content
+    const contentData = await this.contentProvider.getContent(job.itemKind, job.itemId);
+
+    if (!contentData) {
+      // Item not found, mark as done (skip)
+      return;
+    }
+
+    // Generate embedding
+    const embedding = await this.embedder.embed(contentData.content);
+
+    // Upsert to vector store
+    const record: VectorRecord = {
+      id: `${job.itemKind}_${job.itemId}_${job.embeddingVersion}`,
+      eventId: job.itemKind === 'event' ? job.itemId : '',
+      sessionId: (contentData.metadata.sessionId as string) ?? '',
+      eventType: (contentData.metadata.eventType as string) ?? job.itemKind,
+      content: contentData.content,
+      vector: embedding.vector,
+      timestamp: new Date().toISOString(),
+      metadata: {
+        ...contentData.metadata,
+        embeddingVersion: job.embeddingVersion
+      }
+    };
+
+    // Use idempotent upsert (delete + add)
+    await this.vectorStore.upsertBatch([record]);
+  }
+
+  /**
+   * Poll for new jobs
+   */
+  private async poll(): Promise<void> {
+    if (!this.running) return;
+
+    try {
+      await this.processBatch();
+    } catch (error) {
+      console.error('Vector worker V2 error:', error);
+    }
+
+    // Schedule next poll
+    this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+  }
+
+  /**
+   * Process all pending jobs (blocking)
+   */
+  async processAll(): Promise<number> {
+    let totalProcessed = 0;
+    let processed: number;
+
+    do {
+      processed = await this.processBatch();
+      totalProcessed += processed;
+    } while (processed > 0);
+
+    return totalProcessed;
+  }
+
+  /**
+   * Run reconciliation
+   */
+  async reconcile(): Promise<{ recovered: number; retried: number }> {
+    return this.outbox.reconcile();
+  }
+
+  /**
+   * Get metrics
+   */
+  async getMetrics() {
+    return this.outbox.getMetrics();
+  }
+
+  /**
+   * Check if worker is running
+   */
+  isRunning(): boolean {
+    return this.running;
+  }
+
+  /**
+   * Get the outbox instance for direct access
+   */
+  getOutbox(): VectorOutbox {
+    return this.outbox;
+  }
+}
+
+/**
+ * Create a Vector Worker V2 instance
+ */
+export function createVectorWorkerV2(
+  db: Database,
+  vectorStore: VectorStore,
+  embedder: Embedder,
+  config?: Partial<WorkerConfigV2>
+): VectorWorkerV2 {
+  return new VectorWorkerV2(db, vectorStore, embedder, config);
+}

package/src/core/working-set-store.ts

@@ -0,0 +1,244 @@
+/**
+ * Working Set Store
+ * Manages the active memory window for Endless Mode
+ * Biomimetic: Simulates human working memory (7±2 items, 15-30s duration)
+ */
+
+import { randomUUID } from 'crypto';
+import { Database } from 'duckdb';
+import type {
+  MemoryEvent,
+  EndlessModeConfig,
+  WorkingSet,
+  WorkingSetItem
+} from './types.js';
+import { EventStore } from './event-store.js';
+
+export class WorkingSetStore {
+  constructor(
+    private eventStore: EventStore,
+    private config: EndlessModeConfig
+  ) {}
+
+  private get db(): Database {
+    return this.eventStore.getDatabase();
+  }
+
+  /**
+   * Add an event to the working set
+   */
+  async add(eventId: string, relevanceScore: number = 1.0, topics?: string[]): Promise<void> {
+    const expiresAt = new Date(
+      Date.now() + this.config.workingSet.timeWindowHours * 60 * 60 * 1000
+    );
+
+    await this.db.run(
+      `INSERT OR REPLACE INTO working_set (id, event_id, added_at, relevance_score, topics, expires_at)
+       VALUES (?, ?, CURRENT_TIMESTAMP, ?, ?, ?)`,
+      [
+        randomUUID(),
+        eventId,
+        relevanceScore,
+        JSON.stringify(topics || []),
+        expiresAt.toISOString()
+      ]
+    );
+
+    // Enforce size limit
+    await this.enforceLimit();
+  }
+
+  /**
+   * Get the current working set
+   */
+  async get(): Promise<WorkingSet> {
+    // Clean up expired items first
+    await this.cleanup();
+
+    // Get working set items with their events
+    const rows = await this.db.all<Array<Record<string, unknown>>>(
+      `SELECT ws.*, e.*
+       FROM working_set ws
+       JOIN events e ON ws.event_id = e.id
+       ORDER BY ws.relevance_score DESC, ws.added_at DESC
+       LIMIT ?`,
+      [this.config.workingSet.maxEvents]
+    );
+
+    const events: MemoryEvent[] = rows.map(row => ({
+      id: row.id as string,
+      eventType: row.event_type as 'user_prompt' | 'agent_response' | 'session_summary' | 'tool_observation',
+      sessionId: row.session_id as string,
+      timestamp: new Date(row.timestamp as string),
+      content: row.content as string,
+      canonicalKey: row.canonical_key as string,
+      dedupeKey: row.dedupe_key as string,
+      metadata: row.metadata ? JSON.parse(row.metadata as string) : undefined
+    }));
+
+    return {
+      recentEvents: events,
+      lastActivity: events.length > 0 ? events[0].timestamp : new Date(),
+      continuityScore: await this.calculateContinuityScore()
+    };
+  }
+
+  /**
+   * Get working set items (metadata only)
+   */
+  async getItems(): Promise<WorkingSetItem[]> {
+    const rows = await this.db.all<Array<Record<string, unknown>>>(
+      `SELECT * FROM working_set ORDER BY relevance_score DESC, added_at DESC`
+    );
+
+    return rows.map(row => ({
+      id: row.id as string,
+      eventId: row.event_id as string,
+      addedAt: new Date(row.added_at as string),
+      relevanceScore: row.relevance_score as number,
+      topics: row.topics ? JSON.parse(row.topics as string) : undefined,
+      expiresAt: new Date(row.expires_at as string)
+    }));
+  }
+
+  /**
+   * Update relevance score for an event
+   */
+  async updateRelevance(eventId: string, score: number): Promise<void> {
+    await this.db.run(
+      `UPDATE working_set SET relevance_score = ? WHERE event_id = ?`,
+      [score, eventId]
+    );
+  }
+
+  /**
+   * Prune specific events from working set (after consolidation)
+   */
+  async prune(eventIds: string[]): Promise<void> {
+    if (eventIds.length === 0) return;
+
+    const placeholders = eventIds.map(() => '?').join(',');
+    await this.db.run(
+      `DELETE FROM working_set WHERE event_id IN (${placeholders})`,
+      eventIds
+    );
+  }
+
+  /**
+   * Get the count of items in working set
+   */
+  async count(): Promise<number> {
+    const result = await this.db.all<Array<{ count: number }>>(
+      `SELECT COUNT(*) as count FROM working_set`
+    );
+    return result[0]?.count || 0;
+  }
+
+  /**
+   * Clear the entire working set
+   */
+  async clear(): Promise<void> {
+    await this.db.run(`DELETE FROM working_set`);
+  }
+
+  /**
+   * Check if an event is in the working set
+   */
+  async contains(eventId: string): Promise<boolean> {
+    const result = await this.db.all<Array<{ count: number }>>(
+      `SELECT COUNT(*) as count FROM working_set WHERE event_id = ?`,
+      [eventId]
+    );
+    return (result[0]?.count || 0) > 0;
+  }
+
+  /**
+   * Refresh expiration for an event (rehears al - keep relevant items longer)
+   */
+  async refresh(eventId: string): Promise<void> {
+    const newExpiresAt = new Date(
+      Date.now() + this.config.workingSet.timeWindowHours * 60 * 60 * 1000
+    );
+
+    await this.db.run(
+      `UPDATE working_set SET expires_at = ? WHERE event_id = ?`,
+      [newExpiresAt.toISOString(), eventId]
+    );
+  }
+
+  /**
+   * Clean up expired items
+   */
+  private async cleanup(): Promise<void> {
+    await this.db.run(
+      `DELETE FROM working_set WHERE expires_at < datetime('now')`
+    );
+  }
+
+  /**
+   * Enforce the maximum size limit
+   * Removes lowest relevance items when over limit
+   */
+  private async enforceLimit(): Promise<void> {
+    const maxEvents = this.config.workingSet.maxEvents;
+
+    // Get IDs to keep (highest relevance, most recent)
+    const keepIds = await this.db.all<Array<{ id: string }>>(
+      `SELECT id FROM working_set
+       ORDER BY relevance_score DESC, added_at DESC
+       LIMIT ?`,
+      [maxEvents]
+    );
+
+    if (keepIds.length === 0) return;
+
+    const keepIdList = keepIds.map(r => r.id);
+    const placeholders = keepIdList.map(() => '?').join(',');
+
+    // Delete everything not in the keep list
+    await this.db.run(
+      `DELETE FROM working_set WHERE id NOT IN (${placeholders})`,
+      keepIdList
+    );
+  }
+
+  /**
+   * Calculate continuity score based on recent context transitions
+   */
+  private async calculateContinuityScore(): Promise<number> {
+    const result = await this.db.all<Array<{ avg_score: number | null }>>(
+      `SELECT AVG(continuity_score) as avg_score
+       FROM continuity_log
+       WHERE created_at > datetime('now', '-1 hour')`
+    );
+
+    return result[0]?.avg_score ?? 0.5;
+  }
+
+  /**
+   * Get topics from current working set for context matching
+   */
+  async getActiveTopics(): Promise<string[]> {
+    const rows = await this.db.all<Array<{ topics: string }>>(
+      `SELECT topics FROM working_set WHERE topics IS NOT NULL`
+    );
+
+    const allTopics = new Set<string>();
+    for (const row of rows) {
+      const topics = JSON.parse(row.topics) as string[];
+      topics.forEach(t => allTopics.add(t));
+    }
+
+    return Array.from(allTopics);
+  }
+}
+
+/**
+ * Create a Working Set Store instance
+ */
+export function createWorkingSetStore(
+  eventStore: EventStore,
+  config: EndlessModeConfig
+): WorkingSetStore {
+  return new WorkingSetStore(eventStore, config);
+}