@tenex-chat/backend 0.9.1

package/src/conversations/search/embeddings/ConversationIndexingJob.ts

@@ -0,0 +1,320 @@
+/**
+ * ConversationIndexingJob - Automatic batch indexing service
+ *
+ * This service runs periodically (every 5 minutes) to automatically index
+ * conversations that need embedding updates. It tracks which conversations
+ * have been indexed and ensures new/updated conversations are processed
+ * without requiring manual agent intervention.
+ *
+ * Key features:
+ * - Runs every 5 minutes as a background job
+ * - Indexes conversations across all projects
+ * - Tracks indexing state durably to avoid redundant work
+ * - Re-indexes when conversation metadata changes
+ * - Graceful error handling - failures don't break the service
+ * - Prevents overlapping batches
+ * - Decoupled from ConversationRegistry for multi-project support
+ */
+
+import { logger } from "@/utils/logger";
+import { getTenexBasePath } from "@/constants";
+import { join } from "path";
+import { getConversationEmbeddingService } from "./ConversationEmbeddingService";
+import type { BuildDocumentResult } from "./ConversationEmbeddingService";
+import { IndexingStateManager } from "./IndexingStateManager";
+import { listProjectIdsFromDisk, listConversationIdsFromDiskForProject } from "@/conversations/ConversationDiskReader";
+import { RAGService, type RAGDocument } from "@/services/rag/RAGService";
+
+/** Default interval: 5 minutes (in milliseconds) */
+const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
+
+/**
+ * Automatic conversation indexing job service
+ */
+export class ConversationIndexingJob {
+    private static instance: ConversationIndexingJob | null = null;
+    private timer: NodeJS.Timeout | null = null;
+    private isRunning = false;
+    private isBatchRunning = false; // Prevent overlapping batches
+    private intervalMs: number;
+    private projectsBasePath: string;
+    private stateManager: IndexingStateManager;
+
+    private constructor(intervalMs: number = DEFAULT_INTERVAL_MS) {
+        this.intervalMs = intervalMs;
+        // Use stable projects root from config, not mutable registry
+        this.projectsBasePath = join(getTenexBasePath(), "projects");
+        this.stateManager = new IndexingStateManager(this.projectsBasePath);
+    }
+
+    /**
+     * Get singleton instance
+     */
+    public static getInstance(intervalMs?: number): ConversationIndexingJob {
+        if (!ConversationIndexingJob.instance) {
+            ConversationIndexingJob.instance = new ConversationIndexingJob(intervalMs);
+        }
+        return ConversationIndexingJob.instance;
+    }
+
+    /**
+     * Start the periodic indexing job
+     */
+    public start(): void {
+        if (this.isRunning) {
+            logger.warn("ConversationIndexingJob is already running");
+            return;
+        }
+
+        this.isRunning = true;
+        logger.info("Starting ConversationIndexingJob", {
+            intervalMs: this.intervalMs,
+            intervalMinutes: this.intervalMs / 60000,
+            projectsBasePath: this.projectsBasePath,
+        });
+
+        // Run immediately on start
+        this.scheduleNextBatch(0);
+    }
+
+    /**
+     * Schedule the next batch run (self-scheduling to prevent overlaps)
+     */
+    private scheduleNextBatch(delayMs: number): void {
+        if (!this.isRunning) return;
+
+        this.timer = setTimeout(async () => {
+            try {
+                await this.runIndexingBatch();
+            } catch (error) {
+                logger.error("Indexing batch failed", { error });
+            } finally {
+                // Schedule next run only after this one completes
+                this.scheduleNextBatch(this.intervalMs);
+            }
+        }, delayMs);
+    }
+
+    /**
+     * Stop the periodic indexing job
+     */
+    public stop(): void {
+        if (!this.isRunning) {
+            logger.warn("ConversationIndexingJob is not running");
+            return;
+        }
+
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+
+        this.isRunning = false;
+
+        // Save state before stopping
+        this.stateManager.dispose();
+
+        logger.info("ConversationIndexingJob stopped");
+    }
+
+    /**
+     * Run a single indexing batch.
+     *
+     * Collects all conversations needing re-indexing across all projects,
+     * builds RAGDocuments for each, then flushes them via bulkUpsert
+     * (mergeInsert). This creates one LanceDB version per chunk of
+     * BATCH_SIZE instead of 2N versions (delete+insert per conversation).
+     *
+     * Failures are isolated per chunk: documents in a successful chunk are
+     * marked indexed even when other chunks fail. Only truly failed
+     * documents are retried next cycle.
+     */
+    private async runIndexingBatch(): Promise<void> {
+        // Prevent overlapping batches
+        if (this.isBatchRunning) {
+            logger.warn("Skipping indexing batch - previous batch still running");
+            return;
+        }
+
+        this.isBatchRunning = true;
+        logger.debug("Running conversation indexing batch");
+
+        const conversationEmbeddingService = getConversationEmbeddingService();
+
+        try {
+            // Ensure the embedding service is initialized
+            await conversationEmbeddingService.initialize();
+
+            let totalChecked = 0;
+            let totalSkipped = 0;
+            let totalFailed = 0;
+
+            // Collect all documents that need indexing across all projects
+            const pendingDocuments: RAGDocument[] = [];
+            // Track which conversations were successfully built (for marking indexed after flush)
+            // Indices into pendingDocuments correspond 1:1 with pendingMarkIndexed
+            const pendingMarkIndexed: Array<{ projectId: string; conversationId: string }> = [];
+            const pendingMarkNoContent: Array<{ projectId: string; conversationId: string }> = [];
+
+            // Get all projects directly from disk
+            const projectIds = listProjectIdsFromDisk(this.projectsBasePath);
+
+            for (const projectId of projectIds) {
+                try {
+                    const conversationIds = listConversationIdsFromDiskForProject(
+                        this.projectsBasePath,
+                        projectId
+                    );
+
+                    for (const conversationId of conversationIds) {
+                        totalChecked++;
+
+                        // Check if conversation needs indexing using durable state
+                        const needsIndexing = this.stateManager.needsIndexing(
+                            this.projectsBasePath,
+                            projectId,
+                            conversationId
+                        );
+
+                        if (!needsIndexing) {
+                            totalSkipped++;
+                            continue;
+                        }
+
+                        // Build document without writing
+                        const result: BuildDocumentResult = conversationEmbeddingService.buildDocument(
+                            conversationId,
+                            projectId
+                        );
+
+                        switch (result.kind) {
+                            case "ok":
+                                pendingDocuments.push(result.document);
+                                pendingMarkIndexed.push({ projectId, conversationId });
+                                break;
+                            case "noContent":
+                                pendingMarkNoContent.push({ projectId, conversationId });
+                                break;
+                            case "error":
+                                // Transient error — leave unmarked so it retries next cycle
+                                totalFailed++;
+                                logger.warn("Transient error building document, will retry", {
+                                    conversationId: conversationId.substring(0, 8),
+                                    projectId,
+                                    reason: result.reason,
+                                });
+                                break;
+                        }
+                    }
+                } catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    logger.error("Failed to process project conversations", {
+                        projectId,
+                        error: message,
+                    });
+                }
+            }
+
+            // Flush all pending documents via bulkUpsert (per-chunk failure isolation)
+            let totalIndexed = 0;
+            if (pendingDocuments.length > 0) {
+                const ragService = RAGService.getInstance();
+                const collectionName = conversationEmbeddingService.getCollectionName();
+                const { upsertedCount, failedIndices } = await ragService.bulkUpsert(
+                    collectionName,
+                    pendingDocuments
+                );
+
+                totalIndexed = upsertedCount;
+
+                // Build a set of failed indices for O(1) lookup
+                const failedSet = new Set(failedIndices);
+                totalFailed += failedIndices.length;
+
+                // Mark only successfully flushed conversations as indexed
+                for (let i = 0; i < pendingMarkIndexed.length; i++) {
+                    if (failedSet.has(i)) {
+                        // This document's chunk failed — leave unmarked for retry
+                        continue;
+                    }
+                    const { projectId, conversationId } = pendingMarkIndexed[i];
+                    this.stateManager.markIndexed(
+                        this.projectsBasePath,
+                        projectId,
+                        conversationId
+                    );
+                }
+            }
+
+            // Mark no-content conversations to avoid re-trying every batch
+            for (const { projectId, conversationId } of pendingMarkNoContent) {
+                this.stateManager.markIndexed(
+                    this.projectsBasePath,
+                    projectId,
+                    conversationId,
+                    true
+                );
+            }
+
+            if (totalChecked > 0) {
+                logger.info("Conversation indexing batch complete", {
+                    projectsProcessed: projectIds.length,
+                    conversationsChecked: totalChecked,
+                    newlyIndexed: totalIndexed,
+                    skipped: totalSkipped,
+                    failed: totalFailed,
+                });
+            } else {
+                logger.debug("No conversations to index in this batch");
+            }
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            logger.error("Conversation indexing batch failed", { error: message });
+            // Don't throw - we want the job to keep running even if one batch fails
+        } finally {
+            this.isBatchRunning = false;
+        }
+    }
+
+    /**
+     * Force a full re-index of all conversations
+     * This clears the tracking state and re-indexes everything
+     */
+    public async forceFullReindex(): Promise<void> {
+        logger.info("Forcing full conversation re-index");
+        this.stateManager.clearAllState();
+        await this.runIndexingBatch();
+    }
+
+    /**
+     * Get current job status
+     */
+    public getStatus(): {
+        isRunning: boolean;
+        isBatchRunning: boolean;
+        intervalMs: number;
+        stateStats: ReturnType<IndexingStateManager["getStats"]>;
+    } {
+        return {
+            isRunning: this.isRunning,
+            isBatchRunning: this.isBatchRunning,
+            intervalMs: this.intervalMs,
+            stateStats: this.stateManager.getStats(),
+        };
+    }
+
+    /**
+     * Reset singleton instance (for testing)
+     */
+    public static resetInstance(): void {
+        if (ConversationIndexingJob.instance) {
+            ConversationIndexingJob.instance.stop();
+            ConversationIndexingJob.instance = null;
+        }
+    }
+}
+
+// Export lazy getter to avoid eagerly initializing at module load time
+export function getConversationIndexingJob(): ConversationIndexingJob {
+    return ConversationIndexingJob.getInstance();
+}

package/src/conversations/search/embeddings/IndexingStateManager.ts

@@ -0,0 +1,338 @@
+/**
+ * IndexingStateManager - Durable per-conversation indexing state
+ *
+ * This manager tracks the indexing state of conversations to determine when
+ * they need to be re-indexed. It uses a hash-based approach to detect changes
+ * in conversation metadata (title, summary, last_user_message, lastActivity).
+ *
+ * Key features:
+ * - Durable state: persisted to disk in a KV store
+ * - Detects metadata changes via hash comparison
+ * - Bounded memory: only loads state for active projects
+ * - No unbounded growth: old entries naturally expire
+ */
+
+import { createHash } from "crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { logger } from "@/utils/logger";
+import { readLightweightMetadata } from "@/conversations/ConversationDiskReader";
+
+/**
+ * State for a single conversation
+ */
+interface ConversationIndexState {
+    /** Hash of indexed metadata */
+    metadataHash: string;
+    /** Timestamp of last indexing */
+    lastIndexedAt: number;
+    /** Last known activity timestamp */
+    lastActivity?: number;
+    /** Whether this conversation has no indexable content */
+    noContent?: boolean;
+}
+
+/**
+ * State file structure
+ */
+interface StateFile {
+    version: number;
+    states: Record<string, ConversationIndexState>; // Key: "projectId:conversationId"
+}
+
+/**
+ * Manager for conversation indexing state with bounded storage
+ * Eviction strategy: FIFO based on lastIndexedAt timestamp
+ */
+export class IndexingStateManager {
+    private static readonly STATE_VERSION = 1;
+    private static readonly MAX_ENTRIES = 10000; // Prevent unbounded growth
+    private static readonly EVICTION_BATCH_SIZE = 1000; // Remove oldest 10% when full
+
+    private stateFilePath: string;
+    private states: Map<string, ConversationIndexState> = new Map();
+    private dirty = false;
+    private saveTimer: NodeJS.Timeout | null = null;
+
+    constructor(baseDir: string) {
+        // Store state file in the projects directory
+        this.stateFilePath = join(baseDir, "indexing-state.json");
+        this.loadState();
+    }
+
+    /**
+     * Load state from disk
+     */
+    private loadState(): void {
+        try {
+            if (!existsSync(this.stateFilePath)) {
+                logger.debug("No indexing state file found, starting fresh");
+                return;
+            }
+
+            const content = readFileSync(this.stateFilePath, "utf-8");
+            const stateFile: StateFile = JSON.parse(content);
+
+            if (stateFile.version !== IndexingStateManager.STATE_VERSION) {
+                logger.warn("Indexing state version mismatch, resetting state");
+                return;
+            }
+
+            // Load into map
+            for (const [key, value] of Object.entries(stateFile.states)) {
+                this.states.set(key, value);
+            }
+
+            logger.info(`Loaded indexing state: ${this.states.size} entries`);
+        } catch (error) {
+            logger.error("Failed to load indexing state, starting fresh", { error });
+            this.states.clear();
+        }
+    }
+
+    /**
+     * Save state to disk (debounced)
+     */
+    private scheduleSave(): void {
+        if (!this.dirty) {
+            this.dirty = true;
+        }
+
+        // Clear existing timer
+        if (this.saveTimer) {
+            clearTimeout(this.saveTimer);
+        }
+
+        // Schedule save in 5 seconds
+        this.saveTimer = setTimeout(() => {
+            this.saveNow();
+        }, 5000);
+    }
+
+    /**
+     * Immediately save state to disk
+     */
+    public saveNow(): void {
+        if (!this.dirty) return;
+
+        try {
+            // Ensure directory exists
+            const dir = join(this.stateFilePath, "..");
+            if (!existsSync(dir)) {
+                mkdirSync(dir, { recursive: true });
+            }
+
+            // Evict old entries if needed
+            if (this.states.size > IndexingStateManager.MAX_ENTRIES) {
+                this.evictOldEntries();
+            }
+
+            // Convert map to object
+            const statesObj: Record<string, ConversationIndexState> = {};
+            for (const [key, value] of this.states.entries()) {
+                statesObj[key] = value;
+            }
+
+            const stateFile: StateFile = {
+                version: IndexingStateManager.STATE_VERSION,
+                states: statesObj,
+            };
+
+            writeFileSync(this.stateFilePath, JSON.stringify(stateFile, null, 2), "utf-8");
+            this.dirty = false;
+
+            logger.debug(`Saved indexing state: ${this.states.size} entries`);
+        } catch (error) {
+            logger.error("Failed to save indexing state", { error });
+        }
+    }
+
+    /**
+     * Evict oldest entries when reaching max size
+     * Uses FIFO eviction based on lastIndexedAt timestamp (oldest indexed first)
+     */
+    private evictOldEntries(): void {
+        // Sort by lastIndexedAt (oldest first)
+        const entries = Array.from(this.states.entries()).sort(
+            (a, b) => a[1].lastIndexedAt - b[1].lastIndexedAt
+        );
+
+        // Remove oldest batch (FIFO)
+        const toRemove = entries.slice(0, IndexingStateManager.EVICTION_BATCH_SIZE);
+        for (const [key] of toRemove) {
+            this.states.delete(key);
+        }
+
+        logger.info(
+            `Evicted ${toRemove.length} old indexing state entries via FIFO (max: ${IndexingStateManager.MAX_ENTRIES})`
+        );
+    }
+
+    /**
+     * Build a composite key for state lookup
+     */
+    private buildKey(projectId: string, conversationId: string): string {
+        return `${projectId}:${conversationId}`;
+    }
+
+    /**
+     * Calculate metadata hash for a conversation
+     * Hash includes: title, summary, lastUserMessage, lastActivity
+     * Returns both hash and lastActivity to avoid double reads
+     */
+    private calculateMetadataHash(
+        basePath: string,
+        projectId: string,
+        conversationId: string
+    ): { hash: string; lastActivity: number } | null {
+        try {
+            const metadata = readLightweightMetadata(basePath, projectId, conversationId);
+            if (!metadata) return null;
+
+            // Build a stable string representation
+            const parts: string[] = [
+                metadata.title || "",
+                metadata.summary || "",
+                metadata.lastUserMessage || "",
+                String(metadata.lastActivity || 0),
+            ];
+
+            const hashInput = parts.join("|");
+            const hash = createHash("sha256").update(hashInput).digest("hex");
+
+            return {
+                hash,
+                lastActivity: metadata.lastActivity || 0,
+            };
+        } catch (error) {
+            logger.debug(`Failed to calculate metadata hash for ${conversationId.substring(0, 8)}`, {
+                error,
+            });
+            return null;
+        }
+    }
+
+    /**
+     * Check if a conversation needs indexing
+     * Returns true if:
+     * - Never indexed before
+     * - Metadata has changed (different hash)
+     * - Activity timestamp advanced (only if conversation had content before)
+     */
+    public needsIndexing(
+        basePath: string,
+        projectId: string,
+        conversationId: string
+    ): boolean {
+        const key = this.buildKey(projectId, conversationId);
+        const currentState = this.states.get(key);
+
+        // Calculate current metadata hash (includes lastActivity to avoid double read)
+        const result = this.calculateMetadataHash(basePath, projectId, conversationId);
+        if (!result) {
+            // Can't read metadata - skip
+            return false;
+        }
+
+        const { hash: currentHash, lastActivity: currentActivity } = result;
+
+        // Never indexed before
+        if (!currentState) {
+            return true;
+        }
+
+        // If previously marked as no-content, check if activity changed
+        // (which might mean new messages were added)
+        if (currentState.noContent) {
+            const lastActivity = currentState.lastActivity || 0;
+            return currentActivity > lastActivity;
+        }
+
+        // Metadata changed
+        if (currentState.metadataHash !== currentHash) {
+            return true;
+        }
+
+        // Check if activity advanced
+        const lastActivity = currentState.lastActivity || 0;
+        if (currentActivity > lastActivity) {
+            return true;
+        }
+
+        // Already indexed and unchanged
+        return false;
+    }
+
+    /**
+     * Mark a conversation as indexed
+     * @param noContent - Set to true if the conversation has no indexable content
+     */
+    public markIndexed(
+        basePath: string,
+        projectId: string,
+        conversationId: string,
+        noContent = false
+    ): void {
+        const key = this.buildKey(projectId, conversationId);
+        const result = this.calculateMetadataHash(basePath, projectId, conversationId);
+
+        if (!result) {
+            logger.debug(`Cannot mark ${conversationId.substring(0, 8)} as indexed - no metadata`);
+            return;
+        }
+
+        const { hash, lastActivity } = result;
+
+        this.states.set(key, {
+            metadataHash: hash,
+            lastIndexedAt: Date.now(),
+            lastActivity,
+            noContent,
+        });
+
+        this.scheduleSave();
+    }
+
+    /**
+     * Clear a conversation's indexing state (forces re-index)
+     */
+    public clearState(projectId: string, conversationId: string): void {
+        const key = this.buildKey(projectId, conversationId);
+        this.states.delete(key);
+        this.scheduleSave();
+    }
+
+    /**
+     * Clear all state (forces full re-index)
+     */
+    public clearAllState(): void {
+        this.states.clear();
+        this.scheduleSave();
+    }
+
+    /**
+     * Get statistics about the state
+     */
+    public getStats(): {
+        totalEntries: number;
+        maxEntries: number;
+        isDirty: boolean;
+    } {
+        return {
+            totalEntries: this.states.size,
+            maxEntries: IndexingStateManager.MAX_ENTRIES,
+            isDirty: this.dirty,
+        };
+    }
+
+    /**
+     * Cleanup and save before shutdown
+     */
+    public dispose(): void {
+        if (this.saveTimer) {
+            clearTimeout(this.saveTimer);
+            this.saveTimer = null;
+        }
+        this.saveNow();
+    }
+}

package/src/conversations/search/embeddings/index.ts

@@ -0,0 +1,18 @@
+/**
+ * Conversation Embeddings Module
+ *
+ * Provides semantic search capabilities for conversations using RAG.
+ */
+
+export {
+    ConversationEmbeddingService,
+    getConversationEmbeddingService,
+    type ConversationEmbeddingDocument,
+    type SemanticSearchResult,
+    type SemanticSearchOptions,
+} from "./ConversationEmbeddingService";
+
+export {
+    ConversationIndexingJob,
+    getConversationIndexingJob,
+} from "./ConversationIndexingJob";