@xfabric/memory 0.1.0 → 0.2.1

package/dist/memory-manager.js

@@ -1,15 +1,18 @@
 import { join, relative, resolve, dirname, basename } from "node:path";
-import { readFile, stat, readdir, copyFile, rename, unlink } from "node:fs/promises";
+import { readFile, stat, readdir, copyFile, rename, unlink, open as fsOpen } from "node:fs/promises";
 import { existsSync, mkdirSync } from "node:fs";
+import { SESSION_DIRTY_DEBOUNCE_MS, EMBEDDING_RETRY_MAX_ATTEMPTS, EMBEDDING_RETRY_BASE_DELAY_MS, EMBEDDING_RETRY_MAX_DELAY_MS, OPENAI_BATCH_ENDPOINT, } from "./types.js";
 import { hashText } from "./utils/hash.js";
 import { retry } from "./utils/retry.js";
 import { Semaphore } from "./utils/concurrency.js";
-import { createEmbeddingProvider, } from "./providers/index.js";
+import { isRetryableEmbeddingError, isBatchTimeoutError, isBatchUnavailableError, } from "./utils/error-detection.js";
+import { createEmbeddingProvider, runOpenAiEmbeddingBatches, runGeminiEmbeddingBatches, BatchFailureTracker, } from "./providers/index.js";
 import { MemoryStorage } from "./storage/sqlite.js";
 import { loadSqliteVec } from "./storage/sqlite-vec.js";
 import { chunkMarkdown } from "./chunking/markdown.js";
 import { chunkSession } from "./chunking/session.js";
 import { FileWatcher } from "./sync/watcher.js";
+import { onSessionTranscriptUpdate } from "./sync/session-events.js";
 import { searchVector, searchVectorWithSqliteVec } from "./search/vector.js";
 import { searchFts } from "./search/fts.js";
 import { mergeHybridResults } from "./search/hybrid.js";
@@ -24,6 +27,10 @@ const DEFAULT_BATCH_CONFIG = {
     timeoutMs: 60000,
     maxConsecutiveFailures: 3,
 };
+// Additional constants for extended features
+const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
+const EMBEDDING_BATCH_MAX_TOKENS = 8000;
+const EMBEDDING_APPROX_CHARS_PER_TOKEN = 1;
 /**
  * Memory manager for indexing and searching workspace content
  */
@@ -47,6 +54,10 @@ export class MemoryManager {
     // Batch processing state
     batchDisabled = false;
     batchDisableReason;
+    batchFailureTracker = new BatchFailureTracker();
+    // OpenAI/Gemini client references for batch API
+    openAiClient;
+    geminiClient;
     // sqlite-vec availability
     sqliteVecLoaded = false;
     sqliteVecError;
@@ -54,6 +65,15 @@ export class MemoryManager {
     syncIntervalTimer = null;
     // Last sync timestamp
     lastSyncAt;
+    // Session delta tracking for incremental sync
+    sessionDeltas = new Map();
+    // Session warm tracking (dedupe warmSession calls)
+    sessionWarm = new Set();
+    // Session pending files for debounced processing
+    sessionPendingFiles = new Set();
+    sessionWatchTimer = null;
+    // Session event listener unsubscribe function
+    sessionUnsubscribe = null;
     constructor(config, storage) {
         this.config = config;
         this.storage = storage;
@@ -85,10 +105,21 @@ export class MemoryManager {
                 enabled: config.sync?.enabled ?? true,
                 intervalMs: extConfig.sync?.intervalMs ?? 0,
                 syncSessions: extConfig.sync?.syncSessions ?? false,
+                onSearch: extConfig.sync?.onSearch ?? false,
+                onSessionStart: extConfig.sync?.onSessionStart ?? false,
+                intervalMinutes: extConfig.sync?.intervalMinutes ?? 0,
+                sessions: extConfig.sync?.sessions
+                    ? {
+                        deltaBytes: extConfig.sync.sessions.deltaBytes ?? 0,
+                        deltaMessages: extConfig.sync.sessions.deltaMessages ?? 0,
+                    }
+                    : undefined,
             },
             batch: {
                 ...DEFAULT_BATCH_CONFIG,
                 ...extConfig.batch,
+                wait: extConfig.batch?.wait ?? true,
+                pollIntervalMs: extConfig.batch?.pollIntervalMs ?? 2000,
             },
             sessionsDir: extConfig.sessionsDir,
             extraPaths: extConfig.extraPaths,
@@ -117,9 +148,11 @@ export class MemoryManager {
             await manager.startWatching();
         }
         // Start interval sync if configured
-        if (normalizedConfig.sync.intervalMs > 0) {
+        if (normalizedConfig.sync.intervalMs > 0 || normalizedConfig.sync.intervalMinutes > 0) {
             manager.startIntervalSync();
         }
+        // Set up session event listener for push-based notifications
+        manager.setupSessionListener();
         // Cache instance
         INDEX_CACHE.set(cacheKey, manager);
         return manager;
@@ -168,10 +201,29 @@ export class MemoryManager {
     get providerKey() {
         return computeProviderKey(this.provider.id, this.config.remote?.baseUrl, this.config.remote?.model);
     }
+    /**
+     * Pre-sync on session start with deduplication
+     */
+    async warmSession(sessionKey) {
+        if (!this.config.sync.onSessionStart)
+            return;
+        const key = sessionKey?.trim() || "";
+        if (key && this.sessionWarm.has(key))
+            return;
+        void this.sync().catch(() => { });
+        if (key)
+            this.sessionWarm.add(key);
+    }
     /**
      * Search memory for relevant content
      */
     async search(query, options = {}) {
+        // Warm session if configured
+        void this.warmSession(options.sessionKey);
+        // Sync before search if configured and dirty
+        if (this.config.sync.onSearch && (this.memoryDirty || this.sessionsDirty)) {
+            void this.sync().catch(() => { });
+        }
         const maxResults = options.maxResults ?? this.config.query.maxResults;
         const minScore = options.minScore ?? this.config.query.minScore;
         // Get query embedding
@@ -827,6 +879,252 @@ export class MemoryManager {
         }
         return results;
     }
+    /**
+     * Embed batch with exponential backoff retry (500ms-8s, 3 attempts)
+     */
+    async embedBatchWithRetry(texts) {
+        if (texts.length === 0)
+            return [];
+        let attempt = 0;
+        let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS;
+        while (true) {
+            try {
+                return await this.provider.embedBatch(texts);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                if (!isRetryableEmbeddingError(message) || attempt >= EMBEDDING_RETRY_MAX_ATTEMPTS) {
+                    throw err;
+                }
+                const waitMs = Math.min(EMBEDDING_RETRY_MAX_DELAY_MS, Math.round(delayMs * (1 + Math.random() * 0.2)));
+                await new Promise((resolve) => setTimeout(resolve, waitMs));
+                delayMs *= 2;
+                attempt += 1;
+            }
+        }
+    }
+    /**
+     * Build embedding batches respecting token limits
+     */
+    buildEmbeddingBatches(texts) {
+        const batches = [];
+        let current = [];
+        let currentTokens = 0;
+        for (const text of texts) {
+            const estimate = Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN);
+            const wouldExceed = current.length > 0 && currentTokens + estimate > EMBEDDING_BATCH_MAX_TOKENS;
+            if (wouldExceed) {
+                batches.push(current);
+                current = [];
+                currentTokens = 0;
+            }
+            if (current.length === 0 && estimate > EMBEDDING_BATCH_MAX_TOKENS) {
+                batches.push([text]);
+                continue;
+            }
+            current.push(text);
+            currentTokens += estimate;
+        }
+        if (current.length > 0) {
+            batches.push(current);
+        }
+        return batches;
+    }
+    /**
+     * Run batch with timeout retry (single retry on timeout)
+     */
+    async runBatchWithTimeoutRetry(params) {
+        try {
+            return await params.run();
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            if (isBatchTimeoutError(message)) {
+                try {
+                    return await params.run();
+                }
+                catch (retryErr) {
+                    retryErr.batchAttempts = 2;
+                    throw retryErr;
+                }
+            }
+            throw err;
+        }
+    }
+    /**
+     * Run batch with fallback to inline embedding on failure
+     */
+    async runBatchWithFallback(params) {
+        if (this.batchDisabled || this.batchFailureTracker.disabled) {
+            return await params.fallback();
+        }
+        try {
+            const result = await this.runBatchWithTimeoutRetry({
+                provider: params.provider,
+                run: params.run,
+            });
+            await this.batchFailureTracker.reset();
+            return result;
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            const attempts = err.batchAttempts ?? 1;
+            const forceDisable = isBatchUnavailableError(message);
+            await this.batchFailureTracker.recordFailure({
+                provider: params.provider,
+                message,
+                attempts,
+                forceDisable,
+            });
+            return await params.fallback();
+        }
+    }
+    /**
+     * Embed chunks using async batch API (OpenAI or Gemini)
+     * Exposed for advanced use cases where caller controls chunking
+     */
+    async embedChunksWithAsyncBatch(chunks, entry, source) {
+        const provider = this.provider;
+        // Check cache for existing embeddings
+        const embeddings = Array.from({ length: chunks.length }, () => []);
+        const missing = [];
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            const cached = this.storage.getCachedEmbedding(provider.id, provider.model, this.providerKey, chunk.hash);
+            if (cached && cached.length > 0) {
+                embeddings[i] = cached;
+            }
+            else {
+                missing.push({ index: i, chunk });
+            }
+        }
+        if (missing.length === 0)
+            return embeddings;
+        // Use async batch API if available
+        if (provider.id === "openai" && this.openAiClient && this.config.batch.enabled) {
+            return this.embedChunksWithOpenAiBatch(missing, embeddings, entry, source);
+        }
+        if (provider.id === "gemini" && this.geminiClient && this.config.batch.enabled) {
+            return this.embedChunksWithGeminiBatch(missing, embeddings, entry, source);
+        }
+        // Fallback to inline batch
+        return this.embedChunksInline(missing, embeddings);
+    }
+    /**
+     * Embed chunks using OpenAI async batch API
+     */
+    async embedChunksWithOpenAiBatch(missing, embeddings, entry, source) {
+        const openAi = this.openAiClient;
+        if (!openAi)
+            return this.embedChunksInline(missing, embeddings);
+        const requests = [];
+        const mapping = new Map();
+        for (const item of missing) {
+            const chunk = item.chunk;
+            const customId = hashText(`${source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${item.index}`);
+            mapping.set(customId, { index: item.index, hash: chunk.hash });
+            requests.push({
+                custom_id: customId,
+                method: "POST",
+                url: OPENAI_BATCH_ENDPOINT,
+                body: {
+                    model: openAi.model,
+                    input: chunk.text,
+                },
+            });
+        }
+        const batchResult = await this.runBatchWithFallback({
+            provider: "openai",
+            run: async () => await runOpenAiEmbeddingBatches({
+                openAi,
+                agentId: this.config.agentId,
+                requests,
+                wait: this.config.batch.wait,
+                concurrency: this.config.batch.concurrency,
+                pollIntervalMs: this.config.batch.pollIntervalMs,
+                timeoutMs: this.config.batch.timeoutMs,
+            }),
+            fallback: async () => await this.embedChunksInline(missing, embeddings),
+        });
+        if (Array.isArray(batchResult))
+            return batchResult;
+        const byCustomId = batchResult;
+        // Map results back and cache
+        for (const [customId, embedding] of byCustomId.entries()) {
+            const mapped = mapping.get(customId);
+            if (!mapped)
+                continue;
+            embeddings[mapped.index] = embedding;
+            this.storage.cacheEmbedding(this.provider.id, this.provider.model, this.providerKey, mapped.hash, embedding);
+        }
+        return embeddings;
+    }
+    /**
+     * Embed chunks using Gemini async batch API
+     */
+    async embedChunksWithGeminiBatch(missing, embeddings, entry, source) {
+        const gemini = this.geminiClient;
+        if (!gemini)
+            return this.embedChunksInline(missing, embeddings);
+        const requests = [];
+        const mapping = new Map();
+        for (const item of missing) {
+            const chunk = item.chunk;
+            const customId = hashText(`${source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${item.index}`);
+            mapping.set(customId, { index: item.index, hash: chunk.hash });
+            requests.push({
+                custom_id: customId,
+                content: { parts: [{ text: chunk.text }] },
+                taskType: "RETRIEVAL_DOCUMENT",
+            });
+        }
+        const batchResult = await this.runBatchWithFallback({
+            provider: "gemini",
+            run: async () => await runGeminiEmbeddingBatches({
+                gemini,
+                agentId: this.config.agentId,
+                requests,
+                wait: this.config.batch.wait,
+                concurrency: this.config.batch.concurrency,
+                pollIntervalMs: this.config.batch.pollIntervalMs,
+                timeoutMs: this.config.batch.timeoutMs,
+            }),
+            fallback: async () => await this.embedChunksInline(missing, embeddings),
+        });
+        if (Array.isArray(batchResult))
+            return batchResult;
+        const byCustomId = batchResult;
+        // Map results back and cache
+        for (const [customId, embedding] of byCustomId.entries()) {
+            const mapped = mapping.get(customId);
+            if (!mapped)
+                continue;
+            embeddings[mapped.index] = embedding;
+            this.storage.cacheEmbedding(this.provider.id, this.provider.model, this.providerKey, mapped.hash, embedding);
+        }
+        return embeddings;
+    }
+    /**
+     * Embed chunks using inline batch API (fallback)
+     */
+    async embedChunksInline(missing, embeddings) {
+        const texts = missing.map((m) => m.chunk.text);
+        const batches = this.buildEmbeddingBatches(texts);
+        let cursor = 0;
+        for (const batch of batches) {
+            const batchEmbeddings = await this.embedBatchWithRetry(batch);
+            for (let i = 0; i < batch.length; i++) {
+                const item = missing[cursor + i];
+                const embedding = batchEmbeddings[i] ?? [];
+                if (item) {
+                    embeddings[item.index] = embedding;
+                    this.storage.cacheEmbedding(this.provider.id, this.provider.model, this.providerKey, item.chunk.hash, embedding);
+                }
+            }
+            cursor += batch.length;
+        }
+        return embeddings;
+    }
     /**
      * Start watching for file changes
      */
@@ -847,6 +1145,175 @@ export class MemoryManager {
         });
         await this.watcher.start();
     }
+    /**
+     * Schedule session dirty check with debouncing
+     * Call this when a session file is modified externally
+     */
+    scheduleSessionDirty(sessionFile) {
+        this.sessionPendingFiles.add(sessionFile);
+        if (this.sessionWatchTimer)
+            return;
+        this.sessionWatchTimer = setTimeout(() => {
+            this.sessionWatchTimer = null;
+            void this.processSessionDeltaBatch().catch(() => { });
+        }, SESSION_DIRTY_DEBOUNCE_MS);
+    }
+    /**
+     * Process pending session files and check delta thresholds
+     */
+    async processSessionDeltaBatch() {
+        if (this.sessionPendingFiles.size === 0)
+            return;
+        const pending = Array.from(this.sessionPendingFiles);
+        this.sessionPendingFiles.clear();
+        let shouldSync = false;
+        for (const sessionFile of pending) {
+            const delta = await this.updateSessionDelta(sessionFile);
+            if (!delta)
+                continue;
+            const bytesThreshold = delta.deltaBytes;
+            const messagesThreshold = delta.deltaMessages;
+            const bytesHit = bytesThreshold <= 0 ? delta.pendingBytes > 0 : delta.pendingBytes >= bytesThreshold;
+            const messagesHit = messagesThreshold <= 0
+                ? delta.pendingMessages > 0
+                : delta.pendingMessages >= messagesThreshold;
+            if (!bytesHit && !messagesHit)
+                continue;
+            this.sessionsDirtyFiles.add(sessionFile);
+            this.sessionsDirty = true;
+            delta.pendingBytes =
+                bytesThreshold > 0 ? Math.max(0, delta.pendingBytes - bytesThreshold) : 0;
+            delta.pendingMessages =
+                messagesThreshold > 0 ? Math.max(0, delta.pendingMessages - messagesThreshold) : 0;
+            shouldSync = true;
+        }
+        if (shouldSync) {
+            void this.sync().catch(() => { });
+        }
+    }
+    /**
+     * Update session delta tracking for a file
+     */
+    async updateSessionDelta(sessionFile) {
+        const thresholds = this.config.sync.sessions;
+        if (!thresholds)
+            return null;
+        let fileStat;
+        try {
+            fileStat = await stat(sessionFile);
+        }
+        catch {
+            return null;
+        }
+        const size = fileStat.size;
+        let state = this.sessionDeltas.get(sessionFile);
+        if (!state) {
+            state = { lastSize: 0, pendingBytes: 0, pendingMessages: 0 };
+            this.sessionDeltas.set(sessionFile, state);
+        }
+        const deltaBytes = Math.max(0, size - state.lastSize);
+        if (deltaBytes === 0 && size === state.lastSize) {
+            return {
+                deltaBytes: thresholds.deltaBytes,
+                deltaMessages: thresholds.deltaMessages,
+                pendingBytes: state.pendingBytes,
+                pendingMessages: state.pendingMessages,
+            };
+        }
+        if (size < state.lastSize) {
+            // File was truncated, reset
+            state.lastSize = size;
+            state.pendingBytes += size;
+            const shouldCountMessages = thresholds.deltaMessages > 0 &&
+                (thresholds.deltaBytes <= 0 || state.pendingBytes < thresholds.deltaBytes);
+            if (shouldCountMessages) {
+                state.pendingMessages += await this.countNewlines(sessionFile, 0, size);
+            }
+        }
+        else {
+            state.pendingBytes += deltaBytes;
+            const shouldCountMessages = thresholds.deltaMessages > 0 &&
+                (thresholds.deltaBytes <= 0 || state.pendingBytes < thresholds.deltaBytes);
+            if (shouldCountMessages) {
+                state.pendingMessages += await this.countNewlines(sessionFile, state.lastSize, size);
+            }
+            state.lastSize = size;
+        }
+        this.sessionDeltas.set(sessionFile, state);
+        return {
+            deltaBytes: thresholds.deltaBytes,
+            deltaMessages: thresholds.deltaMessages,
+            pendingBytes: state.pendingBytes,
+            pendingMessages: state.pendingMessages,
+        };
+    }
+    /**
+     * Count newlines in a file range (for message counting)
+     */
+    async countNewlines(absPath, start, end) {
+        if (end <= start)
+            return 0;
+        const handle = await fsOpen(absPath, "r");
+        try {
+            let offset = start;
+            let count = 0;
+            const buffer = Buffer.alloc(SESSION_DELTA_READ_CHUNK_BYTES);
+            while (offset < end) {
+                const toRead = Math.min(buffer.length, end - offset);
+                const { bytesRead } = await handle.read(buffer, 0, toRead, offset);
+                if (bytesRead <= 0)
+                    break;
+                for (let i = 0; i < bytesRead; i += 1) {
+                    if (buffer[i] === 10)
+                        count += 1;
+                }
+                offset += bytesRead;
+            }
+            return count;
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    /**
+     * Reset session delta tracking for a file
+     * Call this after successfully processing a session file
+     */
+    resetSessionDelta(absPath, size) {
+        const state = this.sessionDeltas.get(absPath);
+        if (!state)
+            return;
+        state.lastSize = size;
+        state.pendingBytes = 0;
+        state.pendingMessages = 0;
+    }
+    /**
+     * Check if a session file belongs to this agent's sessions directory
+     */
+    isSessionFileForAgent(sessionFile) {
+        if (!sessionFile || !this.config.sessionsDir)
+            return false;
+        const resolvedFile = resolve(sessionFile);
+        const resolvedDir = resolve(this.config.sessionsDir);
+        return resolvedFile.startsWith(`${resolvedDir}/`) || resolvedFile.startsWith(`${resolvedDir}\\`);
+    }
+    /**
+     * Set up session transcript event listener
+     * This provides push-based notifications when session files change
+     * Called automatically during create(), but can be called manually if needed
+     */
+    setupSessionListener() {
+        if (!this.config.sync.syncSessions || this.sessionUnsubscribe)
+            return;
+        this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => {
+            if (this.closed)
+                return;
+            const sessionFile = update.sessionFile;
+            if (!this.isSessionFileForAgent(sessionFile))
+                return;
+            this.scheduleSessionDirty(sessionFile);
+        });
+    }
     /**
      * Start interval-based sync
      */
@@ -854,11 +1321,17 @@ export class MemoryManager {
         if (this.syncIntervalTimer) {
             return;
         }
+        // Support both intervalMs and intervalMinutes
+        const ms = this.config.sync.intervalMinutes > 0
+            ? this.config.sync.intervalMinutes * 60 * 1000
+            : this.config.sync.intervalMs;
+        if (ms <= 0)
+            return;
         this.syncIntervalTimer = setInterval(() => {
             if (this.memoryDirty || this.sessionsDirty) {
                 this.sync().catch(() => { });
             }
-        }, this.config.sync.intervalMs);
+        }, ms);
     }
     /**
      * Stop interval-based sync
@@ -884,6 +1357,16 @@ export class MemoryManager {
     async close() {
         this.stopIntervalSync();
         await this.stopWatching();
+        // Clean up session watch timer
+        if (this.sessionWatchTimer) {
+            clearTimeout(this.sessionWatchTimer);
+            this.sessionWatchTimer = null;
+        }
+        // Unsubscribe from session events
+        if (this.sessionUnsubscribe) {
+            this.sessionUnsubscribe();
+            this.sessionUnsubscribe = null;
+        }
         this.storage.close();
         this.closed = true;
         // Remove from cache
@@ -951,10 +1434,10 @@ export class MemoryManager {
                 consecutiveFailures: this.consecutiveEmbeddingFailures,
             },
             batch: {
-                enabled: this.config.batch.enabled && !this.batchDisabled,
-                pendingJobs: 0, // TODO: track pending batch jobs
-                autoDisabled: this.batchDisabled,
-                disableReason: this.batchDisableReason,
+                enabled: this.config.batch.enabled && !this.batchDisabled && !this.batchFailureTracker.disabled,
+                pendingJobs: 0,
+                autoDisabled: this.batchDisabled || this.batchFailureTracker.disabled,
+                disableReason: this.batchDisableReason ?? this.batchFailureTracker.error,
             },
             sync: {
                 watching: this.watcher !== null,