dsh-tdai-memory 0.1.0

package/vendor/tdai/utils/pipeline-factory.js

@@ -0,0 +1,517 @@
+/**
+ * Pipeline factory: shared infrastructure for creating and wiring
+ * MemoryPipelineManager instances with VectorStore, EmbeddingService,
+ * L1 runner, L2 runner, L3 runner, and persister.
+ *
+ * Used by both:
+ * - `index.ts` (live plugin runtime)
+ * - `seed-runtime.ts` (standalone seed CLI command)
+ *
+ * This avoids duplicating VectorStore init, L1/L2/L3 extraction logic,
+ * persister wiring, and destroy sequences across multiple callers.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { MemoryPipelineManager } from "./pipeline-manager.js";
+import { SessionFilter } from "./session-filter.js";
+import { extractL1Memories } from "../core/record/l1-extractor.js";
+import { readConversationMessagesGroupedBySessionId } from "../core/conversation/l0-recorder.js";
+import { CheckpointManager } from "./checkpoint.js";
+import { createStoreBundle } from "../core/store/factory.js";
+import { readManifest, writeManifest, buildStoreInfo, diffStoreBinding, } from "./manifest.js";
+import { SceneExtractor } from "../core/scene/scene-extractor.js";
+import { PersonaTrigger } from "../core/persona/persona-trigger.js";
+import { PersonaGenerator } from "../core/persona/persona-generator.js";
+import { pullProfilesToLocal, syncLocalProfilesToStore } from "../core/profile/profile-sync.js";
+const TAG = "[memory-tdai] [pipeline-factory]";
+function supportsProfileSyncWrite(store) {
+    return !!(store?.syncProfiles || store?.deleteProfiles);
+}
+// ============================
+// Data directory init
+// ============================
+/**
+ * Ensure all required data subdirectories exist under `pluginDataDir`.
+ * Safe to call multiple times (mkdirSync with `recursive: true`).
+ */
+export function initDataDirectories(dataDir) {
+    const dirs = ["conversations", "records", "scene_blocks", ".metadata", ".backup"];
+    for (const sub of dirs) {
+        fs.mkdirSync(path.join(dataDir, sub), { recursive: true });
+    }
+}
+/**
+ * Cached store init promises — keyed by `pluginDataDir` so that different
+ * data directories (e.g. live runtime vs. seed output) each get their own
+ * store instance, while concurrent callers for the *same* directory share
+ * one initialization.
+ */
+const _storeInitCache = new Map();
+/**
+ * Initialize store backend and (optionally) EmbeddingService.
+ *
+ * **Once-async semantics per dataDir**: the first call for a given
+ * `pluginDataDir` creates the store and caches the result; subsequent
+ * calls with the same dir return the cached Promise immediately.
+ * Call `resetStores()` during shutdown to clear the cache.
+ *
+ * Supports both SQLite (sync init) and TCVDB (async init) backends.
+ */
+export function initStores(cfg, pluginDataDir, logger) {
+    const key = pluginDataDir;
+    if (!_storeInitCache.has(key)) {
+        _storeInitCache.set(key, _doInitStores(cfg, pluginDataDir, logger));
+    }
+    return _storeInitCache.get(key);
+}
+/**
+ * Reset the cached store singleton(s).
+ *
+ * Call this during `gateway_stop` (after closing the actual store/embedding
+ * resources) so that a subsequent `register()` on hot-restart can
+ * re-initialize fresh instances.
+ *
+ * @param pluginDataDir  If provided, only clear the cache for that dir.
+ *                       If omitted, clear all cached stores.
+ */
+export function resetStores(pluginDataDir) {
+    if (pluginDataDir) {
+        _storeInitCache.delete(pluginDataDir);
+    }
+    else {
+        _storeInitCache.clear();
+    }
+}
+/**
+ * Internal: actual store initialization logic (called once by the cache).
+ */
+async function _doInitStores(cfg, pluginDataDir, logger) {
+    let vectorStore;
+    let embeddingService;
+    let needsReindex = false;
+    let reindexReason;
+    try {
+        const bundle = createStoreBundle(cfg, {
+            dataDir: pluginDataDir,
+            logger,
+        });
+        vectorStore = bundle.store;
+        embeddingService = bundle.embedding ?? undefined;
+        const providerInfo = embeddingService?.getProviderInfo();
+        const initResult = await vectorStore.init(providerInfo);
+        if (vectorStore.isDegraded()) {
+            logger.warn(`${TAG} Store is in degraded mode, falling back to keyword dedup`);
+            vectorStore = undefined;
+            embeddingService = undefined;
+        }
+        else {
+            logger.debug?.(`${TAG} Store initialized: backend=${cfg.storeBackend}, provider=${cfg.embedding.provider}`);
+            needsReindex = initResult.needsReindex;
+            reindexReason = initResult.reason;
+            // ── Manifest: first-write + config-drift detection ──
+            try {
+                const currentStoreInfo = buildStoreInfo(bundle.storeSnapshot);
+                const existing = readManifest(pluginDataDir);
+                if (!existing) {
+                    // First init — write manifest
+                    const manifest = {
+                        version: 1,
+                        createdAt: new Date().toISOString(),
+                        store: currentStoreInfo,
+                        seed: null,
+                    };
+                    writeManifest(pluginDataDir, manifest);
+                    logger.debug?.(`${TAG} Manifest created: ${JSON.stringify(currentStoreInfo)}`);
+                }
+                else {
+                    // Compare persisted store binding against current config
+                    const diffs = diffStoreBinding(existing.store, currentStoreInfo);
+                    if (diffs.length > 0) {
+                        logger.debug?.(`${TAG} Store config differs from initial binding recorded in manifest ` +
+                            `(${diffs.join("; ")}). ` +
+                            `This is expected if the storage backend was switched intentionally.`);
+                    }
+                }
+            }
+            catch (err) {
+                logger.warn(`${TAG} Failed to read/write manifest (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    catch (err) {
+        logger.warn(`${TAG} Store init failed; vector/FTS recall and dedup conflict detection will be unavailable: ${err instanceof Error ? err.message : String(err)}`);
+        vectorStore = undefined;
+        embeddingService = undefined;
+    }
+    return { vectorStore, embeddingService, needsReindex, reindexReason };
+}
+// ============================
+// L1 Runner factory
+// ============================
+/**
+ * Create the standard L1 runner function.
+ *
+ * Reads L0 messages (from VectorStore DB or JSONL fallback), groups by sessionId,
+ * runs extractL1Memories for each group, and updates the checkpoint cursor.
+ */
+export function createL1Runner(opts) {
+    const { pluginDataDir, cfg, openclawConfig, vectorStore, embeddingService, logger, getInstanceId, llmRunner } = opts;
+    const config = openclawConfig;
+    return async ({ sessionKey }) => {
+        if (!config && !llmRunner) {
+            logger.debug?.(`${TAG} [l1] No OpenClaw config and no LLM runner, skipping L1 extraction`);
+            return { processedCount: 0 };
+        }
+        const checkpoint = new CheckpointManager(pluginDataDir, logger);
+        const cp = await checkpoint.read();
+        const runnerState = checkpoint.getRunnerState(cp, sessionKey);
+        logger.info(`${TAG} [l1] Session ${sessionKey}: l1_cursor=${runnerState.last_l1_cursor || "(start)"}`);
+        try {
+            let groups;
+            let maxRecordedAtMs = 0;
+            if (vectorStore && !vectorStore.isDegraded()) {
+                const l1Cursor = runnerState.last_l1_cursor > 0
+                    ? runnerState.last_l1_cursor
+                    : undefined;
+                const dbGroups = await vectorStore.queryL0GroupedBySessionId(sessionKey, l1Cursor);
+                groups = dbGroups.map((g) => ({
+                    sessionId: g.sessionId,
+                    messages: g.messages.map((m) => ({
+                        id: m.id,
+                        role: m.role,
+                        content: m.content,
+                        timestamp: m.timestamp,
+                    })),
+                }));
+                // Compute max recordedAtMs across all groups for cursor advancement
+                for (const g of dbGroups) {
+                    for (const m of g.messages) {
+                        if (m.recordedAtMs > maxRecordedAtMs)
+                            maxRecordedAtMs = m.recordedAtMs;
+                    }
+                }
+                logger.debug?.(`${TAG} [l1] L0 data source: VectorStore DB`);
+            }
+            else {
+                logger.debug?.(`${TAG} [l1] L0 data source: JSONL files (VectorStore unavailable)`);
+                const jsonlGroups = await readConversationMessagesGroupedBySessionId(sessionKey, pluginDataDir, runnerState.last_l1_cursor || undefined, logger, 50);
+                groups = jsonlGroups.map((g) => ({
+                    sessionId: g.sessionId,
+                    messages: g.messages,
+                }));
+                // Compute max recordedAtMs from JSONL groups
+                for (const g of jsonlGroups) {
+                    for (const m of g.messages) {
+                        if (m.recordedAtMs > maxRecordedAtMs)
+                            maxRecordedAtMs = m.recordedAtMs;
+                    }
+                }
+            }
+            if (groups.length === 0) {
+                logger.debug?.(`${TAG} [l1] No new L0 messages for session ${sessionKey}`);
+                return { processedCount: 0 };
+            }
+            const totalMessages = groups.reduce((sum, g) => sum + g.messages.length, 0);
+            logger.info(`${TAG} [l1] Processing ${totalMessages} L0 messages across ${groups.length} sessionId group(s) for session ${sessionKey}`);
+            let totalExtracted = 0;
+            let totalStored = 0;
+            let lastSceneName;
+            for (const group of groups) {
+                logger.debug?.(`${TAG} [l1] Group sessionId=${group.sessionId || "(empty)"}: ${group.messages.length} messages`);
+                const l1Result = await extractL1Memories({
+                    messages: group.messages,
+                    sessionKey,
+                    sessionId: group.sessionId,
+                    baseDir: pluginDataDir,
+                    config,
+                    options: {
+                        enableDedup: cfg.extraction.enableDedup,
+                        maxMemoriesPerSession: cfg.extraction.maxMemoriesPerSession,
+                        model: cfg.extraction.model,
+                        previousSceneName: lastSceneName ?? (runnerState.last_scene_name || undefined),
+                        vectorStore,
+                        embeddingService,
+                        conflictRecallTopK: cfg.embedding.conflictRecallTopK,
+                        embeddingTimeoutMs: cfg.embedding.captureTimeoutMs ?? cfg.embedding.timeoutMs,
+                        llmRunner,
+                    },
+                    logger,
+                    instanceId: getInstanceId?.(),
+                });
+                totalExtracted += l1Result.extractedCount;
+                totalStored += l1Result.storedCount;
+                if (l1Result.lastSceneName) {
+                    lastSceneName = l1Result.lastSceneName;
+                }
+            }
+            // Use maxRecordedAtMs (write time) as cursor — always positive, TCVDB-safe
+            await checkpoint.markL1ExtractionComplete(sessionKey, totalStored, maxRecordedAtMs || undefined, lastSceneName);
+            logger.info(`${TAG} [l1] L1 complete: extracted=${totalExtracted}, stored=${totalStored} (${groups.length} group(s))`);
+            return { processedCount: totalMessages };
+        }
+        catch (err) {
+            logger.error(`${TAG} [l1] L1 failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
+            throw err;
+        }
+    };
+}
+// ============================
+// Persister factory
+// ============================
+/**
+ * Create the standard pipeline state persister.
+ * Saves pipeline session states to the checkpoint file.
+ */
+export function createPersister(pluginDataDir, logger) {
+    return async (states) => {
+        const checkpoint = new CheckpointManager(pluginDataDir, logger);
+        await checkpoint.mergePipelineStates(states);
+    };
+}
+// ============================
+// L2 Runner factory
+// ============================
+/**
+ * Create the standard L2 runner function (scene extraction).
+ *
+ * Reads L1 memory records (incremental via VectorStore or JSONL fallback),
+ * runs SceneExtractor, and returns the latest cursor for pipeline-manager
+ * to track incremental progress.
+ *
+ * Used by both `index.ts` (live runtime) and `seed-runtime.ts` (seed CLI).
+ */
+export function createL2Runner(opts) {
+    const { pluginDataDir, cfg, openclawConfig, vectorStore, logger, instanceId, llmRunner } = opts;
+    let profileBaseline = new Map();
+    return async (sessionKey, cursor) => {
+        logger.debug?.(`${TAG} [L2] session=${sessionKey}, updatedAfter=${cursor ?? "(full)"}`);
+        if (!openclawConfig && !llmRunner) {
+            logger.warn(`${TAG} [L2] No OpenClaw config and no LLM runner, skipping scene extraction`);
+            return;
+        }
+        let records;
+        if (vectorStore?.pullProfiles && !vectorStore.isDegraded()) {
+            profileBaseline = await pullProfilesToLocal(pluginDataDir, vectorStore, logger);
+        }
+        if (vectorStore && !vectorStore.isDegraded()) {
+            const { queryMemoryRecords } = await import("../core/record/l1-reader.js");
+            const memRecords = await queryMemoryRecords(vectorStore, {
+                sessionKey,
+                updatedAfter: cursor,
+            }, logger);
+            if (memRecords.length === 0) {
+                logger.debug?.(`${TAG} [L2] No new L1 records since cursor (session=${sessionKey}, updatedAfter=${cursor ?? "(full)"}), skipping scene extraction`);
+                return { skipped: true, latestCursor: cursor || undefined };
+            }
+            logger.debug?.(`${TAG} [L2] Incremental query returned ${memRecords.length} record(s) (session=${sessionKey})`);
+            records = memRecords.map((r) => ({
+                content: r.content,
+                created_at: r.createdAt,
+                id: r.id,
+                updatedAt: r.updatedAt,
+            }));
+        }
+        else {
+            logger.debug?.(`${TAG} [L2] VectorStore unavailable, falling back to JSONL read (session=${sessionKey})`);
+            const { readMemoryRecords } = await import("../core/record/l1-reader.js");
+            let sessionRecords = await readMemoryRecords(sessionKey, pluginDataDir, logger);
+            if (cursor) {
+                const beforeCount = sessionRecords.length;
+                sessionRecords = sessionRecords.filter((r) => {
+                    const t = r.updatedAt || r.createdAt || "";
+                    return t > cursor;
+                });
+                logger.debug?.(`${TAG} [L2] JSONL time filter: ${beforeCount} → ${sessionRecords.length} record(s) (updatedAfter=${cursor})`);
+            }
+            if (sessionRecords.length === 0) {
+                logger.debug?.(`${TAG} [L2] No new L1 records found (JSONL fallback, session=${sessionKey}), skipping scene extraction`);
+                return { latestCursor: cursor || undefined };
+            }
+            records = sessionRecords.map((r) => ({
+                content: r.content,
+                created_at: r.createdAt,
+                id: r.id,
+                updatedAt: r.updatedAt,
+            }));
+        }
+        const extractor = new SceneExtractor({
+            dataDir: pluginDataDir,
+            config: openclawConfig,
+            model: cfg.persona.model,
+            maxScenes: cfg.persona.maxScenes,
+            sceneBackupCount: cfg.persona.sceneBackupCount,
+            logger,
+            instanceId,
+            llmRunner,
+        });
+        const memories = records.map((r) => ({
+            content: r.content,
+            created_at: r.created_at,
+            id: r.id,
+        }));
+        const preCheckpoint = new CheckpointManager(pluginDataDir, logger);
+        const preState = await preCheckpoint.read();
+        const preScenesProcessed = preState.scenes_processed;
+        const preMemoriesSince = preState.memories_since_last_persona;
+        const preTotalProcessed = preState.total_processed;
+        const extractResult = await extractor.extract(memories);
+        if (extractResult.success && extractResult.memoriesProcessed > 0) {
+            const checkpoint = new CheckpointManager(pluginDataDir, logger);
+            const postState = await checkpoint.read();
+            if (postState.scenes_processed < preScenesProcessed ||
+                postState.total_processed < preTotalProcessed) {
+                logger.warn(`${TAG} [L2] ⚠️ Checkpoint corruption detected! ` +
+                    `scenes_processed: ${preScenesProcessed} → ${postState.scenes_processed}, ` +
+                    `total_processed: ${preTotalProcessed} → ${postState.total_processed}, ` +
+                    `memories_since: ${preMemoriesSince} → ${postState.memories_since_last_persona}. ` +
+                    `Repairing...`);
+                await checkpoint.write({
+                    ...postState,
+                    scenes_processed: Math.max(postState.scenes_processed, preScenesProcessed),
+                    total_processed: Math.max(postState.total_processed, preTotalProcessed),
+                    memories_since_last_persona: Math.max(postState.memories_since_last_persona, preMemoriesSince),
+                });
+                logger.info(`${TAG} [L2] Checkpoint repaired`);
+            }
+            if (vectorStore && supportsProfileSyncWrite(vectorStore)) {
+                await syncLocalProfilesToStore(pluginDataDir, vectorStore, profileBaseline, logger);
+            }
+            await checkpoint.incrementScenesProcessed();
+            const latestCursor = records.reduce((latest, r) => {
+                return r.updatedAt > latest ? r.updatedAt : latest;
+            }, "");
+            logger.debug?.(`${TAG} [L2] Extraction complete: processed=${extractResult.memoriesProcessed}, latestCursor=${latestCursor}`);
+            return { latestCursor: latestCursor || undefined };
+        }
+    };
+}
+// ============================
+// L3 Runner factory
+// ============================
+/**
+ * Create the standard L3 runner function (persona generation).
+ *
+ * Uses PersonaTrigger to check if generation is needed, then runs
+ * PersonaGenerator. Used by both `index.ts` and `seed-runtime.ts`.
+ */
+export function createL3Runner(opts) {
+    const { pluginDataDir, cfg, openclawConfig, vectorStore, logger, instanceId, llmRunner } = opts;
+    return async () => {
+        const trigger = new PersonaTrigger({
+            dataDir: pluginDataDir,
+            interval: cfg.persona.triggerEveryN,
+            logger,
+        });
+        const { should, reason } = await trigger.shouldGenerate();
+        if (!should) {
+            logger.debug?.(`${TAG} [L3] Persona generation not needed`);
+            return;
+        }
+        if (!openclawConfig && !llmRunner) {
+            logger.warn(`${TAG} [L3] No OpenClaw config and no LLM runner, skipping persona generation`);
+            return;
+        }
+        // Pull remote profiles to establish fresh baseline before generation.
+        // This ensures syncLocalProfilesToStore() has correct baselineVersion
+        // for the optimistic-lock check instead of defaulting to 0.
+        let profileBaseline = new Map();
+        if (vectorStore?.pullProfiles && !vectorStore.isDegraded()) {
+            profileBaseline = await pullProfilesToLocal(pluginDataDir, vectorStore, logger);
+        }
+        logger.info(`${TAG} [L3] Starting persona generation: ${reason}`);
+        const generator = new PersonaGenerator({
+            dataDir: pluginDataDir,
+            config: openclawConfig,
+            model: cfg.persona.model,
+            backupCount: cfg.persona.backupCount,
+            logger,
+            instanceId,
+            llmRunner,
+        });
+        const genResult = await generator.generateLocalPersona(reason);
+        if (!genResult) {
+            logger.info(`${TAG} [L3] Persona generation skipped (no changes)`);
+            return;
+        }
+        if (vectorStore && supportsProfileSyncWrite(vectorStore)) {
+            await syncLocalProfilesToStore(pluginDataDir, vectorStore, profileBaseline, logger);
+        }
+        const checkpoint = new CheckpointManager(pluginDataDir, logger);
+        const cp = await checkpoint.read();
+        await checkpoint.markPersonaGenerated(cp.total_processed);
+        logger.info(`${TAG} [L3] Persona generation succeeded`);
+    };
+}
+// ============================
+// Pipeline Manager factory
+// ============================
+/**
+ * Create a MemoryPipelineManager with the standard config mapping.
+ */
+export function createPipelineManager(cfg, logger, sessionFilter) {
+    return new MemoryPipelineManager({
+        everyNConversations: cfg.pipeline.everyNConversations,
+        enableWarmup: cfg.pipeline.enableWarmup,
+        l1: { idleTimeoutSeconds: cfg.pipeline.l1IdleTimeoutSeconds },
+        l2: {
+            delayAfterL1Seconds: cfg.pipeline.l2DelayAfterL1Seconds,
+            minIntervalSeconds: cfg.pipeline.l2MinIntervalSeconds,
+            maxIntervalSeconds: cfg.pipeline.l2MaxIntervalSeconds,
+            sessionActiveWindowHours: cfg.pipeline.sessionActiveWindowHours,
+        },
+    }, logger, sessionFilter ?? new SessionFilter([]));
+}
+// ============================
+// Full pipeline factory
+// ============================
+/**
+ * Create a fully wired pipeline instance: VectorStore + EmbeddingService +
+ * MemoryPipelineManager with L1 runner and persister attached.
+ *
+ * This is the high-level entry point used by both `index.ts` and `seed-runtime.ts`.
+ * Callers should attach L2/L3 runners after creation using `createL2Runner()`
+ * and `createL3Runner()` from this module.
+ */
+export async function createPipeline(opts) {
+    const { pluginDataDir, cfg, openclawConfig, logger, sessionFilter, l1LlmRunner } = opts;
+    // Ensure data directories exist
+    initDataDirectories(pluginDataDir);
+    // Initialize stores (once-async: reuses cached result if already initialized)
+    const stores = await initStores(cfg, pluginDataDir, logger);
+    const { vectorStore, embeddingService } = stores;
+    // Create pipeline manager
+    const scheduler = createPipelineManager(cfg, logger, sessionFilter);
+    // Wire L1 runner
+    scheduler.setL1Runner(createL1Runner({
+        pluginDataDir,
+        cfg,
+        openclawConfig,
+        vectorStore,
+        embeddingService,
+        logger,
+        llmRunner: l1LlmRunner,
+    }));
+    // Wire persister
+    scheduler.setPersister(createPersister(pluginDataDir, logger));
+    // Destroy function
+    const destroy = async () => {
+        logger.info(`${TAG} Destroying pipeline...`);
+        await scheduler.destroy();
+        if (vectorStore) {
+            logger.info(`${TAG} Closing VectorStore`);
+            vectorStore.close();
+        }
+        if (embeddingService?.close) {
+            try {
+                logger.info(`${TAG} Closing EmbeddingService`);
+                await embeddingService.close();
+            }
+            catch (err) {
+                logger.warn(`${TAG} Error closing EmbeddingService: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        resetStores(pluginDataDir);
+        logger.info(`${TAG} Pipeline destroyed`);
+    };
+    return { scheduler, vectorStore, embeddingService, destroy };
+}