@theowlops/boombot 0.0.0-stage → 1.1.1

package/dist/gateway/services/memory.js

@@ -0,0 +1,511 @@
+/**
+ * Gateway memory service — wires `@boombot/core` (SQLite FTS5) into the
+ * gateway runtime.
+ *
+ * Two-tier retrieval (mirrors OpenClaw's active-memory escalation):
+ * - `recall` — LIGHT tier. Bounded FTS snapshot used by the agent turn loop
+ *   before the first model round. Returns agent-core's `MemoryHit`
+ *   (`{ id, content, score }`) so it plugs directly into `MemoryPort`.
+ * - `search` — DEEP tier. Larger result set with full snippets for the
+ *   `memory.search` wire command. Returns the gateway protocol's `MemoryHit`
+ *   (`{ id, excerpt, score }`).
+ *
+ * The two `MemoryHit` shapes differ (`content` vs `excerpt`); the
+ * adaptation happens here, inside the service, so callers on either side
+ * never see the other side's shape.
+ *
+ * Curation runs on its own timer (never inside the agent loop), uses the
+ * package's `TtlMemoryCurator`, and persists its run state in the store's
+ * audit trail under the sentinel id `memory:curation`. A failing curation
+ * run is logged and swallowed — it never breaks request handling.
+ *
+ * Consolidation (LLM extraction of durable facts from past conversations)
+ * runs on a second timer the same way, via `@boombot/core`'s
+ * `ConsolidationJob`. Its watermark cursor lives in the audit trail under
+ * the sentinel id `memory:consolidation`, mirroring the curation convention.
+ * Consolidation never runs inline after a turn — the turn loop stays hot
+ * and LLM spend stays bounded by the job's budget.
+ *
+ * Lifecycle: `startMemoryService(ctx)` attaches the service as
+ * `ctx.memoryService` and returns an async stop function that halts both
+ * timers and closes the database.
+ */
+import { randomUUID } from "node:crypto";
+import { mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { GLOBAL_SCOPE, conversationScope, CONSOLIDATION_STATE_ID, ConsolidationJob, DEFAULT_CONSOLIDATION_BUDGET, MAX_QUERY_CHARS, MAX_QUERY_TERMS, MemoryStore, principalScope, recallMemory, searchForPrincipal, SqliteFtsMemoryProvider, TtlMemoryCurator, InMemoryVectorIndex, reciprocalRankFusion, } from "@boombot/core";
+export { CONSOLIDATION_STATE_ID };
+/**
+ * Terms the FTS layer will actually index — mirrors `extractTerms`'s
+ * `TERM_PATTERN` (`packages/core/src/memory/search.ts`). Duplicated rather
+ * than imported because `extractTerms` is precisely what throws on an
+ * over-cap query, so calling it here to clamp would defeat the purpose.
+ */
+const QUERY_TERM_PATTERN = /[\p{L}\p{N}]+/gu;
+/**
+ * Bound recall queries by characters and unique terms before calling FTS.
+ *
+ * Recall queries are whole user messages. The store's term cap exists to
+ * bound FTS work, but it was rejecting a turn's own context: any message over
+ * 32 terms made `extractTerms` throw `query exceeds 32 terms`, aborting the
+ * turn before the model ran. Clamping keeps recall working for long messages
+ * instead of silently dropping to zero hits. Repeated terms spend no extra budget.
+ */
+function clampRecallQuery(query) {
+    const terms = query.slice(0, MAX_QUERY_CHARS).match(QUERY_TERM_PATTERN) ?? [];
+    return [...new Set(terms.map((term) => term.toLowerCase()))].slice(0, MAX_QUERY_TERMS).join(" ");
+}
+const DEFAULT_RECALL_LIMIT = 5;
+const DEFAULT_RECALL_CHARS_PER_HIT = 500;
+const DEFAULT_RECALL_TOTAL_CHARS = 2500;
+const DEFAULT_CURATION_INTERVAL_MS = 3_600_000; // 1h
+const DEFAULT_CURATION_MAX_AGE_DAYS = 90;
+const DEFAULT_CONSOLIDATION_INTERVAL_MS = 1_800_000; // 30min
+/** Hard cap on raw extraction output accumulated from a streaming LLM. */
+const MAX_CONSOLIDATION_RESPONSE_CHARS = 32_768;
+/** Sentinel memory id under which curation run state is kept in the audit trail. */
+export const CURATION_STATE_ID = "memory:curation";
+function resolveOptions(options) {
+    const curationMaxAgeDays = options.curationMaxAgeDays ?? DEFAULT_CURATION_MAX_AGE_DAYS;
+    if (!Number.isFinite(curationMaxAgeDays) || curationMaxAgeDays < 1) {
+        throw new Error("MemoryService: curationMaxAgeDays must be >= 1");
+    }
+    const curationIntervalMs = options.curationIntervalMs ?? DEFAULT_CURATION_INTERVAL_MS;
+    if (!Number.isFinite(curationIntervalMs) || curationIntervalMs <= 0) {
+        throw new Error("MemoryService: curationIntervalMs must be > 0");
+    }
+    const consolidationIntervalMs = options.consolidationIntervalMs ?? DEFAULT_CONSOLIDATION_INTERVAL_MS;
+    if (!Number.isFinite(consolidationIntervalMs) || consolidationIntervalMs <= 0) {
+        throw new Error("MemoryService: consolidationIntervalMs must be > 0");
+    }
+    return {
+        principalId: options.principalId ?? "local",
+        recallLimit: Math.min(Math.max(Math.floor(options.recallLimit ?? DEFAULT_RECALL_LIMIT), 1), 50),
+        curationIntervalMs,
+        curationMaxAgeDays,
+        curationDryRun: options.curationDryRun ?? false,
+        consolidationIntervalMs,
+        consolidationEnabled: options.consolidationEnabled ?? true,
+    };
+}
+/**
+ * Flatten a session's messages into the transcript the extractor sees.
+ * Only user + assistant turns are kept; system prompts and tool outputs
+ * are cut entirely (tool results can carry secrets and bloat the prompt).
+ */
+function flattenTranscript(messages) {
+    const lines = [];
+    for (const message of messages) {
+        if (message.role !== "user" && message.role !== "assistant")
+            continue;
+        const text = message.content.trim();
+        if (text === "")
+            continue;
+        lines.push(`${message.role}: ${text}`);
+    }
+    return lines.join("\n");
+}
+/**
+ * Accumulate a streaming LLM completion into plain text for the extractor.
+ * Only `delta` is collected — `reasoningDelta` is deliberately ignored so
+ * chain-of-thought never lands in memory.
+ */
+async function collectLlmText(client, prompt) {
+    const message = { role: "user", content: prompt };
+    let text = "";
+    for await (const chunk of client.complete([message], [])) {
+        if (typeof chunk.delta === "string" && chunk.delta.length > 0) {
+            text += chunk.delta;
+            if (text.length >= MAX_CONSOLIDATION_RESPONSE_CHARS)
+                break;
+        }
+    }
+    return text.slice(0, MAX_CONSOLIDATION_RESPONSE_CHARS);
+}
+export class GatewayMemoryService {
+    store;
+    curationProvider;
+    curator = new TtlMemoryCurator();
+    consolidationProvider;
+    consolidationJob;
+    vectorIndex = new InMemoryVectorIndex();
+    resolveEmbedding;
+    principalId;
+    recallLimit;
+    curationIntervalMs;
+    curationMaxAgeDays;
+    curationDryRun;
+    log;
+    curationTimer;
+    curationRuns = 0;
+    lastCuration;
+    consolidationRuns = 0;
+    lastConsolidation;
+    constructor(dbPath, log, options = {}) {
+        const resolved = resolveOptions(options);
+        this.log = log;
+        this.principalId = resolved.principalId;
+        this.recallLimit = resolved.recallLimit;
+        this.curationIntervalMs = resolved.curationIntervalMs;
+        this.curationMaxAgeDays = resolved.curationMaxAgeDays;
+        this.curationDryRun = resolved.curationDryRun;
+        this.resolveEmbedding = options.resolveEmbedding;
+        mkdirSync(dirname(dbPath), { recursive: true });
+        this.store = MemoryStore.open(dbPath);
+        // Separate provider handle for the curator (its own connection; the
+        // curator only needs list/isPinned/archive and never touches recall).
+        this.curationProvider = new SqliteFtsMemoryProvider({
+            path: dbPath,
+            principalId: `${this.principalId}:curator`,
+        });
+        // Consolidation needs the session store (conversation source) and an LLM
+        // resolver. Without either it stays disabled — curation is unaffected.
+        // Like the curator, it gets its own provider handle, this time scoped to
+        // the real principal so dedupe search sees the principal's own facts.
+        if (resolved.consolidationEnabled &&
+            options.sessions !== undefined &&
+            options.resolveLlm !== undefined) {
+            const sessions = options.sessions;
+            const resolveLlm = options.resolveLlm;
+            const principalId = resolved.principalId;
+            this.consolidationProvider = new SqliteFtsMemoryProvider({
+                path: dbPath,
+                principalId,
+            });
+            const llm = {
+                completeText: async (prompt) => {
+                    const client = await resolveLlm();
+                    if (client === undefined) {
+                        throw new Error("memory-consolidation: no LLM provider configured");
+                    }
+                    return collectLlmText(client, prompt);
+                },
+            };
+            this.consolidationJob = new ConsolidationJob({
+                llm,
+                store: this.consolidationProvider,
+                conversations: this.buildConversationSource(sessions, principalId),
+                watermarks: this.buildWatermarkStore(),
+                budget: DEFAULT_CONSOLIDATION_BUDGET,
+                intervalMs: resolved.consolidationIntervalMs,
+                onError: (error) => {
+                    this.log.error("[memory] consolidation run failed; continuing", error);
+                },
+            });
+        }
+        else {
+            this.consolidationProvider = undefined;
+            this.consolidationJob = undefined;
+        }
+    }
+    async recall(query, opts) {
+        const principal = this.principalId;
+        const readableScopes = opts?.readableScopes ?? [
+            ...(opts?.channel ? [] : [principalScope(principal)]),
+            GLOBAL_SCOPE,
+            ...(opts?.sessionScope ? [opts.sessionScope] : []),
+            ...(opts?.channel ? [opts.chatId
+                    ? conversationScope(opts.channel, opts.chatId)
+                    : principalScope(`channel:${opts.channel}`)] : [])
+        ];
+        const normalizedQuery = clampRecallQuery(query);
+        if (normalizedQuery === '')
+            return [];
+        const maxCharsPerHit = Number.parseInt(process.env['BOOMBOT_MEMORY_RECALL_CHARS_PER_HIT'] || '', 10) || DEFAULT_RECALL_CHARS_PER_HIT;
+        const maxTotalChars = Number.parseInt(process.env['BOOMBOT_MEMORY_RECALL_TOTAL_CHARS'] || '', 10) || DEFAULT_RECALL_TOTAL_CHARS;
+        const snapshot = recallMemory(this.store, {
+            authorization: { principalId: principal, readableScopes },
+            query: normalizedQuery,
+            maxResults: this.recallLimit,
+            maxCharsPerHit,
+            maxTotalChars,
+        });
+        const lexicalHits = snapshot.hits.map((hit) => ({
+            id: hit.record.id,
+            score: hit.score,
+            item: {
+                id: hit.record.id,
+                content: hit.snippet,
+                score: hit.score,
+            },
+        }));
+        // Hybrid Vector RAG (Reciprocal Rank Fusion) when embedding provider and indexed vectors are active
+        if (this.resolveEmbedding && this.vectorIndex.size() > 0) {
+            try {
+                const client = await this.resolveEmbedding();
+                if (client) {
+                    const vectors = await client.embed([normalizedQuery]);
+                    const queryVec = vectors[0];
+                    if (queryVec) {
+                        const allowedScopeIds = readableScopes.map((s) => typeof s === 'object' && s !== null && 'kind' in s
+                            ? `${s.kind}:${s.id ?? ''}`
+                            : String(s));
+                        const vectorHits = this.vectorIndex.search(new Float32Array(queryVec), {
+                            topK: this.recallLimit,
+                            allowedScopeIds,
+                        });
+                        if (vectorHits.length > 0) {
+                            const semanticHits = vectorHits.map((hit) => {
+                                const snippet = hit.metadata?.['snippet'] ?? '';
+                                return {
+                                    id: hit.id,
+                                    score: hit.similarity,
+                                    item: {
+                                        id: hit.id,
+                                        content: snippet,
+                                        score: hit.similarity,
+                                    },
+                                };
+                            });
+                            const fused = reciprocalRankFusion(lexicalHits, semanticHits, { limit: this.recallLimit });
+                            return fused.map((res) => res.item);
+                        }
+                    }
+                }
+            }
+            catch (err) {
+                this.log.debug?.('[memory] vector recall fallback to lexical due to error', err);
+            }
+        }
+        // Default: Adapt store SearchHit -> agent-core MemoryHit { id, content, score }.
+        return lexicalHits.map((h) => h.item);
+    }
+    async search(query, opts) {
+        const limit = Math.min(Math.max(Math.floor(opts?.limit ?? 20), 1), 100);
+        const hits = searchForPrincipal(this.store, this.principalId, { query, limit });
+        // Adapt store SearchHit -> protocol MemoryHit { id, excerpt, score }.
+        return hits.map((hit) => ({
+            id: hit.record.id,
+            excerpt: hit.snippet,
+            score: hit.score,
+        }));
+    }
+    async note(text, meta, scope) {
+        const now = new Date().toISOString();
+        const record = this.store.create({
+            scope: scope ?? principalScope(this.principalId),
+            text,
+            provenance: {
+                source: "model",
+                reference: "memory.note",
+                actorId: this.principalId,
+                at: now,
+            },
+            actorId: this.principalId,
+        });
+        // The store schema has no metadata column: meta is preserved in the
+        // audit trail (never in recall content) until the schema grows one.
+        if (meta !== undefined && Object.keys(meta).length > 0) {
+            this.store.insertAudit(randomUUID(), record.id, "create", this.principalId, now, `meta:${JSON.stringify(meta).slice(0, 2000)}`);
+        }
+        // Pin policy (P0-3): pin ONLY when the caller explicitly asks
+        // (`meta.pinned === 'true'`) or when the record lives in the durable
+        // global scope. Everything else stays unpinned so the TTL curator can
+        // archive stale records — pin-by-default had made curation dead code.
+        // An explicit `pinned: 'false'` always wins, even on the global scope.
+        const pinnedExplicitly = meta?.pinned === 'true';
+        const pinnedByScope = meta?.pinned !== 'false' && record.scope.kind === 'global';
+        if (pinnedExplicitly || pinnedByScope) {
+            this.store.pin(record.id, record.scope);
+        }
+        // Index vector in background for Hybrid RAG semantic recall
+        if (this.resolveEmbedding) {
+            this.resolveEmbedding()
+                .then(async (client) => {
+                if (!client)
+                    return;
+                const vectors = await client.embed([text]);
+                const vec = vectors[0];
+                if (vec) {
+                    const scopeId = typeof record.scope === 'object' && record.scope !== null && 'kind' in record.scope
+                        ? `${record.scope.kind}:${record.scope.id ?? ''}`
+                        : String(record.scope);
+                    this.vectorIndex.upsert({
+                        id: record.id,
+                        scopeId,
+                        vector: new Float32Array(vec),
+                        metadata: { snippet: text.slice(0, DEFAULT_RECALL_CHARS_PER_HIT) },
+                    });
+                }
+            })
+                .catch((err) => {
+                this.log.debug?.('[memory] vector indexing failed for note', err);
+            });
+        }
+        return { id: record.id };
+    }
+    /** Diagnostics for health checks / tests. Not part of the MemoryService contract. */
+    curationStatus() {
+        return { runs: this.curationRuns, lastRun: this.lastCuration };
+    }
+    /** Start the background curation timer. Idempotent. */
+    startCuration() {
+        if (this.curationTimer !== undefined)
+            return;
+        this.curationTimer = setInterval(() => {
+            this.runCurationOnce();
+        }, this.curationIntervalMs);
+        // Never hold the process open for curation.
+        if (typeof this.curationTimer.unref === "function")
+            this.curationTimer.unref();
+    }
+    /** Drive one curation run now (used by the timer and by tests). */
+    runCurationOnce() {
+        try {
+            const report = this.curator.curate(this.curationProvider, {
+                maxAgeDays: this.curationMaxAgeDays,
+                dryRun: this.curationDryRun,
+                scopes: [GLOBAL_SCOPE, principalScope(this.principalId)],
+            });
+            this.curationRuns += 1;
+            this.lastCuration = {
+                at: report.at,
+                scanned: report.scanned,
+                archived: report.archived.length,
+            };
+            // Persist job state in the store (audit trail, stable sentinel id).
+            this.store.insertAudit(randomUUID(), CURATION_STATE_ID, "reindex", `${this.principalId}:curator`, report.at, JSON.stringify({
+                scanned: report.scanned,
+                kept: report.kept,
+                archived: report.archived.length,
+                skipped: report.skipped,
+                dryRun: report.dryRun,
+            }).slice(0, 2000));
+            this.log.info(`[memory] curation run #${this.curationRuns}: scanned=${report.scanned} archived=${report.archived.length}${report.dryRun ? " (dry-run)" : ""}`);
+        }
+        catch (error) {
+            // Curation must never break request handling — log and continue.
+            this.log.error("[memory] curation run failed; continuing", error);
+        }
+    }
+    /** Diagnostics for health checks / tests. Not part of the MemoryService contract. */
+    consolidationStatus() {
+        return { runs: this.consolidationRuns, lastRun: this.lastConsolidation };
+    }
+    /** Start the background consolidation timer. Idempotent; no-op when disabled. */
+    startConsolidation() {
+        this.consolidationJob?.start();
+    }
+    /** Drive one consolidation run now (used by the timer and by tests). */
+    async runConsolidationOnce() {
+        const job = this.consolidationJob;
+        if (job === undefined) {
+            this.log.warn("[memory] consolidation skipped: no session store or LLM resolver wired");
+            return undefined;
+        }
+        try {
+            const report = await job.runOnce();
+            this.consolidationRuns += 1;
+            this.lastConsolidation = {
+                at: report.at,
+                processed: report.processed,
+                factsCreated: report.factsCreated,
+                stopReason: report.stopReason,
+            };
+            this.log.info(`[memory] consolidation run #${this.consolidationRuns}: processed=${report.processed} facts=${report.factsCreated} stop=${report.stopReason}`);
+            return report;
+        }
+        catch (error) {
+            // Consolidation must never break request handling — log and continue.
+            this.log.error("[memory] consolidation run failed; continuing", error);
+            return undefined;
+        }
+    }
+    /**
+     * Conversation source over the gateway session store: sessions updated
+     * after the watermark cursor, oldest first. Facts are written to the
+     * principal scope — durable facts are cross-session by definition.
+     */
+    buildConversationSource(sessions, principalId) {
+        return {
+            listEndedAfter: async (watermark, limit) => {
+                const all = sessions.list();
+                const after = watermark === null ? all : all.filter((s) => s.updatedAt > watermark);
+                after.sort((a, b) => (a.updatedAt < b.updatedAt ? -1 : a.updatedAt > b.updatedAt ? 1 : 0));
+                return after
+                    .slice(0, Math.max(0, Math.floor(limit)))
+                    .map((session) => ({
+                    id: session.id,
+                    principalId,
+                    scope: principalScope(principalId),
+                    transcript: flattenTranscript(sessions.getMessages(session.id)),
+                    endedAt: session.updatedAt,
+                }));
+            },
+        };
+    }
+    /**
+     * Watermark store over the memory store's audit trail, mirroring the
+     * `memory:curation` convention: run state is an append-only entry under a
+     * sentinel id, never a separate KV table. The cursor is the maximum
+     * `cursor` value seen, so concurrent writers cannot move it backwards.
+     */
+    buildWatermarkStore() {
+        const store = this.store;
+        const actorId = `${this.principalId}:consolidation`;
+        return {
+            get: (key) => {
+                let cursor = null;
+                for (const entry of store.auditFor(key)) {
+                    let parsed;
+                    try {
+                        parsed = JSON.parse(entry.detail);
+                    }
+                    catch {
+                        continue;
+                    }
+                    if (parsed !== null && typeof parsed === "object") {
+                        const candidate = parsed.cursor;
+                        if (typeof candidate === "string" && (cursor === null || candidate > cursor)) {
+                            cursor = candidate;
+                        }
+                    }
+                }
+                return cursor;
+            },
+            set: (key, value) => {
+                store.insertAudit(randomUUID(), key, "reindex", actorId, new Date().toISOString(), JSON.stringify({ cursor: value }).slice(0, 2000));
+            },
+        };
+    }
+    stop() {
+        if (this.curationTimer !== undefined) {
+            clearInterval(this.curationTimer);
+            this.curationTimer = undefined;
+        }
+        this.consolidationJob?.stop();
+        this.consolidationProvider?.close();
+        this.curationProvider.close();
+        this.store.close();
+    }
+}
+/**
+ * Start the memory service: open the FTS5 store at
+ * `${ctx.dataDir}/memory.sqlite`, attach it as `ctx.memoryService`, and
+ * start the background curation + consolidation timers.
+ *
+ * The context may additionally carry `sessions` and `resolveLlm` (see
+ * `MemoryServiceContext`); without them consolidation stays disabled while
+ * curation keeps working.
+ *
+ * Returns an async stop function that halts both timers and closes the store.
+ * Throws on startup failure (unreadable dataDir, corrupt DB) — the gateway
+ * must not boot with a half-wired memory service.
+ */
+export async function startMemoryService(ctx, options = {}) {
+    const dbPath = join(ctx.dataDir, "memory.sqlite");
+    const service = new GatewayMemoryService(dbPath, ctx.log, {
+        ...options,
+        sessions: options.sessions ?? ctx.sessions,
+        resolveLlm: options.resolveLlm ?? ctx.resolveLlm,
+        resolveEmbedding: options.resolveEmbedding ?? ctx.resolveEmbedding,
+    });
+    service.startCuration();
+    service.startConsolidation();
+    ctx.memoryService = service;
+    ctx.log.info(`[memory] service started (db=${dbPath})`);
+    return async () => {
+        service.stop();
+        ctx.log.info("[memory] service stopped");
+    };
+}