@unblocklabs/unblock-memory 0.7.0 → 0.7.2

package/dist/src/analysis.js

@@ -200,22 +200,21 @@ export function latestAnalysisCollections(db) {
         return undefined;
     }
 }
-function count(db, sql, runId) {
-    return db.prepare(sql).get(runId)?.count ?? 0;
-}
 function analysisSummary(db, run) {
-    const clusters = count(db, "SELECT COUNT(*) AS count FROM memory_analysis_clusters WHERE run_id = ?", run.id);
-    const members = count(db, "SELECT COUNT(*) AS count FROM memory_analysis_memberships WHERE run_id = ?", run.id);
-    const expectedNonNoise = count(db, "SELECT COALESCE(SUM(size), 0) AS count FROM memory_analysis_clusters WHERE run_id = ?", run.id);
-    const nonNoise = count(db, "SELECT COUNT(*) AS count FROM memory_analysis_memberships WHERE run_id = ? AND cluster_id <> -1", run.id);
-    const noise = count(db, "SELECT COUNT(*) AS count FROM memory_analysis_memberships WHERE run_id = ? AND cluster_id = -1", run.id);
-    const unassigned = count(db, `
-    SELECT COUNT(*) AS count
+    const { clusters, expectedNonNoise } = db.prepare(`
+    SELECT COUNT(*) AS clusters, COALESCE(SUM(size), 0) AS expectedNonNoise
+    FROM memory_analysis_clusters WHERE run_id = ?
+  `).get(run.id);
+    const { members, nonNoise, noise, unassigned } = db.prepare(`
+    SELECT COUNT(*) AS members,
+      COALESCE(SUM(m.cluster_id <> -1), 0) AS nonNoise,
+      COALESCE(SUM(m.cluster_id = -1), 0) AS noise,
+      COALESCE(SUM(m.cluster_id <> -1 AND c.cluster_id IS NULL), 0) AS unassigned
     FROM memory_analysis_memberships m
     LEFT JOIN memory_analysis_clusters c
       ON c.run_id = m.run_id AND c.cluster_id = m.cluster_id
-    WHERE m.run_id = ? AND m.cluster_id <> -1 AND c.cluster_id IS NULL
-  `, run.id);
+    WHERE m.run_id = ?
+  `).get(run.id);
     if (nonNoise !== expectedNonNoise || members !== nonNoise + noise || unassigned > 0)
         return undefined;
     return {

package/dist/src/contracts.d.ts

@@ -29,7 +29,7 @@ export type SessionSearchFilter = {
     conversationId?: string;
 };
 export type MemoryRequestContext = Pick<OpenClawPluginToolContext, "sessionKey" | "sessionId" | "messageChannel" | "agentAccountId" | "nativeChannelId" | "deliveryContext">;
-export type CorpusSearchOptions = NonNullable<Parameters<MemorySearchManagerContract["search"]>[1]> & {
+export type CorpusSearchOptions = Omit<NonNullable<Parameters<MemorySearchManagerContract["search"]>[1]>, "lexicalOnly"> & {
     corpora?: readonly string[];
     /** Internal vector-hint budget; oversized matched chunks are omitted, never sliced. */
     maxSnippetChars?: number;

package/dist/src/extraction-chunk.d.ts

@@ -25,5 +25,6 @@ export declare function readExtractionChunk(params: {
     newIds: string[];
     entryCount: number;
     fence: string;
+    exhausted: boolean;
     cursor: string;
 }>;

package/dist/src/extraction-chunk.js

@@ -98,6 +98,7 @@ export async function readExtractionChunk(params) {
         const finish = (index, offset = 0) => {
             const next = page.messages[index];
             return { kind: "page", messages, newIds, entryCount, fence: startingFence ?? page.cursor,
+                exhausted: !next && !page.hasMore,
                 cursor: next ? PREFIX + JSON.stringify({ before, fence: page.cursor, index, offset, id: next.id, hash: digest(next) }) : page.cursor };
         };
         for (let i = resume?.index ?? 0; i < page.messages.length; i++) {

package/dist/src/extraction-model.js

@@ -2,7 +2,7 @@ import { Type } from "typebox";
 import { Value } from "typebox/value";
 import { Tiktoken } from "js-tiktoken/lite";
 import o200kBase from "js-tiktoken/ranks/o200k_base";
-import { requestTypeSafe } from "./typesafe-client.js";
+import { requestTypeSafe, TYPESAFE_NOUL_SCHEMA as noul } from "./typesafe-client.js";
 export const EXTRACTION_VERSION = "lasting-facts-v11";
 // Jev allows 32k for state plus its longest question. Leave room for validation
 // instructions, proposal evidence and tokenizer differences, not just Luna input.
@@ -34,6 +34,7 @@ Only extract facts supported by at least one NEW message. Context-only messages
 If a fact is already in existing memories, omit it. Source dates control chronology, not processing order: during backfill, never replace a newer observed fact with an older one, or emit the older value as an undated duplicate. If new evidence clearly corrects/changes an existing memory, set replaces to its exact id and emit the concise replacement. Otherwise replaces is null. Never replace unrelated facts. If a previous claim is explicitly withdrawn without a replacement, retain a concise qualified correction rather than inventing a value.
 Cite exact contiguous substrings copied from the supplied messages using their ids. Preserve Markdown markers, mentions, whitespace and punctuation inside each quote. Never join separated spans into one quote or clean up its formatting. Prefer short exact spans; use separate evidence entries when needed. Include enough evidence to resolve corrections and speaker attribution. Output an empty memories array when nothing qualifies. Do not fill a quota.`;
 let tokenizer;
+let promptTokens;
 // o200k_base is an explicit budgeting proxy, not a claim about the host's private tokenizer.
 function extractionTokens(text) {
     return (tokenizer ??= new Tiktoken(o200kBase)).encode(text, [], []).length;
@@ -45,7 +46,7 @@ function extractionInput(messages, newIds, existing) {
     return JSON.stringify({ messages: messages.map(datedMessage), newMessageIds: newIds, existing, schema: proposalSchema });
 }
 export function extractionOverhead(existing) {
-    return extractionTokens(EXTRACTION_PROMPT) + extractionTokens(extractionInput([], [], existing)) + 512;
+    return (promptTokens ??= extractionTokens(EXTRACTION_PROMPT)) + extractionTokens(extractionInput([], [], existing)) + 512;
 }
 export function extractionMessageTokens(message) {
     return extractionTokens(JSON.stringify(datedMessage(message))) + extractionTokens(JSON.stringify(message.id)) + 8;
@@ -84,8 +85,8 @@ export async function extractWithLuna(runtime, agentId, messages, newIds, existi
 const judgments = Type.Object({ answers: Type.Object({
         supported: Type.Object({ type: Type.Literal("choice"), choice: Type.Union([Type.Literal("supported"), Type.Literal("contradicted"), Type.Literal("unsupported")]),
             probabilities: Type.Object({ supported: Type.Number({ minimum: 0, maximum: 1 }), contradicted: Type.Number({ minimum: 0, maximum: 1 }), unsupported: Type.Number({ minimum: 0, maximum: 1 }) }) }),
-        useful: Type.Object({ type: Type.Literal("noul"), noul: Type.Number({ minimum: 0, maximum: 1 }) }),
-        replacement: Type.Optional(Type.Object({ type: Type.Literal("noul"), noul: Type.Number({ minimum: 0, maximum: 1 }) })),
+        useful: noul,
+        replacement: Type.Optional(noul),
     }) });
 export async function validateExtractedMemory(params) {
     const { proposal, messages, existing } = params;

package/dist/src/extraction-runtime.js

@@ -7,6 +7,7 @@ import { resolveAgentDir, resolveStateDir } from "openclaw/plugin-sdk/memory-cor
 import { MEMORY_DATABASE, hasMemoryTable } from "./memory-database.js";
 import { ExtractionStore } from "./extraction-store.js";
 import { extractionSessions } from "./extraction-source.js";
+import { dateOption } from "./date-option.js";
 export function registerExtraction(api, config, runtime) {
     let lifetime = new AbortController();
     let timer, running;
@@ -36,13 +37,7 @@ export function registerExtraction(api, config, runtime) {
             .option("--since <date>", "Explicit bounded backfill from YYYY-MM-DD UTC; repeat to resume")
             .option("--session <id>", "Restrict this run to one session")
             .action(async (opts) => {
-            let since;
-            if (opts.since !== undefined) {
-                since = Date.parse(opts.since);
-                if (!/^\d{4}-\d{2}-\d{2}$/.test(opts.since) || !Number.isFinite(since) || new Date(since).toISOString().slice(0, 10) !== opts.since) {
-                    throw new Error("since must be a valid YYYY-MM-DD UTC date");
-                }
-            }
+            const since = opts.since === undefined ? undefined : dateOption(opts.since, 0);
             try {
                 // Native harnesses live in the Gateway, not the cold CLI metadata loader.
                 const result = await callGatewayFromCli("unblock-memory.extract", { timeout: "600000" }, { agentId: opts.agent, ...(since === undefined ? {} : { since }), ...(opts.session ? { sessionId: opts.session } : {}) });

package/dist/src/extraction-source.js

@@ -11,13 +11,17 @@ export function extractionSessions(databasePath, agentId, chatTypes) {
         }
         return db.prepare(`SELECT w.session_id,w.session_key,w.chat_type,COALESCE(w.channel,c.channel) AS provider,
       COALESCE(w.account_id,c.account_id) AS account_id,COALESCE(c.native_channel_id,c.native_direct_user_id,c.peer_id,w.primary_conversation_id) AS conversation_id,
-      COALESCE(w.started_at,w.created_at) AS started_at, COALESCE(w.transcript_updated_at,w.updated_at,w.created_at) AS changed_at FROM session_windows w
+      COALESCE(w.started_at,w.created_at) AS started_at, COALESCE(w.transcript_updated_at,w.updated_at,w.created_at) AS changed_at,
+      r.generation,(SELECT MAX(seq) FROM transcript_events WHERE session_id=w.session_id) AS tail FROM session_windows w
       LEFT JOIN conversations c ON c.conversation_id=w.primary_conversation_id
+      LEFT JOIN transcript_rewrite_watermarks r ON r.session_id=w.session_id
       WHERE w.chat_type IN (${chatTypes.map(() => "?").join(",")}) ORDER BY w.created_at,w.session_id`)
             .all(...chatTypes).map(r => ({ sessionId: String(r.session_id), sessionKey: String(r.session_key),
             chatType: r.chat_type, provider: r.provider === null ? undefined : String(r.provider),
             accountId: r.account_id === null ? undefined : String(r.account_id),
-            conversationId: r.conversation_id === null ? undefined : String(r.conversation_id), startedAt: Number(r.started_at), changedAt: Number(r.changed_at) }));
+            conversationId: r.conversation_id === null ? undefined : String(r.conversation_id), startedAt: Number(r.started_at), changedAt: Number(r.changed_at),
+            // The host uses this append-stable watermark for transcript cache validation.
+            sourceRevision: r.generation === null ? undefined : JSON.stringify([r.generation, r.tail]) }));
     }
     finally {
         db.close();

package/dist/src/extraction-store.d.ts

@@ -3,11 +3,13 @@ import type { SessionMetadata } from "./session-projector.js";
 export type ExtractionSession = SessionMetadata & {
     sessionKey: string;
     changedAt?: number;
+    sourceRevision?: string;
 };
 type Checkpoint = {
     cursor: string | null;
     context: ExtractionMessage[];
     since: number;
+    completeRevision: string | null;
 };
 export type ExtractedRecord = {
     id: string;
@@ -42,6 +44,7 @@ export declare class ExtractionStore {
         }[];
         owner: string;
         version: string;
+        completeRevision?: string;
     }): number;
     error(sessionId: string, code: string): void;
     report(): {

package/dist/src/extraction-store.js

@@ -20,7 +20,7 @@ export class ExtractionStore {
         session_id TEXT PRIMARY KEY, metadata TEXT NOT NULL CHECK(json_valid(metadata)), cursor TEXT,
         extract_since INTEGER NOT NULL,
         context TEXT NOT NULL DEFAULT '[]' CHECK(json_valid(context)), updated_at INTEGER NOT NULL,
-        error TEXT
+        error TEXT, complete_revision TEXT
       ) STRICT;
       CREATE TABLE IF NOT EXISTS extracted_memories (
         id TEXT NOT NULL, revision INTEGER NOT NULL, session_id TEXT NOT NULL REFERENCES extraction_sessions(session_id),
@@ -33,6 +33,9 @@ export class ExtractionStore {
       CREATE UNIQUE INDEX IF NOT EXISTS extracted_duplicate ON extracted_memories(session_id,digest) WHERE status='active';
       INSERT OR IGNORE INTO memory_schema VALUES('extraction',1);
     `);
+            if (!this.db.prepare("SELECT 1 FROM pragma_table_info('extraction_sessions') WHERE name='complete_revision'").get()) {
+                this.db.exec("ALTER TABLE extraction_sessions ADD COLUMN complete_revision TEXT");
+            }
             this.db.prepare("INSERT OR IGNORE INTO extraction_worker(id,live_since) VALUES(1,?)").run(Date.now());
         }
         catch (error) {
@@ -58,23 +61,24 @@ export class ExtractionStore {
     checkpoint(session, since) {
         this.db.prepare(`INSERT INTO extraction_sessions(session_id,metadata,updated_at,extract_since) VALUES(?,?,?,?)
       ON CONFLICT(session_id) DO UPDATE SET metadata=excluded.metadata`).run(session.sessionId, JSON.stringify(session), Date.now(), since ?? this.liveSince());
-        const row = this.db.prepare("SELECT cursor,context,extract_since FROM extraction_sessions WHERE session_id=?").get(session.sessionId);
+        const row = this.db.prepare("SELECT cursor,context,extract_since,complete_revision FROM extraction_sessions WHERE session_id=?").get(session.sessionId);
         if (since !== undefined && since < Number(row.extract_since)) {
             // A broader explicitly approved backfill replays this session. Existing facts are
             // supplied to Luna and exact duplicates are ignored; history is never deleted.
-            this.db.prepare("UPDATE extraction_sessions SET cursor=NULL,context='[]',extract_since=? WHERE session_id=?").run(since, session.sessionId);
-            return { cursor: null, context: [], since };
+            this.db.prepare("UPDATE extraction_sessions SET cursor=NULL,context='[]',complete_revision=NULL,extract_since=? WHERE session_id=?").run(since, session.sessionId);
+            return { cursor: null, context: [], since, completeRevision: null };
         }
-        return { cursor: row.cursor, context: JSON.parse(String(row.context)), since: Number(row.extract_since) };
+        return { cursor: row.cursor, context: JSON.parse(String(row.context)), since: Number(row.extract_since),
+            completeRevision: row.complete_revision };
     }
     records(sessionId, paths) {
         const keys = paths?.filter(path => /^[a-f0-9-]+\/[1-9]\d*$/.test(path)).map(path => path.split("/"));
         if (keys && !keys.length)
             return [];
         return this.db.prepare(`SELECT m.*,s.metadata FROM extracted_memories m JOIN extraction_sessions s USING(session_id)
-      WHERE status='active' AND (? IS NULL OR session_id=?)
+      WHERE status='active' ${sessionId === undefined ? "" : "AND session_id=?"}
       ${keys ? `AND (m.id,m.revision) IN (VALUES ${keys.map(() => "(?,?)").join(",")})` : ""}
-      ORDER BY observed_at,id`).all(sessionId ?? null, sessionId ?? null, ...(keys?.flat() ?? []))
+      ORDER BY observed_at,id`).all(...(sessionId === undefined ? [] : [sessionId]), ...(keys?.flat() ?? []))
             .map(r => ({ id: String(r.id), revision: Number(r.revision), text: String(r.text), sessionId: String(r.session_id),
             observedAt: Number(r.observed_at), evidence: JSON.parse(String(r.evidence)), metadata: JSON.parse(String(r.metadata)), judgment: JSON.parse(String(r.judgment)) }));
     }
@@ -83,7 +87,7 @@ export class ExtractionStore {
         try {
             this.renew(owner);
             this.db.prepare("UPDATE extracted_memories SET status='withdrawn' WHERE session_id=? AND status='active'").run(sessionId);
-            this.db.prepare("UPDATE extraction_sessions SET cursor=?,context='[]',error=NULL WHERE session_id=?").run(cursor, sessionId);
+            this.db.prepare("UPDATE extraction_sessions SET cursor=?,context='[]',complete_revision=NULL,error=NULL WHERE session_id=?").run(cursor, sessionId);
             this.db.exec("COMMIT");
         }
         catch (error) {
@@ -119,8 +123,8 @@ export class ExtractionStore {
                     .run(id, revision, params.session.sessionId, proposal.text.trim(), digest, JSON.stringify(proposal.evidence), observedAt, JSON.stringify(judgment), params.version, Date.now());
                 written++;
             }
-            this.db.prepare("UPDATE extraction_sessions SET cursor=?,context=?,updated_at=?,error=NULL WHERE session_id=?")
-                .run(params.cursor, JSON.stringify(params.context), Date.now(), params.session.sessionId);
+            this.db.prepare("UPDATE extraction_sessions SET cursor=?,context=?,updated_at=?,complete_revision=?,error=NULL WHERE session_id=?")
+                .run(params.cursor, JSON.stringify(params.context), Date.now(), params.completeRevision ?? null, params.session.sessionId);
             this.db.exec("COMMIT");
             return written;
         }

package/dist/src/extraction-worker.js

@@ -32,9 +32,10 @@ export async function runExtraction(params) {
         const visible = new Set(sessions.map(s => s.sessionId));
         if (params.sessionId && !visible.has(params.sessionId))
             throw new Error("Unknown or unapproved extraction session");
-        for (const memory of store.records())
-            if (!visible.has(memory.sessionId))
-                store.reset(memory.sessionId, null, owner);
+        for (const { session_id } of store.db.prepare("SELECT DISTINCT session_id FROM extracted_memories WHERE status='active'").all()) {
+            if (!visible.has(String(session_id)))
+                store.reset(String(session_id), null, owner);
+        }
         // Oldest last-checked first: bounded runs must not starve later sessions.
         const checked = new Map(store.db.prepare("SELECT session_id,updated_at FROM extraction_sessions").all()
             .map(r => [String(r.session_id), Number(r.updated_at)]));
@@ -48,6 +49,10 @@ export async function runExtraction(params) {
             if (processed >= config.maxBatches)
                 break;
             const checkpoint = store.checkpoint(session, params.since);
+            if (session.sourceRevision !== undefined && checkpoint.completeRevision === session.sourceRevision) {
+                unchanged++;
+                continue;
+            }
             let stage = "read";
             try {
                 const existing = store.records(session.sessionId);
@@ -69,7 +74,8 @@ export async function runExtraction(params) {
                     continue;
                 }
                 if (!page.entryCount) {
-                    store.db.prepare("UPDATE extraction_sessions SET updated_at=? WHERE session_id=?").run(Date.now(), session.sessionId);
+                    store.db.prepare("UPDATE extraction_sessions SET updated_at=?,complete_revision=? WHERE session_id=?")
+                        .run(Date.now(), page.exhausted ? session.sourceRevision ?? null : null, session.sessionId);
                     unchanged++;
                     continue;
                 }
@@ -82,17 +88,26 @@ export async function runExtraction(params) {
                 const proposals = newIds.length ? await (params.extract ?? extractWithLuna)(params.runtime, params.agentId, messages, newIds, existing.map(m => ({ id: m.id, text: m.text, observedAt: m.observedAt })), signal) : [];
                 const accepted = [];
                 stage = "validate";
-                for (const proposal of proposals) {
+                for (let start = 0; start < proposals.length; start += 4) {
                     signal.throwIfAborted();
-                    const judgment = await (params.validate ?? validateExtractedMemory)({ proposal, messages, newIds,
-                        existing, apiKey, signal, thresholds: config });
-                    const observedAt = judgment.accepted ? Math.max(...proposal.evidence.map(e => messages.find(m => m.id === e.messageId).timestamp)) : 0;
-                    const prior = existing.find(m => m.id === proposal.replaces);
-                    if (judgment.accepted && (!prior || prior.observedAt <= observedAt))
-                        accepted.push({ proposal: { ...proposal,
-                                evidence: proposal.evidence.map(e => ({ ...e, messageId: messages.find(m => m.id === e.messageId).sourceMessageId ?? e.messageId })) }, judgment, observedAt });
-                    else
-                        rejectedCount++;
+                    // Isolated judgments overlap, but failed chunks drain before their lease is released.
+                    const judgments = await Promise.allSettled(proposals.slice(start, start + 4).map(async (proposal) => ({ proposal,
+                        judgment: await (params.validate ?? validateExtractedMemory)({ proposal, messages, newIds, existing, apiKey, signal, thresholds: config }) })));
+                    const failure = judgments.find(result => result.status === "rejected");
+                    if (failure)
+                        throw failure.reason;
+                    for (const result of judgments) {
+                        if (result.status !== "fulfilled")
+                            continue;
+                        const { proposal, judgment } = result.value;
+                        const observedAt = judgment.accepted ? Math.max(...proposal.evidence.map(e => messages.find(m => m.id === e.messageId).timestamp)) : 0;
+                        const prior = existing.find(m => m.id === proposal.replaces);
+                        if (judgment.accepted && (!prior || prior.observedAt <= observedAt))
+                            accepted.push({ proposal: { ...proposal,
+                                    evidence: proposal.evidence.map(e => ({ ...e, messageId: messages.find(m => m.id === e.messageId).sourceMessageId ?? e.messageId })) }, judgment, observedAt });
+                        else
+                            rejectedCount++;
+                    }
                 }
                 // A branch rewrite during inference invalidates its proposals. Appends are safe and processed next time.
                 const check = await readPage(params.agentId, params.agentName, session, page.fence);
@@ -107,7 +122,8 @@ export async function runExtraction(params) {
                 const context = extractionHistory(messages, config.historyMessages);
                 stage = "commit";
                 const written = store.commit({ session, expected: checkpoint.cursor, cursor: page.cursor,
-                    context, accepted, owner, version: EXTRACTION_VERSION });
+                    context, accepted, owner, version: EXTRACTION_VERSION,
+                    completeRevision: page.exhausted ? session.sourceRevision : undefined });
                 processed++;
                 acceptedCount += written;
             }

package/dist/src/manager.d.ts

@@ -12,7 +12,7 @@ import { reviewIndexedClaim } from "./evidence-review.js";
 import { reviewClusterIngestion } from "./cluster-review.js";
 import type { QueryPair } from "./query-contract.js";
 export { expandSessionSearchHit } from "./memory-passage.js";
-export type ManagerStore = Pick<QMDStore, "update" | "embed" | "getStatus" | "listCollections" | "searchLex" | "vsearch" | "get" | "getDocumentBody" | "close">;
+export type ManagerStore = Pick<QMDStore, "update" | "embed" | "getStatus" | "listCollections" | "vsearch" | "get" | "getDocumentBody" | "close">;
 export type ManagerSessionConfig = {
     agentId: string;
     agentName: string;

package/dist/src/manager.js

@@ -74,16 +74,6 @@ function frontmatterValue(body, key) {
         .find((value) => value !== undefined);
     return raw?.replace(/^(?:"(.*)"|'(.*)')$/u, "$1$2").trim();
 }
-function embeddingText(name, description, model) {
-    return model.toLowerCase().includes("qwen3-embedding")
-        ? `${name}\n${description}`
-        : `title: ${name} | text: ${description}`;
-}
-function queryText(query, model) {
-    return model.toLowerCase().includes("qwen3-embedding")
-        ? `Instruct: Retrieve relevant documents for the given query\nQuery: ${query}`
-        : `task: search result | query: ${query}`;
-}
 function cosineSimilarity(left, right) {
     if (left.length !== right.length || left.length === 0)
         return 0;
@@ -192,22 +182,6 @@ function lineSpan(body, position, text) {
     const endLine = startLine + Math.max(0, text.split("\n").length - 1);
     return { startLine, endLine };
 }
-function lexicalResult(hit, corpus, session) {
-    const body = hit.body ?? hit.title;
-    const endLine = Math.max(1, body.split("\n").length);
-    return {
-        path: hit.filepath,
-        startLine: 1,
-        endLine,
-        score: hit.score,
-        textScore: hit.score,
-        snippet: body,
-        source: "memory",
-        corpus,
-        ...(session ? { session } : {}),
-        citation: `${hit.displayPath}#L1-L${endLine}`,
-    };
-}
 function sessionAllowedPaths(metadataByPath, collection, filter) {
     const startedFrom = filter.startedFrom === undefined ? undefined : Date.parse(filter.startedFrom);
     const startedTo = filter.startedTo === undefined ? undefined : Date.parse(filter.startedTo);
@@ -251,6 +225,7 @@ export class QmdMemoryManager {
     #watcher;
     #watchReady;
     #watchTimer;
+    #watchCollections = new Set();
     #watchError;
     #closed = false;
     #files = 0;
@@ -360,12 +335,17 @@ export class QmdMemoryManager {
                 this.#skillIndex = undefined;
             if (!matchingSources.some((source) => source.kind !== "skills"))
                 return;
+            for (const source of matchingSources)
+                if (source.kind !== "skills")
+                    this.#watchCollections.add(source.collection);
             this.#dirty = true;
             if (this.#watchTimer)
                 clearTimeout(this.#watchTimer);
             this.#watchTimer = setTimeout(() => {
                 this.#watchTimer = undefined;
-                void this.sync({ reason: "watch" }).catch(() => undefined);
+                const collections = [...this.#watchCollections];
+                this.#watchCollections.clear();
+                void this.#sync({ reason: "watch" }, collections).catch(() => undefined);
             }, WATCH_DEBOUNCE_MS);
         });
     }
@@ -489,23 +469,36 @@ export class QmdMemoryManager {
         const allowed = new Set(sessionAllowedPaths(new Map(records.map(r => [extractedPath(r), r.metadata])), EXTRACTED_COLLECTION, filter)[EXTRACTED_COLLECTION]);
         return records.filter(r => allowed.has(extractedPath(r)));
     }
-    async #updateAndEmbed(store, collection, force) {
+    async #updateAndEmbed(store, collections, force) {
         // Extracted facts are DB projections, not files: never run filesystem sync on them.
-        const update = await store.update({ collections: collection ? [collection] : this.#qmdSources().map(source => source.collection) });
-        this.#cleanupRemovedDocuments?.(update.updated + update.removed);
         const analysisStore = store;
-        const changed = update.indexed + update.updated + update.removed > 0 || update.needsEmbedding > 0;
-        if ((changed || force) && analysisStore.internal)
-            markMemoryAnalysisStale(analysisStore.internal.db);
-        const embed = await store.embed({
-            ...(collection ? { collection } : {}),
-            force,
-            chunkStrategy: "semantic",
-        });
-        this.#recordEmbedding(embed);
-        const chunksEmbedded = completedEmbeddingCount(embed);
-        if (!changed && !force && chunksEmbedded > 0 && analysisStore.internal) {
-            markMemoryAnalysisStale(analysisStore.internal.db);
+        let changedDocuments = 0;
+        try {
+            const update = await store.update({ collections: collections ? [...collections] : this.#qmdSources().map(source => source.collection) });
+            changedDocuments = update.updated + update.removed;
+            if ((update.indexed + changedDocuments > 0 || update.needsEmbedding > 0 || force) && analysisStore.internal) {
+                markMemoryAnalysisStale(analysisStore.internal.db);
+            }
+        }
+        catch (error) {
+            // QMD may commit earlier collections before a later collection fails.
+            if (analysisStore.internal)
+                markMemoryAnalysisStale(analysisStore.internal.db);
+            changedDocuments = Math.max(1, changedDocuments);
+            throw error;
+        }
+        finally {
+            // A later collection failing must not leave earlier removed plaintext behind.
+            this.#cleanupRemovedDocuments?.(changedDocuments);
+        }
+        let chunksEmbedded = 0;
+        for (const collection of collections?.length ? collections : [undefined]) {
+            const embed = await store.embed({ ...(collection ? { collection } : {}), force, chunkStrategy: "semantic" });
+            this.#recordEmbedding(embed);
+            const count = completedEmbeddingCount(embed);
+            chunksEmbedded += count;
+            if (count > 0 && analysisStore.internal)
+                markMemoryAnalysisStale(analysisStore.internal.db);
         }
         return chunksEmbedded;
     }
@@ -516,16 +509,14 @@ export class QmdMemoryManager {
         this.#dirty = status.needsEmbedding > 0;
     }
     sync(params) {
+        return this.#sync(params);
+    }
+    #sync(params, affectedCollections) {
         const run = async () => {
             const store = await this.#getStore();
             this.#dirty = true;
-            const collections = this.#qmdSources().filter((source) => source.kind !== "sessions");
-            if (collections.length === 0) {
-                await this.#updateAndEmbed(store, undefined, params?.force);
-            }
-            for (const source of collections) {
-                await this.#updateAndEmbed(store, source.collection, params?.force);
-            }
+            const collections = affectedCollections ?? this.#qmdSources().filter((source) => source.kind !== "sessions").map(source => source.collection);
+            await this.#updateAndEmbed(store, collections.length ? collections : undefined, params?.force);
             await this.#refreshIndexStatus(store);
         };
         return this.#enqueue(run);
@@ -544,7 +535,7 @@ export class QmdMemoryManager {
                 index: async () => {
                     onPhase?.("indexing");
                     const store = await this.#getStore();
-                    return this.#updateAndEmbed(store, sessions.collection);
+                    return this.#updateAndEmbed(store, [sessions.collection]);
                 },
             });
             this.#sessionMetadata = sessionMetadataByPath(synced.manifest);
@@ -699,18 +690,28 @@ export class QmdMemoryManager {
       VALUES (?, ?, ?, ?, ?, ?, ?)
     `);
         const curation = this.#getCuration();
+        const fingerprints = new Map();
         for (const annotation of curation.annotations()) {
             if (!annotation.contentFingerprint) {
                 insert.run(annotation.collection, annotation.path, null, null, annotation.eventTime, annotation.basis, 1);
                 continue;
             }
-            const rows = findChunks.all(annotation.collection, annotation.path);
-            const matched = rows.find((row) => chunkFingerprint(row.doc.slice(row.pos, row.pos + row.chunk_len)) === annotation.contentFingerprint);
-            curation.updateAnnotationLocation({
-                annotation,
-                qmdHash: matched?.hash ?? null,
-                qmdSeq: matched?.seq ?? null,
-            });
+            const key = JSON.stringify([annotation.collection, annotation.path]);
+            let chunks = fingerprints.get(key);
+            if (!chunks) {
+                chunks = new Map();
+                const rows = findChunks.all(annotation.collection, annotation.path);
+                for (const row of rows) {
+                    const fingerprint = chunkFingerprint(row.doc.slice(row.pos, row.pos + row.chunk_len));
+                    if (!chunks.has(fingerprint))
+                        chunks.set(fingerprint, row);
+                }
+                fingerprints.set(key, chunks);
+            }
+            const matched = chunks.get(annotation.contentFingerprint);
+            if (annotation.qmdHash !== (matched?.hash ?? null) || annotation.qmdSeq !== (matched?.seq ?? null)) {
+                curation.updateAnnotationLocation({ annotation, qmdHash: matched?.hash ?? null, qmdSeq: matched?.seq ?? null });
+            }
             if (matched) {
                 insert.run(annotation.collection, annotation.path, matched.hash, matched.seq, annotation.eventTime, annotation.basis, 0);
             }
@@ -779,7 +780,7 @@ export class QmdMemoryManager {
     }
     async search(query, opts) {
         const started = performance.now();
-        const operation = opts?.lexicalOnly ? "lexical" : "vector";
+        const operation = "vector";
         let results;
         try {
             results = await this.#search(query, opts);
@@ -820,25 +821,6 @@ export class QmdMemoryManager {
         if (collections.includes(EXTRACTED_COLLECTION)) {
             allowedPaths = { ...allowedPaths, [EXTRACTED_COLLECTION]: currentRecords().map(extractedPath) };
         }
-        if (opts?.lexicalOnly) {
-            const hits = await store.searchLex(query, {
-                limit: opts.maxResults ?? 5,
-                collection: collections,
-            });
-            const current = new Map(currentRecords(hits.map(hit => hit.filepath)).map(r => [`qmd://${EXTRACTED_COLLECTION}/${extractedPath(r)}`, r]));
-            return hits.flatMap((hit) => {
-                if (hit.collectionName === EXTRACTED_COLLECTION) {
-                    const record = current.get(hit.filepath);
-                    return record && hit.score >= (opts.minScore ?? 0) ? [extractedHit(record, hit.score)] : [];
-                }
-                const corpus = this.#sources.get(hit.collectionName)?.corpus;
-                const prefix = `qmd://${hit.collectionName}/`;
-                const session = corpus === "sessions" && hit.filepath.startsWith(prefix)
-                    ? this.#sessionMetadata.get(hit.filepath.slice(prefix.length))
-                    : undefined;
-                return hit.score >= (opts.minScore ?? 0) && corpus ? [lexicalResult(hit, corpus, session)] : [];
-            });
-        }
         const hits = await store.vsearch(query, {
             collection: collections,
             limit: opts?.maxResults ?? 5,
@@ -944,6 +926,7 @@ export class QmdMemoryManager {
         if (!store.internal?.llm)
             throw new Error("Skill Whisperer requires the QMD embedding model");
         const llm = store.internal.llm;
+        const { formatDocForEmbedding, formatQueryForEmbedding } = await qmdModule;
         let skillIndex = this.#skillIndex;
         if (!skillIndex) {
             const pending = (async () => {
@@ -968,7 +951,7 @@ export class QmdMemoryManager {
                     }
                 }
                 const skills = [...metadata.values()];
-                const embeddings = await llm.embedBatch(skills.map(({ candidate, description }) => embeddingText(candidate.name, description, llm.embedModelName)));
+                const embeddings = await llm.embedBatch(skills.map(({ candidate, description }) => formatDocForEmbedding(description, candidate.name, llm.embedModelName)));
                 return skills.flatMap(({ candidate }, index) => {
                     const embedding = embeddings[index]?.embedding;
                     return embedding ? [{ ...candidate, score: 0, embedding }] : [];
@@ -981,7 +964,7 @@ export class QmdMemoryManager {
             });
             this.#skillIndex = skillIndex;
         }
-        const queryEmbedding = await llm.embed(queryText(query, llm.embedModelName), { isQuery: true });
+        const queryEmbedding = await llm.embed(formatQueryForEmbedding(query, llm.embedModelName), { isQuery: true });
         if (!queryEmbedding)
             return [];
         const candidates = await skillIndex;
@@ -1064,6 +1047,7 @@ export class QmdMemoryManager {
         if (this.#watchTimer)
             clearTimeout(this.#watchTimer);
         this.#watchTimer = undefined;
+        this.#watchCollections.clear();
         await this.#watcher?.close();
         this.#watcher = undefined;
         this.#watchReady = undefined;

package/dist/src/memory-whisperer.js

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { judgeTypeSafeMemories } from "./typesafe.js";
+import { judgeMemoryPassage, memoryUsefulnessRequest } from "./typesafe.js";
 import { resolveTypeSafeApiKey, TypeSafeRequestError } from "./typesafe-client.js";
 import { complementaryIndices, reviewMemoryRedundancy } from "./typesafe-review.js";
 import { ApiQueryGenerator, QueryApiError, queryConversation } from "./query-generator.js";
@@ -164,15 +164,11 @@ export function registerMemoryWhisperer(api, runtime, config, typesafe, diagnost
                         timeoutMs: typesafe.timeoutMs };
                     try {
                         const { hit, excerpt } = candidate;
-                        const [probability] = await judgeTypeSafeMemories({
-                            apiKey, timeoutMs: typesafe.timeoutMs, signal,
-                            conversation, asOf,
-                            candidates: [{
-                                    excerpt, corpus: hit.corpus, sourcePath: hit.path,
-                                    dates: [...new Set(hit.sessionMessages?.flatMap(message => message.timestamp ? [message.timestamp] : [])
-                                            ?? (hit.messageTimestamp ? [hit.messageTimestamp] : []))],
-                                }],
-                        });
+                        const { probability } = await judgeMemoryPassage(memoryUsefulnessRequest(conversation, {
+                            excerpt, corpus: hit.corpus, sourcePath: hit.path,
+                            dates: [...new Set(hit.sessionMessages?.flatMap(message => message.timestamp ? [message.timestamp] : [])
+                                    ?? (hit.messageTimestamp ? [hit.messageTimestamp] : []))],
+                        }, asOf), { apiKey, timeoutMs: typesafe.timeoutMs, signal });
                         if (signal.aborted)
                             return [];
                         requestsSucceeded++;