@unblocklabs/unblock-memory 0.3.25 → 0.3.27

package/dist/src/training-retrieval.d.ts

@@ -20,7 +20,7 @@ export declare function historicalPrefix(body: string, spans: readonly SessionMe
     body: string;
     spans: SessionMessageSpan[];
 } | undefined;
-/** A read-only source snapshot, copied into a disposable in-memory QMD index.
+/** Fingerprint a read-only source snapshot; build its search index only on demand.
  * No filesystem projection, live-index mutation, model re-embedding or dependency patch. */
 export declare function historicalTrainingSearch(stateDir: string, chatTypes: readonly ChatType[], cutoff: number, openStore?: typeof import("@unblocklabs/qmd")["createStore"]): Promise<{
     corpusHash: string;

package/dist/src/training-retrieval.js

@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
 import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { createRequire } from "node:module";
+import { setImmediate as yieldToEventLoop } from "node:timers/promises";
 import { readSessionManifest } from "./session-sync.js";
 import { resolveSessionSource } from "./sources.js";
 import { trainingCandidates } from "./training-candidates.js";
@@ -42,7 +43,7 @@ export function historicalPrefix(body, spans, cutoff) {
         return;
     return { body: body.slice(0, parsed[safe.length]?.span.start ?? body.length), spans: safe.map(s => s.span) };
 }
-/** A read-only source snapshot, copied into a disposable in-memory QMD index.
+/** Fingerprint a read-only source snapshot; build its search index only on demand.
  * No filesystem projection, live-index mutation, model re-embedding or dependency patch. */
 export async function historicalTrainingSearch(stateDir, chatTypes, cutoff, openStore) {
     const indexPath = join(stateDir, "index.sqlite");
@@ -51,120 +52,175 @@ export async function historicalTrainingSearch(stateDir, chatTypes, cutoff, open
     const manifest = await readSessionManifest(join(stateDir, "sessions-manifest.json"));
     const source = resolveSessionSource(join(stateDir, "sessions"), chatTypes);
     const createStore = openStore ?? (await import("@unblocklabs/qmd")).createStore;
-    const qmd = await createStore({ dbPath: ":memory:", keepModelsWarm: true,
-        config: { collections: { [source.collection]: { path: source.root, pattern: "**/*.md" } } } });
-    // A snapshot has one native embedding context. Serialize its vector stage only;
-    // QMD's remote TypeSafe scoring remains concurrent across all ten queries.
-    const searchVec = qmd.internal.searchVec;
-    let vectorTail = Promise.resolve();
-    qmd.internal.searchVec = (...args) => {
-        const result = vectorTail.then(() => searchVec(...args));
-        vectorTail = result.then(() => { }, () => { });
-        return result;
-    };
-    const db = qmd.internal.db;
     const maxDate = new Date(cutoff).toISOString();
     const report = { sessions: 0, chunks: 0, excluded: 0, truncated: 0, excludedChunks: 0 };
     const metadata = new Map();
     const fingerprint = createHash("sha256").update(JSON.stringify([TRAINING_RETRIEVAL_VERSION, chatTypes.toSorted(), maxDate]));
+    // Use QMD's exact binding, never mix node:sqlite with sqlite-vec.
+    const requireQmd = createRequire(import.meta.resolve("@unblocklabs/qmd"));
+    const Database = requireQmd("better-sqlite3");
+    const sourceDb = new Database(indexPath, { readonly: true, fileMustExist: true });
+    const documents = [];
+    let sourceClosed = false, closed = false;
+    const closeSource = () => { if (!sourceClosed) {
+        sourceDb.close();
+        sourceClosed = true;
+    } documents.length = 0; };
+    let indexed;
     try {
-        // QMD's SDK only opens writable stores. Use its exact SQLite binding in
-        // read-only mode; mixing node:sqlite with better-sqlite3 corrupts sqlite-vec's
-        // process-global SQLite API pointer when both load the native extension.
-        const requireQmd = createRequire(import.meta.resolve("@unblocklabs/qmd"));
-        const Database = requireQmd("better-sqlite3");
-        const sourceDb = new Database(indexPath, { readonly: true, fileMustExist: true });
-        try {
-            const extension = requireQmd("sqlite-vec");
-            if (!extension || typeof extension !== "object" || !("getLoadablePath" in extension) || typeof extension.getLoadablePath !== "function") {
-                throw new Error("QMD sqlite-vec extension unavailable");
-            }
-            const extensionPath = extension.getLoadablePath();
-            if (typeof extensionPath !== "string")
-                throw new Error("QMD sqlite-vec path unavailable");
-            sourceDb.loadExtension(extensionPath);
-            sourceDb.exec("PRAGMA query_only=ON; PRAGMA busy_timeout=5000; BEGIN");
-            db.exec("BEGIN");
-            const settings = sourceDb.prepare("SELECT key,value FROM store_config WHERE key='embedding_chunk_strategy'").all();
-            for (const { key, value } of settings)
-                db.prepare("INSERT OR REPLACE INTO store_config VALUES (?,?)").run(key, value);
-            fingerprint.update(JSON.stringify(settings));
-            const document = sourceDb.prepare(`SELECT d.hash,c.doc FROM documents d JOIN content c ON c.hash=d.hash
+        const extension = requireQmd("sqlite-vec");
+        if (!extension || typeof extension !== "object" || !("getLoadablePath" in extension) || typeof extension.getLoadablePath !== "function") {
+            throw new Error("QMD sqlite-vec extension unavailable");
+        }
+        const extensionPath = extension.getLoadablePath();
+        if (typeof extensionPath !== "string")
+            throw new Error("QMD sqlite-vec path unavailable");
+        sourceDb.loadExtension(extensionPath);
+        sourceDb.exec("PRAGMA query_only=ON; PRAGMA busy_timeout=5000; BEGIN");
+        const settings = sourceDb.prepare("SELECT key,value FROM store_config WHERE key='embedding_chunk_strategy'").all();
+        fingerprint.update(JSON.stringify(settings));
+        const document = sourceDb.prepare(`SELECT d.hash,c.doc FROM documents d JOIN content c ON c.hash=d.hash
         WHERE d.active=1 AND d.collection=? AND d.path=?`);
-            const chunks = sourceDb.prepare(`SELECT cv.seq,cv.pos,cv.chunk_len,cv.model,cv.embed_fingerprint,v.embedding
+        const chunks = sourceDb.prepare(`SELECT cv.seq,cv.pos,cv.chunk_len,cv.model,cv.embed_fingerprint,v.embedding
         FROM content_vectors cv JOIN vectors_vec v ON v.hash_seq=cv.hash||'_'||cv.seq
         WHERE cv.hash=? ORDER BY cv.seq`);
-            let dimensions;
-            for (const session of Object.values(manifest.sessions).sort((a, b) => a.documentPath.localeCompare(b.documentPath))) {
-                // Loggie projections can retroactively annotate old text using later revisions.
-                // Workspace files and meetings lack immutable historical content proof here.
-                if (!chatTypes.includes(session.chatType) || session.provider === "loggie" || session.startedAt >= cutoff) {
-                    report.excluded++;
-                    continue;
-                }
-                const row = document.get(source.collection, session.documentPath);
-                if (!row || createHash("sha256").update(row.doc).digest("hex") !== session.projectionHash) {
-                    report.excluded++;
-                    continue;
+        let dimensions;
+        // Both passes use this same read transaction. Cached evaluations still
+        // verify every prefix/vector byte, but never build an index or load a model.
+        let yieldAt = performance.now() + 25;
+        for (const session of Object.values(manifest.sessions).sort((a, b) => a.documentPath.localeCompare(b.documentPath))) {
+            if (performance.now() >= yieldAt) {
+                await yieldToEventLoop();
+                yieldAt = performance.now() + 25;
+            }
+            // Loggie projections can retroactively annotate old text using later revisions.
+            // Workspace files and meetings lack immutable historical content proof here.
+            if (!chatTypes.includes(session.chatType) || session.provider === "loggie" || session.startedAt >= cutoff) {
+                report.excluded++;
+                continue;
+            }
+            const row = document.get(source.collection, session.documentPath);
+            if (!row || createHash("sha256").update(row.doc).digest("hex") !== session.projectionHash) {
+                report.excluded++;
+                continue;
+            }
+            const prefix = historicalPrefix(row.doc, session.messages, cutoff);
+            if (!prefix) {
+                report.excluded++;
+                continue;
+            }
+            const hash = createHash("sha256").update(prefix.body).digest("hex");
+            documents.push({ path: session.documentPath, sourceHash: row.hash, hash, body: prefix.body });
+            metadata.set(`qmd://${source.collection}/${session.documentPath}`, prefix.spans);
+            fingerprint.update(JSON.stringify([session.documentPath, hash]));
+            report.sessions++;
+            if (prefix.spans.length < (session.messages?.length ?? 0))
+                report.truncated++;
+            for (const chunk of chunks.iterate(row.hash)) {
+                if (performance.now() >= yieldAt) {
+                    await yieldToEventLoop();
+                    yieldAt = performance.now() + 25;
                 }
-                const prefix = historicalPrefix(row.doc, session.messages, cutoff);
-                if (!prefix) {
-                    report.excluded++;
+                if (!historicalChunk(chunk, prefix.body.length)) {
+                    report.excludedChunks++;
                     continue;
                 }
-                const hash = createHash("sha256").update(prefix.body).digest("hex");
-                qmd.internal.insertContent(hash, prefix.body, maxDate);
-                qmd.internal.insertDocument(source.collection, session.documentPath, "Transcript", hash, maxDate, maxDate);
-                metadata.set(`qmd://${source.collection}/${session.documentPath}`, prefix.spans);
-                fingerprint.update(JSON.stringify([session.documentPath, hash]));
-                report.sessions++;
-                if (prefix.spans.length < (session.messages?.length ?? 0))
-                    report.truncated++;
-                for (const chunk of chunks.all(row.hash)) {
-                    if (!Number.isSafeInteger(chunk.pos) || !Number.isSafeInteger(chunk.chunk_len) || chunk.pos < 0 || chunk.chunk_len <= 0 ||
-                        chunk.pos + chunk.chunk_len > prefix.body.length) {
-                        report.excludedChunks++;
-                        continue;
+                const bytes = Buffer.from(chunk.embedding);
+                if (bytes.length % 4 || !bytes.length)
+                    throw new Error("Invalid historical embedding");
+                const size = bytes.length / 4;
+                dimensions ??= size;
+                if (dimensions !== size)
+                    throw new Error("Mixed historical embedding dimensions");
+                fingerprint.update(JSON.stringify([chunk.seq, chunk.pos, chunk.chunk_len, chunk.model, chunk.embed_fingerprint])).update(bytes);
+                report.chunks++;
+            }
+        }
+        const corpusHash = fingerprint.digest("hex");
+        const materialize = async () => {
+            let qmd;
+            try {
+                qmd = await createStore({ dbPath: ":memory:", keepModelsWarm: true,
+                    config: { collections: { [source.collection]: { path: source.root, pattern: "**/*.md" } } } });
+                // One native embedding context per snapshot; serialize only vector search.
+                const searchVec = qmd.internal.searchVec;
+                let vectorTail = Promise.resolve();
+                qmd.internal.searchVec = (...args) => {
+                    const result = vectorTail.then(() => searchVec(...args));
+                    vectorTail = result.then(() => { }, () => { });
+                    return result;
+                };
+                const db = qmd.internal.db;
+                db.exec("BEGIN");
+                for (const { key, value } of settings)
+                    db.prepare("INSERT OR REPLACE INTO store_config VALUES (?,?)").run(key, value);
+                if (dimensions !== undefined)
+                    qmd.internal.ensureVecTable(dimensions);
+                yieldAt = performance.now() + 25;
+                for (const document of documents) {
+                    if (performance.now() >= yieldAt) {
+                        await yieldToEventLoop();
+                        yieldAt = performance.now() + 25;
                     }
-                    const bytes = Buffer.from(chunk.embedding);
-                    if (bytes.length % 4 || !bytes.length)
-                        throw new Error("Invalid historical embedding");
-                    const size = bytes.length / 4;
-                    if (dimensions === undefined) {
-                        dimensions = size;
-                        qmd.internal.ensureVecTable(size);
+                    qmd.internal.insertContent(document.hash, document.body, maxDate);
+                    qmd.internal.insertDocument(source.collection, document.path, "Transcript", document.hash, maxDate, maxDate);
+                    for (const chunk of chunks.iterate(document.sourceHash)) {
+                        if (performance.now() >= yieldAt) {
+                            await yieldToEventLoop();
+                            yieldAt = performance.now() + 25;
+                        }
+                        if (!historicalChunk(chunk, document.body.length))
+                            continue;
+                        const bytes = Buffer.from(chunk.embedding), vector = new Float32Array(bytes.length / 4);
+                        for (let i = 0; i < vector.length; i++)
+                            vector[i] = bytes.readFloatLE(i * 4);
+                        qmd.internal.insertEmbedding(document.hash, chunk.seq, chunk.pos, vector, chunk.model, maxDate, 1, chunk.embed_fingerprint, chunk.chunk_len);
                     }
-                    if (dimensions !== size)
-                        throw new Error("Mixed historical embedding dimensions");
-                    const vector = new Float32Array(size);
-                    for (let i = 0; i < size; i++)
-                        vector[i] = bytes.readFloatLE(i * 4);
-                    qmd.internal.insertEmbedding(hash, chunk.seq, chunk.pos, vector, chunk.model, maxDate, 1, chunk.embed_fingerprint, chunk.chunk_len);
-                    fingerprint.update(JSON.stringify([chunk.seq, chunk.pos, chunk.chunk_len, chunk.model, chunk.embed_fingerprint])).update(bytes);
-                    report.chunks++;
                 }
+                db.exec("COMMIT");
+                return qmd;
             }
-            db.exec("COMMIT");
-        }
-        finally {
-            sourceDb.close();
-        }
-        const corpusHash = fingerprint.digest("hex");
+            catch (error) {
+                await qmd?.close();
+                throw error;
+            }
+            finally {
+                closeSource();
+            }
+        };
         return { corpusHash, report, maxDate,
             search: async (query) => {
+                if (closed)
+                    throw new Error("Historical snapshot is closed");
                 if (!report.sessions)
                     return [];
                 if (!report.chunks)
                     throw new Error("Historical snapshot has no vectors; refusing a BM25-only evaluation");
+                const qmd = await (indexed ??= materialize());
                 const hits = await trainingCandidates(qmd, query, source.collection, `Historical request made at ${maxDate}. Current, now and latest refer to that timestamp.`);
                 return hits.map(hit => trainingHit(hit, metadata, cutoff));
-            }, close: () => qmd.close() };
+            }, close: async () => {
+                if (closed)
+                    return;
+                closed = true;
+                try {
+                    if (indexed)
+                        await indexed.then(qmd => qmd.close(), () => { });
+                }
+                finally {
+                    closeSource();
+                }
+            } };
     }
     catch (error) {
-        await qmd.close();
+        closeSource();
         throw error;
     }
 }
+function historicalChunk(chunk, length) {
+    return Number.isSafeInteger(chunk.pos) && Number.isSafeInteger(chunk.chunk_len) && chunk.pos >= 0 && chunk.chunk_len > 0 &&
+        chunk.pos + chunk.chunk_len <= length;
+}
 function trainingHit(hit, metadata, cutoff) {
     const spans = metadata.get(hit.file)?.filter(s => s.start < hit.bestChunkPos + hit.bestChunk.length && s.end > hit.bestChunkPos);
     if (!spans?.length || spans.some(s => !(Date.parse(s.timestamp) < Math.floor(cutoff / 1000) * 1000)) ||

package/dist/src/training-store.js

@@ -43,6 +43,8 @@ export class TrainingStore {
           status TEXT NOT NULL CHECK(status IN ('pending','attempted','complete','failed','ambiguous')),
           result_json TEXT, error TEXT, completed_at INTEGER
         ) STRICT;
+        CREATE INDEX IF NOT EXISTS training_judgment_exclusions ON training_steps(json_extract(request_json,'$.identity'))
+          WHERE stage='judge' AND status='complete' AND json_extract(result_json,'$.excluded')=1;
         CREATE TABLE IF NOT EXISTS training_step_attempts (
           id INTEGER PRIMARY KEY, step_id TEXT NOT NULL REFERENCES training_steps(id), started_at INTEGER NOT NULL,
           finished_at INTEGER, status TEXT NOT NULL, error TEXT, result_json TEXT

package/docs/memory-training.md

@@ -56,6 +56,11 @@ Native vector work is serialized per snapshot. Raise concurrency within machine
 and provider capacity. Failures stop new dispatch; in-flight operations drain
 and persist before snapshots close. Dry runs make no inference calls.
 
+Cached evaluations still validate the exact historical text and vector fingerprint,
+but do not rebuild a QMD index. The index is built only on the first uncached search,
+from the same read-only SQLite transaction used to compute that fingerprint.
+Concurrent queries share that index; changed corpus content still invalidates caches.
+
 ## Input and historical boundaries
 
 Read original agent SQLite active-branch conversations (schema 17–19), including
@@ -128,8 +133,13 @@ No blanket failure skipping.
 
 The host must support isolated completion and grant
 `plugins.entries.unblock-memory.llm.allowModelOverride: true` with
-`allowedModels: ["openai/gpt-6-luna"]`. Preserve other grants. The plugin does not
-change its own permissions. Credentials stay with the host, never in provenance.
+`allowedModels: ["openai/gpt-6-luna"]`. The operator CLI explicitly selects an
+agent, so an unbound CLI runtime also requires
+`plugins.entries.unblock-memory.llm.allowAgentIdOverride: true`. Preserve other
+grants. The host's model catalog and selected runtime must support that exact
+model with usable host-managed authentication; a model permission grant alone
+does not establish model availability. The plugin does not change its own
+permissions. Credentials stay with the host, never in provenance.
 
 ## Export and consolidation
 
@@ -145,3 +155,16 @@ identical-input groups together across nodes for train/validation. Use the actua
 **LFM2.5-230M-Base** tokenizer and an explicit causal-LM input/target format before
 fine-tuning; the interim byte limit is not a token count. Runtime abstention remains
 a separate TypeSafe gate; positive-only query training does not teach abstention.
+
+The repository helper uses a pinned official tokenizer revision and chat template,
+with assistant-only loss labels, a conservative 32,768-token training ceiling
+(the model config supports 128,000 positions), and deterministic grouped splits:
+
+```sh
+python scripts/prepare-query-training.py /private/node1-queries.jsonl /private/node2-queries.jsonl \
+  --output /private/new-prepared-directory --cache-dir /private/tokenizer-cache
+```
+
+Install `transformers` and `jinja2` in a separate environment first. It downloads
+tokenizer files only, never model weights. Regex secret screening is not an
+exhaustive privacy audit; inspect quarantine references before including those rows.

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "unblock-memory",
   "name": "Unblock Memory",
-  "version": "0.3.25",
+  "version": "0.3.27",
   "description": "Indexes, retrieves, and analyzes configured workspace memory with existing QMD vectors.",
   "kind": "memory",
   "activation": { "onStartup": true },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@unblocklabs/unblock-memory",
-  "version": "0.3.25",
+  "version": "0.3.27",
   "description": "Workspace-native memory for OpenClaw, powered by QMD",
   "type": "module",
   "license": "MIT",