@unblocklabs/unblock-memory 0.3.24 → 0.3.25

package/dist/src/training-store.js

@@ -0,0 +1,300 @@
+import { randomUUID } from "node:crypto";
+import { openMemoryDatabase } from "./memory-database.js";
+import { trainingHash, TRAINING_PREPARATION } from "./training-input.js";
+import { TRAINING_GATE_MODEL, TRAINING_GATE_THRESHOLD, TRAINING_GATE_VERSION } from "./training-gate.js";
+const LEASE_MS = 120_000;
+export class TrainingStore {
+    #db;
+    #owner = randomUUID();
+    #nodeId;
+    #agentId;
+    constructor(path, agentId) {
+        this.#db = openMemoryDatabase(path);
+        this.#agentId = agentId;
+        try {
+            const version = this.#db.prepare("SELECT version FROM memory_schema WHERE component='training'").get()?.version;
+            if (version !== undefined && version !== 1)
+                throw new Error("Unsupported training database version");
+            this.#db.exec(`
+        CREATE TABLE IF NOT EXISTS training_identity (singleton INTEGER PRIMARY KEY CHECK(singleton=1), node_id TEXT NOT NULL, agent_id TEXT NOT NULL) STRICT;
+        CREATE TABLE IF NOT EXISTS training_lock (singleton INTEGER PRIMARY KEY CHECK(singleton=1), owner TEXT NOT NULL, expires INTEGER NOT NULL) STRICT;
+        CREATE TABLE IF NOT EXISTS training_inputs (hash TEXT PRIMARY KEY, preparation TEXT NOT NULL, input_json TEXT NOT NULL) STRICT;
+        CREATE TABLE IF NOT EXISTS training_examples (
+          id TEXT PRIMARY KEY, session_id TEXT NOT NULL, event_seq INTEGER NOT NULL, timestamp INTEGER NOT NULL,
+          input_hash TEXT NOT NULL REFERENCES training_inputs(hash), context_limited INTEGER NOT NULL,
+          active INTEGER NOT NULL, UNIQUE(session_id,event_seq)
+        ) STRICT;
+        CREATE TABLE IF NOT EXISTS training_gates (
+          id TEXT PRIMARY KEY, input_hash TEXT NOT NULL REFERENCES training_inputs(hash),
+          prompt_version TEXT NOT NULL, requested_model TEXT NOT NULL,
+          status TEXT NOT NULL CHECK(status IN ('pending','attempted','complete','failed','ambiguous')),
+          probability REAL, actual_model TEXT, input_tokens INTEGER, output_tokens INTEGER,
+          error TEXT, completed_at INTEGER,
+          UNIQUE(input_hash,prompt_version,requested_model)
+        ) STRICT;
+        CREATE TABLE IF NOT EXISTS training_attempts (
+          id INTEGER PRIMARY KEY, gate_id TEXT NOT NULL REFERENCES training_gates(id),
+          started_at INTEGER NOT NULL, finished_at INTEGER, status TEXT NOT NULL,
+          error TEXT, result_json TEXT
+        ) STRICT;
+        CREATE INDEX IF NOT EXISTS training_examples_input ON training_examples(input_hash,active);
+        CREATE TABLE IF NOT EXISTS training_steps (
+          id TEXT PRIMARY KEY, stage TEXT NOT NULL, request_json TEXT NOT NULL,
+          status TEXT NOT NULL CHECK(status IN ('pending','attempted','complete','failed','ambiguous')),
+          result_json TEXT, error TEXT, completed_at INTEGER
+        ) STRICT;
+        CREATE TABLE IF NOT EXISTS training_step_attempts (
+          id INTEGER PRIMARY KEY, step_id TEXT NOT NULL REFERENCES training_steps(id), started_at INTEGER NOT NULL,
+          finished_at INTEGER, status TEXT NOT NULL, error TEXT, result_json TEXT
+        ) STRICT;
+        INSERT OR IGNORE INTO memory_schema VALUES ('training',1);
+      `);
+            this.#db.prepare("INSERT OR IGNORE INTO training_identity VALUES (1,?,?)").run(randomUUID(), agentId);
+            const identity = this.#db.prepare("SELECT node_id,agent_id FROM training_identity WHERE singleton=1").get();
+            if (identity.agent_id !== agentId)
+                throw new Error("Training database belongs to a different agent");
+            this.#nodeId = String(identity.node_id);
+        }
+        catch (error) {
+            this.#db.close();
+            throw error;
+        }
+    }
+    #transaction(fn) {
+        this.#db.exec("BEGIN IMMEDIATE");
+        try {
+            const result = fn();
+            this.#db.exec("COMMIT");
+            return result;
+        }
+        catch (error) {
+            this.#db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    #checkLease() {
+        const lease = this.#db.prepare("SELECT owner,expires FROM training_lock WHERE singleton=1").get();
+        if (lease?.owner !== this.#owner || Number(lease.expires) <= Date.now())
+            throw new Error("Training lease lost; no further API requests permitted");
+    }
+    async locked(fn) {
+        this.#transaction(() => {
+            const lock = this.#db.prepare(`INSERT INTO training_lock VALUES (1,?,?) ON CONFLICT(singleton)
+        DO UPDATE SET owner=excluded.owner,expires=excluded.expires WHERE training_lock.expires<=?`)
+                .run(this.#owner, Date.now() + LEASE_MS, Date.now());
+            if (!lock.changes)
+                throw new Error("Another memory-training command is running");
+            // A request may have been billed before a crashed process saved the response.
+            this.#db.exec(`UPDATE training_gates SET status='ambiguous',error='interrupted' WHERE status='attempted';
+        UPDATE training_attempts SET status='ambiguous',error='interrupted' WHERE status='attempted';
+        UPDATE training_steps SET status='ambiguous',error='interrupted' WHERE status='attempted';
+        UPDATE training_step_attempts SET status='ambiguous',error='interrupted' WHERE status='attempted';`);
+        });
+        let leaseError;
+        const heartbeat = setInterval(() => { try {
+            this.renew();
+        }
+        catch (error) {
+            leaseError = error;
+        } }, 10_000);
+        heartbeat.unref();
+        try {
+            const value = await fn();
+            if (leaseError)
+                throw leaseError;
+            return value;
+        }
+        finally {
+            clearInterval(heartbeat);
+            this.#db.prepare("DELETE FROM training_lock WHERE owner=?").run(this.#owner);
+        }
+    }
+    renew() {
+        this.#transaction(() => {
+            this.#checkLease();
+            this.#db.prepare("UPDATE training_lock SET expires=? WHERE owner=?").run(Date.now() + LEASE_MS, this.#owner);
+        });
+    }
+    sessions() { return this.#db.prepare("SELECT DISTINCT session_id FROM training_examples").all().map(r => String(r.session_id)); }
+    syncSession(sessionId, examples, since, until, existingOnly) {
+        return this.#transaction(() => {
+            this.#checkLease();
+            const previous = new Map(this.#db.prepare("SELECT event_seq,input_hash,active FROM training_examples WHERE session_id=?").all(sessionId)
+                .map(row => [Number(row.event_seq), row]));
+            this.#db.prepare("UPDATE training_examples SET active=0 WHERE session_id=?").run(sessionId);
+            const counts = { added: 0, changed: 0, unchanged: 0, retired: 0 };
+            const retained = new Set();
+            for (const example of examples) {
+                const old = previous.get(example.seq);
+                if (!old && (existingOnly || example.timestamp < since || example.timestamp >= until))
+                    continue;
+                retained.add(example.seq);
+                this.#db.prepare("INSERT OR IGNORE INTO training_inputs VALUES (?,?,?)")
+                    .run(example.inputHash, TRAINING_PREPARATION, JSON.stringify(example.input));
+                this.#db.prepare(`INSERT INTO training_examples VALUES (?,?,?,?,?,?,1)
+          ON CONFLICT(session_id,event_seq) DO UPDATE SET timestamp=excluded.timestamp,input_hash=excluded.input_hash,
+          context_limited=excluded.context_limited,active=1`)
+                    .run(trainingHash([this.#nodeId, this.#agentId, sessionId, example.seq]), sessionId, example.seq, example.timestamp, example.inputHash, Number(example.contextLimited));
+                const id = trainingHash([example.inputHash, TRAINING_GATE_VERSION, TRAINING_GATE_MODEL]);
+                this.#db.prepare(`INSERT OR IGNORE INTO training_gates (id,input_hash,prompt_version,requested_model,status) VALUES (?,?,?,?,'pending')`)
+                    .run(id, example.inputHash, TRAINING_GATE_VERSION, TRAINING_GATE_MODEL);
+                if (!old)
+                    counts.added++;
+                else if (old.input_hash !== example.inputHash || old.active !== 1)
+                    counts.changed++;
+                else
+                    counts.unchanged++;
+            }
+            counts.retired = [...previous].filter(([seq, row]) => row.active === 1 && !retained.has(seq)).length;
+            return counts;
+        });
+    }
+    unavailable(sessionId) {
+        this.#checkLease();
+        this.#db.prepare("UPDATE training_examples SET active=-1 WHERE session_id=? AND active=1").run(sessionId);
+    }
+    #scope = `g.prompt_version=? AND g.requested_model=? AND EXISTS
+    (SELECT 1 FROM training_examples e WHERE e.input_hash=g.input_hash AND e.active=1)`;
+    pending(limit) {
+        return this.#db.prepare(`SELECT g.id,g.input_hash inputHash,i.input_json inputJson
+      FROM training_gates g JOIN training_inputs i ON i.hash=g.input_hash
+      WHERE ${this.#scope} AND g.status='pending' ORDER BY
+      (SELECT MAX(timestamp) FROM training_examples WHERE input_hash=g.input_hash AND active=1) DESC,g.id LIMIT ?`)
+            .all(TRAINING_GATE_VERSION, TRAINING_GATE_MODEL, limit ?? -1);
+    }
+    start(id) {
+        return this.#transaction(() => {
+            this.#checkLease();
+            if (!this.#db.prepare("UPDATE training_gates SET status='attempted',error=NULL WHERE id=? AND status='pending'").run(id).changes) {
+                throw new Error("Training gate is not pending");
+            }
+            return Number(this.#db.prepare("INSERT INTO training_attempts (gate_id,started_at,status) VALUES (?,?,'attempted')")
+                .run(id, Date.now()).lastInsertRowid);
+        });
+    }
+    finish(id, attempt, result) {
+        this.#transaction(() => {
+            this.#checkLease();
+            const ok = "probability" in result;
+            const status = ok ? "complete" : result.status, error = ok ? null : result.error;
+            this.#db.prepare(`UPDATE training_gates SET status=?,probability=?,actual_model=?,input_tokens=?,output_tokens=?,error=?,completed_at=? WHERE id=?`)
+                .run(status, ok ? result.probability : null, ok ? result.model : null, ok ? result.usage.input_tokens : null, ok ? result.usage.output_tokens : null, error, Date.now(), id);
+            this.#db.prepare("UPDATE training_attempts SET status=?,finished_at=?,error=?,result_json=? WHERE id=? AND gate_id=?")
+                .run(status, Date.now(), error, ok ? JSON.stringify(result) : null, attempt, id);
+        });
+    }
+    retry(includeAmbiguous) {
+        this.#checkLease();
+        return this.#transaction(() => ["training_gates", "training_steps"].reduce((n, table) => n + Number(this.#db.prepare(`UPDATE ${table} SET status='pending',error=NULL WHERE status='failed' ${includeAmbiguous ? "OR status='ambiguous'" : ""}`).run().changes), 0));
+    }
+    activeExamples() {
+        return this.#db.prepare(`SELECT e.id,e.input_hash inputHash,i.input_json inputJson,e.session_id sessionId,e.timestamp
+      FROM training_examples e JOIN training_inputs i ON i.hash=e.input_hash
+      WHERE e.active=1 ORDER BY e.timestamp DESC,e.id`).all();
+    }
+    queryExamples(threshold = TRAINING_GATE_THRESHOLD) {
+        if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1)
+            throw new Error("Threshold must be between 0 and 1");
+        const probabilities = new Map(this.#db.prepare(`SELECT g.input_hash,g.probability FROM training_gates g
+      WHERE ${this.#scope} AND g.status='complete' AND g.probability>=?`)
+            .all(TRAINING_GATE_VERSION, TRAINING_GATE_MODEL, threshold).map(row => [String(row.input_hash), Number(row.probability)]));
+        return this.activeExamples().flatMap(example => {
+            const recallProbability = probabilities.get(example.inputHash);
+            return recallProbability === undefined ? [] : [{ ...example, recallProbability }];
+        });
+    }
+    step(stage, request) {
+        const id = trainingHash([stage, request]);
+        const row = this.#db.prepare("SELECT status,result_json FROM training_steps WHERE id=? AND stage=?").get(id, stage);
+        return { id, stage, request, status: (row?.status ?? "pending"),
+            result: row?.status === "complete" ? JSON.parse(String(row.result_json)) : undefined };
+    }
+    judgmentExcluded(identity) {
+        return !!this.#db.prepare(`SELECT 1 FROM training_steps WHERE stage='judge' AND status='complete'
+      AND json_extract(request_json,'$.identity')=? AND json_extract(result_json,'$.excluded')=1 LIMIT 1`).get(identity);
+    }
+    startStep(stage, id, request) {
+        return this.#transaction(() => {
+            this.#checkLease();
+            this.#db.prepare("INSERT OR IGNORE INTO training_steps (id,stage,request_json,status) VALUES (?,?,?,'pending')")
+                .run(id, stage, JSON.stringify(request));
+            if (!this.#db.prepare("UPDATE training_steps SET status='attempted',error=NULL WHERE id=? AND status='pending'").run(id).changes) {
+                throw new Error("Training step is not pending");
+            }
+            return Number(this.#db.prepare("INSERT INTO training_step_attempts (step_id,started_at,status) VALUES (?,?,'attempted')")
+                .run(id, Date.now()).lastInsertRowid);
+        });
+    }
+    finishStep(stage, id, attempt, outcome) {
+        this.#transaction(() => {
+            this.#checkLease();
+            const ok = "result" in outcome, status = ok ? "complete" : outcome.status;
+            const json = ok ? JSON.stringify(outcome.result) : null, error = ok ? null : outcome.error;
+            if (!this.#db.prepare(`UPDATE training_steps SET status=?,result_json=?,error=?,completed_at=? WHERE id=? AND stage=? AND status='attempted'`)
+                .run(status, json, error, Date.now(), id, stage).changes)
+                throw new Error("Training step attempt lost");
+            if (!this.#db.prepare(`UPDATE training_step_attempts SET status=?,finished_at=?,error=?,result_json=? WHERE id=? AND step_id=? AND status='attempted'`)
+                .run(status, Date.now(), error, json, attempt, id).changes)
+                throw new Error("Training attempt lost");
+        });
+    }
+    completedEvaluations(versions) {
+        return this.#db.prepare(`SELECT result_json FROM training_steps WHERE stage='evaluate' AND status='complete'
+      AND (? IS NULL OR (json_extract(request_json,'$.version')=? AND json_extract(request_json,'$.retrievalVersion')=?)) ORDER BY completed_at DESC,id`)
+            .all(versions?.selection ?? null, versions?.selection ?? null, versions?.retrieval ?? null)
+            .map(row => JSON.parse(String(row.result_json)));
+    }
+    sourceDetails(id) {
+        const row = this.#db.prepare(`SELECT session_id sessionId,event_seq userEventId,timestamp,context_limited contextLimited
+      FROM training_examples WHERE id=? AND active=1`).get(id);
+        if (!row)
+            throw new Error("Inactive training source");
+        return { nodeId: this.#nodeId, agentId: this.#agentId, sourceId: id, ...row };
+    }
+    stepRecord(id) {
+        const row = this.#db.prepare("SELECT id,stage,request_json,result_json,completed_at FROM training_steps WHERE id=? AND status='complete'").get(id);
+        if (!row)
+            throw new Error("Missing training provenance checkpoint");
+        return { id, stage: row.stage, request: JSON.parse(String(row.request_json)),
+            result: JSON.parse(String(row.result_json)), completedAt: row.completed_at };
+    }
+    status(threshold) {
+        const stages = this.#db.prepare(`SELECT g.status,COUNT(*) count FROM training_gates g WHERE ${this.#scope} GROUP BY g.status`)
+            .all(TRAINING_GATE_VERSION, TRAINING_GATE_MODEL);
+        const labels = this.#db.prepare(`SELECT COUNT(*) complete,COALESCE(SUM(g.probability>=?),0) positive,
+      COALESCE(SUM(g.input_tokens),0) inputTokens,COALESCE(SUM(g.output_tokens),0) outputTokens
+      FROM training_gates g WHERE ${this.#scope} AND g.status='complete'`).get(threshold, TRAINING_GATE_VERSION, TRAINING_GATE_MODEL);
+        return { nodeId: this.#nodeId, agentId: this.#agentId, preparation: TRAINING_PREPARATION,
+            promptVersion: TRAINING_GATE_VERSION, requestedModel: TRAINING_GATE_MODEL, threshold,
+            examples: this.#db.prepare("SELECT active,COUNT(*) count FROM training_examples GROUP BY active").all(),
+            collectedInputs: Number(this.#db.prepare("SELECT COUNT(DISTINCT input_hash) count FROM training_examples WHERE active=1").get().count),
+            queryInputs: Number(labels.positive),
+            stages, complete: Number(labels.complete), positive: Number(labels.positive),
+            inputTokens: Number(labels.inputTokens), outputTokens: Number(labels.outputTokens),
+            negative: Number(labels.complete) - Number(labels.positive),
+            queryStages: this.#db.prepare("SELECT stage,status,COUNT(*) count FROM training_steps GROUP BY stage,status ORDER BY stage,status").all(),
+            queryAttempts: this.#db.prepare(`SELECT s.stage,a.status,COUNT(*) count,
+        COUNT(json_extract(a.result_json,'$.usage.input_tokens')) usageReported,
+        SUM(json_extract(a.result_json,'$.usage.input_tokens')) inputTokens,
+        SUM(json_extract(a.result_json,'$.usage.output_tokens')) outputTokens
+        FROM training_step_attempts a JOIN training_steps s ON s.id=a.step_id GROUP BY s.stage,a.status`).all(),
+            attempts: this.#db.prepare(`SELECT status,COUNT(*) count,
+        COALESCE(SUM(json_extract(result_json,'$.usage.input_tokens')),0) inputTokens,
+        COALESCE(SUM(json_extract(result_json,'$.usage.output_tokens')),0) outputTokens
+        FROM training_attempts GROUP BY status`).all() };
+    }
+    *exportRows(threshold) {
+        const rows = this.#db.prepare(`SELECT g.*,i.input_json FROM training_gates g JOIN training_inputs i ON i.hash=g.input_hash
+      WHERE ${this.#scope} AND g.status='complete' ORDER BY g.id`).iterate(TRAINING_GATE_VERSION, TRAINING_GATE_MODEL);
+        for (const row of rows) {
+            yield { stage: "recall-gate", inputHash: row.input_hash, preparation: TRAINING_PREPARATION,
+                input: JSON.parse(String(row.input_json)), recallProbability: row.probability,
+                recallNeeded: Number(row.probability) >= threshold, threshold, model: row.actual_model, promptVersion: row.prompt_version,
+                usage: { input_tokens: row.input_tokens, output_tokens: row.output_tokens },
+                sources: this.#db.prepare(`SELECT id,session_id sessionId,event_seq userEventId,timestamp,context_limited contextLimited
+          FROM training_examples WHERE input_hash=? AND active=1 ORDER BY session_id,event_seq`).all(String(row.input_hash))
+                    .map(source => ({ nodeId: this.#nodeId, agentId: this.#agentId, ...source })) };
+        }
+    }
+    close() { this.#db.close(); }
+}

package/dist/src/training.d.ts

@@ -0,0 +1,57 @@
+import type { UnblockMemoryConfig } from "./config.js";
+import type { TrainingStore } from "./training-store.js";
+type Source = {
+    databasePath: string;
+    agentId: string;
+};
+/** A full active-branch rescan is cheap/local; only changed exact inputs need inference. */
+export declare function collectTraining(source: Source, store?: TrainingStore, options?: {
+    since?: number;
+    until?: number;
+    existingOnly?: boolean;
+}): {
+    sessions: number;
+    excludedSessions: number;
+    oversizedSessions: number;
+    eligible: number;
+    users: number;
+    filtered: number;
+    oversized: number;
+    unanswered: number;
+    added: number;
+    changed: number;
+    unchanged: number;
+    retired: number;
+};
+/** Parallel, bounded paid work. Negative results are just as cacheable as positives. */
+export declare function runTraining(source: Source, store: TrainingStore, config: UnblockMemoryConfig, options: {
+    maxExamples?: number;
+    maxInputBytes: number;
+    concurrency?: number;
+    dryRun?: boolean;
+}): Promise<{
+    refreshed: {
+        sessions: number;
+        excludedSessions: number;
+        oversizedSessions: number;
+        eligible: number;
+        users: number;
+        filtered: number;
+        oversized: number;
+        unanswered: number;
+        added: number;
+        changed: number;
+        unchanged: number;
+        retired: number;
+    };
+    pendingSelected: number;
+    calls: number;
+    completed: number;
+    failed: number;
+    ambiguous: number;
+    inputBytes: number;
+    inputTokens: number;
+    outputTokens: number;
+    budgetLimited: boolean;
+}>;
+export {};

package/dist/src/training.js

@@ -0,0 +1,104 @@
+import { TypeSafeHttpError } from "./typesafe-transport.js";
+import { resolveTypeSafeApiKey } from "./typesafe.js";
+import { TrainingTranscriptReader } from "./training-input.js";
+import { judgeTrainingInput, TRAINING_GATE_QUESTIONS } from "./training-gate.js";
+/** A full active-branch rescan is cheap/local; only changed exact inputs need inference. */
+export function collectTraining(source, store, options = {}) {
+    const since = options.since ?? 0, until = options.until ?? Number.MAX_SAFE_INTEGER;
+    const reader = new TrainingTranscriptReader(source.databasePath, source.agentId);
+    const result = { sessions: 0, excludedSessions: 0, oversizedSessions: 0, eligible: 0,
+        users: 0, filtered: 0, oversized: 0, unanswered: 0, added: 0, changed: 0, unchanged: 0, retired: 0 };
+    try {
+        const sessions = new Set(options.existingOnly ? store?.sessions() : [...reader.sessions(), ...store?.sessions() ?? []]);
+        for (const id of sessions) {
+            store?.renew();
+            const extracted = reader.read(id);
+            result.sessions++;
+            if (extracted && "oversized" in extracted) {
+                result.oversizedSessions++;
+                store?.unavailable(id);
+                continue;
+            }
+            if (!extracted)
+                result.excludedSessions++;
+            else {
+                for (const key of ["users", "filtered", "oversized", "unanswered"])
+                    result[key] += extracted.coverage[key];
+                result.eligible += extracted.examples.filter(e => e.timestamp >= since && e.timestamp < until).length;
+            }
+            const counts = store?.syncSession(id, extracted?.examples ?? [], since, until, options.existingOnly ?? false);
+            if (counts)
+                for (const key of ["added", "changed", "unchanged", "retired"])
+                    result[key] += counts[key];
+        }
+        return result;
+    }
+    finally {
+        reader.close();
+    }
+}
+/** Parallel, bounded paid work. Negative results are just as cacheable as positives. */
+export async function runTraining(source, store, config, options) {
+    if ((options.maxExamples !== undefined && (!Number.isSafeInteger(options.maxExamples) || options.maxExamples < 1)) ||
+        !Number.isSafeInteger(options.maxInputBytes) || options.maxInputBytes < 1)
+        throw new Error("Invalid training run bounds");
+    const refreshed = collectTraining(source, store, { existingOnly: true });
+    const concurrency = options.concurrency ?? 256;
+    if (!Number.isSafeInteger(concurrency) || concurrency < 1)
+        throw new Error("Invalid training concurrency");
+    const jobs = store.pending(options.maxExamples);
+    const result = { refreshed, pendingSelected: jobs.length, calls: 0, completed: 0, failed: 0, ambiguous: 0,
+        inputBytes: 0, inputTokens: 0, outputTokens: 0, budgetLimited: false };
+    // Resolve only when there is work. Reruns with no pending inputs need no credentials.
+    const key = jobs.length && !options.dryRun ? await resolveTypeSafeApiKey(config.typesafe) : undefined;
+    if (jobs.length && !options.dryRun && !key)
+        throw new Error("TypeSafe is disabled or its credential is unavailable");
+    let next = 0, stopped = false;
+    const workers = await Promise.allSettled(Array.from({ length: Math.min(concurrency, jobs.length) }, async () => {
+        while (!stopped && !result.budgetLimited && !result.failed && !result.ambiguous) {
+            const job = jobs[next++];
+            if (!job)
+                return;
+            try {
+                const input = JSON.parse(job.inputJson);
+                const bytes = Buffer.byteLength(JSON.stringify({ state: input, questions: TRAINING_GATE_QUESTIONS }));
+                if (result.inputBytes + bytes > options.maxInputBytes) {
+                    result.budgetLimited = true;
+                    return;
+                }
+                result.inputBytes += bytes;
+                if (options.dryRun)
+                    continue;
+                store.renew();
+                const attempt = store.start(job.id); // Commit BEFORE the request can leave this process.
+                result.calls++;
+                let judgment;
+                try {
+                    // Offline labeling has its own deadline, not Whisperer's latency-sensitive timeout.
+                    judgment = await judgeTrainingInput(input, key, AbortSignal.timeout(30_000));
+                }
+                catch (error) {
+                    const definite = error instanceof TypeSafeHttpError && error.status >= 400 && error.status < 500;
+                    const status = definite ? "failed" : "ambiguous";
+                    store.finish(job.id, attempt, { status, error: error instanceof TypeSafeHttpError ? `http_${error.status}` : "request_or_response_uncertain" });
+                    result[status]++;
+                    // Stop on the first failure rather than spending the rest of the budget during an outage.
+                    return;
+                }
+                // Storage failures must not be misclassified as provider failures/retried.
+                store.finish(job.id, attempt, judgment);
+                result.completed++;
+                result.inputTokens += judgment.usage.input_tokens;
+                result.outputTokens += judgment.usage.output_tokens;
+            }
+            catch (error) {
+                stopped = true;
+                throw error;
+            }
+        }
+    }));
+    const failure = workers.find(item => item.status === "rejected");
+    if (failure)
+        throw failure.reason;
+    return result;
+}

package/docs/memory-training.md

@@ -0,0 +1,147 @@
+# Memory training collector
+
+Operator-only collection for **LFM2.5-230M-Base**. No scheduler, runtime recall
+changes, memory writes, live-index mutation or automatic background inference.
+
+## Recipe
+
+1. Collect eligible historical user turns and preceding visible conversation.
+2. TypeSafe `jev-1.13.0` recall probability **>=0.7** gates query generation.
+   Preserve negative labels for audit; greetings do not need query targets.
+3. Isolated `openai/gpt-6-luna`, **xhigh**, generates exactly **10 distinct
+   single-line queries**, using the tested v3 prompt, 12,000 output-token allowance
+   and 300-second deadline. No fallback model or agent tools.
+4. Retrieve **10 literal vector + 10 BM25 matches** from the originating agent's
+   historical sessions snapshot. Retain all unique eligible passages, with no
+   merged passage-count cap.
+5. TypeSafe judges each passage's additional utility for the **original
+   conversation**. It receives conversation, as-of time, passage text/source/dates,
+   never generated query/ID, rank, retrieval score or method.
+6. Sum the **five highest** normalized passage grades; retain the top **three exact
+   queries**. Ties preserve teacher order. No score cutoff, answer requirement or
+   cross-query novelty rule. An empty retrieval scores zero; failures never do.
+
+The four-level rubric distinguishes no, marginal, useful and direct high-value
+additional context, requires correct identity and temporal applicability, discounts
+repetition and unsupported premises, and treats all content as untrusted.
+Full distributions and reported usage are persisted.
+
+## Commands
+
+```sh
+openclaw memory-training collect --agent main --dry-run
+openclaw memory-training collect --agent main
+openclaw memory-training run --agent main --concurrency 256
+openclaw memory-training generate --agent main --concurrency 8
+openclaw memory-training evaluate --agent main --concurrency 4
+openclaw memory-training status --agent main
+openclaw memory-training export --agent main --output /private/query-training.jsonl
+openclaw memory-training export --agent main --stage recall-gate --output /private/recall-gate.jsonl
+```
+
+Collection supports inclusive `--since` and exclusive `--until YYYY-MM-DD` UTC,
+using user-event time, not session start. Omit dates for all eligible history.
+Appends enter only through a later collect; narrow bounds do not delete old cohorts.
+Use the same `--threshold` for generate/evaluate/export/status (default 0.7).
+
+Optional `--max-examples` bounds new work. Run and generate default to a
+3,000,000 serialized input-byte budget, **not tokens**; rerun to drain pending work.
+Evaluate's optional `--max-calls` counts new retrieval operations plus uncached
+passage judgments. There is no default example/call-count cap.
+
+Recall defaults to 256 concurrent requests, generation to 8 isolated completions.
+Evaluation defaults to 4 inputs with 10 parallel queries each. Distinct remote
+passage judgments overlap (up to 800 memberships before dedup at default depth).
+Native vector work is serialized per snapshot. Raise concurrency within machine
+and provider capacity. Failures stop new dispatch; in-flight operations drain
+and persist before snapshots close. Dry runs make no inference calls.
+
+## Input and historical boundaries
+
+Read original agent SQLite active-branch conversations (schema 17–19), including
+direct/group/channel chats, excluding cron, heartbeat, spawned/subagent, hook/plugin
+and untyped diagnostic sessions. Exclude explicit bots, synthetic messages,
+analysis/thinking, errors and tool payloads. Ordinary older users need not have
+enriched sender metadata. Strip recognized transport envelopes.
+
+A following assistant reply/tool action establishes eligibility before the next
+user/context boundary; delivery mirrors count, duplicate visible replies appear once.
+The qualifying **future answer never enters its input**. Earlier visible replies
+may appear in later inputs. Compaction/internal messages break history continuity.
+Keep at most 32 preceding whole messages within 24,000 serialized UTF-8 bytes;
+drop oldest whole messages, never slice the latest request. Oversized requests
+are skipped. Sessions above 50,000 events or 32 MB become unavailable, not deleted.
+
+Snapshots require matching projection hashes and trusted message spans. Copy
+only prefixes before the **entire second of the user's timestamp**, stopping at
+unknown/future dates. Never infer dates from quoted headings. Copy existing vectors
+only for complete chunks within the safe prefix; BM25 sees that same prefix.
+Validate returned passages/dates again. No reembedding or temporary transcripts.
+
+QMD 2.10.1 cannot independently set per-method depth through its public search API.
+A small discovery adapter retains its tokenization, FTS-highlight chunk selection,
+source-aware dedup and installed chunk helpers, but requests ten per method and
+omits query-conditioned scoring. It does not rewrite QMD. Its existing
+12,000-character passage eligibility rule remains; no passage is truncated.
+
+Honor configured session chat types and each node's DM policy. Time-unversioned
+files and Loggie projections are excluded. This is a historical text-prefix
+evaluation of the currently retained corpus, not a reconstruction of the old index:
+later edits/deletions cannot be undone. Persist coverage/exclusion counts.
+
+## Checkpoints, retries and permissions
+
+Private database: `<state>/agents/<agent>/unblock-memory/training.sqlite` (0600).
+Never commit, publish or index this file or its exports.
+
+Source identity includes persisted node ID, agent, session and user-event sequence.
+Exact inputs share recall/teacher checkpoints. The new teacher policy
+`query-teacher-v3-xhigh` distinguishes old low-reasoning results without deleting
+them. Retrieval keys include query, source/cutoff, corpus fingerprint and settings.
+Passage keys include original input, as-of time, exact passage/position and full
+rubric: unchanged judgments survive changes in queries/corpus. Selection is
+separately versioned.
+
+Run/generate/evaluate/export revalidate collected inputs; edits change affected
+hashes, branch removals retire sources, and paid checkpoints remain intact.
+Status does not rescan; stage/attempt totals include historical recipes, not just
+current-cohort progress. Exports filter to the current recipe and active recall gate.
+
+A renewable SQLite lease serializes modifying commands. Attempts commit **before**
+dispatch. Crashes, uncertain transport and malformed responses become ambiguous;
+4xx/host authorization errors are definite failures. Storage errors propagate
+separately. There are no automatic paid retries or model/route fallbacks.
+
+```sh
+openclaw memory-training retry-failed --agent main
+# Explicit acceptance of possible duplicate billing:
+openclaw memory-training retry-failed --agent main --include-ambiguous
+```
+
+These reset that agent's failed checkpoints, including historical recipes, preserving
+attempts. Inspect before use. They do not themselves send requests.
+
+Explicitly authorized exclusions use `evaluate --exclude-judgment <sha256>`.
+The hash is SHA256 of JSON `[judgeVersion,inputHash,passagePosition,fullJudgeRequest]`.
+Exclusions persist, appear in provenance, and are omitted rather than scored zero.
+No blanket failure skipping.
+
+The host must support isolated completion and grant
+`plugins.entries.unblock-memory.llm.allowModelOverride: true` with
+`allowedModels: ["openai/gpt-6-luna"]`. Preserve other grants. The plugin does not
+change its own permissions. Credentials stay with the host, never in provenance.
+
+## Export and consolidation
+
+Export creates a new 0600 JSONL file and refuses overwrite. Query rows contain
+exact inputs, three targets, all query totals/passage references, source/time,
+recall probability, corpus coverage and teacher/retrieval/judgment provenance.
+Recall-gate exports include negatives. Export revalidates input sources but does
+not rerun retrieval; evaluate first if a fresh corpus assessment is desired.
+
+Transfer privately and verify hashes. Deduplicate identical inputs while retaining
+source provenance; quarantine suspected secrets. Keep connected session and
+identical-input groups together across nodes for train/validation. Use the actual
+**LFM2.5-230M-Base** tokenizer and an explicit causal-LM input/target format before
+fine-tuning; the interim byte limit is not a token count. Runtime abstention remains
+a separate TypeSafe gate; positive-only query training does not teach abstention.

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "unblock-memory",
   "name": "Unblock Memory",
-  "version": "0.3.24",
+  "version": "0.3.25",
   "description": "Indexes, retrieves, and analyzes configured workspace memory with existing QMD vectors.",
   "kind": "memory",
   "activation": { "onStartup": true },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@unblocklabs/unblock-memory",
-  "version": "0.3.24",
+  "version": "0.3.25",
   "description": "Workspace-native memory for OpenClaw, powered by QMD",
   "type": "module",
   "license": "MIT",
@@ -26,6 +26,7 @@
     "docs/retrieval.md",
     "docs/peoplesql.md",
     "docs/response-audit.md",
+    "docs/memory-training.md",
     "openclaw.plugin.json"
   ],
   "scripts": {