@unblocklabs/unblock-memory 0.3.24 → 0.3.25

package/dist/src/training-retrieval.js

@@ -0,0 +1,176 @@
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { createRequire } from "node:module";
+import { readSessionManifest } from "./session-sync.js";
+import { resolveSessionSource } from "./sources.js";
+import { trainingCandidates } from "./training-candidates.js";
+export const TRAINING_RETRIEVAL_VERSION = "qmd-2.10.1-historical-prefix-depth10-v2";
+export const TRAINING_SEARCH_OPTIONS = { vector: 10, bm25: 10, mergedLimit: null, rerank: false };
+/** Never infer dates by parsing message bodies: headings can be quoted or forged. */
+export function historicalPrefix(body, spans, cutoff) {
+    if (!spans?.length || !body.startsWith("# Transcript\n\n") || !Number.isFinite(cutoff))
+        return;
+    let previousEnd = 14;
+    const parsed = [];
+    for (const span of spans) {
+        if (![span.start, span.bodyStart, span.end].every(Number.isSafeInteger) || span.start < previousEnd ||
+            span.bodyStart <= span.start || span.end < span.bodyStart || span.end > body.length ||
+            body.slice(previousEnd, span.start).trim())
+            return;
+        const heading = `## ${span.type === "user" ? "User" : "Assistant"} — ${span.name} — ${span.timestamp}\n\n`;
+        if (body.slice(span.start, span.bodyStart) !== heading)
+            return;
+        // Explicit zones only: no process-local timezone interpretation. Unknown zones fail closed.
+        const time = typeof span.timestamp === "string" && / (?:UTC|GMT(?:[+-]\d{1,2}(?::\d{2})?)?|[ECMP][SD]T)$/u.test(span.timestamp)
+            ? Date.parse(span.timestamp) : NaN;
+        parsed.push({ span, time });
+        previousEnd = span.end;
+    }
+    if (body.slice(previousEnd).trim())
+        return;
+    // Projection dates have second precision. Exclude the entire cutoff second,
+    // including the current question, equal-time replies and crossing chunks.
+    const before = Math.floor(cutoff / 1000) * 1000;
+    const safe = [];
+    for (const item of parsed) {
+        if (!Number.isFinite(item.time) || item.time >= before)
+            break;
+        safe.push(item);
+    }
+    if (!safe.length)
+        return;
+    return { body: body.slice(0, parsed[safe.length]?.span.start ?? body.length), spans: safe.map(s => s.span) };
+}
+/** A read-only source snapshot, copied into a disposable in-memory QMD index.
+ * No filesystem projection, live-index mutation, model re-embedding or dependency patch. */
+export async function historicalTrainingSearch(stateDir, chatTypes, cutoff, openStore) {
+    const indexPath = join(stateDir, "index.sqlite");
+    if (!existsSync(indexPath))
+        throw new Error("No QMD index; sync sessions before evaluating training queries");
+    const manifest = await readSessionManifest(join(stateDir, "sessions-manifest.json"));
+    const source = resolveSessionSource(join(stateDir, "sessions"), chatTypes);
+    const createStore = openStore ?? (await import("@unblocklabs/qmd")).createStore;
+    const qmd = await createStore({ dbPath: ":memory:", keepModelsWarm: true,
+        config: { collections: { [source.collection]: { path: source.root, pattern: "**/*.md" } } } });
+    // A snapshot has one native embedding context. Serialize its vector stage only;
+    // QMD's remote TypeSafe scoring remains concurrent across all ten queries.
+    const searchVec = qmd.internal.searchVec;
+    let vectorTail = Promise.resolve();
+    qmd.internal.searchVec = (...args) => {
+        const result = vectorTail.then(() => searchVec(...args));
+        vectorTail = result.then(() => { }, () => { });
+        return result;
+    };
+    const db = qmd.internal.db;
+    const maxDate = new Date(cutoff).toISOString();
+    const report = { sessions: 0, chunks: 0, excluded: 0, truncated: 0, excludedChunks: 0 };
+    const metadata = new Map();
+    const fingerprint = createHash("sha256").update(JSON.stringify([TRAINING_RETRIEVAL_VERSION, chatTypes.toSorted(), maxDate]));
+    try {
+        // QMD's SDK only opens writable stores. Use its exact SQLite binding in
+        // read-only mode; mixing node:sqlite with better-sqlite3 corrupts sqlite-vec's
+        // process-global SQLite API pointer when both load the native extension.
+        const requireQmd = createRequire(import.meta.resolve("@unblocklabs/qmd"));
+        const Database = requireQmd("better-sqlite3");
+        const sourceDb = new Database(indexPath, { readonly: true, fileMustExist: true });
+        try {
+            const extension = requireQmd("sqlite-vec");
+            if (!extension || typeof extension !== "object" || !("getLoadablePath" in extension) || typeof extension.getLoadablePath !== "function") {
+                throw new Error("QMD sqlite-vec extension unavailable");
+            }
+            const extensionPath = extension.getLoadablePath();
+            if (typeof extensionPath !== "string")
+                throw new Error("QMD sqlite-vec path unavailable");
+            sourceDb.loadExtension(extensionPath);
+            sourceDb.exec("PRAGMA query_only=ON; PRAGMA busy_timeout=5000; BEGIN");
+            db.exec("BEGIN");
+            const settings = sourceDb.prepare("SELECT key,value FROM store_config WHERE key='embedding_chunk_strategy'").all();
+            for (const { key, value } of settings)
+                db.prepare("INSERT OR REPLACE INTO store_config VALUES (?,?)").run(key, value);
+            fingerprint.update(JSON.stringify(settings));
+            const document = sourceDb.prepare(`SELECT d.hash,c.doc FROM documents d JOIN content c ON c.hash=d.hash
+        WHERE d.active=1 AND d.collection=? AND d.path=?`);
+            const chunks = sourceDb.prepare(`SELECT cv.seq,cv.pos,cv.chunk_len,cv.model,cv.embed_fingerprint,v.embedding
+        FROM content_vectors cv JOIN vectors_vec v ON v.hash_seq=cv.hash||'_'||cv.seq
+        WHERE cv.hash=? ORDER BY cv.seq`);
+            let dimensions;
+            for (const session of Object.values(manifest.sessions).sort((a, b) => a.documentPath.localeCompare(b.documentPath))) {
+                // Loggie projections can retroactively annotate old text using later revisions.
+                // Workspace files and meetings lack immutable historical content proof here.
+                if (!chatTypes.includes(session.chatType) || session.provider === "loggie" || session.startedAt >= cutoff) {
+                    report.excluded++;
+                    continue;
+                }
+                const row = document.get(source.collection, session.documentPath);
+                if (!row || createHash("sha256").update(row.doc).digest("hex") !== session.projectionHash) {
+                    report.excluded++;
+                    continue;
+                }
+                const prefix = historicalPrefix(row.doc, session.messages, cutoff);
+                if (!prefix) {
+                    report.excluded++;
+                    continue;
+                }
+                const hash = createHash("sha256").update(prefix.body).digest("hex");
+                qmd.internal.insertContent(hash, prefix.body, maxDate);
+                qmd.internal.insertDocument(source.collection, session.documentPath, "Transcript", hash, maxDate, maxDate);
+                metadata.set(`qmd://${source.collection}/${session.documentPath}`, prefix.spans);
+                fingerprint.update(JSON.stringify([session.documentPath, hash]));
+                report.sessions++;
+                if (prefix.spans.length < (session.messages?.length ?? 0))
+                    report.truncated++;
+                for (const chunk of chunks.all(row.hash)) {
+                    if (!Number.isSafeInteger(chunk.pos) || !Number.isSafeInteger(chunk.chunk_len) || chunk.pos < 0 || chunk.chunk_len <= 0 ||
+                        chunk.pos + chunk.chunk_len > prefix.body.length) {
+                        report.excludedChunks++;
+                        continue;
+                    }
+                    const bytes = Buffer.from(chunk.embedding);
+                    if (bytes.length % 4 || !bytes.length)
+                        throw new Error("Invalid historical embedding");
+                    const size = bytes.length / 4;
+                    if (dimensions === undefined) {
+                        dimensions = size;
+                        qmd.internal.ensureVecTable(size);
+                    }
+                    if (dimensions !== size)
+                        throw new Error("Mixed historical embedding dimensions");
+                    const vector = new Float32Array(size);
+                    for (let i = 0; i < size; i++)
+                        vector[i] = bytes.readFloatLE(i * 4);
+                    qmd.internal.insertEmbedding(hash, chunk.seq, chunk.pos, vector, chunk.model, maxDate, 1, chunk.embed_fingerprint, chunk.chunk_len);
+                    fingerprint.update(JSON.stringify([chunk.seq, chunk.pos, chunk.chunk_len, chunk.model, chunk.embed_fingerprint])).update(bytes);
+                    report.chunks++;
+                }
+            }
+            db.exec("COMMIT");
+        }
+        finally {
+            sourceDb.close();
+        }
+        const corpusHash = fingerprint.digest("hex");
+        return { corpusHash, report, maxDate,
+            search: async (query) => {
+                if (!report.sessions)
+                    return [];
+                if (!report.chunks)
+                    throw new Error("Historical snapshot has no vectors; refusing a BM25-only evaluation");
+                const hits = await trainingCandidates(qmd, query, source.collection, `Historical request made at ${maxDate}. Current, now and latest refer to that timestamp.`);
+                return hits.map(hit => trainingHit(hit, metadata, cutoff));
+            }, close: () => qmd.close() };
+    }
+    catch (error) {
+        await qmd.close();
+        throw error;
+    }
+}
+function trainingHit(hit, metadata, cutoff) {
+    const spans = metadata.get(hit.file)?.filter(s => s.start < hit.bestChunkPos + hit.bestChunk.length && s.end > hit.bestChunkPos);
+    if (!spans?.length || spans.some(s => !(Date.parse(s.timestamp) < Math.floor(cutoff / 1000) * 1000)) ||
+        hit.body.slice(hit.bestChunkPos, hit.bestChunkPos + hit.bestChunk.length) !== hit.bestChunk) {
+        throw new Error("QMD returned evidence outside the historical snapshot");
+    }
+    return { path: hit.file, position: hit.bestChunkPos, text: hit.bestChunk, dates: [...new Set(spans.map(s => s.timestamp))],
+        score: hit.score, methods: hit.explain?.methods ?? [] };
+}

package/dist/src/training-runtime.d.ts

@@ -0,0 +1,4 @@
+import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
+import type { UnblockMemoryConfig } from "./config.js";
+/** CLI only: no scheduler, tools, hooks, or live memory-index changes. */
+export declare function registerMemoryTraining(api: OpenClawPluginApi, config: UnblockMemoryConfig): void;

package/dist/src/training-runtime.js

@@ -0,0 +1,140 @@
+import { existsSync, openSync, writeSync, closeSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { resolveAgentDir, resolveStateDir } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
+import { listAgentIds } from "openclaw/plugin-sdk/agent-runtime";
+import { normalizeAgentIdStrict } from "openclaw/plugin-sdk/routing";
+import { TrainingStore } from "./training-store.js";
+import { TRAINING_GATE_THRESHOLD } from "./training-gate.js";
+import { collectTraining, runTraining } from "./training.js";
+import { generateTrainingQueries, evaluateTrainingQueries, exportQueryTraining, TRAINING_EVALUATION_CONCURRENCY } from "./training-queries.js";
+function dateOption(value, fallback) {
+    if (value === undefined)
+        return fallback;
+    if (!/^\d{4}-\d{2}-\d{2}$/.test(value) || !Number.isFinite(Date.parse(value)) || new Date(value).toISOString().slice(0, 10) !== value) {
+        throw new Error("Dates must be valid YYYY-MM-DD UTC dates");
+    }
+    return Date.parse(value);
+}
+function thresholdOption(value) {
+    const result = Number(value);
+    if (!value.trim() || !Number.isFinite(result) || result < 0 || result > 1)
+        throw new Error("Threshold must be between 0 and 1");
+    return result;
+}
+/** CLI only: no scheduler, tools, hooks, or live memory-index changes. */
+export function registerMemoryTraining(api, config) {
+    const paths = (cfg, id) => {
+        const normalized = normalizeAgentIdStrict(id);
+        if (!normalized.ok || !listAgentIds(cfg).includes(normalized.value))
+            throw new Error("Unknown memory-training agent");
+        const agentId = normalized.value;
+        const stateDir = join(resolveStateDir(), "agents", agentId, "unblock-memory");
+        return { agentId, databasePath: join(resolveAgentDir(cfg, agentId), "openclaw-agent.sqlite"),
+            stateDir, storePath: join(stateDir, "training.sqlite") };
+    };
+    api.registerCli(({ program, config: cfg }) => {
+        const root = program.command("memory-training").description("Operator-only resumable LFM query-training pipeline");
+        const withStore = async (agent, options, fn) => {
+            const source = paths(cfg, agent);
+            if (!options.create && !existsSync(source.storePath))
+                throw new Error("No training database; run memory-training collect first");
+            const store = new TrainingStore(source.storePath, source.agentId);
+            try {
+                return await (options.readOnly ? fn(store, source) : store.locked(() => fn(store, source)));
+            }
+            finally {
+                store.close();
+            }
+        };
+        root.command("collect").option("--agent <id>", "Agent id", "main")
+            .option("--since <date>", "Inclusive user-message date YYYY-MM-DD UTC")
+            .option("--until <date>", "Exclusive user-message date YYYY-MM-DD UTC")
+            .option("--dry-run", "Coverage only; no database writes or API calls")
+            .action(async (opts) => {
+            const since = dateOption(opts.since, 0), until = dateOption(opts.until, Number.MAX_SAFE_INTEGER);
+            if (until <= since)
+                throw new Error("until must be later than since");
+            const result = opts.dryRun ? collectTraining(paths(cfg, opts.agent), undefined, { since, until }) :
+                await withStore(opts.agent, { create: true }, (store, source) => collectTraining(source, store, { since, until }));
+            console.log(JSON.stringify(result, null, 2));
+        });
+        root.command("run").description("Judge whether historical recall would help each collected input").option("--agent <id>", "Agent id", "main")
+            .option("--max-examples <n>", "Optional maximum new API calls; no example-count limit by default")
+            .option("--max-input-bytes <n>", "Maximum total serialized state + question bytes, not tokens", "3000000")
+            .option("--concurrency <n>", "Concurrent TypeSafe recall requests", "256")
+            .option("--dry-run", "Refresh local checkpoints and preview pending work without API calls")
+            .action(async (opts) => {
+            const result = await withStore(opts.agent, {}, (store, source) => runTraining(source, store, config, {
+                maxExamples: opts.maxExamples === undefined ? undefined : Number(opts.maxExamples),
+                maxInputBytes: Number(opts.maxInputBytes), concurrency: Number(opts.concurrency), dryRun: opts.dryRun,
+            }));
+            console.log(JSON.stringify(result, null, 2));
+            if (result.failed || result.ambiguous)
+                process.exitCode = 1;
+        });
+        for (const command of ["generate", "evaluate"]) {
+            const stage = root.command(command).description(command === "generate" ? "Generate ten exact queries with isolated gpt-6-luna" : "Rank queries using existing historical QMD results")
+                .option("--agent <id>", "Agent id", "main")
+                .option("--threshold <p>", "Minimum completed recall probability", String(TRAINING_GATE_THRESHOLD))
+                .option("--max-examples <n>", "Optional maximum uncached examples")
+                .option("--dry-run", "Refresh local sources and preview work without provider calls");
+            if (command === "generate")
+                stage.option("--max-input-bytes <n>", "Teacher input byte budget", "3000000")
+                    .option("--concurrency <n>", "Concurrent isolated teacher completions", "8");
+            else
+                stage.option("--max-calls <n>", "Maximum new retrieval operations plus uncached passage judgments")
+                    .option("--exclude-judgment <hash...>", "Explicitly exclude exact judgment hashes; persist exclusions, never score as zero")
+                    .option("--concurrency <n>", "Concurrent historical inputs; remote passage judgments run concurrently", String(TRAINING_EVALUATION_CONCURRENCY));
+            stage.action(async (opts) => {
+                const options = { maxExamples: opts.maxExamples === undefined ? undefined : Number(opts.maxExamples),
+                    dryRun: opts.dryRun, threshold: thresholdOption(opts.threshold) };
+                const result = await withStore(opts.agent, {}, async (store, source) => command === "generate"
+                    ? await generateTrainingQueries(source, store, api.runtime, { ...options, maxInputBytes: Number(opts.maxInputBytes), concurrency: Number(opts.concurrency) })
+                    : await evaluateTrainingQueries(source, store, config, { ...options, maxCalls: opts.maxCalls === undefined ? undefined : Number(opts.maxCalls),
+                        concurrency: Number(opts.concurrency), excludeJudgments: opts.excludeJudgment }));
+                console.log(JSON.stringify(result, null, 2));
+                if (result.failed || result.ambiguous || result.blocked)
+                    process.exitCode = 1;
+            });
+        }
+        root.command("status").option("--agent <id>", "Agent id", "main")
+            .option("--threshold <p>", "Minimum completed recall probability for query eligibility", String(TRAINING_GATE_THRESHOLD))
+            .action(async (opts) => {
+            const threshold = thresholdOption(opts.threshold);
+            console.log(JSON.stringify(await withStore(opts.agent, { readOnly: true }, store => store.status(threshold)), null, 2));
+        });
+        root.command("retry-failed").option("--agent <id>", "Agent id", "main")
+            .option("--include-ambiguous", "Explicitly permit retrying requests that may already have been billed")
+            .action(async (opts) => {
+            const reset = await withStore(opts.agent, {}, store => store.retry(opts.includeAmbiguous === true));
+            console.log(JSON.stringify({ reset, calls: 0 }));
+        });
+        root.command("export").option("--agent <id>", "Agent id", "main")
+            .option("--threshold <p>", "Minimum completed recall probability", String(TRAINING_GATE_THRESHOLD))
+            .option("--stage <stage>", "query-training or recall-gate", "query-training")
+            .requiredOption("--output <path>", "New private JSONL file; refuses overwrite")
+            .action(async (opts) => {
+            if (!["recall-gate", "query-training"].includes(opts.stage))
+                throw new Error("Unknown training export stage");
+            const output = resolve(opts.output);
+            const threshold = thresholdOption(opts.threshold);
+            const result = await withStore(opts.agent, {}, (store, source) => {
+                collectTraining(source, store, { existingOnly: true });
+                const fd = openSync(output, "wx", 0o600);
+                let rows = 0;
+                const data = opts.stage === "query-training" ? exportQueryTraining(store, threshold) : store.exportRows(threshold);
+                try {
+                    for (const row of data) {
+                        writeSync(fd, JSON.stringify(row) + "\n");
+                        rows++;
+                    }
+                }
+                finally {
+                    closeSync(fd);
+                }
+                return { output, rows, stage: opts.stage };
+            });
+            console.log(JSON.stringify(result, null, 2));
+        });
+    }, { descriptors: [{ name: "memory-training", description: "Collect, generate and rank resumable LFM query examples", hasSubcommands: true }] });
+}

package/dist/src/training-store.d.ts

@@ -0,0 +1,160 @@
+import { type TrainingExample, type TrainingInput } from "./training-input.js";
+import { type judgeTrainingInput } from "./training-gate.js";
+import type { TeacherResult } from "./training-models.js";
+import type { TrainingHit } from "./training-retrieval.js";
+import type { parseContextJudgment } from "./training-judge.js";
+type GateResult = Awaited<ReturnType<typeof judgeTrainingInput>>;
+type Job = {
+    id: string;
+    inputHash: string;
+    inputJson: string;
+};
+export type TrainingSourceExample = {
+    id: string;
+    inputHash: string;
+    inputJson: string;
+    sessionId: string;
+    timestamp: number;
+};
+export type QueryEvaluation = {
+    query: string;
+    retrievalId: string;
+    score: number;
+    judgments?: {
+        id: string;
+        excluded: boolean;
+    }[];
+};
+type TrainingEvaluation = {
+    sourceId: string;
+    inputHash: string;
+    timestamp: number;
+    corpusHash: string;
+    teacherId: string;
+    corpusReport: {
+        sessions: number;
+        chunks: number;
+        excluded: number;
+        truncated: number;
+        excludedChunks: number;
+    };
+    queries: QueryEvaluation[];
+    selected: string[];
+};
+export type TrainingStepResults = {
+    generate: TeacherResult;
+    retrieve: {
+        query: string;
+        maxDate: string;
+        corpusHash: string;
+        hits: TrainingHit[];
+    };
+    judge: ReturnType<typeof parseContextJudgment> | {
+        excluded: true;
+        reason: "operator-exclusion";
+    };
+    evaluate: TrainingEvaluation;
+};
+type StepStatus = "pending" | "attempted" | "complete" | "failed" | "ambiguous";
+export declare class TrainingStore {
+    #private;
+    constructor(path: string, agentId: string);
+    locked<T>(fn: () => T | Promise<T>): Promise<T>;
+    renew(): void;
+    sessions(): string[];
+    syncSession(sessionId: string, examples: TrainingExample[], since: number, until: number, existingOnly: boolean): {
+        added: number;
+        changed: number;
+        unchanged: number;
+        retired: number;
+    };
+    unavailable(sessionId: string): void;
+    pending(limit?: number): Job[];
+    start(id: string): number;
+    finish(id: string, attempt: number, result: GateResult | {
+        status: "failed" | "ambiguous";
+        error: string;
+    }): void;
+    retry(includeAmbiguous: boolean): number;
+    activeExamples(): TrainingSourceExample[];
+    queryExamples(threshold?: number): {
+        recallProbability: number;
+        id: string;
+        inputHash: string;
+        inputJson: string;
+        sessionId: string;
+        timestamp: number;
+    }[];
+    step<S extends keyof TrainingStepResults>(stage: S, request: unknown): {
+        id: string;
+        stage: S;
+        request: unknown;
+        status: StepStatus;
+        result: TrainingStepResults[S] | undefined;
+    };
+    judgmentExcluded(identity: string): boolean;
+    startStep(stage: keyof TrainingStepResults, id: string, request: unknown): number;
+    finishStep<S extends keyof TrainingStepResults>(stage: S, id: string, attempt: number, outcome: {
+        result: TrainingStepResults[S];
+    } | {
+        status: "failed" | "ambiguous";
+        error: string;
+    }): void;
+    completedEvaluations(versions?: {
+        selection: string;
+        retrieval: string;
+    }): TrainingEvaluation[];
+    sourceDetails(id: string): {
+        nodeId: string;
+        agentId: string;
+        sourceId: string;
+    };
+    stepRecord(id: string): {
+        id: string;
+        stage: import("node:sqlite").SQLOutputValue;
+        request: unknown;
+        result: unknown;
+        completedAt: import("node:sqlite").SQLOutputValue;
+    };
+    status(threshold: number): {
+        nodeId: string;
+        agentId: string;
+        preparation: string;
+        promptVersion: string;
+        requestedModel: string;
+        threshold: number;
+        examples: Record<string, import("node:sqlite").SQLOutputValue>[];
+        collectedInputs: number;
+        queryInputs: number;
+        stages: Record<string, import("node:sqlite").SQLOutputValue>[];
+        complete: number;
+        positive: number;
+        inputTokens: number;
+        outputTokens: number;
+        negative: number;
+        queryStages: Record<string, import("node:sqlite").SQLOutputValue>[];
+        queryAttempts: Record<string, import("node:sqlite").SQLOutputValue>[];
+        attempts: Record<string, import("node:sqlite").SQLOutputValue>[];
+    };
+    exportRows(threshold: number): Generator<{
+        stage: string;
+        inputHash: import("node:sqlite").SQLOutputValue;
+        preparation: string;
+        input: TrainingInput;
+        recallProbability: import("node:sqlite").SQLOutputValue;
+        recallNeeded: boolean;
+        threshold: number;
+        model: import("node:sqlite").SQLOutputValue;
+        promptVersion: import("node:sqlite").SQLOutputValue;
+        usage: {
+            input_tokens: import("node:sqlite").SQLOutputValue;
+            output_tokens: import("node:sqlite").SQLOutputValue;
+        };
+        sources: {
+            nodeId: string;
+            agentId: string;
+        }[];
+    }, void, unknown>;
+    close(): void;
+}
+export {};