@unblocklabs/unblock-memory 0.6.2 → 0.7.0

package/dist/src/extraction-runtime.js

@@ -0,0 +1,121 @@
+import { join } from "node:path";
+import { Type } from "typebox";
+import { Value } from "typebox/value";
+import { callGatewayFromCli, errorShape, ErrorCodes } from "openclaw/plugin-sdk/gateway-runtime";
+import { listAgentIds, resolveAgentIdentity } from "openclaw/plugin-sdk/agent-runtime";
+import { resolveAgentDir, resolveStateDir } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
+import { MEMORY_DATABASE, hasMemoryTable } from "./memory-database.js";
+import { ExtractionStore } from "./extraction-store.js";
+import { extractionSessions } from "./extraction-source.js";
+export function registerExtraction(api, config, runtime) {
+    let lifetime = new AbortController();
+    let timer, running;
+    const options = (cfg, agentId) => {
+        if (!listAgentIds(cfg).includes(agentId))
+            throw new Error("Unknown extraction agent");
+        return { agentId, agentName: resolveAgentIdentity(cfg, agentId)?.name?.trim() || agentId,
+            config, runtime: api.runtime, storePath: join(resolveStateDir(), "agents", agentId, "unblock-memory", MEMORY_DATABASE),
+            sessions: () => extractionSessions(join(resolveAgentDir(cfg, agentId), "openclaw-agent.sqlite"), agentId, config.extraction.chatTypes) };
+    };
+    const run = async (cfg, agentId, signal, since, sessionId, scheduled = false) => {
+        const opts = options(cfg, agentId);
+        // Keep the extraction tokenizer/model modules out of normal plugin startup.
+        const { runExtraction } = await import("./extraction-worker.js");
+        const result = await runExtraction({ ...opts, signal, since, scheduled, sessionId });
+        if (result.status === "completed" && config.extraction.publish && runtime) {
+            const { manager, error } = await runtime.getMemorySearchManager({ cfg, agentId });
+            if (!manager)
+                throw new Error(error ?? "Extraction index unavailable");
+            await manager.syncExtracted();
+        }
+        return result;
+    };
+    api.registerCli(({ program, config: cfg }) => {
+        const root = program.command("memory-extract").description("Optional session fact extraction; no daily Markdown writes");
+        root.command("run").option("--agent <id>", "Agent id", "main")
+            .option("--since <date>", "Explicit bounded backfill from YYYY-MM-DD UTC; repeat to resume")
+            .option("--session <id>", "Restrict this run to one session")
+            .action(async (opts) => {
+            let since;
+            if (opts.since !== undefined) {
+                since = Date.parse(opts.since);
+                if (!/^\d{4}-\d{2}-\d{2}$/.test(opts.since) || !Number.isFinite(since) || new Date(since).toISOString().slice(0, 10) !== opts.since) {
+                    throw new Error("since must be a valid YYYY-MM-DD UTC date");
+                }
+            }
+            try {
+                // Native harnesses live in the Gateway, not the cold CLI metadata loader.
+                const result = await callGatewayFromCli("unblock-memory.extract", { timeout: "600000" }, { agentId: opts.agent, ...(since === undefined ? {} : { since }), ...(opts.session ? { sessionId: opts.session } : {}) });
+                console.log(JSON.stringify(result, null, 2));
+                if (result.status === "unavailable" || (result.status === "completed" && result.failed))
+                    process.exitCode = 1;
+            }
+            catch {
+                console.log(JSON.stringify({ status: "unavailable", reason: "Extraction failed; check the running Gateway, host prerequisites and operator report" }));
+                process.exitCode = 1;
+            }
+        });
+        root.command("report").option("--agent <id>", "Agent id", "main").action((opts) => {
+            const { storePath } = options(cfg, opts.agent);
+            if (!hasMemoryTable(storePath, "extracted_memories")) {
+                console.log(JSON.stringify({ status: "not_run" }));
+                return;
+            }
+            const store = new ExtractionStore(storePath);
+            try {
+                console.log(JSON.stringify(store.report(), null, 2));
+            }
+            finally {
+                store.close();
+            }
+        });
+    }, { descriptors: [{ name: "memory-extract", description: "Extract and inspect source-linked session facts", hasSubcommands: true }] });
+    if (api.registrationMode === "cli-metadata")
+        return;
+    const runParams = Type.Object({ agentId: Type.String(), since: Type.Optional(Type.Integer({ minimum: 0 })),
+        sessionId: Type.Optional(Type.String()) }, { additionalProperties: false });
+    api.registerGatewayMethod("unblock-memory.extract", async ({ params, respond }) => {
+        if (!Value.Check(runParams, params)) {
+            respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, "Invalid extraction parameters"));
+            return;
+        }
+        try {
+            respond(true, await run(api.config, params.agentId, lifetime.signal, params.since, params.sessionId));
+        }
+        catch {
+            respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "Extraction failed; check host prerequisites and operator report"));
+        }
+    }, { scope: "operator.admin" });
+    if (!config.extraction.enabled)
+        return;
+    api.on("gateway_stop", async () => { lifetime.abort(); if (timer)
+        clearInterval(timer); timer = undefined; await running; });
+    if (!config.extraction.intervalMinutes || !config.typesafe.enabled)
+        return;
+    const tick = () => {
+        if (running || lifetime.signal.aborted)
+            return;
+        running = (async () => {
+            for (const agentId of listAgentIds(api.config)) {
+                if (lifetime.signal.aborted)
+                    break;
+                try {
+                    const result = await run(api.config, agentId, lifetime.signal, undefined, undefined, true);
+                    if (result.status === "unavailable" || (result.status === "completed" && result.failed))
+                        api.logger.warn("unblock-memory extraction unavailable; inspect memory-extract report");
+                }
+                catch {
+                    api.logger.warn("unblock-memory extraction unavailable");
+                }
+            }
+        })().finally(() => { running = undefined; });
+    };
+    api.on("gateway_start", () => {
+        if (timer)
+            clearInterval(timer);
+        lifetime = new AbortController();
+        timer = setInterval(tick, 60_000);
+        timer.unref();
+        tick();
+    });
+}

package/dist/src/extraction-source.d.ts

@@ -0,0 +1,26 @@
+import type { ChatType } from "./config.js";
+import type { ExtractionSession } from "./extraction-store.js";
+import type { ExtractionMessage } from "./extraction-model.js";
+export declare function extractionSessions(databasePath: string, agentId: string, chatTypes: ChatType[]): ExtractionSession[];
+export type ExtractionPage = {
+    kind: "page";
+    cursor: string;
+    hasMore: boolean;
+    messages: ExtractionMessage[];
+    entryCount: number;
+} | {
+    kind: "reset";
+    cursor: string;
+} | {
+    kind: "missing";
+} | {
+    kind: "unavailable";
+};
+export declare function readExtractionPage(agentId: string, agentName: string, session: ExtractionSession, cursor: string | null, read?: (params: {
+    agentId: string;
+    sessionId: string;
+    sessionKey: string;
+    cursor?: string;
+    maxMessages: number;
+    maxBytes: number;
+}) => Promise<unknown>): Promise<ExtractionPage>;

package/dist/src/extraction-source.js

@@ -0,0 +1,66 @@
+import { DatabaseSync } from "node:sqlite";
+import { Type } from "typebox";
+import { Value } from "typebox/value";
+import { projectMessage } from "./session-projector.js";
+export function extractionSessions(databasePath, agentId, chatTypes) {
+    const db = new DatabaseSync(databasePath, { readOnly: true });
+    try {
+        db.exec("PRAGMA query_only=ON; PRAGMA busy_timeout=5000");
+        if (db.prepare("SELECT agent_id FROM schema_meta WHERE meta_key='primary'").get()?.agent_id !== agentId) {
+            throw new Error("Wrong extraction source agent");
+        }
+        return db.prepare(`SELECT w.session_id,w.session_key,w.chat_type,COALESCE(w.channel,c.channel) AS provider,
+      COALESCE(w.account_id,c.account_id) AS account_id,COALESCE(c.native_channel_id,c.native_direct_user_id,c.peer_id,w.primary_conversation_id) AS conversation_id,
+      COALESCE(w.started_at,w.created_at) AS started_at, COALESCE(w.transcript_updated_at,w.updated_at,w.created_at) AS changed_at FROM session_windows w
+      LEFT JOIN conversations c ON c.conversation_id=w.primary_conversation_id
+      WHERE w.chat_type IN (${chatTypes.map(() => "?").join(",")}) ORDER BY w.created_at,w.session_id`)
+            .all(...chatTypes).map(r => ({ sessionId: String(r.session_id), sessionKey: String(r.session_key),
+            chatType: r.chat_type, provider: r.provider === null ? undefined : String(r.provider),
+            accountId: r.account_id === null ? undefined : String(r.account_id),
+            conversationId: r.conversation_id === null ? undefined : String(r.conversation_id), startedAt: Number(r.started_at), changedAt: Number(r.changed_at) }));
+    }
+    finally {
+        db.close();
+    }
+}
+const pageSchema = Type.Object({ kind: Type.Literal("page"), cursor: Type.String(), hasMore: Type.Boolean(),
+    requiredBytes: Type.Optional(Type.Number()), entries: Type.Array(Type.Object({ entryId: Type.String(),
+        message: Type.Unknown(), createdAt: Type.Optional(Type.String()) })) });
+export async function readExtractionPage(agentId, agentName, session, cursor, read) {
+    // Runtime-only import: the supported host export ships JS without a declaration file.
+    const sdkPath = "openclaw/plugin-sdk/session-transcript-runtime";
+    const sdk = read ? undefined : await import(sdkPath);
+    const readDelta = read ?? sdk?.readSessionTranscriptVisibleMessageDelta;
+    if (!readDelta)
+        throw new Error("Host visible transcript API unavailable");
+    let maxBytes = 1_000_000;
+    let result;
+    for (;;) {
+        result = await readDelta({ agentId, sessionId: session.sessionId,
+            sessionKey: session.sessionKey, ...(cursor ? { cursor } : {}), maxMessages: 40, maxBytes });
+        if (!Value.Check(pageSchema, result) || result.entries.length || !result.requiredBytes)
+            break;
+        // The host caps reads at 64 MiB. This is an allocation bound, not a model budget.
+        if (!Number.isSafeInteger(result.requiredBytes) || result.requiredBytes <= maxBytes || result.requiredBytes > 64 * 1024 * 1024) {
+            throw new Error("Transcript entry exceeds host read capacity");
+        }
+        maxBytes = result.requiredBytes;
+    }
+    if (Value.Check(pageSchema, result)) {
+        const messages = result.entries.flatMap(entry => {
+            const createdAt = Date.parse(entry.createdAt ?? "");
+            if (!Number.isFinite(createdAt))
+                throw new Error("Extraction message has no source date");
+            const projected = projectMessage({ eventJson: JSON.stringify({ type: "message", timestamp: entry.createdAt, message: entry.message }), createdAt }, { ...session, agentName, timezone: "UTC", events: [] });
+            return projected?.text ? [{ id: entry.entryId, speaker: projected.speaker, role: projected.role, text: projected.text, timestamp: projected.timestamp }] : [];
+        });
+        return { kind: "page", cursor: result.cursor, hasMore: result.hasMore, messages, entryCount: result.entries.length };
+    }
+    const resetSchema = Type.Object({ kind: Type.Literal("reset"), cursor: Type.String() });
+    if (Value.Check(resetSchema, result))
+        return { kind: "reset", cursor: result.cursor };
+    const missingSchema = Type.Object({ kind: Type.Union([Type.Literal("missing"), Type.Literal("unavailable")]) });
+    if (Value.Check(missingSchema, result))
+        return { kind: result.kind };
+    throw new Error("Invalid host transcript page");
+}

package/dist/src/extraction-store.d.ts

@@ -0,0 +1,52 @@
+import type { ExtractionMessage, MemoryProposal } from "./extraction-model.js";
+import type { SessionMetadata } from "./session-projector.js";
+export type ExtractionSession = SessionMetadata & {
+    sessionKey: string;
+    changedAt?: number;
+};
+type Checkpoint = {
+    cursor: string | null;
+    context: ExtractionMessage[];
+    since: number;
+};
+export type ExtractedRecord = {
+    id: string;
+    revision: number;
+    text: string;
+    sessionId: string;
+    observedAt: number;
+    evidence: MemoryProposal["evidence"];
+    metadata: ExtractionSession;
+    judgment: unknown;
+};
+export declare class ExtractionStore {
+    readonly db: import("node:sqlite").DatabaseSync;
+    constructor(path: string);
+    close(): void;
+    liveSince(): number;
+    claim(scheduled: boolean, intervalMs: number): string | undefined;
+    renew(owner: string): void;
+    release(owner: string): void;
+    checkpoint(session: ExtractionSession, since?: number): Checkpoint;
+    records(sessionId?: string, paths?: readonly string[]): ExtractedRecord[];
+    reset(sessionId: string, cursor: string | null, owner: string): void;
+    commit(params: {
+        session: ExtractionSession;
+        expected: string | null;
+        cursor: string;
+        context: ExtractionMessage[];
+        accepted: {
+            proposal: MemoryProposal;
+            observedAt: number;
+            judgment: unknown;
+        }[];
+        owner: string;
+        version: string;
+    }): number;
+    error(sessionId: string, code: string): void;
+    report(): {
+        memories: ExtractedRecord[];
+        sessions: Record<string, import("node:sqlite").SQLOutputValue>[];
+    };
+}
+export {};

package/dist/src/extraction-store.js

@@ -0,0 +1,138 @@
+import { createHash, randomUUID } from "node:crypto";
+import { openMemoryDatabase } from "./memory-database.js";
+function memoryDigest(text) {
+    return createHash("sha256").update(text.trim().toLowerCase().replace(/\s+/g, " ").replace(/[.!]+$/, "")).digest("hex");
+}
+export class ExtractionStore {
+    db;
+    constructor(path) {
+        this.db = openMemoryDatabase(path);
+        try {
+            const version = this.db.prepare("SELECT version FROM memory_schema WHERE component='extraction'").get()?.version;
+            if (version !== undefined && version !== 1)
+                throw new Error("Unsupported extraction schema");
+            this.db.exec(`
+      CREATE TABLE IF NOT EXISTS extraction_worker (
+        id INTEGER PRIMARY KEY CHECK(id=1), owner TEXT, lease_until INTEGER NOT NULL DEFAULT 0,
+        next_run INTEGER NOT NULL DEFAULT 0, live_since INTEGER NOT NULL
+      ) STRICT;
+      CREATE TABLE IF NOT EXISTS extraction_sessions (
+        session_id TEXT PRIMARY KEY, metadata TEXT NOT NULL CHECK(json_valid(metadata)), cursor TEXT,
+        extract_since INTEGER NOT NULL,
+        context TEXT NOT NULL DEFAULT '[]' CHECK(json_valid(context)), updated_at INTEGER NOT NULL,
+        error TEXT
+      ) STRICT;
+      CREATE TABLE IF NOT EXISTS extracted_memories (
+        id TEXT NOT NULL, revision INTEGER NOT NULL, session_id TEXT NOT NULL REFERENCES extraction_sessions(session_id),
+        text TEXT NOT NULL, digest TEXT NOT NULL, evidence TEXT NOT NULL CHECK(json_valid(evidence)),
+        observed_at INTEGER NOT NULL, status TEXT NOT NULL CHECK(status IN ('active','superseded','withdrawn')),
+        judgment TEXT NOT NULL CHECK(json_valid(judgment)), prompt_version TEXT NOT NULL, created_at INTEGER NOT NULL,
+        PRIMARY KEY(id,revision)
+      ) STRICT;
+      CREATE UNIQUE INDEX IF NOT EXISTS extracted_current ON extracted_memories(id) WHERE status='active';
+      CREATE UNIQUE INDEX IF NOT EXISTS extracted_duplicate ON extracted_memories(session_id,digest) WHERE status='active';
+      INSERT OR IGNORE INTO memory_schema VALUES('extraction',1);
+    `);
+            this.db.prepare("INSERT OR IGNORE INTO extraction_worker(id,live_since) VALUES(1,?)").run(Date.now());
+        }
+        catch (error) {
+            this.db.close();
+            throw error;
+        }
+    }
+    close() { this.db.close(); }
+    liveSince() { return Number(this.db.prepare("SELECT live_since FROM extraction_worker WHERE id=1").get().live_since); }
+    claim(scheduled, intervalMs) {
+        const owner = randomUUID(), now = Date.now();
+        const result = this.db.prepare(`UPDATE extraction_worker SET owner=?,lease_until=?,next_run=?
+      WHERE id=1 AND lease_until<? AND (?=0 OR next_run<=?)`).run(owner, now + 180_000, now + intervalMs, now, Number(scheduled), now);
+        return result.changes ? owner : undefined;
+    }
+    renew(owner) {
+        const now = Date.now();
+        if (!this.db.prepare("UPDATE extraction_worker SET lease_until=? WHERE id=1 AND owner=? AND lease_until>=?")
+            .run(now + 180_000, owner, now).changes)
+            throw new Error("Extraction lease lost");
+    }
+    release(owner) { this.db.prepare("UPDATE extraction_worker SET owner=NULL,lease_until=0 WHERE id=1 AND owner=?").run(owner); }
+    checkpoint(session, since) {
+        this.db.prepare(`INSERT INTO extraction_sessions(session_id,metadata,updated_at,extract_since) VALUES(?,?,?,?)
+      ON CONFLICT(session_id) DO UPDATE SET metadata=excluded.metadata`).run(session.sessionId, JSON.stringify(session), Date.now(), since ?? this.liveSince());
+        const row = this.db.prepare("SELECT cursor,context,extract_since FROM extraction_sessions WHERE session_id=?").get(session.sessionId);
+        if (since !== undefined && since < Number(row.extract_since)) {
+            // A broader explicitly approved backfill replays this session. Existing facts are
+            // supplied to Luna and exact duplicates are ignored; history is never deleted.
+            this.db.prepare("UPDATE extraction_sessions SET cursor=NULL,context='[]',extract_since=? WHERE session_id=?").run(since, session.sessionId);
+            return { cursor: null, context: [], since };
+        }
+        return { cursor: row.cursor, context: JSON.parse(String(row.context)), since: Number(row.extract_since) };
+    }
+    records(sessionId, paths) {
+        const keys = paths?.filter(path => /^[a-f0-9-]+\/[1-9]\d*$/.test(path)).map(path => path.split("/"));
+        if (keys && !keys.length)
+            return [];
+        return this.db.prepare(`SELECT m.*,s.metadata FROM extracted_memories m JOIN extraction_sessions s USING(session_id)
+      WHERE status='active' AND (? IS NULL OR session_id=?)
+      ${keys ? `AND (m.id,m.revision) IN (VALUES ${keys.map(() => "(?,?)").join(",")})` : ""}
+      ORDER BY observed_at,id`).all(sessionId ?? null, sessionId ?? null, ...(keys?.flat() ?? []))
+            .map(r => ({ id: String(r.id), revision: Number(r.revision), text: String(r.text), sessionId: String(r.session_id),
+            observedAt: Number(r.observed_at), evidence: JSON.parse(String(r.evidence)), metadata: JSON.parse(String(r.metadata)), judgment: JSON.parse(String(r.judgment)) }));
+    }
+    reset(sessionId, cursor, owner) {
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            this.renew(owner);
+            this.db.prepare("UPDATE extracted_memories SET status='withdrawn' WHERE session_id=? AND status='active'").run(sessionId);
+            this.db.prepare("UPDATE extraction_sessions SET cursor=?,context='[]',error=NULL WHERE session_id=?").run(cursor, sessionId);
+            this.db.exec("COMMIT");
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    commit(params) {
+        this.db.exec("BEGIN IMMEDIATE");
+        let written = 0;
+        try {
+            this.renew(params.owner);
+            const current = this.db.prepare("SELECT cursor FROM extraction_sessions WHERE session_id=?").get(params.session.sessionId);
+            if (current?.cursor !== params.expected)
+                throw new Error("Extraction checkpoint changed");
+            for (const { proposal, observedAt, judgment } of params.accepted) {
+                const digest = memoryDigest(proposal.text);
+                const duplicate = this.db.prepare("SELECT id FROM extracted_memories WHERE session_id=? AND digest=? AND status='active'")
+                    .get(params.session.sessionId, digest);
+                if (duplicate)
+                    continue;
+                let id = randomUUID(), revision = 1;
+                if (proposal.replaces) {
+                    const previous = this.db.prepare("SELECT revision,observed_at FROM extracted_memories WHERE id=? AND session_id=? AND status='active'")
+                        .get(proposal.replaces, params.session.sessionId);
+                    if (!previous || Number(previous.observed_at) > observedAt)
+                        throw new Error("Invalid or stale memory replacement");
+                    id = proposal.replaces;
+                    revision = Number(previous.revision) + 1;
+                    this.db.prepare("UPDATE extracted_memories SET status='superseded' WHERE id=? AND status='active'").run(id);
+                }
+                this.db.prepare(`INSERT INTO extracted_memories VALUES(?,?,?,?,?,?,?,'active',?,?,?)`)
+                    .run(id, revision, params.session.sessionId, proposal.text.trim(), digest, JSON.stringify(proposal.evidence), observedAt, JSON.stringify(judgment), params.version, Date.now());
+                written++;
+            }
+            this.db.prepare("UPDATE extraction_sessions SET cursor=?,context=?,updated_at=?,error=NULL WHERE session_id=?")
+                .run(params.cursor, JSON.stringify(params.context), Date.now(), params.session.sessionId);
+            this.db.exec("COMMIT");
+            return written;
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    error(sessionId, code) {
+        this.db.prepare("UPDATE extraction_sessions SET error=?,updated_at=? WHERE session_id=?").run(code, Date.now(), sessionId);
+    }
+    report() {
+        return { memories: this.records(), sessions: this.db.prepare("SELECT session_id,updated_at,error,cursor IS NOT NULL AS checkpointed FROM extraction_sessions").all() };
+    }
+}

package/dist/src/extraction-worker.d.ts

@@ -0,0 +1,51 @@
+import type { UnblockMemoryConfig } from "./config.js";
+import { extractWithLuna, validateExtractedMemory } from "./extraction-model.js";
+import { type ExtractionSession } from "./extraction-store.js";
+import { readExtractionPage } from "./extraction-source.js";
+export declare function runExtraction(params: {
+    config: UnblockMemoryConfig;
+    storePath: string;
+    agentId: string;
+    runtime: unknown;
+    agentName: string;
+    sessions: () => ExtractionSession[];
+    signal: AbortSignal;
+    since?: number;
+    sessionId?: string;
+    scheduled?: boolean;
+    readPage?: typeof readExtractionPage;
+    extract?: typeof extractWithLuna;
+    validate?: typeof validateExtractedMemory;
+}): Promise<{
+    status: "disabled";
+    reason?: undefined;
+    processed?: undefined;
+    accepted?: undefined;
+    rejected?: undefined;
+    unchanged?: undefined;
+    failed?: undefined;
+} | {
+    status: "unavailable";
+    reason: string;
+    processed?: undefined;
+    accepted?: undefined;
+    rejected?: undefined;
+    unchanged?: undefined;
+    failed?: undefined;
+} | {
+    status: "not_due_or_busy";
+    reason?: undefined;
+    processed?: undefined;
+    accepted?: undefined;
+    rejected?: undefined;
+    unchanged?: undefined;
+    failed?: undefined;
+} | {
+    status: "completed";
+    processed: number;
+    accepted: number;
+    rejected: number;
+    unchanged: number;
+    failed: number;
+    reason?: undefined;
+}>;

package/dist/src/extraction-worker.js

@@ -0,0 +1,128 @@
+import { resolveTypeSafeApiKey } from "./typesafe-client.js";
+import { extractWithLuna, validateExtractedMemory, EXTRACTION_VERSION } from "./extraction-model.js";
+import { ExtractionStore } from "./extraction-store.js";
+import { readExtractionPage } from "./extraction-source.js";
+import { extractionHistory, readExtractionChunk } from "./extraction-chunk.js";
+export async function runExtraction(params) {
+    const config = params.config.extraction;
+    if (!config.enabled)
+        return { status: "disabled" };
+    const apiKey = await resolveTypeSafeApiKey(params.config.typesafe);
+    if (!apiKey)
+        return { status: "unavailable", reason: "TypeSafe API key not configured" };
+    const store = new ExtractionStore(params.storePath);
+    const owner = store.claim(params.scheduled ?? false, config.intervalMinutes * 60_000);
+    if (!owner) {
+        store.close();
+        return { status: "not_due_or_busy" };
+    }
+    const leaseAbort = new AbortController();
+    const signal = AbortSignal.any([params.signal, leaseAbort.signal, AbortSignal.timeout(600_000)]);
+    const heartbeat = setInterval(() => { try {
+        store.renew(owner);
+    }
+    catch {
+        leaseAbort.abort();
+    } }, 30_000);
+    heartbeat.unref();
+    let processed = 0, acceptedCount = 0, rejectedCount = 0, unchanged = 0, failed = 0;
+    const readPage = params.readPage ?? readExtractionPage;
+    try {
+        const sessions = params.sessions();
+        const visible = new Set(sessions.map(s => s.sessionId));
+        if (params.sessionId && !visible.has(params.sessionId))
+            throw new Error("Unknown or unapproved extraction session");
+        for (const memory of store.records())
+            if (!visible.has(memory.sessionId))
+                store.reset(memory.sessionId, null, owner);
+        // Oldest last-checked first: bounded runs must not starve later sessions.
+        const checked = new Map(store.db.prepare("SELECT session_id,updated_at FROM extraction_sessions").all()
+            .map(r => [String(r.session_id), Number(r.updated_at)]));
+        sessions.sort((a, b) => (checked.get(a.sessionId) ?? 0) - (checked.get(b.sessionId) ?? 0));
+        for (const session of sessions) {
+            signal.throwIfAborted();
+            if (params.sessionId && params.sessionId !== session.sessionId)
+                continue;
+            if (!checked.has(session.sessionId) && (session.changedAt ?? Infinity) < (params.since ?? store.liveSince()))
+                continue;
+            if (processed >= config.maxBatches)
+                break;
+            const checkpoint = store.checkpoint(session, params.since);
+            let stage = "read";
+            try {
+                const existing = store.records(session.sessionId);
+                if (existing.length > 100 || JSON.stringify(existing.map(m => ({ id: m.id, text: m.text }))).length > 40_000) {
+                    throw new Error("Existing memories exceed review budget");
+                }
+                const page = await readExtractionChunk({ agentId: params.agentId, agentName: params.agentName, session,
+                    cursor: checkpoint.cursor, context: extractionHistory(checkpoint.context, config.historyMessages),
+                    existing: existing.map(m => ({ id: m.id, text: m.text, observedAt: m.observedAt })), signal, readPage });
+                if (page.kind === "unavailable") {
+                    store.error(session.sessionId, "source_unavailable");
+                    failed++;
+                    processed++;
+                    continue;
+                }
+                if (page.kind === "reset" || page.kind === "missing") {
+                    store.reset(session.sessionId, page.kind === "reset" ? page.cursor : null, owner);
+                    processed++;
+                    continue;
+                }
+                if (!page.entryCount) {
+                    store.db.prepare("UPDATE extraction_sessions SET updated_at=? WHERE session_id=?").run(Date.now(), session.sessionId);
+                    unchanged++;
+                    continue;
+                }
+                const since = checkpoint.since;
+                const freshIds = new Set(page.newIds);
+                const fresh = page.messages.filter(m => freshIds.has(m.id) && m.timestamp >= since);
+                const messages = page.messages;
+                const newIds = fresh.map(m => m.id);
+                stage = "propose";
+                const proposals = newIds.length ? await (params.extract ?? extractWithLuna)(params.runtime, params.agentId, messages, newIds, existing.map(m => ({ id: m.id, text: m.text, observedAt: m.observedAt })), signal) : [];
+                const accepted = [];
+                stage = "validate";
+                for (const proposal of proposals) {
+                    signal.throwIfAborted();
+                    const judgment = await (params.validate ?? validateExtractedMemory)({ proposal, messages, newIds,
+                        existing, apiKey, signal, thresholds: config });
+                    const observedAt = judgment.accepted ? Math.max(...proposal.evidence.map(e => messages.find(m => m.id === e.messageId).timestamp)) : 0;
+                    const prior = existing.find(m => m.id === proposal.replaces);
+                    if (judgment.accepted && (!prior || prior.observedAt <= observedAt))
+                        accepted.push({ proposal: { ...proposal,
+                                evidence: proposal.evidence.map(e => ({ ...e, messageId: messages.find(m => m.id === e.messageId).sourceMessageId ?? e.messageId })) }, judgment, observedAt });
+                    else
+                        rejectedCount++;
+                }
+                // A branch rewrite during inference invalidates its proposals. Appends are safe and processed next time.
+                const check = await readPage(params.agentId, params.agentName, session, page.fence);
+                if (check.kind === "reset" || check.kind === "missing") {
+                    store.reset(session.sessionId, check.kind === "reset" ? check.cursor : null, owner);
+                    processed++;
+                    continue;
+                }
+                if (check.kind === "unavailable")
+                    throw new Error("Source changed during extraction");
+                signal.throwIfAborted();
+                const context = extractionHistory(messages, config.historyMessages);
+                stage = "commit";
+                const written = store.commit({ session, expected: checkpoint.cursor, cursor: page.cursor,
+                    context, accepted, owner, version: EXTRACTION_VERSION });
+                processed++;
+                acceptedCount += written;
+            }
+            catch {
+                signal.throwIfAborted();
+                failed++;
+                processed++;
+                store.error(session.sessionId, `${stage}_failed`);
+            }
+        }
+        return { status: "completed", processed, accepted: acceptedCount, rejected: rejectedCount, unchanged, failed };
+    }
+    finally {
+        clearInterval(heartbeat);
+        store.release(owner);
+        store.close();
+    }
+}

package/dist/src/manager.d.ts

@@ -1,3 +1,4 @@
+import type { ExtractionConfig } from "./extraction-config.js";
 import type { QMDStore } from "@unblocklabs/qmd";
 import { type AnalysisRunner, type MemoryAnalysisSummary, type MemoryClusterDetail, type MemoryClusterList, type MemoryClusterSort, type MemoryReclusterOptions } from "./analysis.js";
 import type { CorpusMemorySearchResult, CorpusSearchOptions, MemoryEmbeddingProbeResult, MemoryProviderStatus, MemoryReadResult, MemoryRequestContext, MemorySearchManagerContract, MemorySyncParams } from "./contracts.js";
@@ -77,6 +78,7 @@ export declare class QmdMemoryManager implements MemorySearchManagerContract {
     constructor(params: {
         dbPath: string;
         curationPath?: string;
+        extraction?: ExtractionConfig;
         workspaceDir: string;
         sources: readonly ResolvedSource[];
         storeFactory?: () => Promise<ManagerStore>;
@@ -86,6 +88,7 @@ export declare class QmdMemoryManager implements MemorySearchManagerContract {
         sessions?: ManagerSessionConfig;
     });
     start(): Promise<void>;
+    syncExtracted(): Promise<void>;
     sync(params?: MemorySyncParams): Promise<void>;
     syncSessions(force?: boolean, onPhase?: (phase: "projecting" | "indexing") => void): Promise<SessionSyncResult>;
     recluster(options?: MemoryReclusterOptions, signal?: AbortSignal): Promise<MemoryAnalysisSummary>;