dsh-continual-evolve 0.14.1 → 0.15.0

package/lib/extraction.js

@@ -0,0 +1,269 @@
+import { boundaryCursorOf, evaluateSlice, eventsAfterCursor, isInternalAgent, seqOfCursor, serializeIncrement } from "./extraction-surface.js";
+import { EXTRACTION_SYSTEM_PROMPT, extractionPrompt, manifestLineOf, parseExtractionAnswer } from "./extraction-prompt.js";
+import { cwdOf } from "./memory-section.js";
+/**
+ * Run one extraction pass for a target. Never throws: every terminal path
+ * is either a landed application, a ledgered skip, or a ledgered failure.
+ */
+export async function runExtraction(deps, target, opts) {
+    const { store } = deps;
+    const ledgerBase = {
+        sessionId: target.sessionId,
+        workspaceId: target.workspaceId,
+        trigger: opts.trigger,
+    };
+    if (target.internalAgent) {
+        store.logSkipped({ ...ledgerBase, skipReason: "internal-agent", status: "skipped" });
+        return { status: "skipped", skipReason: "internal-agent" };
+    }
+    if (target.workspaceId === undefined) {
+        store.logSkipped({ ...ledgerBase, skipReason: "no-workspace", status: "skipped" });
+        return { status: "skipped", skipReason: "no-workspace" };
+    }
+    if (deps.surface === undefined) {
+        store.logSkipped({ ...ledgerBase, skipReason: "no-surface", status: "skipped" });
+        return { status: "skipped", skipReason: "no-surface" };
+    }
+    if (target.provider === undefined || target.model === undefined || deps.llm === undefined) {
+        store.logSkipped({ ...ledgerBase, skipReason: "no-route", status: "skipped" });
+        return { status: "skipped", skipReason: "no-route" };
+    }
+    const cursor = store.cursor(target.workspaceId, target.sessionId);
+    let all;
+    try {
+        const snapshot = await deps.surface.readSurface(target.sessionId);
+        all = Array.isArray(snapshot.events) ? snapshot.events : [];
+    }
+    catch (error) {
+        store.logSkipped({ ...ledgerBase, skipReason: "surface-read-failed", status: "failed" });
+        return { status: "failed", skipReason: `surface read failed: ${error instanceof Error ? error.message : String(error)}` };
+    }
+    const events = eventsAfterCursor(all, cursor === undefined ? undefined : `seq:${cursor}`);
+    const slice = evaluateSlice({ events, internalAgent: false });
+    if (!slice.eligible) {
+        store.logSkipped({ ...ledgerBase, skipReason: slice.skipReason, status: "skipped" });
+        return { status: "skipped", skipReason: slice.skipReason };
+    }
+    const boundary = boundaryCursorOf(all);
+    const boundarySeq = seqOfCursor(boundary);
+    const workspaceId = target.workspaceId;
+    const advanceCursor = () => {
+        if (boundarySeq !== undefined) {
+            store.setCursor(workspaceId, target.sessionId, boundarySeq);
+        }
+    };
+    const increment = serializeIncrement(slice.events, opts.incrementMaxChars);
+    const manifest = store.list(target.workspaceId).slice(0, 50).map(manifestLineOf);
+    const candidates = candidateRecords(store, target.workspaceId, slice.userText);
+    const prompt = extractionPrompt({
+        increment,
+        manifest,
+        candidates: candidates.map(formatCandidate),
+        userSeqs: slice.userSeqs,
+    });
+    const started = Date.now();
+    let text;
+    let usage;
+    try {
+        const answer = await streamOnce(deps.llm, target, prompt);
+        text = answer.text;
+        usage = answer.usage;
+    }
+    catch (error) {
+        store.logSkipped({ ...ledgerBase, skipReason: "llm-failed", status: "failed", durationMs: Date.now() - started });
+        return { status: "failed", skipReason: `llm call failed: ${error instanceof Error ? error.message : String(error)}` };
+    }
+    const parsed = parseExtractionAnswer(text);
+    if (parsed.decision === "error") {
+        // A malformed answer is a failed run: the increment stays unconsumed
+        // and the next trigger retries it.
+        store.logSkipped({
+            ...ledgerBase,
+            skipReason: parsed.reason,
+            status: "failed",
+            model: target.model,
+            usage,
+            durationMs: Date.now() - started,
+        });
+        return { status: "failed", skipReason: parsed.reason };
+    }
+    if (parsed.decision === "skip") {
+        // A real model decision (even "nothing worth keeping") consumed the
+        // increment — advancing the cursor here is what keeps retries honest.
+        advanceCursor();
+        store.logSkipped({
+            ...ledgerBase,
+            skipReason: `model-skip: ${parsed.reason}`,
+            status: "skipped",
+            model: target.model,
+            usage,
+            durationMs: Date.now() - started,
+        });
+        return { status: "skipped", skipReason: parsed.reason, cursor: boundarySeq };
+    }
+    const outcomes = store.applyProposals({
+        workspaceId: target.workspaceId,
+        trigger: opts.trigger,
+        sessionId: target.sessionId,
+        runId: `extract:${target.sessionId}:${boundarySeq ?? "tail"}`,
+        model: target.model,
+        usage,
+        decision: parsed.reason,
+    }, parsed.proposals);
+    if (!outcomes.every((outcome) => outcome.ok)) {
+        return { status: "rejected", skipReason: outcomes.find((outcome) => !outcome.ok)?.reason };
+    }
+    advanceCursor();
+    // Hygiene rides on every applied run: orphan FTS rows dropped, secret
+    // leakage quarantined — never a separate scheduler.
+    const patrol = store.patrol();
+    void patrol;
+    return { status: "applied", applied: outcomes.filter((outcome) => outcome.ok).length, cursor: boundarySeq };
+}
+/** Similar existing memories (full bodies) for the model to update against. */
+function candidateRecords(store, workspaceId, userText) {
+    const query = userText.split("\n")[0]?.slice(0, 100).trim() ?? "";
+    const hits = query.length >= 3 ? store.search(workspaceId, query, 8) : [];
+    return hits.length > 0 ? hits : store.list(workspaceId).slice(0, 8);
+}
+function formatCandidate(record) {
+    return [`# 候选 id=${record.id}（type=${record.type}）`, `title=${record.title}`, `钩子=${record.description}`, "全文：", record.body].join("\n");
+}
+/** One streaming text call with the host session identity forwarded. */
+async function streamOnce(llm, target, prompt) {
+    let text = "";
+    let usage;
+    let finish;
+    const options = {
+        provider: target.provider,
+        model: target.model,
+        messages: [{ role: "user", content: [{ type: "text", text: prompt }] }],
+        system: EXTRACTION_SYSTEM_PROMPT,
+        maxTokens: 4096,
+        // MissingSessionID pitfall: direct calls must carry the host session.
+        sessionId: target.sessionId,
+    };
+    for await (const chunk of llm.stream(options)) {
+        const piece = chunk;
+        if (piece?.type === "text-delta" && typeof piece.text === "string") {
+            text += piece.text;
+        }
+        else if (piece?.type === "usage") {
+            usage = piece.usage;
+        }
+        else if (piece?.type === "finish") {
+            finish = piece.reason;
+        }
+    }
+    if (finish !== undefined && finish.kind !== "stop") {
+        throw new Error(`stream finished with ${finish.kind ?? "unknown"}`);
+    }
+    if (text.trim().length === 0) {
+        throw new Error("empty model output");
+    }
+    return { text, usage };
+}
+/**
+ * Wire the extraction triggers onto the host context. Returns a disposer
+ * that removes the listeners and clears the timer. The scheduler is the
+ * only background automation the plugin runs, and it is bounded by design:
+ * one timer, one in-flight run, one pending slot.
+ */
+export function createExtractionScheduler(host, deps, options = {}) {
+    const debounceMs = Math.max(0, (options.debounceMin ?? 10) * 60_000);
+    const listeners = [];
+    let timer = null;
+    let running = false;
+    let pending = null;
+    const targetOf = (agent) => {
+        const id = agent?.id;
+        if (typeof id !== "string" || id.length === 0) {
+            return null;
+        }
+        const route = agent?.options;
+        return {
+            agent,
+            sessionId: id,
+            provider: typeof route?.provider === "string" ? route.provider : undefined,
+            model: typeof route?.model === "string" ? route.model : undefined,
+        };
+    };
+    const flush = (trigger) => {
+        if (timer !== null) {
+            clearTimeout(timer);
+            timer = null;
+        }
+        if (running || pending === null) {
+            return;
+        }
+        const target = pending;
+        pending = null;
+        running = true;
+        void runExtraction(deps, {
+            workspaceId: cwdOf(target.agent),
+            sessionId: target.sessionId,
+            internalAgent: isInternalAgent(target.agent),
+            provider: target.provider,
+            model: target.model,
+        }, { trigger }).catch((error) => {
+            host.logger("continual-evolve").warn(`extraction run failed unexpectedly: ${error instanceof Error ? error.message : String(error)}`);
+        }).finally(() => {
+            running = false;
+            if (pending !== null) {
+                // A turn arrived mid-run and refreshed the pending slot: drain it.
+                flush("turn-debounce");
+            }
+        });
+    };
+    const arm = (agent) => {
+        const target = targetOf(agent);
+        if (target === null) {
+            return;
+        }
+        pending = target; // Coalescing: the latest boundary wins, nothing queues.
+        if (debounceMs === 0) {
+            flush("turn-debounce");
+            return;
+        }
+        if (timer !== null) {
+            clearTimeout(timer);
+        }
+        timer = setTimeout(() => {
+            timer = null;
+            flush("turn-debounce");
+        }, debounceMs);
+    };
+    listeners.push(host.on("agent/turn-stopping", (payload) => {
+        if (payload.agent === undefined) {
+            return;
+        }
+        arm(payload.agent);
+    }));
+    listeners.push(host.on("session/event", (session, event) => {
+        if (event?.type !== "compaction/start" || pending === null || pending.sessionId !== session?.id) {
+            return;
+        }
+        flush("compaction");
+    }));
+    listeners.push(host.on("agent/disposed", (payload) => {
+        const id = payload.agent?.id;
+        if (pending === null || pending.sessionId !== id) {
+            return;
+        }
+        flush("close-drain");
+    }));
+    host.logger("continual-evolve").info(`extraction scheduler armed (debounce ${options.debounceMin ?? 10}min, single-flight with coalescing)`);
+    return () => {
+        if (timer !== null) {
+            clearTimeout(timer);
+            timer = null;
+        }
+        pending = null;
+        for (const listener of listeners) {
+            if (typeof listener === "function") {
+                listener();
+            }
+        }
+    };
+}
+//# sourceMappingURL=extraction.js.map

package/lib/import-md.d.ts

@@ -0,0 +1,15 @@
+import { type MemoryStore } from "./store.js";
+/** What one workspace's migration did. */
+export interface MdImportResult {
+    workspaceId: string;
+    /** Records imported (existing ids from a partial earlier run don't count). */
+    imported: number;
+    /** Records skipped by a hard gate (reasons in `errors`). */
+    skipped: number;
+    /** The renamed-aside directory, when the rename succeeded. */
+    archiveDir: string | null;
+    errors: string[];
+}
+/** Run the migration for one workspace root. Never throws past its result. */
+export declare function importWorkspaceMd(store: MemoryStore, cwd: string): MdImportResult;
+//# sourceMappingURL=import-md.d.ts.map

package/lib/import-md.js

@@ -0,0 +1,140 @@
+/**
+ * One-time migration from the MD-era store: `<workspace>/.evolve/memory/`
+ * files are imported into the central database under the workspace's own
+ * partition, then the directory is renamed aside (`memory-imported-<date>/`)
+ * — never deleted. Runs lazily per workspace on first contact (injection
+ * assembly or card read); the `md-imported` state flag keeps it idempotent
+ * even when the rename fails.
+ *
+ * Import is lossless by design: unknown types map to `reference`, missing
+ * hooks are synthesized from the body, and only the hard gates (secret
+ * screen) can skip a file — skipped files stay in the renamed directory for
+ * manual salvage.
+ */
+import { existsSync, readdirSync, readFileSync, renameSync, statSync } from "node:fs";
+import { basename, join } from "node:path";
+import { slugifyId } from "./memory-rules.js";
+import { MD_IMPORT_STATE_KEY } from "./store.js";
+/** The MD-era index file (imported rows come from the other .md files). */
+const MD_INDEX_FILE = "MEMORY.md";
+/** Run the migration for one workspace root. Never throws past its result. */
+export function importWorkspaceMd(store, cwd) {
+    const workspaceId = cwd;
+    const absent = { workspaceId, imported: 0, skipped: 0, archiveDir: null, errors: [] };
+    if (store.state(workspaceId, MD_IMPORT_STATE_KEY) !== undefined) {
+        return absent;
+    }
+    const memoryDir = join(cwd, ".evolve", "memory");
+    if (!existsSync(memoryDir) || !statSync(memoryDir).isDirectory()) {
+        store.setState(workspaceId, MD_IMPORT_STATE_KEY, new Date().toISOString());
+        return absent;
+    }
+    const result = importMemoryDir(store, workspaceId, memoryDir);
+    const archiveDir = renameAside(memoryDir);
+    if (archiveDir !== null) {
+        result.archiveDir = archiveDir;
+    }
+    else {
+        result.errors.push("the old directory could not be renamed; import flag set anyway (imports are idempotent)");
+    }
+    store.setState(workspaceId, MD_IMPORT_STATE_KEY, new Date().toISOString());
+    return result;
+}
+/** Import every non-index .md file of an MD-era store directory. */
+function importMemoryDir(store, workspaceId, memoryDir) {
+    const result = { workspaceId, imported: 0, skipped: 0, archiveDir: null, errors: [] };
+    const files = readdirSync(memoryDir)
+        .filter((file) => file.endsWith(".md") && file !== MD_INDEX_FILE)
+        .sort();
+    for (const file of files) {
+        try {
+            const parsed = parseMdMemory(readFileSync(join(memoryDir, file), "utf8"), basename(file, ".md"));
+            if (store.get(workspaceId, parsed.id) !== undefined) {
+                continue;
+            }
+            const outcomes = store.applyProposals({ workspaceId, trigger: "import", decision: `md migration: ${file}` }, [
+                {
+                    action: "create",
+                    id: parsed.id,
+                    type: parsed.type,
+                    title: parsed.title,
+                    description: parsed.description,
+                    body: parsed.body,
+                },
+            ]);
+            if (outcomes[0]?.ok === true) {
+                result.imported += 1;
+            }
+            else {
+                result.skipped += 1;
+                result.errors.push(`${file}: ${outcomes[0]?.reason ?? "unknown rejection"}`);
+            }
+        }
+        catch (error) {
+            result.skipped += 1;
+            result.errors.push(`${file}: unreadable (${error instanceof Error ? error.message : String(error)})`);
+        }
+    }
+    return result;
+}
+const VALID_TYPES = ["user", "feedback", "reference"];
+/**
+ * Parse one MD-era file: frontmatter `name` / `description` / `type` (type
+ * may sit under a `metadata:` block), body = everything after the closing
+ * `---`. Anything missing or malformed is synthesized from the filename and
+ * body rather than dropped — migration must not lose.
+ */
+function parseMdMemory(text, fileBase) {
+    const fields = { name: "", description: "", type: "" };
+    let body = text;
+    if (text.startsWith("---")) {
+        const blockEnd = text.indexOf("\n---", 3);
+        if (blockEnd >= 0) {
+            for (const line of text.slice(3, blockEnd).split("\n")) {
+                const match = /^(name|description|type):\s*(.*)$/.exec(line.trim());
+                if (match?.[1] !== undefined && match?.[2] !== undefined) {
+                    const key = match[1];
+                    fields[key] = match[2].trim().replace(/^["']|["']$/g, "");
+                }
+            }
+            body = text.slice(blockEnd + 4);
+        }
+    }
+    const title = fields.name.length > 0 ? fields.name : fileBase;
+    const type = (VALID_TYPES.includes(fields.type) ? fields.type : "reference");
+    const description = fields.description.length > 0 ? fields.description : firstLine(body, 200);
+    return {
+        id: fields.name.length > 0 && slugifyId(fields.name) === fields.name ? fields.name : slugifyId(title),
+        type,
+        title,
+        description: description.length > 0 ? description : fileBase,
+        body: body.trim().length > 0 ? body.trim() : fields.name,
+    };
+}
+/** First non-empty line, capped — the synthesized relevance hook. */
+function firstLine(text, cap) {
+    for (const line of text.split("\n")) {
+        const trimmed = line.trim().replace(/^#+\s*/, "").replace(/^-\s*/, "");
+        if (trimmed.length > 0) {
+            return trimmed.length > cap ? `${trimmed.slice(0, cap - 1)}…` : trimmed;
+        }
+    }
+    return "";
+}
+/** Rename the imported directory aside; null when the rename failed. */
+function renameAside(memoryDir) {
+    const stamp = new Date().toISOString().slice(0, 10);
+    const base = join(memoryDir, "..", `memory-imported-${stamp}`);
+    let target = base;
+    for (let attempt = 2; existsSync(target); attempt += 1) {
+        target = `${base}-${attempt}`;
+    }
+    try {
+        renameSync(memoryDir, target);
+        return target;
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=import-md.js.map

package/lib/index.d.ts

@@ -1,25 +1,29 @@
 /**
  * dsh-continual-evolve — plugin entry.
  *
- * After the 2026-10-04 ZCode-alignment teardown the plugin is one thing: a
- * session-start system-prompt section that injects the workspace memory
- * index (`<cwd>/.evolve/memory/MEMORY.md`), hands out the store path, and
- * teaches the when-to-save guide. The model reads and writes the markdown
- * files with its native tools; there are no plugin tools, no commands, no
- * background extraction, and no governance. See
- * .agents/notes/implemented/architecture/2026-10-04-zcode-alignment-teardown.md.
+ * The v0.15 SQLite single-store redesign (ADR
+ * `2026-10-06-sqlite-single-store`): every memory lives in one central
+ * database (`~/.dsh/evolve/memory.db`, rows partitioned by workspace path)
+ * and the plugin's code is the literal sole writer. This entry wires four
+ * surfaces over the store:
  *
- * The 2026-10-04 plugin-management card adds a read-only surface: the host's
- * workspace registry feeds a catalogue, and a `webServer` route triple projects
- * any catalogued workspace's store for the card in the official plugin manager
- * (`plugins.bundle.config`, client bundle in client/client.js). The card is a
- * viewer — see
- * .agents/notes/implemented/feature/2026-10-04-plugin-management-card.md and
- * .agents/notes/implemented/feature/2026-10-05-zcode-workspace-switcher.md.
+ * - the session-start section (`memory-section.ts`): a store query rendered
+ *   as the injected index, frozen per session for the prompt cache;
+ * - the model tools (`memory-tools.ts`): `memory_write` for explicit
+ *   "记住/忘掉" instructions, `memory_read` for bodies and search;
+ * - the extraction scheduler (`extraction.ts`): turn-level, debounced,
+ *   single-flight; proposals land only through the store's gates;
+ * - the read-only card routes (`card-routes.ts`) in the official plugin
+ *   manager.
  *
- * The host API surface is declared locally (the minimal shape this plugin
- * touches) instead of importing the DSH type packages — the section call is
- * the same one every prior release used, verified in production.
+ * The store opens asynchronously (dynamic `node:sqlite` import); until it is
+ * ready — or forever, if the runtime lacks `node:sqlite` — the section
+ * renders "" and the tools never register. Injection must never break an
+ * assembly, and a degraded plugin must never take the host down.
+ *
+ * The host API surface is declared locally (the minimal shapes this plugin
+ * touches) instead of importing the DSH type packages — verified in
+ * production across every prior release.
  */
 import z from "@deepseek-ai/schemastery";
 export declare const name = "continual-evolve";
@@ -43,22 +47,38 @@ interface HostContext {
 }
 export declare const Config: z<Schemastery.ObjectS<NoInfer<{
     /**
-     * The memory section: injects the workspace memory index at session
-     * start, frozen per session to keep the system prompt byte-stable
-     * (prompt cache). `enabled`/`guide`/`order`/`maxChars` default to
-     * true/true/400/6000; with `guide` off and an empty store the section
-     * renders to "" and costs no tokens.
+     * The memory section: injects the workspace memory index (a store query)
+     * at session start, frozen per session to keep the system prompt
+     * byte-stable (prompt cache). `enabled`/`guide`/`order`/`maxChars`
+     * default to true/true/400/6000; with `guide` off and an empty store the
+     * section renders to "" and costs no tokens.
      */
     memoryIndex: z<Schemastery.ObjectS<NoInfer<{
         enabled: z<boolean, boolean, "plain">;
         guide: z<boolean, boolean, "plain">;
         order: z<number, number, "plain">;
         maxChars: z<number, number, "plain">;
+        /**
+         * The proposal-based extraction run (trigger: turn-level with idle
+         * debounce; writes land only through the store's gates). Off keeps
+         * the plugin injection + explicit tools only.
+         */
+        extraction: z<boolean, boolean, "plain">;
+        /** Idle debounce minutes between turn end and an extraction run (0 = per turn). */
+        debounceMin: z<number, number, "plain">;
     }>>, Schemastery.ObjectT<NoInfer<{
         enabled: z<boolean, boolean, "plain">;
         guide: z<boolean, boolean, "plain">;
         order: z<number, number, "plain">;
         maxChars: z<number, number, "plain">;
+        /**
+         * The proposal-based extraction run (trigger: turn-level with idle
+         * debounce; writes land only through the store's gates). Off keeps
+         * the plugin injection + explicit tools only.
+         */
+        extraction: z<boolean, boolean, "plain">;
+        /** Idle debounce minutes between turn end and an extraction run (0 = per turn). */
+        debounceMin: z<number, number, "plain">;
     }>>, "plain">;
     /**
      * The read-only memory card in the official plugin manager
@@ -73,22 +93,38 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
     }>>, "plain">;
 }>>, Schemastery.ObjectT<NoInfer<{
     /**
-     * The memory section: injects the workspace memory index at session
-     * start, frozen per session to keep the system prompt byte-stable
-     * (prompt cache). `enabled`/`guide`/`order`/`maxChars` default to
-     * true/true/400/6000; with `guide` off and an empty store the section
-     * renders to "" and costs no tokens.
+     * The memory section: injects the workspace memory index (a store query)
+     * at session start, frozen per session to keep the system prompt
+     * byte-stable (prompt cache). `enabled`/`guide`/`order`/`maxChars`
+     * default to true/true/400/6000; with `guide` off and an empty store the
+     * section renders to "" and costs no tokens.
      */
     memoryIndex: z<Schemastery.ObjectS<NoInfer<{
         enabled: z<boolean, boolean, "plain">;
         guide: z<boolean, boolean, "plain">;
         order: z<number, number, "plain">;
         maxChars: z<number, number, "plain">;
+        /**
+         * The proposal-based extraction run (trigger: turn-level with idle
+         * debounce; writes land only through the store's gates). Off keeps
+         * the plugin injection + explicit tools only.
+         */
+        extraction: z<boolean, boolean, "plain">;
+        /** Idle debounce minutes between turn end and an extraction run (0 = per turn). */
+        debounceMin: z<number, number, "plain">;
     }>>, Schemastery.ObjectT<NoInfer<{
         enabled: z<boolean, boolean, "plain">;
         guide: z<boolean, boolean, "plain">;
         order: z<number, number, "plain">;
         maxChars: z<number, number, "plain">;
+        /**
+         * The proposal-based extraction run (trigger: turn-level with idle
+         * debounce; writes land only through the store's gates). Off keeps
+         * the plugin injection + explicit tools only.
+         */
+        extraction: z<boolean, boolean, "plain">;
+        /** Idle debounce minutes between turn end and an extraction run (0 = per turn). */
+        debounceMin: z<number, number, "plain">;
     }>>, "plain">;
     /**
      * The read-only memory card in the official plugin manager