@augmem/cortext-openclaw-plugin 0.1.2 → 0.2.0

package/README.md

@@ -87,6 +87,8 @@ Under `plugins.entries.cortext.config`:
 | `ingestReasoning` | `true` | feed `thinking` deltas, not just answer text |
 | `forceRepass` | `true` | request a revise on interrupt (see limits — may be a no-op) |
 | `autoConsolidate` | `true` | consolidate on compaction |
+| `compactionMode` | `hybrid` | `hybrid`: system + recall + verbatim tail; `full`: system + recall + working memory only |
+| `protectTail` | `6` | hybrid: trailing messages kept verbatim (exchange-aligned) |
 
 ## Design and limits
 
@@ -98,9 +100,25 @@ Under `plugins.entries.cortext.config`:
 - **No cross-turn recall cache.** Recall queries Cortext live every assembly, so
   a correction ingested this turn is reflected immediately (an earlier caching
   bug returned stale facts).
-- **Compaction.** Cortext memory persists out-of-band, so the engine reports
-  `ownsCompaction: false` and delegates transcript compaction to the host; on
-  `compact` it consolidates its own graph.
+- **Compaction is a window, not surgery.** Cortext owns compaction
+  (`ownsCompaction: true`) and never calls a summarizer LLM: every message is
+  already in the durable store, so `compact` picks an exchange-aligned cut,
+  and each `assemble` drops the archived prefix from the model context and
+  bridges it with recalled memory. The on-disk transcript is never mutated —
+  nothing is destroyed, and archived content comes back through query-relevant
+  recall each turn (fresher than a frozen summary). The cut anchor is
+  content-based and self-healing: if the host rotates the transcript, the
+  window clears rather than over-dropping. Two modes (`compactionMode`):
+  - **`hybrid`** (default): keep system prompt + long-term recall + a verbatim
+    tail of the last `protectTail` messages, walked back to a user-message
+    boundary so the tail is a self-contained exchange.
+  - **`full`**: keep system prompt + Cortext memory only (long-term recall plus
+    the live working-memory snapshot); the verbatim window shrinks to the
+    current exchange. Maximum savings — memory IS the context.
+
+  Verified live (gateway + budget-pressure compaction): 16 messages archived
+  with no LLM call, and a fact that existed *only* behind the window was
+  answered correctly from memory injection on the next turn.
 - **The gate cannot splice into a live decode**, but it requests a re-pass.
   The agent event stream is one-way (observe only). On `should_interrupt` the
   plugin (a) stages the recalled memory for the next assembly and (b) via

package/dist/compaction.d.ts

@@ -0,0 +1,59 @@
+import type { AgentMessage } from "./openclaw.js";
+/**
+ * Compaction as a window over the transcript, not transcript surgery.
+ *
+ * When the host asks the engine to compact (ownsCompaction: true), we pick an
+ * exchange-aligned cut point in the last assembled view and remember the first
+ * KEPT message as an anchor. Every subsequent assemble() drops the prefix
+ * before the anchor and bridges it with Cortext memory — the on-disk transcript
+ * is never mutated, so nothing is destroyed and anything dropped remains
+ * recallable from the durable store.
+ *
+ * The anchor is content-based (role + text prefix), not positional: if the
+ * host rotates or rewrites the transcript and the anchor vanishes, the window
+ * self-heals by clearing (worst case the context regrows until the next
+ * compaction — never over-drops).
+ */
+export interface CompactionAnchor {
+    role: string;
+    /** First KEEP_PREFIX_CHARS of the anchor message's extracted text. */
+    textPrefix: string;
+    /** Messages dropped when the anchor was set (telemetry only). */
+    dropped: number;
+    ts: number;
+}
+export declare class CompactionState {
+    private anchors;
+    private loadedFrom;
+    /** Load persisted anchors from the store dir (idempotent per dir). */
+    load(dir: string): void;
+    get(scopeKey: string): CompactionAnchor | undefined;
+    set(scopeKey: string, anchor: CompactionAnchor, dir: string): void;
+    clear(scopeKey: string, dir: string): void;
+    private persist;
+}
+export declare function anchorFor(message: AgentMessage, extract: (content: unknown) => string, dropped: number): CompactionAnchor;
+export declare function matchesAnchor(message: AgentMessage, anchor: CompactionAnchor, extract: (content: unknown) => string): boolean;
+/**
+ * Choose the index of the first message to KEEP.
+ *
+ * - "full": keep from the last user message onward (the current exchange).
+ * - "hybrid": keep the last `protectTail` messages, then walk the cut back to
+ *   a user message so the tail starts on a self-contained exchange (never on a
+ *   tool result or mid tool-exchange).
+ *
+ * System-role messages are ignored here — the caller always keeps them.
+ * Returns 0 when there is nothing worth cutting.
+ */
+export declare function chooseCut(messages: AgentMessage[], mode: "hybrid" | "full", protectTail: number): number;
+/**
+ * Cold-start fallback: when compact() runs before any assemble in this process
+ * (the host's preflight compaction on a fresh gateway), read the message
+ * entries straight from the transcript jsonl. Linear read of `type:"message"`
+ * entries — good enough to pick a cut; if a branched/rotated DAG makes the
+ * anchor stale, assemble's anchor-miss self-heal clears it rather than
+ * over-dropping.
+ */
+export declare function readTranscriptMessages(sessionFile: string | undefined): AgentMessage[];
+/** The bridge message inserted where the archived prefix used to be. */
+export declare function bridgeMessage(mode: "hybrid" | "full"): AgentMessage;

package/dist/compaction.js

@@ -0,0 +1,115 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+const KEEP_PREFIX_CHARS = 200;
+const STATE_FILE = "compaction.json";
+export class CompactionState {
+    anchors = new Map();
+    loadedFrom = null;
+    /** Load persisted anchors from the store dir (idempotent per dir). */
+    load(dir) {
+        if (this.loadedFrom === dir)
+            return;
+        this.loadedFrom = dir;
+        try {
+            const raw = JSON.parse(readFileSync(join(dir, STATE_FILE), "utf-8"));
+            this.anchors = new Map(Object.entries(raw));
+        }
+        catch { /* first run or unreadable — start empty */ }
+    }
+    get(scopeKey) {
+        return this.anchors.get(scopeKey);
+    }
+    set(scopeKey, anchor, dir) {
+        this.anchors.set(scopeKey, anchor);
+        this.persist(dir);
+    }
+    clear(scopeKey, dir) {
+        if (this.anchors.delete(scopeKey))
+            this.persist(dir);
+    }
+    persist(dir) {
+        try {
+            writeFileSync(join(dir, STATE_FILE), JSON.stringify(Object.fromEntries(this.anchors)), "utf-8");
+        }
+        catch { /* best-effort — state also lives in memory */ }
+    }
+}
+export function anchorFor(message, extract, dropped) {
+    return {
+        role: String(message.role ?? ""),
+        textPrefix: extract(message.content).slice(0, KEEP_PREFIX_CHARS),
+        dropped,
+        ts: Date.now(),
+    };
+}
+export function matchesAnchor(message, anchor, extract) {
+    return String(message.role ?? "") === anchor.role &&
+        extract(message.content).slice(0, KEEP_PREFIX_CHARS) === anchor.textPrefix;
+}
+/**
+ * Choose the index of the first message to KEEP.
+ *
+ * - "full": keep from the last user message onward (the current exchange).
+ * - "hybrid": keep the last `protectTail` messages, then walk the cut back to
+ *   a user message so the tail starts on a self-contained exchange (never on a
+ *   tool result or mid tool-exchange).
+ *
+ * System-role messages are ignored here — the caller always keeps them.
+ * Returns 0 when there is nothing worth cutting.
+ */
+export function chooseCut(messages, mode, protectTail) {
+    const lastUser = (from) => {
+        for (let i = from; i >= 0; i--) {
+            if (messages[i]?.role === "user")
+                return i;
+        }
+        return 0;
+    };
+    if (mode === "full")
+        return lastUser(messages.length - 1);
+    let cut = Math.max(0, messages.length - Math.max(1, protectTail));
+    cut = lastUser(cut);
+    return cut;
+}
+/**
+ * Cold-start fallback: when compact() runs before any assemble in this process
+ * (the host's preflight compaction on a fresh gateway), read the message
+ * entries straight from the transcript jsonl. Linear read of `type:"message"`
+ * entries — good enough to pick a cut; if a branched/rotated DAG makes the
+ * anchor stale, assemble's anchor-miss self-heal clears it rather than
+ * over-dropping.
+ */
+export function readTranscriptMessages(sessionFile) {
+    if (!sessionFile)
+        return [];
+    let raw;
+    try {
+        raw = readFileSync(sessionFile, "utf-8");
+    }
+    catch {
+        return [];
+    }
+    const messages = [];
+    for (const line of raw.split("\n")) {
+        if (!line.trim())
+            continue;
+        try {
+            const entry = JSON.parse(line);
+            if (entry.type === "message" && entry.message && typeof entry.message.role === "string") {
+                messages.push(entry.message);
+            }
+        }
+        catch { /* skip malformed line */ }
+    }
+    return messages;
+}
+/** The bridge message inserted where the archived prefix used to be. */
+export function bridgeMessage(mode) {
+    const wm = mode === "full" ? " and its working-memory snapshot" : "";
+    return {
+        role: "user",
+        content: `[Earlier conversation was archived to Cortext durable memory. ` +
+            `Relevant context is recalled into the system prompt each turn${wm}. ` +
+            `Continue the conversation naturally.]`,
+    };
+}

package/dist/config.d.ts

@@ -1,4 +1,11 @@
 export type MemoryScope = "agent" | "session" | "global";
+/** How Cortext compacts the model-visible window (`ownsCompaction`):
+ *  - "hybrid": keep system prompt + recalled long-term memory + a verbatim
+ *    tail of recent messages (exchange-aligned). Safe default.
+ *  - "full": keep system prompt + Cortext memory only (long-term recall plus
+ *    the live working-memory snapshot); the verbatim window shrinks to the
+ *    current exchange. Maximum token savings — memory IS the context. */
+export type CompactionMode = "hybrid" | "full";
 export interface CortextPluginConfig {
     dbPath: string;
     /** Isolation boundary for memory. "session" (default): one store per session
@@ -18,6 +25,11 @@ export interface CortextPluginConfig {
      *  memory (costs one extra pass per trigger). */
     forceRepass: boolean;
     autoConsolidate: boolean;
+    /** Compaction window mode (see CompactionMode). */
+    compactionMode: CompactionMode;
+    /** Hybrid mode: number of trailing messages kept verbatim (the cut is walked
+     *  back to a user-message boundary so the tail is a self-contained exchange). */
+    protectTail: number;
 }
 export declare const DEFAULTS: CortextPluginConfig;
 export declare function resolveConfig(raw: Record<string, unknown> | undefined): CortextPluginConfig;

package/dist/config.js

@@ -11,8 +11,11 @@ export const DEFAULTS = {
     ingestReasoning: true,
     forceRepass: true,
     autoConsolidate: true,
+    compactionMode: "hybrid",
+    protectTail: 6,
 };
 const SCOPES = ["agent", "session", "global"];
+const COMPACTION_MODES = ["hybrid", "full"];
 export function resolveConfig(raw) {
     const cfg = { ...DEFAULTS };
     if (!raw)
@@ -25,5 +28,9 @@ export function resolveConfig(raw) {
     }
     if (!SCOPES.includes(cfg.memoryScope))
         cfg.memoryScope = "session";
+    if (!COMPACTION_MODES.includes(cfg.compactionMode))
+        cfg.compactionMode = "hybrid";
+    if (!Number.isFinite(cfg.protectTail) || cfg.protectTail < 0)
+        cfg.protectTail = 6;
     return cfg;
 }

package/dist/cortext.d.ts

@@ -39,6 +39,10 @@ export declare class CortextStore {
     scopeKey(ids: ScopeIds): string;
     forScope(key: string): CortextEngine;
     for(ids: ScopeIds): CortextEngine;
+    /** The on-disk directory holding this store's scope databases (also used for
+     *  the compaction-state sidecar). safe() permits dots, so "."/".." are
+     *  rejected — dbPath must stay under baseDir. */
+    storeDir(): string;
     disposeAll(): void;
 }
 export declare function memoryText(item: CortextMemory): string;

package/dist/cortext.js

@@ -90,14 +90,7 @@ export class CortextStore {
             this.engines.set(key, existing);
             return existing;
         }
-        // safe() permits dots, so guard "."/".." — dbPath must stay under baseDir.
-        const name = safe(this.cfg.dbPath);
-        const dir = join(this.baseDir, /^\.+$/.test(name) || !name ? "cortext" : name);
-        try {
-            mkdirSync(dir, { recursive: true });
-        }
-        catch { /* exists */ }
-        const engine = new CortextEngine(join(dir, `${key}.sqlite`), this.cfg);
+        const engine = new CortextEngine(join(this.storeDir(), `${key}.sqlite`), this.cfg);
         this.engines.set(key, engine);
         while (this.engines.size > MAX_ENGINES) {
             const oldest = this.engines.keys().next().value;
@@ -109,6 +102,18 @@ export class CortextStore {
     for(ids) {
         return this.forScope(this.scopeKey(ids));
     }
+    /** The on-disk directory holding this store's scope databases (also used for
+     *  the compaction-state sidecar). safe() permits dots, so "."/".." are
+     *  rejected — dbPath must stay under baseDir. */
+    storeDir() {
+        const name = safe(this.cfg.dbPath);
+        const dir = join(this.baseDir, /^\.+$/.test(name) || !name ? "cortext" : name);
+        try {
+            mkdirSync(dir, { recursive: true });
+        }
+        catch { /* exists */ }
+        return dir;
+    }
     disposeAll() {
         for (const e of this.engines.values())
             e.flush();

package/dist/engine.d.ts

@@ -1,4 +1,5 @@
 import type { AssembleParams, AssembleResult, CompactParams, CompactResult, ContextEngine, ContextEngineInfo, IngestParams, IngestResult, Logger } from "./openclaw.js";
+import type { CortextPluginConfig } from "./config.js";
 import { CortextStore } from "./cortext.js";
 import type { InterruptBus } from "./store.js";
 /**
@@ -13,12 +14,18 @@ export declare class CortextContextEngine implements ContextEngine {
     private readonly store;
     private readonly bus;
     private readonly logger;
-    private readonly autoConsolidate;
-    private readonly recallLimit;
+    private readonly cfg;
     readonly info: ContextEngineInfo;
-    constructor(store: CortextStore, bus: InterruptBus, logger: Logger, autoConsolidate: boolean, recallLimit: number);
+    /** Last full (pre-window) assembled view per scope — compact() picks its cut
+     *  from this, since CompactParams carry no messages. */
+    private lastView;
+    private compaction;
+    constructor(store: CortextStore, bus: InterruptBus, logger: Logger, cfg: CortextPluginConfig);
     ingest(params: IngestParams): Promise<IngestResult>;
     assemble(params: AssembleParams): Promise<AssembleResult>;
+    /** Drop the archived prefix (everything before the anchor), keeping system
+     *  messages and bridging with a note. Self-heals if the anchor is gone. */
+    private applyWindow;
     compact(params: CompactParams): Promise<CompactResult>;
     dispose(): Promise<void>;
     private source;

package/dist/engine.js

@@ -1,4 +1,5 @@
 import { formatMemories, memoryBlock, safe } from "./cortext.js";
+import { CompactionState, anchorFor, bridgeMessage, chooseCut, matchesAnchor, readTranscriptMessages } from "./compaction.js";
 // Bound serialized tool-call arguments so a huge payload (a file write, a long
 // patch) doesn't dominate the store; the result text is ingested separately.
 const TOOL_ARGS_MAX_CHARS = 2000;
@@ -66,20 +67,22 @@ export class CortextContextEngine {
     store;
     bus;
     logger;
-    autoConsolidate;
-    recallLimit;
+    cfg;
     info = {
         id: "cortext",
         name: "Cortext Memory",
-        version: "0.1.2",
-        ownsCompaction: false,
+        version: "0.2.0",
+        ownsCompaction: true,
     };
-    constructor(store, bus, logger, autoConsolidate, recallLimit) {
+    /** Last full (pre-window) assembled view per scope — compact() picks its cut
+     *  from this, since CompactParams carry no messages. */
+    lastView = new Map();
+    compaction = new CompactionState();
+    constructor(store, bus, logger, cfg) {
         this.store = store;
         this.bus = bus;
         this.logger = logger;
-        this.autoConsolidate = autoConsolidate;
-        this.recallLimit = recallLimit;
+        this.cfg = cfg;
     }
     async ingest(params) {
         const role = String(params.message?.role ?? "user");
@@ -91,35 +94,91 @@ export class CortextContextEngine {
         return { ingested: ctx !== null };
     }
     async assemble(params) {
-        const estimatedTokens = estimateTokens(params.messages);
-        const query = (params.prompt ?? latestUserText(params.messages)).trim();
-        if (!query)
-            return { messages: params.messages, estimatedTokens };
         const scopeKey = this.store.scopeKey({ sessionKey: params.sessionKey, sessionId: params.sessionId });
+        // Remember the full pre-window view: compact() picks its cut from it.
+        this.lastView.set(scopeKey, params.messages);
+        const { messages, windowed } = this.applyWindow(scopeKey, params.messages);
+        const estimatedTokens = estimateTokens(messages);
+        const query = (params.prompt ?? latestUserText(params.messages)).trim();
         const engine = this.store.forScope(scopeKey);
-        const ctx = engine.recall(query, this.source(params.sessionId, "agent", "assemble"));
-        const recalled = ctx ? formatMemories(ctx.retrieved_memory, this.recallLimit) : "";
+        const ctx = query ? engine.recall(query, this.source(params.sessionId, "agent", "assemble")) : null;
+        const recalled = ctx ? formatMemories(ctx.retrieved_memory, this.cfg.recallLimit) : "";
+        // Full mode with an active window: the verbatim transcript is gone, so the
+        // live working-memory snapshot rides along with long-term recall.
+        const working = windowed && this.cfg.compactionMode === "full" && ctx
+            ? formatMemories(ctx.working_memory, this.cfg.recallLimit)
+            : "";
         // Drain what the gate staged mid-generation, keyed by the SAME scope key —
         // so a different scope's assemble can never pick it up.
         const staged = this.bus.take(scopeKey);
-        const body = [staged, recalled].filter(Boolean).join("\n");
-        if (!body)
-            return { messages: params.messages, estimatedTokens };
+        const body = [staged, recalled, working].filter(Boolean).join("\n");
         return {
-            messages: params.messages,
+            messages,
             estimatedTokens,
-            systemPromptAddition: memoryBlock(body),
+            ...(windowed ? { promptAuthority: "assembled" } : {}),
+            ...(body ? { systemPromptAddition: memoryBlock(body) } : {}),
         };
     }
+    /** Drop the archived prefix (everything before the anchor), keeping system
+     *  messages and bridging with a note. Self-heals if the anchor is gone. */
+    applyWindow(scopeKey, messages) {
+        const dir = this.store.storeDir();
+        this.compaction.load(dir);
+        const anchor = this.compaction.get(scopeKey);
+        if (!anchor)
+            return { messages, windowed: false };
+        let idx = -1;
+        for (let i = 0; i < messages.length; i++) {
+            if (messages[i].role !== "system" && matchesAnchor(messages[i], anchor, messageText)) {
+                idx = i;
+                break;
+            }
+        }
+        if (idx < 0) {
+            // Transcript rotated/rewritten under us — never over-drop; regrow instead.
+            this.compaction.clear(scopeKey, dir);
+            return { messages, windowed: false };
+        }
+        if (idx === 0)
+            return { messages, windowed: false };
+        const head = messages.slice(0, idx).filter((m) => m.role === "system");
+        return { messages: [...head, bridgeMessage(this.cfg.compactionMode), ...messages.slice(idx)], windowed: true };
+    }
     async compact(params) {
-        // Cortext memory persists out-of-band; consolidate its graph, delegate
-        // transcript compaction to the host. (Compact params carry no agentId in
-        // the real openclaw types; agent scope derives from the sessionKey.)
-        const engine = this.store.for({ sessionKey: params.sessionKey, sessionId: params.sessionId });
-        if (this.autoConsolidate)
+        // Compaction = moving the window, not destroying the transcript. Every
+        // message is already in the durable store (ingest), so we pick an
+        // exchange-aligned cut in the last assembled view, anchor it, and let
+        // assemble() drop the archived prefix from the model context. The on-disk
+        // transcript is untouched; dropped content stays recallable. No LLM call.
+        // (Compact params carry no agentId in the real openclaw types; agent scope
+        // derives from the sessionKey.)
+        const scopeKey = this.store.scopeKey({ sessionKey: params.sessionKey, sessionId: params.sessionId });
+        const engine = this.store.forScope(scopeKey);
+        if (this.cfg.autoConsolidate)
             engine.consolidate();
         engine.flush();
-        return { ok: true, compacted: false, reason: "cortext retains memory out-of-band; transcript compaction delegated to host" };
+        // Preflight compaction on a fresh gateway process runs before any
+        // assemble — fall back to reading the transcript file so a cold-start
+        // compact still works (returning compacted:false here fails the turn).
+        const view = this.lastView.get(scopeKey) ?? readTranscriptMessages(params.sessionFile);
+        if (!view.length) {
+            return { ok: true, compacted: false, reason: "no assembled view or readable transcript for this scope" };
+        }
+        const cut = chooseCut(view, this.cfg.compactionMode, this.cfg.protectTail);
+        const dropped = view.slice(0, cut).filter((m) => m.role !== "system").length;
+        if (cut <= 0 || dropped === 0) {
+            return { ok: true, compacted: false, reason: "nothing before the protected window to archive" };
+        }
+        const dir = this.store.storeDir();
+        this.compaction.load(dir);
+        this.compaction.set(scopeKey, anchorFor(view[cut], messageText, dropped), dir);
+        const tokensBefore = estimateTokens(view);
+        const kept = this.applyWindow(scopeKey, view).messages;
+        const tokensAfter = estimateTokens(kept);
+        const summary = `Archived ${dropped} message(s) to Cortext durable memory ` +
+            `(${this.cfg.compactionMode} mode; recalled per turn, no summarizer LLM call).`;
+        this.logger.info(`cortext compaction: ${summary} ~${tokensBefore} -> ~${tokensAfter} tokens (scope ${scopeKey})`);
+        return { ok: true, compacted: true, reason: summary, result: { summary, tokensBefore, tokensAfter } };
     }
     async dispose() {
         this.store.disposeAll();

package/dist/register.js

@@ -18,7 +18,7 @@ export function register(api) {
     const store = new CortextStore(cfg, join(homedir(), ".openclaw", "cortext"));
     api.registerContextEngine("cortext", (ctx) => {
         store.setBaseDir(ctx?.agentDir);
-        return new CortextContextEngine(store, bus, api.logger, cfg.autoConsolidate, cfg.recallLimit);
+        return new CortextContextEngine(store, bus, api.logger, cfg);
     });
     if (cfg.interruptGate) {
         const gate = new InterruptGate(store, bus, api.logger, cfg.ingestReasoning, cfg.recallLimit);

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "cortext",
   "name": "Cortext Memory",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Durable local memory for OpenClaw. Per-conversation-isolated context engine with an interrupt gate over streaming reasoning.",
   "kind": "context-engine",
   "configSchema": {
@@ -66,6 +66,19 @@
         "type": "boolean",
         "description": "Consolidate memory on compaction.",
         "default": true
+      },
+      "compactionMode": {
+        "type": "string",
+        "enum": ["hybrid", "full"],
+        "description": "Cortext owns compaction (no summarizer LLM call). 'hybrid': keep system prompt + long-term recall + a verbatim tail of recent messages. 'full': keep system prompt + Cortext memory only (long-term recall plus the working-memory snapshot); the verbatim window shrinks to the current exchange.",
+        "default": "hybrid"
+      },
+      "protectTail": {
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 64,
+        "description": "Hybrid mode: trailing messages kept verbatim after compaction (cut is walked back to a user-message boundary).",
+        "default": 6
       }
     }
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@augmem/cortext-openclaw-plugin",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Cortext memory for OpenClaw: a per-conversation-isolated context engine plus an interrupt gate over streaming reasoning.",
   "license": "Apache-2.0",
   "type": "module",