@martian-engineering/lossless-claw 0.6.2 → 0.6.3

package/README.md

@@ -246,7 +246,7 @@ Lossless-claw distinguishes OpenClaw's two session-reset commands:
 - `2`: keep d2+ summaries; recommended default
 - `3+`: keep only deeper, more abstract summaries
 
-Lossless-claw currently applies these storage semantics through the `before_reset` hook only. User-facing confirmation text after `/new` or `/reset` must be emitted by OpenClaw's command handlers.
+Lossless-claw applies `/new` pruning through `before_reset` and uses `session_end` to catch transcript rollovers such as `/reset`, idle or daily session rotation, compaction session replacement, and deletions. User-facing confirmation text after `/new` or `/reset` must still be emitted by OpenClaw's command handlers.
 
 Use `ignoreSessionPatterns` or `LCM_IGNORE_SESSION_PATTERNS` to keep low-value sessions completely out of LCM. Matching sessions do not create conversations, do not store messages, and do not participate in compaction or delegated expansion grants.
 

package/docs/configuration.md

@@ -159,7 +159,7 @@ Lossless-claw treats the two OpenClaw reset commands differently:
 - `/reset` archives the active conversation row and creates a fresh active row for the same stable `sessionKey`.
 
 This preserves lossless history while still giving users a real clean-slate command.
-OpenClaw's command handlers still own the user-facing post-command disclosure text; lossless-claw applies only the underlying storage transition through `before_reset`.
+Lossless-claw applies `/new` through `before_reset`, then uses `session_end` to catch the broader rollover cases OpenClaw can emit: `/reset`, idle or daily session rotation, compaction-driven session replacement, and deletions. OpenClaw's command handlers still own the user-facing post-command disclosure text.
 
 Use `ignoreSessionPatterns` or `LCM_IGNORE_SESSION_PATTERNS` to keep low-value sessions completely out of LCM. Matching sessions do not create conversations, do not store messages, and do not participate in compaction or delegated expansion grants.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@martian-engineering/lossless-claw",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Lossless Context Management plugin for OpenClaw — DAG-based conversation summarization with incremental compaction",
   "type": "module",
   "main": "index.ts",

package/src/engine.ts

@@ -48,6 +48,7 @@ import { compileSessionPatterns, matchesSessionPattern } from "./session-pattern
 import { logStartupBannerOnce } from "./startup-banner-log.js";
 import {
   ConversationStore,
+  type ConversationRecord,
   type CreateMessagePartInput,
   type MessagePartRecord,
   type MessagePartType,
@@ -1017,7 +1018,7 @@ function readFileSegment(sessionFile: string, offset: number): string | null {
   }
 }
 
-function readLastJsonlEntryBeforeOffset(sessionFile: string, offset: number): string | null {
+function readLastJsonlEntryBeforeOffset(sessionFile: string, offset: number, messageOnly = false): string | null {
   const chunkSize = 16_384;
   let fd: number | null = null;
   try {
@@ -1029,16 +1030,23 @@ function readLastJsonlEntryBeforeOffset(sessionFile: string, offset: number): st
     fd = openSync(sessionFile, "r");
     let cursor = safeOffset;
     let carry = "";
-    while (cursor > 0) {
-      const start = Math.max(0, cursor - chunkSize);
-      const length = cursor - start;
-      const buffer = Buffer.alloc(length);
-      readSync(fd, buffer, 0, length, start);
-      carry = buffer.toString("utf8") + carry;
+    let reachedStart = false;
+    while (cursor > 0 || (reachedStart && carry.length > 0)) {
+      if (!reachedStart) {
+        const start = Math.max(0, cursor - chunkSize);
+        const length = cursor - start;
+        const buffer = Buffer.alloc(length);
+        readSync(fd, buffer, 0, length, start);
+        carry = buffer.toString("utf8") + carry;
+        cursor = start;
+        if (start === 0) {
+          reachedStart = true;
+        }
+      }
 
       const trimmedEnd = carry.replace(/\s+$/u, "");
       if (!trimmedEnd) {
-        cursor = start;
+        if (reachedStart) break;
         carry = "";
         continue;
       }
@@ -1047,17 +1055,36 @@ function readLastJsonlEntryBeforeOffset(sessionFile: string, offset: number): st
       if (newlineIndex >= 0) {
         const candidate = trimmedEnd.slice(newlineIndex + 1).trim();
         if (candidate) {
+          if (messageOnly) {
+            let isMessage = false;
+            try {
+              isMessage = extractBootstrapMessageCandidate(JSON.parse(candidate)) != null;
+            } catch { /* not valid JSON, skip */ }
+            if (!isMessage) {
+              carry = trimmedEnd.slice(0, newlineIndex);
+              continue;
+            }
+          }
           return candidate;
         }
         carry = trimmedEnd.slice(0, newlineIndex);
-        cursor = start;
         continue;
       }
 
-      if (start === 0) {
-        return trimmedEnd.trim() || null;
+      // No newline found — entire trimmedEnd is one line
+      if (reachedStart) {
+        const firstLine = trimmedEnd.trim() || null;
+        if (firstLine && messageOnly) {
+          let isMessage = false;
+          try {
+            isMessage = extractBootstrapMessageCandidate(JSON.parse(firstLine)) != null;
+          } catch { /* not valid JSON */ }
+          if (!isMessage) return null;
+        }
+        return firstLine;
       }
-      cursor = start;
+      // Need more data from earlier in the file
+      continue;
     }
     return null;
   } catch {
@@ -1826,17 +1853,18 @@ export class LcmContextEngine implements ContextEngine {
     conversationId: number;
     historicalMessages: AgentMessage[];
   }): Promise<{
+    blockedByImportCap: boolean;
     importedMessages: number;
     hasOverlap: boolean;
   }> {
     const { sessionId, conversationId, historicalMessages } = params;
     if (historicalMessages.length === 0) {
-      return { importedMessages: 0, hasOverlap: false };
+      return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
 
     const latestDbMessage = await this.conversationStore.getLastMessage(conversationId);
     if (!latestDbMessage) {
-      return { importedMessages: 0, hasOverlap: false };
+      return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
 
     const storedHistoricalMessages = historicalMessages.map((message) => toStoredMessage(message));
@@ -1857,7 +1885,7 @@ export class LcmContextEngine implements ContextEngine {
         }
       }
       if (dbOccurrences === historicalOccurrences) {
-        return { importedMessages: 0, hasOverlap: true };
+        return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
       }
     }
 
@@ -1909,13 +1937,20 @@ export class LcmContextEngine implements ContextEngine {
     }
 
     if (anchorIndex < 0) {
-      return { importedMessages: 0, hasOverlap: false };
+      return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
     if (anchorIndex >= historicalMessages.length - 1) {
-      return { importedMessages: 0, hasOverlap: true };
+      return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
     }
 
     const missingTail = historicalMessages.slice(anchorIndex + 1);
+
+    const existingDbCount = await this.conversationStore.getMessageCount(conversationId);
+    if (existingDbCount > 0 && missingTail.length > Math.max(existingDbCount * 0.2, 50)) {
+      console.error(`[lcm] reconcileSessionTail: import cap exceeded — would import ${missingTail.length} messages (existing: ${existingDbCount}). Aborting to prevent flood.`);
+      return { blockedByImportCap: true, importedMessages: 0, hasOverlap: true };
+    }
+
     let importedMessages = 0;
     for (const message of missingTail) {
       const result = await this.ingestSingle({ sessionId, sessionKey: params.sessionKey, message });
@@ -1924,7 +1959,7 @@ export class LcmContextEngine implements ContextEngine {
       }
     }
 
-    return { importedMessages, hasOverlap: true };
+    return { blockedByImportCap: false, importedMessages, hasOverlap: true };
   }
 
   async bootstrap(params: {
@@ -2019,6 +2054,7 @@ export class LcmContextEngine implements ContextEngine {
             const tailEntryRaw = readLastJsonlEntryBeforeOffset(
               params.sessionFile,
               bootstrapState.lastProcessedOffset,
+              true,
             );
             const tailEntryMessage = readBootstrapMessageFromJsonLine(tailEntryRaw);
             const tailEntryHash = tailEntryMessage
@@ -2143,6 +2179,14 @@ export class LcmContextEngine implements ContextEngine {
             historicalMessages,
           });
 
+          if (reconcile.blockedByImportCap) {
+            return {
+              bootstrapped: false,
+              importedMessages: 0,
+              reason: "reconcile import capped",
+            };
+          }
+
           if (!conversation.bootstrappedAt) {
             await this.conversationStore.markConversationBootstrapped(conversationId);
           }
@@ -2405,9 +2449,34 @@ export class LcmContextEngine implements ContextEngine {
           };
         }
 
-        return params.runtimeContext.rewriteTranscriptEntries({
+        const result = await params.runtimeContext.rewriteTranscriptEntries({
           replacements,
         });
+
+        if (result.changed) {
+          try {
+            const fileStat = statSync(params.sessionFile);
+            const newSize = fileStat.size;
+            const newMtimeMs = Math.trunc(fileStat.mtimeMs);
+            const lastEntryRaw = readLastJsonlEntryBeforeOffset(params.sessionFile, newSize, true);
+            const lastEntryMsg = readBootstrapMessageFromJsonLine(lastEntryRaw);
+            const lastEntryHash = lastEntryMsg ? createBootstrapEntryHash(toStoredMessage(lastEntryMsg)) : null;
+            if (lastEntryHash) {
+              await this.summaryStore.upsertConversationBootstrapState({
+                conversationId: conversation.conversationId,
+                sessionFilePath: params.sessionFile,
+                lastSeenSize: newSize,
+                lastSeenMtimeMs: newMtimeMs,
+                lastProcessedOffset: newSize,
+                lastProcessedEntryHash: lastEntryHash,
+              });
+            }
+          } catch (e) {
+            console.error("[lcm] Failed to update bootstrap checkpoint after maintain:", e);
+          }
+        }
+
+        return result;
       },
     );
   }
@@ -3219,6 +3288,69 @@ export class LcmContextEngine implements ContextEngine {
     // The shared connection is managed for the lifetime of the plugin process.
   }
 
+  /** Detect the empty replacement row created during a prior lifecycle rollover. */
+  private async isFreshLifecycleConversation(conversation: ConversationRecord): Promise<boolean> {
+    const currentMessageCount = await this.conversationStore.getMessageCount(conversation.conversationId);
+    if (currentMessageCount !== 0) {
+      return false;
+    }
+    const currentContextItems = await this.summaryStore.getContextItems(conversation.conversationId);
+    return currentContextItems.length === 0 && !conversation.bootstrappedAt;
+  }
+
+  /**
+   * Archive the current active conversation and optionally create the replacement
+   * row that bootstrap should attach to for the next session transcript.
+   */
+  private async applySessionReplacement(params: {
+    reason: string;
+    sessionId?: string;
+    sessionKey?: string;
+    nextSessionId?: string;
+    nextSessionKey?: string;
+    createReplacement: boolean;
+    createReplacementWhenMissing?: boolean;
+  }): Promise<void> {
+    const current = await this.conversationStore.getConversationForSession({
+      sessionId: params.sessionId,
+      sessionKey: params.sessionKey,
+    });
+    if (!current && !params.createReplacementWhenMissing) {
+      return;
+    }
+
+    if (current?.active) {
+      if (params.createReplacement && await this.isFreshLifecycleConversation(current)) {
+        this.deps.log.info(
+          `[lcm] ${params.reason} lifecycle no-op for already fresh conversation ${current.conversationId}`,
+        );
+        return;
+      }
+      await this.conversationStore.archiveConversation(current.conversationId);
+    }
+
+    if (!params.createReplacement) {
+      this.deps.log.info(
+        `[lcm] ${params.reason} lifecycle archived conversation ${current?.conversationId ?? "(none)"}`,
+      );
+      return;
+    }
+
+    const nextSessionId = params.nextSessionId?.trim() || params.sessionId?.trim() || current?.sessionId;
+    if (!nextSessionId) {
+      this.deps.log.warn(`[lcm] ${params.reason} lifecycle skipped: no session identity available`);
+      return;
+    }
+    const nextSessionKey = params.nextSessionKey?.trim() || params.sessionKey?.trim() || current?.sessionKey;
+    const freshConversation = await this.conversationStore.createConversation({
+      sessionId: nextSessionId,
+      sessionKey: nextSessionKey,
+    });
+    this.deps.log.info(
+      `[lcm] ${params.reason} lifecycle archived prior conversation and created ${freshConversation.conversationId}`,
+    );
+  }
+
   /** Apply LCM lifecycle semantics for OpenClaw's /new and /reset commands. */
   async handleBeforeReset(params: {
     reason?: string;
@@ -3261,44 +3393,50 @@ export class LcmContextEngine implements ContextEngine {
             );
             return;
           }
-
-          const current = await this.conversationStore.getConversationForSession({
+          await this.applySessionReplacement({
+            reason: "/reset",
             sessionId: params.sessionId,
             sessionKey: params.sessionKey,
+            createReplacement: true,
+            createReplacementWhenMissing: true,
           });
-          if (current?.active) {
-            const currentMessageCount = await this.conversationStore.getMessageCount(
-              current.conversationId,
-            );
-            const currentContextItems = await this.summaryStore.getContextItems(
-              current.conversationId,
-            );
-            if (
-              currentMessageCount === 0
-              && currentContextItems.length === 0
-              && !current.bootstrappedAt
-            ) {
-              this.deps.log.info(
-                `[lcm] /reset no-op for already fresh conversation ${current.conversationId}`,
-              );
-              return;
-            }
-            await this.conversationStore.archiveConversation(current.conversationId);
-          }
+        }),
+    );
+  }
 
-          const nextSessionId = params.sessionId?.trim() || current?.sessionId;
-          if (!nextSessionId) {
-            this.deps.log.warn("[lcm] /reset skipped: no session identity available");
-            return;
-          }
+  /** Apply generic lifecycle semantics for session rollover and deletion hooks. */
+  async handleSessionEnd(params: {
+    reason?: string;
+    sessionId?: string;
+    sessionKey?: string;
+    nextSessionId?: string;
+    nextSessionKey?: string;
+  }): Promise<void> {
+    const reason = params.reason?.trim();
+    if (!reason || reason === "new" || reason === "unknown") {
+      return;
+    }
+    if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
+      return;
+    }
+    if (this.isStatelessSession(params.sessionKey ?? params.nextSessionKey)) {
+      return;
+    }
 
-          const freshConversation = await this.conversationStore.createConversation({
-            sessionId: nextSessionId,
-            sessionKey: params.sessionKey?.trim(),
+    const createReplacement = reason !== "deleted";
+    this.ensureMigrated();
+    await this.withSessionQueue(
+      this.resolveSessionQueueKey(params.nextSessionId ?? params.sessionId, params.sessionKey ?? params.nextSessionKey),
+      async () =>
+        this.conversationStore.withTransaction(async () => {
+          await this.applySessionReplacement({
+            reason: `session_end:${reason}`,
+            sessionId: params.sessionId,
+            sessionKey: params.sessionKey ?? params.nextSessionKey,
+            nextSessionId: params.nextSessionId,
+            nextSessionKey: params.nextSessionKey,
+            createReplacement,
           });
-          this.deps.log.info(
-            `[lcm] /reset archived prior conversation and created ${freshConversation.conversationId}`,
-          );
         }),
     );
   }
@@ -3446,3 +3584,6 @@ function createEmergencyFallbackSummarize(): (
     return text.slice(0, maxChars) + "\n[Truncated for context management]";
   };
 }
+
+/** @internal Exposed for unit tests only. */
+export const __testing = { readLastJsonlEntryBeforeOffset };

package/src/plugin/index.ts

@@ -66,6 +66,14 @@ type RuntimeModelAuthResult = {
   apiKey?: string;
 };
 
+type SessionEndLifecycleEvent = {
+  sessionId?: string;
+  sessionKey?: string;
+  reason?: string;
+  nextSessionId?: string;
+  nextSessionKey?: string;
+};
+
 type RuntimeModelAuthModel = {
   id: string;
   provider: string;
@@ -1529,9 +1537,9 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
     },
     agentLaneSubagent: "subagent",
     log: {
-      info: (msg) => api.logger.info(msg),
-      warn: (msg) => api.logger.warn(msg),
-      error: (msg) => api.logger.error(msg),
+      info: (msg) => console.error(msg),
+      warn: (msg) => console.error(msg),
+      error: (msg) => console.error(msg),
       debug: (msg) => api.logger.debug?.(msg),
     },
   };
@@ -1568,6 +1576,16 @@ const lcmPlugin = {
     api.on("before_prompt_build", () => ({
       prependSystemContext: LOSSLESS_RECALL_POLICY_PROMPT,
     }));
+    api.on("session_end", async (event) => {
+      const lifecycleEvent = event as SessionEndLifecycleEvent;
+      await lcm.handleSessionEnd({
+        reason: lifecycleEvent.reason,
+        sessionId: lifecycleEvent.sessionId,
+        sessionKey: lifecycleEvent.sessionKey,
+        nextSessionId: lifecycleEvent.nextSessionId,
+        nextSessionKey: lifecycleEvent.nextSessionKey,
+      });
+    });
     api.registerContextEngine("lossless-claw", () => lcm);
     api.registerContextEngine("default", () => lcm);
     api.registerTool((ctx) =>
@@ -1609,12 +1627,12 @@ const lcmPlugin = {
 
     logStartupBannerOnce({
       key: "plugin-loaded",
-      log: (message) => api.logger.info(message),
+      log: (message) => console.error(message),
       message: `[lcm] Plugin loaded (enabled=${deps.config.enabled}, db=${deps.config.databasePath}, threshold=${deps.config.contextThreshold})`,
     });
     logStartupBannerOnce({
       key: "compaction-model",
-      log: (message) => api.logger.info(message),
+      log: (message) => console.error(message),
       message: buildCompactionModelLog({
         config: deps.config,
         openClawConfig: api.config,