@martian-engineering/lossless-claw 0.2.3 → 0.2.4

package/README.md

@@ -22,7 +22,7 @@ Nothing is lost. Raw messages stay in the database. Summaries link back to their
 
 ### Prerequisites
 
-- OpenClaw with context engine support (josh/context-engine branch or equivalent)
+- OpenClaw with plugin context engine support
 - Node.js 22+
 - An LLM provider configured in OpenClaw (used for summarization)
 
@@ -243,7 +243,12 @@ Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
   "plugins": {
     "entries": {
       "lossless-claw": {
-        "enabled": true
+        "enabled": true,
+        "config": {
+          "freshTailCount": 32,
+          "contextThreshold": 0.75,
+          "incrementalMaxDepth": -1
+        }
       }
     }
   }
@@ -267,9 +272,12 @@ Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
 | `LCM_CONDENSED_TARGET_TOKENS` | `2000` | Target token count for condensed summaries |
 | `LCM_MAX_EXPAND_TOKENS` | `4000` | Token cap for sub-agent expansion queries |
 | `LCM_LARGE_FILE_TOKEN_THRESHOLD` | `25000` | File blocks above this size are intercepted and stored separately |
+| `LCM_LARGE_FILE_SUMMARY_PROVIDER` | `""` | Provider override for large-file summarization |
+| `LCM_LARGE_FILE_SUMMARY_MODEL` | `""` | Model override for large-file summarization |
 | `LCM_SUMMARY_MODEL` | *(from OpenClaw)* | Model for summarization (e.g. `anthropic/claude-sonnet-4-20250514`) |
 | `LCM_SUMMARY_PROVIDER` | *(from OpenClaw)* | Provider override for summarization |
-| `LCM_INCREMENTAL_MAX_DEPTH` | `0` | Depth limit for incremental condensation after leaf passes (-1 = unlimited) |
+| `LCM_AUTOCOMPACT_DISABLED` | `false` | Disable automatic compaction after turns |
+| `LCM_PRUNE_HEARTBEAT_OK` | `false` | Retroactively delete `HEARTBEAT_OK` turn cycles from LCM storage |
 
 ### Recommended starting configuration
 
@@ -283,6 +291,47 @@ LCM_CONTEXT_THRESHOLD=0.75
 - **incrementalMaxDepth=-1** enables unlimited automatic condensation after each compaction pass — the DAG cascades as deep as needed. Set to `0` (default) for leaf-only, or a positive integer for a specific depth cap.
 - **contextThreshold=0.75** triggers compaction when context reaches 75% of the model's window, leaving headroom for the model's response.
 
+### OpenClaw session reset settings
+
+LCM preserves history through compaction, but it does **not** change OpenClaw's core session reset policy. If sessions are resetting sooner than you want, increase OpenClaw's `session.reset.idleMinutes` or use a channel/type-specific override.
+
+```json
+{
+  "session": {
+    "reset": {
+      "mode": "idle",
+      "idleMinutes": 10080
+    }
+  }
+}
+```
+
+- `session.reset.mode: "idle"` keeps a session alive until the idle window expires.
+- `session.reset.idleMinutes` is the actual reset interval in minutes.
+- OpenClaw does **not** currently enforce a maximum `idleMinutes`; in source it is validated only as a positive integer.
+- If you also use daily reset mode, `idleMinutes` acts as a secondary guard and the session resets when **either** the daily boundary or the idle window is reached first.
+- Legacy `session.idleMinutes` still works, but OpenClaw prefers `session.reset.idleMinutes`.
+
+Useful values:
+
+- `1440` = 1 day
+- `10080` = 7 days
+- `43200` = 30 days
+- `525600` = 365 days
+
+For most long-lived LCM setups, a good starting point is:
+
+```json
+{
+  "session": {
+    "reset": {
+      "mode": "idle",
+      "idleMinutes": 10080
+    }
+  }
+}
+```
+
 ## How it works
 
 See [docs/architecture.md](docs/architecture.md) for the full technical deep-dive. Here's the summary:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@martian-engineering/lossless-claw",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "Lossless Context Management plugin for OpenClaw — DAG-based conversation summarization with incremental compaction",
   "type": "module",
   "main": "index.ts",

package/src/assembler.ts

@@ -144,6 +144,137 @@ function getOriginalRole(parts: MessagePartRecord[]): string | null {
   return null;
 }
 
+function getPartMetadata(part: MessagePartRecord): {
+  originalRole?: string;
+  rawType?: string;
+  raw?: unknown;
+} {
+  const decoded = parseJson(part.metadata);
+  if (!decoded || typeof decoded !== "object") {
+    return {};
+  }
+
+  const record = decoded as {
+    originalRole?: unknown;
+    rawType?: unknown;
+    raw?: unknown;
+  };
+  return {
+    originalRole:
+      typeof record.originalRole === "string" && record.originalRole.length > 0
+        ? record.originalRole
+        : undefined,
+    rawType:
+      typeof record.rawType === "string" && record.rawType.length > 0
+        ? record.rawType
+        : undefined,
+    raw: record.raw,
+  };
+}
+
+function parseStoredValue(value: string | null): unknown {
+  if (typeof value !== "string" || value.length === 0) {
+    return undefined;
+  }
+  const parsed = parseJson(value);
+  return parsed !== undefined ? parsed : value;
+}
+
+function reasoningBlockFromPart(part: MessagePartRecord, rawType?: string): unknown {
+  const type = rawType === "thinking" ? "thinking" : "reasoning";
+  if (typeof part.textContent === "string" && part.textContent.length > 0) {
+    return type === "thinking"
+      ? { type, thinking: part.textContent }
+      : { type, text: part.textContent };
+  }
+  return { type };
+}
+
+/**
+ * Detect if a raw block is an OpenClaw-normalised OpenAI reasoning item.
+ * OpenClaw converts OpenAI `{type:"reasoning", id:"rs_…", encrypted_content:"…"}`
+ * into `{type:"thinking", thinking:"", thinkingSignature:"{…}"}`.
+ * When we reassemble for the OpenAI provider we need the original back.
+ */
+function tryRestoreOpenAIReasoning(raw: Record<string, unknown>): Record<string, unknown> | null {
+  if (raw.type !== "thinking") return null;
+  const sig = raw.thinkingSignature;
+  if (typeof sig !== "string" || !sig.startsWith("{")) return null;
+  try {
+    const parsed = JSON.parse(sig) as Record<string, unknown>;
+    if (parsed.type === "reasoning" && typeof parsed.id === "string") {
+      return parsed;
+    }
+  } catch {
+    // not valid JSON — leave as-is
+  }
+  return null;
+}
+
+function toolCallBlockFromPart(part: MessagePartRecord, rawType?: string): unknown {
+  const type =
+    rawType === "function_call" ||
+    rawType === "functionCall" ||
+    rawType === "tool_use" ||
+    rawType === "tool-use" ||
+    rawType === "toolUse" ||
+    rawType === "toolCall"
+      ? rawType
+      : "toolCall";
+  const input = parseStoredValue(part.toolInput);
+  const block: Record<string, unknown> = { type };
+
+  if (type === "function_call") {
+    if (typeof part.toolCallId === "string" && part.toolCallId.length > 0) {
+      block.call_id = part.toolCallId;
+    }
+    if (typeof part.toolName === "string" && part.toolName.length > 0) {
+      block.name = part.toolName;
+    }
+    if (input !== undefined) {
+      block.arguments = input;
+    }
+    return block;
+  }
+
+  if (typeof part.toolCallId === "string" && part.toolCallId.length > 0) {
+    block.id = part.toolCallId;
+  }
+  if (typeof part.toolName === "string" && part.toolName.length > 0) {
+    block.name = part.toolName;
+  }
+
+  if (input !== undefined) {
+    if (type === "functionCall") {
+      block.arguments = input;
+    } else {
+      block.input = input;
+    }
+  }
+  return block;
+}
+
+function toolResultBlockFromPart(part: MessagePartRecord, rawType?: string): unknown {
+  const type =
+    rawType === "function_call_output" || rawType === "toolResult" || rawType === "tool_result"
+      ? rawType
+      : "tool_result";
+  const output = parseStoredValue(part.toolOutput) ?? part.textContent ?? "";
+  const block: Record<string, unknown> = { type, output };
+
+  if (type === "function_call_output") {
+    if (typeof part.toolCallId === "string" && part.toolCallId.length > 0) {
+      block.call_id = part.toolCallId;
+    }
+    return block;
+  }
+
+  if (typeof part.toolCallId === "string" && part.toolCallId.length > 0) {
+    block.tool_use_id = part.toolCallId;
+  }
+  return block;
+}
+
 function toRuntimeRole(
   dbRole: MessageRole,
   parts: MessagePartRecord[],
@@ -173,26 +304,43 @@ function toRuntimeRole(
 }
 
 function blockFromPart(part: MessagePartRecord): unknown {
-  const decoded = parseJson(part.metadata);
-  if (decoded && typeof decoded === "object") {
-    const raw = (decoded as { raw?: unknown }).raw;
-    if (raw && typeof raw === "object") {
-      return raw;
-    }
+  const metadata = getPartMetadata(part);
+  if (metadata.raw && typeof metadata.raw === "object") {
+    // If this is an OpenClaw-normalised OpenAI reasoning block, restore the original
+    // OpenAI format so the Responses API gets the {type:"reasoning", id:"rs_…"} it expects.
+    const restored = tryRestoreOpenAIReasoning(metadata.raw as Record<string, unknown>);
+    if (restored) return restored;
+    return metadata.raw;
   }
 
-  if (part.partType === "text" || part.partType === "reasoning") {
-    return { type: "text", text: part.textContent ?? "" };
+  if (part.partType === "reasoning") {
+    return reasoningBlockFromPart(part, metadata.rawType);
   }
   if (part.partType === "tool") {
-    const toolOutput = parseJson(part.toolOutput);
-    if (toolOutput !== undefined) {
-      return toolOutput;
-    }
-    if (typeof part.textContent === "string") {
-      return { type: "text", text: part.textContent };
+    if (metadata.originalRole === "toolResult" || metadata.rawType === "function_call_output") {
+      return toolResultBlockFromPart(part, metadata.rawType);
     }
-    return { type: "text", text: part.toolOutput ?? part.toolInput ?? "" };
+    return toolCallBlockFromPart(part, metadata.rawType);
+  }
+  if (
+    metadata.rawType === "function_call" ||
+    metadata.rawType === "functionCall" ||
+    metadata.rawType === "tool_use" ||
+    metadata.rawType === "tool-use" ||
+    metadata.rawType === "toolUse" ||
+    metadata.rawType === "toolCall"
+  ) {
+    return toolCallBlockFromPart(part, metadata.rawType);
+  }
+  if (
+    metadata.rawType === "function_call_output" ||
+    metadata.rawType === "tool_result" ||
+    metadata.rawType === "toolResult"
+  ) {
+    return toolResultBlockFromPart(part, metadata.rawType);
+  }
+  if (part.partType === "text") {
+    return { type: "text", text: part.textContent ?? "" };
   }
 
   if (typeof part.textContent === "string" && part.textContent.length > 0) {

package/src/engine.ts

@@ -60,6 +60,39 @@ function safeString(value: unknown): string | undefined {
   return typeof value === "string" ? value : undefined;
 }
 
+function appendTextValue(value: unknown, out: string[]): void {
+  if (typeof value === "string") {
+    out.push(value);
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      appendTextValue(entry, out);
+    }
+    return;
+  }
+  if (!value || typeof value !== "object") {
+    return;
+  }
+
+  const record = value as Record<string, unknown>;
+  appendTextValue(record.text, out);
+  appendTextValue(record.value, out);
+}
+
+function extractReasoningText(record: Record<string, unknown>): string | undefined {
+  const chunks: string[] = [];
+  appendTextValue(record.summary, chunks);
+  if (chunks.length === 0) {
+    return undefined;
+  }
+
+  const normalized = chunks
+    .map((chunk) => chunk.trim())
+    .filter((chunk, idx, arr) => chunk.length > 0 && arr.indexOf(chunk) === idx);
+  return normalized.length > 0 ? normalized.join("\n") : undefined;
+}
+
 function normalizeUnknownBlock(value: unknown): {
   type: string;
   text?: string;
@@ -76,7 +109,12 @@ function normalizeUnknownBlock(value: unknown): {
   const rawType = safeString(record.type);
   return {
     type: rawType ?? "agent",
-    text: safeString(record.text) ?? safeString(record.thinking),
+    text:
+      safeString(record.text) ??
+      safeString(record.thinking) ??
+      ((rawType === "reasoning" || rawType === "thinking")
+        ? extractReasoningText(record)
+        : undefined),
     metadata: { raw: record },
   };
 }
@@ -89,7 +127,12 @@ function toPartType(type: string): MessagePartType {
     case "reasoning":
       return "reasoning";
     case "tool_use":
+    case "toolUse":
     case "tool-use":
+    case "toolCall":
+    case "functionCall":
+    case "function_call":
+    case "function_call_output":
     case "tool_result":
     case "toolResult":
     case "tool":
@@ -215,7 +258,12 @@ function buildMessageParts(params: {
   const role = typeof message.role === "string" ? message.role : "unknown";
   const topLevel = message as unknown as Record<string, unknown>;
   const topLevelToolCallId =
-    safeString(topLevel.toolCallId) ?? safeString(topLevel.tool_call_id) ?? safeString(topLevel.id);
+    safeString(topLevel.toolCallId) ??
+    safeString(topLevel.tool_call_id) ??
+    safeString(topLevel.toolUseId) ??
+    safeString(topLevel.tool_use_id) ??
+    safeString(topLevel.call_id) ??
+    safeString(topLevel.id);
 
   // BashExecutionMessage: preserve a synthetic text part so output is round-trippable.
   if (!("content" in message) && "command" in message && "output" in message) {
@@ -284,14 +332,19 @@ function buildMessageParts(params: {
   for (let ordinal = 0; ordinal < message.content.length; ordinal++) {
     const block = normalizeUnknownBlock(message.content[ordinal]);
     const metadataRecord = block.metadata.raw as Record<string, unknown> | undefined;
+    const partType = toPartType(block.type);
     const toolCallId =
       safeString(metadataRecord?.toolCallId) ??
       safeString(metadataRecord?.tool_call_id) ??
+      safeString(metadataRecord?.toolUseId) ??
+      safeString(metadataRecord?.tool_use_id) ??
+      safeString(metadataRecord?.call_id) ??
+      (partType === "tool" ? safeString(metadataRecord?.id) : undefined) ??
       topLevelToolCallId;
 
     parts.push({
       sessionId,
-      partType: toPartType(block.type),
+      partType,
       ordinal,
       textContent: block.text ?? null,
       toolCallId,
@@ -302,6 +355,8 @@ function buildMessageParts(params: {
       toolInput:
         metadataRecord?.input !== undefined
           ? toJson(metadataRecord.input)
+          : metadataRecord?.arguments !== undefined
+            ? toJson(metadataRecord.arguments)
           : metadataRecord?.toolInput !== undefined
             ? toJson(metadataRecord.toolInput)
             : (safeString(metadataRecord?.tool_input) ?? null),

package/src/transcript-repair.ts

@@ -27,7 +27,80 @@ type ToolCallLike = {
 
 // -- Extraction helpers (from tool-call-id.ts) --
 
-const TOOL_CALL_TYPES = new Set(["toolCall", "toolUse", "functionCall"]);
+const TOOL_CALL_TYPES = new Set([
+  "toolCall",
+  "toolUse",
+  "tool_use",
+  "tool-use",
+  "functionCall",
+  "function_call",
+]);
+const OPENAI_FUNCTION_CALL_TYPES = new Set(["functionCall", "function_call"]);
+
+function extractToolCallId(block: { id?: unknown; call_id?: unknown }): string | null {
+  if (typeof block.id === "string" && block.id) {
+    return block.id;
+  }
+  if (typeof block.call_id === "string" && block.call_id) {
+    return block.call_id;
+  }
+  return null;
+}
+
+function normalizeAssistantReasoningBlocks<T extends AgentMessageLike>(message: T): T {
+  if (!Array.isArray(message.content)) {
+    return message;
+  }
+
+  let sawToolCall = false;
+  let reasoningAfterToolCall = false;
+  let functionCallCount = 0;
+
+  for (const block of message.content) {
+    if (!block || typeof block !== "object") {
+      return message;
+    }
+
+    const type = (block as { type?: unknown }).type;
+    if (type === "reasoning" || type === "thinking") {
+      if (sawToolCall) {
+        reasoningAfterToolCall = true;
+      }
+      continue;
+    }
+
+    if (typeof type === "string" && TOOL_CALL_TYPES.has(type)) {
+      sawToolCall = true;
+      if (OPENAI_FUNCTION_CALL_TYPES.has(type)) {
+        functionCallCount += 1;
+      }
+      continue;
+    }
+
+    return message;
+  }
+
+  // Only repair the specific OpenAI shape we need: a single function call that
+  // has one or more reasoning blocks after it. Multi-call turns may use
+  // interleaved reasoning intentionally, so leave them untouched.
+  if (!reasoningAfterToolCall || functionCallCount !== 1) {
+    return message;
+  }
+
+  const reasoning = message.content.filter((block) => {
+    const type = (block as { type?: unknown }).type;
+    return type === "reasoning" || type === "thinking";
+  });
+  const toolCalls = message.content.filter((block) => {
+    const type = (block as { type?: unknown }).type;
+    return typeof type === "string" && TOOL_CALL_TYPES.has(type);
+  });
+
+  return {
+    ...message,
+    content: [...reasoning, ...toolCalls],
+  };
+}
 
 function extractToolCallsFromAssistant(msg: AgentMessageLike): ToolCallLike[] {
   const content = msg.content;
@@ -40,13 +113,14 @@ function extractToolCallsFromAssistant(msg: AgentMessageLike): ToolCallLike[] {
     if (!block || typeof block !== "object") {
       continue;
     }
-    const rec = block as { type?: unknown; id?: unknown; name?: unknown };
-    if (typeof rec.id !== "string" || !rec.id) {
+    const rec = block as { type?: unknown; id?: unknown; call_id?: unknown; name?: unknown };
+    const id = extractToolCallId(rec);
+    if (!id) {
       continue;
     }
     if (typeof rec.type === "string" && TOOL_CALL_TYPES.has(rec.type)) {
       toolCalls.push({
-        id: rec.id,
+        id,
         name: typeof rec.name === "string" ? rec.name : undefined,
       });
     }
@@ -134,18 +208,23 @@ export function sanitizeToolUseResultPairing<T extends AgentMessageLike>(message
       continue;
     }
 
+    const normalizedAssistant = normalizeAssistantReasoningBlocks(msg);
+    if (normalizedAssistant !== msg) {
+      changed = true;
+    }
+
     // Skip tool call extraction for aborted or errored assistant messages.
     // When stopReason is "error" or "aborted", the tool_use blocks may be incomplete
     // and should not have synthetic tool_results created.
-    const stopReason = msg.stopReason;
+    const stopReason = normalizedAssistant.stopReason;
     if (stopReason === "error" || stopReason === "aborted") {
-      out.push(msg);
+      out.push(normalizedAssistant as T);
       continue;
     }
 
-    const toolCalls = extractToolCallsFromAssistant(msg);
+    const toolCalls = extractToolCallsFromAssistant(normalizedAssistant);
     if (toolCalls.length === 0) {
-      out.push(msg);
+      out.push(normalizedAssistant as T);
       continue;
     }
 
@@ -190,7 +269,7 @@ export function sanitizeToolUseResultPairing<T extends AgentMessageLike>(message
       }
     }
 
-    out.push(msg);
+    out.push(normalizedAssistant as T);
 
     if (spanResultsById.size > 0 && remainder.length > 0) {
       moved = true;