@martian-engineering/lossless-claw 0.5.3 → 0.6.1

package/skills/lossless-claw/references/recall-tools.md

@@ -0,0 +1,55 @@
+# Recall Tools
+
+Use recall tools when the question depends on exact historical evidence from compacted context.
+
+## Tool selection
+
+### `lcm_grep`
+
+Use for:
+
+- finding whether a term, file name, error string, or identifier appears in compacted history
+- narrowing the search space before deeper inspection
+
+Do not use it for:
+
+- answering detail-heavy questions by itself
+
+### `lcm_describe`
+
+Use for:
+
+- inspecting a specific summary or stored-file record by ID
+- reading lineage and content for a known summary node
+
+Do not use it for:
+
+- broad discovery when you do not know the target ID yet
+
+### `lcm_expand_query`
+
+Use for:
+
+- focused questions that need richer detail recovered from summaries
+- evidence-oriented follow-up after `lcm_grep` or `lcm_describe`
+
+This is the best recall tool when the user asks for:
+
+- exact commands
+- exact file paths
+- precise timestamps
+- root-cause chains
+
+### `lcm_expand`
+
+Treat as a specialized sub-agent flow, not the default first step.
+
+## Recommended workflow
+
+1. Start with `lcm_grep` to find likely evidence.
+2. Use `lcm_describe` when you have a summary or file ID.
+3. Use `lcm_expand_query` when the answer requires precise recovery rather than a high-level summary.
+
+## Important guardrail
+
+Do not infer exact details from summaries alone when the user needs evidence. Expand first or state that the answer still needs expansion.

package/skills/lossless-claw/references/session-lifecycle.md

@@ -0,0 +1,59 @@
+# Session lifecycle (`/new` and `/reset`)
+
+This reference describes the current behavior on `main`.
+
+## Short version
+
+For stock `lossless-claw` on current main:
+
+- OpenClaw handles `/new` and `/reset` as session-reset operations.
+- `lossless-claw` does **not** currently register its own `before_reset` hook or a custom reset policy.
+- `lossless-claw` prefers **`sessionKey`** as the stable identity for an LCM conversation.
+- When the same `sessionKey` reappears with a new `sessionId`, `lossless-claw` updates the stored `sessionId` on the existing LCM conversation row instead of creating a brand-new LCM conversation.
+
+## What that means in practice
+
+If a user asks whether `/new` or `/reset` gives them a fresh LCM conversation, the answer is usually **no** under the current implementation.
+
+They get a fresh OpenClaw session runtime, but LCM continuity still follows the stable `sessionKey` when one is available.
+
+So today:
+
+- `/new` and `/reset` can reset the runtime session
+- but LCM history may continue in the same conversation row if the chat/thread keeps the same `sessionKey`
+
+## Why
+
+Current lossless-claw conversation resolution does this:
+
+1. look up by `sessionKey` first
+2. fall back to `sessionId` only when no `sessionKey` match exists
+3. if the `sessionKey` already exists but the `sessionId` changed, update the stored `sessionId` on that same conversation
+
+That behavior preserves continuity across session resets for the same chat identity.
+
+## Important limitation
+
+There is currently **no plugin-specific `/new` vs `/reset` split** in stock lossless-claw docs or runtime behavior.
+
+If someone is asking for semantics like:
+
+- `/new` keeps LCM history but rotates transcript
+- `/reset` archives old LCM conversation and starts a new one
+
+that is a **design/spec topic**, not current stock behavior.
+
+## Safe operator guidance
+
+When answering users:
+
+- do not promise that `/new` or `/reset` clears LCM history
+- explain that current stock behavior follows `sessionKey` continuity
+- if they need a truly separate LCM history, use a different session key context (for example a different chat/thread/binding) or explicit non-MVP migration/surgery tools
+
+## Relation to `/status`
+
+This session behavior is separate from `/status` metrics.
+
+- `/status` reflects runtime session state and the last assembled request snapshot
+- `/lossless` reflects LCM conversation state keyed by the plugin's conversation mapping rules

package/src/assembler.ts

@@ -25,6 +25,8 @@ export interface AssembleContextInput {
   tokenBudget: number;
   /** Number of most recent raw turns to always include (default: 8) */
   freshTailCount?: number;
+  /** Optional user query for relevance-based eviction scoring (BM25-lite). When absent or unsearchable, falls back to chronological eviction. */
+  prompt?: string;
 }
 
 export interface AssembleContextResult {
@@ -52,10 +54,11 @@ function estimateTokens(text: string): number {
 type SummaryPromptSignal = Pick<SummaryRecord, "kind" | "depth" | "descendantCount">;
 
 /**
- * Build LCM usage guidance for the runtime system prompt.
+ * Build dynamic prompt guidance for compacted session context.
  *
  * Guidance is emitted only when summaries are present in assembled context.
- * Depth-aware: minimal for shallow compaction, full guidance for deep trees.
+ * Static recall policy lives in the plugin prompt hook so this addition
+ * remains session-specific and reflects only the current compaction state.
  */
 function buildSystemPromptAddition(summarySignals: SummaryPromptSignal[]): string | undefined {
   if (summarySignals.length === 0) {
@@ -68,36 +71,24 @@ function buildSystemPromptAddition(summarySignals: SummaryPromptSignal[]): strin
 
   const sections: string[] = [];
 
-  // Core recall workflow — always present when summaries exist
+  // Dynamic compaction reminder — always present when summaries exist.
   sections.push(
-    "## LCM Recall",
+    "## Compacted Conversation Context",
     "",
-    "Summaries above are compressed context — maps to details, not the details themselves.",
+    "Summaries above are compressed context, not full detail.",
     "",
-    "**Recall priority:** Use LCM tools first for compacted conversation history. If LCM does not cover the needed data, prefer any available memory/recall tool before falling back to raw text search.",
+    "Treat summaries as compressed recall cues rather than proof of exact wording or exact values.",
     "",
-    "**Conflict handling:** If newer evidence conflicts with an older summary or recollection, prefer the newer evidence. Do not trust a stale summary over fresher contradictory information.",
-    "",
-    "**Contradictions/uncertainty:** If facts seem contradictory or uncertain, verify with LCM tools before answering instead of trusting the summary at face value.",
-    "",
-    "**Tool escalation:**",
-    "1. `lcm_grep` — search by regex or full-text across messages and summaries",
-    "2. `lcm_describe` — inspect a specific summary (cheap, no sub-agent)",
-    "3. `lcm_expand_query` — deep recall: spawns bounded sub-agent, expands DAG, returns answer with cited summary IDs (~120s, don't ration it)",
-    "",
-    "**`lcm_expand_query` usage** — two patterns (always requires `prompt`):",
-    "- With IDs: `lcm_expand_query(summaryIds: [\"sum_xxx\"], prompt: \"What config changes were discussed?\")`",
-    "- With search: `lcm_expand_query(query: \"database migration\", prompt: \"What strategy was decided?\")`",
-    "- Optional: `maxTokens` (default 2000), `conversationId`, `allConversations: true`",
-    "",
-    "**Summaries include \"Expand for details about:\" footers** listing compressed specifics. Use `lcm_expand_query` with that summary's ID to retrieve them.",
+    "If a summary includes an \"Expand for details about:\" footer, use it as a cue to expand before asserting specifics.",
   );
 
-  // Precision/evidence rules — always present but stronger when heavily compacted
+  // Precision/evidence rules — always present but stronger when heavily compacted.
   if (heavilyCompacted) {
     sections.push(
       "",
-      "**\u26a0 Deeply compacted context — expand before asserting specifics.**",
+      "**Deeply compacted context: expand before asserting specifics.**",
+      "",
+      "Before answering with exact commands, SHAs, paths, timestamps, config values, or causal chains, expand for the missing detail.",
       "",
       "Default recall flow for precision work:",
       "1) `lcm_grep` to locate relevant summary/message IDs",
@@ -105,20 +96,20 @@ function buildSystemPromptAddition(summarySignals: SummaryPromptSignal[]): strin
       "3) Answer with citations to summary IDs used",
       "",
       "**Uncertainty checklist (run before answering):**",
-      "- Am I relying on an older summary even though newer evidence disagrees?",
-      "- Am I making exact factual claims from a condensed summary?",
+      "- Am I making an exact factual claim from a compressed or condensed summary?",
       "- Could compaction have omitted a crucial detail?",
-      "- Would this answer fail if the user asks for proof?",
+      "- Would I need an expansion step if the user asks for proof or the exact text?",
+      "- Should I state uncertainty instead of asserting specifics until I expand?",
       "",
-      "If yes to any \u2192 expand first.",
+      "If yes to any item, expand first or explicitly say that you need to expand.",
       "",
-      "**Do not guess** exact commands, SHAs, file paths, timestamps, config values, or causal claims from condensed summaries. Expand first or state that you need to expand.",
+      "Do not guess exact commands, SHAs, file paths, timestamps, config values, or causal claims from condensed summaries. Expand first or explicitly say that you need to expand.",
     );
   } else {
     sections.push(
       "",
-      "**For precision/evidence questions** (exact commands, SHAs, paths, timestamps, config values, root-cause chains): expand before answering.",
-      "Do not guess from condensed summaries — expand first or state uncertainty.",
+      "For exact commands, SHAs, paths, timestamps, config values, or causal chains, expand for details before answering.",
+      "State uncertainty instead of guessing from compressed summaries.",
     );
   }
 
@@ -281,6 +272,20 @@ export function toolResultBlockFromPart(
   rawType?: string,
   raw?: Record<string, unknown>,
 ): unknown {
+  if (
+    raw &&
+    typeof raw.text === "string" &&
+    raw.output === undefined &&
+    raw.content === undefined &&
+    (part.toolOutput == null || part.toolOutput === "") &&
+    (part.textContent == null || part.textContent === raw.text)
+  ) {
+    return {
+      type: "text",
+      text: raw.text,
+    };
+  }
+
   const type =
     rawType === "function_call_output" || rawType === "toolResult" || rawType === "tool_result"
       ? rawType
@@ -468,7 +473,8 @@ export function blockFromPart(part: MessagePartRecord): unknown {
   return { type: "text", text: "" };
 }
 
-function contentFromParts(
+/** @internal Exported for transcript-maintenance reconstruction. */
+export function contentFromParts(
   parts: MessagePartRecord[],
   role: "user" | "assistant" | "toolResult",
   fallbackContent: string,
@@ -497,7 +503,8 @@ function contentFromParts(
   return blocks;
 }
 
-function pickToolCallId(parts: MessagePartRecord[]): string | undefined {
+/** @internal Exported for transcript-maintenance reconstruction. */
+export function pickToolCallId(parts: MessagePartRecord[]): string | undefined {
   for (const part of parts) {
     if (typeof part.toolCallId === "string" && part.toolCallId.length > 0) {
       return part.toolCallId;
@@ -526,7 +533,8 @@ function pickToolCallId(parts: MessagePartRecord[]): string | undefined {
   return undefined;
 }
 
-function pickToolName(parts: MessagePartRecord[]): string | undefined {
+/** @internal Exported for transcript-maintenance reconstruction. */
+export function pickToolName(parts: MessagePartRecord[]): string | undefined {
   for (const part of parts) {
     if (typeof part.toolName === "string" && part.toolName.length > 0) {
       return part.toolName;
@@ -555,7 +563,8 @@ function pickToolName(parts: MessagePartRecord[]): string | undefined {
   return undefined;
 }
 
-function pickToolIsError(parts: MessagePartRecord[]): boolean | undefined {
+/** @internal Exported for transcript-maintenance reconstruction. */
+export function pickToolIsError(parts: MessagePartRecord[]): boolean | undefined {
   for (const part of parts) {
     const decoded = parseJson(part.metadata);
     if (!decoded || typeof decoded !== "object") {
@@ -814,10 +823,60 @@ interface ResolvedItem {
   tokens: number;
   /** Whether this came from a raw message (vs. a summary) */
   isMessage: boolean;
+  /** Pre-extracted plain text used for relevance scoring */
+  text: string;
   /** Summary metadata used for dynamic system prompt guidance */
   summarySignal?: SummaryPromptSignal;
 }
 
+// ── BM25-lite relevance scorer ────────────────────────────────────────────────
+
+/** @internal Exported for testing only. Tokenize text into lowercase alphanumeric terms. */
+export function tokenizeText(text: string): string[] {
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((t) => t.length > 1);
+}
+
+/**
+ * @internal Exported for testing only.
+ * Score an item's text against a prompt using BM25-lite (term-frequency overlap).
+ * Higher scores indicate stronger keyword overlap. Returns 0 when either input is empty.
+ */
+export function scoreRelevance(itemText: string, prompt: string): number {
+  const promptTerms = tokenizeText(prompt);
+  if (promptTerms.length === 0) return 0;
+
+  const itemTerms = tokenizeText(itemText);
+  if (itemTerms.length === 0) return 0;
+
+  // Build term-frequency map for the item
+  const freq = new Map<string, number>();
+  for (const term of itemTerms) {
+    freq.set(term, (freq.get(term) ?? 0) + 1);
+  }
+
+  // Sum TF contribution for each unique prompt term
+  const seen = new Set<string>();
+  let score = 0;
+  for (const term of promptTerms) {
+    if (seen.has(term)) continue;
+    seen.add(term);
+    const tf = freq.get(term) ?? 0;
+    if (tf > 0) {
+      // Normalised TF: tf / itemLength (BM25-lite saturation skipped for simplicity)
+      score += tf / itemTerms.length;
+    }
+  }
+  return score;
+}
+
+/** Return true when a prompt contains at least one searchable term. */
+function hasSearchablePrompt(prompt?: string): prompt is string {
+  return typeof prompt === "string" && tokenizeText(prompt).length > 0;
+}
+
 // ── ContextAssembler ─────────────────────────────────────────────────────────
 
 export class ContextAssembler {
@@ -910,8 +969,32 @@ export class ContextAssembler {
       // Everything fits
       selected.push(...evictable);
       evictableTokens = evictableTotalTokens;
+    } else if (hasSearchablePrompt(input.prompt)) {
+      // Prompt-aware eviction: score each evictable item by relevance to the
+      // prompt, then greedily fill budget from highest-scoring items down.
+      // Re-sort selected items by ordinal to restore chronological order.
+      const scored = evictable.map((item, idx) => ({
+        item,
+        score: scoreRelevance(item.text, input.prompt),
+        idx, // original index — higher = more recent, used as tiebreaker
+      }));
+      // Sort: highest relevance first; most recent (higher idx) breaks ties
+      scored.sort((a, b) => b.score - a.score || b.idx - a.idx);
+
+      const kept: ResolvedItem[] = [];
+      let accum = 0;
+      for (const { item } of scored) {
+        if (accum + item.tokens <= remainingBudget) {
+          kept.push(item);
+          accum += item.tokens;
+        }
+      }
+      // Restore chronological order by ordinal before appending freshTail
+      kept.sort((a, b) => a.ordinal - b.ordinal);
+      selected.push(...kept);
+      evictableTokens = accum;
     } else {
-      // Need to drop oldest items until we fit.
+      // Chronological eviction (default): drop oldest items until we fit.
       // Walk from the END of evictable (newest first) accumulating tokens,
       // then reverse to restore chronological order.
       const kept: ResolvedItem[] = [];
@@ -949,8 +1032,19 @@ export class ContextAssembler {
       }
     }
 
+    // Filter out assistant messages with empty content — these can occur when
+    // tool-use-only turns are stored with content="" and zero message_parts,
+    // or when filterNonFreshAssistantToolCalls strips all tool_use blocks.
+    // Anthropic (and other providers) reject empty content arrays/strings.
+    const cleaned = rawMessages.filter(
+      (m) =>
+        !(
+          m?.role === "assistant" &&
+          (Array.isArray(m.content) ? m.content.length === 0 : !m.content)
+        ),
+    );
     return {
-      messages: sanitizeToolUseResultPairing(rawMessages) as AgentMessage[],
+      messages: sanitizeToolUseResultPairing(cleaned) as AgentMessage[],
       estimatedTokens,
       systemPromptAddition,
       stats: {
@@ -1056,6 +1150,7 @@ export class ContextAssembler {
             } as AgentMessage),
       tokens: tokenCount,
       isMessage: true,
+      text: contentText,
     };
   }
 
@@ -1078,6 +1173,7 @@ export class ContextAssembler {
       message: { role: "user" as const, content } as AgentMessage,
       tokens,
       isMessage: false,
+      text: summary.content,
       summarySignal: {
         kind: summary.kind,
         depth: summary.depth,

package/src/compaction.ts

@@ -25,6 +25,8 @@ export interface CompactionResult {
   condensed: boolean;
   /** Escalation level used: "normal" | "aggressive" | "fallback" */
   level?: CompactionLevel;
+  /** Whether compaction was blocked by a provider auth failure */
+  authFailure?: boolean;
 }
 
 export interface CompactionConfig {
@@ -465,6 +467,7 @@ export class CompactionEngine {
         tokensBefore,
         tokensAfter: tokensBefore,
         condensed: false,
+        authFailure: true,
       };
     }
     const tokensAfterLeaf = await this.summaryStore.getContextTokenCount(conversationId);
@@ -581,6 +584,7 @@ export class CompactionEngine {
     let level: CompactionLevel | undefined;
     let previousSummaryContent: string | undefined;
     let previousTokens = tokensBefore;
+    let hadAuthFailure = false;
 
     // Phase 1: leaf passes over oldest raw chunks outside the protected tail.
     while (true) {
@@ -598,6 +602,7 @@ export class CompactionEngine {
         input.summaryModel,
       );
       if (!leafResult) {
+        hadAuthFailure = true;
         break;
       }
       const passTokensAfter = await this.summaryStore.getContextTokenCount(conversationId);
@@ -644,6 +649,7 @@ export class CompactionEngine {
         input.summaryModel,
       );
       if (!condenseResult) {
+        hadAuthFailure = true;
         break;
       }
       const passTokensAfter = await this.summaryStore.getContextTokenCount(conversationId);
@@ -680,6 +686,7 @@ export class CompactionEngine {
       createdSummaryId,
       condensed,
       level,
+      ...(hadAuthFailure ? { authFailure: true } : {}),
     };
   }
 
@@ -693,7 +700,7 @@ export class CompactionEngine {
     currentTokens?: number;
     summarize: CompactionSummarizeFn;
     summaryModel?: string;
-  }): Promise<{ success: boolean; rounds: number; finalTokens: number }> {
+  }): Promise<{ success: boolean; rounds: number; finalTokens: number; authFailure?: boolean }> {
     const { conversationId, tokenBudget, summarize } = input;
     const targetTokens =
       typeof input.targetTokens === "number" &&
@@ -727,6 +734,15 @@ export class CompactionEngine {
         summaryModel: input.summaryModel,
       });
 
+      if (result.authFailure) {
+        return {
+          success: false,
+          rounds: round,
+          finalTokens: result.tokensAfter,
+          authFailure: true,
+        };
+      }
+
       if (result.tokensAfter <= targetTokens) {
         return {
           success: true,
@@ -1542,12 +1558,7 @@ export class CompactionEngine {
     return { summaryId, level: condensed.level };
   }
 
-  /**
-   * Persist durable compaction events into canonical history as message parts.
-   *
-   * Event persistence is best-effort: failures are swallowed to avoid
-   * compromising the core compaction path.
-   */
+  /** Emit compaction telemetry without mutating canonical conversation history. */
   private async persistCompactionEvents(input: {
     conversationId: number;
     tokensBefore: number;
@@ -1608,7 +1619,7 @@ export class CompactionEngine {
     }
   }
 
-  /** Write one compaction event message + part atomically where possible. */
+  /** Log one compaction event without appending a synthetic chat message. */
   private async persistCompactionEvent(input: {
     conversationId: number;
     sessionId: string;
@@ -1621,43 +1632,8 @@ export class CompactionEngine {
     condensedPassOccurred: boolean;
   }): Promise<void> {
     const content = `LCM compaction ${input.pass} pass (${input.level}): ${input.tokensBefore} -> ${input.tokensAfter}`;
-    const metadata = JSON.stringify({
-      conversationId: input.conversationId,
-      pass: input.pass,
-      level: input.level,
-      tokensBefore: input.tokensBefore,
-      tokensAfter: input.tokensAfter,
-      createdSummaryId: input.createdSummaryId,
-      createdSummaryIds: input.createdSummaryIds,
-      condensedPassOccurred: input.condensedPassOccurred,
-    });
-
-    const writeEvent = async (): Promise<void> => {
-      const seq = (await this.conversationStore.getMaxSeq(input.conversationId)) + 1;
-      const eventMessage = await this.conversationStore.createMessage({
-        conversationId: input.conversationId,
-        seq,
-        role: "system",
-        content,
-        tokenCount: estimateTokens(content),
-      });
-
-      const parts: CreateMessagePartInput[] = [
-        {
-          sessionId: input.sessionId,
-          partType: "compaction",
-          ordinal: 0,
-          textContent: content,
-          metadata,
-        },
-      ];
-      await this.conversationStore.createMessageParts(eventMessage.messageId, parts);
-    };
-
-    try {
-      await this.conversationStore.withTransaction(() => writeEvent());
-    } catch {
-      // Compaction should still succeed if event persistence fails.
-    }
+    console.info(
+      `[lcm] ${content} conversation=${input.conversationId} summary=${input.createdSummaryId}`,
+    );
   }
 }