@martian-engineering/lossless-claw 0.1.4 → 0.1.6

package/docs/configuration.md

@@ -102,6 +102,13 @@ export LCM_SUMMARY_PROVIDER=anthropic
 
 Using a cheaper/faster model for summarization can reduce costs, but quality matters — poor summaries compound as they're condensed into higher-level nodes.
 
+## TUI conversation window size
+
+`LCM_TUI_CONVERSATION_WINDOW_SIZE` (default `200`) controls how many messages `lcm-tui` loads per keyset-paged conversation window when a session has an LCM `conversation_id`.
+
+- Smaller values reduce render/query cost for very large conversations.
+- Larger values show more context per page but increase render time.
+
 ## Database management
 
 The SQLite database lives at `LCM_DATABASE_PATH` (default `~/.openclaw/lcm.db`). 

package/docs/tui.md

@@ -72,12 +72,16 @@ A scrollable, color-coded view of the raw session messages. Each message shows i
 
 This is the raw session data, not the LCM-managed context. Use it to understand what actually happened in the conversation.
 
+For sessions with an LCM `conv_id`, the conversation view uses keyset-paged windows by `message_id` (newest window first) instead of hydrating full history.
+
 | Key | Action |
 |-----|--------|
 | `↑`/`↓` or `k`/`j` | Scroll one line |
 | `PgUp`/`PgDn` | Scroll half page |
 | `g` | Jump to top |
 | `G` | Jump to bottom |
+| `[` | Load older message window |
+| `]` | Load newer message window |
 | `l` | Open **Summary DAG** view |
 | `c` | Open **Context** view |
 | `f` | Open **Large Files** view |
@@ -475,6 +479,7 @@ If the provider auth profile mode is `oauth` (not `api_key`), set the provider A
 Interactive rewrite (`w`/`W`) can be configured with:
 - `LCM_TUI_SUMMARY_PROVIDER`
 - `LCM_TUI_SUMMARY_MODEL`
+- `LCM_TUI_CONVERSATION_WINDOW_SIZE` (default `200`)
 
 It also honors `LCM_SUMMARY_PROVIDER` / `LCM_SUMMARY_MODEL` as fallback.
 

package/index.ts

@@ -49,6 +49,13 @@ type PluginEnvSnapshot = {
 
 type ReadEnvFn = (key: string) => string | undefined;
 
+type CompleteSimpleOptions = {
+  apiKey?: string;
+  maxTokens: number;
+  temperature?: number;
+  reasoning?: string;
+};
+
 /** Capture plugin env values once during initialization. */
 function snapshotPluginEnv(env: NodeJS.ProcessEnv = process.env): PluginEnvSnapshot {
   return {
@@ -130,13 +137,17 @@ type PiAiModule = {
       contextWindow?: number;
       maxTokens?: number;
     },
-    request: { messages: Array<{ role: string; content: unknown; timestamp?: number }> },
+    request: {
+      systemPrompt?: string;
+      messages: Array<{ role: string; content: unknown; timestamp?: number }>;
+    },
     options: {
       apiKey?: string;
       maxTokens: number;
       temperature?: number;
+      reasoning?: string;
     },
-  ) => Promise<{ content?: Array<{ type: string; text?: string }> }>;
+  ) => Promise<Record<string, unknown> & { content?: Array<{ type: string; text?: string }> }>;
   getModel?: (provider: string, modelId: string) => unknown;
   getModels?: (provider: string) => unknown[];
   getEnvApiKey?: (provider: string) => string | undefined;
@@ -173,6 +184,39 @@ function inferApiFromProvider(provider: string): string {
   return map[normalized] ?? "openai-responses";
 }
 
+/** Codex Responses rejects `temperature`; omit it for that API family. */
+export function shouldOmitTemperatureForApi(api: string | undefined): boolean {
+  return (api ?? "").trim().toLowerCase() === "openai-codex-responses";
+}
+
+/** Build provider-aware options for pi-ai completeSimple. */
+export function buildCompleteSimpleOptions(params: {
+  api: string | undefined;
+  apiKey: string | undefined;
+  maxTokens: number;
+  temperature: number | undefined;
+  reasoning: string | undefined;
+}): CompleteSimpleOptions {
+  const options: CompleteSimpleOptions = {
+    apiKey: params.apiKey,
+    maxTokens: params.maxTokens,
+  };
+
+  if (
+    typeof params.temperature === "number" &&
+    Number.isFinite(params.temperature) &&
+    !shouldOmitTemperatureForApi(params.api)
+  ) {
+    options.temperature = params.temperature;
+  }
+
+  if (typeof params.reasoning === "string" && params.reasoning.trim()) {
+    options.reasoning = params.reasoning.trim();
+  }
+
+  return options;
+}
+
 /** Select provider-specific config values with case-insensitive provider keys. */
 function findProviderConfigValue<T>(
   map: Record<string, T> | undefined,
@@ -566,8 +610,10 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
       agentDir,
       runtimeConfig,
       messages,
+      system,
       maxTokens,
       temperature,
+      reasoning,
     }) => {
       try {
         const piAiModuleId = "@mariozechner/pi-ai";
@@ -644,24 +690,62 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
           });
         }
 
+        const completeOptions = buildCompleteSimpleOptions({
+          api: resolvedModel.api,
+          apiKey: resolvedApiKey,
+          maxTokens,
+          temperature,
+          reasoning,
+        });
+
         const result = await mod.completeSimple(
           resolvedModel,
           {
+            ...(typeof system === "string" && system.trim()
+              ? { systemPrompt: system.trim() }
+              : {}),
             messages: messages.map((message) => ({
               role: message.role,
               content: message.content,
               timestamp: Date.now(),
             })),
           },
-          {
-            apiKey: resolvedApiKey,
-            maxTokens,
-            temperature,
-          },
+          completeOptions,
         );
 
+        if (!isRecord(result)) {
+          return {
+            content: [],
+            request_provider: providerId,
+            request_model: modelId,
+            request_api: resolvedModel.api,
+            request_reasoning:
+              typeof reasoning === "string" && reasoning.trim() ? reasoning.trim() : "(none)",
+            request_has_system:
+              typeof system === "string" && system.trim().length > 0 ? "true" : "false",
+            request_temperature:
+              typeof completeOptions.temperature === "number"
+                ? String(completeOptions.temperature)
+                : "(omitted)",
+            request_temperature_sent:
+              typeof completeOptions.temperature === "number" ? "true" : "false",
+          };
+        }
+
         return {
-          content: Array.isArray(result?.content) ? result.content : [],
+          ...result,
+          content: Array.isArray(result.content) ? result.content : [],
+          request_provider: providerId,
+          request_model: modelId,
+          request_api: resolvedModel.api,
+          request_reasoning:
+            typeof reasoning === "string" && reasoning.trim() ? reasoning.trim() : "(none)",
+          request_has_system: typeof system === "string" && system.trim().length > 0 ? "true" : "false",
+          request_temperature:
+            typeof completeOptions.temperature === "number"
+              ? String(completeOptions.temperature)
+              : "(omitted)",
+          request_temperature_sent: typeof completeOptions.temperature === "number" ? "true" : "false",
         };
       } catch (err) {
         console.error(`[lcm] completeSimple error:`, err instanceof Error ? err.message : err);
@@ -715,8 +799,8 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
       }
 
       const provider = (
-        providerHint?.trim() ||
         envSnapshot.lcmSummaryProvider ||
+        providerHint?.trim() ||
         envSnapshot.openclawProvider ||
         "openai"
       ).trim();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@martian-engineering/lossless-claw",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Lossless Context Management plugin for OpenClaw — DAG-based conversation summarization with incremental compaction",
   "type": "module",
   "main": "index.ts",

package/src/compaction.ts

@@ -747,6 +747,18 @@ export class CompactionEngine {
     return estimateTokens(summary.content);
   }
 
+  /** Resolve message token count with content-length fallback. */
+  private resolveMessageTokenCount(message: { tokenCount: number; content: string }): number {
+    if (
+      typeof message.tokenCount === "number" &&
+      Number.isFinite(message.tokenCount) &&
+      message.tokenCount > 0
+    ) {
+      return message.tokenCount;
+    }
+    return estimateTokens(message.content);
+  }
+
   private resolveLeafMinFanout(): number {
     if (
       typeof this.config.leafMinFanout === "number" &&
@@ -995,7 +1007,8 @@ export class CompactionEngine {
     previousSummaryContent?: string,
   ): Promise<{ summaryId: string; level: CompactionLevel; content: string }> {
     // Fetch full message content for each context item
-    const messageContents: { messageId: number; content: string; createdAt: Date }[] = [];
+    const messageContents: { messageId: number; content: string; createdAt: Date; tokenCount: number }[] =
+      [];
     for (const item of messageItems) {
       if (item.messageId == null) {
         continue;
@@ -1006,6 +1019,7 @@ export class CompactionEngine {
           messageId: msg.messageId,
           content: msg.content,
           createdAt: msg.createdAt,
+          tokenCount: this.resolveMessageTokenCount(msg),
         });
       }
     }
@@ -1046,6 +1060,11 @@ export class CompactionEngine {
           ? new Date(Math.max(...messageContents.map((message) => message.createdAt.getTime())))
           : undefined,
       descendantCount: 0,
+      descendantTokenCount: 0,
+      sourceMessageTokenCount: messageContents.reduce(
+        (sum, message) => sum + Math.max(0, Math.floor(message.tokenCount)),
+        0,
+      ),
     });
 
     // Link to source messages
@@ -1156,6 +1175,22 @@ export class CompactionEngine {
             : 0;
         return count + childDescendants + 1;
       }, 0),
+      descendantTokenCount: summaryRecords.reduce((count, summary) => {
+        const childDescendantTokens =
+          typeof summary.descendantTokenCount === "number" &&
+          Number.isFinite(summary.descendantTokenCount)
+            ? Math.max(0, Math.floor(summary.descendantTokenCount))
+            : 0;
+        return count + Math.max(0, Math.floor(summary.tokenCount)) + childDescendantTokens;
+      }, 0),
+      sourceMessageTokenCount: summaryRecords.reduce((count, summary) => {
+        const sourceTokens =
+          typeof summary.sourceMessageTokenCount === "number" &&
+          Number.isFinite(summary.sourceMessageTokenCount)
+            ? Math.max(0, Math.floor(summary.sourceMessageTokenCount))
+            : 0;
+        return count + sourceTokens;
+      }, 0),
     });
 
     // Link to parent summaries

package/src/db/migration.ts

@@ -9,6 +9,7 @@ type SummaryDepthRow = {
   conversation_id: number;
   kind: "leaf" | "condensed";
   depth: number;
+  token_count: number;
   created_at: string;
 };
 
@@ -16,6 +17,7 @@ type SummaryMessageTimeRangeRow = {
   summary_id: string;
   earliest_at: string | null;
   latest_at: string | null;
+  source_message_token_count: number | null;
 };
 
 type SummaryParentEdgeRow = {
@@ -36,6 +38,10 @@ function ensureSummaryMetadataColumns(db: DatabaseSync): void {
   const hasEarliestAt = summaryColumns.some((col) => col.name === "earliest_at");
   const hasLatestAt = summaryColumns.some((col) => col.name === "latest_at");
   const hasDescendantCount = summaryColumns.some((col) => col.name === "descendant_count");
+  const hasDescendantTokenCount = summaryColumns.some((col) => col.name === "descendant_token_count");
+  const hasSourceMessageTokenCount = summaryColumns.some(
+    (col) => col.name === "source_message_token_count",
+  );
 
   if (!hasEarliestAt) {
     db.exec(`ALTER TABLE summaries ADD COLUMN earliest_at TEXT`);
@@ -46,6 +52,12 @@ function ensureSummaryMetadataColumns(db: DatabaseSync): void {
   if (!hasDescendantCount) {
     db.exec(`ALTER TABLE summaries ADD COLUMN descendant_count INTEGER NOT NULL DEFAULT 0`);
   }
+  if (!hasDescendantTokenCount) {
+    db.exec(`ALTER TABLE summaries ADD COLUMN descendant_token_count INTEGER NOT NULL DEFAULT 0`);
+  }
+  if (!hasSourceMessageTokenCount) {
+    db.exec(`ALTER TABLE summaries ADD COLUMN source_message_token_count INTEGER NOT NULL DEFAULT 0`);
+  }
 }
 
 function parseTimestamp(value: string | null | undefined): Date | null {
@@ -84,7 +96,7 @@ function backfillSummaryDepths(db: DatabaseSync): void {
     const conversationId = row.conversation_id;
     const summaries = db
       .prepare(
-        `SELECT summary_id, conversation_id, kind, depth, created_at
+        `SELECT summary_id, conversation_id, kind, depth, token_count, created_at
          FROM summaries
          WHERE conversation_id = ?`,
       )
@@ -180,7 +192,8 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
 
   const updateMetadataStmt = db.prepare(
     `UPDATE summaries
-     SET earliest_at = ?, latest_at = ?, descendant_count = ?
+     SET earliest_at = ?, latest_at = ?, descendant_count = ?,
+         descendant_token_count = ?, source_message_token_count = ?
      WHERE summary_id = ?`,
   );
 
@@ -188,7 +201,7 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
     const conversationId = conversationRow.conversation_id;
     const summaries = db
       .prepare(
-        `SELECT summary_id, conversation_id, kind, depth, created_at
+        `SELECT summary_id, conversation_id, kind, depth, token_count, created_at
          FROM summaries
          WHERE conversation_id = ?
          ORDER BY depth ASC, created_at ASC`,
@@ -200,7 +213,11 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
 
     const leafRanges = db
       .prepare(
-        `SELECT sm.summary_id, MIN(m.created_at) AS earliest_at, MAX(m.created_at) AS latest_at
+        `SELECT
+           sm.summary_id,
+           MIN(m.created_at) AS earliest_at,
+           MAX(m.created_at) AS latest_at,
+           COALESCE(SUM(m.token_count), 0) AS source_message_token_count
          FROM summary_messages sm
          JOIN messages m ON m.message_id = sm.message_id
          JOIN summaries s ON s.summary_id = sm.summary_id
@@ -209,7 +226,14 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
       )
       .all(conversationId) as SummaryMessageTimeRangeRow[];
     const leafRangeBySummaryId = new Map(
-      leafRanges.map((row) => [row.summary_id, { earliestAt: row.earliest_at, latestAt: row.latest_at }]),
+      leafRanges.map((row) => [
+        row.summary_id,
+        {
+          earliestAt: row.earliest_at,
+          latestAt: row.latest_at,
+          sourceMessageTokenCount: row.source_message_token_count,
+        },
+      ]),
     );
 
     const edges = db
@@ -230,8 +254,17 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
 
     const metadataBySummaryId = new Map<
       string,
-      { earliestAt: Date | null; latestAt: Date | null; descendantCount: number }
+      {
+        earliestAt: Date | null;
+        latestAt: Date | null;
+        descendantCount: number;
+        descendantTokenCount: number;
+        sourceMessageTokenCount: number;
+      }
     >();
+    const tokenCountBySummaryId = new Map(
+      summaries.map((summary) => [summary.summary_id, Math.max(0, Math.floor(summary.token_count ?? 0))]),
+    );
 
     for (const summary of summaries) {
       const fallbackDate = parseTimestamp(summary.created_at);
@@ -244,6 +277,11 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
           earliestAt,
           latestAt,
           descendantCount: 0,
+          descendantTokenCount: 0,
+          sourceMessageTokenCount: Math.max(
+            0,
+            Math.floor(range?.sourceMessageTokenCount ?? 0),
+          ),
         });
         continue;
       }
@@ -254,6 +292,8 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
           earliestAt: fallbackDate,
           latestAt: fallbackDate,
           descendantCount: 0,
+          descendantTokenCount: 0,
+          sourceMessageTokenCount: 0,
         });
         continue;
       }
@@ -261,6 +301,8 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
       let earliestAt: Date | null = null;
       let latestAt: Date | null = null;
       let descendantCount = 0;
+      let descendantTokenCount = 0;
+      let sourceMessageTokenCount = 0;
 
       for (const parentId of parentIds) {
         const parentMetadata = metadataBySummaryId.get(parentId);
@@ -279,12 +321,18 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
         }
 
         descendantCount += Math.max(0, parentMetadata.descendantCount) + 1;
+        const parentTokenCount = tokenCountBySummaryId.get(parentId) ?? 0;
+        descendantTokenCount +=
+          Math.max(0, parentTokenCount) + Math.max(0, parentMetadata.descendantTokenCount);
+        sourceMessageTokenCount += Math.max(0, parentMetadata.sourceMessageTokenCount);
       }
 
       metadataBySummaryId.set(summary.summary_id, {
         earliestAt: earliestAt ?? fallbackDate,
         latestAt: latestAt ?? fallbackDate,
         descendantCount: Math.max(0, descendantCount),
+        descendantTokenCount: Math.max(0, descendantTokenCount),
+        sourceMessageTokenCount: Math.max(0, sourceMessageTokenCount),
       });
     }
 
@@ -298,6 +346,8 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
         isoStringOrNull(metadata.earliestAt),
         isoStringOrNull(metadata.latestAt),
         Math.max(0, metadata.descendantCount),
+        Math.max(0, metadata.descendantTokenCount),
+        Math.max(0, metadata.sourceMessageTokenCount),
         summary.summary_id,
       );
     }
@@ -336,6 +386,8 @@ export function runLcmMigrations(db: DatabaseSync): void {
       earliest_at TEXT,
       latest_at TEXT,
       descendant_count INTEGER NOT NULL DEFAULT 0,
+      descendant_token_count INTEGER NOT NULL DEFAULT 0,
+      source_message_token_count INTEGER NOT NULL DEFAULT 0,
       created_at TEXT NOT NULL DEFAULT (datetime('now')),
       file_ids TEXT NOT NULL DEFAULT '[]'
     );

package/src/expansion-auth.ts

@@ -56,6 +56,7 @@ const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes
 
 export class ExpansionAuthManager {
   private grants: Map<string, ExpansionGrant> = new Map();
+  private consumedTokensByGrantId: Map<string, number> = new Map();
 
   /**
    * Create a new expansion grant with the given parameters.
@@ -79,6 +80,7 @@ export class ExpansionAuthManager {
     };
 
     this.grants.set(grantId, grant);
+    this.consumedTokensByGrantId.set(grantId, 0);
     return grant;
   }
 
@@ -113,6 +115,36 @@ export class ExpansionAuthManager {
     return true;
   }
 
+  /**
+   * Resolve remaining token budget for an active grant.
+   */
+  getRemainingTokenBudget(grantId: string): number | null {
+    const grant = this.getGrant(grantId);
+    if (!grant) {
+      return null;
+    }
+    const consumed = Math.max(0, this.consumedTokensByGrantId.get(grantId) ?? 0);
+    return Math.max(0, Math.floor(grant.tokenCap) - consumed);
+  }
+
+  /**
+   * Consume token budget for a grant, clamped to the grant token cap.
+   */
+  consumeTokenBudget(grantId: string, consumedTokens: number): number | null {
+    const grant = this.getGrant(grantId);
+    if (!grant) {
+      return null;
+    }
+    const safeConsumed =
+      typeof consumedTokens === "number" && Number.isFinite(consumedTokens)
+        ? Math.max(0, Math.floor(consumedTokens))
+        : 0;
+    const previous = Math.max(0, this.consumedTokensByGrantId.get(grantId) ?? 0);
+    const next = Math.min(Math.max(1, Math.floor(grant.tokenCap)), previous + safeConsumed);
+    this.consumedTokensByGrantId.set(grantId, next);
+    return Math.max(0, Math.floor(grant.tokenCap) - next);
+  }
+
   /**
    * Validate an expansion request against a grant.
    * Checks existence, expiry, revocation, conversation scope, and summary scope.
@@ -177,6 +209,7 @@ export class ExpansionAuthManager {
     for (const [grantId, grant] of this.grants) {
       if (grant.revoked || grant.expiresAt.getTime() <= now) {
         this.grants.delete(grantId);
+        this.consumedTokensByGrantId.delete(grantId);
         removed++;
       }
     }
@@ -293,7 +326,26 @@ export function wrapWithAuth(
       if (!validation.valid) {
         throw new Error(`Expansion authorization failed: ${validation.reason}`);
       }
-      return orchestrator.expand(request);
+
+      const remainingBudget = authManager.getRemainingTokenBudget(grantId);
+      if (remainingBudget == null) {
+        throw new Error("Expansion authorization failed: Grant not found");
+      }
+      if (remainingBudget <= 0) {
+        throw new Error("Expansion authorization failed: Grant token budget exhausted");
+      }
+
+      const requestedTokenCap =
+        typeof request.tokenCap === "number" && Number.isFinite(request.tokenCap)
+          ? Math.max(1, Math.trunc(request.tokenCap))
+          : remainingBudget;
+      const effectiveTokenCap = Math.max(1, Math.min(requestedTokenCap, remainingBudget));
+      const result = await orchestrator.expand({
+        ...request,
+        tokenCap: effectiveTokenCap,
+      });
+      authManager.consumeTokenBudget(grantId, result.totalTokens);
+      return result;
     },
   };
 }

package/src/retrieval.ts

@@ -20,11 +20,32 @@ export interface DescribeResult {
     conversationId: number;
     kind: "leaf" | "condensed";
     content: string;
+    depth: number;
     tokenCount: number;
+    descendantCount: number;
+    descendantTokenCount: number;
+    sourceMessageTokenCount: number;
     fileIds: string[];
     parentIds: string[];
     childIds: string[];
     messageIds: number[];
+    earliestAt: Date | null;
+    latestAt: Date | null;
+    subtree: Array<{
+      summaryId: string;
+      parentSummaryId: string | null;
+      depthFromRoot: number;
+      kind: "leaf" | "condensed";
+      depth: number;
+      tokenCount: number;
+      descendantCount: number;
+      descendantTokenCount: number;
+      sourceMessageTokenCount: number;
+      earliestAt: Date | null;
+      latestAt: Date | null;
+      childCount: number;
+      path: string;
+    }>;
     createdAt: Date;
   };
   /** File-specific fields */
@@ -127,10 +148,11 @@ export class RetrievalEngine {
     }
 
     // Fetch lineage in parallel
-    const [parents, children, messageIds] = await Promise.all([
+    const [parents, children, messageIds, subtree] = await Promise.all([
       this.summaryStore.getSummaryParents(id),
       this.summaryStore.getSummaryChildren(id),
       this.summaryStore.getSummaryMessages(id),
+      this.summaryStore.getSummarySubtree(id),
     ]);
 
     return {
@@ -140,11 +162,32 @@ export class RetrievalEngine {
         conversationId: summary.conversationId,
         kind: summary.kind,
         content: summary.content,
+        depth: summary.depth,
         tokenCount: summary.tokenCount,
+        descendantCount: summary.descendantCount,
+        descendantTokenCount: summary.descendantTokenCount,
+        sourceMessageTokenCount: summary.sourceMessageTokenCount,
         fileIds: summary.fileIds,
         parentIds: parents.map((p) => p.summaryId),
         childIds: children.map((c) => c.summaryId),
         messageIds,
+        earliestAt: summary.earliestAt,
+        latestAt: summary.latestAt,
+        subtree: subtree.map((node) => ({
+          summaryId: node.summaryId,
+          parentSummaryId: node.parentSummaryId,
+          depthFromRoot: node.depthFromRoot,
+          kind: node.kind,
+          depth: node.depth,
+          tokenCount: node.tokenCount,
+          descendantCount: node.descendantCount,
+          descendantTokenCount: node.descendantTokenCount,
+          sourceMessageTokenCount: node.sourceMessageTokenCount,
+          earliestAt: node.earliestAt,
+          latestAt: node.latestAt,
+          childCount: node.childCount,
+          path: node.path,
+        })),
         createdAt: summary.createdAt,
       },
     };