@martian-engineering/lossless-claw 0.5.2 → 0.5.3

package/src/compaction.ts

@@ -38,7 +38,7 @@ export interface CompactionConfig {
   condensedMinFanout: number;
   /** Relaxed minimum fanout for hard-trigger sweeps. */
   condensedMinFanoutHard: number;
-  /** Incremental depth passes to run after each leaf compaction (default 0). */
+  /** Incremental depth passes to run after each leaf compaction (default 1). */
   incrementalMaxDepth: number;
   /** Max source tokens to compact per leaf/condensed chunk (default 20000) */
   leafChunkTokens?: number;
@@ -50,9 +50,11 @@ export interface CompactionConfig {
   maxRounds: number;
   /** IANA timezone for timestamps in summaries (default: UTC) */
   timezone?: string;
+  /** Maximum allowed overage factor for summaries relative to target tokens (default 3). */
+  summaryMaxOverageFactor: number;
 }
 
-type CompactionLevel = "normal" | "aggressive" | "fallback";
+type CompactionLevel = "normal" | "aggressive" | "fallback" | "capped";
 type CompactionPass = "leaf" | "condensed";
 type CompactionSummarizeOptions = {
   previousSummary?: string;
@@ -86,6 +88,30 @@ function estimateTokens(content: string): number {
   return Math.ceil(content.length / 4);
 }
 
+/** Deterministically cap summary text so the persisted output stays within maxTokens. */
+function capSummaryText(
+  content: string,
+  originalTokens: number,
+  maxTokens: number,
+): string {
+  const suffixes = [
+    `\n[Capped from ${originalTokens} tokens to ~${maxTokens}]`,
+    `\n[Capped to ~${maxTokens}]`,
+    "\n[Capped]",
+    "",
+  ];
+
+  for (const suffix of suffixes) {
+    const maxChars = Math.max(0, maxTokens * 4 - suffix.length);
+    const capped = `${content.slice(0, maxChars)}${suffix}`;
+    if (estimateTokens(capped) <= maxTokens) {
+      return capped;
+    }
+  }
+
+  return content.slice(0, Math.max(0, maxTokens * 4));
+}
+
 /** Format a timestamp as `YYYY-MM-DD HH:mm TZ` for prompt source text. */
 export function formatTimestamp(value: Date, timezone: string = "UTC"): string {
   try {
@@ -150,6 +176,11 @@ const DEFAULT_LEAF_CHUNK_TOKENS = 20_000;
  * with no meaningful text.
  */
 const MEDIA_PATH_RE = /^MEDIA:\/.+$/;
+const EMBEDDED_DATA_URL_RE = /data:[^;\s"'`]+;base64,[A-Za-z0-9+/=\s]+/gi;
+const MEDIA_ATTACHMENT_PART_TYPES = new Set(["file", "snapshot"]);
+const MEDIA_ATTACHMENT_RAW_TYPES = new Set(["file", "image", "snapshot"]);
+const STRUCTURED_MEDIA_TEXT_KEYS = ["text", "caption", "alt", "title", "summary"] as const;
+const STRUCTURED_MEDIA_NESTED_KEYS = ["content", "parts", "items", "message", "messages"] as const;
 
 const CONDENSED_MIN_INPUT_RATIO = 0.1;
 
@@ -165,6 +196,140 @@ function dedupeOrderedIds(ids: Iterable<string>): string[] {
   return ordered;
 }
 
+/** Parse message-part metadata without throwing on malformed JSON. */
+function parseMessagePartMetadata(part: CreateMessagePartInput | { metadata: string | null }): Record<string, unknown> {
+  if (typeof part.metadata !== "string" || !part.metadata.trim()) {
+    return {};
+  }
+  try {
+    const parsed = JSON.parse(part.metadata) as unknown;
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+      ? (parsed as Record<string, unknown>)
+      : {};
+  } catch {
+    return {};
+  }
+}
+
+/** Detect whether a string is mostly binary/base64 payload and not meaningful prose. */
+function looksLikeBinaryPayload(value: string): boolean {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    return false;
+  }
+  if (/^data:[^;\s"'`]+;base64,/i.test(trimmed)) {
+    return true;
+  }
+  const compact = trimmed.replace(/\s+/g, "");
+  if (compact.length < 256 || compact.length % 4 !== 0) {
+    return false;
+  }
+  if (!/^[A-Za-z0-9+/=]+$/.test(compact)) {
+    return false;
+  }
+  return !/[ .,:;!?()[\]{}]/.test(trimmed);
+}
+
+/** Strip attachment payloads from plain strings before they reach the summarizer. */
+function stripEmbeddedMediaPayloads(content: string): string {
+  const withoutDataUrls = content.replace(EMBEDDED_DATA_URL_RE, "[embedded media omitted]");
+  const sanitizedLines = withoutDataUrls
+    .split(/\r?\n/)
+    .map((line) => line.trimEnd())
+    .filter((line) => {
+      const trimmed = line.trim();
+      if (!trimmed) {
+        return false;
+      }
+      if (MEDIA_PATH_RE.test(trimmed)) {
+        return false;
+      }
+      if (looksLikeBinaryPayload(trimmed)) {
+        return false;
+      }
+      return true;
+    });
+  return sanitizedLines.join("\n").trim();
+}
+
+/** Extract human-readable text from structured content while ignoring attachment payload fields. */
+function extractSanitizedStructuredText(value: unknown, depth = 0): string[] {
+  if (depth >= 4 || value == null) {
+    return [];
+  }
+  if (typeof value === "string") {
+    const sanitized = stripEmbeddedMediaPayloads(value);
+    return sanitized ? [sanitized] : [];
+  }
+  if (Array.isArray(value)) {
+    return value.flatMap((entry) => extractSanitizedStructuredText(entry, depth + 1));
+  }
+  if (typeof value !== "object") {
+    return [];
+  }
+
+  const record = value as Record<string, unknown>;
+  const rawType = typeof record.type === "string" ? record.type.trim().toLowerCase() : "";
+  const textFragments: string[] = [];
+
+  for (const key of STRUCTURED_MEDIA_TEXT_KEYS) {
+    const candidate = record[key];
+    if (typeof candidate !== "string") {
+      continue;
+    }
+    const sanitized = stripEmbeddedMediaPayloads(candidate);
+    if (sanitized) {
+      textFragments.push(sanitized);
+    }
+  }
+
+  if (MEDIA_ATTACHMENT_RAW_TYPES.has(rawType)) {
+    return textFragments;
+  }
+
+  for (const key of STRUCTURED_MEDIA_NESTED_KEYS) {
+    textFragments.push(...extractSanitizedStructuredText(record[key], depth + 1));
+  }
+
+  return textFragments;
+}
+
+/** Normalize message content down to human-readable text, excluding binary/media payloads. */
+function extractMeaningfulMessageText(content: string): string {
+  const trimmed = content.trim();
+  if (!trimmed) {
+    return "";
+  }
+  if ((trimmed.startsWith("[") && trimmed.endsWith("]")) || (trimmed.startsWith("{") && trimmed.endsWith("}"))) {
+    try {
+      const parsed = JSON.parse(trimmed) as unknown;
+      const extracted = extractSanitizedStructuredText(parsed)
+        .map((fragment) => fragment.trim())
+        .filter(Boolean);
+      return extracted.join("\n").trim();
+    } catch {
+      // Fall back to plain-text sanitation below.
+    }
+  }
+  return stripEmbeddedMediaPayloads(content);
+}
+
+/** Identify whether a stored message part represents a media attachment. */
+function isMediaAttachmentPart(part: CreateMessagePartInput | { partType: string; metadata: string | null }): boolean {
+  if (MEDIA_ATTACHMENT_PART_TYPES.has(part.partType)) {
+    return true;
+  }
+  const metadata = parseMessagePartMetadata(part);
+  const rawType =
+    typeof metadata.rawType === "string"
+      ? metadata.rawType.trim().toLowerCase()
+      : metadata.raw && typeof metadata.raw === "object" && !Array.isArray(metadata.raw) &&
+          typeof (metadata.raw as Record<string, unknown>).type === "string"
+        ? ((metadata.raw as Record<string, unknown>).type as string).trim().toLowerCase()
+        : "";
+  return MEDIA_ATTACHMENT_RAW_TYPES.has(rawType);
+}
+
 // ── CompactionEngine ─────────────────────────────────────────────────────────
 
 export class CompactionEngine {
@@ -1010,6 +1175,8 @@ export class CompactionEngine {
     sourceText: string;
     summarize: CompactionSummarizeFn;
     options?: CompactionSummarizeOptions;
+    /** Target token count for this summary kind (leaf or condensed). Used for hard-cap enforcement. */
+    targetTokens: number;
   }): Promise<{ content: string; level: CompactionLevel } | null> {
     const sourceText = params.sourceText.trim();
     if (!sourceText) {
@@ -1076,6 +1243,21 @@ export class CompactionEngine {
       }
     }
 
+    // Hard cap: enforce maximum summary size relative to the kind-appropriate target.
+    const summaryTokens = estimateTokens(summaryText);
+    const maxTokens = Math.ceil(params.targetTokens * this.config.summaryMaxOverageFactor);
+
+    if (summaryTokens > Math.ceil(params.targetTokens * 1.5)) {
+      console.warn(
+        `[lcm] summary exceeds target by ${Math.round((summaryTokens / params.targetTokens - 1) * 100)}%: ${summaryTokens} tokens vs target ${params.targetTokens}`,
+      );
+    }
+
+    if (summaryTokens > maxTokens) {
+      summaryText = capSummaryText(summaryText, summaryTokens, maxTokens);
+      level = "capped";
+    }
+
     return { content: summaryText, level };
   }
 
@@ -1086,10 +1268,9 @@ export class CompactionEngine {
    * attachments. This gives the summarizer enough context to produce a
    * meaningful summary instead of trying to compress raw file paths.
    *
-   * - Media-only messages (just a file path, no text): content is replaced
-   *   with "[Media attachment]" or "[Image attachment]" etc.
-   * - Media-mostly messages (any real text + attachment): content is annotated
-   *   with " [with media attachment]" suffix.
+   * - Media-only messages: content is replaced with "[Media attachment]".
+   * - Media-mostly messages: text is preserved and annotated with
+   *   " [with media attachment]".
    * - Text-only messages: returned unchanged.
    */
   private async annotateMediaContent(
@@ -1097,27 +1278,29 @@ export class CompactionEngine {
     content: string,
   ): Promise<string> {
     const parts = await this.conversationStore.getMessageParts(messageId);
-    const hasMediaParts = parts.some(
-      (p) => p.partType === "file" || p.partType === "snapshot",
-    );
+    const hasMediaParts = parts.some((part) => isMediaAttachmentPart(part));
     if (!hasMediaParts) {
       return content;
     }
 
-    // Strip MEDIA:/... paths to see how much actual text remains
-    const textWithoutPaths = content
-      .split("\n")
-      .filter((line) => !MEDIA_PATH_RE.test(line.trim()))
+    const partText = parts
+      .filter((part) => !isMediaAttachmentPart(part))
+      .map((part) => (typeof part.textContent === "string" ? part.textContent : ""))
+      .map((text) => stripEmbeddedMediaPayloads(text))
+      .map((text) => text.trim())
+      .filter(Boolean)
       .join("\n")
       .trim();
+    const fallbackText = extractMeaningfulMessageText(content);
+    const meaningfulText = (partText || fallbackText).trim();
 
-    if (textWithoutPaths.length === 0) {
-      // Media-only: replace with descriptive annotation
+    if (!meaningfulText) {
       return "[Media attachment]";
     }
-
-    // Media-mostly: keep the text, add annotation
-    return `${textWithoutPaths} [with media attachment]`;
+    if (meaningfulText.includes("[with media attachment]")) {
+      return meaningfulText;
+    }
+    return `${meaningfulText} [with media attachment]`;
   }
 
   // ── Private: Leaf Pass ───────────────────────────────────────────────────
@@ -1167,6 +1350,7 @@ export class CompactionEngine {
         previousSummary: previousSummaryContent,
         isCondensed: false,
       },
+      targetTokens: this.config.leafTargetTokens,
     });
     if (!summary) {
       console.warn(
@@ -1274,6 +1458,7 @@ export class CompactionEngine {
         isCondensed: true,
         depth: targetDepth + 1,
       },
+      targetTokens: this.config.condensedTargetTokens,
     });
     if (!condensed) {
       console.warn(

package/src/db/config.ts

@@ -37,11 +37,19 @@ export type LcmConfig = {
   expansionProvider: string;
   /** Model override for lcm_expand_query sub-agent. */
   expansionModel: string;
+  /** Max time to wait for delegated lcm_expand_query sub-agent completion. */
+  delegationTimeoutMs: number;
   autocompactDisabled: boolean;
   /** IANA timezone for timestamps in summaries (from TZ env or system default) */
   timezone: string;
   /** When true, retroactively delete HEARTBEAT_OK turn cycles from LCM storage. */
   pruneHeartbeatOk: boolean;
+  /** Hard ceiling for assembly token budget — caps runtime-provided and fallback budgets. */
+  maxAssemblyTokenBudget?: number;
+  /** Maximum allowed overage factor for summaries relative to target tokens (default 3). */
+  summaryMaxOverageFactor: number;
+  /** Custom instructions injected into all summarization prompts. */
+  customInstructions: string;
 };
 
 /** Safely coerce an unknown value to a finite number, or return undefined. */
@@ -100,6 +108,10 @@ export function resolveLcmConfig(
   pluginConfig?: Record<string, unknown>,
 ): LcmConfig {
   const pc = pluginConfig ?? {};
+  const envDelegationTimeoutMs =
+    env.LCM_DELEGATION_TIMEOUT_MS !== undefined
+      ? toNumber(env.LCM_DELEGATION_TIMEOUT_MS)
+      : undefined;
 
   return {
     enabled:
@@ -134,7 +146,7 @@ export function resolveLcmConfig(
         ?? toNumber(pc.contextThreshold) ?? 0.75,
     freshTailCount:
       (env.LCM_FRESH_TAIL_COUNT !== undefined ? parseInt(env.LCM_FRESH_TAIL_COUNT, 10) : undefined)
-        ?? toNumber(pc.freshTailCount) ?? 32,
+        ?? toNumber(pc.freshTailCount) ?? 64,
     leafMinFanout:
       (env.LCM_LEAF_MIN_FANOUT !== undefined ? parseInt(env.LCM_LEAF_MIN_FANOUT, 10) : undefined)
         ?? toNumber(pc.leafMinFanout) ?? 8,
@@ -146,13 +158,13 @@ export function resolveLcmConfig(
         ?? toNumber(pc.condensedMinFanoutHard) ?? 2,
     incrementalMaxDepth:
       (env.LCM_INCREMENTAL_MAX_DEPTH !== undefined ? parseInt(env.LCM_INCREMENTAL_MAX_DEPTH, 10) : undefined)
-        ?? toNumber(pc.incrementalMaxDepth) ?? 0,
+        ?? toNumber(pc.incrementalMaxDepth) ?? 1,
     leafChunkTokens:
       (env.LCM_LEAF_CHUNK_TOKENS !== undefined ? parseInt(env.LCM_LEAF_CHUNK_TOKENS, 10) : undefined)
         ?? toNumber(pc.leafChunkTokens) ?? 20000,
     leafTargetTokens:
       (env.LCM_LEAF_TARGET_TOKENS !== undefined ? parseInt(env.LCM_LEAF_TARGET_TOKENS, 10) : undefined)
-        ?? toNumber(pc.leafTargetTokens) ?? 1200,
+        ?? toNumber(pc.leafTargetTokens) ?? 2400,
     condensedTargetTokens:
       (env.LCM_CONDENSED_TARGET_TOKENS !== undefined ? parseInt(env.LCM_CONDENSED_TARGET_TOKENS, 10) : undefined)
         ?? toNumber(pc.condensedTargetTokens) ?? 2000,
@@ -176,6 +188,7 @@ export function resolveLcmConfig(
       env.LCM_EXPANSION_PROVIDER?.trim() ?? toStr(pc.expansionProvider) ?? "",
     expansionModel:
       env.LCM_EXPANSION_MODEL?.trim() ?? toStr(pc.expansionModel) ?? "",
+    delegationTimeoutMs: envDelegationTimeoutMs ?? toNumber(pc.delegationTimeoutMs) ?? 120000,
     autocompactDisabled:
       env.LCM_AUTOCOMPACT_DISABLED !== undefined
         ? env.LCM_AUTOCOMPACT_DISABLED === "true"
@@ -185,5 +198,13 @@ export function resolveLcmConfig(
       env.LCM_PRUNE_HEARTBEAT_OK !== undefined
         ? env.LCM_PRUNE_HEARTBEAT_OK === "true"
         : toBool(pc.pruneHeartbeatOk) ?? false,
+    maxAssemblyTokenBudget:
+      (env.LCM_MAX_ASSEMBLY_TOKEN_BUDGET !== undefined ? parseInt(env.LCM_MAX_ASSEMBLY_TOKEN_BUDGET, 10) : undefined)
+        ?? toNumber(pc.maxAssemblyTokenBudget) ?? undefined,
+    summaryMaxOverageFactor:
+      (env.LCM_SUMMARY_MAX_OVERAGE_FACTOR !== undefined ? parseFloat(env.LCM_SUMMARY_MAX_OVERAGE_FACTOR) : undefined)
+        ?? toNumber(pc.summaryMaxOverageFactor) ?? 3,
+    customInstructions:
+      env.LCM_CUSTOM_INSTRUCTIONS?.trim() ?? toStr(pc.customInstructions) ?? "",
   };
 }

package/src/engine.ts

@@ -1064,6 +1064,7 @@ export class LcmContextEngine implements ContextEngine {
       condensedTargetTokens: this.config.condensedTargetTokens,
       maxRounds: 10,
       timezone: this.config.timezone,
+      summaryMaxOverageFactor: this.config.summaryMaxOverageFactor,
     };
     this.compaction = new CompactionEngine(
       this.conversationStore,
@@ -1189,6 +1190,12 @@ export class LcmContextEngine implements ContextEngine {
     return undefined;
   }
 
+  /** Cap a resolved token budget against the configured maxAssemblyTokenBudget. */
+  private applyAssemblyBudgetCap(budget: number): number {
+    const cap = this.config.maxAssemblyTokenBudget;
+    return cap != null && cap > 0 ? Math.min(budget, cap) : budget;
+  }
+
   /** Resolve an LCM conversation id from a session key via the session store. */
   private async resolveConversationIdForSessionKey(
     sessionKey: string,
@@ -1231,10 +1238,14 @@ export class LcmContextEngine implements ContextEngine {
       };
     }
     try {
+      const customInstructions =
+        params.customInstructions !== undefined
+          ? params.customInstructions
+          : (this.config.customInstructions || undefined);
       const runtimeSummarizer = await createLcmSummarizeFromLegacyParams({
         deps: this.deps,
         legacyParams: lp,
-        customInstructions: params.customInstructions,
+        customInstructions,
       });
       if (runtimeSummarizer) {
         return { summarize: runtimeSummarizer.fn, summaryModel: runtimeSummarizer.model };
@@ -1271,6 +1282,7 @@ export class LcmContextEngine implements ContextEngine {
       const result = await createLcmSummarizeFromLegacyParams({
         deps: this.deps,
         legacyParams: { provider, model },
+        customInstructions: this.config.customInstructions || undefined,
       });
       if (!result) {
         return undefined;
@@ -2133,7 +2145,7 @@ export class LcmContextEngine implements ContextEngine {
       runtimeContext: params.runtimeContext,
       legacyParams,
     });
-    const tokenBudget = resolvedTokenBudget ?? DEFAULT_AFTER_TURN_TOKEN_BUDGET;
+    const tokenBudget = this.applyAssemblyBudgetCap(resolvedTokenBudget ?? DEFAULT_AFTER_TURN_TOKEN_BUDGET);
     if (resolvedTokenBudget === undefined) {
       console.warn(
         `[lcm] afterTurn: tokenBudget not provided; using default ${DEFAULT_AFTER_TURN_TOKEN_BUDGET}`,
@@ -2220,12 +2232,13 @@ export class LcmContextEngine implements ContextEngine {
         };
       }
 
-      const tokenBudget =
+      const tokenBudget = this.applyAssemblyBudgetCap(
         typeof params.tokenBudget === "number" &&
         Number.isFinite(params.tokenBudget) &&
         params.tokenBudget > 0
           ? Math.floor(params.tokenBudget)
-          : 128_000;
+          : 128_000,
+      );
 
       const assembled = await this.assembler.assemble({
         conversationId: conversation.conversationId,
@@ -2324,11 +2337,14 @@ export class LcmContextEngine implements ContextEngine {
         }
 
         const legacyParams = asRecord(params.runtimeContext) ?? params.legacyParams;
-        const tokenBudget = this.resolveTokenBudget({
+        const resolvedTokenBudget = this.resolveTokenBudget({
           tokenBudget: params.tokenBudget,
           runtimeContext: params.runtimeContext,
           legacyParams,
         });
+        const tokenBudget = resolvedTokenBudget
+          ? this.applyAssemblyBudgetCap(resolvedTokenBudget)
+          : resolvedTokenBudget;
         if (!tokenBudget) {
           return {
             ok: false,
@@ -2438,11 +2454,14 @@ export class LcmContextEngine implements ContextEngine {
           }
         ).manualCompaction === true;
       const forceCompaction = force || manualCompactionRequested;
-      const tokenBudget = this.resolveTokenBudget({
+      const resolvedTokenBudget = this.resolveTokenBudget({
         tokenBudget: params.tokenBudget,
         runtimeContext: params.runtimeContext,
         legacyParams,
       });
+      const tokenBudget = resolvedTokenBudget
+        ? this.applyAssemblyBudgetCap(resolvedTokenBudget)
+        : resolvedTokenBudget;
       if (!tokenBudget) {
         return {
           ok: false,