@martian-engineering/lossless-claw 0.5.1 → 0.5.3

package/src/tools/lcm-expand-query-tool.ts

@@ -21,7 +21,7 @@ import {
   stampDelegatedExpansionContext,
 } from "./lcm-expansion-recursion-guard.js";
 
-const DELEGATED_WAIT_TIMEOUT_MS = 120_000;
+const DEFAULT_DELEGATED_WAIT_TIMEOUT_MS = 120_000;
 const GATEWAY_TIMEOUT_MS = 10_000;
 const DEFAULT_MAX_ANSWER_TOKENS = 2_000;
 
@@ -75,16 +75,98 @@ type ExpandQueryReply = {
   truncated: boolean;
 };
 
+type ParsedExpandQueryReply =
+  | {
+      ok: true;
+      value: ExpandQueryReply;
+    }
+  | {
+      ok: false;
+      error: string;
+    };
+
 type SummaryCandidate = {
   summaryId: string;
   conversationId: number;
+  requiresMessageExpansion: boolean;
 };
 
+function collectExpansionFailureText(value: unknown, parts: string[], depth = 0): void {
+  if (depth > 3 || value == null) {
+    return;
+  }
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (trimmed) {
+      parts.push(trimmed);
+    }
+    return;
+  }
+  if (typeof value === "number" || typeof value === "boolean") {
+    parts.push(String(value));
+    return;
+  }
+  if (value instanceof Error) {
+    if (value.message.trim()) {
+      parts.push(value.message.trim());
+    }
+    collectExpansionFailureText(value.cause, parts, depth + 1);
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      collectExpansionFailureText(entry, parts, depth + 1);
+    }
+    return;
+  }
+  if (typeof value === "object") {
+    const record = value as Record<string, unknown>;
+    for (const key of ["message", "error", "reason", "details", "response", "cause", "code"]) {
+      collectExpansionFailureText(record[key], parts, depth + 1);
+    }
+  }
+}
+
+function formatExpansionFailure(error: unknown): string {
+  const parts: string[] = [];
+  collectExpansionFailureText(error, parts);
+  const message = parts.join(" ").replace(/\s+/g, " ").trim();
+  if (message) {
+    return message;
+  }
+  if (typeof error === "string" && error.trim()) {
+    return error.trim();
+  }
+  return "Delegated expansion query failed.";
+}
+
+function shouldRetryWithoutOverride(message: string): boolean {
+  const normalized = message.toLowerCase();
+  return [
+    "model.request",
+    "missing scopes",
+    "insufficient scope",
+    "unauthorized",
+    "not authorized",
+    "forbidden",
+    "provider/model overrides are not authorized",
+    "model override is not authorized",
+    "unknown model",
+    "model not found",
+    "invalid model",
+    "not available",
+    "not supported",
+    "401",
+    "403",
+  ].some((signal) => normalized.includes(signal));
+}
+
 /**
  * Build the sub-agent task message for delegated expansion and prompt answering.
  */
 function buildDelegatedExpandQueryTask(params: {
   summaryIds: string[];
+  messageBackedSummaryIds: string[];
   conversationId: number;
   query?: string;
   prompt: string;
@@ -95,6 +177,10 @@ function buildDelegatedExpandQueryTask(params: {
   originSessionKey: string;
 }) {
   const seedSummaryIds = params.summaryIds.length > 0 ? params.summaryIds.join(", ") : "(none)";
+  const messageBackedSummaryIds =
+    params.messageBackedSummaryIds.length > 0
+      ? params.messageBackedSummaryIds.join(", ")
+      : "(none)";
   return [
     "You are an autonomous LCM retrieval navigator. Plan and execute retrieval before answering.",
     "",
@@ -102,6 +188,7 @@ function buildDelegatedExpandQueryTask(params: {
     `Conversation scope: ${params.conversationId}`,
     `Expansion token budget (total across this run): ${params.tokenCap}`,
     `Seed summary IDs: ${seedSummaryIds}`,
+    `Seed summaries requiring raw message expansion: ${messageBackedSummaryIds}`,
     params.query ? `Routing query: ${params.query}` : undefined,
     "",
     "Strategy:",
@@ -109,7 +196,7 @@ function buildDelegatedExpandQueryTask(params: {
     "2. If additional candidates are needed, use `lcm_grep` scoped to summaries.",
     "3. Select branches that fit remaining budget; prefer high-signal paths first.",
     "4. Call `lcm_expand` selectively (do not expand everything blindly).",
-    "5. Keep includeMessages=false by default; use includeMessages=true only for specific leaf evidence.",
+    "5. Keep includeMessages=false by default; use includeMessages=true for the message-backed seed summaries above and any other specific leaf evidence.",
     `6. Stay within ${params.tokenCap} total expansion tokens across all lcm_expand calls.`,
     "",
     "User prompt to answer:",
@@ -141,24 +228,25 @@ function buildDelegatedExpandQueryTask(params: {
   ].join("\n");
 }
 
+function formatInvalidDelegatedReply(reply: string, reason: string): string {
+  const compact = reply.replace(/\s+/g, " ").trim();
+  const snippet = compact.length <= 240 ? compact : `${compact.slice(0, 240)}...`;
+  return `Delegated expansion query returned ${reason}: ${snippet}`;
+}
+
 /**
- * Parse the child reply; accepts plain JSON or fenced JSON.
+ * Parse the child reply; accepts plain JSON or fenced JSON and rejects malformed fallbacks.
  */
 function parseDelegatedExpandQueryReply(
   rawReply: string | undefined,
   fallbackExpandedSummaryCount: number,
-): ExpandQueryReply {
-  const fallback: ExpandQueryReply = {
-    answer: (rawReply ?? "").trim(),
-    citedIds: [],
-    expandedSummaryCount: fallbackExpandedSummaryCount,
-    totalSourceTokens: 0,
-    truncated: false,
-  };
-
+): ParsedExpandQueryReply {
   const reply = rawReply?.trim();
   if (!reply) {
-    return fallback;
+    return {
+      ok: false,
+      error: "Delegated expansion query returned an empty reply.",
+    };
   }
 
   const candidates: string[] = [reply];
@@ -177,6 +265,12 @@ function parseDelegatedExpandQueryReply(
         truncated?: unknown;
       };
       const answer = typeof parsed.answer === "string" ? parsed.answer.trim() : "";
+      if (!answer) {
+        return {
+          ok: false,
+          error: formatInvalidDelegatedReply(reply, 'JSON without a non-empty "answer"'),
+        };
+      }
       const citedIds = normalizeSummaryIds(
         Array.isArray(parsed.citedIds)
           ? parsed.citedIds.filter((value): value is string => typeof value === "string")
@@ -194,18 +288,24 @@ function parseDelegatedExpandQueryReply(
       const truncated = parsed.truncated === true;
 
       return {
-        answer: answer || fallback.answer,
-        citedIds,
-        expandedSummaryCount,
-        totalSourceTokens,
-        truncated,
+        ok: true,
+        value: {
+          answer,
+          citedIds,
+          expandedSummaryCount,
+          totalSourceTokens,
+          truncated,
+        },
       };
     } catch {
       // Try next candidate.
     }
   }
 
-  return fallback;
+  return {
+    ok: false,
+    error: formatInvalidDelegatedReply(reply, "non-JSON output"),
+  };
 }
 
 /**
@@ -246,6 +346,22 @@ function resolveSourceConversationId(params: {
   );
 }
 
+function upsertSummaryCandidate(
+  candidates: Map<string, SummaryCandidate>,
+  candidate: SummaryCandidate,
+): void {
+  const existing = candidates.get(candidate.summaryId);
+  if (!existing) {
+    candidates.set(candidate.summaryId, candidate);
+    return;
+  }
+  candidates.set(candidate.summaryId, {
+    ...existing,
+    requiresMessageExpansion:
+      existing.requiresMessageExpansion || candidate.requiresMessageExpansion,
+  });
+}
+
 /**
  * Resolve summary candidates from explicit IDs and/or query matches.
  */
@@ -263,13 +379,15 @@ async function resolveSummaryCandidates(params: {
     if (!described || described.type !== "summary" || !described.summary) {
       throw new Error(`Summary not found: ${summaryId}`);
     }
-    candidates.set(summaryId, {
+    upsertSummaryCandidate(candidates, {
       summaryId,
       conversationId: described.summary.conversationId,
+      requiresMessageExpansion: false,
     });
   }
 
   if (params.query) {
+    const summaryStore = params.lcm.getSummaryStore();
     const grepResult = await retrieval.grep({
       query: params.query,
       mode: "full_text",
@@ -277,11 +395,46 @@ async function resolveSummaryCandidates(params: {
       conversationId: params.conversationId,
     });
     for (const summary of grepResult.summaries) {
-      candidates.set(summary.summaryId, {
+      upsertSummaryCandidate(candidates, {
         summaryId: summary.summaryId,
         conversationId: summary.conversationId,
+        requiresMessageExpansion: false,
       });
     }
+
+    if (grepResult.summaries.length === 0 && typeof params.conversationId === "number") {
+      const maxDepth = await summaryStore.getConversationMaxSummaryDepth(params.conversationId);
+      if (typeof maxDepth === "number" && maxDepth <= 1) {
+        const messageResult = await retrieval.grep({
+          query: params.query,
+          mode: "full_text",
+          scope: "messages",
+          conversationId: params.conversationId,
+        });
+        const messageIds = messageResult.messages.map((message) => message.messageId);
+        const leafLinks = await summaryStore.getLeafSummaryLinksForMessageIds(
+          params.conversationId,
+          messageIds,
+        );
+        const summaryIdsByMessageId = new Map<number, string[]>();
+        for (const link of leafLinks) {
+          const linkedSummaryIds = summaryIdsByMessageId.get(link.messageId) ?? [];
+          if (!linkedSummaryIds.includes(link.summaryId)) {
+            linkedSummaryIds.push(link.summaryId);
+            summaryIdsByMessageId.set(link.messageId, linkedSummaryIds);
+          }
+        }
+        for (const message of messageResult.messages) {
+          for (const summaryId of summaryIdsByMessageId.get(message.messageId) ?? []) {
+            upsertSummaryCandidate(candidates, {
+              summaryId,
+              conversationId: params.conversationId,
+              requiresMessageExpansion: true,
+            });
+          }
+        }
+      }
+    }
   }
 
   return Array.from(candidates.values());
@@ -297,6 +450,10 @@ export function createLcmExpandQueryTool(input: {
   /** Session key for scope fallback when sessionId is unavailable. */
   sessionKey?: string;
 }): AnyAgentTool {
+  const delegatedWaitTimeoutMs =
+    input.deps.config.delegationTimeoutMs || DEFAULT_DELEGATED_WAIT_TIMEOUT_MS;
+  const delegatedWaitTimeoutSeconds = Math.ceil(delegatedWaitTimeoutMs / 1000);
+
   return {
     name: "lcm_expand_query",
     label: "LCM Expand Query",
@@ -401,9 +558,6 @@ export function createLcmExpandQueryTool(input: {
         });
       }
 
-      let childSessionKey = "";
-      let grantCreated = false;
-
       try {
         const candidates = await resolveSummaryCandidates({
           lcm: input.lcm,
@@ -438,6 +592,15 @@ export function createLcmExpandQueryTool(input: {
             .filter((candidate) => candidate.conversationId === sourceConversationId)
             .map((candidate) => candidate.summaryId),
         );
+        const messageBackedSummaryIds = normalizeSummaryIds(
+          candidates
+            .filter(
+              (candidate) =>
+                candidate.conversationId === sourceConversationId &&
+                candidate.requiresMessageExpansion,
+            )
+            .map((candidate) => candidate.summaryId),
+        );
 
         if (summaryIds.length === 0) {
           return jsonResult({
@@ -448,28 +611,12 @@ export function createLcmExpandQueryTool(input: {
         const requesterAgentId = input.deps.normalizeAgentId(
           input.deps.parseAgentSessionKey(callerSessionKey)?.agentId,
         );
-        childSessionKey = `agent:${requesterAgentId}:subagent:${crypto.randomUUID()}`;
         const childExpansionDepth = resolveNextExpansionDepth(callerSessionKey);
         const originSessionKey = recursionCheck.originSessionKey || callerSessionKey || "main";
 
-        createDelegatedExpansionGrant({
-          delegatedSessionKey: childSessionKey,
-          issuerSessionId: callerSessionKey || "main",
-          allowedConversationIds: [sourceConversationId],
-          tokenCap: expansionTokenCap,
-          ttlMs: DELEGATED_WAIT_TIMEOUT_MS + 30_000,
-        });
-        stampDelegatedExpansionContext({
-          sessionKey: childSessionKey,
-          requestId,
-          expansionDepth: childExpansionDepth,
-          originSessionKey,
-          stampedBy: "lcm_expand_query",
-        });
-        grantCreated = true;
-
         const task = buildDelegatedExpandQueryTask({
           summaryIds,
+          messageBackedSummaryIds,
           conversationId: sourceConversationId,
           query: query || undefined,
           prompt,
@@ -480,118 +627,166 @@ export function createLcmExpandQueryTool(input: {
           originSessionKey,
         });
 
-        const childIdem = crypto.randomUUID();
         const expansionProvider = input.deps.config.expansionProvider || undefined;
         const expansionModel = input.deps.config.expansionModel || undefined;
-        const response = (await input.deps.callGateway({
-          method: "agent",
-          params: {
-            message: task,
-            sessionKey: childSessionKey,
-            deliver: false,
-            lane: input.deps.agentLaneSubagent,
-            idempotencyKey: childIdem,
-            ...(expansionProvider ? { provider: expansionProvider } : {}),
-            ...(expansionModel ? { model: expansionModel } : {}),
-            extraSystemPrompt: input.deps.buildSubagentSystemPrompt({
-              depth: 1,
-              maxDepth: 8,
-              taskSummary: "Run lcm_expand and return prompt-focused JSON answer",
-            }),
-          },
-          timeoutMs: GATEWAY_TIMEOUT_MS,
-        })) as { runId?: string };
-
-        const runId = typeof response?.runId === "string" ? response.runId.trim() : "";
-        if (!runId) {
-          return jsonResult({
-            error: "Delegated expansion did not return a runId.",
-          });
-        }
-
-        const wait = (await input.deps.callGateway({
-          method: "agent.wait",
-          params: {
-            runId,
-            timeoutMs: DELEGATED_WAIT_TIMEOUT_MS,
-          },
-          timeoutMs: DELEGATED_WAIT_TIMEOUT_MS,
-        })) as { status?: string; error?: string };
-        const status = typeof wait?.status === "string" ? wait.status : "error";
-        if (status === "timeout") {
-          recordExpansionDelegationTelemetry({
-            deps: input.deps,
-            component: "lcm_expand_query",
-            event: "timeout",
-            requestId,
-            sessionKey: callerSessionKey,
-            expansionDepth: childExpansionDepth,
-            originSessionKey,
-            runId,
-          });
-          return jsonResult({
-            error: "lcm_expand_query timed out waiting for delegated expansion (120s).",
-          });
-        }
-        if (status !== "ok") {
-          return jsonResult({
-            error:
-              typeof wait?.error === "string" && wait.error.trim()
-                ? wait.error
-                : "Delegated expansion query failed.",
-          });
-        }
-
-        const replyPayload = (await input.deps.callGateway({
-          method: "sessions.get",
-          params: { key: childSessionKey, limit: 80 },
-          timeoutMs: GATEWAY_TIMEOUT_MS,
-        })) as { messages?: unknown[] };
-        const reply = input.deps.readLatestAssistantReply(
-          Array.isArray(replyPayload.messages) ? replyPayload.messages : [],
-        );
-        const parsed = parseDelegatedExpandQueryReply(reply, summaryIds.length);
-        recordExpansionDelegationTelemetry({
-          deps: input.deps,
-          component: "lcm_expand_query",
-          event: "success",
-          requestId,
-          sessionKey: callerSessionKey,
-          expansionDepth: childExpansionDepth,
-          originSessionKey,
-          runId,
-        });
+        const canonicalExpansionModel = expansionModel?.includes("/") ? expansionModel : undefined;
+        const delegatedOverrideProvider = canonicalExpansionModel ? undefined : expansionProvider;
+        const delegatedOverrideModel = canonicalExpansionModel || expansionModel;
+        const configuredOverrideLabel =
+          delegatedOverrideProvider && delegatedOverrideModel
+            ? `${delegatedOverrideProvider}/${delegatedOverrideModel}`
+            : delegatedOverrideModel || delegatedOverrideProvider || "configured override";
+
+        const runDelegatedQuery = async (provider?: string, model?: string) => {
+          const childSessionKey = `agent:${requesterAgentId}:subagent:${crypto.randomUUID()}`;
+          const childIdem = crypto.randomUUID();
+          let grantCreated = false;
 
-        return jsonResult({
-          answer: parsed.answer,
-          citedIds: parsed.citedIds,
-          sourceConversationId,
-          expandedSummaryCount: parsed.expandedSummaryCount,
-          totalSourceTokens: parsed.totalSourceTokens,
-          truncated: parsed.truncated,
-        });
-      } catch (error) {
-        return jsonResult({
-          error: error instanceof Error ? error.message : String(error),
-        });
-      } finally {
-        if (childSessionKey) {
           try {
-            await input.deps.callGateway({
-              method: "sessions.delete",
-              params: { key: childSessionKey, deleteTranscript: true },
+            createDelegatedExpansionGrant({
+              delegatedSessionKey: childSessionKey,
+              issuerSessionId: callerSessionKey || "main",
+              allowedConversationIds: [sourceConversationId],
+              tokenCap: expansionTokenCap,
+              ttlMs: delegatedWaitTimeoutMs + 30_000,
+            });
+            stampDelegatedExpansionContext({
+              sessionKey: childSessionKey,
+              requestId,
+              expansionDepth: childExpansionDepth,
+              originSessionKey,
+              stampedBy: "lcm_expand_query",
+            });
+            grantCreated = true;
+
+            const response = (await input.deps.callGateway({
+              method: "agent",
+              params: {
+                message: task,
+                sessionKey: childSessionKey,
+                deliver: false,
+                lane: input.deps.agentLaneSubagent,
+                idempotencyKey: childIdem,
+                ...(provider ? { provider } : {}),
+                ...(model ? { model } : {}),
+                extraSystemPrompt: input.deps.buildSubagentSystemPrompt({
+                  depth: 1,
+                  maxDepth: 8,
+                  taskSummary: "Run lcm_expand and return prompt-focused JSON answer",
+                }),
+              },
               timeoutMs: GATEWAY_TIMEOUT_MS,
+            })) as { runId?: unknown; error?: unknown };
+
+            const runId = typeof response?.runId === "string" ? response.runId.trim() : "";
+            if (!runId) {
+              throw new Error(
+                formatExpansionFailure(response?.error ?? response)
+                  || "Delegated expansion did not return a runId.",
+              );
+            }
+
+            const wait = (await input.deps.callGateway({
+              method: "agent.wait",
+              params: {
+                runId,
+                timeoutMs: delegatedWaitTimeoutMs,
+              },
+              timeoutMs: delegatedWaitTimeoutMs,
+            })) as { status?: string; error?: unknown };
+            const status = typeof wait?.status === "string" ? wait.status : "error";
+            if (status === "timeout") {
+              recordExpansionDelegationTelemetry({
+                deps: input.deps,
+                component: "lcm_expand_query",
+                event: "timeout",
+                requestId,
+                sessionKey: callerSessionKey,
+                expansionDepth: childExpansionDepth,
+                originSessionKey,
+                runId,
+              });
+              throw new Error(
+                `lcm_expand_query timed out waiting for delegated expansion (${delegatedWaitTimeoutSeconds}s).`,
+              );
+            }
+            if (status !== "ok") {
+              throw new Error(formatExpansionFailure(wait?.error));
+            }
+
+            const replyPayload = (await input.deps.callGateway({
+              method: "sessions.get",
+              params: { key: childSessionKey, limit: 80 },
+              timeoutMs: GATEWAY_TIMEOUT_MS,
+            })) as { messages?: unknown[] };
+            const reply = input.deps.readLatestAssistantReply(
+              Array.isArray(replyPayload.messages) ? replyPayload.messages : [],
+            );
+            const parsed = parseDelegatedExpandQueryReply(reply, summaryIds.length);
+            if (!parsed.ok) {
+              throw new Error(parsed.error);
+            }
+            recordExpansionDelegationTelemetry({
+              deps: input.deps,
+              component: "lcm_expand_query",
+              event: "success",
+              requestId,
+              sessionKey: callerSessionKey,
+              expansionDepth: childExpansionDepth,
+              originSessionKey,
+              runId,
+            });
+
+            return jsonResult({
+              answer: parsed.value.answer,
+              citedIds: parsed.value.citedIds,
+              sourceConversationId,
+              expandedSummaryCount: parsed.value.expandedSummaryCount,
+              totalSourceTokens: parsed.value.totalSourceTokens,
+              truncated: parsed.value.truncated,
             });
-          } catch {
-            // Cleanup is best-effort.
+          } finally {
+            try {
+              await input.deps.callGateway({
+                method: "sessions.delete",
+                params: { key: childSessionKey, deleteTranscript: true },
+                timeoutMs: GATEWAY_TIMEOUT_MS,
+              });
+            } catch {
+              // Cleanup is best-effort.
+            }
+            if (grantCreated) {
+              revokeDelegatedExpansionGrantForSession(childSessionKey, { removeBinding: true });
+            }
+            clearDelegatedExpansionContext(childSessionKey);
           }
+        };
+
+        if (!expansionProvider && !expansionModel) {
+          return await runDelegatedQuery();
         }
-        if (grantCreated && childSessionKey) {
-          revokeDelegatedExpansionGrantForSession(childSessionKey, { removeBinding: true });
-        }
-        if (childSessionKey) {
-          clearDelegatedExpansionContext(childSessionKey);
+
+        try {
+          return await runDelegatedQuery(delegatedOverrideProvider, delegatedOverrideModel);
+        } catch (error) {
+          const failure = formatExpansionFailure(error);
+          input.deps.log.warn(
+            `[lcm] delegated expansion override failed (${configuredOverrideLabel}): ${failure}`,
+          );
+          if (!shouldRetryWithoutOverride(failure)) {
+            throw new Error(failure);
+          }
+          input.deps.log.warn(
+            `[lcm] retrying delegated expansion without provider/model override after: ${failure}`,
+          );
+          return await runDelegatedQuery();
         }
+      } catch (error) {
+        const failure = formatExpansionFailure(error);
+        input.deps.log.error(`[lcm] delegated expansion query failed: ${failure}`);
+        return jsonResult({
+          error: failure,
+        });
       }
     },
   };

package/src/types.ts

@@ -72,6 +72,7 @@ export type ApiKeyLookupOptions = {
   preferredProfile?: string;
   agentDir?: string;
   runtimeConfig?: unknown;
+  skipModelAuth?: boolean;
 };
 
 export type GetApiKeyFn = (