@martian-engineering/lossless-claw 0.5.2 → 0.6.0

package/src/tools/lcm-expand-query-tool.ts

@@ -13,15 +13,17 @@ import {
   resolveRequesterConversationScopeId,
 } from "./lcm-expand-tool.delegation.js";
 import {
+  acquireExpansionConcurrencySlot,
   clearDelegatedExpansionContext,
   evaluateExpansionRecursionGuard,
   recordExpansionDelegationTelemetry,
+  releaseExpansionConcurrencySlot,
   resolveExpansionRequestId,
   resolveNextExpansionDepth,
   stampDelegatedExpansionContext,
 } from "./lcm-expansion-recursion-guard.js";
 
-const DELEGATED_WAIT_TIMEOUT_MS = 120_000;
+const DEFAULT_DELEGATED_WAIT_TIMEOUT_MS = 120_000;
 const GATEWAY_TIMEOUT_MS = 10_000;
 const DEFAULT_MAX_ANSWER_TOKENS = 2_000;
 
@@ -75,9 +77,20 @@ type ExpandQueryReply = {
   truncated: boolean;
 };
 
+type ParsedExpandQueryReply =
+  | {
+      ok: true;
+      value: ExpandQueryReply;
+    }
+  | {
+      ok: false;
+      error: string;
+    };
+
 type SummaryCandidate = {
   summaryId: string;
   conversationId: number;
+  requiresMessageExpansion: boolean;
 };
 
 function collectExpansionFailureText(value: unknown, parts: string[], depth = 0): void {
@@ -155,6 +168,7 @@ function shouldRetryWithoutOverride(message: string): boolean {
  */
 function buildDelegatedExpandQueryTask(params: {
   summaryIds: string[];
+  messageBackedSummaryIds: string[];
   conversationId: number;
   query?: string;
   prompt: string;
@@ -165,6 +179,10 @@ function buildDelegatedExpandQueryTask(params: {
   originSessionKey: string;
 }) {
   const seedSummaryIds = params.summaryIds.length > 0 ? params.summaryIds.join(", ") : "(none)";
+  const messageBackedSummaryIds =
+    params.messageBackedSummaryIds.length > 0
+      ? params.messageBackedSummaryIds.join(", ")
+      : "(none)";
   return [
     "You are an autonomous LCM retrieval navigator. Plan and execute retrieval before answering.",
     "",
@@ -172,6 +190,7 @@ function buildDelegatedExpandQueryTask(params: {
     `Conversation scope: ${params.conversationId}`,
     `Expansion token budget (total across this run): ${params.tokenCap}`,
     `Seed summary IDs: ${seedSummaryIds}`,
+    `Seed summaries requiring raw message expansion: ${messageBackedSummaryIds}`,
     params.query ? `Routing query: ${params.query}` : undefined,
     "",
     "Strategy:",
@@ -179,7 +198,7 @@ function buildDelegatedExpandQueryTask(params: {
     "2. If additional candidates are needed, use `lcm_grep` scoped to summaries.",
     "3. Select branches that fit remaining budget; prefer high-signal paths first.",
     "4. Call `lcm_expand` selectively (do not expand everything blindly).",
-    "5. Keep includeMessages=false by default; use includeMessages=true only for specific leaf evidence.",
+    "5. Keep includeMessages=false by default; use includeMessages=true for the message-backed seed summaries above and any other specific leaf evidence.",
     `6. Stay within ${params.tokenCap} total expansion tokens across all lcm_expand calls.`,
     "",
     "User prompt to answer:",
@@ -211,24 +230,25 @@ function buildDelegatedExpandQueryTask(params: {
   ].join("\n");
 }
 
+function formatInvalidDelegatedReply(reply: string, reason: string): string {
+  const compact = reply.replace(/\s+/g, " ").trim();
+  const snippet = compact.length <= 240 ? compact : `${compact.slice(0, 240)}...`;
+  return `Delegated expansion query returned ${reason}: ${snippet}`;
+}
+
 /**
- * Parse the child reply; accepts plain JSON or fenced JSON.
+ * Parse the child reply; accepts plain JSON or fenced JSON and rejects malformed fallbacks.
  */
 function parseDelegatedExpandQueryReply(
   rawReply: string | undefined,
   fallbackExpandedSummaryCount: number,
-): ExpandQueryReply {
-  const fallback: ExpandQueryReply = {
-    answer: (rawReply ?? "").trim(),
-    citedIds: [],
-    expandedSummaryCount: fallbackExpandedSummaryCount,
-    totalSourceTokens: 0,
-    truncated: false,
-  };
-
+): ParsedExpandQueryReply {
   const reply = rawReply?.trim();
   if (!reply) {
-    return fallback;
+    return {
+      ok: false,
+      error: "Delegated expansion query returned an empty reply.",
+    };
   }
 
   const candidates: string[] = [reply];
@@ -247,6 +267,12 @@ function parseDelegatedExpandQueryReply(
         truncated?: unknown;
       };
       const answer = typeof parsed.answer === "string" ? parsed.answer.trim() : "";
+      if (!answer) {
+        return {
+          ok: false,
+          error: formatInvalidDelegatedReply(reply, 'JSON without a non-empty "answer"'),
+        };
+      }
       const citedIds = normalizeSummaryIds(
         Array.isArray(parsed.citedIds)
           ? parsed.citedIds.filter((value): value is string => typeof value === "string")
@@ -264,18 +290,24 @@ function parseDelegatedExpandQueryReply(
       const truncated = parsed.truncated === true;
 
       return {
-        answer: answer || fallback.answer,
-        citedIds,
-        expandedSummaryCount,
-        totalSourceTokens,
-        truncated,
+        ok: true,
+        value: {
+          answer,
+          citedIds,
+          expandedSummaryCount,
+          totalSourceTokens,
+          truncated,
+        },
       };
     } catch {
       // Try next candidate.
     }
   }
 
-  return fallback;
+  return {
+    ok: false,
+    error: formatInvalidDelegatedReply(reply, "non-JSON output"),
+  };
 }
 
 /**
@@ -316,6 +348,22 @@ function resolveSourceConversationId(params: {
   );
 }
 
+function upsertSummaryCandidate(
+  candidates: Map<string, SummaryCandidate>,
+  candidate: SummaryCandidate,
+): void {
+  const existing = candidates.get(candidate.summaryId);
+  if (!existing) {
+    candidates.set(candidate.summaryId, candidate);
+    return;
+  }
+  candidates.set(candidate.summaryId, {
+    ...existing,
+    requiresMessageExpansion:
+      existing.requiresMessageExpansion || candidate.requiresMessageExpansion,
+  });
+}
+
 /**
  * Resolve summary candidates from explicit IDs and/or query matches.
  */
@@ -333,13 +381,15 @@ async function resolveSummaryCandidates(params: {
     if (!described || described.type !== "summary" || !described.summary) {
       throw new Error(`Summary not found: ${summaryId}`);
     }
-    candidates.set(summaryId, {
+    upsertSummaryCandidate(candidates, {
       summaryId,
       conversationId: described.summary.conversationId,
+      requiresMessageExpansion: false,
     });
   }
 
   if (params.query) {
+    const summaryStore = params.lcm.getSummaryStore();
     const grepResult = await retrieval.grep({
       query: params.query,
       mode: "full_text",
@@ -347,11 +397,46 @@ async function resolveSummaryCandidates(params: {
       conversationId: params.conversationId,
     });
     for (const summary of grepResult.summaries) {
-      candidates.set(summary.summaryId, {
+      upsertSummaryCandidate(candidates, {
         summaryId: summary.summaryId,
         conversationId: summary.conversationId,
+        requiresMessageExpansion: false,
       });
     }
+
+    if (grepResult.summaries.length === 0 && typeof params.conversationId === "number") {
+      const maxDepth = await summaryStore.getConversationMaxSummaryDepth(params.conversationId);
+      if (typeof maxDepth === "number" && maxDepth <= 1) {
+        const messageResult = await retrieval.grep({
+          query: params.query,
+          mode: "full_text",
+          scope: "messages",
+          conversationId: params.conversationId,
+        });
+        const messageIds = messageResult.messages.map((message) => message.messageId);
+        const leafLinks = await summaryStore.getLeafSummaryLinksForMessageIds(
+          params.conversationId,
+          messageIds,
+        );
+        const summaryIdsByMessageId = new Map<number, string[]>();
+        for (const link of leafLinks) {
+          const linkedSummaryIds = summaryIdsByMessageId.get(link.messageId) ?? [];
+          if (!linkedSummaryIds.includes(link.summaryId)) {
+            linkedSummaryIds.push(link.summaryId);
+            summaryIdsByMessageId.set(link.messageId, linkedSummaryIds);
+          }
+        }
+        for (const message of messageResult.messages) {
+          for (const summaryId of summaryIdsByMessageId.get(message.messageId) ?? []) {
+            upsertSummaryCandidate(candidates, {
+              summaryId,
+              conversationId: params.conversationId,
+              requiresMessageExpansion: true,
+            });
+          }
+        }
+      }
+    }
   }
 
   return Array.from(candidates.values());
@@ -367,6 +452,10 @@ export function createLcmExpandQueryTool(input: {
   /** Session key for scope fallback when sessionId is unavailable. */
   sessionKey?: string;
 }): AnyAgentTool {
+  const delegatedWaitTimeoutMs =
+    input.deps.config.delegationTimeoutMs || DEFAULT_DELEGATED_WAIT_TIMEOUT_MS;
+  const delegatedWaitTimeoutSeconds = Math.ceil(delegatedWaitTimeoutMs / 1000);
+
   return {
     name: "lcm_expand_query",
     label: "LCM Expand Query",
@@ -444,34 +533,36 @@ export function createLcmExpandQueryTool(input: {
         });
       }
 
-      const conversationScope = await resolveLcmConversationScope({
-        lcm: input.lcm,
-        deps: input.deps,
-        sessionId: input.sessionId,
-        sessionKey: input.sessionKey,
-        params: p,
-      });
-      let scopedConversationId = conversationScope.conversationId;
-      if (
-        !conversationScope.allConversations &&
-        scopedConversationId == null &&
-        callerSessionKey
-      ) {
-        scopedConversationId = await resolveRequesterConversationScopeId({
-          deps: input.deps,
-          requesterSessionKey: callerSessionKey,
+      const originSessionKey = recursionCheck.originSessionKey || callerSessionKey || "main";
+
+      try {
+        const conversationScope = await resolveLcmConversationScope({
           lcm: input.lcm,
+          deps: input.deps,
+          sessionId: input.sessionId,
+          sessionKey: input.sessionKey,
+          params: p,
         });
-      }
+        let scopedConversationId = conversationScope.conversationId;
+        if (
+          !conversationScope.allConversations &&
+          scopedConversationId == null &&
+          callerSessionKey
+        ) {
+          scopedConversationId = await resolveRequesterConversationScopeId({
+            deps: input.deps,
+            requesterSessionKey: callerSessionKey,
+            lcm: input.lcm,
+          });
+        }
 
-      if (!conversationScope.allConversations && scopedConversationId == null) {
-        return jsonResult({
-          error:
-            "No LCM conversation found for this session. Provide conversationId or set allConversations=true.",
-        });
-      }
+        if (!conversationScope.allConversations && scopedConversationId == null) {
+          return jsonResult({
+            error:
+              "No LCM conversation found for this session. Provide conversationId or set allConversations=true.",
+          });
+        }
 
-      try {
         const candidates = await resolveSummaryCandidates({
           lcm: input.lcm,
           explicitSummaryIds,
@@ -505,6 +596,15 @@ export function createLcmExpandQueryTool(input: {
             .filter((candidate) => candidate.conversationId === sourceConversationId)
             .map((candidate) => candidate.summaryId),
         );
+        const messageBackedSummaryIds = normalizeSummaryIds(
+          candidates
+            .filter(
+              (candidate) =>
+                candidate.conversationId === sourceConversationId &&
+                candidate.requiresMessageExpansion,
+            )
+            .map((candidate) => candidate.summaryId),
+        );
 
         if (summaryIds.length === 0) {
           return jsonResult({
@@ -512,14 +612,39 @@ export function createLcmExpandQueryTool(input: {
           });
         }
 
+        const concurrencyCheck = acquireExpansionConcurrencySlot({
+          originSessionKey,
+          requestId,
+        });
+        if (concurrencyCheck.blocked) {
+          recordExpansionDelegationTelemetry({
+            deps: input.deps,
+            component: "lcm_expand_query",
+            event: "block",
+            requestId,
+            sessionKey: callerSessionKey,
+            expansionDepth: recursionCheck.expansionDepth,
+            originSessionKey: concurrencyCheck.originSessionKey,
+            reason: concurrencyCheck.reason,
+          });
+          return jsonResult({
+            errorCode: concurrencyCheck.code,
+            error: concurrencyCheck.message,
+            requestId: concurrencyCheck.requestId,
+            expansionDepth: recursionCheck.expansionDepth,
+            originSessionKey: concurrencyCheck.originSessionKey,
+            reason: concurrencyCheck.reason,
+          });
+        }
+
         const requesterAgentId = input.deps.normalizeAgentId(
           input.deps.parseAgentSessionKey(callerSessionKey)?.agentId,
         );
         const childExpansionDepth = resolveNextExpansionDepth(callerSessionKey);
-        const originSessionKey = recursionCheck.originSessionKey || callerSessionKey || "main";
 
         const task = buildDelegatedExpandQueryTask({
           summaryIds,
+          messageBackedSummaryIds,
           conversationId: sourceConversationId,
           query: query || undefined,
           prompt,
@@ -532,10 +657,13 @@ export function createLcmExpandQueryTool(input: {
 
         const expansionProvider = input.deps.config.expansionProvider || undefined;
         const expansionModel = input.deps.config.expansionModel || undefined;
+        const canonicalExpansionModel = expansionModel?.includes("/") ? expansionModel : undefined;
+        const delegatedOverrideProvider = canonicalExpansionModel ? undefined : expansionProvider;
+        const delegatedOverrideModel = canonicalExpansionModel || expansionModel;
         const configuredOverrideLabel =
-          expansionProvider && expansionModel
-            ? `${expansionProvider}/${expansionModel}`
-            : expansionModel || expansionProvider || "configured override";
+          delegatedOverrideProvider && delegatedOverrideModel
+            ? `${delegatedOverrideProvider}/${delegatedOverrideModel}`
+            : delegatedOverrideModel || delegatedOverrideProvider || "configured override";
 
         const runDelegatedQuery = async (provider?: string, model?: string) => {
           const childSessionKey = `agent:${requesterAgentId}:subagent:${crypto.randomUUID()}`;
@@ -548,7 +676,7 @@ export function createLcmExpandQueryTool(input: {
               issuerSessionId: callerSessionKey || "main",
               allowedConversationIds: [sourceConversationId],
               tokenCap: expansionTokenCap,
-              ttlMs: DELEGATED_WAIT_TIMEOUT_MS + 30_000,
+              ttlMs: delegatedWaitTimeoutMs + 30_000,
             });
             stampDelegatedExpansionContext({
               sessionKey: childSessionKey,
@@ -590,9 +718,9 @@ export function createLcmExpandQueryTool(input: {
               method: "agent.wait",
               params: {
                 runId,
-                timeoutMs: DELEGATED_WAIT_TIMEOUT_MS,
+                timeoutMs: delegatedWaitTimeoutMs,
               },
-              timeoutMs: DELEGATED_WAIT_TIMEOUT_MS,
+              timeoutMs: delegatedWaitTimeoutMs,
             })) as { status?: string; error?: unknown };
             const status = typeof wait?.status === "string" ? wait.status : "error";
             if (status === "timeout") {
@@ -607,7 +735,7 @@ export function createLcmExpandQueryTool(input: {
                 runId,
               });
               throw new Error(
-                "lcm_expand_query timed out waiting for delegated expansion (120s).",
+                `lcm_expand_query timed out waiting for delegated expansion (${delegatedWaitTimeoutSeconds}s).`,
               );
             }
             if (status !== "ok") {
@@ -623,6 +751,9 @@ export function createLcmExpandQueryTool(input: {
               Array.isArray(replyPayload.messages) ? replyPayload.messages : [],
             );
             const parsed = parseDelegatedExpandQueryReply(reply, summaryIds.length);
+            if (!parsed.ok) {
+              throw new Error(parsed.error);
+            }
             recordExpansionDelegationTelemetry({
               deps: input.deps,
               component: "lcm_expand_query",
@@ -635,12 +766,12 @@ export function createLcmExpandQueryTool(input: {
             });
 
             return jsonResult({
-              answer: parsed.answer,
-              citedIds: parsed.citedIds,
+              answer: parsed.value.answer,
+              citedIds: parsed.value.citedIds,
               sourceConversationId,
-              expandedSummaryCount: parsed.expandedSummaryCount,
-              totalSourceTokens: parsed.totalSourceTokens,
-              truncated: parsed.truncated,
+              expandedSummaryCount: parsed.value.expandedSummaryCount,
+              totalSourceTokens: parsed.value.totalSourceTokens,
+              truncated: parsed.value.truncated,
             });
           } finally {
             try {
@@ -664,7 +795,7 @@ export function createLcmExpandQueryTool(input: {
         }
 
         try {
-          return await runDelegatedQuery(expansionProvider, expansionModel);
+          return await runDelegatedQuery(delegatedOverrideProvider, delegatedOverrideModel);
         } catch (error) {
           const failure = formatExpansionFailure(error);
           input.deps.log.warn(
@@ -684,6 +815,11 @@ export function createLcmExpandQueryTool(input: {
         return jsonResult({
           error: failure,
         });
+      } finally {
+        releaseExpansionConcurrencySlot({
+          originSessionKey,
+          requestId,
+        });
       }
     },
   };

package/src/tools/lcm-expansion-recursion-guard.ts

@@ -3,6 +3,7 @@ import { resolveDelegatedExpansionGrantId } from "../expansion-auth.js";
 import type { LcmDependencies } from "../types.js";
 
 export const EXPANSION_RECURSION_ERROR_CODE = "EXPANSION_RECURSION_BLOCKED";
+export const EXPANSION_CONCURRENCY_ERROR_CODE = "EXPANSION_CONCURRENCY_BLOCKED";
 const EXPANSION_DELEGATION_DEPTH_CAP = 1;
 
 type TelemetryEvent = "start" | "block" | "timeout" | "success";
@@ -23,6 +24,7 @@ export type DelegatedExpansionContext = {
 };
 
 export type ExpansionRecursionBlockReason = "depth_cap" | "idempotent_reentry";
+export type ExpansionConcurrencyBlockReason = "origin_session_in_flight";
 
 export type ExpansionRecursionGuardDecision =
   | {
@@ -41,8 +43,24 @@ export type ExpansionRecursionGuardDecision =
       originSessionKey: string;
     };
 
+export type ExpansionConcurrencyGuardDecision =
+  | {
+      blocked: false;
+      requestId: string;
+      originSessionKey: string;
+    }
+  | {
+      blocked: true;
+      code: typeof EXPANSION_CONCURRENCY_ERROR_CODE;
+      reason: ExpansionConcurrencyBlockReason;
+      message: string;
+      requestId: string;
+      originSessionKey: string;
+    };
+
 const delegatedContextBySessionKey = new Map<string, DelegatedExpansionContext>();
 const blockedRequestIdsBySessionKey = new Map<string, Set<string>>();
+const activeRequestIdByOriginSessionKey = new Map<string, string>();
 
 function normalizeSessionKey(sessionKey?: string): string {
   return typeof sessionKey === "string" ? sessionKey.trim() : "";
@@ -90,6 +108,14 @@ function buildExpansionRecursionRecoveryGuidance(originSessionKey: string): stri
   );
 }
 
+function buildExpansionConcurrencyRecoveryGuidance(originSessionKey: string): string {
+  return (
+    "Recovery: Wait for the active expansion to finish before retrying. " +
+    `If you need an immediate fallback, stay in the origin session (${originSessionKey}) ` +
+    "and use `lcm_grep` or `lcm_describe` instead."
+  );
+}
+
 /**
  * Create a stable request identifier for delegated expansion orchestration.
  */
@@ -213,6 +239,66 @@ export function evaluateExpansionRecursionGuard(params: {
   };
 }
 
+/**
+ * Acquire the single active delegated-expansion slot for an origin session.
+ * A second concurrent caller from the same origin session is blocked
+ * immediately so sub-agents do not deadlock on a shared lane.
+ */
+export function acquireExpansionConcurrencySlot(params: {
+  originSessionKey?: string;
+  requestId: string;
+}): ExpansionConcurrencyGuardDecision {
+  const originSessionKey = normalizeSessionKey(params.originSessionKey) || "main";
+  const requestId = params.requestId.trim();
+  const activeRequestId = activeRequestIdByOriginSessionKey.get(originSessionKey);
+
+  if (activeRequestId && activeRequestId !== requestId) {
+    return {
+      blocked: true,
+      code: EXPANSION_CONCURRENCY_ERROR_CODE,
+      reason: "origin_session_in_flight",
+      message:
+        `${EXPANSION_CONCURRENCY_ERROR_CODE}: Another lcm_expand_query delegation is already ` +
+        `in flight for origin session (${originSessionKey}; activeRequestId=${activeRequestId}). ` +
+        buildExpansionConcurrencyRecoveryGuidance(originSessionKey),
+      requestId,
+      originSessionKey,
+    };
+  }
+
+  if (!activeRequestId) {
+    activeRequestIdByOriginSessionKey.set(originSessionKey, requestId);
+  }
+
+  return {
+    blocked: false,
+    requestId,
+    originSessionKey,
+  };
+}
+
+/**
+ * Release the active delegated-expansion slot for an origin session.
+ */
+export function releaseExpansionConcurrencySlot(params: {
+  originSessionKey?: string;
+  requestId?: string;
+}): void {
+  const originSessionKey = normalizeSessionKey(params.originSessionKey);
+  if (!originSessionKey) {
+    return;
+  }
+  const activeRequestId = activeRequestIdByOriginSessionKey.get(originSessionKey);
+  if (!activeRequestId) {
+    return;
+  }
+  const requestId = params.requestId?.trim();
+  if (requestId && activeRequestId !== requestId) {
+    return;
+  }
+  activeRequestIdByOriginSessionKey.delete(originSessionKey);
+}
+
 /**
  * Emit structured delegated expansion telemetry with monotonic counters.
  */
@@ -279,6 +365,7 @@ export function getExpansionDelegationTelemetrySnapshotForTests(): Record<Teleme
 export function resetExpansionDelegationGuardForTests(): void {
   delegatedContextBySessionKey.clear();
   blockedRequestIdsBySessionKey.clear();
+  activeRequestIdByOriginSessionKey.clear();
   telemetryCounters.start = 0;
   telemetryCounters.block = 0;
   telemetryCounters.timeout = 0;

package/src/types.ts

@@ -72,6 +72,7 @@ export type ApiKeyLookupOptions = {
   preferredProfile?: string;
   agentDir?: string;
   runtimeConfig?: unknown;
+  skipModelAuth?: boolean;
 };
 
 export type GetApiKeyFn = (