@martian-engineering/lossless-claw 0.6.3 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +26 -6
  2. package/docs/agent-tools.md +16 -5
  3. package/docs/configuration.md +223 -214
  4. package/openclaw.plugin.json +123 -0
  5. package/package.json +1 -1
  6. package/skills/lossless-claw/SKILL.md +3 -2
  7. package/skills/lossless-claw/references/architecture.md +12 -0
  8. package/skills/lossless-claw/references/config.md +135 -3
  9. package/skills/lossless-claw/references/diagnostics.md +13 -0
  10. package/src/assembler.ts +17 -5
  11. package/src/compaction.ts +161 -53
  12. package/src/db/config.ts +102 -4
  13. package/src/db/connection.ts +35 -7
  14. package/src/db/features.ts +24 -5
  15. package/src/db/migration.ts +257 -78
  16. package/src/engine.ts +1007 -110
  17. package/src/estimate-tokens.ts +80 -0
  18. package/src/lcm-log.ts +37 -0
  19. package/src/plugin/index.ts +493 -101
  20. package/src/plugin/lcm-command.ts +288 -7
  21. package/src/plugin/lcm-doctor-apply.ts +1 -3
  22. package/src/plugin/lcm-doctor-cleaners.ts +655 -0
  23. package/src/plugin/shared-init.ts +59 -0
  24. package/src/prune.ts +391 -0
  25. package/src/retrieval.ts +8 -9
  26. package/src/startup-banner-log.ts +1 -0
  27. package/src/store/compaction-telemetry-store.ts +156 -0
  28. package/src/store/conversation-store.ts +6 -1
  29. package/src/store/fts5-sanitize.ts +25 -4
  30. package/src/store/full-text-sort.ts +21 -0
  31. package/src/store/index.ts +8 -0
  32. package/src/store/summary-store.ts +21 -14
  33. package/src/summarize.ts +55 -34
  34. package/src/tools/lcm-describe-tool.ts +9 -4
  35. package/src/tools/lcm-expand-query-tool.ts +609 -200
  36. package/src/tools/lcm-expand-tool.ts +9 -4
  37. package/src/tools/lcm-grep-tool.ts +22 -8
  38. package/src/types.ts +1 -0
@@ -26,6 +26,7 @@ import {
26
26
  const DEFAULT_DELEGATED_WAIT_TIMEOUT_MS = 120_000;
27
27
  const GATEWAY_TIMEOUT_MS = 10_000;
28
28
  const DEFAULT_MAX_ANSWER_TOKENS = 2_000;
29
+ const DEFAULT_MAX_CONVERSATION_BUCKETS = 3;
29
30
 
30
31
  const LcmExpandQuerySchema = Type.Object({
31
32
  summaryIds: Type.Optional(
@@ -36,11 +37,12 @@ const LcmExpandQuerySchema = Type.Object({
36
37
  query: Type.Optional(
37
38
  Type.String({
38
39
  description:
39
- "Text query used to find summaries via grep before expansion. Required when summaryIds is not provided.",
40
+ "FTS5 query used to find summaries via the same full-text search path as lcm_grep before expansion. Use 1-3 distinctive terms or a quoted phrase; FTS5 defaults to AND matching, so extra terms make matches stricter. Required when summaryIds is not provided.",
40
41
  }),
41
42
  ),
42
43
  prompt: Type.String({
43
- description: "Question to answer using expanded context.",
44
+ description:
45
+ "Natural-language question or task to answer using expanded context. Put the answer request here, not in query.",
44
46
  }),
45
47
  conversationId: Type.Optional(
46
48
  Type.Number({
@@ -69,7 +71,28 @@ const LcmExpandQuerySchema = Type.Object({
69
71
  ),
70
72
  });
71
73
 
74
+ type ConversationBreakdown = {
75
+ conversationId: number;
76
+ expandedSummaryCount: number;
77
+ citedIds: string[];
78
+ totalSourceTokens: number;
79
+ truncated: boolean;
80
+ status?: "success" | "failed" | "skipped";
81
+ error?: string;
82
+ };
83
+
72
84
  type ExpandQueryReply = {
85
+ answer: string;
86
+ citedIds: string[];
87
+ sourceConversationIds: number[];
88
+ expandedSummaryCount: number;
89
+ totalSourceTokens: number;
90
+ truncated: boolean;
91
+ conversationBreakdown?: ConversationBreakdown[];
92
+ sourceConversationId?: number;
93
+ };
94
+
95
+ type DelegatedExpandQueryReply = {
73
96
  answer: string;
74
97
  citedIds: string[];
75
98
  expandedSummaryCount: number;
@@ -80,7 +103,7 @@ type ExpandQueryReply = {
80
103
  type ParsedExpandQueryReply =
81
104
  | {
82
105
  ok: true;
83
- value: ExpandQueryReply;
106
+ value: DelegatedExpandQueryReply;
84
107
  }
85
108
  | {
86
109
  ok: false;
@@ -91,6 +114,48 @@ type SummaryCandidate = {
91
114
  summaryId: string;
92
115
  conversationId: number;
93
116
  requiresMessageExpansion: boolean;
117
+ isExplicit: boolean;
118
+ matchedAt?: Date;
119
+ };
120
+
121
+ type ConversationBucket = {
122
+ conversationId: number;
123
+ summaryIds: string[];
124
+ messageBackedSummaryIds: string[];
125
+ candidateCount: number;
126
+ explicitSummaryCount: number;
127
+ messageBackedCount: number;
128
+ newestMatchAt?: Date;
129
+ };
130
+
131
+ type BucketExecutionResult =
132
+ | {
133
+ conversationId: number;
134
+ status: "success";
135
+ candidateCount: number;
136
+ reply: DelegatedExpandQueryReply;
137
+ }
138
+ | {
139
+ conversationId: number;
140
+ status: "failed" | "skipped";
141
+ candidateCount: number;
142
+ error: string;
143
+ };
144
+
145
+ type RunDelegatedExpandQueryParams = {
146
+ deps: LcmDependencies;
147
+ callerSessionKey: string;
148
+ requesterAgentId: string;
149
+ bucket: ConversationBucket;
150
+ query?: string;
151
+ prompt: string;
152
+ maxTokens: number;
153
+ tokenCap: number;
154
+ requestId: string;
155
+ childExpansionDepth: number;
156
+ originSessionKey: string;
157
+ delegatedWaitTimeoutMs: number;
158
+ delegatedWaitTimeoutSeconds: number;
94
159
  };
95
160
 
96
161
  function collectExpansionFailureText(value: unknown, parts: string[], depth = 0): void {
@@ -163,6 +228,16 @@ function shouldRetryWithoutOverride(message: string): boolean {
163
228
  ].some((signal) => normalized.includes(signal));
164
229
  }
165
230
 
231
+ function maxDate(left?: Date, right?: Date): Date | undefined {
232
+ if (!left) {
233
+ return right;
234
+ }
235
+ if (!right) {
236
+ return left;
237
+ }
238
+ return left.getTime() >= right.getTime() ? left : right;
239
+ }
240
+
166
241
  /**
167
242
  * Build the sub-agent task message for delegated expansion and prompt answering.
168
243
  */
@@ -195,7 +270,7 @@ function buildDelegatedExpandQueryTask(params: {
195
270
  "",
196
271
  "Strategy:",
197
272
  "1. Start with `lcm_describe` on seed summaries to inspect subtree manifests and branch costs.",
198
- "2. If additional candidates are needed, use `lcm_grep` scoped to summaries.",
273
+ "2. If additional candidates are needed, use `lcm_grep` scoped to summaries. Prefer `mode: \"full_text\"`, quote exact multi-word phrases, use `sort: \"relevance\"` for older-topic recall, and `sort: \"hybrid\"` when recency should still matter.",
199
274
  "3. Select branches that fit remaining budget; prefer high-signal paths first.",
200
275
  "4. Call `lcm_expand` selectively (do not expand everything blindly).",
201
276
  "5. Keep includeMessages=false by default; use includeMessages=true for the message-backed seed summaries above and any other specific leaf evidence.",
@@ -227,7 +302,9 @@ function buildDelegatedExpandQueryTask(params: {
227
302
  "- expandedSummaryCount should reflect how many summaries were expanded/used.",
228
303
  "- totalSourceTokens should estimate total tokens consumed from expansion calls.",
229
304
  "- truncated should indicate whether source expansion appears truncated.",
230
- ].join("\n");
305
+ ]
306
+ .filter((line): line is string => typeof line === "string")
307
+ .join("\n");
231
308
  }
232
309
 
233
310
  function formatInvalidDelegatedReply(reply: string, reason: string): string {
@@ -236,6 +313,174 @@ function formatInvalidDelegatedReply(reply: string, reason: string): string {
236
313
  return `Delegated expansion query returned ${reason}: ${snippet}`;
237
314
  }
238
315
 
316
+ function buildConversationBuckets(candidates: SummaryCandidate[]): ConversationBucket[] {
317
+ const buckets = new Map<
318
+ number,
319
+ {
320
+ conversationId: number;
321
+ summaryIds: string[];
322
+ messageBackedSummaryIds: string[];
323
+ summaryIdSet: Set<string>;
324
+ explicitSummaryIdSet: Set<string>;
325
+ messageBackedSummaryIdSet: Set<string>;
326
+ newestMatchAt?: Date;
327
+ }
328
+ >();
329
+
330
+ for (const candidate of candidates) {
331
+ const bucket =
332
+ buckets.get(candidate.conversationId) ??
333
+ {
334
+ conversationId: candidate.conversationId,
335
+ summaryIds: [],
336
+ messageBackedSummaryIds: [],
337
+ summaryIdSet: new Set<string>(),
338
+ explicitSummaryIdSet: new Set<string>(),
339
+ messageBackedSummaryIdSet: new Set<string>(),
340
+ newestMatchAt: undefined,
341
+ };
342
+
343
+ if (!bucket.summaryIdSet.has(candidate.summaryId)) {
344
+ bucket.summaryIds.push(candidate.summaryId);
345
+ bucket.summaryIdSet.add(candidate.summaryId);
346
+ }
347
+ if (candidate.isExplicit) {
348
+ bucket.explicitSummaryIdSet.add(candidate.summaryId);
349
+ }
350
+ if (
351
+ candidate.requiresMessageExpansion &&
352
+ !bucket.messageBackedSummaryIdSet.has(candidate.summaryId)
353
+ ) {
354
+ bucket.messageBackedSummaryIds.push(candidate.summaryId);
355
+ bucket.messageBackedSummaryIdSet.add(candidate.summaryId);
356
+ }
357
+ bucket.newestMatchAt = maxDate(bucket.newestMatchAt, candidate.matchedAt);
358
+ buckets.set(candidate.conversationId, bucket);
359
+ }
360
+
361
+ return Array.from(buckets.values()).map((bucket) => ({
362
+ conversationId: bucket.conversationId,
363
+ summaryIds: normalizeSummaryIds(bucket.summaryIds),
364
+ messageBackedSummaryIds: normalizeSummaryIds(bucket.messageBackedSummaryIds),
365
+ candidateCount: bucket.summaryIds.length,
366
+ explicitSummaryCount: bucket.explicitSummaryIdSet.size,
367
+ messageBackedCount: bucket.messageBackedSummaryIds.length,
368
+ newestMatchAt: bucket.newestMatchAt,
369
+ }));
370
+ }
371
+
372
+ function compareConversationBuckets(left: ConversationBucket, right: ConversationBucket): number {
373
+ const explicitDelta = right.explicitSummaryCount - left.explicitSummaryCount;
374
+ if (explicitDelta !== 0) {
375
+ return explicitDelta;
376
+ }
377
+
378
+ const candidateDelta = right.candidateCount - left.candidateCount;
379
+ if (candidateDelta !== 0) {
380
+ return candidateDelta;
381
+ }
382
+
383
+ const recencyDelta =
384
+ (right.newestMatchAt?.getTime() ?? 0) - (left.newestMatchAt?.getTime() ?? 0);
385
+ if (recencyDelta !== 0) {
386
+ return recencyDelta;
387
+ }
388
+
389
+ const messageBackedDelta = right.messageBackedCount - left.messageBackedCount;
390
+ if (messageBackedDelta !== 0) {
391
+ return messageBackedDelta;
392
+ }
393
+
394
+ return left.conversationId - right.conversationId;
395
+ }
396
+
397
+ function buildExpandQueryReply(params: {
398
+ answer: string;
399
+ citedIds: string[];
400
+ sourceConversationIds: number[];
401
+ expandedSummaryCount: number;
402
+ totalSourceTokens: number;
403
+ truncated: boolean;
404
+ conversationBreakdown?: ConversationBreakdown[];
405
+ }): ExpandQueryReply {
406
+ const sourceConversationIds = [...params.sourceConversationIds].sort((left, right) => left - right);
407
+
408
+ return {
409
+ answer: params.answer,
410
+ citedIds: normalizeSummaryIds(params.citedIds),
411
+ sourceConversationIds,
412
+ ...(sourceConversationIds.length === 1
413
+ ? { sourceConversationId: sourceConversationIds[0] }
414
+ : {}),
415
+ expandedSummaryCount: params.expandedSummaryCount,
416
+ totalSourceTokens: params.totalSourceTokens,
417
+ truncated: params.truncated,
418
+ ...(params.conversationBreakdown ? { conversationBreakdown: params.conversationBreakdown } : {}),
419
+ };
420
+ }
421
+
422
+ function synthesizeConversationAnswers(params: {
423
+ prompt: string;
424
+ results: BucketExecutionResult[];
425
+ }): string {
426
+ const successfulResults = params.results.filter(
427
+ (result): result is Extract<BucketExecutionResult, { status: "success" }> =>
428
+ result.status === "success",
429
+ );
430
+ const failedResults = params.results.filter(
431
+ (result): result is Extract<BucketExecutionResult, { status: "failed" }> =>
432
+ result.status === "failed",
433
+ );
434
+ const skippedResults = params.results.filter(
435
+ (result): result is Extract<BucketExecutionResult, { status: "skipped" }> =>
436
+ result.status === "skipped",
437
+ );
438
+
439
+ if (successfulResults.length === 1 && failedResults.length === 0 && skippedResults.length === 0) {
440
+ return successfulResults[0].reply.answer;
441
+ }
442
+
443
+ const lines: string[] = [];
444
+ if (successfulResults.length > 1) {
445
+ lines.push(`Merged findings across ${successfulResults.length} conversations:`);
446
+ lines.push("");
447
+ }
448
+
449
+ for (const result of successfulResults) {
450
+ if (successfulResults.length > 1) {
451
+ lines.push(`Conversation ${result.conversationId}:`);
452
+ }
453
+ lines.push(result.reply.answer);
454
+ if (successfulResults.length > 1) {
455
+ lines.push("");
456
+ }
457
+ }
458
+
459
+ const notes: string[] = [];
460
+ if (failedResults.length > 0) {
461
+ notes.push(
462
+ `failed conversations: ${failedResults
463
+ .map((result) => `${result.conversationId} (${result.error})`)
464
+ .join("; ")}`,
465
+ );
466
+ }
467
+ if (skippedResults.length > 0) {
468
+ notes.push(
469
+ `skipped conversations: ${skippedResults
470
+ .map((result) => `${result.conversationId} (${result.error})`)
471
+ .join("; ")}`,
472
+ );
473
+ }
474
+ if (notes.length > 0) {
475
+ if (lines.length > 0 && lines[lines.length - 1] !== "") {
476
+ lines.push("");
477
+ }
478
+ lines.push(`Partial coverage for "${params.prompt}": ${notes.join("; ")}`);
479
+ }
480
+
481
+ return lines.join("\n").trim();
482
+ }
483
+
239
484
  /**
240
485
  * Parse the child reply; accepts plain JSON or fenced JSON and rejects malformed fallbacks.
241
486
  */
@@ -348,6 +593,19 @@ function resolveSourceConversationId(params: {
348
593
  );
349
594
  }
350
595
 
596
+ function selectSingleConversationBucket(params: {
597
+ sourceConversationId: number;
598
+ buckets: ConversationBucket[];
599
+ }): ConversationBucket {
600
+ const bucket = params.buckets.find(
601
+ (candidateBucket) => candidateBucket.conversationId === params.sourceConversationId,
602
+ );
603
+ if (!bucket || bucket.summaryIds.length === 0) {
604
+ throw new Error("No summaryIds available after applying conversation scope.");
605
+ }
606
+ return bucket;
607
+ }
608
+
351
609
  function upsertSummaryCandidate(
352
610
  candidates: Map<string, SummaryCandidate>,
353
611
  candidate: SummaryCandidate,
@@ -361,6 +619,8 @@ function upsertSummaryCandidate(
361
619
  ...existing,
362
620
  requiresMessageExpansion:
363
621
  existing.requiresMessageExpansion || candidate.requiresMessageExpansion,
622
+ isExplicit: existing.isExplicit || candidate.isExplicit,
623
+ matchedAt: maxDate(existing.matchedAt, candidate.matchedAt),
364
624
  });
365
625
  }
366
626
 
@@ -385,6 +645,8 @@ async function resolveSummaryCandidates(params: {
385
645
  summaryId,
386
646
  conversationId: described.summary.conversationId,
387
647
  requiresMessageExpansion: false,
648
+ isExplicit: true,
649
+ matchedAt: described.summary.latestAt ?? described.summary.createdAt,
388
650
  });
389
651
  }
390
652
 
@@ -401,6 +663,8 @@ async function resolveSummaryCandidates(params: {
401
663
  summaryId: summary.summaryId,
402
664
  conversationId: summary.conversationId,
403
665
  requiresMessageExpansion: false,
666
+ isExplicit: false,
667
+ matchedAt: summary.createdAt,
404
668
  });
405
669
  }
406
670
 
@@ -432,6 +696,8 @@ async function resolveSummaryCandidates(params: {
432
696
  summaryId,
433
697
  conversationId: params.conversationId,
434
698
  requiresMessageExpansion: true,
699
+ isExplicit: false,
700
+ matchedAt: message.createdAt,
435
701
  });
436
702
  }
437
703
  }
@@ -442,9 +708,181 @@ async function resolveSummaryCandidates(params: {
442
708
  return Array.from(candidates.values());
443
709
  }
444
710
 
711
+ /**
712
+ * Run a single delegated lcm_expand_query bucket against one conversation.
713
+ */
714
+ async function runDelegatedExpandQuery(
715
+ params: RunDelegatedExpandQueryParams,
716
+ ): Promise<DelegatedExpandQueryReply> {
717
+ const task = buildDelegatedExpandQueryTask({
718
+ summaryIds: params.bucket.summaryIds,
719
+ messageBackedSummaryIds: params.bucket.messageBackedSummaryIds,
720
+ conversationId: params.bucket.conversationId,
721
+ query: params.query,
722
+ prompt: params.prompt,
723
+ maxTokens: params.maxTokens,
724
+ tokenCap: params.tokenCap,
725
+ requestId: params.requestId,
726
+ expansionDepth: params.childExpansionDepth,
727
+ originSessionKey: params.originSessionKey,
728
+ });
729
+
730
+ const expansionProvider = params.deps.config.expansionProvider || undefined;
731
+ const expansionModel = params.deps.config.expansionModel || undefined;
732
+ const canonicalExpansionModel = expansionModel?.includes("/") ? expansionModel : undefined;
733
+ const delegatedOverrideProvider = canonicalExpansionModel ? undefined : expansionProvider;
734
+ const delegatedOverrideModel = canonicalExpansionModel || expansionModel;
735
+ const configuredOverrideLabel =
736
+ delegatedOverrideProvider && delegatedOverrideModel
737
+ ? `${delegatedOverrideProvider}/${delegatedOverrideModel}`
738
+ : delegatedOverrideModel || delegatedOverrideProvider || "configured override";
739
+
740
+ const runDelegatedQuery = async (provider?: string, model?: string) => {
741
+ const childSessionKey = `agent:${params.requesterAgentId}:subagent:${crypto.randomUUID()}`;
742
+ const childIdem = crypto.randomUUID();
743
+ let grantCreated = false;
744
+
745
+ try {
746
+ createDelegatedExpansionGrant({
747
+ delegatedSessionKey: childSessionKey,
748
+ issuerSessionId: params.callerSessionKey || "main",
749
+ allowedConversationIds: [params.bucket.conversationId],
750
+ tokenCap: params.tokenCap,
751
+ ttlMs: params.delegatedWaitTimeoutMs + 30_000,
752
+ });
753
+ stampDelegatedExpansionContext({
754
+ sessionKey: childSessionKey,
755
+ requestId: params.requestId,
756
+ expansionDepth: params.childExpansionDepth,
757
+ originSessionKey: params.originSessionKey,
758
+ stampedBy: "lcm_expand_query",
759
+ });
760
+ grantCreated = true;
761
+
762
+ const response = (await params.deps.callGateway({
763
+ method: "agent",
764
+ params: {
765
+ message: task,
766
+ sessionKey: childSessionKey,
767
+ deliver: false,
768
+ lane: params.deps.agentLaneSubagent,
769
+ idempotencyKey: childIdem,
770
+ ...(provider ? { provider } : {}),
771
+ ...(model ? { model } : {}),
772
+ extraSystemPrompt: params.deps.buildSubagentSystemPrompt({
773
+ depth: 1,
774
+ maxDepth: 8,
775
+ taskSummary: "Run lcm_expand and return prompt-focused JSON answer",
776
+ }),
777
+ },
778
+ timeoutMs: GATEWAY_TIMEOUT_MS,
779
+ })) as { runId?: unknown; error?: unknown };
780
+
781
+ const runId = typeof response?.runId === "string" ? response.runId.trim() : "";
782
+ if (!runId) {
783
+ throw new Error(
784
+ formatExpansionFailure(response?.error ?? response)
785
+ || "Delegated expansion did not return a runId.",
786
+ );
787
+ }
788
+
789
+ const wait = (await params.deps.callGateway({
790
+ method: "agent.wait",
791
+ params: {
792
+ runId,
793
+ timeoutMs: params.delegatedWaitTimeoutMs,
794
+ },
795
+ timeoutMs: params.delegatedWaitTimeoutMs,
796
+ })) as { status?: string; error?: unknown };
797
+ const status = typeof wait?.status === "string" ? wait.status : "error";
798
+ if (status === "timeout") {
799
+ recordExpansionDelegationTelemetry({
800
+ deps: params.deps,
801
+ component: "lcm_expand_query",
802
+ event: "timeout",
803
+ requestId: params.requestId,
804
+ sessionKey: params.callerSessionKey,
805
+ expansionDepth: params.childExpansionDepth,
806
+ originSessionKey: params.originSessionKey,
807
+ runId,
808
+ });
809
+ throw new Error(
810
+ `lcm_expand_query timed out waiting for delegated expansion (${params.delegatedWaitTimeoutSeconds}s).`,
811
+ );
812
+ }
813
+ if (status !== "ok") {
814
+ throw new Error(formatExpansionFailure(wait?.error));
815
+ }
816
+
817
+ const replyPayload = (await params.deps.callGateway({
818
+ method: "sessions.get",
819
+ params: { key: childSessionKey, limit: 80 },
820
+ timeoutMs: GATEWAY_TIMEOUT_MS,
821
+ })) as { messages?: unknown[] };
822
+ const reply = params.deps.readLatestAssistantReply(
823
+ Array.isArray(replyPayload.messages) ? replyPayload.messages : [],
824
+ );
825
+ const parsed = parseDelegatedExpandQueryReply(reply, params.bucket.summaryIds.length);
826
+ if (!parsed.ok) {
827
+ throw new Error(parsed.error);
828
+ }
829
+ recordExpansionDelegationTelemetry({
830
+ deps: params.deps,
831
+ component: "lcm_expand_query",
832
+ event: "success",
833
+ requestId: params.requestId,
834
+ sessionKey: params.callerSessionKey,
835
+ expansionDepth: params.childExpansionDepth,
836
+ originSessionKey: params.originSessionKey,
837
+ runId,
838
+ });
839
+
840
+ return parsed.value;
841
+ } finally {
842
+ try {
843
+ await params.deps.callGateway({
844
+ method: "sessions.delete",
845
+ params: { key: childSessionKey, deleteTranscript: true },
846
+ timeoutMs: GATEWAY_TIMEOUT_MS,
847
+ });
848
+ } catch {
849
+ // Cleanup is best-effort.
850
+ }
851
+ if (grantCreated) {
852
+ revokeDelegatedExpansionGrantForSession(childSessionKey, { removeBinding: true });
853
+ }
854
+ clearDelegatedExpansionContext(childSessionKey);
855
+ }
856
+ };
857
+
858
+ if (!expansionProvider && !expansionModel) {
859
+ return await runDelegatedQuery();
860
+ }
861
+
862
+ try {
863
+ return await runDelegatedQuery(delegatedOverrideProvider, delegatedOverrideModel);
864
+ } catch (error) {
865
+ const failure = formatExpansionFailure(error);
866
+ params.deps.log.warn(
867
+ `[lcm] delegated expansion override failed (${configuredOverrideLabel}) for conversation ${params.bucket.conversationId}: ${failure}`,
868
+ );
869
+ if (!shouldRetryWithoutOverride(failure)) {
870
+ throw new Error(failure);
871
+ }
872
+ params.deps.log.warn(
873
+ `[lcm] retrying delegated expansion without provider/model override after: ${failure}`,
874
+ );
875
+ return await runDelegatedQuery();
876
+ }
877
+ }
878
+
879
+ /**
880
+ * Create the top-level lcm_expand_query tool wrapper for main-agent use.
881
+ */
445
882
  export function createLcmExpandQueryTool(input: {
446
883
  deps: LcmDependencies;
447
- lcm: LcmContextEngine;
884
+ lcm?: LcmContextEngine;
885
+ getLcm?: () => Promise<LcmContextEngine>;
448
886
  /** Session id used for LCM conversation scoping. */
449
887
  sessionId?: string;
450
888
  /** Requester agent session key used for delegated child session/auth scoping. */
@@ -460,11 +898,15 @@ export function createLcmExpandQueryTool(input: {
460
898
  name: "lcm_expand_query",
461
899
  label: "LCM Expand Query",
462
900
  description:
463
- "Answer a focused question using delegated LCM expansion. " +
464
- "Find candidate summaries (by IDs or query), expand them in a delegated sub-agent, " +
465
- "and return a compact prompt-focused answer with cited summary IDs.",
901
+ "Answer a focused natural-language question using delegated LCM expansion. " +
902
+ "Find candidate summaries (by IDs or a short FTS5 query that follows the same full-text rules as lcm_grep), expand them in a delegated sub-agent, " +
903
+ "and return a compact prompt-focused answer. Tool output includes cited summary IDs for follow-up.",
466
904
  parameters: LcmExpandQuerySchema,
467
905
  async execute(_toolCallId, params) {
906
+ const lcm = input.lcm ?? (await input.getLcm?.());
907
+ if (!lcm) {
908
+ throw new Error("LCM engine is unavailable.");
909
+ }
468
910
  const p = params as Record<string, unknown>;
469
911
  const explicitSummaryIds = normalizeSummaryIds(p.summaryIds as string[] | undefined);
470
912
  const query = typeof p.query === "string" ? p.query.trim() : "";
@@ -475,7 +917,8 @@ export function createLcmExpandQueryTool(input: {
475
917
  typeof requestedMaxTokens === "number" && Number.isFinite(requestedMaxTokens)
476
918
  ? Math.max(1, requestedMaxTokens)
477
919
  : DEFAULT_MAX_ANSWER_TOKENS;
478
- const requestedTokenCap = typeof p.tokenCap === "number" ? Math.trunc(p.tokenCap) : undefined;
920
+ const requestedTokenCap =
921
+ typeof p.tokenCap === "number" ? Math.trunc(p.tokenCap) : undefined;
479
922
  const expansionTokenCap =
480
923
  typeof requestedTokenCap === "number" && Number.isFinite(requestedTokenCap)
481
924
  ? Math.max(1, requestedTokenCap)
@@ -537,7 +980,7 @@ export function createLcmExpandQueryTool(input: {
537
980
 
538
981
  try {
539
982
  const conversationScope = await resolveLcmConversationScope({
540
- lcm: input.lcm,
983
+ lcm,
541
984
  deps: input.deps,
542
985
  sessionId: input.sessionId,
543
986
  sessionKey: input.sessionKey,
@@ -552,7 +995,7 @@ export function createLcmExpandQueryTool(input: {
552
995
  scopedConversationId = await resolveRequesterConversationScopeId({
553
996
  deps: input.deps,
554
997
  requesterSessionKey: callerSessionKey,
555
- lcm: input.lcm,
998
+ lcm,
556
999
  });
557
1000
  }
558
1001
 
@@ -564,7 +1007,7 @@ export function createLcmExpandQueryTool(input: {
564
1007
  }
565
1008
 
566
1009
  const candidates = await resolveSummaryCandidates({
567
- lcm: input.lcm,
1010
+ lcm,
568
1011
  explicitSummaryIds,
569
1012
  query: query || undefined,
570
1013
  conversationId: scopedConversationId,
@@ -576,41 +1019,19 @@ export function createLcmExpandQueryTool(input: {
576
1019
  error: "No matching summaries found.",
577
1020
  });
578
1021
  }
579
- return jsonResult({
580
- answer: "No matching summaries found for this scope.",
581
- citedIds: [],
582
- sourceConversationId: scopedConversationId,
583
- expandedSummaryCount: 0,
584
- totalSourceTokens: 0,
585
- truncated: false,
586
- });
1022
+ return jsonResult(
1023
+ buildExpandQueryReply({
1024
+ answer: "No matching summaries found for this scope.",
1025
+ citedIds: [],
1026
+ sourceConversationIds: [scopedConversationId],
1027
+ expandedSummaryCount: 0,
1028
+ totalSourceTokens: 0,
1029
+ truncated: false,
1030
+ }),
1031
+ );
587
1032
  }
588
1033
 
589
- const sourceConversationId = resolveSourceConversationId({
590
- scopedConversationId,
591
- allConversations: conversationScope.allConversations,
592
- candidates,
593
- });
594
- const summaryIds = normalizeSummaryIds(
595
- candidates
596
- .filter((candidate) => candidate.conversationId === sourceConversationId)
597
- .map((candidate) => candidate.summaryId),
598
- );
599
- const messageBackedSummaryIds = normalizeSummaryIds(
600
- candidates
601
- .filter(
602
- (candidate) =>
603
- candidate.conversationId === sourceConversationId &&
604
- candidate.requiresMessageExpansion,
605
- )
606
- .map((candidate) => candidate.summaryId),
607
- );
608
-
609
- if (summaryIds.length === 0) {
610
- return jsonResult({
611
- error: "No summaryIds available after applying conversation scope.",
612
- });
613
- }
1034
+ const conversationBuckets = buildConversationBuckets(candidates);
614
1035
 
615
1036
  const concurrencyCheck = acquireExpansionConcurrencySlot({
616
1037
  originSessionKey,
@@ -642,173 +1063,161 @@ export function createLcmExpandQueryTool(input: {
642
1063
  );
643
1064
  const childExpansionDepth = resolveNextExpansionDepth(callerSessionKey);
644
1065
 
645
- const task = buildDelegatedExpandQueryTask({
646
- summaryIds,
647
- messageBackedSummaryIds,
648
- conversationId: sourceConversationId,
649
- query: query || undefined,
650
- prompt,
651
- maxTokens,
652
- tokenCap: expansionTokenCap,
653
- requestId,
654
- expansionDepth: childExpansionDepth,
655
- originSessionKey,
656
- });
1066
+ if (!conversationScope.allConversations) {
1067
+ const sourceConversationId = resolveSourceConversationId({
1068
+ scopedConversationId,
1069
+ allConversations: conversationScope.allConversations,
1070
+ candidates,
1071
+ });
1072
+ const bucket = selectSingleConversationBucket({
1073
+ sourceConversationId,
1074
+ buckets: conversationBuckets,
1075
+ });
1076
+ const delegatedReply = await runDelegatedExpandQuery({
1077
+ deps: input.deps,
1078
+ callerSessionKey,
1079
+ requesterAgentId,
1080
+ bucket,
1081
+ query: query || undefined,
1082
+ prompt,
1083
+ maxTokens,
1084
+ tokenCap: expansionTokenCap,
1085
+ requestId,
1086
+ childExpansionDepth,
1087
+ originSessionKey,
1088
+ delegatedWaitTimeoutMs,
1089
+ delegatedWaitTimeoutSeconds,
1090
+ });
657
1091
 
658
- const expansionProvider = input.deps.config.expansionProvider || undefined;
659
- const expansionModel = input.deps.config.expansionModel || undefined;
660
- const canonicalExpansionModel = expansionModel?.includes("/") ? expansionModel : undefined;
661
- const delegatedOverrideProvider = canonicalExpansionModel ? undefined : expansionProvider;
662
- const delegatedOverrideModel = canonicalExpansionModel || expansionModel;
663
- const configuredOverrideLabel =
664
- delegatedOverrideProvider && delegatedOverrideModel
665
- ? `${delegatedOverrideProvider}/${delegatedOverrideModel}`
666
- : delegatedOverrideModel || delegatedOverrideProvider || "configured override";
667
-
668
- const runDelegatedQuery = async (provider?: string, model?: string) => {
669
- const childSessionKey = `agent:${requesterAgentId}:subagent:${crypto.randomUUID()}`;
670
- const childIdem = crypto.randomUUID();
671
- let grantCreated = false;
1092
+ return jsonResult(
1093
+ buildExpandQueryReply({
1094
+ answer: delegatedReply.answer,
1095
+ citedIds: delegatedReply.citedIds,
1096
+ sourceConversationIds: [sourceConversationId],
1097
+ expandedSummaryCount: delegatedReply.expandedSummaryCount,
1098
+ totalSourceTokens: delegatedReply.totalSourceTokens,
1099
+ truncated: delegatedReply.truncated,
1100
+ }),
1101
+ );
1102
+ }
672
1103
 
673
- try {
674
- createDelegatedExpansionGrant({
675
- delegatedSessionKey: childSessionKey,
676
- issuerSessionId: callerSessionKey || "main",
677
- allowedConversationIds: [sourceConversationId],
678
- tokenCap: expansionTokenCap,
679
- ttlMs: delegatedWaitTimeoutMs + 30_000,
680
- });
681
- stampDelegatedExpansionContext({
682
- sessionKey: childSessionKey,
683
- requestId,
684
- expansionDepth: childExpansionDepth,
685
- originSessionKey,
686
- stampedBy: "lcm_expand_query",
1104
+ const rankedBuckets = [...conversationBuckets].sort(compareConversationBuckets);
1105
+ const bucketResults: BucketExecutionResult[] = [];
1106
+ const bucketsToExpand = rankedBuckets.slice(0, DEFAULT_MAX_CONVERSATION_BUCKETS);
1107
+ const skippedBuckets = rankedBuckets.slice(DEFAULT_MAX_CONVERSATION_BUCKETS);
1108
+ let remainingTokenCap = expansionTokenCap;
1109
+ let firstFailure: string | undefined;
1110
+
1111
+ for (const bucket of bucketsToExpand) {
1112
+ if (remainingTokenCap <= 0) {
1113
+ bucketResults.push({
1114
+ conversationId: bucket.conversationId,
1115
+ status: "skipped",
1116
+ candidateCount: bucket.candidateCount,
1117
+ error: "global token budget exhausted",
687
1118
  });
688
- grantCreated = true;
689
-
690
- const response = (await input.deps.callGateway({
691
- method: "agent",
692
- params: {
693
- message: task,
694
- sessionKey: childSessionKey,
695
- deliver: false,
696
- lane: input.deps.agentLaneSubagent,
697
- idempotencyKey: childIdem,
698
- ...(provider ? { provider } : {}),
699
- ...(model ? { model } : {}),
700
- extraSystemPrompt: input.deps.buildSubagentSystemPrompt({
701
- depth: 1,
702
- maxDepth: 8,
703
- taskSummary: "Run lcm_expand and return prompt-focused JSON answer",
704
- }),
705
- },
706
- timeoutMs: GATEWAY_TIMEOUT_MS,
707
- })) as { runId?: unknown; error?: unknown };
708
-
709
- const runId = typeof response?.runId === "string" ? response.runId.trim() : "";
710
- if (!runId) {
711
- throw new Error(
712
- formatExpansionFailure(response?.error ?? response)
713
- || "Delegated expansion did not return a runId.",
714
- );
715
- }
716
-
717
- const wait = (await input.deps.callGateway({
718
- method: "agent.wait",
719
- params: {
720
- runId,
721
- timeoutMs: delegatedWaitTimeoutMs,
722
- },
723
- timeoutMs: delegatedWaitTimeoutMs,
724
- })) as { status?: string; error?: unknown };
725
- const status = typeof wait?.status === "string" ? wait.status : "error";
726
- if (status === "timeout") {
727
- recordExpansionDelegationTelemetry({
728
- deps: input.deps,
729
- component: "lcm_expand_query",
730
- event: "timeout",
731
- requestId,
732
- sessionKey: callerSessionKey,
733
- expansionDepth: childExpansionDepth,
734
- originSessionKey,
735
- runId,
736
- });
737
- throw new Error(
738
- `lcm_expand_query timed out waiting for delegated expansion (${delegatedWaitTimeoutSeconds}s).`,
739
- );
740
- }
741
- if (status !== "ok") {
742
- throw new Error(formatExpansionFailure(wait?.error));
743
- }
744
-
745
- const replyPayload = (await input.deps.callGateway({
746
- method: "sessions.get",
747
- params: { key: childSessionKey, limit: 80 },
748
- timeoutMs: GATEWAY_TIMEOUT_MS,
749
- })) as { messages?: unknown[] };
750
- const reply = input.deps.readLatestAssistantReply(
751
- Array.isArray(replyPayload.messages) ? replyPayload.messages : [],
752
- );
753
- const parsed = parseDelegatedExpandQueryReply(reply, summaryIds.length);
754
- if (!parsed.ok) {
755
- throw new Error(parsed.error);
756
- }
757
- recordExpansionDelegationTelemetry({
1119
+ continue;
1120
+ }
1121
+
1122
+ try {
1123
+ const delegatedReply = await runDelegatedExpandQuery({
758
1124
  deps: input.deps,
759
- component: "lcm_expand_query",
760
- event: "success",
1125
+ callerSessionKey,
1126
+ requesterAgentId,
1127
+ bucket,
1128
+ query: query || undefined,
1129
+ prompt,
1130
+ maxTokens,
1131
+ tokenCap: remainingTokenCap,
761
1132
  requestId,
762
- sessionKey: callerSessionKey,
763
- expansionDepth: childExpansionDepth,
1133
+ childExpansionDepth,
764
1134
  originSessionKey,
765
- runId,
1135
+ delegatedWaitTimeoutMs,
1136
+ delegatedWaitTimeoutSeconds,
766
1137
  });
767
-
768
- return jsonResult({
769
- answer: parsed.value.answer,
770
- citedIds: parsed.value.citedIds,
771
- sourceConversationId,
772
- expandedSummaryCount: parsed.value.expandedSummaryCount,
773
- totalSourceTokens: parsed.value.totalSourceTokens,
774
- truncated: parsed.value.truncated,
1138
+ bucketResults.push({
1139
+ conversationId: bucket.conversationId,
1140
+ status: "success",
1141
+ candidateCount: bucket.candidateCount,
1142
+ reply: delegatedReply,
1143
+ });
1144
+ remainingTokenCap = Math.max(
1145
+ 0,
1146
+ remainingTokenCap - Math.max(0, delegatedReply.totalSourceTokens),
1147
+ );
1148
+ } catch (error) {
1149
+ const failure = formatExpansionFailure(error);
1150
+ firstFailure ??= failure;
1151
+ bucketResults.push({
1152
+ conversationId: bucket.conversationId,
1153
+ status: "failed",
1154
+ candidateCount: bucket.candidateCount,
1155
+ error: failure,
775
1156
  });
776
- } finally {
777
- try {
778
- await input.deps.callGateway({
779
- method: "sessions.delete",
780
- params: { key: childSessionKey, deleteTranscript: true },
781
- timeoutMs: GATEWAY_TIMEOUT_MS,
782
- });
783
- } catch {
784
- // Cleanup is best-effort.
785
- }
786
- if (grantCreated) {
787
- revokeDelegatedExpansionGrantForSession(childSessionKey, { removeBinding: true });
788
- }
789
- clearDelegatedExpansionContext(childSessionKey);
790
1157
  }
791
- };
1158
+ }
1159
+
1160
+ for (const bucket of skippedBuckets) {
1161
+ bucketResults.push({
1162
+ conversationId: bucket.conversationId,
1163
+ status: "skipped",
1164
+ candidateCount: bucket.candidateCount,
1165
+ error: `skipped after reaching max conversation bucket limit (${DEFAULT_MAX_CONVERSATION_BUCKETS})`,
1166
+ });
1167
+ }
792
1168
 
793
- if (!expansionProvider && !expansionModel) {
794
- return await runDelegatedQuery();
1169
+ const successfulResults = bucketResults.filter(
1170
+ (result): result is Extract<BucketExecutionResult, { status: "success" }> =>
1171
+ result.status === "success",
1172
+ );
1173
+ if (successfulResults.length === 0) {
1174
+ throw new Error(firstFailure ?? "Delegated expansion query failed.");
795
1175
  }
796
1176
 
797
- try {
798
- return await runDelegatedQuery(delegatedOverrideProvider, delegatedOverrideModel);
799
- } catch (error) {
800
- const failure = formatExpansionFailure(error);
801
- input.deps.log.warn(
802
- `[lcm] delegated expansion override failed (${configuredOverrideLabel}): ${failure}`,
803
- );
804
- if (!shouldRetryWithoutOverride(failure)) {
805
- throw new Error(failure);
1177
+ const conversationBreakdown: ConversationBreakdown[] = bucketResults.map((result) => {
1178
+ if (result.status === "success") {
1179
+ return {
1180
+ conversationId: result.conversationId,
1181
+ expandedSummaryCount: result.reply.expandedSummaryCount,
1182
+ citedIds: result.reply.citedIds,
1183
+ totalSourceTokens: result.reply.totalSourceTokens,
1184
+ truncated: result.reply.truncated,
1185
+ status: "success",
1186
+ };
806
1187
  }
807
- input.deps.log.warn(
808
- `[lcm] retrying delegated expansion without provider/model override after: ${failure}`,
809
- );
810
- return await runDelegatedQuery();
811
- }
1188
+ return {
1189
+ conversationId: result.conversationId,
1190
+ expandedSummaryCount: 0,
1191
+ citedIds: [],
1192
+ totalSourceTokens: 0,
1193
+ truncated: true,
1194
+ status: result.status,
1195
+ error: result.error,
1196
+ };
1197
+ });
1198
+
1199
+ return jsonResult(
1200
+ buildExpandQueryReply({
1201
+ answer: synthesizeConversationAnswers({
1202
+ prompt,
1203
+ results: bucketResults,
1204
+ }),
1205
+ citedIds: successfulResults.flatMap((result) => result.reply.citedIds),
1206
+ sourceConversationIds: successfulResults.map((result) => result.conversationId),
1207
+ expandedSummaryCount: successfulResults.reduce(
1208
+ (total, result) => total + result.reply.expandedSummaryCount,
1209
+ 0,
1210
+ ),
1211
+ totalSourceTokens: successfulResults.reduce(
1212
+ (total, result) => total + result.reply.totalSourceTokens,
1213
+ 0,
1214
+ ),
1215
+ truncated:
1216
+ successfulResults.some((result) => result.reply.truncated)
1217
+ || bucketResults.some((result) => result.status !== "success"),
1218
+ conversationBreakdown,
1219
+ }),
1220
+ );
812
1221
  } catch (error) {
813
1222
  const failure = formatExpansionFailure(error);
814
1223
  input.deps.log.error(`[lcm] delegated expansion query failed: ${failure}`);