@martian-engineering/lossless-claw 0.8.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +8 -0
  2. package/dist/index.js +19240 -0
  3. package/docs/configuration.md +15 -5
  4. package/openclaw.plugin.json +27 -3
  5. package/package.json +7 -6
  6. package/skills/lossless-claw/references/config.md +37 -0
  7. package/index.ts +0 -2
  8. package/src/assembler.ts +0 -1196
  9. package/src/compaction.ts +0 -1753
  10. package/src/db/config.ts +0 -345
  11. package/src/db/connection.ts +0 -151
  12. package/src/db/features.ts +0 -61
  13. package/src/db/migration.ts +0 -868
  14. package/src/engine.ts +0 -4486
  15. package/src/estimate-tokens.ts +0 -80
  16. package/src/expansion-auth.ts +0 -365
  17. package/src/expansion-policy.ts +0 -303
  18. package/src/expansion.ts +0 -383
  19. package/src/integrity.ts +0 -600
  20. package/src/large-files.ts +0 -546
  21. package/src/lcm-log.ts +0 -37
  22. package/src/openclaw-bridge.ts +0 -22
  23. package/src/plugin/index.ts +0 -2037
  24. package/src/plugin/lcm-command.ts +0 -1040
  25. package/src/plugin/lcm-doctor-apply.ts +0 -540
  26. package/src/plugin/lcm-doctor-cleaners.ts +0 -655
  27. package/src/plugin/lcm-doctor-shared.ts +0 -210
  28. package/src/plugin/shared-init.ts +0 -59
  29. package/src/prune.ts +0 -391
  30. package/src/retrieval.ts +0 -360
  31. package/src/session-patterns.ts +0 -23
  32. package/src/startup-banner-log.ts +0 -49
  33. package/src/store/compaction-telemetry-store.ts +0 -156
  34. package/src/store/conversation-store.ts +0 -929
  35. package/src/store/fts5-sanitize.ts +0 -50
  36. package/src/store/full-text-fallback.ts +0 -83
  37. package/src/store/full-text-sort.ts +0 -21
  38. package/src/store/index.ts +0 -39
  39. package/src/store/parse-utc-timestamp.ts +0 -25
  40. package/src/store/summary-store.ts +0 -1519
  41. package/src/summarize.ts +0 -1508
  42. package/src/tools/common.ts +0 -53
  43. package/src/tools/lcm-conversation-scope.ts +0 -127
  44. package/src/tools/lcm-describe-tool.ts +0 -245
  45. package/src/tools/lcm-expand-query-tool.ts +0 -1235
  46. package/src/tools/lcm-expand-tool.delegation.ts +0 -580
  47. package/src/tools/lcm-expand-tool.ts +0 -453
  48. package/src/tools/lcm-expansion-recursion-guard.ts +0 -373
  49. package/src/tools/lcm-grep-tool.ts +0 -228
  50. package/src/transaction-mutex.ts +0 -136
  51. package/src/transcript-repair.ts +0 -301
  52. package/src/types.ts +0 -165
package/src/engine.ts DELETED
@@ -1,4486 +0,0 @@
1
- import { createHash, randomUUID } from "node:crypto";
2
- import { closeSync, createReadStream, openSync, readSync, statSync } from "node:fs";
3
- import { mkdir, writeFile } from "node:fs/promises";
4
- import { homedir } from "node:os";
5
- import { join } from "node:path";
6
- import type { DatabaseSync } from "node:sqlite";
7
- import { createInterface } from "node:readline";
8
- import { SessionManager } from "@mariozechner/pi-coding-agent";
9
- import type {
10
- ContextEngine,
11
- ContextEngineInfo,
12
- AssembleResult,
13
- BootstrapResult,
14
- CompactResult,
15
- IngestBatchResult,
16
- IngestResult,
17
- SubagentEndReason,
18
- SubagentSpawnPreparation,
19
- } from "openclaw/plugin-sdk";
20
- import {
21
- blockFromPart,
22
- contentFromParts,
23
- ContextAssembler,
24
- pickToolCallId,
25
- pickToolIsError,
26
- pickToolName,
27
- } from "./assembler.js";
28
- import { CompactionEngine, type CompactionConfig } from "./compaction.js";
29
- import type { LcmConfig } from "./db/config.js";
30
- import { getLcmDbFeatures } from "./db/features.js";
31
- import { runLcmMigrations } from "./db/migration.js";
32
- import {
33
- createDelegatedExpansionGrant,
34
- getRuntimeExpansionAuthManager,
35
- removeDelegatedExpansionGrantForSession,
36
- resolveDelegatedExpansionGrantId,
37
- revokeDelegatedExpansionGrantForSession,
38
- } from "./expansion-auth.js";
39
- import {
40
- extensionFromNameOrMime,
41
- formatFileReference,
42
- formatToolOutputReference,
43
- generateExplorationSummary,
44
- parseFileBlocks,
45
- } from "./large-files.js";
46
- import { describeLogError } from "./lcm-log.js";
47
- import { RetrievalEngine } from "./retrieval.js";
48
- import { compileSessionPatterns, matchesSessionPattern } from "./session-patterns.js";
49
- import { logStartupBannerOnce } from "./startup-banner-log.js";
50
- import {
51
- CompactionTelemetryStore,
52
- type ConversationCompactionTelemetryRecord,
53
- type CacheState,
54
- type ActivityBand,
55
- } from "./store/compaction-telemetry-store.js";
56
- import {
57
- ConversationStore,
58
- type ConversationRecord,
59
- type CreateMessagePartInput,
60
- type MessagePartRecord,
61
- type MessagePartType,
62
- } from "./store/conversation-store.js";
63
- import { SummaryStore } from "./store/summary-store.js";
64
- import { createLcmSummarizeFromLegacyParams, LcmProviderAuthError } from "./summarize.js";
65
- import type { LcmDependencies } from "./types.js";
66
- import { estimateTokens } from "./estimate-tokens.js";
67
-
68
- type AgentMessage = Parameters<ContextEngine["ingest"]>[0]["message"];
69
- type AssembleResultWithSystemPrompt = AssembleResult & { systemPromptAddition?: string };
70
- type CircuitBreakerState = {
71
- failures: number;
72
- openSince: number | null;
73
- };
74
- type PromptCacheSnapshot = {
75
- lastObservedCacheRead?: number;
76
- lastObservedCacheWrite?: number;
77
- cacheState: CacheState;
78
- retention?: string;
79
- sawExplicitBreak: boolean;
80
- };
81
- type IncrementalCompactionDecision = {
82
- shouldCompact: boolean;
83
- cacheState: CacheState;
84
- maxPasses: number;
85
- rawTokensOutsideTail: number;
86
- threshold: number;
87
- leafChunkTokens: number;
88
- fallbackLeafChunkTokens: number[];
89
- activityBand: ActivityBand;
90
- allowCondensedPasses: boolean;
91
- };
92
- type DynamicLeafChunkBounds = {
93
- floor: number;
94
- medium: number;
95
- high: number;
96
- max: number;
97
- };
98
- type TranscriptRewriteReplacement = {
99
- entryId: string;
100
- message: AgentMessage;
101
- };
102
- type TranscriptRewriteRequest = {
103
- replacements: TranscriptRewriteReplacement[];
104
- };
105
- type ContextEngineMaintenanceResult = {
106
- changed: boolean;
107
- bytesFreed: number;
108
- rewrittenEntries: number;
109
- reason?: string;
110
- };
111
- type ContextEngineMaintenanceRuntimeContext = Record<string, unknown> & {
112
- rewriteTranscriptEntries?: (
113
- request: TranscriptRewriteRequest,
114
- ) => Promise<ContextEngineMaintenanceResult>;
115
- };
116
-
117
- const TRANSCRIPT_GC_BATCH_SIZE = 12;
118
- const HOT_CACHE_HYSTERESIS_TURNS = 2;
119
- const DYNAMIC_LEAF_CHUNK_MEDIUM_MULTIPLIER = 1.5;
120
- const DYNAMIC_LEAF_CHUNK_HIGH_MULTIPLIER = 2;
121
- const DYNAMIC_ACTIVITY_MEDIUM_UPSHIFT_FACTOR = 0.5;
122
- const DYNAMIC_ACTIVITY_MEDIUM_DOWNSHIFT_FACTOR = 0.35;
123
- const DYNAMIC_ACTIVITY_HIGH_UPSHIFT_FACTOR = 1.0;
124
- const DYNAMIC_ACTIVITY_HIGH_DOWNSHIFT_FACTOR = 0.75;
125
-
126
- // ── Helpers ──────────────────────────────────────────────────────────────────
127
-
128
- function toJson(value: unknown): string {
129
- const encoded = JSON.stringify(value);
130
- return typeof encoded === "string" ? encoded : "";
131
- }
132
-
133
- function safeString(value: unknown): string | undefined {
134
- return typeof value === "string" ? value : undefined;
135
- }
136
-
137
- function formatDurationMs(durationMs: number): string {
138
- return `${durationMs}ms`;
139
- }
140
-
141
- function asRecord(value: unknown): Record<string, unknown> | undefined {
142
- return value && typeof value === "object" && !Array.isArray(value)
143
- ? (value as Record<string, unknown>)
144
- : undefined;
145
- }
146
-
147
- function safeBoolean(value: unknown): boolean | undefined {
148
- return typeof value === "boolean" ? value : undefined;
149
- }
150
-
151
- function extractTranscriptToolCallId(message: AgentMessage): string | undefined {
152
- const topLevel = message as Record<string, unknown>;
153
- const direct =
154
- safeString(topLevel.toolCallId) ??
155
- safeString(topLevel.tool_call_id) ??
156
- safeString(topLevel.toolUseId) ??
157
- safeString(topLevel.tool_use_id) ??
158
- safeString(topLevel.call_id) ??
159
- safeString(topLevel.id);
160
- if (direct) {
161
- return direct;
162
- }
163
-
164
- if (!Array.isArray(topLevel.content)) {
165
- return undefined;
166
- }
167
-
168
- for (const item of topLevel.content) {
169
- const record = asRecord(item);
170
- if (!record) {
171
- continue;
172
- }
173
- const nested =
174
- safeString(record.toolCallId) ??
175
- safeString(record.tool_call_id) ??
176
- safeString(record.toolUseId) ??
177
- safeString(record.tool_use_id) ??
178
- safeString(record.call_id) ??
179
- safeString(record.id);
180
- if (nested) {
181
- return nested;
182
- }
183
- }
184
-
185
- return undefined;
186
- }
187
-
188
- function listTranscriptToolResultEntryIdsByCallId(sessionFile: string): Map<string, string> {
189
- const sessionManager = SessionManager.open(sessionFile);
190
- const branch = sessionManager.getBranch();
191
- const entryIdsByCallId = new Map<string, string>();
192
- const duplicateCallIds = new Set<string>();
193
-
194
- for (const entry of branch) {
195
- if (entry.type !== "message" || entry.message.role !== "toolResult") {
196
- continue;
197
- }
198
- const toolCallId = extractTranscriptToolCallId(entry.message as AgentMessage);
199
- if (!toolCallId) {
200
- continue;
201
- }
202
- if (entryIdsByCallId.has(toolCallId)) {
203
- duplicateCallIds.add(toolCallId);
204
- continue;
205
- }
206
- entryIdsByCallId.set(toolCallId, entry.id);
207
- }
208
-
209
- for (const duplicateCallId of duplicateCallIds) {
210
- entryIdsByCallId.delete(duplicateCallId);
211
- }
212
-
213
- return entryIdsByCallId;
214
- }
215
-
216
- function appendTextValue(value: unknown, out: string[]): void {
217
- if (typeof value === "string") {
218
- out.push(value);
219
- return;
220
- }
221
- if (Array.isArray(value)) {
222
- for (const entry of value) {
223
- appendTextValue(entry, out);
224
- }
225
- return;
226
- }
227
- if (!value || typeof value !== "object") {
228
- return;
229
- }
230
-
231
- const record = value as Record<string, unknown>;
232
- appendTextValue(record.text, out);
233
- appendTextValue(record.value, out);
234
- }
235
-
236
- const STRUCTURED_TEXT_FIELD_KEYS = ["text", "transcript", "transcription", "message", "summary"];
237
- const STRUCTURED_ARRAY_FIELD_KEYS = [
238
- "segments",
239
- "utterances",
240
- "paragraphs",
241
- "alternatives",
242
- "words",
243
- "items",
244
- "results",
245
- ];
246
- const STRUCTURED_NESTED_FIELD_KEYS = ["content", "output", "result", "payload", "data", "value"];
247
- const MAX_STRUCTURED_TEXT_DEPTH = 6;
248
- const TOOL_RAW_TYPES: ReadonlySet<string> = new Set([
249
- "tool_use",
250
- "toolUse",
251
- "tool-use",
252
- "toolCall",
253
- "tool_call",
254
- "functionCall",
255
- "function_call",
256
- "function_call_output",
257
- "tool_result",
258
- "toolResult",
259
- "tool_use_result",
260
- ]);
261
-
262
- function looksLikeJsonPayload(value: string): boolean {
263
- const trimmed = value.trim();
264
- if (!trimmed) {
265
- return false;
266
- }
267
- return (
268
- (trimmed.startsWith("{") && trimmed.endsWith("}")) ||
269
- (trimmed.startsWith("[") && trimmed.endsWith("]"))
270
- );
271
- }
272
-
273
- function extractStructuredText(value: unknown, depth: number = 0): string | undefined {
274
- if (value == null || depth > MAX_STRUCTURED_TEXT_DEPTH) {
275
- return undefined;
276
- }
277
- if (typeof value === "string") {
278
- if (looksLikeJsonPayload(value)) {
279
- try {
280
- const parsed = JSON.parse(value.trim());
281
- const parsedText = extractStructuredText(parsed, depth + 1);
282
- if (typeof parsedText === "string" && parsedText.length > 0) {
283
- return parsedText;
284
- }
285
- } catch {
286
- // Fall through to returning the original string when parsing fails.
287
- }
288
- }
289
- return value;
290
- }
291
- if (Array.isArray(value)) {
292
- const texts: string[] = [];
293
- for (const entry of value) {
294
- const text = extractStructuredText(entry, depth + 1);
295
- if (typeof text === "string" && text.trim().length > 0) {
296
- texts.push(text);
297
- }
298
- }
299
- return texts.length > 0 ? texts.join("\n") : undefined;
300
- }
301
- if (typeof value !== "object") {
302
- return undefined;
303
- }
304
-
305
- const record = value as Record<string, unknown>;
306
-
307
- // Skip tool call/result objects — their structured data belongs in the parts table, not content
308
- if (typeof record.type === "string" && TOOL_RAW_TYPES.has(record.type)) {
309
- if (safeBoolean(record.toolOutputExternalized)) {
310
- const externalizedText =
311
- extractStructuredText(record.output, depth + 1) ??
312
- extractStructuredText(record.content, depth + 1) ??
313
- extractStructuredText(record.result, depth + 1);
314
- if (typeof externalizedText === "string" && externalizedText.trim().length > 0) {
315
- return externalizedText;
316
- }
317
- }
318
- return undefined;
319
- }
320
-
321
- for (const key of STRUCTURED_TEXT_FIELD_KEYS) {
322
- const candidate = record[key];
323
- if (typeof candidate === "string" && candidate.trim().length > 0) {
324
- return candidate;
325
- }
326
- }
327
-
328
- for (const key of STRUCTURED_ARRAY_FIELD_KEYS) {
329
- const candidate = record[key];
330
- if (Array.isArray(candidate)) {
331
- const texts: string[] = [];
332
- for (const entry of candidate) {
333
- const text = extractStructuredText(entry, depth + 1);
334
- if (typeof text === "string" && text.trim().length > 0) {
335
- texts.push(text);
336
- }
337
- }
338
- if (texts.length > 0) {
339
- return texts.join("\n");
340
- }
341
- }
342
- }
343
-
344
- for (const key of STRUCTURED_NESTED_FIELD_KEYS) {
345
- const nested = record[key];
346
- const nestedText = extractStructuredText(nested, depth + 1);
347
- if (typeof nestedText === "string" && nestedText.trim().length > 0) {
348
- return nestedText;
349
- }
350
- }
351
-
352
- return undefined;
353
- }
354
-
355
- function extractReasoningText(record: Record<string, unknown>): string | undefined {
356
- const chunks: string[] = [];
357
- appendTextValue(record.summary, chunks);
358
- if (chunks.length === 0) {
359
- return undefined;
360
- }
361
-
362
- const normalized = chunks
363
- .map((chunk) => chunk.trim())
364
- .filter((chunk, idx, arr) => chunk.length > 0 && arr.indexOf(chunk) === idx);
365
- return normalized.length > 0 ? normalized.join("\n") : undefined;
366
- }
367
-
368
- function normalizeUnknownBlock(value: unknown): {
369
- type: string;
370
- text?: string;
371
- metadata: Record<string, unknown>;
372
- } {
373
- if (!value || typeof value !== "object" || Array.isArray(value)) {
374
- return {
375
- type: "agent",
376
- metadata: { raw: value },
377
- };
378
- }
379
-
380
- const record = value as Record<string, unknown>;
381
- const rawType = safeString(record.type);
382
- return {
383
- type: rawType ?? "agent",
384
- text:
385
- safeString(record.text) ??
386
- safeString(record.thinking) ??
387
- ((rawType === "reasoning" || rawType === "thinking")
388
- ? extractReasoningText(record)
389
- : undefined),
390
- metadata: { raw: record },
391
- };
392
- }
393
-
394
- function toPartType(type: string): MessagePartType {
395
- switch (type) {
396
- case "text":
397
- return "text";
398
- case "thinking":
399
- case "reasoning":
400
- return "reasoning";
401
- case "tool_use":
402
- case "toolUse":
403
- case "tool-use":
404
- case "toolCall":
405
- case "functionCall":
406
- case "function_call":
407
- case "function_call_output":
408
- case "tool_result":
409
- case "toolResult":
410
- case "tool":
411
- return "tool";
412
- case "patch":
413
- return "patch";
414
- case "file":
415
- case "image":
416
- return "file";
417
- case "subtask":
418
- return "subtask";
419
- case "compaction":
420
- return "compaction";
421
- case "step_start":
422
- case "step-start":
423
- return "step_start";
424
- case "step_finish":
425
- case "step-finish":
426
- return "step_finish";
427
- case "snapshot":
428
- return "snapshot";
429
- case "retry":
430
- return "retry";
431
- case "agent":
432
- return "agent";
433
- default:
434
- return "agent";
435
- }
436
- }
437
-
438
- /**
439
- * Convert AgentMessage content into plain text for DB storage.
440
- *
441
- * For content block arrays we keep only text blocks to avoid persisting raw
442
- * JSON syntax that can later pollute assembled model context.
443
- */
444
- function extractMessageContent(content: unknown): string {
445
- const extracted = extractStructuredText(content);
446
- if (typeof extracted === "string") {
447
- return extracted;
448
- }
449
- if (content == null) {
450
- return "";
451
- }
452
- if (Array.isArray(content) && content.length === 0) {
453
- return "";
454
- }
455
- // If content is an array of only tool call/result objects, store as empty
456
- // (structured data is preserved in the message parts table)
457
- if (Array.isArray(content) && content.length > 0 && content.every(
458
- (item) => typeof item === "object" && item !== null && !Array.isArray(item) &&
459
- typeof (item as Record<string, unknown>).type === "string" &&
460
- TOOL_RAW_TYPES.has((item as Record<string, unknown>).type as string)
461
- )) {
462
- return "";
463
- }
464
-
465
- const serialized = JSON.stringify(content);
466
- return typeof serialized === "string" ? serialized : "";
467
- }
468
-
469
- function toRuntimeRoleForTokenEstimate(role: string): "user" | "assistant" | "toolResult" {
470
- if (role === "tool" || role === "toolResult") {
471
- return "toolResult";
472
- }
473
- if (role === "user" || role === "system") {
474
- return "user";
475
- }
476
- return "assistant";
477
- }
478
-
479
- function isTextBlock(value: unknown): value is { type: "text"; text: string } {
480
- if (!value || typeof value !== "object" || Array.isArray(value)) {
481
- return false;
482
- }
483
- const record = value as Record<string, unknown>;
484
- return record.type === "text" && typeof record.text === "string";
485
- }
486
-
487
- function toSyntheticMessagePartRecord(
488
- part: CreateMessagePartInput,
489
- messageId: number,
490
- ): MessagePartRecord {
491
- return {
492
- partId: `estimate-part-${part.ordinal}`,
493
- messageId,
494
- sessionId: part.sessionId,
495
- partType: part.partType,
496
- ordinal: part.ordinal,
497
- textContent: part.textContent ?? null,
498
- toolCallId: part.toolCallId ?? null,
499
- toolName: part.toolName ?? null,
500
- toolInput: part.toolInput ?? null,
501
- toolOutput: part.toolOutput ?? null,
502
- metadata: part.metadata ?? null,
503
- };
504
- }
505
-
506
- function normalizeMessageContentForStorage(params: {
507
- message: AgentMessage;
508
- fallbackContent: string;
509
- }): unknown {
510
- const { message, fallbackContent } = params;
511
- if (!("content" in message)) {
512
- return fallbackContent;
513
- }
514
-
515
- const role = toRuntimeRoleForTokenEstimate(message.role);
516
- const parts = buildMessageParts({
517
- sessionId: "storage-estimate",
518
- message,
519
- fallbackContent,
520
- }).map((part) => toSyntheticMessagePartRecord(part, 0));
521
-
522
- if (parts.length === 0) {
523
- if (role === "assistant") {
524
- return fallbackContent ? [{ type: "text", text: fallbackContent }] : [];
525
- }
526
- if (role === "toolResult") {
527
- return [{ type: "text", text: fallbackContent }];
528
- }
529
- return fallbackContent;
530
- }
531
-
532
- const blocks = parts.map(blockFromPart);
533
- if (role === "user" && blocks.length === 1 && isTextBlock(blocks[0])) {
534
- return blocks[0].text;
535
- }
536
- return blocks;
537
- }
538
-
539
- /**
540
- * Estimate token usage for the content shape that the assembler will emit.
541
- *
542
- * LCM stores a plain-text fallback copy in messages.content, but message_parts
543
- * can rehydrate larger structured/raw blocks. This estimator mirrors the
544
- * rehydrated shape so compaction decisions use realistic token totals.
545
- */
546
- function estimateContentTokensForRole(params: {
547
- role: "user" | "assistant" | "toolResult";
548
- content: unknown;
549
- fallbackContent: string;
550
- }): number {
551
- const { role, content, fallbackContent } = params;
552
-
553
- if (typeof content === "string") {
554
- return estimateTokens(content);
555
- }
556
-
557
- if (Array.isArray(content)) {
558
- if (content.length === 0) {
559
- return estimateTokens(fallbackContent);
560
- }
561
-
562
- if (role === "user" && content.length === 1 && isTextBlock(content[0])) {
563
- return estimateTokens(content[0].text);
564
- }
565
-
566
- const serialized = JSON.stringify(content);
567
- return estimateTokens(typeof serialized === "string" ? serialized : "");
568
- }
569
-
570
- if (content && typeof content === "object") {
571
- if (role === "user" && isTextBlock(content)) {
572
- return estimateTokens(content.text);
573
- }
574
-
575
- const serialized = JSON.stringify([content]);
576
- return estimateTokens(typeof serialized === "string" ? serialized : "");
577
- }
578
-
579
- return estimateTokens(fallbackContent);
580
- }
581
-
582
- function buildMessageParts(params: {
583
- sessionId: string;
584
- message: AgentMessage;
585
- fallbackContent: string;
586
- }): import("./store/conversation-store.js").CreateMessagePartInput[] {
587
- const { sessionId, message, fallbackContent } = params;
588
- const role = typeof message.role === "string" ? message.role : "unknown";
589
- const topLevel = message as unknown as Record<string, unknown>;
590
- const topLevelToolCallId =
591
- safeString(topLevel.toolCallId) ??
592
- safeString(topLevel.tool_call_id) ??
593
- safeString(topLevel.toolUseId) ??
594
- safeString(topLevel.tool_use_id) ??
595
- safeString(topLevel.call_id) ??
596
- safeString(topLevel.id);
597
- const topLevelToolName =
598
- safeString(topLevel.toolName) ??
599
- safeString(topLevel.tool_name);
600
- const topLevelIsError =
601
- safeBoolean(topLevel.isError) ??
602
- safeBoolean(topLevel.is_error);
603
-
604
- // BashExecutionMessage: preserve a synthetic text part so output is round-trippable.
605
- if (!("content" in message) && "command" in message && "output" in message) {
606
- return [
607
- {
608
- sessionId,
609
- partType: "text",
610
- ordinal: 0,
611
- textContent: fallbackContent,
612
- metadata: toJson({
613
- originalRole: role,
614
- source: "bash-exec",
615
- command: safeString((message as { command?: unknown }).command),
616
- }),
617
- },
618
- ];
619
- }
620
-
621
- if (!("content" in message)) {
622
- return [
623
- {
624
- sessionId,
625
- partType: "agent",
626
- ordinal: 0,
627
- textContent: fallbackContent || null,
628
- metadata: toJson({
629
- originalRole: role,
630
- source: "unknown-message-shape",
631
- raw: message,
632
- }),
633
- },
634
- ];
635
- }
636
-
637
- if (typeof message.content === "string") {
638
- return [
639
- {
640
- sessionId,
641
- partType: "text",
642
- ordinal: 0,
643
- textContent: message.content,
644
- metadata: toJson({
645
- originalRole: role,
646
- toolCallId: topLevelToolCallId,
647
- toolName: topLevelToolName,
648
- isError: topLevelIsError,
649
- }),
650
- },
651
- ];
652
- }
653
-
654
- if (!Array.isArray(message.content)) {
655
- return [
656
- {
657
- sessionId,
658
- partType: "agent",
659
- ordinal: 0,
660
- textContent: fallbackContent || null,
661
- metadata: toJson({
662
- originalRole: role,
663
- source: "non-array-content",
664
- raw: message.content,
665
- }),
666
- },
667
- ];
668
- }
669
-
670
- const parts: CreateMessagePartInput[] = [];
671
- for (let ordinal = 0; ordinal < message.content.length; ordinal++) {
672
- const block = normalizeUnknownBlock(message.content[ordinal]);
673
- const metadataRecord = block.metadata.raw as Record<string, unknown> | undefined;
674
- const rawBlockType = safeString(metadataRecord?.rawType) ?? block.type;
675
- const partType = toPartType(rawBlockType);
676
- const rawBlock =
677
- metadataRecord && rawBlockType !== block.type
678
- ? {
679
- ...metadataRecord,
680
- type: rawBlockType,
681
- }
682
- : (metadataRecord ?? message.content[ordinal]);
683
- const toolCallId =
684
- safeString(metadataRecord?.toolCallId) ??
685
- safeString(metadataRecord?.tool_call_id) ??
686
- safeString(metadataRecord?.toolUseId) ??
687
- safeString(metadataRecord?.tool_use_id) ??
688
- safeString(metadataRecord?.call_id) ??
689
- (partType === "tool" ? safeString(metadataRecord?.id) : undefined) ??
690
- topLevelToolCallId;
691
-
692
- parts.push({
693
- sessionId,
694
- partType,
695
- ordinal,
696
- textContent: block.text ?? null,
697
- toolCallId,
698
- toolName:
699
- safeString(metadataRecord?.name) ??
700
- safeString(metadataRecord?.toolName) ??
701
- safeString(metadataRecord?.tool_name) ??
702
- topLevelToolName,
703
- toolInput:
704
- metadataRecord?.input !== undefined
705
- ? toJson(metadataRecord.input)
706
- : metadataRecord?.arguments !== undefined
707
- ? toJson(metadataRecord.arguments)
708
- : metadataRecord?.toolInput !== undefined
709
- ? toJson(metadataRecord.toolInput)
710
- : (safeString(metadataRecord?.tool_input) ?? null),
711
- toolOutput:
712
- metadataRecord?.output !== undefined
713
- ? toJson(metadataRecord.output)
714
- : metadataRecord?.toolOutput !== undefined
715
- ? toJson(metadataRecord.toolOutput)
716
- : (safeString(metadataRecord?.tool_output) ?? null),
717
- metadata: toJson({
718
- originalRole: role,
719
- toolCallId: topLevelToolCallId,
720
- toolName: topLevelToolName,
721
- isError: topLevelIsError,
722
- externalizedFileId: safeString(metadataRecord?.externalizedFileId),
723
- originalByteSize:
724
- typeof metadataRecord?.originalByteSize === "number"
725
- ? metadataRecord.originalByteSize
726
- : undefined,
727
- toolOutputExternalized: safeBoolean(metadataRecord?.toolOutputExternalized),
728
- externalizationReason: safeString(metadataRecord?.externalizationReason),
729
- rawType: rawBlockType,
730
- raw: rawBlock,
731
- }),
732
- });
733
- }
734
-
735
- return parts;
736
- }
737
-
738
- /**
739
- * Map AgentMessage role to the DB enum.
740
- *
741
- * "user" -> "user"
742
- * "assistant" -> "assistant"
743
- *
744
- * AgentMessage only has user/assistant roles, but we keep the mapping
745
- * explicit for clarity and future-proofing.
746
- */
747
- function toDbRole(role: string): "user" | "assistant" | "system" | "tool" {
748
- if (role === "tool" || role === "toolResult") {
749
- return "tool";
750
- }
751
- if (role === "system") {
752
- return "system";
753
- }
754
- if (role === "user") {
755
- return "user";
756
- }
757
- if (role === "assistant") {
758
- return "assistant";
759
- }
760
- // Unknown roles are preserved via message_parts metadata and treated as assistant.
761
- return "assistant";
762
- }
763
-
764
- type StoredMessage = {
765
- role: "user" | "assistant" | "system" | "tool";
766
- content: string;
767
- tokenCount: number;
768
- };
769
-
770
- /**
771
- * Normalize AgentMessage variants into the storage shape used by LCM.
772
- */
773
- function toStoredMessage(message: AgentMessage): StoredMessage {
774
- const content =
775
- "content" in message
776
- ? extractMessageContent(message.content)
777
- : "output" in message
778
- ? `$ ${(message as { command: string; output: string }).command}\n${(message as { command: string; output: string }).output}`
779
- : "";
780
- const runtimeRole = toRuntimeRoleForTokenEstimate(message.role);
781
- const normalizedContent =
782
- "content" in message
783
- ? normalizeMessageContentForStorage({
784
- message,
785
- fallbackContent: content,
786
- })
787
- : content;
788
- const tokenCount =
789
- "content" in message
790
- ? estimateContentTokensForRole({
791
- role: runtimeRole,
792
- content: normalizedContent,
793
- fallbackContent: content,
794
- })
795
- : estimateTokens(content);
796
-
797
- return {
798
- role: toDbRole(message.role),
799
- content,
800
- tokenCount,
801
- };
802
- }
803
-
804
- function createBootstrapEntryHash(message: StoredMessage | null): string | null {
805
- if (!message) {
806
- return null;
807
- }
808
- return createHash("sha256")
809
- .update(JSON.stringify({ role: message.role, content: message.content }))
810
- .digest("hex");
811
- }
812
-
813
- function estimateMessageContentTokensForAfterTurn(content: unknown): number {
814
- if (typeof content === "string") {
815
- return estimateTokens(content);
816
- }
817
- if (Array.isArray(content)) {
818
- let total = 0;
819
- for (const part of content) {
820
- if (!part || typeof part !== "object") {
821
- continue;
822
- }
823
- const record = part as Record<string, unknown>;
824
- const text =
825
- typeof record.text === "string"
826
- ? record.text
827
- : typeof record.thinking === "string"
828
- ? record.thinking
829
- : "";
830
- if (text) {
831
- total += estimateTokens(text);
832
- }
833
- }
834
- return total;
835
- }
836
- if (content == null) {
837
- return 0;
838
- }
839
- const serialized = JSON.stringify(content);
840
- return estimateTokens(typeof serialized === "string" ? serialized : "");
841
- }
842
-
843
- function estimateSessionTokenCountForAfterTurn(messages: AgentMessage[]): number {
844
- let total = 0;
845
- for (const message of messages) {
846
- if ("content" in message) {
847
- total += estimateMessageContentTokensForAfterTurn(message.content);
848
- continue;
849
- }
850
- if ("command" in message || "output" in message) {
851
- const commandText =
852
- typeof (message as { command?: unknown }).command === "string"
853
- ? (message as { command?: string }).command
854
- : "";
855
- const outputText =
856
- typeof (message as { output?: unknown }).output === "string"
857
- ? (message as { output?: string }).output
858
- : "";
859
- total += estimateTokens(`${commandText}\n${outputText}`);
860
- }
861
- }
862
- return total;
863
- }
864
-
865
- function isBootstrapMessage(value: unknown): value is AgentMessage {
866
- if (!value || typeof value !== "object") {
867
- return false;
868
- }
869
- const msg = value as { role?: unknown; content?: unknown; command?: unknown; output?: unknown };
870
- if (typeof msg.role !== "string") {
871
- return false;
872
- }
873
- return "content" in msg || ("command" in msg && "output" in msg);
874
- }
875
-
876
- function extractCanonicalBootstrapMessage(value: unknown): AgentMessage | null {
877
- if (isBootstrapMessage(value)) {
878
- return value;
879
- }
880
- if (!value || typeof value !== "object" || Array.isArray(value)) {
881
- return null;
882
- }
883
- const entry = value as { type?: unknown; message?: unknown };
884
- if ("message" in entry) {
885
- if (entry.type !== undefined && entry.type !== "message") {
886
- return null;
887
- }
888
- return isBootstrapMessage(entry.message) ? entry.message : null;
889
- }
890
- return null;
891
- }
892
-
893
- function extractBootstrapMessageCandidate(value: unknown): AgentMessage | null {
894
- return extractCanonicalBootstrapMessage(value);
895
- }
896
-
897
- function parseBootstrapJsonl(raw: string, options?: {
898
- strict?: boolean;
899
- }): { messages: AgentMessage[]; sawNonWhitespace: boolean; hadMalformedLine: boolean } {
900
- const messages: AgentMessage[] = [];
901
- const lines = raw.split(/\r?\n/);
902
- let sawNonWhitespace = false;
903
- let hadMalformedLine = false;
904
- for (const line of lines) {
905
- const item = line.trim();
906
- if (!item) {
907
- continue;
908
- }
909
- sawNonWhitespace = true;
910
- try {
911
- const parsed = JSON.parse(item);
912
- const candidate = extractBootstrapMessageCandidate(parsed);
913
- if (candidate) {
914
- messages.push(candidate);
915
- continue;
916
- }
917
- } catch {
918
- if (options?.strict) {
919
- hadMalformedLine = true;
920
- }
921
- }
922
- }
923
- return { messages, sawNonWhitespace, hadMalformedLine };
924
- }
925
-
926
- /** Load recoverable messages from a JSON/JSONL session file without full-file reads for JSONL. */
927
- async function readLeafPathMessages(sessionFile: string): Promise<AgentMessage[]> {
928
- try {
929
- let sawNonWhitespace = false;
930
- let jsonArrayMode = false;
931
- let jsonArrayBuffer = "";
932
- const messages: AgentMessage[] = [];
933
- const stream = createReadStream(sessionFile, { encoding: "utf8" });
934
- const lines = createInterface({
935
- input: stream,
936
- crlfDelay: Infinity,
937
- });
938
-
939
- for await (const line of lines) {
940
- if (!sawNonWhitespace) {
941
- const trimmed = line.trim();
942
- if (trimmed) {
943
- sawNonWhitespace = true;
944
- if (trimmed.startsWith("[")) {
945
- jsonArrayMode = true;
946
- }
947
- }
948
- }
949
-
950
- if (jsonArrayMode) {
951
- jsonArrayBuffer += `${line}\n`;
952
- continue;
953
- }
954
-
955
- const parsed = parseBootstrapJsonl(line);
956
- if (parsed.messages.length > 0) {
957
- messages.push(...parsed.messages);
958
- }
959
- }
960
-
961
- if (jsonArrayMode) {
962
- const trimmed = jsonArrayBuffer.trim();
963
- if (!trimmed) {
964
- return [];
965
- }
966
- try {
967
- const parsed = JSON.parse(trimmed);
968
- if (!Array.isArray(parsed)) {
969
- return [];
970
- }
971
- return parsed.filter(isBootstrapMessage);
972
- } catch {
973
- return [];
974
- }
975
- }
976
-
977
- return messages;
978
- } catch {
979
- return [];
980
- }
981
- }
982
-
983
- /**
984
- * Resolve the first-time bootstrap token budget.
985
- *
986
- * When unset, bootstrap keeps a modest suffix of the parent session rather than
987
- * inheriting the full raw history into a brand-new conversation.
988
- */
989
- function resolveBootstrapMaxTokens(config: Pick<LcmConfig, "bootstrapMaxTokens" | "leafChunkTokens">): number {
990
- if (
991
- typeof config.bootstrapMaxTokens === "number" &&
992
- Number.isFinite(config.bootstrapMaxTokens) &&
993
- config.bootstrapMaxTokens > 0
994
- ) {
995
- return Math.floor(config.bootstrapMaxTokens);
996
- }
997
-
998
- const leafChunkTokens =
999
- typeof config.leafChunkTokens === "number" &&
1000
- Number.isFinite(config.leafChunkTokens) &&
1001
- config.leafChunkTokens > 0
1002
- ? Math.floor(config.leafChunkTokens)
1003
- : 20_000;
1004
- return Math.max(6000, Math.floor(leafChunkTokens * 0.3));
1005
- }
1006
-
1007
- /**
1008
- * Keep only the newest bootstrap messages that fit within the token budget.
1009
- *
1010
- * The newest message is always preserved so a fork never starts empty when the
1011
- * parent transcript has any recoverable content at all.
1012
- */
1013
- function trimBootstrapMessagesToBudget(messages: AgentMessage[], maxTokens: number): AgentMessage[] {
1014
- if (messages.length === 0) {
1015
- return [];
1016
- }
1017
-
1018
- const safeMaxTokens = Number.isFinite(maxTokens) ? Math.floor(maxTokens) : 0;
1019
- if (safeMaxTokens <= 0) {
1020
- return [messages[messages.length - 1]!];
1021
- }
1022
-
1023
- const kept: AgentMessage[] = [];
1024
- let totalTokens = 0;
1025
-
1026
- for (let index = messages.length - 1; index >= 0; index -= 1) {
1027
- const message = messages[index]!;
1028
- const tokenCount = toStoredMessage(message).tokenCount;
1029
- if (kept.length > 0 && totalTokens + tokenCount > safeMaxTokens) {
1030
- break;
1031
- }
1032
- kept.push(message);
1033
- totalTokens += tokenCount;
1034
- }
1035
-
1036
- // If a single oversized tail message exceeds the budget, return empty
1037
- // rather than silently bypassing the budget cap. An empty bootstrap is
1038
- // safer than an exploding one.
1039
- if (kept.length === 1 && totalTokens > safeMaxTokens) {
1040
- return [];
1041
- }
1042
-
1043
- kept.reverse();
1044
- return kept;
1045
- }
1046
-
1047
- function readFileSegment(sessionFile: string, offset: number): string | null {
1048
- let fd: number | null = null;
1049
- try {
1050
- fd = openSync(sessionFile, "r");
1051
- const stats = statSync(sessionFile);
1052
- const safeOffset = Math.max(0, Math.min(Math.floor(offset), stats.size));
1053
- const length = stats.size - safeOffset;
1054
- if (length <= 0) {
1055
- return "";
1056
- }
1057
- const buffer = Buffer.alloc(length);
1058
- readSync(fd, buffer, 0, length, safeOffset);
1059
- return buffer.toString("utf8");
1060
- } catch {
1061
- return null;
1062
- } finally {
1063
- if (fd != null) {
1064
- closeSync(fd);
1065
- }
1066
- }
1067
- }
1068
-
1069
- function readLastJsonlEntryBeforeOffset(
1070
- sessionFile: string,
1071
- offset: number,
1072
- messageOnly = false,
1073
- matcher?: (message: AgentMessage) => boolean,
1074
- ): string | null {
1075
- const chunkSize = 16_384;
1076
- let fd: number | null = null;
1077
- try {
1078
- const safeOffset = Math.max(0, Math.floor(offset));
1079
- if (safeOffset <= 0) {
1080
- return null;
1081
- }
1082
-
1083
- fd = openSync(sessionFile, "r");
1084
- let cursor = safeOffset;
1085
- let carry = "";
1086
- let reachedStart = false;
1087
- while (cursor > 0 || (reachedStart && carry.length > 0)) {
1088
- if (!reachedStart) {
1089
- const start = Math.max(0, cursor - chunkSize);
1090
- const length = cursor - start;
1091
- const buffer = Buffer.alloc(length);
1092
- readSync(fd, buffer, 0, length, start);
1093
- carry = buffer.toString("utf8") + carry;
1094
- cursor = start;
1095
- if (start === 0) {
1096
- reachedStart = true;
1097
- }
1098
- }
1099
-
1100
- const trimmedEnd = carry.replace(/\s+$/u, "");
1101
- if (!trimmedEnd) {
1102
- if (reachedStart) break;
1103
- carry = "";
1104
- continue;
1105
- }
1106
-
1107
- const newlineIndex = Math.max(trimmedEnd.lastIndexOf("\n"), trimmedEnd.lastIndexOf("\r"));
1108
- if (newlineIndex >= 0) {
1109
- const candidate = trimmedEnd.slice(newlineIndex + 1).trim();
1110
- if (candidate) {
1111
- if (messageOnly) {
1112
- let matchedMessage: AgentMessage | null = null;
1113
- try {
1114
- matchedMessage = extractBootstrapMessageCandidate(JSON.parse(candidate));
1115
- } catch { /* not valid JSON, skip */ }
1116
- if (!matchedMessage || (matcher && !matcher(matchedMessage))) {
1117
- carry = trimmedEnd.slice(0, newlineIndex);
1118
- continue;
1119
- }
1120
- }
1121
- return candidate;
1122
- }
1123
- carry = trimmedEnd.slice(0, newlineIndex);
1124
- continue;
1125
- }
1126
-
1127
- // No newline found — entire trimmedEnd is one line
1128
- if (reachedStart) {
1129
- const firstLine = trimmedEnd.trim() || null;
1130
- if (firstLine && messageOnly) {
1131
- let matchedMessage: AgentMessage | null = null;
1132
- try {
1133
- matchedMessage = extractBootstrapMessageCandidate(JSON.parse(firstLine));
1134
- } catch { /* not valid JSON */ }
1135
- if (!matchedMessage || (matcher && !matcher(matchedMessage))) return null;
1136
- }
1137
- return firstLine;
1138
- }
1139
- // Need more data from earlier in the file
1140
- continue;
1141
- }
1142
- return null;
1143
- } catch {
1144
- return null;
1145
- } finally {
1146
- if (fd != null) {
1147
- closeSync(fd);
1148
- }
1149
- }
1150
- }
1151
-
1152
- function readAppendedLeafPathMessages(params: {
1153
- sessionFile: string;
1154
- offset: number;
1155
- }): { messages: AgentMessage[]; canUseAppendOnly: boolean; sawNonWhitespace: boolean } {
1156
- const raw = readFileSegment(params.sessionFile, params.offset);
1157
- if (raw == null) {
1158
- return { messages: [], canUseAppendOnly: false, sawNonWhitespace: false };
1159
- }
1160
-
1161
- const trimmed = raw.trim();
1162
- if (!trimmed) {
1163
- return { messages: [], canUseAppendOnly: true, sawNonWhitespace: false };
1164
- }
1165
-
1166
- if (trimmed.startsWith("[")) {
1167
- return { messages: [], canUseAppendOnly: false, sawNonWhitespace: true };
1168
- }
1169
-
1170
- const parsed = parseBootstrapJsonl(raw, { strict: true });
1171
- if (parsed.hadMalformedLine) {
1172
- return { messages: [], canUseAppendOnly: false, sawNonWhitespace: parsed.sawNonWhitespace };
1173
- }
1174
-
1175
- return {
1176
- messages: parsed.messages,
1177
- canUseAppendOnly: true,
1178
- sawNonWhitespace: parsed.sawNonWhitespace,
1179
- };
1180
- }
1181
-
1182
- function readBootstrapMessageFromJsonLine(line: string | null): AgentMessage | null {
1183
- if (!line) {
1184
- return null;
1185
- }
1186
- try {
1187
- return extractBootstrapMessageCandidate(JSON.parse(line));
1188
- } catch {
1189
- return null;
1190
- }
1191
- }
1192
-
1193
- function messageIdentity(role: string, content: string): string {
1194
- return `${role}\u0000${content}`;
1195
- }
1196
-
1197
- // ── LcmContextEngine ────────────────────────────────────────────────────────
1198
-
1199
- export class LcmContextEngine implements ContextEngine {
1200
- readonly info: ContextEngineInfo;
1201
-
1202
- private config: LcmConfig;
1203
-
1204
- /** Get the configured timezone, falling back to system timezone. */
1205
- get timezone(): string {
1206
- return this.config.timezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone;
1207
- }
1208
-
1209
- private conversationStore: ConversationStore;
1210
- private summaryStore: SummaryStore;
1211
- private compactionTelemetryStore: CompactionTelemetryStore;
1212
- private assembler: ContextAssembler;
1213
- private compaction: CompactionEngine;
1214
- private retrieval: RetrievalEngine;
1215
- private readonly db: DatabaseSync;
1216
- private migrated = false;
1217
- private readonly fts5Available: boolean;
1218
- private readonly ignoreSessionPatterns: RegExp[];
1219
- private readonly statelessSessionPatterns: RegExp[];
1220
- private sessionOperationQueues = new Map<
1221
- string,
1222
- { promise: Promise<void>; refCount: number }
1223
- >();
1224
- private largeFileTextSummarizerResolved = false;
1225
- private largeFileTextSummarizer?: (prompt: string) => Promise<string | null>;
1226
- private deps: LcmDependencies;
1227
-
1228
- // ── Circuit breaker for compaction auth failures ──
1229
- private circuitBreakerStates = new Map<string, CircuitBreakerState>();
1230
-
1231
- constructor(deps: LcmDependencies, database: DatabaseSync) {
1232
- this.deps = deps;
1233
- this.config = deps.config;
1234
- this.ignoreSessionPatterns = compileSessionPatterns(this.config.ignoreSessionPatterns);
1235
- this.statelessSessionPatterns = compileSessionPatterns(this.config.statelessSessionPatterns);
1236
- this.db = database;
1237
-
1238
- // Run migrations eagerly at construction time so the schema exists
1239
- // before any lifecycle hook fires.
1240
- let migrationOk = false;
1241
- const migrationStartedAt = Date.now();
1242
- try {
1243
- runLcmMigrations(this.db, {
1244
- log: this.deps.log,
1245
- });
1246
- this.migrated = true;
1247
-
1248
- // Verify tables were actually created
1249
- const tables = this.db
1250
- .prepare("SELECT name FROM sqlite_master WHERE type='table'")
1251
- .all() as Array<{ name: string }>;
1252
- if (tables.length === 0) {
1253
- this.deps.log.warn(
1254
- "[lcm] Migration completed but database has zero tables — DB may be non-functional",
1255
- );
1256
- } else {
1257
- migrationOk = true;
1258
- this.deps.log.info(
1259
- `[lcm] Migration run completed during engine init: duration=${formatDurationMs(Date.now() - migrationStartedAt)} fts5=${this.fts5Available}`,
1260
- );
1261
- this.deps.log.debug(
1262
- `[lcm] Migration successful — ${tables.length} tables: ${tables.map((t) => t.name).join(", ")}`,
1263
- );
1264
- }
1265
- } catch (err) {
1266
- this.deps.log.error(
1267
- `[lcm] Migration failed after ${formatDurationMs(Date.now() - migrationStartedAt)}: ${err instanceof Error ? err.message : String(err)}`,
1268
- );
1269
- }
1270
-
1271
- this.fts5Available = getLcmDbFeatures(this.db).fts5Available;
1272
-
1273
- // Only claim ownership of compaction when the DB is operational.
1274
- // Without a working schema, ownsCompaction would disable the runtime's
1275
- // built-in compaction safeguard and inflate the context budget.
1276
- this.info = {
1277
- id: "lcm",
1278
- name: "Lossless Context Management Engine",
1279
- version: "0.1.0",
1280
- ownsCompaction: migrationOk,
1281
- };
1282
-
1283
- this.conversationStore = new ConversationStore(this.db, {
1284
- fts5Available: this.fts5Available,
1285
- });
1286
- this.summaryStore = new SummaryStore(this.db, { fts5Available: this.fts5Available });
1287
- this.compactionTelemetryStore = new CompactionTelemetryStore(this.db);
1288
-
1289
- if (!this.fts5Available) {
1290
- this.deps.log.warn(
1291
- "[lcm] FTS5 unavailable in the current Node runtime; full_text search will fall back to LIKE and indexing is disabled",
1292
- );
1293
- }
1294
- if (this.config.ignoreSessionPatterns.length > 0) {
1295
- logStartupBannerOnce({
1296
- key: "ignore-session-patterns",
1297
- log: (message) => this.deps.log.info(message),
1298
- message: `[lcm] Ignoring sessions matching ${this.config.ignoreSessionPatterns.length} pattern(s): ${this.config.ignoreSessionPatterns.join(", ")}`,
1299
- });
1300
- }
1301
- if (this.config.skipStatelessSessions && this.config.statelessSessionPatterns.length > 0) {
1302
- logStartupBannerOnce({
1303
- key: "stateless-session-patterns",
1304
- log: (message) => this.deps.log.info(message),
1305
- message: `[lcm] Stateless session patterns: ${this.config.statelessSessionPatterns.length} pattern(s): ${this.config.statelessSessionPatterns.join(", ")}`,
1306
- });
1307
- }
1308
-
1309
- this.assembler = new ContextAssembler(
1310
- this.conversationStore,
1311
- this.summaryStore,
1312
- this.config.timezone,
1313
- );
1314
-
1315
- const compactionConfig: CompactionConfig = {
1316
- contextThreshold: this.config.contextThreshold,
1317
- freshTailCount: this.config.freshTailCount,
1318
- leafMinFanout: this.config.leafMinFanout,
1319
- condensedMinFanout: this.config.condensedMinFanout,
1320
- condensedMinFanoutHard: this.config.condensedMinFanoutHard,
1321
- incrementalMaxDepth: this.config.incrementalMaxDepth,
1322
- leafChunkTokens: this.config.leafChunkTokens,
1323
- leafTargetTokens: this.config.leafTargetTokens,
1324
- condensedTargetTokens: this.config.condensedTargetTokens,
1325
- maxRounds: 10,
1326
- timezone: this.config.timezone,
1327
- summaryMaxOverageFactor: this.config.summaryMaxOverageFactor,
1328
- };
1329
- this.compaction = new CompactionEngine(
1330
- this.conversationStore,
1331
- this.summaryStore,
1332
- compactionConfig,
1333
- this.deps.log,
1334
- );
1335
-
1336
- this.retrieval = new RetrievalEngine(this.conversationStore, this.summaryStore);
1337
- }
1338
-
1339
- /**
1340
- * Check whether a session should be excluded from LCM processing.
1341
- *
1342
- * We prefer sessionKey matching because the configured glob patterns are
1343
- * documented in terms of session keys, but we fall back to sessionId for
1344
- * older call sites that may not provide the key yet.
1345
- */
1346
- private shouldIgnoreSession(params: { sessionId?: string; sessionKey?: string }): boolean {
1347
- if (this.ignoreSessionPatterns.length === 0) {
1348
- return false;
1349
- }
1350
-
1351
- const candidate =
1352
- typeof params.sessionKey === "string" && params.sessionKey.trim()
1353
- ? params.sessionKey.trim()
1354
- : (params.sessionId?.trim() ?? "");
1355
- if (!candidate) {
1356
- return false;
1357
- }
1358
-
1359
- return matchesSessionPattern(candidate, this.ignoreSessionPatterns);
1360
- }
1361
-
1362
- /** Check whether a session key should skip all LCM writes while remaining readable. */
1363
- isStatelessSession(sessionKey: string | undefined): boolean {
1364
- const trimmedKey = typeof sessionKey === "string" ? sessionKey.trim() : "";
1365
- if (
1366
- !this.config.skipStatelessSessions
1367
- || !trimmedKey
1368
- || this.statelessSessionPatterns.length === 0
1369
- ) {
1370
- return false;
1371
- }
1372
- return matchesSessionPattern(trimmedKey, this.statelessSessionPatterns);
1373
- }
1374
-
1375
- // ── Circuit breaker helpers ──────────────────────────────────────────────
1376
-
1377
- private getCircuitBreakerState(key: string): CircuitBreakerState {
1378
- let state = this.circuitBreakerStates.get(key);
1379
- if (!state) {
1380
- state = { failures: 0, openSince: null };
1381
- this.circuitBreakerStates.set(key, state);
1382
- }
1383
- return state;
1384
- }
1385
-
1386
- private isCircuitBreakerOpen(key: string): boolean {
1387
- const state = this.circuitBreakerStates.get(key);
1388
- if (!state || state.openSince === null) return false;
1389
- const elapsed = Date.now() - state.openSince;
1390
- if (elapsed >= this.config.circuitBreakerCooldownMs) {
1391
- this.resetCircuitBreaker(key);
1392
- return false;
1393
- }
1394
- return true;
1395
- }
1396
-
1397
- private recordCompactionAuthFailure(key: string): void {
1398
- const state = this.getCircuitBreakerState(key);
1399
- state.failures++;
1400
- const halfThreshold = Math.ceil(this.config.circuitBreakerThreshold / 2);
1401
- if (state.failures === halfThreshold && state.failures < this.config.circuitBreakerThreshold) {
1402
- this.deps.log.warn(
1403
- `[lcm] WARNING: compaction degraded — ${state.failures}/${this.config.circuitBreakerThreshold} consecutive auth failures for ${key}`,
1404
- );
1405
- }
1406
- if (state.failures >= this.config.circuitBreakerThreshold) {
1407
- state.openSince = Date.now();
1408
- const cooldownMin = Math.round(this.config.circuitBreakerCooldownMs / 60000);
1409
- this.deps.log.warn(
1410
- `[lcm] CIRCUIT BREAKER OPEN: compaction disabled for ${key}. Auto-retry in ${cooldownMin}m. LCM is operating in degraded mode.`,
1411
- );
1412
- }
1413
- }
1414
-
1415
- private recordCompactionSuccess(key: string): void {
1416
- const state = this.circuitBreakerStates.get(key);
1417
- if (!state) {
1418
- return;
1419
- }
1420
- if (state.failures > 0 || state.openSince !== null) {
1421
- this.deps.log.info(
1422
- `[lcm] compaction circuit breaker CLOSED: successful compaction for ${key} after ${state.failures} prior failures.`,
1423
- );
1424
- }
1425
- this.resetCircuitBreaker(key);
1426
- }
1427
-
1428
- private resetCircuitBreaker(key: string): void {
1429
- this.circuitBreakerStates.delete(key);
1430
- }
1431
-
1432
- /** Ensure DB schema is up-to-date. Called lazily on first bootstrap/ingest/assemble/compact. */
1433
- private ensureMigrated(): void {
1434
- if (this.migrated) {
1435
- return;
1436
- }
1437
- const migrationStartedAt = Date.now();
1438
- this.deps.log.info("[lcm] ensureMigrated: running migrations lazily");
1439
- runLcmMigrations(this.db, {
1440
- log: this.deps.log,
1441
- });
1442
- this.migrated = true;
1443
- this.deps.log.info(
1444
- `[lcm] ensureMigrated: completed in ${formatDurationMs(Date.now() - migrationStartedAt)}`,
1445
- );
1446
- }
1447
-
1448
- /**
1449
- * Serialize mutating operations per stable session identity to prevent
1450
- * ingest/compaction races across runtime UUID recycling.
1451
- */
1452
- private async withSessionQueue<T>(
1453
- queueKey: string,
1454
- operation: () => Promise<T>,
1455
- options?: { operationName?: string; context?: string },
1456
- ): Promise<T> {
1457
- const entry = this.sessionOperationQueues.get(queueKey);
1458
- const previous = entry?.promise ?? Promise.resolve();
1459
- const queuedAhead = entry?.refCount ?? 0;
1460
- let releaseQueue: () => void = () => {};
1461
- const current = new Promise<void>((resolve) => {
1462
- releaseQueue = resolve;
1463
- });
1464
- const next = previous.catch(() => {}).then(() => current);
1465
-
1466
- if (entry) {
1467
- entry.promise = next;
1468
- entry.refCount++;
1469
- } else {
1470
- this.sessionOperationQueues.set(queueKey, { promise: next, refCount: 1 });
1471
- }
1472
-
1473
- const waitStartedAt = Date.now();
1474
- await previous.catch(() => {});
1475
- const waitMs = Date.now() - waitStartedAt;
1476
- if (options?.operationName) {
1477
- const detail = options.context ? ` ${options.context}` : "";
1478
- this.deps.log.info(
1479
- `[lcm] ${options.operationName}: session queue acquired queueKey=${queueKey} queuedAhead=${queuedAhead} wait=${formatDurationMs(waitMs)}${detail}`,
1480
- );
1481
- }
1482
- try {
1483
- return await operation();
1484
- } finally {
1485
- releaseQueue();
1486
- const cur = this.sessionOperationQueues.get(queueKey);
1487
- if (cur && --cur.refCount === 0) {
1488
- this.sessionOperationQueues.delete(queueKey);
1489
- }
1490
- }
1491
- }
1492
-
1493
- /** Prefer stable session keys for queue serialization when available. */
1494
- private resolveSessionQueueKey(sessionId?: string, sessionKey?: string): string {
1495
- const normalizedSessionKey = sessionKey?.trim();
1496
- const normalizedSessionId = sessionId?.trim();
1497
- return normalizedSessionKey || normalizedSessionId || "__lcm__";
1498
- }
1499
-
1500
- /** Normalize optional live token estimates supplied by runtime callers. */
1501
- private normalizeObservedTokenCount(value: unknown): number | undefined {
1502
- if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
1503
- return undefined;
1504
- }
1505
- return Math.floor(value);
1506
- }
1507
-
1508
- /** Resolve token budget from direct params or legacy fallback input. */
1509
- private resolveTokenBudget(params: {
1510
- tokenBudget?: number;
1511
- runtimeContext?: Record<string, unknown>;
1512
- legacyParams?: Record<string, unknown>;
1513
- }): number | undefined {
1514
- const lp = asRecord(params.runtimeContext) ?? params.legacyParams ?? {};
1515
- if (
1516
- typeof params.tokenBudget === "number" &&
1517
- Number.isFinite(params.tokenBudget) &&
1518
- params.tokenBudget > 0
1519
- ) {
1520
- return Math.floor(params.tokenBudget);
1521
- }
1522
- if (
1523
- typeof lp.tokenBudget === "number" &&
1524
- Number.isFinite(lp.tokenBudget) &&
1525
- lp.tokenBudget > 0
1526
- ) {
1527
- return Math.floor(lp.tokenBudget);
1528
- }
1529
- return undefined;
1530
- }
1531
-
1532
- /** Cap a resolved token budget against the configured maxAssemblyTokenBudget. */
1533
- private applyAssemblyBudgetCap(budget: number): number {
1534
- const cap = this.config.maxAssemblyTokenBudget;
1535
- return cap != null && cap > 0 ? Math.min(budget, cap) : budget;
1536
- }
1537
-
1538
- /** Normalize token counters that may legitimately be zero. */
1539
- private normalizeOptionalCount(value: unknown): number | undefined {
1540
- if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
1541
- return undefined;
1542
- }
1543
- return Math.floor(value);
1544
- }
1545
-
1546
- /** Treat a recent cache hit as still-hot for a couple of turns unless telemetry observed a later break. */
1547
- private shouldApplyHotCacheHysteresis(
1548
- telemetry: ConversationCompactionTelemetryRecord | null,
1549
- ): boolean {
1550
- if (!telemetry?.lastObservedCacheHitAt) {
1551
- return false;
1552
- }
1553
- if (
1554
- telemetry.lastObservedCacheBreakAt
1555
- && telemetry.lastObservedCacheBreakAt >= telemetry.lastObservedCacheHitAt
1556
- ) {
1557
- return false;
1558
- }
1559
- return telemetry.turnsSinceLeafCompaction <= HOT_CACHE_HYSTERESIS_TURNS;
1560
- }
1561
-
1562
- /** Resolve the effective cache state the incremental compaction policy should react to. */
1563
- private resolveCacheAwareState(
1564
- telemetry: ConversationCompactionTelemetryRecord | null,
1565
- ): CacheState {
1566
- if (!telemetry) {
1567
- return "unknown";
1568
- }
1569
- if (telemetry.cacheState === "hot") {
1570
- return "hot";
1571
- }
1572
- if (this.shouldApplyHotCacheHysteresis(telemetry)) {
1573
- return "hot";
1574
- }
1575
- return telemetry.cacheState;
1576
- }
1577
-
1578
- /** Decide whether a hot cache still has enough real token-budget headroom to skip incremental maintenance. */
1579
- private isComfortablyUnderTokenBudget(params: {
1580
- currentTokenCount?: number;
1581
- tokenBudget: number;
1582
- }): boolean {
1583
- if (
1584
- typeof params.currentTokenCount !== "number"
1585
- || !Number.isFinite(params.currentTokenCount)
1586
- || params.currentTokenCount < 0
1587
- ) {
1588
- return false;
1589
- }
1590
- const budget = Math.max(1, Math.floor(params.tokenBudget));
1591
- const safeBudget = Math.floor(
1592
- budget * (1 - this.config.cacheAwareCompaction.hotCacheBudgetHeadroomRatio),
1593
- );
1594
- return params.currentTokenCount <= safeBudget;
1595
- }
1596
-
1597
- /** Resolve bounded dynamic leaf chunk sizes from config and the active token budget. */
1598
- private resolveDynamicLeafChunkBounds(tokenBudget?: number): DynamicLeafChunkBounds {
1599
- const floor = Math.max(1, Math.floor(this.config.leafChunkTokens));
1600
- const configuredMax = this.config.dynamicLeafChunkTokens.enabled
1601
- ? Math.max(floor, Math.floor(this.config.dynamicLeafChunkTokens.max))
1602
- : floor;
1603
- const budgetCap =
1604
- typeof tokenBudget === "number" &&
1605
- Number.isFinite(tokenBudget) &&
1606
- tokenBudget > 0
1607
- ? Math.max(floor, Math.floor(tokenBudget * this.config.contextThreshold))
1608
- : configuredMax;
1609
- const max = Math.max(floor, Math.min(configuredMax, budgetCap));
1610
- const medium = Math.max(
1611
- floor,
1612
- Math.min(max, Math.floor(floor * DYNAMIC_LEAF_CHUNK_MEDIUM_MULTIPLIER)),
1613
- );
1614
- const high = Math.max(
1615
- floor,
1616
- Math.min(max, Math.floor(floor * DYNAMIC_LEAF_CHUNK_HIGH_MULTIPLIER)),
1617
- );
1618
- return { floor, medium, high, max };
1619
- }
1620
-
1621
- /** Classify the current refill rate into a simple step band with downshift hysteresis. */
1622
- private classifyDynamicLeafActivityBand(params: {
1623
- lastActivityBand?: ActivityBand;
1624
- tokensAccumulatedSinceLeafCompaction: number;
1625
- turnsSinceLeafCompaction: number;
1626
- floor: number;
1627
- }): ActivityBand {
1628
- const turns = Math.max(1, params.turnsSinceLeafCompaction);
1629
- const tokensPerTurn = params.tokensAccumulatedSinceLeafCompaction / turns;
1630
- const mediumUpshift = params.floor * DYNAMIC_ACTIVITY_MEDIUM_UPSHIFT_FACTOR;
1631
- const mediumDownshift = params.floor * DYNAMIC_ACTIVITY_MEDIUM_DOWNSHIFT_FACTOR;
1632
- const highUpshift = params.floor * DYNAMIC_ACTIVITY_HIGH_UPSHIFT_FACTOR;
1633
- const highDownshift = params.floor * DYNAMIC_ACTIVITY_HIGH_DOWNSHIFT_FACTOR;
1634
- const lastBand = params.lastActivityBand ?? "low";
1635
-
1636
- if (lastBand === "high") {
1637
- if (tokensPerTurn >= highDownshift) {
1638
- return "high";
1639
- }
1640
- return tokensPerTurn >= mediumDownshift ? "medium" : "low";
1641
- }
1642
- if (lastBand === "medium") {
1643
- if (tokensPerTurn >= highUpshift) {
1644
- return "high";
1645
- }
1646
- if (tokensPerTurn < mediumDownshift) {
1647
- return "low";
1648
- }
1649
- return "medium";
1650
- }
1651
- if (tokensPerTurn >= highUpshift) {
1652
- return "high";
1653
- }
1654
- if (tokensPerTurn >= mediumUpshift) {
1655
- return "medium";
1656
- }
1657
- return "low";
1658
- }
1659
-
1660
- /** Map an activity band to the corresponding working leaf chunk size. */
1661
- private resolveLeafChunkTokensForBand(
1662
- band: ActivityBand,
1663
- bounds: DynamicLeafChunkBounds,
1664
- ): number {
1665
- switch (band) {
1666
- case "high":
1667
- return bounds.high;
1668
- case "medium":
1669
- return bounds.medium;
1670
- default:
1671
- return bounds.floor;
1672
- }
1673
- }
1674
-
1675
- /** Build descending fallback chunk sizes used when a provider rejects a larger chunk. */
1676
- private buildLeafChunkFallbacks(params: {
1677
- preferred: number;
1678
- bounds: DynamicLeafChunkBounds;
1679
- }): number[] {
1680
- const ordered = [params.preferred, params.bounds.max, params.bounds.high, params.bounds.medium, params.bounds.floor];
1681
- const seen = new Set<number>();
1682
- const fallbacks: number[] = [];
1683
- for (const value of ordered) {
1684
- const normalized = Math.max(params.bounds.floor, Math.floor(value));
1685
- if (seen.has(normalized)) {
1686
- continue;
1687
- }
1688
- seen.add(normalized);
1689
- fallbacks.push(normalized);
1690
- }
1691
- return fallbacks.sort((a, b) => b - a);
1692
- }
1693
-
1694
- /** Detect provider/model token-limit failures that should trigger a lower chunk retry. */
1695
- private isRecoverableLeafChunkOverflowError(error: unknown): boolean {
1696
- const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
1697
- if (!message) {
1698
- return false;
1699
- }
1700
- return [
1701
- "context length",
1702
- "context window",
1703
- "maximum context",
1704
- "max context",
1705
- "too many tokens",
1706
- "too many input tokens",
1707
- "input tokens",
1708
- "token limit",
1709
- "context limit",
1710
- "input is too large",
1711
- "input too large",
1712
- "prompt is too long",
1713
- "request too large",
1714
- "exceeds the model",
1715
- "exceeds context",
1716
- ].some((fragment) => message.includes(fragment));
1717
- }
1718
-
1719
- /** Extract the current prompt-cache snapshot from runtime context, if present. */
1720
- private readPromptCacheSnapshot(runtimeContext?: Record<string, unknown>): PromptCacheSnapshot | null {
1721
- const promptCache = asRecord(runtimeContext?.promptCache);
1722
- if (!promptCache) {
1723
- return null;
1724
- }
1725
-
1726
- const lastCallUsage = asRecord(promptCache.lastCallUsage);
1727
- const observation = asRecord(promptCache.observation);
1728
- const cacheRead = this.normalizeOptionalCount(lastCallUsage?.cacheRead);
1729
- const cacheWrite = this.normalizeOptionalCount(lastCallUsage?.cacheWrite);
1730
- const sawExplicitBreak = safeBoolean(observation?.broke) === true;
1731
- const retention = safeString(promptCache.retention)?.trim();
1732
- const hasUsageSignal = cacheRead !== undefined || cacheWrite !== undefined;
1733
- const hasObservationSignal =
1734
- typeof observation?.cacheRead === "number"
1735
- || typeof observation?.previousCacheRead === "number"
1736
- || sawExplicitBreak;
1737
-
1738
- let cacheState: CacheState = "unknown";
1739
- if (sawExplicitBreak) {
1740
- cacheState = "cold";
1741
- } else if (typeof cacheRead === "number" && cacheRead > 0) {
1742
- cacheState = "hot";
1743
- } else if (hasUsageSignal || hasObservationSignal) {
1744
- cacheState = "cold";
1745
- }
1746
-
1747
- return {
1748
- ...(cacheRead !== undefined ? { lastObservedCacheRead: cacheRead } : {}),
1749
- ...(cacheWrite !== undefined ? { lastObservedCacheWrite: cacheWrite } : {}),
1750
- cacheState,
1751
- ...(retention ? { retention } : {}),
1752
- sawExplicitBreak,
1753
- };
1754
- }
1755
-
1756
- /** Persist the current turn's compaction telemetry for later policy decisions. */
1757
- private async updateCompactionTelemetry(params: {
1758
- conversationId: number;
1759
- runtimeContext?: Record<string, unknown>;
1760
- tokenBudget?: number;
1761
- rawTokensOutsideTail?: number;
1762
- }): Promise<ConversationCompactionTelemetryRecord | null> {
1763
- const snapshot = this.readPromptCacheSnapshot(params.runtimeContext);
1764
- const existing = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
1765
- params.conversationId,
1766
- );
1767
- if (!snapshot && params.rawTokensOutsideTail === undefined) {
1768
- return existing;
1769
- }
1770
-
1771
- const now = new Date();
1772
- const bounds = this.resolveDynamicLeafChunkBounds(params.tokenBudget);
1773
- const turnsSinceLeafCompaction =
1774
- (existing?.turnsSinceLeafCompaction ?? 0) + 1;
1775
- const tokensAccumulatedSinceLeafCompaction =
1776
- params.rawTokensOutsideTail ?? existing?.tokensAccumulatedSinceLeafCompaction ?? 0;
1777
- const lastActivityBand = this.classifyDynamicLeafActivityBand({
1778
- lastActivityBand: existing?.lastActivityBand,
1779
- tokensAccumulatedSinceLeafCompaction,
1780
- turnsSinceLeafCompaction,
1781
- floor: bounds.floor,
1782
- });
1783
- await this.compactionTelemetryStore.upsertConversationCompactionTelemetry({
1784
- conversationId: params.conversationId,
1785
- lastObservedCacheRead: snapshot?.lastObservedCacheRead ?? existing?.lastObservedCacheRead ?? null,
1786
- lastObservedCacheWrite:
1787
- snapshot?.lastObservedCacheWrite ?? existing?.lastObservedCacheWrite ?? null,
1788
- lastObservedCacheHitAt:
1789
- snapshot?.cacheState === "hot"
1790
- ? now
1791
- : existing?.lastObservedCacheHitAt ?? null,
1792
- lastObservedCacheBreakAt:
1793
- snapshot?.sawExplicitBreak
1794
- ? now
1795
- : existing?.lastObservedCacheBreakAt ?? null,
1796
- cacheState: snapshot?.cacheState ?? existing?.cacheState ?? "unknown",
1797
- retention: snapshot?.retention ?? existing?.retention ?? null,
1798
- lastLeafCompactionAt: existing?.lastLeafCompactionAt ?? null,
1799
- turnsSinceLeafCompaction,
1800
- tokensAccumulatedSinceLeafCompaction,
1801
- lastActivityBand,
1802
- });
1803
- const updated = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
1804
- params.conversationId,
1805
- );
1806
- if (updated) {
1807
- this.deps.log.debug(
1808
- `[lcm] compaction telemetry updated: conversation=${params.conversationId} cacheState=${updated.cacheState} cacheRead=${updated.lastObservedCacheRead ?? "null"} cacheWrite=${updated.lastObservedCacheWrite ?? "null"} retention=${updated.retention ?? "null"} turnsSinceLeafCompaction=${updated.turnsSinceLeafCompaction} tokensSinceLeafCompaction=${updated.tokensAccumulatedSinceLeafCompaction} activityBand=${updated.lastActivityBand} rawTokensOutsideTail=${params.rawTokensOutsideTail ?? "null"} tokenBudget=${params.tokenBudget ?? "null"}`,
1809
- );
1810
- }
1811
- return updated;
1812
- }
1813
-
1814
- /** Reset refill counters after any successful leaf-producing compaction. */
1815
- private async markLeafCompactionTelemetrySuccess(params: {
1816
- conversationId: number;
1817
- activityBand?: ActivityBand;
1818
- }): Promise<void> {
1819
- const existing = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
1820
- params.conversationId,
1821
- );
1822
- await this.compactionTelemetryStore.upsertConversationCompactionTelemetry({
1823
- conversationId: params.conversationId,
1824
- lastObservedCacheRead: existing?.lastObservedCacheRead ?? null,
1825
- lastObservedCacheWrite: existing?.lastObservedCacheWrite ?? null,
1826
- lastObservedCacheHitAt: existing?.lastObservedCacheHitAt ?? null,
1827
- lastObservedCacheBreakAt: existing?.lastObservedCacheBreakAt ?? null,
1828
- cacheState: existing?.cacheState ?? "unknown",
1829
- retention: existing?.retention ?? null,
1830
- lastLeafCompactionAt: new Date(),
1831
- turnsSinceLeafCompaction: 0,
1832
- tokensAccumulatedSinceLeafCompaction: 0,
1833
- lastActivityBand: params.activityBand ?? existing?.lastActivityBand ?? "low",
1834
- });
1835
- this.deps.log.debug(
1836
- `[lcm] compaction telemetry reset after leaf compaction: conversation=${params.conversationId} cacheState=${existing?.cacheState ?? "unknown"} activityBand=${params.activityBand ?? existing?.lastActivityBand ?? "low"}`,
1837
- );
1838
- }
1839
-
1840
- /** Emit an operational trace for the incremental compaction policy decision. */
1841
- private logIncrementalCompactionDecision(params: {
1842
- conversationId: number;
1843
- cacheState: CacheState;
1844
- activityBand: ActivityBand;
1845
- triggerLeafChunkTokens: number;
1846
- preferredLeafChunkTokens: number;
1847
- fallbackLeafChunkTokens: number[];
1848
- rawTokensOutsideTail: number;
1849
- threshold: number;
1850
- shouldCompact: boolean;
1851
- maxPasses: number;
1852
- allowCondensedPasses: boolean;
1853
- reason: string;
1854
- }): IncrementalCompactionDecision {
1855
- this.deps.log.info(
1856
- `[lcm] incremental compaction decision: conversation=${params.conversationId} cacheState=${params.cacheState} activityBand=${params.activityBand} triggerLeafChunkTokens=${params.triggerLeafChunkTokens} preferredLeafChunkTokens=${params.preferredLeafChunkTokens} fallbackLeafChunkTokens=${params.fallbackLeafChunkTokens.join(",")} rawTokensOutsideTail=${params.rawTokensOutsideTail} threshold=${params.threshold} shouldCompact=${params.shouldCompact} maxPasses=${params.maxPasses} allowCondensedPasses=${params.allowCondensedPasses} reason=${params.reason}`,
1857
- );
1858
- return {
1859
- shouldCompact: params.shouldCompact,
1860
- cacheState: params.cacheState,
1861
- maxPasses: params.maxPasses,
1862
- rawTokensOutsideTail: params.rawTokensOutsideTail,
1863
- threshold: params.threshold,
1864
- leafChunkTokens: params.preferredLeafChunkTokens,
1865
- fallbackLeafChunkTokens: params.fallbackLeafChunkTokens,
1866
- activityBand: params.activityBand,
1867
- allowCondensedPasses: params.allowCondensedPasses,
1868
- };
1869
- }
1870
-
1871
- /** Resolve the cache-aware incremental-compaction policy for the current session. */
1872
- private async evaluateIncrementalCompaction(params: {
1873
- conversationId: number;
1874
- tokenBudget: number;
1875
- currentTokenCount?: number;
1876
- }): Promise<IncrementalCompactionDecision> {
1877
- const telemetry = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
1878
- params.conversationId,
1879
- );
1880
- const cacheState =
1881
- this.config.cacheAwareCompaction.enabled
1882
- ? this.resolveCacheAwareState(telemetry)
1883
- : "unknown";
1884
- const bounds = this.resolveDynamicLeafChunkBounds(params.tokenBudget);
1885
- const activityBand =
1886
- this.config.dynamicLeafChunkTokens.enabled
1887
- ? this.classifyDynamicLeafActivityBand({
1888
- lastActivityBand: telemetry?.lastActivityBand,
1889
- tokensAccumulatedSinceLeafCompaction:
1890
- telemetry?.tokensAccumulatedSinceLeafCompaction ?? 0,
1891
- turnsSinceLeafCompaction: telemetry?.turnsSinceLeafCompaction ?? 0,
1892
- floor: bounds.floor,
1893
- })
1894
- : "low";
1895
- const triggerLeafChunkTokens =
1896
- this.config.dynamicLeafChunkTokens.enabled && cacheState === "hot"
1897
- ? bounds.max
1898
- : this.config.dynamicLeafChunkTokens.enabled
1899
- ? this.resolveLeafChunkTokensForBand(activityBand, bounds)
1900
- : bounds.floor;
1901
- const preferredLeafChunkTokens =
1902
- this.config.cacheAwareCompaction.enabled && (cacheState === "cold" || cacheState === "hot")
1903
- ? bounds.max
1904
- : triggerLeafChunkTokens;
1905
- const fallbackLeafChunkTokens = this.buildLeafChunkFallbacks({
1906
- preferred: preferredLeafChunkTokens,
1907
- bounds,
1908
- });
1909
- const leafTrigger = await this.compaction.evaluateLeafTrigger(
1910
- params.conversationId,
1911
- triggerLeafChunkTokens,
1912
- );
1913
- if (!leafTrigger.shouldCompact) {
1914
- return this.logIncrementalCompactionDecision({
1915
- conversationId: params.conversationId,
1916
- cacheState,
1917
- activityBand,
1918
- triggerLeafChunkTokens,
1919
- preferredLeafChunkTokens,
1920
- fallbackLeafChunkTokens,
1921
- rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
1922
- threshold: leafTrigger.threshold,
1923
- shouldCompact: false,
1924
- maxPasses: 1,
1925
- allowCondensedPasses: false,
1926
- reason: "below-leaf-trigger",
1927
- });
1928
- }
1929
-
1930
- const budgetDecision = await this.compaction.evaluate(
1931
- params.conversationId,
1932
- params.tokenBudget,
1933
- params.currentTokenCount,
1934
- );
1935
- if (budgetDecision.shouldCompact) {
1936
- return this.logIncrementalCompactionDecision({
1937
- conversationId: params.conversationId,
1938
- cacheState,
1939
- activityBand,
1940
- triggerLeafChunkTokens,
1941
- preferredLeafChunkTokens,
1942
- fallbackLeafChunkTokens,
1943
- rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
1944
- threshold: leafTrigger.threshold,
1945
- shouldCompact: true,
1946
- maxPasses: 1,
1947
- allowCondensedPasses: true,
1948
- reason: "budget-trigger",
1949
- });
1950
- }
1951
-
1952
- if (
1953
- cacheState === "hot"
1954
- && this.isComfortablyUnderTokenBudget({
1955
- currentTokenCount: params.currentTokenCount,
1956
- tokenBudget: params.tokenBudget,
1957
- })
1958
- ) {
1959
- return this.logIncrementalCompactionDecision({
1960
- conversationId: params.conversationId,
1961
- cacheState,
1962
- activityBand,
1963
- triggerLeafChunkTokens,
1964
- preferredLeafChunkTokens,
1965
- fallbackLeafChunkTokens,
1966
- rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
1967
- threshold: leafTrigger.threshold,
1968
- shouldCompact: false,
1969
- maxPasses: 1,
1970
- allowCondensedPasses: false,
1971
- reason: "hot-cache-budget-headroom",
1972
- });
1973
- }
1974
-
1975
- if (
1976
- cacheState === "hot"
1977
- && leafTrigger.rawTokensOutsideTail
1978
- < Math.floor(
1979
- leafTrigger.threshold * this.config.cacheAwareCompaction.hotCachePressureFactor,
1980
- )
1981
- ) {
1982
- return this.logIncrementalCompactionDecision({
1983
- conversationId: params.conversationId,
1984
- cacheState,
1985
- activityBand,
1986
- triggerLeafChunkTokens,
1987
- preferredLeafChunkTokens,
1988
- fallbackLeafChunkTokens,
1989
- rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
1990
- threshold: leafTrigger.threshold,
1991
- shouldCompact: false,
1992
- maxPasses: 1,
1993
- allowCondensedPasses: false,
1994
- reason: "hot-cache-defer",
1995
- });
1996
- }
1997
-
1998
- const maxPasses =
1999
- cacheState === "cold"
2000
- ? Math.max(1, this.config.cacheAwareCompaction.maxColdCacheCatchupPasses)
2001
- : 1;
2002
- return this.logIncrementalCompactionDecision({
2003
- conversationId: params.conversationId,
2004
- cacheState,
2005
- activityBand,
2006
- triggerLeafChunkTokens,
2007
- preferredLeafChunkTokens,
2008
- fallbackLeafChunkTokens,
2009
- rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
2010
- threshold: leafTrigger.threshold,
2011
- shouldCompact: true,
2012
- maxPasses,
2013
- allowCondensedPasses: cacheState !== "hot",
2014
- reason: cacheState === "cold" ? "cold-cache-catchup" : "leaf-trigger",
2015
- });
2016
- }
2017
-
2018
- /** Resolve an LCM conversation id from a session key via the session store. */
2019
- private async resolveConversationIdForSessionKey(
2020
- sessionKey: string,
2021
- ): Promise<number | undefined> {
2022
- const trimmedKey = sessionKey.trim();
2023
- if (!trimmedKey) {
2024
- return undefined;
2025
- }
2026
- try {
2027
- const bySessionKey = await this.conversationStore.getConversationForSession({
2028
- sessionKey: trimmedKey,
2029
- });
2030
- if (bySessionKey) {
2031
- return bySessionKey.conversationId;
2032
- }
2033
-
2034
- const runtimeSessionId = await this.deps.resolveSessionIdFromSessionKey(trimmedKey);
2035
- if (!runtimeSessionId) {
2036
- return undefined;
2037
- }
2038
- const conversation = await this.conversationStore.getConversationForSession({
2039
- sessionId: runtimeSessionId,
2040
- });
2041
- return conversation?.conversationId;
2042
- } catch {
2043
- return undefined;
2044
- }
2045
- }
2046
-
2047
- /** Format stable session identifiers for LCM diagnostic logs. */
2048
- private formatSessionLogContext(params: {
2049
- conversationId: number;
2050
- sessionId: string;
2051
- sessionKey?: string;
2052
- }): string {
2053
- const parts = [
2054
- `conversation=${params.conversationId}`,
2055
- `session=${params.sessionId}`,
2056
- ];
2057
- const trimmedSessionKey = params.sessionKey?.trim();
2058
- if (trimmedSessionKey) {
2059
- parts.push(`sessionKey=${trimmedSessionKey}`);
2060
- }
2061
- return parts.join(" ");
2062
- }
2063
-
2064
- /** Build a summarize callback with runtime provider fallback handling. */
2065
- private async resolveSummarize(params: {
2066
- legacyParams?: Record<string, unknown>;
2067
- customInstructions?: string;
2068
- breakerScope: string;
2069
- }): Promise<{
2070
- summarize: (text: string, aggressive?: boolean) => Promise<string>;
2071
- summaryModel: string;
2072
- breakerKey?: string;
2073
- }> {
2074
- const lp = params.legacyParams ?? {};
2075
- if (typeof lp.summarize === "function") {
2076
- return {
2077
- summarize: lp.summarize as (text: string, aggressive?: boolean) => Promise<string>,
2078
- summaryModel: "unknown",
2079
- breakerKey: `custom:${params.breakerScope}`,
2080
- };
2081
- }
2082
- try {
2083
- const customInstructions =
2084
- params.customInstructions !== undefined
2085
- ? params.customInstructions
2086
- : (this.config.customInstructions || undefined);
2087
- const runtimeSummarizer = await createLcmSummarizeFromLegacyParams({
2088
- deps: this.deps,
2089
- legacyParams: lp,
2090
- customInstructions,
2091
- });
2092
- if (runtimeSummarizer) {
2093
- return {
2094
- summarize: runtimeSummarizer.fn,
2095
- summaryModel: runtimeSummarizer.model,
2096
- breakerKey: runtimeSummarizer.breakerKey,
2097
- };
2098
- }
2099
- this.deps.log.error(`[lcm] resolveSummarize: createLcmSummarizeFromLegacyParams returned undefined`);
2100
- } catch (err) {
2101
- this.deps.log.error(
2102
- `[lcm] resolveSummarize failed, using emergency fallback: ${describeLogError(err)}`,
2103
- );
2104
- }
2105
- this.deps.log.error(`[lcm] resolveSummarize: FALLING BACK TO EMERGENCY TRUNCATION`);
2106
- return { summarize: createEmergencyFallbackSummarize(), summaryModel: "unknown" };
2107
- }
2108
-
2109
- /**
2110
- * Resolve an optional model-backed summarizer for large text file exploration.
2111
- *
2112
- * This is opt-in via env so ingest remains deterministic and lightweight when
2113
- * no summarization model is configured.
2114
- */
2115
- private async resolveLargeFileTextSummarizer(): Promise<
2116
- ((prompt: string) => Promise<string | null>) | undefined
2117
- > {
2118
- if (this.largeFileTextSummarizerResolved) {
2119
- return this.largeFileTextSummarizer;
2120
- }
2121
- this.largeFileTextSummarizerResolved = true;
2122
-
2123
- const provider = this.deps.config.largeFileSummaryProvider;
2124
- const model = this.deps.config.largeFileSummaryModel;
2125
- if (!provider || !model) {
2126
- return undefined;
2127
- }
2128
-
2129
- try {
2130
- const result = await createLcmSummarizeFromLegacyParams({
2131
- deps: this.deps,
2132
- legacyParams: { provider, model },
2133
- customInstructions: this.config.customInstructions || undefined,
2134
- });
2135
- if (!result) {
2136
- return undefined;
2137
- }
2138
-
2139
- this.largeFileTextSummarizer = async (prompt: string): Promise<string | null> => {
2140
- let summary: string;
2141
- try {
2142
- summary = await result.fn(prompt, false);
2143
- } catch (err) {
2144
- if (err instanceof LcmProviderAuthError) {
2145
- return null;
2146
- }
2147
- throw err;
2148
- }
2149
- if (typeof summary !== "string") {
2150
- return null;
2151
- }
2152
- const trimmed = summary.trim();
2153
- return trimmed.length > 0 ? trimmed : null;
2154
- };
2155
- return this.largeFileTextSummarizer;
2156
- } catch {
2157
- return undefined;
2158
- }
2159
- }
2160
-
2161
- /** Persist intercepted large-file text payloads to ~/.openclaw/lcm-files. */
2162
- private async storeLargeFileContent(params: {
2163
- conversationId: number;
2164
- fileId: string;
2165
- extension: string;
2166
- content: string;
2167
- }): Promise<string> {
2168
- const dir = join(homedir(), ".openclaw", "lcm-files", String(params.conversationId));
2169
- await mkdir(dir, { recursive: true });
2170
-
2171
- const normalizedExtension = params.extension.replace(/[^a-z0-9]/gi, "").toLowerCase() || "txt";
2172
- const filePath = join(dir, `${params.fileId}.${normalizedExtension}`);
2173
- await writeFile(filePath, params.content, "utf8");
2174
- return filePath;
2175
- }
2176
-
2177
- /** Persist a large text payload and return the resulting compact placeholder. */
2178
- private async externalizeLargeTextPayload(params: {
2179
- conversationId: number;
2180
- content: string;
2181
- fileName?: string;
2182
- mimeType?: string;
2183
- formatReference: (input: { fileId: string; byteSize: number; summary: string }) => string;
2184
- }): Promise<{ fileId: string; byteSize: number; summary: string; reference: string }> {
2185
- const summarizeText = await this.resolveLargeFileTextSummarizer();
2186
- const fileId = `file_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
2187
- const extension = extensionFromNameOrMime(params.fileName, params.mimeType);
2188
- const storageUri = await this.storeLargeFileContent({
2189
- conversationId: params.conversationId,
2190
- fileId,
2191
- extension,
2192
- content: params.content,
2193
- });
2194
- const byteSize = Buffer.byteLength(params.content, "utf8");
2195
- const explorationSummary = await generateExplorationSummary({
2196
- content: params.content,
2197
- fileName: params.fileName,
2198
- mimeType: params.mimeType,
2199
- summarizeText,
2200
- });
2201
-
2202
- await this.summaryStore.insertLargeFile({
2203
- fileId,
2204
- conversationId: params.conversationId,
2205
- fileName: params.fileName,
2206
- mimeType: params.mimeType,
2207
- byteSize,
2208
- storageUri,
2209
- explorationSummary,
2210
- });
2211
-
2212
- return {
2213
- fileId,
2214
- byteSize,
2215
- summary: explorationSummary,
2216
- reference: params.formatReference({
2217
- fileId,
2218
- byteSize,
2219
- summary: explorationSummary,
2220
- }),
2221
- };
2222
- }
2223
-
2224
- /**
2225
- * Intercept oversized <file> blocks before persistence and replace them with
2226
- * compact file references backed by large_files records.
2227
- */
2228
- private async interceptLargeFiles(params: {
2229
- conversationId: number;
2230
- content: string;
2231
- }): Promise<{ rewrittenContent: string; fileIds: string[] } | null> {
2232
- const blocks = parseFileBlocks(params.content);
2233
- if (blocks.length === 0) {
2234
- return null;
2235
- }
2236
-
2237
- const threshold = Math.max(1, this.config.largeFileTokenThreshold);
2238
- const fileIds: string[] = [];
2239
- const rewrittenSegments: string[] = [];
2240
- let cursor = 0;
2241
- let interceptedAny = false;
2242
-
2243
- for (const block of blocks) {
2244
- const blockTokens = estimateTokens(block.text);
2245
- if (blockTokens < threshold) {
2246
- continue;
2247
- }
2248
-
2249
- interceptedAny = true;
2250
- const externalized = await this.externalizeLargeTextPayload({
2251
- conversationId: params.conversationId,
2252
- content: block.text,
2253
- fileName: block.fileName,
2254
- mimeType: block.mimeType,
2255
- formatReference: ({ fileId, byteSize, summary }) =>
2256
- formatFileReference({
2257
- fileId,
2258
- fileName: block.fileName,
2259
- mimeType: block.mimeType,
2260
- byteSize,
2261
- summary,
2262
- }),
2263
- });
2264
-
2265
- rewrittenSegments.push(params.content.slice(cursor, block.start));
2266
- rewrittenSegments.push(externalized.reference);
2267
- cursor = block.end;
2268
- fileIds.push(externalized.fileId);
2269
- }
2270
-
2271
- if (!interceptedAny) {
2272
- return null;
2273
- }
2274
-
2275
- rewrittenSegments.push(params.content.slice(cursor));
2276
- return {
2277
- rewrittenContent: rewrittenSegments.join(""),
2278
- fileIds,
2279
- };
2280
- }
2281
-
2282
- /** Externalize oversized textual tool outputs before they are persisted inline. */
2283
- private async interceptLargeToolResults(params: {
2284
- conversationId: number;
2285
- message: AgentMessage;
2286
- }): Promise<{ rewrittenMessage: AgentMessage; fileIds: string[] } | null> {
2287
- if (
2288
- (params.message.role !== "toolResult" && params.message.role !== "tool") ||
2289
- !("content" in params.message)
2290
- ) {
2291
- return null;
2292
- }
2293
- if (!Array.isArray(params.message.content)) {
2294
- return null;
2295
- }
2296
-
2297
- const threshold = Math.max(1, this.config.largeFileTokenThreshold);
2298
- const rewrittenContent: unknown[] = [];
2299
- const fileIds: string[] = [];
2300
- let interceptedAny = false;
2301
- const topLevel = params.message as Record<string, unknown>;
2302
- const topLevelToolCallId =
2303
- safeString(topLevel.toolCallId) ??
2304
- safeString(topLevel.tool_call_id) ??
2305
- safeString(topLevel.toolUseId) ??
2306
- safeString(topLevel.tool_use_id) ??
2307
- safeString(topLevel.call_id) ??
2308
- safeString(topLevel.id);
2309
- const topLevelToolName =
2310
- safeString(topLevel.toolName) ??
2311
- safeString(topLevel.tool_name);
2312
- const topLevelIsError =
2313
- safeBoolean(topLevel.isError) ??
2314
- safeBoolean(topLevel.is_error);
2315
-
2316
- for (const item of params.message.content) {
2317
- if (!item || typeof item !== "object" || Array.isArray(item)) {
2318
- rewrittenContent.push(item);
2319
- continue;
2320
- }
2321
-
2322
- const record = item as Record<string, unknown>;
2323
- const rawType = safeString(record.type);
2324
- const isStructuredToolResult =
2325
- rawType !== "tool_result" &&
2326
- rawType !== "toolResult" &&
2327
- rawType !== "function_call_output";
2328
- const isPlainTextToolResult =
2329
- rawType === "text" &&
2330
- typeof record.text === "string";
2331
- if (isStructuredToolResult && !isPlainTextToolResult) {
2332
- rewrittenContent.push(item);
2333
- continue;
2334
- }
2335
-
2336
- const textSource =
2337
- isPlainTextToolResult
2338
- ? record.text
2339
- : record.output !== undefined
2340
- ? record.output
2341
- : record.content !== undefined
2342
- ? record.content
2343
- : record;
2344
- const extractedText = extractStructuredText(textSource);
2345
- if (typeof extractedText !== "string" || estimateTokens(extractedText) < threshold) {
2346
- rewrittenContent.push(item);
2347
- continue;
2348
- }
2349
-
2350
- interceptedAny = true;
2351
- const toolName =
2352
- safeString(record.name) ??
2353
- topLevelToolName ??
2354
- "tool-result";
2355
- const externalized = await this.externalizeLargeTextPayload({
2356
- conversationId: params.conversationId,
2357
- content: extractedText,
2358
- fileName: `${toolName}.txt`,
2359
- mimeType: "text/plain",
2360
- formatReference: ({ fileId, byteSize, summary }) =>
2361
- formatToolOutputReference({
2362
- fileId,
2363
- toolName,
2364
- byteSize,
2365
- summary,
2366
- }),
2367
- });
2368
-
2369
- const normalizedRawType =
2370
- rawType === "function_call_output" ? "function_call_output" : "tool_result";
2371
- const compactBlock: Record<string, unknown> = isPlainTextToolResult
2372
- ? {
2373
- type: "text",
2374
- text: externalized.reference,
2375
- rawType: normalizedRawType,
2376
- externalizedFileId: externalized.fileId,
2377
- originalByteSize: externalized.byteSize,
2378
- toolOutputExternalized: true,
2379
- externalizationReason: "large_tool_result",
2380
- }
2381
- : {
2382
- type: normalizedRawType,
2383
- output: externalized.reference,
2384
- externalizedFileId: externalized.fileId,
2385
- originalByteSize: externalized.byteSize,
2386
- toolOutputExternalized: true,
2387
- externalizationReason: "large_tool_result",
2388
- };
2389
- const callId =
2390
- safeString(record.tool_use_id) ??
2391
- safeString(record.toolUseId) ??
2392
- safeString(record.tool_call_id) ??
2393
- safeString(record.toolCallId) ??
2394
- safeString(record.call_id) ??
2395
- safeString(record.id) ??
2396
- topLevelToolCallId;
2397
- if (callId) {
2398
- if (normalizedRawType === "function_call_output") {
2399
- compactBlock.call_id = callId;
2400
- } else {
2401
- compactBlock.tool_use_id = callId;
2402
- }
2403
- }
2404
- if (typeof record.is_error === "boolean") {
2405
- compactBlock.is_error = record.is_error;
2406
- } else if (typeof record.isError === "boolean") {
2407
- compactBlock.isError = record.isError;
2408
- } else if (typeof topLevelIsError === "boolean") {
2409
- compactBlock.isError = topLevelIsError;
2410
- }
2411
- if (toolName) {
2412
- compactBlock.name = toolName;
2413
- }
2414
-
2415
- rewrittenContent.push(compactBlock);
2416
- fileIds.push(externalized.fileId);
2417
- }
2418
-
2419
- if (!interceptedAny) {
2420
- return null;
2421
- }
2422
-
2423
- return {
2424
- rewrittenMessage: {
2425
- ...params.message,
2426
- content: rewrittenContent,
2427
- } as AgentMessage,
2428
- fileIds,
2429
- };
2430
- }
2431
-
2432
- // ── ContextEngine interface ─────────────────────────────────────────────
2433
-
2434
- /**
2435
- * Reconcile session-file history with persisted messages and append only the
2436
- * tail that is present in JSONL but missing from LCM.
2437
- */
2438
- private async reconcileSessionTail(params: {
2439
- sessionId: string;
2440
- sessionKey?: string;
2441
- conversationId: number;
2442
- historicalMessages: AgentMessage[];
2443
- }): Promise<{
2444
- blockedByImportCap: boolean;
2445
- importedMessages: number;
2446
- hasOverlap: boolean;
2447
- }> {
2448
- const { sessionId, conversationId, historicalMessages } = params;
2449
- const startedAt = Date.now();
2450
- const sessionContext = this.formatSessionLogContext({
2451
- conversationId,
2452
- sessionId,
2453
- sessionKey: params.sessionKey,
2454
- });
2455
- if (historicalMessages.length === 0) {
2456
- this.deps.log.info(
2457
- `[lcm] reconcileSessionTail: skipped for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=0 reason=empty-history`,
2458
- );
2459
- return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
2460
- }
2461
-
2462
- const latestDbMessage = await this.conversationStore.getLastMessage(conversationId);
2463
- if (!latestDbMessage) {
2464
- this.deps.log.info(
2465
- `[lcm] reconcileSessionTail: skipped for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} reason=no-db-tail`,
2466
- );
2467
- return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
2468
- }
2469
-
2470
- const storedHistoricalMessages = historicalMessages.map((message) => toStoredMessage(message));
2471
-
2472
- // Fast path: one tail comparison for the common in-sync case.
2473
- const latestHistorical = storedHistoricalMessages[storedHistoricalMessages.length - 1];
2474
- const latestIdentity = messageIdentity(latestDbMessage.role, latestDbMessage.content);
2475
- if (latestIdentity === messageIdentity(latestHistorical.role, latestHistorical.content)) {
2476
- const dbOccurrences = await this.conversationStore.countMessagesByIdentity(
2477
- conversationId,
2478
- latestDbMessage.role,
2479
- latestDbMessage.content,
2480
- );
2481
- let historicalOccurrences = 0;
2482
- for (const stored of storedHistoricalMessages) {
2483
- if (messageIdentity(stored.role, stored.content) === latestIdentity) {
2484
- historicalOccurrences += 1;
2485
- }
2486
- }
2487
- if (dbOccurrences === historicalOccurrences) {
2488
- this.deps.log.info(
2489
- `[lcm] reconcileSessionTail: fast path for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=true`,
2490
- );
2491
- return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
2492
- }
2493
- }
2494
-
2495
- // Slow path: walk backward through JSONL to find the most recent anchor
2496
- // message that already exists in LCM, then append everything after it.
2497
- let anchorIndex = -1;
2498
- const historicalIdentityTotals = new Map<string, number>();
2499
- for (const stored of storedHistoricalMessages) {
2500
- const identity = messageIdentity(stored.role, stored.content);
2501
- historicalIdentityTotals.set(identity, (historicalIdentityTotals.get(identity) ?? 0) + 1);
2502
- }
2503
-
2504
- const historicalIdentityCountsAfterIndex = new Map<string, number>();
2505
- const dbIdentityCounts = new Map<string, number>();
2506
- for (let index = storedHistoricalMessages.length - 1; index >= 0; index--) {
2507
- const stored = storedHistoricalMessages[index];
2508
- const identity = messageIdentity(stored.role, stored.content);
2509
- const seenAfter = historicalIdentityCountsAfterIndex.get(identity) ?? 0;
2510
- const total = historicalIdentityTotals.get(identity) ?? 0;
2511
- const occurrencesThroughIndex = total - seenAfter;
2512
- const exists = await this.conversationStore.hasMessage(
2513
- conversationId,
2514
- stored.role,
2515
- stored.content,
2516
- );
2517
- historicalIdentityCountsAfterIndex.set(identity, seenAfter + 1);
2518
- if (!exists) {
2519
- continue;
2520
- }
2521
-
2522
- let dbCountForIdentity = dbIdentityCounts.get(identity);
2523
- if (dbCountForIdentity === undefined) {
2524
- dbCountForIdentity = await this.conversationStore.countMessagesByIdentity(
2525
- conversationId,
2526
- stored.role,
2527
- stored.content,
2528
- );
2529
- dbIdentityCounts.set(identity, dbCountForIdentity);
2530
- }
2531
-
2532
- // Match the same occurrence index as the DB tail so repeated empty
2533
- // tool messages do not anchor against a later, still-missing entry.
2534
- if (dbCountForIdentity !== occurrencesThroughIndex) {
2535
- continue;
2536
- }
2537
-
2538
- anchorIndex = index;
2539
- break;
2540
- }
2541
-
2542
- if (anchorIndex < 0) {
2543
- this.deps.log.info(
2544
- `[lcm] reconcileSessionTail: no anchor for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=false`,
2545
- );
2546
- return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
2547
- }
2548
- if (anchorIndex >= historicalMessages.length - 1) {
2549
- this.deps.log.info(
2550
- `[lcm] reconcileSessionTail: anchor at tip for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=true`,
2551
- );
2552
- return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
2553
- }
2554
-
2555
- const missingTail = historicalMessages.slice(anchorIndex + 1);
2556
-
2557
- const existingDbCount = await this.conversationStore.getMessageCount(conversationId);
2558
- if (existingDbCount > 0 && missingTail.length > Math.max(existingDbCount * 0.2, 50)) {
2559
- this.deps.log.warn(
2560
- `[lcm] reconcileSessionTail: import cap exceeded for ${sessionContext} — would import ${missingTail.length} messages (existing: ${existingDbCount}). Aborting to prevent flood.`,
2561
- );
2562
- this.deps.log.info(
2563
- `[lcm] reconcileSessionTail: blocked for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} missingTail=${missingTail.length} existingDbCount=${existingDbCount}`,
2564
- );
2565
- return { blockedByImportCap: true, importedMessages: 0, hasOverlap: true };
2566
- }
2567
-
2568
- let importedMessages = 0;
2569
- for (const message of missingTail) {
2570
- const result = await this.ingestSingle({ sessionId, sessionKey: params.sessionKey, message });
2571
- if (result.ingested) {
2572
- importedMessages += 1;
2573
- }
2574
- }
2575
-
2576
- this.deps.log.info(
2577
- `[lcm] reconcileSessionTail: slow path for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} anchorIndex=${anchorIndex} missingTail=${missingTail.length} importedMessages=${importedMessages}`,
2578
- );
2579
- return { blockedByImportCap: false, importedMessages, hasOverlap: true };
2580
- }
2581
-
2582
- /**
2583
- * Persist bootstrap checkpoint metadata anchored to the current DB frontier.
2584
- *
2585
- * We intentionally checkpoint the session file's current EOF while hashing the
2586
- * latest persisted DB message. This keeps append-only recovery aligned with the
2587
- * canonical LCM frontier even when trailing transcript entries are pruned or
2588
- * otherwise noncanonical.
2589
- */
2590
- private async refreshBootstrapState(params: {
2591
- conversationId: number;
2592
- sessionFile: string;
2593
- fileStats?: { size: number; mtimeMs: number };
2594
- }): Promise<void> {
2595
- const latestDbMessage = await this.conversationStore.getLastMessage(params.conversationId);
2596
- const fileStats = params.fileStats ?? statSync(params.sessionFile);
2597
- await this.summaryStore.upsertConversationBootstrapState({
2598
- conversationId: params.conversationId,
2599
- sessionFilePath: params.sessionFile,
2600
- lastSeenSize: fileStats.size,
2601
- lastSeenMtimeMs: Math.trunc(fileStats.mtimeMs),
2602
- lastProcessedOffset: fileStats.size,
2603
- lastProcessedEntryHash: latestDbMessage
2604
- ? createBootstrapEntryHash({
2605
- role: latestDbMessage.role,
2606
- content: latestDbMessage.content,
2607
- tokenCount: latestDbMessage.tokenCount,
2608
- })
2609
- : null,
2610
- });
2611
- }
2612
-
2613
- async bootstrap(params: {
2614
- sessionId: string;
2615
- sessionFile: string;
2616
- sessionKey?: string;
2617
- }): Promise<BootstrapResult> {
2618
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
2619
- return {
2620
- bootstrapped: false,
2621
- importedMessages: 0,
2622
- reason: "session excluded by pattern",
2623
- };
2624
- }
2625
- if (this.isStatelessSession(params.sessionKey)) {
2626
- return {
2627
- bootstrapped: false,
2628
- importedMessages: 0,
2629
- reason: "stateless session",
2630
- };
2631
- }
2632
- this.ensureMigrated();
2633
- const startedAt = Date.now();
2634
- const sessionLabel = [
2635
- `session=${params.sessionId}`,
2636
- ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
2637
- ].join(" ");
2638
- const sessionFileStats = statSync(params.sessionFile);
2639
- const sessionFileSize = sessionFileStats.size;
2640
- const sessionFileMtimeMs = Math.trunc(sessionFileStats.mtimeMs);
2641
-
2642
- const result = await this.withSessionQueue(
2643
- this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
2644
- async () =>
2645
- this.conversationStore.withTransaction(async () => {
2646
- const persistBootstrapState = async (
2647
- conversationId: number,
2648
- ): Promise<void> => {
2649
- await this.refreshBootstrapState({
2650
- conversationId,
2651
- sessionFile: params.sessionFile,
2652
- fileStats: {
2653
- size: sessionFileSize,
2654
- mtimeMs: sessionFileMtimeMs,
2655
- },
2656
- });
2657
- };
2658
-
2659
- const conversation = await this.conversationStore.getOrCreateConversation(params.sessionId, {
2660
- sessionKey: params.sessionKey,
2661
- });
2662
- const conversationId = conversation.conversationId;
2663
- const existingCount = await this.conversationStore.getMessageCount(conversationId);
2664
- const bootstrapState =
2665
- existingCount > 0
2666
- ? await this.summaryStore.getConversationBootstrapState(conversationId)
2667
- : null;
2668
-
2669
- // If the transcript file is byte-for-byte unchanged from the last
2670
- // successful bootstrap checkpoint, skip reopening and reparsing it.
2671
- if (
2672
- bootstrapState &&
2673
- bootstrapState.sessionFilePath === params.sessionFile &&
2674
- bootstrapState.lastSeenSize === sessionFileSize &&
2675
- bootstrapState.lastSeenMtimeMs === sessionFileMtimeMs
2676
- ) {
2677
- if (!conversation.bootstrappedAt) {
2678
- await this.conversationStore.markConversationBootstrapped(conversationId);
2679
- }
2680
- this.deps.log.info(
2681
- `[lcm] bootstrap: checkpoint hit conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} duration=${formatDurationMs(Date.now() - startedAt)}`,
2682
- );
2683
- return {
2684
- bootstrapped: false,
2685
- importedMessages: 0,
2686
- reason: conversation.bootstrappedAt ? "already bootstrapped" : "conversation already up to date",
2687
- };
2688
- }
2689
-
2690
- if (
2691
- existingCount > 0 &&
2692
- bootstrapState &&
2693
- bootstrapState.sessionFilePath === params.sessionFile &&
2694
- sessionFileSize > bootstrapState.lastSeenSize &&
2695
- sessionFileMtimeMs >= bootstrapState.lastSeenMtimeMs
2696
- ) {
2697
- const latestDbMessage = await this.conversationStore.getLastMessage(conversationId);
2698
- const latestDbHash = latestDbMessage
2699
- ? createBootstrapEntryHash({
2700
- role: latestDbMessage.role,
2701
- content: latestDbMessage.content,
2702
- tokenCount: latestDbMessage.tokenCount,
2703
- })
2704
- : null;
2705
- const tailEntryRaw = readLastJsonlEntryBeforeOffset(
2706
- params.sessionFile,
2707
- bootstrapState.lastProcessedOffset,
2708
- true,
2709
- (message) => createBootstrapEntryHash(toStoredMessage(message)) === latestDbHash,
2710
- );
2711
- const tailEntryMessage = readBootstrapMessageFromJsonLine(tailEntryRaw);
2712
- const tailEntryHash = tailEntryMessage
2713
- ? createBootstrapEntryHash(toStoredMessage(tailEntryMessage))
2714
- : null;
2715
-
2716
- if (
2717
- latestDbHash &&
2718
- latestDbHash === bootstrapState.lastProcessedEntryHash &&
2719
- tailEntryHash &&
2720
- tailEntryHash === bootstrapState.lastProcessedEntryHash
2721
- ) {
2722
- const appended = readAppendedLeafPathMessages({
2723
- sessionFile: params.sessionFile,
2724
- offset: bootstrapState.lastProcessedOffset,
2725
- });
2726
- if (appended.canUseAppendOnly) {
2727
- if (!conversation.bootstrappedAt) {
2728
- await this.conversationStore.markConversationBootstrapped(conversationId);
2729
- }
2730
-
2731
- let importedMessages = 0;
2732
- for (const message of appended.messages) {
2733
- const ingestResult = await this.ingestSingle({
2734
- sessionId: params.sessionId,
2735
- sessionKey: params.sessionKey,
2736
- message,
2737
- });
2738
- if (ingestResult.ingested) {
2739
- importedMessages += 1;
2740
- }
2741
- }
2742
-
2743
- await persistBootstrapState(conversationId);
2744
- this.deps.log.info(
2745
- `[lcm] bootstrap: append-only conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} appendedMessages=${appended.messages.length} importedMessages=${importedMessages} duration=${formatDurationMs(Date.now() - startedAt)}`,
2746
- );
2747
-
2748
- if (importedMessages > 0) {
2749
- return {
2750
- bootstrapped: true,
2751
- importedMessages,
2752
- reason: "reconciled missing session messages",
2753
- };
2754
- }
2755
-
2756
- return {
2757
- bootstrapped: false,
2758
- importedMessages: 0,
2759
- reason: conversation.bootstrappedAt ? "already bootstrapped" : "conversation already up to date",
2760
- };
2761
- }
2762
- }
2763
- }
2764
-
2765
- const historicalMessages = await readLeafPathMessages(params.sessionFile);
2766
- this.deps.log.info(
2767
- `[lcm] bootstrap: full transcript read conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} historicalMessages=${historicalMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
2768
- );
2769
-
2770
- // First-time import path: no LCM rows yet, so seed directly from the
2771
- // active leaf context snapshot.
2772
- if (existingCount === 0) {
2773
- const bootstrapMessages = trimBootstrapMessagesToBudget(
2774
- historicalMessages,
2775
- resolveBootstrapMaxTokens(this.config),
2776
- );
2777
-
2778
- if (bootstrapMessages.length === 0) {
2779
- await this.conversationStore.markConversationBootstrapped(conversationId);
2780
- await persistBootstrapState(conversationId);
2781
- return {
2782
- bootstrapped: false,
2783
- importedMessages: 0,
2784
- reason: "no leaf-path messages in session",
2785
- };
2786
- }
2787
-
2788
- const nextSeq = (await this.conversationStore.getMaxSeq(conversationId)) + 1;
2789
- const bulkInput = bootstrapMessages.map((message, index) => {
2790
- const stored = toStoredMessage(message);
2791
- return {
2792
- conversationId,
2793
- seq: nextSeq + index,
2794
- role: stored.role,
2795
- content: stored.content,
2796
- tokenCount: stored.tokenCount,
2797
- };
2798
- });
2799
-
2800
- const inserted = await this.conversationStore.createMessagesBulk(bulkInput);
2801
- await this.summaryStore.appendContextMessages(
2802
- conversationId,
2803
- inserted.map((record) => record.messageId),
2804
- );
2805
- await this.conversationStore.markConversationBootstrapped(conversationId);
2806
-
2807
- // Prune HEARTBEAT_OK turns from the freshly imported data
2808
- if (this.config.pruneHeartbeatOk) {
2809
- const pruned = await this.pruneHeartbeatOkTurns(conversationId);
2810
- if (pruned > 0) {
2811
- this.deps.log.info(
2812
- `[lcm] bootstrap: pruned ${pruned} HEARTBEAT_OK messages from conversation ${conversationId}`,
2813
- );
2814
- }
2815
- }
2816
-
2817
- await persistBootstrapState(conversationId);
2818
- this.deps.log.info(
2819
- `[lcm] bootstrap: initial import conversation=${conversationId} ${sessionLabel} importedMessages=${inserted.length} sourceMessages=${historicalMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
2820
- );
2821
-
2822
- return {
2823
- bootstrapped: true,
2824
- importedMessages: inserted.length,
2825
- };
2826
- }
2827
-
2828
- // Existing conversation path: reconcile crash gaps by appending JSONL
2829
- // messages that were never persisted to LCM.
2830
- const reconcile = await this.reconcileSessionTail({
2831
- sessionId: params.sessionId,
2832
- sessionKey: params.sessionKey,
2833
- conversationId,
2834
- historicalMessages,
2835
- });
2836
- this.deps.log.info(
2837
- `[lcm] bootstrap: reconcile finished conversation=${conversationId} ${sessionLabel} importedMessages=${reconcile.importedMessages} overlap=${reconcile.hasOverlap} blockedByImportCap=${reconcile.blockedByImportCap} duration=${formatDurationMs(Date.now() - startedAt)}`,
2838
- );
2839
-
2840
- if (reconcile.blockedByImportCap) {
2841
- return {
2842
- bootstrapped: false,
2843
- importedMessages: 0,
2844
- reason: "reconcile import capped",
2845
- };
2846
- }
2847
-
2848
- if (!conversation.bootstrappedAt) {
2849
- await this.conversationStore.markConversationBootstrapped(conversationId);
2850
- }
2851
-
2852
- if (reconcile.importedMessages > 0) {
2853
- await persistBootstrapState(conversationId);
2854
- return {
2855
- bootstrapped: true,
2856
- importedMessages: reconcile.importedMessages,
2857
- reason: "reconciled missing session messages",
2858
- };
2859
- }
2860
-
2861
- if (reconcile.hasOverlap) {
2862
- await persistBootstrapState(conversationId);
2863
- }
2864
-
2865
- if (conversation.bootstrappedAt) {
2866
- return {
2867
- bootstrapped: false,
2868
- importedMessages: 0,
2869
- reason: "already bootstrapped",
2870
- };
2871
- }
2872
-
2873
- return {
2874
- bootstrapped: false,
2875
- importedMessages: 0,
2876
- reason: reconcile.hasOverlap
2877
- ? "conversation already up to date"
2878
- : "conversation already has messages",
2879
- };
2880
- }),
2881
- { operationName: "bootstrap", context: sessionLabel },
2882
- );
2883
-
2884
- // Post-bootstrap pruning: clean HEARTBEAT_OK turns that were already
2885
- // in the DB from prior bootstrap cycles (before pruning was enabled).
2886
- if (this.config.pruneHeartbeatOk && result.bootstrapped === false) {
2887
- try {
2888
- const conversation = await this.conversationStore.getConversationForSession({
2889
- sessionId: params.sessionId,
2890
- sessionKey: params.sessionKey,
2891
- });
2892
- if (conversation) {
2893
- const pruned = await this.pruneHeartbeatOkTurns(conversation.conversationId);
2894
- if (pruned > 0) {
2895
- await this.refreshBootstrapState({
2896
- conversationId: conversation.conversationId,
2897
- sessionFile: params.sessionFile,
2898
- });
2899
- this.deps.log.info(
2900
- `[lcm] bootstrap: retroactively pruned ${pruned} HEARTBEAT_OK messages from conversation ${conversation.conversationId}`,
2901
- );
2902
- }
2903
- }
2904
- } catch (err) {
2905
- this.deps.log.warn(
2906
- `[lcm] bootstrap: heartbeat pruning failed: ${describeLogError(err)}`,
2907
- );
2908
- }
2909
- }
2910
-
2911
- this.deps.log.info(
2912
- `[lcm] bootstrap: done ${sessionLabel} bootstrapped=${result.bootstrapped} importedMessages=${result.importedMessages} reason=${result.reason ?? "none"} duration=${formatDurationMs(Date.now() - startedAt)}`,
2913
- );
2914
- return result;
2915
- }
2916
-
2917
- /**
2918
- * Remove messages from the batch that already exist in the DB for this session.
2919
- * Conservative replay detection: only strip a prefix when the incoming
2920
- * batch begins with the entire stored transcript for the session.
2921
- *
2922
- * Fixes two issues from #246:
2923
- * 1. Replaced hasMessage() fast-path with aligned-tail check — the old
2924
- * approach false-positives on legitimate repeated first messages
2925
- * 2. Dedup now runs on newMessages only, before autoCompactionSummary
2926
- * is prepended — synthetic summaries can no longer interfere with
2927
- * replay detection
2928
- */
2929
- private async deduplicateAfterTurnBatch(
2930
- sessionId: string,
2931
- sessionKey: string | undefined,
2932
- batch: AgentMessage[],
2933
- ): Promise<AgentMessage[]> {
2934
- if (batch.length === 0) return batch;
2935
-
2936
- const conversation = await this.conversationStore.getConversationForSession({
2937
- sessionId,
2938
- sessionKey,
2939
- });
2940
- if (!conversation) return batch;
2941
-
2942
- const conversationId = conversation.conversationId;
2943
- const storedMessageCount = await this.conversationStore.getMessageCount(conversationId);
2944
- if (storedMessageCount === 0 || storedMessageCount > batch.length) {
2945
- return batch;
2946
- }
2947
-
2948
- // Aligned-tail check: DB's last message must match the message at the
2949
- // exact replay boundary in the incoming batch. This replaces the
2950
- // hasMessage() check which could false-positive on any repeated content.
2951
- const lastDbMessage = await this.conversationStore.getLastMessage(conversationId);
2952
- if (!lastDbMessage) return batch;
2953
-
2954
- const storedBatch = batch.map((m) => toStoredMessage(m));
2955
- const batchAtBoundary = storedBatch[storedMessageCount - 1]!;
2956
- if (
2957
- messageIdentity(lastDbMessage.role, lastDbMessage.content) !==
2958
- messageIdentity(batchAtBoundary.role, batchAtBoundary.content)
2959
- ) {
2960
- return batch;
2961
- }
2962
-
2963
- // Full proof: incoming batch must start with the entire stored transcript
2964
- // in exact order before we trim anything.
2965
- const storedMessages = await this.conversationStore.getMessages(conversationId, {
2966
- limit: storedMessageCount,
2967
- });
2968
- if (storedMessages.length !== storedMessageCount) {
2969
- return batch;
2970
- }
2971
- for (let i = 0; i < storedMessageCount; i += 1) {
2972
- const storedConversationMessage = storedMessages[i]!;
2973
- const incomingMessage = storedBatch[i]!;
2974
- if (
2975
- messageIdentity(storedConversationMessage.role, storedConversationMessage.content) !==
2976
- messageIdentity(incomingMessage.role, incomingMessage.content)
2977
- ) {
2978
- return batch;
2979
- }
2980
- }
2981
-
2982
- return batch.slice(storedMessageCount);
2983
- }
2984
- /**
2985
- * Rebuild a compact tool-result message from stored message parts.
2986
- *
2987
- * The first transcript-GC pass only rewrites tool results that were already
2988
- * externalized into large_files during ingest, so the stored placeholder is
2989
- * the canonical replacement content.
2990
- */
2991
- private async buildTranscriptGcReplacementMessage(
2992
- messageId: number,
2993
- ): Promise<AgentMessage | null> {
2994
- const message = await this.conversationStore.getMessageById(messageId);
2995
- if (!message) {
2996
- return null;
2997
- }
2998
-
2999
- const parts = await this.conversationStore.getMessageParts(messageId);
3000
- const toolCallId = pickToolCallId(parts);
3001
- if (!toolCallId) {
3002
- return null;
3003
- }
3004
-
3005
- const content = contentFromParts(parts, "toolResult", message.content);
3006
- const toolName = pickToolName(parts) ?? "unknown";
3007
- const isError = pickToolIsError(parts);
3008
-
3009
- return {
3010
- role: "toolResult",
3011
- toolCallId,
3012
- toolName,
3013
- content,
3014
- ...(isError !== undefined ? { isError } : {}),
3015
- } as AgentMessage;
3016
- }
3017
-
3018
- /**
3019
- * Run transcript GC for summarized tool-result messages that already have a
3020
- * large_files-backed placeholder stored in LCM.
3021
- */
3022
- async maintain(params: {
3023
- sessionId: string;
3024
- sessionFile: string;
3025
- sessionKey?: string;
3026
- runtimeContext?: ContextEngineMaintenanceRuntimeContext;
3027
- }): Promise<ContextEngineMaintenanceResult> {
3028
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
3029
- return {
3030
- changed: false,
3031
- bytesFreed: 0,
3032
- rewrittenEntries: 0,
3033
- reason: "session excluded by pattern",
3034
- };
3035
- }
3036
- if (this.isStatelessSession(params.sessionKey)) {
3037
- return {
3038
- changed: false,
3039
- bytesFreed: 0,
3040
- rewrittenEntries: 0,
3041
- reason: "stateless session",
3042
- };
3043
- }
3044
- if (typeof params.runtimeContext?.rewriteTranscriptEntries !== "function") {
3045
- return {
3046
- changed: false,
3047
- bytesFreed: 0,
3048
- rewrittenEntries: 0,
3049
- reason: "runtime rewrite helper unavailable",
3050
- };
3051
- }
3052
-
3053
- const rewriteTranscriptEntries = params.runtimeContext.rewriteTranscriptEntries;
3054
- const startedAt = Date.now();
3055
- const sessionLabel = [
3056
- `session=${params.sessionId}`,
3057
- ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
3058
- ].join(" ");
3059
- return this.withSessionQueue(
3060
- this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
3061
- async () => {
3062
- const conversation = await this.conversationStore.getConversationForSession({
3063
- sessionId: params.sessionId,
3064
- sessionKey: params.sessionKey,
3065
- });
3066
- if (!conversation) {
3067
- return {
3068
- changed: false,
3069
- bytesFreed: 0,
3070
- rewrittenEntries: 0,
3071
- reason: "conversation not found",
3072
- };
3073
- }
3074
-
3075
- const candidates = await this.summaryStore.listTranscriptGcCandidates(
3076
- conversation.conversationId,
3077
- { limit: TRANSCRIPT_GC_BATCH_SIZE },
3078
- );
3079
- if (candidates.length === 0) {
3080
- this.deps.log.info(
3081
- `[lcm] maintain: no transcript GC candidates conversation=${conversation.conversationId} ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
3082
- );
3083
- return {
3084
- changed: false,
3085
- bytesFreed: 0,
3086
- rewrittenEntries: 0,
3087
- reason: "no transcript GC candidates",
3088
- };
3089
- }
3090
-
3091
- const transcriptEntryIdsByCallId = listTranscriptToolResultEntryIdsByCallId(
3092
- params.sessionFile,
3093
- );
3094
- const replacements: TranscriptRewriteReplacement[] = [];
3095
- const seenEntryIds = new Set<string>();
3096
-
3097
- for (const candidate of candidates) {
3098
- const entryId = transcriptEntryIdsByCallId.get(candidate.toolCallId);
3099
- if (!entryId || seenEntryIds.has(entryId)) {
3100
- continue;
3101
- }
3102
-
3103
- const replacementMessage = await this.buildTranscriptGcReplacementMessage(
3104
- candidate.messageId,
3105
- );
3106
- if (!replacementMessage) {
3107
- continue;
3108
- }
3109
-
3110
- seenEntryIds.add(entryId);
3111
- replacements.push({
3112
- entryId,
3113
- message: replacementMessage,
3114
- });
3115
- }
3116
-
3117
- if (replacements.length === 0) {
3118
- this.deps.log.info(
3119
- `[lcm] maintain: no matching transcript entries conversation=${conversation.conversationId} ${sessionLabel} candidates=${candidates.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
3120
- );
3121
- return {
3122
- changed: false,
3123
- bytesFreed: 0,
3124
- rewrittenEntries: 0,
3125
- reason: "no matching transcript entries",
3126
- };
3127
- }
3128
-
3129
- const result = await rewriteTranscriptEntries({
3130
- replacements,
3131
- });
3132
-
3133
- if (result.changed) {
3134
- try {
3135
- await this.refreshBootstrapState({
3136
- conversationId: conversation.conversationId,
3137
- sessionFile: params.sessionFile,
3138
- });
3139
- } catch (e) {
3140
- this.deps.log.warn(
3141
- `[lcm] Failed to update bootstrap checkpoint after maintain: ${describeLogError(e)}`,
3142
- );
3143
- }
3144
- }
3145
-
3146
- this.deps.log.info(
3147
- `[lcm] maintain: done conversation=${conversation.conversationId} ${sessionLabel} candidates=${candidates.length} replacements=${replacements.length} changed=${result.changed} rewrittenEntries=${result.rewrittenEntries} bytesFreed=${result.bytesFreed} duration=${formatDurationMs(Date.now() - startedAt)}`,
3148
- );
3149
- return result;
3150
- },
3151
- { operationName: "maintain", context: sessionLabel },
3152
- );
3153
- }
3154
- private async ingestSingle(params: {
3155
- sessionId: string;
3156
- sessionKey?: string;
3157
- message: AgentMessage;
3158
- isHeartbeat?: boolean;
3159
- }): Promise<IngestResult> {
3160
- const { sessionId, sessionKey, message, isHeartbeat } = params;
3161
- if (isHeartbeat) {
3162
- return { ingested: false };
3163
- }
3164
-
3165
- // Skip assistant messages that failed with an error and have no useful content.
3166
- // These occur when an API call returns a 500 or similar transient error.
3167
- // Ingesting them pollutes the LCM database: on retry, the error messages
3168
- // accumulate and get assembled into context, creating a positive feedback
3169
- // loop where each retry sends an increasingly large (and malformed) payload
3170
- // that continues to fail.
3171
- if (message.role === "assistant") {
3172
- const topLevel = message as unknown as Record<string, unknown>;
3173
- const stopReason =
3174
- typeof topLevel.stopReason === "string"
3175
- ? topLevel.stopReason
3176
- : typeof topLevel.stop_reason === "string"
3177
- ? topLevel.stop_reason
3178
- : undefined;
3179
- if (stopReason === "error" || stopReason === "aborted") {
3180
- const content = topLevel.content;
3181
- const isEmpty =
3182
- content === undefined ||
3183
- content === null ||
3184
- content === "" ||
3185
- (Array.isArray(content) && content.length === 0);
3186
- if (isEmpty) {
3187
- return { ingested: false };
3188
- }
3189
- }
3190
- }
3191
-
3192
- const stored = toStoredMessage(message);
3193
-
3194
- // Get or create conversation for this session
3195
- const conversation = await this.conversationStore.getOrCreateConversation(sessionId, {
3196
- sessionKey,
3197
- });
3198
- const conversationId = conversation.conversationId;
3199
-
3200
- let messageForParts = message;
3201
- if (stored.role === "user") {
3202
- const intercepted = await this.interceptLargeFiles({
3203
- conversationId,
3204
- content: stored.content,
3205
- });
3206
- if (intercepted) {
3207
- stored.content = intercepted.rewrittenContent;
3208
- stored.tokenCount = estimateTokens(stored.content);
3209
- if ("content" in message) {
3210
- messageForParts = {
3211
- ...message,
3212
- content: stored.content,
3213
- } as AgentMessage;
3214
- }
3215
- }
3216
- } else if (stored.role === "tool") {
3217
- const intercepted = await this.interceptLargeToolResults({
3218
- conversationId,
3219
- message,
3220
- });
3221
- if (intercepted) {
3222
- messageForParts = intercepted.rewrittenMessage;
3223
- const rewrittenStored = toStoredMessage(intercepted.rewrittenMessage);
3224
- stored.content = rewrittenStored.content;
3225
- stored.tokenCount = rewrittenStored.tokenCount;
3226
- }
3227
- }
3228
-
3229
- // Determine next sequence number
3230
- const maxSeq = await this.conversationStore.getMaxSeq(conversationId);
3231
- const seq = maxSeq + 1;
3232
-
3233
- // Persist the message
3234
- const msgRecord = await this.conversationStore.createMessage({
3235
- conversationId,
3236
- seq,
3237
- role: stored.role,
3238
- content: stored.content,
3239
- tokenCount: stored.tokenCount,
3240
- });
3241
- await this.conversationStore.createMessageParts(
3242
- msgRecord.messageId,
3243
- buildMessageParts({
3244
- sessionId,
3245
- message: messageForParts,
3246
- fallbackContent: stored.content,
3247
- }),
3248
- );
3249
-
3250
- // Append to context items so assembler can see it
3251
- await this.summaryStore.appendContextMessage(conversationId, msgRecord.messageId);
3252
-
3253
- return { ingested: true };
3254
- }
3255
-
3256
- async ingest(params: {
3257
- sessionId: string;
3258
- sessionKey?: string;
3259
- message: AgentMessage;
3260
- isHeartbeat?: boolean;
3261
- }): Promise<IngestResult> {
3262
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
3263
- return { ingested: false };
3264
- }
3265
- if (this.isStatelessSession(params.sessionKey)) {
3266
- return { ingested: false };
3267
- }
3268
- this.ensureMigrated();
3269
- return this.withSessionQueue(
3270
- this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
3271
- () => this.ingestSingle(params),
3272
- {
3273
- operationName: "ingest",
3274
- context: [
3275
- `session=${params.sessionId}`,
3276
- ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
3277
- ].join(" "),
3278
- },
3279
- );
3280
- }
3281
-
3282
- async ingestBatch(params: {
3283
- sessionId: string;
3284
- sessionKey?: string;
3285
- messages: AgentMessage[];
3286
- isHeartbeat?: boolean;
3287
- }): Promise<IngestBatchResult> {
3288
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
3289
- return { ingestedCount: 0 };
3290
- }
3291
- if (this.isStatelessSession(params.sessionKey)) {
3292
- return { ingestedCount: 0 };
3293
- }
3294
- this.ensureMigrated();
3295
- if (params.messages.length === 0) {
3296
- return { ingestedCount: 0 };
3297
- }
3298
- return this.withSessionQueue(
3299
- this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
3300
- async () => {
3301
- let ingestedCount = 0;
3302
- for (const message of params.messages) {
3303
- const result = await this.ingestSingle({
3304
- sessionId: params.sessionId,
3305
- sessionKey: params.sessionKey,
3306
- message,
3307
- isHeartbeat: params.isHeartbeat,
3308
- });
3309
- if (result.ingested) {
3310
- ingestedCount += 1;
3311
- }
3312
- }
3313
- return { ingestedCount };
3314
- },
3315
- {
3316
- operationName: "ingestBatch",
3317
- context: [
3318
- `session=${params.sessionId}`,
3319
- ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
3320
- `messages=${params.messages.length}`,
3321
- ].join(" "),
3322
- },
3323
- );
3324
- }
3325
-
3326
- async afterTurn(params: {
3327
- sessionId: string;
3328
- sessionKey?: string;
3329
- sessionFile: string;
3330
- messages: AgentMessage[];
3331
- prePromptMessageCount: number;
3332
- autoCompactionSummary?: string;
3333
- isHeartbeat?: boolean;
3334
- tokenBudget?: number;
3335
- /** OpenClaw runtime param name (preferred). */
3336
- runtimeContext?: Record<string, unknown>;
3337
- /** Back-compat param name. */
3338
- legacyCompactionParams?: Record<string, unknown>;
3339
- }): Promise<void> {
3340
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
3341
- return;
3342
- }
3343
- if (this.isStatelessSession(params.sessionKey)) {
3344
- return;
3345
- }
3346
- this.ensureMigrated();
3347
- const startedAt = Date.now();
3348
- const sessionLabel = [
3349
- `session=${params.sessionId}`,
3350
- ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
3351
- ].join(" ");
3352
-
3353
- // Dedup guard: prevent duplicate ingestion when gateway restart replays
3354
- // full history. Run on newMessages BEFORE prepending autoCompactionSummary
3355
- // so synthetic summaries cannot interfere with replay detection.
3356
- const newMessages = params.messages.slice(params.prePromptMessageCount);
3357
- const dedupedNewMessages = await this.deduplicateAfterTurnBatch(
3358
- params.sessionId,
3359
- params.sessionKey,
3360
- newMessages,
3361
- );
3362
-
3363
- const ingestBatch: AgentMessage[] = [];
3364
- if (params.autoCompactionSummary) {
3365
- ingestBatch.push({
3366
- role: "user",
3367
- content: params.autoCompactionSummary,
3368
- } as AgentMessage);
3369
- }
3370
-
3371
- ingestBatch.push(...dedupedNewMessages);
3372
- if (ingestBatch.length === 0) {
3373
- this.deps.log.info(
3374
- `[lcm] afterTurn: nothing to ingest ${sessionLabel} newMessages=${newMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
3375
- );
3376
- return;
3377
- }
3378
-
3379
- try {
3380
- await this.ingestBatch({
3381
- sessionId: params.sessionId,
3382
- sessionKey: params.sessionKey,
3383
- messages: ingestBatch,
3384
- isHeartbeat: params.isHeartbeat === true,
3385
- });
3386
- } catch (err) {
3387
- // Never compact a stale or partially ingested frontier.
3388
- this.deps.log.error(
3389
- `[lcm] afterTurn: ingest failed, skipping compaction: ${describeLogError(err)}`,
3390
- );
3391
- return;
3392
- }
3393
-
3394
- if (batchLooksLikeHeartbeatAckTurn(ingestBatch)) {
3395
- try {
3396
- const conversation = await this.conversationStore.getConversationForSession({
3397
- sessionId: params.sessionId,
3398
- sessionKey: params.sessionKey,
3399
- });
3400
- if (conversation) {
3401
- const pruned = await this.pruneHeartbeatOkTurns(conversation.conversationId);
3402
- if (pruned > 0) {
3403
- const sessionContext = this.formatSessionLogContext({
3404
- conversationId: conversation.conversationId,
3405
- sessionId: params.sessionId,
3406
- sessionKey: params.sessionKey,
3407
- });
3408
- try {
3409
- await this.refreshBootstrapState({
3410
- conversationId: conversation.conversationId,
3411
- sessionFile: params.sessionFile,
3412
- });
3413
- } catch (err) {
3414
- this.deps.log.warn(
3415
- `[lcm] afterTurn: heartbeat pruning checkpoint refresh failed for ${sessionContext}: ${describeLogError(err)}`,
3416
- );
3417
- }
3418
- this.deps.log.info(
3419
- `[lcm] afterTurn: pruned ${pruned} heartbeat ack messages for ${sessionContext}`,
3420
- );
3421
- return;
3422
- }
3423
- }
3424
- } catch (err) {
3425
- this.deps.log.warn(
3426
- `[lcm] afterTurn: heartbeat pruning failed: ${describeLogError(err)}`,
3427
- );
3428
- }
3429
- }
3430
-
3431
- const legacyParams = asRecord(params.runtimeContext) ?? asRecord(params.legacyCompactionParams);
3432
- const DEFAULT_AFTER_TURN_TOKEN_BUDGET = 128_000;
3433
- const resolvedTokenBudget = this.resolveTokenBudget({
3434
- tokenBudget: params.tokenBudget,
3435
- runtimeContext: params.runtimeContext,
3436
- legacyParams,
3437
- });
3438
- const tokenBudget = this.applyAssemblyBudgetCap(resolvedTokenBudget ?? DEFAULT_AFTER_TURN_TOKEN_BUDGET);
3439
- if (resolvedTokenBudget === undefined) {
3440
- this.deps.log.warn(
3441
- `[lcm] afterTurn: tokenBudget not provided; using default ${DEFAULT_AFTER_TURN_TOKEN_BUDGET}`,
3442
- );
3443
- }
3444
-
3445
- const liveContextTokens = estimateSessionTokenCountForAfterTurn(params.messages);
3446
- const conversation = await this.conversationStore.getConversationForSession({
3447
- sessionId: params.sessionId,
3448
- sessionKey: params.sessionKey,
3449
- });
3450
- if (!conversation) {
3451
- this.deps.log.info(
3452
- `[lcm] afterTurn: conversation lookup missed ${sessionLabel} ingestBatch=${ingestBatch.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
3453
- );
3454
- return;
3455
- }
3456
-
3457
- try {
3458
- const rawLeafTrigger = await this.compaction.evaluateLeafTrigger(conversation.conversationId);
3459
- await this.updateCompactionTelemetry({
3460
- conversationId: conversation.conversationId,
3461
- runtimeContext: asRecord(params.runtimeContext),
3462
- tokenBudget,
3463
- rawTokensOutsideTail: rawLeafTrigger.rawTokensOutsideTail,
3464
- });
3465
- } catch (err) {
3466
- this.deps.log.warn(
3467
- `[lcm] afterTurn: compaction telemetry update failed: ${describeLogError(err)}`,
3468
- );
3469
- }
3470
-
3471
- try {
3472
- const leafDecision = await this.evaluateIncrementalCompaction({
3473
- conversationId: conversation.conversationId,
3474
- tokenBudget,
3475
- currentTokenCount: liveContextTokens,
3476
- });
3477
- if (leafDecision.shouldCompact) {
3478
- this.compactLeafAsync({
3479
- sessionId: params.sessionId,
3480
- sessionKey: params.sessionKey,
3481
- sessionFile: params.sessionFile,
3482
- tokenBudget,
3483
- currentTokenCount: liveContextTokens,
3484
- legacyParams,
3485
- maxPasses: leafDecision.maxPasses,
3486
- leafChunkTokens: leafDecision.leafChunkTokens,
3487
- fallbackLeafChunkTokens: leafDecision.fallbackLeafChunkTokens,
3488
- activityBand: leafDecision.activityBand,
3489
- allowCondensedPasses: leafDecision.allowCondensedPasses,
3490
- }).catch(() => {
3491
- // Leaf compaction is best-effort and should not fail the caller.
3492
- });
3493
- }
3494
- } catch {
3495
- // Leaf trigger checks are best-effort.
3496
- }
3497
-
3498
- try {
3499
- await this.compact({
3500
- sessionId: params.sessionId,
3501
- sessionKey: params.sessionKey,
3502
- sessionFile: params.sessionFile,
3503
- tokenBudget,
3504
- currentTokenCount: liveContextTokens,
3505
- compactionTarget: "threshold",
3506
- legacyParams,
3507
- });
3508
- } catch {
3509
- // Proactive compaction is best-effort in the post-turn lifecycle.
3510
- }
3511
-
3512
- this.deps.log.info(
3513
- `[lcm] afterTurn: done conversation=${conversation.conversationId} ${sessionLabel} newMessages=${newMessages.length} dedupedMessages=${dedupedNewMessages.length} ingestedMessages=${ingestBatch.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
3514
- );
3515
- }
3516
-
3517
- async assemble(params: {
3518
- sessionId: string;
3519
- sessionKey?: string;
3520
- messages: AgentMessage[];
3521
- tokenBudget?: number;
3522
- /** Optional user query for relevance-based eviction (BM25-lite). When absent or unsearchable, falls back to chronological eviction. */
3523
- prompt?: string;
3524
- }): Promise<AssembleResult> {
3525
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
3526
- return {
3527
- messages: params.messages,
3528
- estimatedTokens: 0,
3529
- };
3530
- }
3531
- try {
3532
- this.ensureMigrated();
3533
- const startedAt = Date.now();
3534
- const sessionLabel = [
3535
- `session=${params.sessionId}`,
3536
- ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
3537
- ].join(" ");
3538
-
3539
- const conversation = await this.conversationStore.getConversationForSession({
3540
- sessionId: params.sessionId,
3541
- sessionKey: params.sessionKey,
3542
- });
3543
- if (!conversation) {
3544
- this.deps.log.info(
3545
- `[lcm] assemble: conversation lookup missed ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
3546
- );
3547
- return {
3548
- messages: params.messages,
3549
- estimatedTokens: 0,
3550
- };
3551
- }
3552
-
3553
- const contextItems = await this.summaryStore.getContextItems(conversation.conversationId);
3554
- if (contextItems.length === 0) {
3555
- this.deps.log.info(
3556
- `[lcm] assemble: no context items conversation=${conversation.conversationId} ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
3557
- );
3558
- return {
3559
- messages: params.messages,
3560
- estimatedTokens: 0,
3561
- };
3562
- }
3563
-
3564
- // Guard against incomplete bootstrap/coverage: if the DB only has
3565
- // raw context items and clearly trails the current live history, keep
3566
- // the live path to avoid dropping prompt context.
3567
- const hasSummaryItems = contextItems.some((item) => item.itemType === "summary");
3568
- if (!hasSummaryItems && contextItems.length < params.messages.length) {
3569
- this.deps.log.info(
3570
- `[lcm] assemble: falling back to live context conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} liveMessages=${params.messages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
3571
- );
3572
- return {
3573
- messages: params.messages,
3574
- estimatedTokens: 0,
3575
- };
3576
- }
3577
-
3578
- const tokenBudget = this.applyAssemblyBudgetCap(
3579
- typeof params.tokenBudget === "number" &&
3580
- Number.isFinite(params.tokenBudget) &&
3581
- params.tokenBudget > 0
3582
- ? Math.floor(params.tokenBudget)
3583
- : 128_000,
3584
- );
3585
-
3586
- const assembled = await this.assembler.assemble({
3587
- conversationId: conversation.conversationId,
3588
- tokenBudget,
3589
- freshTailCount: this.config.freshTailCount,
3590
- prompt: params.prompt,
3591
- });
3592
-
3593
- // If assembly produced no messages for a non-empty live session,
3594
- // fail safe to the live context.
3595
- if (assembled.messages.length === 0 && params.messages.length > 0) {
3596
- this.deps.log.info(
3597
- `[lcm] assemble: empty assembled output, using live context conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} tokenBudget=${tokenBudget} duration=${formatDurationMs(Date.now() - startedAt)}`,
3598
- );
3599
- return {
3600
- messages: params.messages,
3601
- estimatedTokens: 0,
3602
- };
3603
- }
3604
-
3605
- this.deps.log.info(
3606
- `[lcm] assemble: done conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} hasSummaryItems=${hasSummaryItems} inputMessages=${params.messages.length} outputMessages=${assembled.messages.length} tokenBudget=${tokenBudget} estimatedTokens=${assembled.estimatedTokens} duration=${formatDurationMs(Date.now() - startedAt)}`,
3607
- );
3608
-
3609
- const result: AssembleResultWithSystemPrompt = {
3610
- messages: assembled.messages,
3611
- estimatedTokens: assembled.estimatedTokens,
3612
- ...(assembled.systemPromptAddition
3613
- ? { systemPromptAddition: assembled.systemPromptAddition }
3614
- : {}),
3615
- };
3616
- return result;
3617
- } catch (err) {
3618
- this.deps.log.info(
3619
- `[lcm] assemble: failed for session=${params.sessionId}${params.sessionKey?.trim() ? ` sessionKey=${params.sessionKey.trim()}` : ""} error=${describeLogError(err)}`,
3620
- );
3621
- return {
3622
- messages: params.messages,
3623
- estimatedTokens: 0,
3624
- };
3625
- }
3626
- }
3627
-
3628
- /** Evaluate whether incremental leaf compaction should run for a session. */
3629
- async evaluateLeafTrigger(sessionId: string, sessionKey?: string): Promise<{
3630
- shouldCompact: boolean;
3631
- rawTokensOutsideTail: number;
3632
- threshold: number;
3633
- }> {
3634
- this.ensureMigrated();
3635
- const conversation = await this.conversationStore.getConversationForSession({
3636
- sessionId,
3637
- sessionKey,
3638
- });
3639
- if (!conversation) {
3640
- const fallbackThreshold =
3641
- typeof this.config.leafChunkTokens === "number" &&
3642
- Number.isFinite(this.config.leafChunkTokens) &&
3643
- this.config.leafChunkTokens > 0
3644
- ? Math.floor(this.config.leafChunkTokens)
3645
- : 20_000;
3646
- return {
3647
- shouldCompact: false,
3648
- rawTokensOutsideTail: 0,
3649
- threshold: fallbackThreshold,
3650
- };
3651
- }
3652
- return this.compaction.evaluateLeafTrigger(conversation.conversationId);
3653
- }
3654
-
3655
- /** Run one or more incremental leaf compaction passes in the per-session queue. */
3656
- async compactLeafAsync(params: {
3657
- sessionId: string;
3658
- sessionKey?: string;
3659
- sessionFile: string;
3660
- tokenBudget?: number;
3661
- currentTokenCount?: number;
3662
- customInstructions?: string;
3663
- /** OpenClaw runtime param name (preferred). */
3664
- runtimeContext?: Record<string, unknown>;
3665
- /** Back-compat param name. */
3666
- legacyParams?: Record<string, unknown>;
3667
- force?: boolean;
3668
- previousSummaryContent?: string;
3669
- maxPasses?: number;
3670
- leafChunkTokens?: number;
3671
- fallbackLeafChunkTokens?: number[];
3672
- activityBand?: ActivityBand;
3673
- allowCondensedPasses?: boolean;
3674
- }): Promise<CompactResult> {
3675
- if (this.isStatelessSession(params.sessionKey)) {
3676
- return {
3677
- ok: true,
3678
- compacted: false,
3679
- reason: "stateless session",
3680
- };
3681
- }
3682
- this.ensureMigrated();
3683
- return this.withSessionQueue(
3684
- this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
3685
- async () => {
3686
- const conversation = await this.conversationStore.getConversationForSession({
3687
- sessionId: params.sessionId,
3688
- sessionKey: params.sessionKey,
3689
- });
3690
- if (!conversation) {
3691
- return {
3692
- ok: true,
3693
- compacted: false,
3694
- reason: "no conversation found for session",
3695
- };
3696
- }
3697
-
3698
- const legacyParams = asRecord(params.runtimeContext) ?? params.legacyParams;
3699
- const resolvedTokenBudget = this.resolveTokenBudget({
3700
- tokenBudget: params.tokenBudget,
3701
- runtimeContext: params.runtimeContext,
3702
- legacyParams,
3703
- });
3704
- const tokenBudget = resolvedTokenBudget
3705
- ? this.applyAssemblyBudgetCap(resolvedTokenBudget)
3706
- : resolvedTokenBudget;
3707
- if (!tokenBudget) {
3708
- return {
3709
- ok: false,
3710
- compacted: false,
3711
- reason: "missing token budget in compact params",
3712
- };
3713
- }
3714
-
3715
- const lp = legacyParams ?? {};
3716
- const observedTokens = this.normalizeObservedTokenCount(
3717
- params.currentTokenCount ??
3718
- (
3719
- lp as {
3720
- currentTokenCount?: unknown;
3721
- }
3722
- ).currentTokenCount,
3723
- );
3724
- const { summarize, summaryModel, breakerKey } = await this.resolveSummarize({
3725
- legacyParams,
3726
- customInstructions: params.customInstructions,
3727
- breakerScope: this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
3728
- });
3729
- if (breakerKey && this.isCircuitBreakerOpen(breakerKey)) {
3730
- return {
3731
- ok: true,
3732
- compacted: false,
3733
- reason: "circuit breaker open",
3734
- };
3735
- }
3736
-
3737
- const storedTokensBefore = await this.summaryStore.getContextTokenCount(
3738
- conversation.conversationId,
3739
- );
3740
- const maxPasses =
3741
- typeof params.maxPasses === "number" &&
3742
- Number.isFinite(params.maxPasses) &&
3743
- params.maxPasses > 0
3744
- ? Math.floor(params.maxPasses)
3745
- : 1;
3746
- const fallbackLeafChunkTokens = Array.isArray(params.fallbackLeafChunkTokens)
3747
- ? [...new Set(params.fallbackLeafChunkTokens
3748
- .filter((value): value is number => typeof value === "number" && Number.isFinite(value) && value > 0)
3749
- .map((value) => Math.floor(value)))]
3750
- .sort((a, b) => b - a)
3751
- : [];
3752
- let activeLeafChunkTokens =
3753
- typeof params.leafChunkTokens === "number" &&
3754
- Number.isFinite(params.leafChunkTokens) &&
3755
- params.leafChunkTokens > 0
3756
- ? Math.floor(params.leafChunkTokens)
3757
- : fallbackLeafChunkTokens[0];
3758
- this.deps.log.info(
3759
- `[lcm] compactLeafAsync start: conversation=${conversation.conversationId} session=${params.sessionId} leafChunkTokens=${activeLeafChunkTokens ?? "null"} fallbackLeafChunkTokens=${fallbackLeafChunkTokens.join(",")} maxPasses=${maxPasses} activityBand=${params.activityBand ?? "unknown"} allowCondensedPasses=${params.allowCondensedPasses !== false}`,
3760
- );
3761
-
3762
- let rounds = 0;
3763
- let finalTokens = observedTokens ?? storedTokensBefore;
3764
- let authFailure = false;
3765
-
3766
- for (let pass = 0; pass < maxPasses; pass += 1) {
3767
- let leafResult: Awaited<ReturnType<typeof this.compaction.compactLeaf>> | undefined;
3768
- while (true) {
3769
- try {
3770
- leafResult = await this.compaction.compactLeaf({
3771
- conversationId: conversation.conversationId,
3772
- tokenBudget,
3773
- summarize,
3774
- ...(activeLeafChunkTokens !== undefined ? { leafChunkTokens: activeLeafChunkTokens } : {}),
3775
- force: params.force,
3776
- previousSummaryContent: pass === 0 ? params.previousSummaryContent : undefined,
3777
- summaryModel,
3778
- allowCondensedPasses: params.allowCondensedPasses,
3779
- });
3780
- break;
3781
- } catch (err) {
3782
- const nextLeafChunkTokens = fallbackLeafChunkTokens.find(
3783
- (value) => activeLeafChunkTokens !== undefined && value < activeLeafChunkTokens,
3784
- );
3785
- if (!this.isRecoverableLeafChunkOverflowError(err) || nextLeafChunkTokens === undefined) {
3786
- throw err;
3787
- }
3788
- this.deps.log.warn(
3789
- `[lcm] compactLeafAsync: retrying with smaller leafChunkTokens=${nextLeafChunkTokens} after provider token-limit error: ${err instanceof Error ? err.message : String(err)}`,
3790
- );
3791
- activeLeafChunkTokens = nextLeafChunkTokens;
3792
- }
3793
- }
3794
- if (!leafResult) {
3795
- break;
3796
- }
3797
- finalTokens = leafResult.tokensAfter;
3798
-
3799
- if (leafResult.authFailure) {
3800
- authFailure = true;
3801
- break;
3802
- }
3803
- if (!leafResult.actionTaken) {
3804
- break;
3805
- }
3806
- rounds += 1;
3807
- if (leafResult.tokensAfter >= leafResult.tokensBefore) {
3808
- break;
3809
- }
3810
- }
3811
-
3812
- if (authFailure && breakerKey) {
3813
- this.recordCompactionAuthFailure(breakerKey);
3814
- } else if (rounds > 0 && breakerKey) {
3815
- this.recordCompactionSuccess(breakerKey);
3816
- }
3817
- if (rounds > 0) {
3818
- await this.markLeafCompactionTelemetrySuccess({
3819
- conversationId: conversation.conversationId,
3820
- activityBand: params.activityBand,
3821
- });
3822
- }
3823
-
3824
- const tokensBefore = observedTokens ?? storedTokensBefore;
3825
- this.deps.log.debug(
3826
- `[lcm] compactLeafAsync result: conversation=${conversation.conversationId} session=${params.sessionId} rounds=${rounds} compacted=${rounds > 0} authFailure=${authFailure} finalLeafChunkTokens=${activeLeafChunkTokens ?? "null"} finalTokens=${finalTokens}`,
3827
- );
3828
-
3829
- return {
3830
- ok: true,
3831
- compacted: rounds > 0,
3832
- reason: authFailure
3833
- ? "provider auth failure"
3834
- : rounds > 0
3835
- ? "compacted"
3836
- : "below threshold",
3837
- result: {
3838
- tokensBefore,
3839
- tokensAfter: finalTokens,
3840
- details: {
3841
- rounds,
3842
- targetTokens: tokenBudget,
3843
- mode: "leaf",
3844
- maxPasses,
3845
- },
3846
- },
3847
- };
3848
- },
3849
- );
3850
- }
3851
-
3852
- async compact(params: {
3853
- sessionId: string;
3854
- sessionKey?: string;
3855
- sessionFile: string;
3856
- tokenBudget?: number;
3857
- currentTokenCount?: number;
3858
- compactionTarget?: "budget" | "threshold";
3859
- customInstructions?: string;
3860
- /** OpenClaw runtime param name (preferred). */
3861
- runtimeContext?: Record<string, unknown>;
3862
- /** Back-compat param name. */
3863
- legacyParams?: Record<string, unknown>;
3864
- /** Force compaction even if below threshold */
3865
- force?: boolean;
3866
- }): Promise<CompactResult> {
3867
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
3868
- return {
3869
- ok: true,
3870
- compacted: false,
3871
- reason: "session excluded",
3872
- };
3873
- }
3874
- if (this.isStatelessSession(params.sessionKey)) {
3875
- return {
3876
- ok: true,
3877
- compacted: false,
3878
- reason: "stateless session",
3879
- };
3880
- }
3881
- this.ensureMigrated();
3882
- return this.withSessionQueue(
3883
- this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
3884
- async () => {
3885
- const { sessionId, force = false } = params;
3886
-
3887
- // Look up conversation
3888
- const conversation = await this.conversationStore.getConversationForSession({
3889
- sessionId,
3890
- sessionKey: params.sessionKey,
3891
- });
3892
- if (!conversation) {
3893
- return {
3894
- ok: true,
3895
- compacted: false,
3896
- reason: "no conversation found for session",
3897
- };
3898
- }
3899
-
3900
- const conversationId = conversation.conversationId;
3901
-
3902
- const legacyParams = asRecord(params.runtimeContext) ?? params.legacyParams;
3903
- const lp = legacyParams ?? {};
3904
- const manualCompactionRequested =
3905
- (
3906
- lp as {
3907
- manualCompaction?: unknown;
3908
- }
3909
- ).manualCompaction === true;
3910
- const forceCompaction = force || manualCompactionRequested;
3911
- const resolvedTokenBudget = this.resolveTokenBudget({
3912
- tokenBudget: params.tokenBudget,
3913
- runtimeContext: params.runtimeContext,
3914
- legacyParams,
3915
- });
3916
- const tokenBudget = resolvedTokenBudget
3917
- ? this.applyAssemblyBudgetCap(resolvedTokenBudget)
3918
- : resolvedTokenBudget;
3919
- if (!tokenBudget) {
3920
- return {
3921
- ok: false,
3922
- compacted: false,
3923
- reason: "missing token budget in compact params",
3924
- };
3925
- }
3926
-
3927
- const { summarize, summaryModel, breakerKey } = await this.resolveSummarize({
3928
- legacyParams,
3929
- customInstructions: params.customInstructions,
3930
- breakerScope: this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
3931
- });
3932
- if (breakerKey && this.isCircuitBreakerOpen(breakerKey)) {
3933
- return {
3934
- ok: true,
3935
- compacted: false,
3936
- reason: "circuit breaker open",
3937
- };
3938
- }
3939
-
3940
- // Evaluate whether compaction is needed (unless forced)
3941
- const observedTokens = this.normalizeObservedTokenCount(
3942
- params.currentTokenCount ??
3943
- (
3944
- lp as {
3945
- currentTokenCount?: unknown;
3946
- }
3947
- ).currentTokenCount,
3948
- );
3949
- const decision =
3950
- observedTokens !== undefined
3951
- ? await this.compaction.evaluate(conversationId, tokenBudget, observedTokens)
3952
- : await this.compaction.evaluate(conversationId, tokenBudget);
3953
- const targetTokens =
3954
- params.compactionTarget === "threshold" ? decision.threshold : tokenBudget;
3955
- const liveContextStillExceedsTarget =
3956
- observedTokens !== undefined && observedTokens >= targetTokens;
3957
-
3958
- if (!forceCompaction && !decision.shouldCompact) {
3959
- return {
3960
- ok: true,
3961
- compacted: false,
3962
- reason: "below threshold",
3963
- result: {
3964
- tokensBefore: decision.currentTokens,
3965
- },
3966
- };
3967
- }
3968
-
3969
- // Forced budget recovery should use the capped convergence loop so live
3970
- // overflow counts can drive recovery even when persisted context is already small.
3971
- const useSweep = manualCompactionRequested || params.compactionTarget === "threshold";
3972
- if (useSweep) {
3973
- const sweepResult = await this.compaction.compact({
3974
- conversationId,
3975
- tokenBudget,
3976
- summarize,
3977
- force: forceCompaction,
3978
- hardTrigger: false,
3979
- summaryModel,
3980
- });
3981
-
3982
- if (sweepResult.authFailure && breakerKey) {
3983
- this.recordCompactionAuthFailure(breakerKey);
3984
- } else if (sweepResult.actionTaken && breakerKey) {
3985
- this.recordCompactionSuccess(breakerKey);
3986
- }
3987
- if (sweepResult.actionTaken) {
3988
- await this.markLeafCompactionTelemetrySuccess({ conversationId });
3989
- }
3990
-
3991
- return {
3992
- ok: !sweepResult.authFailure && (sweepResult.actionTaken || !liveContextStillExceedsTarget),
3993
- compacted: sweepResult.actionTaken,
3994
- reason: sweepResult.authFailure
3995
- ? (sweepResult.actionTaken
3996
- ? "provider auth failure after partial compaction"
3997
- : "provider auth failure")
3998
- : sweepResult.actionTaken
3999
- ? "compacted"
4000
- : manualCompactionRequested
4001
- ? "nothing to compact"
4002
- : liveContextStillExceedsTarget
4003
- ? "live context still exceeds target"
4004
- : "already under target",
4005
- result: {
4006
- tokensBefore: decision.currentTokens,
4007
- tokensAfter: sweepResult.tokensAfter,
4008
- details: {
4009
- rounds: sweepResult.actionTaken ? 1 : 0,
4010
- targetTokens,
4011
- },
4012
- },
4013
- };
4014
- }
4015
-
4016
- // When forced, use the token budget as target
4017
- const convergenceTargetTokens = forceCompaction
4018
- ? tokenBudget
4019
- : params.compactionTarget === "threshold"
4020
- ? decision.threshold
4021
- : tokenBudget;
4022
-
4023
- // When forced (overflow recovery) and the caller did not supply an
4024
- // observed token count, assume we are at least at the token budget so
4025
- // compactUntilUnder does not bail with "already under target" while the
4026
- // live context is actually overflowing.
4027
- const effectiveCurrentTokens =
4028
- observedTokens !== undefined
4029
- ? observedTokens
4030
- : forceCompaction
4031
- ? tokenBudget
4032
- : undefined;
4033
- const compactResult = await this.compaction.compactUntilUnder({
4034
- conversationId,
4035
- tokenBudget,
4036
- targetTokens: convergenceTargetTokens,
4037
- ...(effectiveCurrentTokens !== undefined ? { currentTokens: effectiveCurrentTokens } : {}),
4038
- summarize,
4039
- summaryModel,
4040
- });
4041
-
4042
- if (compactResult.authFailure && breakerKey) {
4043
- this.recordCompactionAuthFailure(breakerKey);
4044
- } else if (compactResult.rounds > 0 && breakerKey) {
4045
- this.recordCompactionSuccess(breakerKey);
4046
- }
4047
-
4048
- const didCompact = compactResult.rounds > 0;
4049
- if (didCompact) {
4050
- await this.markLeafCompactionTelemetrySuccess({ conversationId });
4051
- }
4052
-
4053
- return {
4054
- ok: compactResult.success,
4055
- compacted: didCompact,
4056
- reason: compactResult.authFailure
4057
- ? (didCompact
4058
- ? "provider auth failure after partial compaction"
4059
- : "provider auth failure")
4060
- : compactResult.success
4061
- ? didCompact
4062
- ? "compacted"
4063
- : "already under target"
4064
- : "could not reach target",
4065
- result: {
4066
- tokensBefore: decision.currentTokens,
4067
- tokensAfter: compactResult.finalTokens,
4068
- details: {
4069
- rounds: compactResult.rounds,
4070
- targetTokens: convergenceTargetTokens,
4071
- },
4072
- },
4073
- };
4074
- },
4075
- );
4076
- }
4077
-
4078
- async prepareSubagentSpawn(params: {
4079
- parentSessionKey: string;
4080
- childSessionKey: string;
4081
- ttlMs?: number;
4082
- }): Promise<SubagentSpawnPreparation | undefined> {
4083
- if (
4084
- this.shouldIgnoreSession({ sessionKey: params.parentSessionKey })
4085
- || this.shouldIgnoreSession({ sessionKey: params.childSessionKey })
4086
- || this.isStatelessSession(params.parentSessionKey)
4087
- || this.isStatelessSession(params.childSessionKey)
4088
- ) {
4089
- return undefined;
4090
- }
4091
- this.ensureMigrated();
4092
-
4093
- const childSessionKey = params.childSessionKey.trim();
4094
- const parentSessionKey = params.parentSessionKey.trim();
4095
- if (!childSessionKey || !parentSessionKey) {
4096
- return undefined;
4097
- }
4098
-
4099
- const conversationId = await this.resolveConversationIdForSessionKey(parentSessionKey);
4100
- if (typeof conversationId !== "number") {
4101
- return undefined;
4102
- }
4103
-
4104
- const ttlMs =
4105
- typeof params.ttlMs === "number" && Number.isFinite(params.ttlMs) && params.ttlMs > 0
4106
- ? Math.floor(params.ttlMs)
4107
- : undefined;
4108
-
4109
- // Inherit scope from parent grant if one exists (prevents privilege escalation)
4110
- const parentGrantId = resolveDelegatedExpansionGrantId(parentSessionKey);
4111
- const parentGrant = parentGrantId
4112
- ? getRuntimeExpansionAuthManager().getGrant(parentGrantId)
4113
- : null;
4114
-
4115
- const childTokenCap = parentGrant
4116
- ? Math.min(
4117
- getRuntimeExpansionAuthManager().getRemainingTokenBudget(parentGrantId!) ?? this.config.maxExpandTokens,
4118
- this.config.maxExpandTokens,
4119
- )
4120
- : this.config.maxExpandTokens;
4121
-
4122
- const childMaxDepth = parentGrant
4123
- ? Math.max(0, parentGrant.maxDepth - 1)
4124
- : undefined;
4125
-
4126
- const childAllowedSummaryIds = parentGrant?.allowedSummaryIds.length
4127
- ? parentGrant.allowedSummaryIds
4128
- : undefined;
4129
-
4130
- createDelegatedExpansionGrant({
4131
- delegatedSessionKey: childSessionKey,
4132
- issuerSessionId: parentSessionKey,
4133
- allowedConversationIds: [conversationId],
4134
- allowedSummaryIds: childAllowedSummaryIds,
4135
- tokenCap: childTokenCap,
4136
- maxDepth: childMaxDepth,
4137
- ttlMs,
4138
- });
4139
-
4140
- return {
4141
- rollback: () => {
4142
- revokeDelegatedExpansionGrantForSession(childSessionKey, { removeBinding: true });
4143
- },
4144
- };
4145
- }
4146
-
4147
- async onSubagentEnded(params: {
4148
- childSessionKey: string;
4149
- reason: SubagentEndReason;
4150
- }): Promise<void> {
4151
- if (
4152
- this.shouldIgnoreSession({ sessionKey: params.childSessionKey })
4153
- || this.isStatelessSession(params.childSessionKey)
4154
- ) {
4155
- return;
4156
- }
4157
- const childSessionKey = params.childSessionKey.trim();
4158
- if (!childSessionKey) {
4159
- return;
4160
- }
4161
-
4162
- switch (params.reason) {
4163
- case "deleted":
4164
- revokeDelegatedExpansionGrantForSession(childSessionKey, { removeBinding: true });
4165
- break;
4166
- case "completed":
4167
- revokeDelegatedExpansionGrantForSession(childSessionKey);
4168
- break;
4169
- case "released":
4170
- case "swept":
4171
- removeDelegatedExpansionGrantForSession(childSessionKey);
4172
- break;
4173
- }
4174
- }
4175
-
4176
- async dispose(): Promise<void> {
4177
- // No-op for plugin singleton — the connection is shared across runs.
4178
- // OpenClaw's runner calls dispose() after every run, but the plugin
4179
- // registers a single engine instance reused by the factory. Closing
4180
- // the DB here would break subsequent runs with "database is not open".
4181
- // The shared connection is managed for the lifetime of the plugin process.
4182
- }
4183
-
4184
- /** Detect the empty replacement row created during a prior lifecycle rollover. */
4185
- private async isFreshLifecycleConversation(conversation: ConversationRecord): Promise<boolean> {
4186
- const currentMessageCount = await this.conversationStore.getMessageCount(conversation.conversationId);
4187
- if (currentMessageCount !== 0) {
4188
- return false;
4189
- }
4190
- const currentContextItems = await this.summaryStore.getContextItems(conversation.conversationId);
4191
- return currentContextItems.length === 0 && !conversation.bootstrappedAt;
4192
- }
4193
-
4194
- /**
4195
- * Archive the current active conversation and optionally create the replacement
4196
- * row that bootstrap should attach to for the next session transcript.
4197
- */
4198
- private async applySessionReplacement(params: {
4199
- reason: string;
4200
- sessionId?: string;
4201
- sessionKey?: string;
4202
- nextSessionId?: string;
4203
- nextSessionKey?: string;
4204
- createReplacement: boolean;
4205
- createReplacementWhenMissing?: boolean;
4206
- }): Promise<void> {
4207
- const current = await this.conversationStore.getConversationForSession({
4208
- sessionId: params.sessionId,
4209
- sessionKey: params.sessionKey,
4210
- });
4211
- if (!current && !params.createReplacementWhenMissing) {
4212
- return;
4213
- }
4214
-
4215
- if (current?.active) {
4216
- if (params.createReplacement && await this.isFreshLifecycleConversation(current)) {
4217
- this.deps.log.info(
4218
- `[lcm] ${params.reason} lifecycle no-op for already fresh conversation ${current.conversationId}`,
4219
- );
4220
- return;
4221
- }
4222
- await this.conversationStore.archiveConversation(current.conversationId);
4223
- }
4224
-
4225
- if (!params.createReplacement) {
4226
- this.deps.log.info(
4227
- `[lcm] ${params.reason} lifecycle archived conversation ${current?.conversationId ?? "(none)"}`,
4228
- );
4229
- return;
4230
- }
4231
-
4232
- const nextSessionId = params.nextSessionId?.trim() || params.sessionId?.trim() || current?.sessionId;
4233
- if (!nextSessionId) {
4234
- this.deps.log.warn(`[lcm] ${params.reason} lifecycle skipped: no session identity available`);
4235
- return;
4236
- }
4237
- const nextSessionKey = params.nextSessionKey?.trim() || params.sessionKey?.trim() || current?.sessionKey;
4238
- const freshConversation = await this.conversationStore.createConversation({
4239
- sessionId: nextSessionId,
4240
- ...(nextSessionKey ? { sessionKey: nextSessionKey } : {}),
4241
- });
4242
- this.deps.log.info(
4243
- `[lcm] ${params.reason} lifecycle archived prior conversation and created ${freshConversation.conversationId}`,
4244
- );
4245
- }
4246
-
4247
- /** Apply LCM lifecycle semantics for OpenClaw's /new and /reset commands. */
4248
- async handleBeforeReset(params: {
4249
- reason?: string;
4250
- sessionId?: string;
4251
- sessionKey?: string;
4252
- }): Promise<void> {
4253
- const reason = params.reason?.trim();
4254
- if (reason !== "new" && reason !== "reset") {
4255
- return;
4256
- }
4257
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
4258
- return;
4259
- }
4260
- if (this.isStatelessSession(params.sessionKey)) {
4261
- return;
4262
- }
4263
-
4264
- this.ensureMigrated();
4265
- await this.withSessionQueue(
4266
- this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
4267
- async () =>
4268
- this.conversationStore.withTransaction(async () => {
4269
- if (reason === "new") {
4270
- const conversation = await this.conversationStore.getConversationForSession({
4271
- sessionId: params.sessionId,
4272
- sessionKey: params.sessionKey,
4273
- });
4274
- if (!conversation) {
4275
- return;
4276
- }
4277
-
4278
- const retainDepth =
4279
- typeof this.config.newSessionRetainDepth === "number"
4280
- && Number.isFinite(this.config.newSessionRetainDepth)
4281
- ? this.config.newSessionRetainDepth
4282
- : 2;
4283
- await this.summaryStore.pruneForNewSession(conversation.conversationId, retainDepth);
4284
- this.deps.log.info(
4285
- `[lcm] /new pruned conversation ${conversation.conversationId} to retain depth ${retainDepth}`,
4286
- );
4287
- return;
4288
- }
4289
- await this.applySessionReplacement({
4290
- reason: "/reset",
4291
- sessionId: params.sessionId,
4292
- sessionKey: params.sessionKey,
4293
- createReplacement: true,
4294
- createReplacementWhenMissing: true,
4295
- });
4296
- }),
4297
- );
4298
- }
4299
-
4300
- /** Apply generic lifecycle semantics for session rollover and deletion hooks. */
4301
- async handleSessionEnd(params: {
4302
- reason?: string;
4303
- sessionId?: string;
4304
- sessionKey?: string;
4305
- nextSessionId?: string;
4306
- nextSessionKey?: string;
4307
- }): Promise<void> {
4308
- const reason = params.reason?.trim();
4309
- if (!reason || reason === "new" || reason === "unknown") {
4310
- return;
4311
- }
4312
- if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
4313
- return;
4314
- }
4315
- if (this.isStatelessSession(params.sessionKey ?? params.nextSessionKey)) {
4316
- return;
4317
- }
4318
-
4319
- const createReplacement = reason !== "deleted";
4320
- this.ensureMigrated();
4321
- await this.withSessionQueue(
4322
- this.resolveSessionQueueKey(params.nextSessionId ?? params.sessionId, params.sessionKey ?? params.nextSessionKey),
4323
- async () =>
4324
- this.conversationStore.withTransaction(async () => {
4325
- await this.applySessionReplacement({
4326
- reason: `session_end:${reason}`,
4327
- sessionId: params.sessionId,
4328
- sessionKey: params.sessionKey ?? params.nextSessionKey,
4329
- nextSessionId: params.nextSessionId,
4330
- nextSessionKey: params.nextSessionKey,
4331
- createReplacement,
4332
- });
4333
- }),
4334
- );
4335
- }
4336
-
4337
- // ── Public accessors for retrieval (used by subagent expansion) ─────────
4338
-
4339
- getRetrieval(): RetrievalEngine {
4340
- return this.retrieval;
4341
- }
4342
-
4343
- getConversationStore(): ConversationStore {
4344
- return this.conversationStore;
4345
- }
4346
-
4347
- getSummaryStore(): SummaryStore {
4348
- return this.summaryStore;
4349
- }
4350
-
4351
- getCompactionTelemetryStore(): CompactionTelemetryStore {
4352
- return this.compactionTelemetryStore;
4353
- }
4354
-
4355
- // ── Heartbeat pruning ──────────────────────────────────────────────────
4356
-
4357
- /**
4358
- * Detect HEARTBEAT_OK turn cycles in a conversation and delete them.
4359
- *
4360
- * A HEARTBEAT_OK turn is: a user message (the heartbeat prompt), followed by
4361
- * any tool call/result messages, ending with an assistant message that is a
4362
- * heartbeat ack. The entire sequence has no durable information value for LCM.
4363
- *
4364
- * Detection: assistant content (trimmed, lowercased) starts with "heartbeat_ok"
4365
- * and any text after is not alphanumeric (matches OpenClaw core's ack detection).
4366
- * This catches both exact "HEARTBEAT_OK" and chatty variants like
4367
- * "HEARTBEAT_OK — weekend, no market".
4368
- *
4369
- * Returns the number of messages deleted.
4370
- */
4371
- private async pruneHeartbeatOkTurns(conversationId: number): Promise<number> {
4372
- const allMessages = await this.conversationStore.getMessages(conversationId);
4373
- if (allMessages.length === 0) {
4374
- return 0;
4375
- }
4376
-
4377
- const toDelete: number[] = [];
4378
-
4379
- // Walk through messages finding HEARTBEAT_OK assistant replies, then
4380
- // collect the entire turn (back to the preceding user message).
4381
- for (let i = 0; i < allMessages.length; i++) {
4382
- const msg = allMessages[i];
4383
- if (msg.role !== "assistant") {
4384
- continue;
4385
- }
4386
- if (!isHeartbeatOkContent(msg.content)) {
4387
- continue;
4388
- }
4389
-
4390
- // Found an exact HEARTBEAT_OK reply. Walk backward to find the turn start
4391
- // (the preceding user message).
4392
- const turnMessages = [msg];
4393
- for (let j = i - 1; j >= 0; j--) {
4394
- const prev = allMessages[j];
4395
- turnMessages.push(prev);
4396
- if (prev.role === "user") {
4397
- break; // Found turn start
4398
- }
4399
- }
4400
-
4401
- if (!turnMessages.some((record) => record.role === "user")) {
4402
- continue;
4403
- }
4404
- if (!turnLooksLikeHeartbeatTurn(turnMessages)) {
4405
- continue;
4406
- }
4407
-
4408
- toDelete.push(...turnMessages.map((record) => record.messageId));
4409
- }
4410
-
4411
- if (toDelete.length === 0) {
4412
- return 0;
4413
- }
4414
-
4415
- // Deduplicate (a message could theoretically appear in multiple turns)
4416
- const uniqueIds = [...new Set(toDelete)];
4417
- return this.conversationStore.deleteMessages(uniqueIds);
4418
- }
4419
- }
4420
-
4421
- // ── Heartbeat detection ─────────────────────────────────────────────────────
4422
-
4423
- const HEARTBEAT_OK_TOKEN = "heartbeat_ok";
4424
- const HEARTBEAT_TURN_MARKER = "heartbeat.md";
4425
-
4426
- /**
4427
- * Detect whether an assistant message is a heartbeat ack.
4428
- *
4429
- * Only exact (case-insensitive) "HEARTBEAT_OK" acknowledgements are pruned.
4430
- * Any additional text indicates the heartbeat carried real content and should remain.
4431
- */
4432
- function isHeartbeatOkContent(content: string): boolean {
4433
- return content.trim().toLowerCase() === HEARTBEAT_OK_TOKEN;
4434
- }
4435
-
4436
- function batchLooksLikeHeartbeatAckTurn(messages: AgentMessage[]): boolean {
4437
- let sawHeartbeatMarker = false;
4438
- let sawHeartbeatAck = false;
4439
-
4440
- for (const message of messages) {
4441
- const stored = toStoredMessage(message);
4442
- if (!sawHeartbeatMarker && stored.content.toLowerCase().includes(HEARTBEAT_TURN_MARKER)) {
4443
- sawHeartbeatMarker = true;
4444
- }
4445
- if (!sawHeartbeatAck && stored.role === "assistant" && isHeartbeatOkContent(stored.content)) {
4446
- sawHeartbeatAck = true;
4447
- }
4448
- if (sawHeartbeatMarker && sawHeartbeatAck) {
4449
- return true;
4450
- }
4451
- }
4452
-
4453
- return false;
4454
- }
4455
-
4456
- function turnLooksLikeHeartbeatTurn(turnMessages: Array<{ content: string }>): boolean {
4457
- return turnMessages.some((message) =>
4458
- message.content.toLowerCase().includes(HEARTBEAT_TURN_MARKER),
4459
- );
4460
- }
4461
-
4462
- // ── Emergency fallback summarization ────────────────────────────────────────
4463
-
4464
- /**
4465
- * Creates a deterministic truncation summarizer used only as an emergency
4466
- * fallback when the model-backed summarizer cannot be created.
4467
- *
4468
- * CompactionEngine already escalates normal -> aggressive -> fallback for
4469
- * convergence. This function simply provides a stable baseline summarize
4470
- * callback to keep compaction operable when runtime setup is unavailable.
4471
- */
4472
- function createEmergencyFallbackSummarize(): (
4473
- text: string,
4474
- aggressive?: boolean,
4475
- ) => Promise<string> {
4476
- return async (text: string, aggressive?: boolean): Promise<string> => {
4477
- const maxChars = aggressive ? 600 * 4 : 900 * 4;
4478
- if (text.length <= maxChars) {
4479
- return text;
4480
- }
4481
- return text.slice(0, maxChars) + "\n[Truncated for context management]";
4482
- };
4483
- }
4484
-
4485
- /** @internal Exposed for unit tests only. */
4486
- export const __testing = { readLastJsonlEntryBeforeOffset };