@martian-engineering/lossless-claw 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/index.js +971 -0
- package/docs/configuration.md +15 -5
- package/openclaw.plugin.json +27 -3
- package/package.json +7 -6
- package/skills/lossless-claw/references/config.md +37 -0
- package/index.ts +0 -2
- package/src/assembler.ts +0 -1196
- package/src/compaction.ts +0 -1753
- package/src/db/config.ts +0 -345
- package/src/db/connection.ts +0 -151
- package/src/db/features.ts +0 -61
- package/src/db/migration.ts +0 -868
- package/src/engine.ts +0 -4486
- package/src/estimate-tokens.ts +0 -80
- package/src/expansion-auth.ts +0 -365
- package/src/expansion-policy.ts +0 -303
- package/src/expansion.ts +0 -383
- package/src/integrity.ts +0 -600
- package/src/large-files.ts +0 -546
- package/src/lcm-log.ts +0 -37
- package/src/openclaw-bridge.ts +0 -22
- package/src/plugin/index.ts +0 -2037
- package/src/plugin/lcm-command.ts +0 -1040
- package/src/plugin/lcm-doctor-apply.ts +0 -540
- package/src/plugin/lcm-doctor-cleaners.ts +0 -655
- package/src/plugin/lcm-doctor-shared.ts +0 -210
- package/src/plugin/shared-init.ts +0 -59
- package/src/prune.ts +0 -391
- package/src/retrieval.ts +0 -360
- package/src/session-patterns.ts +0 -23
- package/src/startup-banner-log.ts +0 -49
- package/src/store/compaction-telemetry-store.ts +0 -156
- package/src/store/conversation-store.ts +0 -929
- package/src/store/fts5-sanitize.ts +0 -50
- package/src/store/full-text-fallback.ts +0 -83
- package/src/store/full-text-sort.ts +0 -21
- package/src/store/index.ts +0 -39
- package/src/store/parse-utc-timestamp.ts +0 -25
- package/src/store/summary-store.ts +0 -1519
- package/src/summarize.ts +0 -1508
- package/src/tools/common.ts +0 -53
- package/src/tools/lcm-conversation-scope.ts +0 -127
- package/src/tools/lcm-describe-tool.ts +0 -245
- package/src/tools/lcm-expand-query-tool.ts +0 -1235
- package/src/tools/lcm-expand-tool.delegation.ts +0 -580
- package/src/tools/lcm-expand-tool.ts +0 -453
- package/src/tools/lcm-expansion-recursion-guard.ts +0 -373
- package/src/tools/lcm-grep-tool.ts +0 -228
- package/src/transaction-mutex.ts +0 -136
- package/src/transcript-repair.ts +0 -301
- package/src/types.ts +0 -165
package/src/engine.ts
DELETED
|
@@ -1,4486 +0,0 @@
|
|
|
1
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { closeSync, createReadStream, openSync, readSync, statSync } from "node:fs";
|
|
3
|
-
import { mkdir, writeFile } from "node:fs/promises";
|
|
4
|
-
import { homedir } from "node:os";
|
|
5
|
-
import { join } from "node:path";
|
|
6
|
-
import type { DatabaseSync } from "node:sqlite";
|
|
7
|
-
import { createInterface } from "node:readline";
|
|
8
|
-
import { SessionManager } from "@mariozechner/pi-coding-agent";
|
|
9
|
-
import type {
|
|
10
|
-
ContextEngine,
|
|
11
|
-
ContextEngineInfo,
|
|
12
|
-
AssembleResult,
|
|
13
|
-
BootstrapResult,
|
|
14
|
-
CompactResult,
|
|
15
|
-
IngestBatchResult,
|
|
16
|
-
IngestResult,
|
|
17
|
-
SubagentEndReason,
|
|
18
|
-
SubagentSpawnPreparation,
|
|
19
|
-
} from "openclaw/plugin-sdk";
|
|
20
|
-
import {
|
|
21
|
-
blockFromPart,
|
|
22
|
-
contentFromParts,
|
|
23
|
-
ContextAssembler,
|
|
24
|
-
pickToolCallId,
|
|
25
|
-
pickToolIsError,
|
|
26
|
-
pickToolName,
|
|
27
|
-
} from "./assembler.js";
|
|
28
|
-
import { CompactionEngine, type CompactionConfig } from "./compaction.js";
|
|
29
|
-
import type { LcmConfig } from "./db/config.js";
|
|
30
|
-
import { getLcmDbFeatures } from "./db/features.js";
|
|
31
|
-
import { runLcmMigrations } from "./db/migration.js";
|
|
32
|
-
import {
|
|
33
|
-
createDelegatedExpansionGrant,
|
|
34
|
-
getRuntimeExpansionAuthManager,
|
|
35
|
-
removeDelegatedExpansionGrantForSession,
|
|
36
|
-
resolveDelegatedExpansionGrantId,
|
|
37
|
-
revokeDelegatedExpansionGrantForSession,
|
|
38
|
-
} from "./expansion-auth.js";
|
|
39
|
-
import {
|
|
40
|
-
extensionFromNameOrMime,
|
|
41
|
-
formatFileReference,
|
|
42
|
-
formatToolOutputReference,
|
|
43
|
-
generateExplorationSummary,
|
|
44
|
-
parseFileBlocks,
|
|
45
|
-
} from "./large-files.js";
|
|
46
|
-
import { describeLogError } from "./lcm-log.js";
|
|
47
|
-
import { RetrievalEngine } from "./retrieval.js";
|
|
48
|
-
import { compileSessionPatterns, matchesSessionPattern } from "./session-patterns.js";
|
|
49
|
-
import { logStartupBannerOnce } from "./startup-banner-log.js";
|
|
50
|
-
import {
|
|
51
|
-
CompactionTelemetryStore,
|
|
52
|
-
type ConversationCompactionTelemetryRecord,
|
|
53
|
-
type CacheState,
|
|
54
|
-
type ActivityBand,
|
|
55
|
-
} from "./store/compaction-telemetry-store.js";
|
|
56
|
-
import {
|
|
57
|
-
ConversationStore,
|
|
58
|
-
type ConversationRecord,
|
|
59
|
-
type CreateMessagePartInput,
|
|
60
|
-
type MessagePartRecord,
|
|
61
|
-
type MessagePartType,
|
|
62
|
-
} from "./store/conversation-store.js";
|
|
63
|
-
import { SummaryStore } from "./store/summary-store.js";
|
|
64
|
-
import { createLcmSummarizeFromLegacyParams, LcmProviderAuthError } from "./summarize.js";
|
|
65
|
-
import type { LcmDependencies } from "./types.js";
|
|
66
|
-
import { estimateTokens } from "./estimate-tokens.js";
|
|
67
|
-
|
|
68
|
-
type AgentMessage = Parameters<ContextEngine["ingest"]>[0]["message"];
|
|
69
|
-
type AssembleResultWithSystemPrompt = AssembleResult & { systemPromptAddition?: string };
|
|
70
|
-
type CircuitBreakerState = {
|
|
71
|
-
failures: number;
|
|
72
|
-
openSince: number | null;
|
|
73
|
-
};
|
|
74
|
-
type PromptCacheSnapshot = {
|
|
75
|
-
lastObservedCacheRead?: number;
|
|
76
|
-
lastObservedCacheWrite?: number;
|
|
77
|
-
cacheState: CacheState;
|
|
78
|
-
retention?: string;
|
|
79
|
-
sawExplicitBreak: boolean;
|
|
80
|
-
};
|
|
81
|
-
type IncrementalCompactionDecision = {
|
|
82
|
-
shouldCompact: boolean;
|
|
83
|
-
cacheState: CacheState;
|
|
84
|
-
maxPasses: number;
|
|
85
|
-
rawTokensOutsideTail: number;
|
|
86
|
-
threshold: number;
|
|
87
|
-
leafChunkTokens: number;
|
|
88
|
-
fallbackLeafChunkTokens: number[];
|
|
89
|
-
activityBand: ActivityBand;
|
|
90
|
-
allowCondensedPasses: boolean;
|
|
91
|
-
};
|
|
92
|
-
type DynamicLeafChunkBounds = {
|
|
93
|
-
floor: number;
|
|
94
|
-
medium: number;
|
|
95
|
-
high: number;
|
|
96
|
-
max: number;
|
|
97
|
-
};
|
|
98
|
-
type TranscriptRewriteReplacement = {
|
|
99
|
-
entryId: string;
|
|
100
|
-
message: AgentMessage;
|
|
101
|
-
};
|
|
102
|
-
type TranscriptRewriteRequest = {
|
|
103
|
-
replacements: TranscriptRewriteReplacement[];
|
|
104
|
-
};
|
|
105
|
-
type ContextEngineMaintenanceResult = {
|
|
106
|
-
changed: boolean;
|
|
107
|
-
bytesFreed: number;
|
|
108
|
-
rewrittenEntries: number;
|
|
109
|
-
reason?: string;
|
|
110
|
-
};
|
|
111
|
-
type ContextEngineMaintenanceRuntimeContext = Record<string, unknown> & {
|
|
112
|
-
rewriteTranscriptEntries?: (
|
|
113
|
-
request: TranscriptRewriteRequest,
|
|
114
|
-
) => Promise<ContextEngineMaintenanceResult>;
|
|
115
|
-
};
|
|
116
|
-
|
|
117
|
-
const TRANSCRIPT_GC_BATCH_SIZE = 12;
|
|
118
|
-
const HOT_CACHE_HYSTERESIS_TURNS = 2;
|
|
119
|
-
const DYNAMIC_LEAF_CHUNK_MEDIUM_MULTIPLIER = 1.5;
|
|
120
|
-
const DYNAMIC_LEAF_CHUNK_HIGH_MULTIPLIER = 2;
|
|
121
|
-
const DYNAMIC_ACTIVITY_MEDIUM_UPSHIFT_FACTOR = 0.5;
|
|
122
|
-
const DYNAMIC_ACTIVITY_MEDIUM_DOWNSHIFT_FACTOR = 0.35;
|
|
123
|
-
const DYNAMIC_ACTIVITY_HIGH_UPSHIFT_FACTOR = 1.0;
|
|
124
|
-
const DYNAMIC_ACTIVITY_HIGH_DOWNSHIFT_FACTOR = 0.75;
|
|
125
|
-
|
|
126
|
-
// ── Helpers ──────────────────────────────────────────────────────────────────
|
|
127
|
-
|
|
128
|
-
function toJson(value: unknown): string {
|
|
129
|
-
const encoded = JSON.stringify(value);
|
|
130
|
-
return typeof encoded === "string" ? encoded : "";
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function safeString(value: unknown): string | undefined {
|
|
134
|
-
return typeof value === "string" ? value : undefined;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
function formatDurationMs(durationMs: number): string {
|
|
138
|
-
return `${durationMs}ms`;
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
142
|
-
return value && typeof value === "object" && !Array.isArray(value)
|
|
143
|
-
? (value as Record<string, unknown>)
|
|
144
|
-
: undefined;
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
function safeBoolean(value: unknown): boolean | undefined {
|
|
148
|
-
return typeof value === "boolean" ? value : undefined;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function extractTranscriptToolCallId(message: AgentMessage): string | undefined {
|
|
152
|
-
const topLevel = message as Record<string, unknown>;
|
|
153
|
-
const direct =
|
|
154
|
-
safeString(topLevel.toolCallId) ??
|
|
155
|
-
safeString(topLevel.tool_call_id) ??
|
|
156
|
-
safeString(topLevel.toolUseId) ??
|
|
157
|
-
safeString(topLevel.tool_use_id) ??
|
|
158
|
-
safeString(topLevel.call_id) ??
|
|
159
|
-
safeString(topLevel.id);
|
|
160
|
-
if (direct) {
|
|
161
|
-
return direct;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
if (!Array.isArray(topLevel.content)) {
|
|
165
|
-
return undefined;
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
for (const item of topLevel.content) {
|
|
169
|
-
const record = asRecord(item);
|
|
170
|
-
if (!record) {
|
|
171
|
-
continue;
|
|
172
|
-
}
|
|
173
|
-
const nested =
|
|
174
|
-
safeString(record.toolCallId) ??
|
|
175
|
-
safeString(record.tool_call_id) ??
|
|
176
|
-
safeString(record.toolUseId) ??
|
|
177
|
-
safeString(record.tool_use_id) ??
|
|
178
|
-
safeString(record.call_id) ??
|
|
179
|
-
safeString(record.id);
|
|
180
|
-
if (nested) {
|
|
181
|
-
return nested;
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
return undefined;
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
function listTranscriptToolResultEntryIdsByCallId(sessionFile: string): Map<string, string> {
|
|
189
|
-
const sessionManager = SessionManager.open(sessionFile);
|
|
190
|
-
const branch = sessionManager.getBranch();
|
|
191
|
-
const entryIdsByCallId = new Map<string, string>();
|
|
192
|
-
const duplicateCallIds = new Set<string>();
|
|
193
|
-
|
|
194
|
-
for (const entry of branch) {
|
|
195
|
-
if (entry.type !== "message" || entry.message.role !== "toolResult") {
|
|
196
|
-
continue;
|
|
197
|
-
}
|
|
198
|
-
const toolCallId = extractTranscriptToolCallId(entry.message as AgentMessage);
|
|
199
|
-
if (!toolCallId) {
|
|
200
|
-
continue;
|
|
201
|
-
}
|
|
202
|
-
if (entryIdsByCallId.has(toolCallId)) {
|
|
203
|
-
duplicateCallIds.add(toolCallId);
|
|
204
|
-
continue;
|
|
205
|
-
}
|
|
206
|
-
entryIdsByCallId.set(toolCallId, entry.id);
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
for (const duplicateCallId of duplicateCallIds) {
|
|
210
|
-
entryIdsByCallId.delete(duplicateCallId);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
return entryIdsByCallId;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
function appendTextValue(value: unknown, out: string[]): void {
|
|
217
|
-
if (typeof value === "string") {
|
|
218
|
-
out.push(value);
|
|
219
|
-
return;
|
|
220
|
-
}
|
|
221
|
-
if (Array.isArray(value)) {
|
|
222
|
-
for (const entry of value) {
|
|
223
|
-
appendTextValue(entry, out);
|
|
224
|
-
}
|
|
225
|
-
return;
|
|
226
|
-
}
|
|
227
|
-
if (!value || typeof value !== "object") {
|
|
228
|
-
return;
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
const record = value as Record<string, unknown>;
|
|
232
|
-
appendTextValue(record.text, out);
|
|
233
|
-
appendTextValue(record.value, out);
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
const STRUCTURED_TEXT_FIELD_KEYS = ["text", "transcript", "transcription", "message", "summary"];
|
|
237
|
-
const STRUCTURED_ARRAY_FIELD_KEYS = [
|
|
238
|
-
"segments",
|
|
239
|
-
"utterances",
|
|
240
|
-
"paragraphs",
|
|
241
|
-
"alternatives",
|
|
242
|
-
"words",
|
|
243
|
-
"items",
|
|
244
|
-
"results",
|
|
245
|
-
];
|
|
246
|
-
const STRUCTURED_NESTED_FIELD_KEYS = ["content", "output", "result", "payload", "data", "value"];
|
|
247
|
-
const MAX_STRUCTURED_TEXT_DEPTH = 6;
|
|
248
|
-
const TOOL_RAW_TYPES: ReadonlySet<string> = new Set([
|
|
249
|
-
"tool_use",
|
|
250
|
-
"toolUse",
|
|
251
|
-
"tool-use",
|
|
252
|
-
"toolCall",
|
|
253
|
-
"tool_call",
|
|
254
|
-
"functionCall",
|
|
255
|
-
"function_call",
|
|
256
|
-
"function_call_output",
|
|
257
|
-
"tool_result",
|
|
258
|
-
"toolResult",
|
|
259
|
-
"tool_use_result",
|
|
260
|
-
]);
|
|
261
|
-
|
|
262
|
-
function looksLikeJsonPayload(value: string): boolean {
|
|
263
|
-
const trimmed = value.trim();
|
|
264
|
-
if (!trimmed) {
|
|
265
|
-
return false;
|
|
266
|
-
}
|
|
267
|
-
return (
|
|
268
|
-
(trimmed.startsWith("{") && trimmed.endsWith("}")) ||
|
|
269
|
-
(trimmed.startsWith("[") && trimmed.endsWith("]"))
|
|
270
|
-
);
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
function extractStructuredText(value: unknown, depth: number = 0): string | undefined {
|
|
274
|
-
if (value == null || depth > MAX_STRUCTURED_TEXT_DEPTH) {
|
|
275
|
-
return undefined;
|
|
276
|
-
}
|
|
277
|
-
if (typeof value === "string") {
|
|
278
|
-
if (looksLikeJsonPayload(value)) {
|
|
279
|
-
try {
|
|
280
|
-
const parsed = JSON.parse(value.trim());
|
|
281
|
-
const parsedText = extractStructuredText(parsed, depth + 1);
|
|
282
|
-
if (typeof parsedText === "string" && parsedText.length > 0) {
|
|
283
|
-
return parsedText;
|
|
284
|
-
}
|
|
285
|
-
} catch {
|
|
286
|
-
// Fall through to returning the original string when parsing fails.
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
return value;
|
|
290
|
-
}
|
|
291
|
-
if (Array.isArray(value)) {
|
|
292
|
-
const texts: string[] = [];
|
|
293
|
-
for (const entry of value) {
|
|
294
|
-
const text = extractStructuredText(entry, depth + 1);
|
|
295
|
-
if (typeof text === "string" && text.trim().length > 0) {
|
|
296
|
-
texts.push(text);
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
return texts.length > 0 ? texts.join("\n") : undefined;
|
|
300
|
-
}
|
|
301
|
-
if (typeof value !== "object") {
|
|
302
|
-
return undefined;
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
const record = value as Record<string, unknown>;
|
|
306
|
-
|
|
307
|
-
// Skip tool call/result objects — their structured data belongs in the parts table, not content
|
|
308
|
-
if (typeof record.type === "string" && TOOL_RAW_TYPES.has(record.type)) {
|
|
309
|
-
if (safeBoolean(record.toolOutputExternalized)) {
|
|
310
|
-
const externalizedText =
|
|
311
|
-
extractStructuredText(record.output, depth + 1) ??
|
|
312
|
-
extractStructuredText(record.content, depth + 1) ??
|
|
313
|
-
extractStructuredText(record.result, depth + 1);
|
|
314
|
-
if (typeof externalizedText === "string" && externalizedText.trim().length > 0) {
|
|
315
|
-
return externalizedText;
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
return undefined;
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
for (const key of STRUCTURED_TEXT_FIELD_KEYS) {
|
|
322
|
-
const candidate = record[key];
|
|
323
|
-
if (typeof candidate === "string" && candidate.trim().length > 0) {
|
|
324
|
-
return candidate;
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
for (const key of STRUCTURED_ARRAY_FIELD_KEYS) {
|
|
329
|
-
const candidate = record[key];
|
|
330
|
-
if (Array.isArray(candidate)) {
|
|
331
|
-
const texts: string[] = [];
|
|
332
|
-
for (const entry of candidate) {
|
|
333
|
-
const text = extractStructuredText(entry, depth + 1);
|
|
334
|
-
if (typeof text === "string" && text.trim().length > 0) {
|
|
335
|
-
texts.push(text);
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
if (texts.length > 0) {
|
|
339
|
-
return texts.join("\n");
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
for (const key of STRUCTURED_NESTED_FIELD_KEYS) {
|
|
345
|
-
const nested = record[key];
|
|
346
|
-
const nestedText = extractStructuredText(nested, depth + 1);
|
|
347
|
-
if (typeof nestedText === "string" && nestedText.trim().length > 0) {
|
|
348
|
-
return nestedText;
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
return undefined;
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
function extractReasoningText(record: Record<string, unknown>): string | undefined {
|
|
356
|
-
const chunks: string[] = [];
|
|
357
|
-
appendTextValue(record.summary, chunks);
|
|
358
|
-
if (chunks.length === 0) {
|
|
359
|
-
return undefined;
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
const normalized = chunks
|
|
363
|
-
.map((chunk) => chunk.trim())
|
|
364
|
-
.filter((chunk, idx, arr) => chunk.length > 0 && arr.indexOf(chunk) === idx);
|
|
365
|
-
return normalized.length > 0 ? normalized.join("\n") : undefined;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
function normalizeUnknownBlock(value: unknown): {
|
|
369
|
-
type: string;
|
|
370
|
-
text?: string;
|
|
371
|
-
metadata: Record<string, unknown>;
|
|
372
|
-
} {
|
|
373
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
374
|
-
return {
|
|
375
|
-
type: "agent",
|
|
376
|
-
metadata: { raw: value },
|
|
377
|
-
};
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
const record = value as Record<string, unknown>;
|
|
381
|
-
const rawType = safeString(record.type);
|
|
382
|
-
return {
|
|
383
|
-
type: rawType ?? "agent",
|
|
384
|
-
text:
|
|
385
|
-
safeString(record.text) ??
|
|
386
|
-
safeString(record.thinking) ??
|
|
387
|
-
((rawType === "reasoning" || rawType === "thinking")
|
|
388
|
-
? extractReasoningText(record)
|
|
389
|
-
: undefined),
|
|
390
|
-
metadata: { raw: record },
|
|
391
|
-
};
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
function toPartType(type: string): MessagePartType {
|
|
395
|
-
switch (type) {
|
|
396
|
-
case "text":
|
|
397
|
-
return "text";
|
|
398
|
-
case "thinking":
|
|
399
|
-
case "reasoning":
|
|
400
|
-
return "reasoning";
|
|
401
|
-
case "tool_use":
|
|
402
|
-
case "toolUse":
|
|
403
|
-
case "tool-use":
|
|
404
|
-
case "toolCall":
|
|
405
|
-
case "functionCall":
|
|
406
|
-
case "function_call":
|
|
407
|
-
case "function_call_output":
|
|
408
|
-
case "tool_result":
|
|
409
|
-
case "toolResult":
|
|
410
|
-
case "tool":
|
|
411
|
-
return "tool";
|
|
412
|
-
case "patch":
|
|
413
|
-
return "patch";
|
|
414
|
-
case "file":
|
|
415
|
-
case "image":
|
|
416
|
-
return "file";
|
|
417
|
-
case "subtask":
|
|
418
|
-
return "subtask";
|
|
419
|
-
case "compaction":
|
|
420
|
-
return "compaction";
|
|
421
|
-
case "step_start":
|
|
422
|
-
case "step-start":
|
|
423
|
-
return "step_start";
|
|
424
|
-
case "step_finish":
|
|
425
|
-
case "step-finish":
|
|
426
|
-
return "step_finish";
|
|
427
|
-
case "snapshot":
|
|
428
|
-
return "snapshot";
|
|
429
|
-
case "retry":
|
|
430
|
-
return "retry";
|
|
431
|
-
case "agent":
|
|
432
|
-
return "agent";
|
|
433
|
-
default:
|
|
434
|
-
return "agent";
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
/**
|
|
439
|
-
* Convert AgentMessage content into plain text for DB storage.
|
|
440
|
-
*
|
|
441
|
-
* For content block arrays we keep only text blocks to avoid persisting raw
|
|
442
|
-
* JSON syntax that can later pollute assembled model context.
|
|
443
|
-
*/
|
|
444
|
-
function extractMessageContent(content: unknown): string {
|
|
445
|
-
const extracted = extractStructuredText(content);
|
|
446
|
-
if (typeof extracted === "string") {
|
|
447
|
-
return extracted;
|
|
448
|
-
}
|
|
449
|
-
if (content == null) {
|
|
450
|
-
return "";
|
|
451
|
-
}
|
|
452
|
-
if (Array.isArray(content) && content.length === 0) {
|
|
453
|
-
return "";
|
|
454
|
-
}
|
|
455
|
-
// If content is an array of only tool call/result objects, store as empty
|
|
456
|
-
// (structured data is preserved in the message parts table)
|
|
457
|
-
if (Array.isArray(content) && content.length > 0 && content.every(
|
|
458
|
-
(item) => typeof item === "object" && item !== null && !Array.isArray(item) &&
|
|
459
|
-
typeof (item as Record<string, unknown>).type === "string" &&
|
|
460
|
-
TOOL_RAW_TYPES.has((item as Record<string, unknown>).type as string)
|
|
461
|
-
)) {
|
|
462
|
-
return "";
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
const serialized = JSON.stringify(content);
|
|
466
|
-
return typeof serialized === "string" ? serialized : "";
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
function toRuntimeRoleForTokenEstimate(role: string): "user" | "assistant" | "toolResult" {
|
|
470
|
-
if (role === "tool" || role === "toolResult") {
|
|
471
|
-
return "toolResult";
|
|
472
|
-
}
|
|
473
|
-
if (role === "user" || role === "system") {
|
|
474
|
-
return "user";
|
|
475
|
-
}
|
|
476
|
-
return "assistant";
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
function isTextBlock(value: unknown): value is { type: "text"; text: string } {
|
|
480
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
481
|
-
return false;
|
|
482
|
-
}
|
|
483
|
-
const record = value as Record<string, unknown>;
|
|
484
|
-
return record.type === "text" && typeof record.text === "string";
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
function toSyntheticMessagePartRecord(
|
|
488
|
-
part: CreateMessagePartInput,
|
|
489
|
-
messageId: number,
|
|
490
|
-
): MessagePartRecord {
|
|
491
|
-
return {
|
|
492
|
-
partId: `estimate-part-${part.ordinal}`,
|
|
493
|
-
messageId,
|
|
494
|
-
sessionId: part.sessionId,
|
|
495
|
-
partType: part.partType,
|
|
496
|
-
ordinal: part.ordinal,
|
|
497
|
-
textContent: part.textContent ?? null,
|
|
498
|
-
toolCallId: part.toolCallId ?? null,
|
|
499
|
-
toolName: part.toolName ?? null,
|
|
500
|
-
toolInput: part.toolInput ?? null,
|
|
501
|
-
toolOutput: part.toolOutput ?? null,
|
|
502
|
-
metadata: part.metadata ?? null,
|
|
503
|
-
};
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
function normalizeMessageContentForStorage(params: {
|
|
507
|
-
message: AgentMessage;
|
|
508
|
-
fallbackContent: string;
|
|
509
|
-
}): unknown {
|
|
510
|
-
const { message, fallbackContent } = params;
|
|
511
|
-
if (!("content" in message)) {
|
|
512
|
-
return fallbackContent;
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
const role = toRuntimeRoleForTokenEstimate(message.role);
|
|
516
|
-
const parts = buildMessageParts({
|
|
517
|
-
sessionId: "storage-estimate",
|
|
518
|
-
message,
|
|
519
|
-
fallbackContent,
|
|
520
|
-
}).map((part) => toSyntheticMessagePartRecord(part, 0));
|
|
521
|
-
|
|
522
|
-
if (parts.length === 0) {
|
|
523
|
-
if (role === "assistant") {
|
|
524
|
-
return fallbackContent ? [{ type: "text", text: fallbackContent }] : [];
|
|
525
|
-
}
|
|
526
|
-
if (role === "toolResult") {
|
|
527
|
-
return [{ type: "text", text: fallbackContent }];
|
|
528
|
-
}
|
|
529
|
-
return fallbackContent;
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
const blocks = parts.map(blockFromPart);
|
|
533
|
-
if (role === "user" && blocks.length === 1 && isTextBlock(blocks[0])) {
|
|
534
|
-
return blocks[0].text;
|
|
535
|
-
}
|
|
536
|
-
return blocks;
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
/**
|
|
540
|
-
* Estimate token usage for the content shape that the assembler will emit.
|
|
541
|
-
*
|
|
542
|
-
* LCM stores a plain-text fallback copy in messages.content, but message_parts
|
|
543
|
-
* can rehydrate larger structured/raw blocks. This estimator mirrors the
|
|
544
|
-
* rehydrated shape so compaction decisions use realistic token totals.
|
|
545
|
-
*/
|
|
546
|
-
function estimateContentTokensForRole(params: {
|
|
547
|
-
role: "user" | "assistant" | "toolResult";
|
|
548
|
-
content: unknown;
|
|
549
|
-
fallbackContent: string;
|
|
550
|
-
}): number {
|
|
551
|
-
const { role, content, fallbackContent } = params;
|
|
552
|
-
|
|
553
|
-
if (typeof content === "string") {
|
|
554
|
-
return estimateTokens(content);
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
if (Array.isArray(content)) {
|
|
558
|
-
if (content.length === 0) {
|
|
559
|
-
return estimateTokens(fallbackContent);
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
if (role === "user" && content.length === 1 && isTextBlock(content[0])) {
|
|
563
|
-
return estimateTokens(content[0].text);
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
const serialized = JSON.stringify(content);
|
|
567
|
-
return estimateTokens(typeof serialized === "string" ? serialized : "");
|
|
568
|
-
}
|
|
569
|
-
|
|
570
|
-
if (content && typeof content === "object") {
|
|
571
|
-
if (role === "user" && isTextBlock(content)) {
|
|
572
|
-
return estimateTokens(content.text);
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
const serialized = JSON.stringify([content]);
|
|
576
|
-
return estimateTokens(typeof serialized === "string" ? serialized : "");
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
return estimateTokens(fallbackContent);
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
function buildMessageParts(params: {
|
|
583
|
-
sessionId: string;
|
|
584
|
-
message: AgentMessage;
|
|
585
|
-
fallbackContent: string;
|
|
586
|
-
}): import("./store/conversation-store.js").CreateMessagePartInput[] {
|
|
587
|
-
const { sessionId, message, fallbackContent } = params;
|
|
588
|
-
const role = typeof message.role === "string" ? message.role : "unknown";
|
|
589
|
-
const topLevel = message as unknown as Record<string, unknown>;
|
|
590
|
-
const topLevelToolCallId =
|
|
591
|
-
safeString(topLevel.toolCallId) ??
|
|
592
|
-
safeString(topLevel.tool_call_id) ??
|
|
593
|
-
safeString(topLevel.toolUseId) ??
|
|
594
|
-
safeString(topLevel.tool_use_id) ??
|
|
595
|
-
safeString(topLevel.call_id) ??
|
|
596
|
-
safeString(topLevel.id);
|
|
597
|
-
const topLevelToolName =
|
|
598
|
-
safeString(topLevel.toolName) ??
|
|
599
|
-
safeString(topLevel.tool_name);
|
|
600
|
-
const topLevelIsError =
|
|
601
|
-
safeBoolean(topLevel.isError) ??
|
|
602
|
-
safeBoolean(topLevel.is_error);
|
|
603
|
-
|
|
604
|
-
// BashExecutionMessage: preserve a synthetic text part so output is round-trippable.
|
|
605
|
-
if (!("content" in message) && "command" in message && "output" in message) {
|
|
606
|
-
return [
|
|
607
|
-
{
|
|
608
|
-
sessionId,
|
|
609
|
-
partType: "text",
|
|
610
|
-
ordinal: 0,
|
|
611
|
-
textContent: fallbackContent,
|
|
612
|
-
metadata: toJson({
|
|
613
|
-
originalRole: role,
|
|
614
|
-
source: "bash-exec",
|
|
615
|
-
command: safeString((message as { command?: unknown }).command),
|
|
616
|
-
}),
|
|
617
|
-
},
|
|
618
|
-
];
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
if (!("content" in message)) {
|
|
622
|
-
return [
|
|
623
|
-
{
|
|
624
|
-
sessionId,
|
|
625
|
-
partType: "agent",
|
|
626
|
-
ordinal: 0,
|
|
627
|
-
textContent: fallbackContent || null,
|
|
628
|
-
metadata: toJson({
|
|
629
|
-
originalRole: role,
|
|
630
|
-
source: "unknown-message-shape",
|
|
631
|
-
raw: message,
|
|
632
|
-
}),
|
|
633
|
-
},
|
|
634
|
-
];
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
if (typeof message.content === "string") {
|
|
638
|
-
return [
|
|
639
|
-
{
|
|
640
|
-
sessionId,
|
|
641
|
-
partType: "text",
|
|
642
|
-
ordinal: 0,
|
|
643
|
-
textContent: message.content,
|
|
644
|
-
metadata: toJson({
|
|
645
|
-
originalRole: role,
|
|
646
|
-
toolCallId: topLevelToolCallId,
|
|
647
|
-
toolName: topLevelToolName,
|
|
648
|
-
isError: topLevelIsError,
|
|
649
|
-
}),
|
|
650
|
-
},
|
|
651
|
-
];
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
if (!Array.isArray(message.content)) {
|
|
655
|
-
return [
|
|
656
|
-
{
|
|
657
|
-
sessionId,
|
|
658
|
-
partType: "agent",
|
|
659
|
-
ordinal: 0,
|
|
660
|
-
textContent: fallbackContent || null,
|
|
661
|
-
metadata: toJson({
|
|
662
|
-
originalRole: role,
|
|
663
|
-
source: "non-array-content",
|
|
664
|
-
raw: message.content,
|
|
665
|
-
}),
|
|
666
|
-
},
|
|
667
|
-
];
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
const parts: CreateMessagePartInput[] = [];
|
|
671
|
-
for (let ordinal = 0; ordinal < message.content.length; ordinal++) {
|
|
672
|
-
const block = normalizeUnknownBlock(message.content[ordinal]);
|
|
673
|
-
const metadataRecord = block.metadata.raw as Record<string, unknown> | undefined;
|
|
674
|
-
const rawBlockType = safeString(metadataRecord?.rawType) ?? block.type;
|
|
675
|
-
const partType = toPartType(rawBlockType);
|
|
676
|
-
const rawBlock =
|
|
677
|
-
metadataRecord && rawBlockType !== block.type
|
|
678
|
-
? {
|
|
679
|
-
...metadataRecord,
|
|
680
|
-
type: rawBlockType,
|
|
681
|
-
}
|
|
682
|
-
: (metadataRecord ?? message.content[ordinal]);
|
|
683
|
-
const toolCallId =
|
|
684
|
-
safeString(metadataRecord?.toolCallId) ??
|
|
685
|
-
safeString(metadataRecord?.tool_call_id) ??
|
|
686
|
-
safeString(metadataRecord?.toolUseId) ??
|
|
687
|
-
safeString(metadataRecord?.tool_use_id) ??
|
|
688
|
-
safeString(metadataRecord?.call_id) ??
|
|
689
|
-
(partType === "tool" ? safeString(metadataRecord?.id) : undefined) ??
|
|
690
|
-
topLevelToolCallId;
|
|
691
|
-
|
|
692
|
-
parts.push({
|
|
693
|
-
sessionId,
|
|
694
|
-
partType,
|
|
695
|
-
ordinal,
|
|
696
|
-
textContent: block.text ?? null,
|
|
697
|
-
toolCallId,
|
|
698
|
-
toolName:
|
|
699
|
-
safeString(metadataRecord?.name) ??
|
|
700
|
-
safeString(metadataRecord?.toolName) ??
|
|
701
|
-
safeString(metadataRecord?.tool_name) ??
|
|
702
|
-
topLevelToolName,
|
|
703
|
-
toolInput:
|
|
704
|
-
metadataRecord?.input !== undefined
|
|
705
|
-
? toJson(metadataRecord.input)
|
|
706
|
-
: metadataRecord?.arguments !== undefined
|
|
707
|
-
? toJson(metadataRecord.arguments)
|
|
708
|
-
: metadataRecord?.toolInput !== undefined
|
|
709
|
-
? toJson(metadataRecord.toolInput)
|
|
710
|
-
: (safeString(metadataRecord?.tool_input) ?? null),
|
|
711
|
-
toolOutput:
|
|
712
|
-
metadataRecord?.output !== undefined
|
|
713
|
-
? toJson(metadataRecord.output)
|
|
714
|
-
: metadataRecord?.toolOutput !== undefined
|
|
715
|
-
? toJson(metadataRecord.toolOutput)
|
|
716
|
-
: (safeString(metadataRecord?.tool_output) ?? null),
|
|
717
|
-
metadata: toJson({
|
|
718
|
-
originalRole: role,
|
|
719
|
-
toolCallId: topLevelToolCallId,
|
|
720
|
-
toolName: topLevelToolName,
|
|
721
|
-
isError: topLevelIsError,
|
|
722
|
-
externalizedFileId: safeString(metadataRecord?.externalizedFileId),
|
|
723
|
-
originalByteSize:
|
|
724
|
-
typeof metadataRecord?.originalByteSize === "number"
|
|
725
|
-
? metadataRecord.originalByteSize
|
|
726
|
-
: undefined,
|
|
727
|
-
toolOutputExternalized: safeBoolean(metadataRecord?.toolOutputExternalized),
|
|
728
|
-
externalizationReason: safeString(metadataRecord?.externalizationReason),
|
|
729
|
-
rawType: rawBlockType,
|
|
730
|
-
raw: rawBlock,
|
|
731
|
-
}),
|
|
732
|
-
});
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
return parts;
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
/**
|
|
739
|
-
* Map AgentMessage role to the DB enum.
|
|
740
|
-
*
|
|
741
|
-
* "user" -> "user"
|
|
742
|
-
* "assistant" -> "assistant"
|
|
743
|
-
*
|
|
744
|
-
* AgentMessage only has user/assistant roles, but we keep the mapping
|
|
745
|
-
* explicit for clarity and future-proofing.
|
|
746
|
-
*/
|
|
747
|
-
function toDbRole(role: string): "user" | "assistant" | "system" | "tool" {
|
|
748
|
-
if (role === "tool" || role === "toolResult") {
|
|
749
|
-
return "tool";
|
|
750
|
-
}
|
|
751
|
-
if (role === "system") {
|
|
752
|
-
return "system";
|
|
753
|
-
}
|
|
754
|
-
if (role === "user") {
|
|
755
|
-
return "user";
|
|
756
|
-
}
|
|
757
|
-
if (role === "assistant") {
|
|
758
|
-
return "assistant";
|
|
759
|
-
}
|
|
760
|
-
// Unknown roles are preserved via message_parts metadata and treated as assistant.
|
|
761
|
-
return "assistant";
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
type StoredMessage = {
|
|
765
|
-
role: "user" | "assistant" | "system" | "tool";
|
|
766
|
-
content: string;
|
|
767
|
-
tokenCount: number;
|
|
768
|
-
};
|
|
769
|
-
|
|
770
|
-
/**
|
|
771
|
-
* Normalize AgentMessage variants into the storage shape used by LCM.
|
|
772
|
-
*/
|
|
773
|
-
function toStoredMessage(message: AgentMessage): StoredMessage {
|
|
774
|
-
const content =
|
|
775
|
-
"content" in message
|
|
776
|
-
? extractMessageContent(message.content)
|
|
777
|
-
: "output" in message
|
|
778
|
-
? `$ ${(message as { command: string; output: string }).command}\n${(message as { command: string; output: string }).output}`
|
|
779
|
-
: "";
|
|
780
|
-
const runtimeRole = toRuntimeRoleForTokenEstimate(message.role);
|
|
781
|
-
const normalizedContent =
|
|
782
|
-
"content" in message
|
|
783
|
-
? normalizeMessageContentForStorage({
|
|
784
|
-
message,
|
|
785
|
-
fallbackContent: content,
|
|
786
|
-
})
|
|
787
|
-
: content;
|
|
788
|
-
const tokenCount =
|
|
789
|
-
"content" in message
|
|
790
|
-
? estimateContentTokensForRole({
|
|
791
|
-
role: runtimeRole,
|
|
792
|
-
content: normalizedContent,
|
|
793
|
-
fallbackContent: content,
|
|
794
|
-
})
|
|
795
|
-
: estimateTokens(content);
|
|
796
|
-
|
|
797
|
-
return {
|
|
798
|
-
role: toDbRole(message.role),
|
|
799
|
-
content,
|
|
800
|
-
tokenCount,
|
|
801
|
-
};
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
function createBootstrapEntryHash(message: StoredMessage | null): string | null {
|
|
805
|
-
if (!message) {
|
|
806
|
-
return null;
|
|
807
|
-
}
|
|
808
|
-
return createHash("sha256")
|
|
809
|
-
.update(JSON.stringify({ role: message.role, content: message.content }))
|
|
810
|
-
.digest("hex");
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
function estimateMessageContentTokensForAfterTurn(content: unknown): number {
|
|
814
|
-
if (typeof content === "string") {
|
|
815
|
-
return estimateTokens(content);
|
|
816
|
-
}
|
|
817
|
-
if (Array.isArray(content)) {
|
|
818
|
-
let total = 0;
|
|
819
|
-
for (const part of content) {
|
|
820
|
-
if (!part || typeof part !== "object") {
|
|
821
|
-
continue;
|
|
822
|
-
}
|
|
823
|
-
const record = part as Record<string, unknown>;
|
|
824
|
-
const text =
|
|
825
|
-
typeof record.text === "string"
|
|
826
|
-
? record.text
|
|
827
|
-
: typeof record.thinking === "string"
|
|
828
|
-
? record.thinking
|
|
829
|
-
: "";
|
|
830
|
-
if (text) {
|
|
831
|
-
total += estimateTokens(text);
|
|
832
|
-
}
|
|
833
|
-
}
|
|
834
|
-
return total;
|
|
835
|
-
}
|
|
836
|
-
if (content == null) {
|
|
837
|
-
return 0;
|
|
838
|
-
}
|
|
839
|
-
const serialized = JSON.stringify(content);
|
|
840
|
-
return estimateTokens(typeof serialized === "string" ? serialized : "");
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
function estimateSessionTokenCountForAfterTurn(messages: AgentMessage[]): number {
|
|
844
|
-
let total = 0;
|
|
845
|
-
for (const message of messages) {
|
|
846
|
-
if ("content" in message) {
|
|
847
|
-
total += estimateMessageContentTokensForAfterTurn(message.content);
|
|
848
|
-
continue;
|
|
849
|
-
}
|
|
850
|
-
if ("command" in message || "output" in message) {
|
|
851
|
-
const commandText =
|
|
852
|
-
typeof (message as { command?: unknown }).command === "string"
|
|
853
|
-
? (message as { command?: string }).command
|
|
854
|
-
: "";
|
|
855
|
-
const outputText =
|
|
856
|
-
typeof (message as { output?: unknown }).output === "string"
|
|
857
|
-
? (message as { output?: string }).output
|
|
858
|
-
: "";
|
|
859
|
-
total += estimateTokens(`${commandText}\n${outputText}`);
|
|
860
|
-
}
|
|
861
|
-
}
|
|
862
|
-
return total;
|
|
863
|
-
}
|
|
864
|
-
|
|
865
|
-
function isBootstrapMessage(value: unknown): value is AgentMessage {
|
|
866
|
-
if (!value || typeof value !== "object") {
|
|
867
|
-
return false;
|
|
868
|
-
}
|
|
869
|
-
const msg = value as { role?: unknown; content?: unknown; command?: unknown; output?: unknown };
|
|
870
|
-
if (typeof msg.role !== "string") {
|
|
871
|
-
return false;
|
|
872
|
-
}
|
|
873
|
-
return "content" in msg || ("command" in msg && "output" in msg);
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
function extractCanonicalBootstrapMessage(value: unknown): AgentMessage | null {
|
|
877
|
-
if (isBootstrapMessage(value)) {
|
|
878
|
-
return value;
|
|
879
|
-
}
|
|
880
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
881
|
-
return null;
|
|
882
|
-
}
|
|
883
|
-
const entry = value as { type?: unknown; message?: unknown };
|
|
884
|
-
if ("message" in entry) {
|
|
885
|
-
if (entry.type !== undefined && entry.type !== "message") {
|
|
886
|
-
return null;
|
|
887
|
-
}
|
|
888
|
-
return isBootstrapMessage(entry.message) ? entry.message : null;
|
|
889
|
-
}
|
|
890
|
-
return null;
|
|
891
|
-
}
|
|
892
|
-
|
|
893
|
-
function extractBootstrapMessageCandidate(value: unknown): AgentMessage | null {
|
|
894
|
-
return extractCanonicalBootstrapMessage(value);
|
|
895
|
-
}
|
|
896
|
-
|
|
897
|
-
function parseBootstrapJsonl(raw: string, options?: {
|
|
898
|
-
strict?: boolean;
|
|
899
|
-
}): { messages: AgentMessage[]; sawNonWhitespace: boolean; hadMalformedLine: boolean } {
|
|
900
|
-
const messages: AgentMessage[] = [];
|
|
901
|
-
const lines = raw.split(/\r?\n/);
|
|
902
|
-
let sawNonWhitespace = false;
|
|
903
|
-
let hadMalformedLine = false;
|
|
904
|
-
for (const line of lines) {
|
|
905
|
-
const item = line.trim();
|
|
906
|
-
if (!item) {
|
|
907
|
-
continue;
|
|
908
|
-
}
|
|
909
|
-
sawNonWhitespace = true;
|
|
910
|
-
try {
|
|
911
|
-
const parsed = JSON.parse(item);
|
|
912
|
-
const candidate = extractBootstrapMessageCandidate(parsed);
|
|
913
|
-
if (candidate) {
|
|
914
|
-
messages.push(candidate);
|
|
915
|
-
continue;
|
|
916
|
-
}
|
|
917
|
-
} catch {
|
|
918
|
-
if (options?.strict) {
|
|
919
|
-
hadMalformedLine = true;
|
|
920
|
-
}
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
return { messages, sawNonWhitespace, hadMalformedLine };
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
/** Load recoverable messages from a JSON/JSONL session file without full-file reads for JSONL. */
|
|
927
|
-
async function readLeafPathMessages(sessionFile: string): Promise<AgentMessage[]> {
|
|
928
|
-
try {
|
|
929
|
-
let sawNonWhitespace = false;
|
|
930
|
-
let jsonArrayMode = false;
|
|
931
|
-
let jsonArrayBuffer = "";
|
|
932
|
-
const messages: AgentMessage[] = [];
|
|
933
|
-
const stream = createReadStream(sessionFile, { encoding: "utf8" });
|
|
934
|
-
const lines = createInterface({
|
|
935
|
-
input: stream,
|
|
936
|
-
crlfDelay: Infinity,
|
|
937
|
-
});
|
|
938
|
-
|
|
939
|
-
for await (const line of lines) {
|
|
940
|
-
if (!sawNonWhitespace) {
|
|
941
|
-
const trimmed = line.trim();
|
|
942
|
-
if (trimmed) {
|
|
943
|
-
sawNonWhitespace = true;
|
|
944
|
-
if (trimmed.startsWith("[")) {
|
|
945
|
-
jsonArrayMode = true;
|
|
946
|
-
}
|
|
947
|
-
}
|
|
948
|
-
}
|
|
949
|
-
|
|
950
|
-
if (jsonArrayMode) {
|
|
951
|
-
jsonArrayBuffer += `${line}\n`;
|
|
952
|
-
continue;
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
const parsed = parseBootstrapJsonl(line);
|
|
956
|
-
if (parsed.messages.length > 0) {
|
|
957
|
-
messages.push(...parsed.messages);
|
|
958
|
-
}
|
|
959
|
-
}
|
|
960
|
-
|
|
961
|
-
if (jsonArrayMode) {
|
|
962
|
-
const trimmed = jsonArrayBuffer.trim();
|
|
963
|
-
if (!trimmed) {
|
|
964
|
-
return [];
|
|
965
|
-
}
|
|
966
|
-
try {
|
|
967
|
-
const parsed = JSON.parse(trimmed);
|
|
968
|
-
if (!Array.isArray(parsed)) {
|
|
969
|
-
return [];
|
|
970
|
-
}
|
|
971
|
-
return parsed.filter(isBootstrapMessage);
|
|
972
|
-
} catch {
|
|
973
|
-
return [];
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
return messages;
|
|
978
|
-
} catch {
|
|
979
|
-
return [];
|
|
980
|
-
}
|
|
981
|
-
}
|
|
982
|
-
|
|
983
|
-
/**
|
|
984
|
-
* Resolve the first-time bootstrap token budget.
|
|
985
|
-
*
|
|
986
|
-
* When unset, bootstrap keeps a modest suffix of the parent session rather than
|
|
987
|
-
* inheriting the full raw history into a brand-new conversation.
|
|
988
|
-
*/
|
|
989
|
-
function resolveBootstrapMaxTokens(config: Pick<LcmConfig, "bootstrapMaxTokens" | "leafChunkTokens">): number {
|
|
990
|
-
if (
|
|
991
|
-
typeof config.bootstrapMaxTokens === "number" &&
|
|
992
|
-
Number.isFinite(config.bootstrapMaxTokens) &&
|
|
993
|
-
config.bootstrapMaxTokens > 0
|
|
994
|
-
) {
|
|
995
|
-
return Math.floor(config.bootstrapMaxTokens);
|
|
996
|
-
}
|
|
997
|
-
|
|
998
|
-
const leafChunkTokens =
|
|
999
|
-
typeof config.leafChunkTokens === "number" &&
|
|
1000
|
-
Number.isFinite(config.leafChunkTokens) &&
|
|
1001
|
-
config.leafChunkTokens > 0
|
|
1002
|
-
? Math.floor(config.leafChunkTokens)
|
|
1003
|
-
: 20_000;
|
|
1004
|
-
return Math.max(6000, Math.floor(leafChunkTokens * 0.3));
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
/**
|
|
1008
|
-
* Keep only the newest bootstrap messages that fit within the token budget.
|
|
1009
|
-
*
|
|
1010
|
-
* The newest message is always preserved so a fork never starts empty when the
|
|
1011
|
-
* parent transcript has any recoverable content at all.
|
|
1012
|
-
*/
|
|
1013
|
-
function trimBootstrapMessagesToBudget(messages: AgentMessage[], maxTokens: number): AgentMessage[] {
|
|
1014
|
-
if (messages.length === 0) {
|
|
1015
|
-
return [];
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
const safeMaxTokens = Number.isFinite(maxTokens) ? Math.floor(maxTokens) : 0;
|
|
1019
|
-
if (safeMaxTokens <= 0) {
|
|
1020
|
-
return [messages[messages.length - 1]!];
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
const kept: AgentMessage[] = [];
|
|
1024
|
-
let totalTokens = 0;
|
|
1025
|
-
|
|
1026
|
-
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
|
1027
|
-
const message = messages[index]!;
|
|
1028
|
-
const tokenCount = toStoredMessage(message).tokenCount;
|
|
1029
|
-
if (kept.length > 0 && totalTokens + tokenCount > safeMaxTokens) {
|
|
1030
|
-
break;
|
|
1031
|
-
}
|
|
1032
|
-
kept.push(message);
|
|
1033
|
-
totalTokens += tokenCount;
|
|
1034
|
-
}
|
|
1035
|
-
|
|
1036
|
-
// If a single oversized tail message exceeds the budget, return empty
|
|
1037
|
-
// rather than silently bypassing the budget cap. An empty bootstrap is
|
|
1038
|
-
// safer than an exploding one.
|
|
1039
|
-
if (kept.length === 1 && totalTokens > safeMaxTokens) {
|
|
1040
|
-
return [];
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
kept.reverse();
|
|
1044
|
-
return kept;
|
|
1045
|
-
}
|
|
1046
|
-
|
|
1047
|
-
function readFileSegment(sessionFile: string, offset: number): string | null {
|
|
1048
|
-
let fd: number | null = null;
|
|
1049
|
-
try {
|
|
1050
|
-
fd = openSync(sessionFile, "r");
|
|
1051
|
-
const stats = statSync(sessionFile);
|
|
1052
|
-
const safeOffset = Math.max(0, Math.min(Math.floor(offset), stats.size));
|
|
1053
|
-
const length = stats.size - safeOffset;
|
|
1054
|
-
if (length <= 0) {
|
|
1055
|
-
return "";
|
|
1056
|
-
}
|
|
1057
|
-
const buffer = Buffer.alloc(length);
|
|
1058
|
-
readSync(fd, buffer, 0, length, safeOffset);
|
|
1059
|
-
return buffer.toString("utf8");
|
|
1060
|
-
} catch {
|
|
1061
|
-
return null;
|
|
1062
|
-
} finally {
|
|
1063
|
-
if (fd != null) {
|
|
1064
|
-
closeSync(fd);
|
|
1065
|
-
}
|
|
1066
|
-
}
|
|
1067
|
-
}
|
|
1068
|
-
|
|
1069
|
-
function readLastJsonlEntryBeforeOffset(
|
|
1070
|
-
sessionFile: string,
|
|
1071
|
-
offset: number,
|
|
1072
|
-
messageOnly = false,
|
|
1073
|
-
matcher?: (message: AgentMessage) => boolean,
|
|
1074
|
-
): string | null {
|
|
1075
|
-
const chunkSize = 16_384;
|
|
1076
|
-
let fd: number | null = null;
|
|
1077
|
-
try {
|
|
1078
|
-
const safeOffset = Math.max(0, Math.floor(offset));
|
|
1079
|
-
if (safeOffset <= 0) {
|
|
1080
|
-
return null;
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
fd = openSync(sessionFile, "r");
|
|
1084
|
-
let cursor = safeOffset;
|
|
1085
|
-
let carry = "";
|
|
1086
|
-
let reachedStart = false;
|
|
1087
|
-
while (cursor > 0 || (reachedStart && carry.length > 0)) {
|
|
1088
|
-
if (!reachedStart) {
|
|
1089
|
-
const start = Math.max(0, cursor - chunkSize);
|
|
1090
|
-
const length = cursor - start;
|
|
1091
|
-
const buffer = Buffer.alloc(length);
|
|
1092
|
-
readSync(fd, buffer, 0, length, start);
|
|
1093
|
-
carry = buffer.toString("utf8") + carry;
|
|
1094
|
-
cursor = start;
|
|
1095
|
-
if (start === 0) {
|
|
1096
|
-
reachedStart = true;
|
|
1097
|
-
}
|
|
1098
|
-
}
|
|
1099
|
-
|
|
1100
|
-
const trimmedEnd = carry.replace(/\s+$/u, "");
|
|
1101
|
-
if (!trimmedEnd) {
|
|
1102
|
-
if (reachedStart) break;
|
|
1103
|
-
carry = "";
|
|
1104
|
-
continue;
|
|
1105
|
-
}
|
|
1106
|
-
|
|
1107
|
-
const newlineIndex = Math.max(trimmedEnd.lastIndexOf("\n"), trimmedEnd.lastIndexOf("\r"));
|
|
1108
|
-
if (newlineIndex >= 0) {
|
|
1109
|
-
const candidate = trimmedEnd.slice(newlineIndex + 1).trim();
|
|
1110
|
-
if (candidate) {
|
|
1111
|
-
if (messageOnly) {
|
|
1112
|
-
let matchedMessage: AgentMessage | null = null;
|
|
1113
|
-
try {
|
|
1114
|
-
matchedMessage = extractBootstrapMessageCandidate(JSON.parse(candidate));
|
|
1115
|
-
} catch { /* not valid JSON, skip */ }
|
|
1116
|
-
if (!matchedMessage || (matcher && !matcher(matchedMessage))) {
|
|
1117
|
-
carry = trimmedEnd.slice(0, newlineIndex);
|
|
1118
|
-
continue;
|
|
1119
|
-
}
|
|
1120
|
-
}
|
|
1121
|
-
return candidate;
|
|
1122
|
-
}
|
|
1123
|
-
carry = trimmedEnd.slice(0, newlineIndex);
|
|
1124
|
-
continue;
|
|
1125
|
-
}
|
|
1126
|
-
|
|
1127
|
-
// No newline found — entire trimmedEnd is one line
|
|
1128
|
-
if (reachedStart) {
|
|
1129
|
-
const firstLine = trimmedEnd.trim() || null;
|
|
1130
|
-
if (firstLine && messageOnly) {
|
|
1131
|
-
let matchedMessage: AgentMessage | null = null;
|
|
1132
|
-
try {
|
|
1133
|
-
matchedMessage = extractBootstrapMessageCandidate(JSON.parse(firstLine));
|
|
1134
|
-
} catch { /* not valid JSON */ }
|
|
1135
|
-
if (!matchedMessage || (matcher && !matcher(matchedMessage))) return null;
|
|
1136
|
-
}
|
|
1137
|
-
return firstLine;
|
|
1138
|
-
}
|
|
1139
|
-
// Need more data from earlier in the file
|
|
1140
|
-
continue;
|
|
1141
|
-
}
|
|
1142
|
-
return null;
|
|
1143
|
-
} catch {
|
|
1144
|
-
return null;
|
|
1145
|
-
} finally {
|
|
1146
|
-
if (fd != null) {
|
|
1147
|
-
closeSync(fd);
|
|
1148
|
-
}
|
|
1149
|
-
}
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
function readAppendedLeafPathMessages(params: {
|
|
1153
|
-
sessionFile: string;
|
|
1154
|
-
offset: number;
|
|
1155
|
-
}): { messages: AgentMessage[]; canUseAppendOnly: boolean; sawNonWhitespace: boolean } {
|
|
1156
|
-
const raw = readFileSegment(params.sessionFile, params.offset);
|
|
1157
|
-
if (raw == null) {
|
|
1158
|
-
return { messages: [], canUseAppendOnly: false, sawNonWhitespace: false };
|
|
1159
|
-
}
|
|
1160
|
-
|
|
1161
|
-
const trimmed = raw.trim();
|
|
1162
|
-
if (!trimmed) {
|
|
1163
|
-
return { messages: [], canUseAppendOnly: true, sawNonWhitespace: false };
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
if (trimmed.startsWith("[")) {
|
|
1167
|
-
return { messages: [], canUseAppendOnly: false, sawNonWhitespace: true };
|
|
1168
|
-
}
|
|
1169
|
-
|
|
1170
|
-
const parsed = parseBootstrapJsonl(raw, { strict: true });
|
|
1171
|
-
if (parsed.hadMalformedLine) {
|
|
1172
|
-
return { messages: [], canUseAppendOnly: false, sawNonWhitespace: parsed.sawNonWhitespace };
|
|
1173
|
-
}
|
|
1174
|
-
|
|
1175
|
-
return {
|
|
1176
|
-
messages: parsed.messages,
|
|
1177
|
-
canUseAppendOnly: true,
|
|
1178
|
-
sawNonWhitespace: parsed.sawNonWhitespace,
|
|
1179
|
-
};
|
|
1180
|
-
}
|
|
1181
|
-
|
|
1182
|
-
function readBootstrapMessageFromJsonLine(line: string | null): AgentMessage | null {
|
|
1183
|
-
if (!line) {
|
|
1184
|
-
return null;
|
|
1185
|
-
}
|
|
1186
|
-
try {
|
|
1187
|
-
return extractBootstrapMessageCandidate(JSON.parse(line));
|
|
1188
|
-
} catch {
|
|
1189
|
-
return null;
|
|
1190
|
-
}
|
|
1191
|
-
}
|
|
1192
|
-
|
|
1193
|
-
function messageIdentity(role: string, content: string): string {
|
|
1194
|
-
return `${role}\u0000${content}`;
|
|
1195
|
-
}
|
|
1196
|
-
|
|
1197
|
-
// ── LcmContextEngine ────────────────────────────────────────────────────────
|
|
1198
|
-
|
|
1199
|
-
export class LcmContextEngine implements ContextEngine {
|
|
1200
|
-
readonly info: ContextEngineInfo;
|
|
1201
|
-
|
|
1202
|
-
private config: LcmConfig;
|
|
1203
|
-
|
|
1204
|
-
/** Get the configured timezone, falling back to system timezone. */
|
|
1205
|
-
get timezone(): string {
|
|
1206
|
-
return this.config.timezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
1207
|
-
}
|
|
1208
|
-
|
|
1209
|
-
private conversationStore: ConversationStore;
|
|
1210
|
-
private summaryStore: SummaryStore;
|
|
1211
|
-
private compactionTelemetryStore: CompactionTelemetryStore;
|
|
1212
|
-
private assembler: ContextAssembler;
|
|
1213
|
-
private compaction: CompactionEngine;
|
|
1214
|
-
private retrieval: RetrievalEngine;
|
|
1215
|
-
private readonly db: DatabaseSync;
|
|
1216
|
-
private migrated = false;
|
|
1217
|
-
private readonly fts5Available: boolean;
|
|
1218
|
-
private readonly ignoreSessionPatterns: RegExp[];
|
|
1219
|
-
private readonly statelessSessionPatterns: RegExp[];
|
|
1220
|
-
private sessionOperationQueues = new Map<
|
|
1221
|
-
string,
|
|
1222
|
-
{ promise: Promise<void>; refCount: number }
|
|
1223
|
-
>();
|
|
1224
|
-
private largeFileTextSummarizerResolved = false;
|
|
1225
|
-
private largeFileTextSummarizer?: (prompt: string) => Promise<string | null>;
|
|
1226
|
-
private deps: LcmDependencies;
|
|
1227
|
-
|
|
1228
|
-
// ── Circuit breaker for compaction auth failures ──
|
|
1229
|
-
private circuitBreakerStates = new Map<string, CircuitBreakerState>();
|
|
1230
|
-
|
|
1231
|
-
constructor(deps: LcmDependencies, database: DatabaseSync) {
|
|
1232
|
-
this.deps = deps;
|
|
1233
|
-
this.config = deps.config;
|
|
1234
|
-
this.ignoreSessionPatterns = compileSessionPatterns(this.config.ignoreSessionPatterns);
|
|
1235
|
-
this.statelessSessionPatterns = compileSessionPatterns(this.config.statelessSessionPatterns);
|
|
1236
|
-
this.db = database;
|
|
1237
|
-
|
|
1238
|
-
// Run migrations eagerly at construction time so the schema exists
|
|
1239
|
-
// before any lifecycle hook fires.
|
|
1240
|
-
let migrationOk = false;
|
|
1241
|
-
const migrationStartedAt = Date.now();
|
|
1242
|
-
try {
|
|
1243
|
-
runLcmMigrations(this.db, {
|
|
1244
|
-
log: this.deps.log,
|
|
1245
|
-
});
|
|
1246
|
-
this.migrated = true;
|
|
1247
|
-
|
|
1248
|
-
// Verify tables were actually created
|
|
1249
|
-
const tables = this.db
|
|
1250
|
-
.prepare("SELECT name FROM sqlite_master WHERE type='table'")
|
|
1251
|
-
.all() as Array<{ name: string }>;
|
|
1252
|
-
if (tables.length === 0) {
|
|
1253
|
-
this.deps.log.warn(
|
|
1254
|
-
"[lcm] Migration completed but database has zero tables — DB may be non-functional",
|
|
1255
|
-
);
|
|
1256
|
-
} else {
|
|
1257
|
-
migrationOk = true;
|
|
1258
|
-
this.deps.log.info(
|
|
1259
|
-
`[lcm] Migration run completed during engine init: duration=${formatDurationMs(Date.now() - migrationStartedAt)} fts5=${this.fts5Available}`,
|
|
1260
|
-
);
|
|
1261
|
-
this.deps.log.debug(
|
|
1262
|
-
`[lcm] Migration successful — ${tables.length} tables: ${tables.map((t) => t.name).join(", ")}`,
|
|
1263
|
-
);
|
|
1264
|
-
}
|
|
1265
|
-
} catch (err) {
|
|
1266
|
-
this.deps.log.error(
|
|
1267
|
-
`[lcm] Migration failed after ${formatDurationMs(Date.now() - migrationStartedAt)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1268
|
-
);
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
this.fts5Available = getLcmDbFeatures(this.db).fts5Available;
|
|
1272
|
-
|
|
1273
|
-
// Only claim ownership of compaction when the DB is operational.
|
|
1274
|
-
// Without a working schema, ownsCompaction would disable the runtime's
|
|
1275
|
-
// built-in compaction safeguard and inflate the context budget.
|
|
1276
|
-
this.info = {
|
|
1277
|
-
id: "lcm",
|
|
1278
|
-
name: "Lossless Context Management Engine",
|
|
1279
|
-
version: "0.1.0",
|
|
1280
|
-
ownsCompaction: migrationOk,
|
|
1281
|
-
};
|
|
1282
|
-
|
|
1283
|
-
this.conversationStore = new ConversationStore(this.db, {
|
|
1284
|
-
fts5Available: this.fts5Available,
|
|
1285
|
-
});
|
|
1286
|
-
this.summaryStore = new SummaryStore(this.db, { fts5Available: this.fts5Available });
|
|
1287
|
-
this.compactionTelemetryStore = new CompactionTelemetryStore(this.db);
|
|
1288
|
-
|
|
1289
|
-
if (!this.fts5Available) {
|
|
1290
|
-
this.deps.log.warn(
|
|
1291
|
-
"[lcm] FTS5 unavailable in the current Node runtime; full_text search will fall back to LIKE and indexing is disabled",
|
|
1292
|
-
);
|
|
1293
|
-
}
|
|
1294
|
-
if (this.config.ignoreSessionPatterns.length > 0) {
|
|
1295
|
-
logStartupBannerOnce({
|
|
1296
|
-
key: "ignore-session-patterns",
|
|
1297
|
-
log: (message) => this.deps.log.info(message),
|
|
1298
|
-
message: `[lcm] Ignoring sessions matching ${this.config.ignoreSessionPatterns.length} pattern(s): ${this.config.ignoreSessionPatterns.join(", ")}`,
|
|
1299
|
-
});
|
|
1300
|
-
}
|
|
1301
|
-
if (this.config.skipStatelessSessions && this.config.statelessSessionPatterns.length > 0) {
|
|
1302
|
-
logStartupBannerOnce({
|
|
1303
|
-
key: "stateless-session-patterns",
|
|
1304
|
-
log: (message) => this.deps.log.info(message),
|
|
1305
|
-
message: `[lcm] Stateless session patterns: ${this.config.statelessSessionPatterns.length} pattern(s): ${this.config.statelessSessionPatterns.join(", ")}`,
|
|
1306
|
-
});
|
|
1307
|
-
}
|
|
1308
|
-
|
|
1309
|
-
this.assembler = new ContextAssembler(
|
|
1310
|
-
this.conversationStore,
|
|
1311
|
-
this.summaryStore,
|
|
1312
|
-
this.config.timezone,
|
|
1313
|
-
);
|
|
1314
|
-
|
|
1315
|
-
const compactionConfig: CompactionConfig = {
|
|
1316
|
-
contextThreshold: this.config.contextThreshold,
|
|
1317
|
-
freshTailCount: this.config.freshTailCount,
|
|
1318
|
-
leafMinFanout: this.config.leafMinFanout,
|
|
1319
|
-
condensedMinFanout: this.config.condensedMinFanout,
|
|
1320
|
-
condensedMinFanoutHard: this.config.condensedMinFanoutHard,
|
|
1321
|
-
incrementalMaxDepth: this.config.incrementalMaxDepth,
|
|
1322
|
-
leafChunkTokens: this.config.leafChunkTokens,
|
|
1323
|
-
leafTargetTokens: this.config.leafTargetTokens,
|
|
1324
|
-
condensedTargetTokens: this.config.condensedTargetTokens,
|
|
1325
|
-
maxRounds: 10,
|
|
1326
|
-
timezone: this.config.timezone,
|
|
1327
|
-
summaryMaxOverageFactor: this.config.summaryMaxOverageFactor,
|
|
1328
|
-
};
|
|
1329
|
-
this.compaction = new CompactionEngine(
|
|
1330
|
-
this.conversationStore,
|
|
1331
|
-
this.summaryStore,
|
|
1332
|
-
compactionConfig,
|
|
1333
|
-
this.deps.log,
|
|
1334
|
-
);
|
|
1335
|
-
|
|
1336
|
-
this.retrieval = new RetrievalEngine(this.conversationStore, this.summaryStore);
|
|
1337
|
-
}
|
|
1338
|
-
|
|
1339
|
-
/**
|
|
1340
|
-
* Check whether a session should be excluded from LCM processing.
|
|
1341
|
-
*
|
|
1342
|
-
* We prefer sessionKey matching because the configured glob patterns are
|
|
1343
|
-
* documented in terms of session keys, but we fall back to sessionId for
|
|
1344
|
-
* older call sites that may not provide the key yet.
|
|
1345
|
-
*/
|
|
1346
|
-
private shouldIgnoreSession(params: { sessionId?: string; sessionKey?: string }): boolean {
|
|
1347
|
-
if (this.ignoreSessionPatterns.length === 0) {
|
|
1348
|
-
return false;
|
|
1349
|
-
}
|
|
1350
|
-
|
|
1351
|
-
const candidate =
|
|
1352
|
-
typeof params.sessionKey === "string" && params.sessionKey.trim()
|
|
1353
|
-
? params.sessionKey.trim()
|
|
1354
|
-
: (params.sessionId?.trim() ?? "");
|
|
1355
|
-
if (!candidate) {
|
|
1356
|
-
return false;
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1359
|
-
return matchesSessionPattern(candidate, this.ignoreSessionPatterns);
|
|
1360
|
-
}
|
|
1361
|
-
|
|
1362
|
-
/** Check whether a session key should skip all LCM writes while remaining readable. */
|
|
1363
|
-
isStatelessSession(sessionKey: string | undefined): boolean {
|
|
1364
|
-
const trimmedKey = typeof sessionKey === "string" ? sessionKey.trim() : "";
|
|
1365
|
-
if (
|
|
1366
|
-
!this.config.skipStatelessSessions
|
|
1367
|
-
|| !trimmedKey
|
|
1368
|
-
|| this.statelessSessionPatterns.length === 0
|
|
1369
|
-
) {
|
|
1370
|
-
return false;
|
|
1371
|
-
}
|
|
1372
|
-
return matchesSessionPattern(trimmedKey, this.statelessSessionPatterns);
|
|
1373
|
-
}
|
|
1374
|
-
|
|
1375
|
-
// ── Circuit breaker helpers ──────────────────────────────────────────────
|
|
1376
|
-
|
|
1377
|
-
private getCircuitBreakerState(key: string): CircuitBreakerState {
|
|
1378
|
-
let state = this.circuitBreakerStates.get(key);
|
|
1379
|
-
if (!state) {
|
|
1380
|
-
state = { failures: 0, openSince: null };
|
|
1381
|
-
this.circuitBreakerStates.set(key, state);
|
|
1382
|
-
}
|
|
1383
|
-
return state;
|
|
1384
|
-
}
|
|
1385
|
-
|
|
1386
|
-
private isCircuitBreakerOpen(key: string): boolean {
|
|
1387
|
-
const state = this.circuitBreakerStates.get(key);
|
|
1388
|
-
if (!state || state.openSince === null) return false;
|
|
1389
|
-
const elapsed = Date.now() - state.openSince;
|
|
1390
|
-
if (elapsed >= this.config.circuitBreakerCooldownMs) {
|
|
1391
|
-
this.resetCircuitBreaker(key);
|
|
1392
|
-
return false;
|
|
1393
|
-
}
|
|
1394
|
-
return true;
|
|
1395
|
-
}
|
|
1396
|
-
|
|
1397
|
-
private recordCompactionAuthFailure(key: string): void {
|
|
1398
|
-
const state = this.getCircuitBreakerState(key);
|
|
1399
|
-
state.failures++;
|
|
1400
|
-
const halfThreshold = Math.ceil(this.config.circuitBreakerThreshold / 2);
|
|
1401
|
-
if (state.failures === halfThreshold && state.failures < this.config.circuitBreakerThreshold) {
|
|
1402
|
-
this.deps.log.warn(
|
|
1403
|
-
`[lcm] WARNING: compaction degraded — ${state.failures}/${this.config.circuitBreakerThreshold} consecutive auth failures for ${key}`,
|
|
1404
|
-
);
|
|
1405
|
-
}
|
|
1406
|
-
if (state.failures >= this.config.circuitBreakerThreshold) {
|
|
1407
|
-
state.openSince = Date.now();
|
|
1408
|
-
const cooldownMin = Math.round(this.config.circuitBreakerCooldownMs / 60000);
|
|
1409
|
-
this.deps.log.warn(
|
|
1410
|
-
`[lcm] CIRCUIT BREAKER OPEN: compaction disabled for ${key}. Auto-retry in ${cooldownMin}m. LCM is operating in degraded mode.`,
|
|
1411
|
-
);
|
|
1412
|
-
}
|
|
1413
|
-
}
|
|
1414
|
-
|
|
1415
|
-
private recordCompactionSuccess(key: string): void {
|
|
1416
|
-
const state = this.circuitBreakerStates.get(key);
|
|
1417
|
-
if (!state) {
|
|
1418
|
-
return;
|
|
1419
|
-
}
|
|
1420
|
-
if (state.failures > 0 || state.openSince !== null) {
|
|
1421
|
-
this.deps.log.info(
|
|
1422
|
-
`[lcm] compaction circuit breaker CLOSED: successful compaction for ${key} after ${state.failures} prior failures.`,
|
|
1423
|
-
);
|
|
1424
|
-
}
|
|
1425
|
-
this.resetCircuitBreaker(key);
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
|
-
private resetCircuitBreaker(key: string): void {
|
|
1429
|
-
this.circuitBreakerStates.delete(key);
|
|
1430
|
-
}
|
|
1431
|
-
|
|
1432
|
-
/** Ensure DB schema is up-to-date. Called lazily on first bootstrap/ingest/assemble/compact. */
|
|
1433
|
-
private ensureMigrated(): void {
|
|
1434
|
-
if (this.migrated) {
|
|
1435
|
-
return;
|
|
1436
|
-
}
|
|
1437
|
-
const migrationStartedAt = Date.now();
|
|
1438
|
-
this.deps.log.info("[lcm] ensureMigrated: running migrations lazily");
|
|
1439
|
-
runLcmMigrations(this.db, {
|
|
1440
|
-
log: this.deps.log,
|
|
1441
|
-
});
|
|
1442
|
-
this.migrated = true;
|
|
1443
|
-
this.deps.log.info(
|
|
1444
|
-
`[lcm] ensureMigrated: completed in ${formatDurationMs(Date.now() - migrationStartedAt)}`,
|
|
1445
|
-
);
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
/**
|
|
1449
|
-
* Serialize mutating operations per stable session identity to prevent
|
|
1450
|
-
* ingest/compaction races across runtime UUID recycling.
|
|
1451
|
-
*/
|
|
1452
|
-
private async withSessionQueue<T>(
|
|
1453
|
-
queueKey: string,
|
|
1454
|
-
operation: () => Promise<T>,
|
|
1455
|
-
options?: { operationName?: string; context?: string },
|
|
1456
|
-
): Promise<T> {
|
|
1457
|
-
const entry = this.sessionOperationQueues.get(queueKey);
|
|
1458
|
-
const previous = entry?.promise ?? Promise.resolve();
|
|
1459
|
-
const queuedAhead = entry?.refCount ?? 0;
|
|
1460
|
-
let releaseQueue: () => void = () => {};
|
|
1461
|
-
const current = new Promise<void>((resolve) => {
|
|
1462
|
-
releaseQueue = resolve;
|
|
1463
|
-
});
|
|
1464
|
-
const next = previous.catch(() => {}).then(() => current);
|
|
1465
|
-
|
|
1466
|
-
if (entry) {
|
|
1467
|
-
entry.promise = next;
|
|
1468
|
-
entry.refCount++;
|
|
1469
|
-
} else {
|
|
1470
|
-
this.sessionOperationQueues.set(queueKey, { promise: next, refCount: 1 });
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
const waitStartedAt = Date.now();
|
|
1474
|
-
await previous.catch(() => {});
|
|
1475
|
-
const waitMs = Date.now() - waitStartedAt;
|
|
1476
|
-
if (options?.operationName) {
|
|
1477
|
-
const detail = options.context ? ` ${options.context}` : "";
|
|
1478
|
-
this.deps.log.info(
|
|
1479
|
-
`[lcm] ${options.operationName}: session queue acquired queueKey=${queueKey} queuedAhead=${queuedAhead} wait=${formatDurationMs(waitMs)}${detail}`,
|
|
1480
|
-
);
|
|
1481
|
-
}
|
|
1482
|
-
try {
|
|
1483
|
-
return await operation();
|
|
1484
|
-
} finally {
|
|
1485
|
-
releaseQueue();
|
|
1486
|
-
const cur = this.sessionOperationQueues.get(queueKey);
|
|
1487
|
-
if (cur && --cur.refCount === 0) {
|
|
1488
|
-
this.sessionOperationQueues.delete(queueKey);
|
|
1489
|
-
}
|
|
1490
|
-
}
|
|
1491
|
-
}
|
|
1492
|
-
|
|
1493
|
-
/** Prefer stable session keys for queue serialization when available. */
|
|
1494
|
-
private resolveSessionQueueKey(sessionId?: string, sessionKey?: string): string {
|
|
1495
|
-
const normalizedSessionKey = sessionKey?.trim();
|
|
1496
|
-
const normalizedSessionId = sessionId?.trim();
|
|
1497
|
-
return normalizedSessionKey || normalizedSessionId || "__lcm__";
|
|
1498
|
-
}
|
|
1499
|
-
|
|
1500
|
-
/** Normalize optional live token estimates supplied by runtime callers. */
|
|
1501
|
-
private normalizeObservedTokenCount(value: unknown): number | undefined {
|
|
1502
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
1503
|
-
return undefined;
|
|
1504
|
-
}
|
|
1505
|
-
return Math.floor(value);
|
|
1506
|
-
}
|
|
1507
|
-
|
|
1508
|
-
/** Resolve token budget from direct params or legacy fallback input. */
|
|
1509
|
-
private resolveTokenBudget(params: {
|
|
1510
|
-
tokenBudget?: number;
|
|
1511
|
-
runtimeContext?: Record<string, unknown>;
|
|
1512
|
-
legacyParams?: Record<string, unknown>;
|
|
1513
|
-
}): number | undefined {
|
|
1514
|
-
const lp = asRecord(params.runtimeContext) ?? params.legacyParams ?? {};
|
|
1515
|
-
if (
|
|
1516
|
-
typeof params.tokenBudget === "number" &&
|
|
1517
|
-
Number.isFinite(params.tokenBudget) &&
|
|
1518
|
-
params.tokenBudget > 0
|
|
1519
|
-
) {
|
|
1520
|
-
return Math.floor(params.tokenBudget);
|
|
1521
|
-
}
|
|
1522
|
-
if (
|
|
1523
|
-
typeof lp.tokenBudget === "number" &&
|
|
1524
|
-
Number.isFinite(lp.tokenBudget) &&
|
|
1525
|
-
lp.tokenBudget > 0
|
|
1526
|
-
) {
|
|
1527
|
-
return Math.floor(lp.tokenBudget);
|
|
1528
|
-
}
|
|
1529
|
-
return undefined;
|
|
1530
|
-
}
|
|
1531
|
-
|
|
1532
|
-
/** Cap a resolved token budget against the configured maxAssemblyTokenBudget. */
|
|
1533
|
-
private applyAssemblyBudgetCap(budget: number): number {
|
|
1534
|
-
const cap = this.config.maxAssemblyTokenBudget;
|
|
1535
|
-
return cap != null && cap > 0 ? Math.min(budget, cap) : budget;
|
|
1536
|
-
}
|
|
1537
|
-
|
|
1538
|
-
/** Normalize token counters that may legitimately be zero. */
|
|
1539
|
-
private normalizeOptionalCount(value: unknown): number | undefined {
|
|
1540
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1541
|
-
return undefined;
|
|
1542
|
-
}
|
|
1543
|
-
return Math.floor(value);
|
|
1544
|
-
}
|
|
1545
|
-
|
|
1546
|
-
/** Treat a recent cache hit as still-hot for a couple of turns unless telemetry observed a later break. */
|
|
1547
|
-
private shouldApplyHotCacheHysteresis(
|
|
1548
|
-
telemetry: ConversationCompactionTelemetryRecord | null,
|
|
1549
|
-
): boolean {
|
|
1550
|
-
if (!telemetry?.lastObservedCacheHitAt) {
|
|
1551
|
-
return false;
|
|
1552
|
-
}
|
|
1553
|
-
if (
|
|
1554
|
-
telemetry.lastObservedCacheBreakAt
|
|
1555
|
-
&& telemetry.lastObservedCacheBreakAt >= telemetry.lastObservedCacheHitAt
|
|
1556
|
-
) {
|
|
1557
|
-
return false;
|
|
1558
|
-
}
|
|
1559
|
-
return telemetry.turnsSinceLeafCompaction <= HOT_CACHE_HYSTERESIS_TURNS;
|
|
1560
|
-
}
|
|
1561
|
-
|
|
1562
|
-
/** Resolve the effective cache state the incremental compaction policy should react to. */
|
|
1563
|
-
private resolveCacheAwareState(
|
|
1564
|
-
telemetry: ConversationCompactionTelemetryRecord | null,
|
|
1565
|
-
): CacheState {
|
|
1566
|
-
if (!telemetry) {
|
|
1567
|
-
return "unknown";
|
|
1568
|
-
}
|
|
1569
|
-
if (telemetry.cacheState === "hot") {
|
|
1570
|
-
return "hot";
|
|
1571
|
-
}
|
|
1572
|
-
if (this.shouldApplyHotCacheHysteresis(telemetry)) {
|
|
1573
|
-
return "hot";
|
|
1574
|
-
}
|
|
1575
|
-
return telemetry.cacheState;
|
|
1576
|
-
}
|
|
1577
|
-
|
|
1578
|
-
/** Decide whether a hot cache still has enough real token-budget headroom to skip incremental maintenance. */
|
|
1579
|
-
private isComfortablyUnderTokenBudget(params: {
|
|
1580
|
-
currentTokenCount?: number;
|
|
1581
|
-
tokenBudget: number;
|
|
1582
|
-
}): boolean {
|
|
1583
|
-
if (
|
|
1584
|
-
typeof params.currentTokenCount !== "number"
|
|
1585
|
-
|| !Number.isFinite(params.currentTokenCount)
|
|
1586
|
-
|| params.currentTokenCount < 0
|
|
1587
|
-
) {
|
|
1588
|
-
return false;
|
|
1589
|
-
}
|
|
1590
|
-
const budget = Math.max(1, Math.floor(params.tokenBudget));
|
|
1591
|
-
const safeBudget = Math.floor(
|
|
1592
|
-
budget * (1 - this.config.cacheAwareCompaction.hotCacheBudgetHeadroomRatio),
|
|
1593
|
-
);
|
|
1594
|
-
return params.currentTokenCount <= safeBudget;
|
|
1595
|
-
}
|
|
1596
|
-
|
|
1597
|
-
/** Resolve bounded dynamic leaf chunk sizes from config and the active token budget. */
|
|
1598
|
-
private resolveDynamicLeafChunkBounds(tokenBudget?: number): DynamicLeafChunkBounds {
|
|
1599
|
-
const floor = Math.max(1, Math.floor(this.config.leafChunkTokens));
|
|
1600
|
-
const configuredMax = this.config.dynamicLeafChunkTokens.enabled
|
|
1601
|
-
? Math.max(floor, Math.floor(this.config.dynamicLeafChunkTokens.max))
|
|
1602
|
-
: floor;
|
|
1603
|
-
const budgetCap =
|
|
1604
|
-
typeof tokenBudget === "number" &&
|
|
1605
|
-
Number.isFinite(tokenBudget) &&
|
|
1606
|
-
tokenBudget > 0
|
|
1607
|
-
? Math.max(floor, Math.floor(tokenBudget * this.config.contextThreshold))
|
|
1608
|
-
: configuredMax;
|
|
1609
|
-
const max = Math.max(floor, Math.min(configuredMax, budgetCap));
|
|
1610
|
-
const medium = Math.max(
|
|
1611
|
-
floor,
|
|
1612
|
-
Math.min(max, Math.floor(floor * DYNAMIC_LEAF_CHUNK_MEDIUM_MULTIPLIER)),
|
|
1613
|
-
);
|
|
1614
|
-
const high = Math.max(
|
|
1615
|
-
floor,
|
|
1616
|
-
Math.min(max, Math.floor(floor * DYNAMIC_LEAF_CHUNK_HIGH_MULTIPLIER)),
|
|
1617
|
-
);
|
|
1618
|
-
return { floor, medium, high, max };
|
|
1619
|
-
}
|
|
1620
|
-
|
|
1621
|
-
/** Classify the current refill rate into a simple step band with downshift hysteresis. */
|
|
1622
|
-
private classifyDynamicLeafActivityBand(params: {
|
|
1623
|
-
lastActivityBand?: ActivityBand;
|
|
1624
|
-
tokensAccumulatedSinceLeafCompaction: number;
|
|
1625
|
-
turnsSinceLeafCompaction: number;
|
|
1626
|
-
floor: number;
|
|
1627
|
-
}): ActivityBand {
|
|
1628
|
-
const turns = Math.max(1, params.turnsSinceLeafCompaction);
|
|
1629
|
-
const tokensPerTurn = params.tokensAccumulatedSinceLeafCompaction / turns;
|
|
1630
|
-
const mediumUpshift = params.floor * DYNAMIC_ACTIVITY_MEDIUM_UPSHIFT_FACTOR;
|
|
1631
|
-
const mediumDownshift = params.floor * DYNAMIC_ACTIVITY_MEDIUM_DOWNSHIFT_FACTOR;
|
|
1632
|
-
const highUpshift = params.floor * DYNAMIC_ACTIVITY_HIGH_UPSHIFT_FACTOR;
|
|
1633
|
-
const highDownshift = params.floor * DYNAMIC_ACTIVITY_HIGH_DOWNSHIFT_FACTOR;
|
|
1634
|
-
const lastBand = params.lastActivityBand ?? "low";
|
|
1635
|
-
|
|
1636
|
-
if (lastBand === "high") {
|
|
1637
|
-
if (tokensPerTurn >= highDownshift) {
|
|
1638
|
-
return "high";
|
|
1639
|
-
}
|
|
1640
|
-
return tokensPerTurn >= mediumDownshift ? "medium" : "low";
|
|
1641
|
-
}
|
|
1642
|
-
if (lastBand === "medium") {
|
|
1643
|
-
if (tokensPerTurn >= highUpshift) {
|
|
1644
|
-
return "high";
|
|
1645
|
-
}
|
|
1646
|
-
if (tokensPerTurn < mediumDownshift) {
|
|
1647
|
-
return "low";
|
|
1648
|
-
}
|
|
1649
|
-
return "medium";
|
|
1650
|
-
}
|
|
1651
|
-
if (tokensPerTurn >= highUpshift) {
|
|
1652
|
-
return "high";
|
|
1653
|
-
}
|
|
1654
|
-
if (tokensPerTurn >= mediumUpshift) {
|
|
1655
|
-
return "medium";
|
|
1656
|
-
}
|
|
1657
|
-
return "low";
|
|
1658
|
-
}
|
|
1659
|
-
|
|
1660
|
-
/** Map an activity band to the corresponding working leaf chunk size. */
|
|
1661
|
-
private resolveLeafChunkTokensForBand(
|
|
1662
|
-
band: ActivityBand,
|
|
1663
|
-
bounds: DynamicLeafChunkBounds,
|
|
1664
|
-
): number {
|
|
1665
|
-
switch (band) {
|
|
1666
|
-
case "high":
|
|
1667
|
-
return bounds.high;
|
|
1668
|
-
case "medium":
|
|
1669
|
-
return bounds.medium;
|
|
1670
|
-
default:
|
|
1671
|
-
return bounds.floor;
|
|
1672
|
-
}
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
|
-
/** Build descending fallback chunk sizes used when a provider rejects a larger chunk. */
|
|
1676
|
-
private buildLeafChunkFallbacks(params: {
|
|
1677
|
-
preferred: number;
|
|
1678
|
-
bounds: DynamicLeafChunkBounds;
|
|
1679
|
-
}): number[] {
|
|
1680
|
-
const ordered = [params.preferred, params.bounds.max, params.bounds.high, params.bounds.medium, params.bounds.floor];
|
|
1681
|
-
const seen = new Set<number>();
|
|
1682
|
-
const fallbacks: number[] = [];
|
|
1683
|
-
for (const value of ordered) {
|
|
1684
|
-
const normalized = Math.max(params.bounds.floor, Math.floor(value));
|
|
1685
|
-
if (seen.has(normalized)) {
|
|
1686
|
-
continue;
|
|
1687
|
-
}
|
|
1688
|
-
seen.add(normalized);
|
|
1689
|
-
fallbacks.push(normalized);
|
|
1690
|
-
}
|
|
1691
|
-
return fallbacks.sort((a, b) => b - a);
|
|
1692
|
-
}
|
|
1693
|
-
|
|
1694
|
-
/** Detect provider/model token-limit failures that should trigger a lower chunk retry. */
|
|
1695
|
-
private isRecoverableLeafChunkOverflowError(error: unknown): boolean {
|
|
1696
|
-
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
1697
|
-
if (!message) {
|
|
1698
|
-
return false;
|
|
1699
|
-
}
|
|
1700
|
-
return [
|
|
1701
|
-
"context length",
|
|
1702
|
-
"context window",
|
|
1703
|
-
"maximum context",
|
|
1704
|
-
"max context",
|
|
1705
|
-
"too many tokens",
|
|
1706
|
-
"too many input tokens",
|
|
1707
|
-
"input tokens",
|
|
1708
|
-
"token limit",
|
|
1709
|
-
"context limit",
|
|
1710
|
-
"input is too large",
|
|
1711
|
-
"input too large",
|
|
1712
|
-
"prompt is too long",
|
|
1713
|
-
"request too large",
|
|
1714
|
-
"exceeds the model",
|
|
1715
|
-
"exceeds context",
|
|
1716
|
-
].some((fragment) => message.includes(fragment));
|
|
1717
|
-
}
|
|
1718
|
-
|
|
1719
|
-
/** Extract the current prompt-cache snapshot from runtime context, if present. */
|
|
1720
|
-
private readPromptCacheSnapshot(runtimeContext?: Record<string, unknown>): PromptCacheSnapshot | null {
|
|
1721
|
-
const promptCache = asRecord(runtimeContext?.promptCache);
|
|
1722
|
-
if (!promptCache) {
|
|
1723
|
-
return null;
|
|
1724
|
-
}
|
|
1725
|
-
|
|
1726
|
-
const lastCallUsage = asRecord(promptCache.lastCallUsage);
|
|
1727
|
-
const observation = asRecord(promptCache.observation);
|
|
1728
|
-
const cacheRead = this.normalizeOptionalCount(lastCallUsage?.cacheRead);
|
|
1729
|
-
const cacheWrite = this.normalizeOptionalCount(lastCallUsage?.cacheWrite);
|
|
1730
|
-
const sawExplicitBreak = safeBoolean(observation?.broke) === true;
|
|
1731
|
-
const retention = safeString(promptCache.retention)?.trim();
|
|
1732
|
-
const hasUsageSignal = cacheRead !== undefined || cacheWrite !== undefined;
|
|
1733
|
-
const hasObservationSignal =
|
|
1734
|
-
typeof observation?.cacheRead === "number"
|
|
1735
|
-
|| typeof observation?.previousCacheRead === "number"
|
|
1736
|
-
|| sawExplicitBreak;
|
|
1737
|
-
|
|
1738
|
-
let cacheState: CacheState = "unknown";
|
|
1739
|
-
if (sawExplicitBreak) {
|
|
1740
|
-
cacheState = "cold";
|
|
1741
|
-
} else if (typeof cacheRead === "number" && cacheRead > 0) {
|
|
1742
|
-
cacheState = "hot";
|
|
1743
|
-
} else if (hasUsageSignal || hasObservationSignal) {
|
|
1744
|
-
cacheState = "cold";
|
|
1745
|
-
}
|
|
1746
|
-
|
|
1747
|
-
return {
|
|
1748
|
-
...(cacheRead !== undefined ? { lastObservedCacheRead: cacheRead } : {}),
|
|
1749
|
-
...(cacheWrite !== undefined ? { lastObservedCacheWrite: cacheWrite } : {}),
|
|
1750
|
-
cacheState,
|
|
1751
|
-
...(retention ? { retention } : {}),
|
|
1752
|
-
sawExplicitBreak,
|
|
1753
|
-
};
|
|
1754
|
-
}
|
|
1755
|
-
|
|
1756
|
-
/** Persist the current turn's compaction telemetry for later policy decisions. */
|
|
1757
|
-
private async updateCompactionTelemetry(params: {
|
|
1758
|
-
conversationId: number;
|
|
1759
|
-
runtimeContext?: Record<string, unknown>;
|
|
1760
|
-
tokenBudget?: number;
|
|
1761
|
-
rawTokensOutsideTail?: number;
|
|
1762
|
-
}): Promise<ConversationCompactionTelemetryRecord | null> {
|
|
1763
|
-
const snapshot = this.readPromptCacheSnapshot(params.runtimeContext);
|
|
1764
|
-
const existing = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
|
|
1765
|
-
params.conversationId,
|
|
1766
|
-
);
|
|
1767
|
-
if (!snapshot && params.rawTokensOutsideTail === undefined) {
|
|
1768
|
-
return existing;
|
|
1769
|
-
}
|
|
1770
|
-
|
|
1771
|
-
const now = new Date();
|
|
1772
|
-
const bounds = this.resolveDynamicLeafChunkBounds(params.tokenBudget);
|
|
1773
|
-
const turnsSinceLeafCompaction =
|
|
1774
|
-
(existing?.turnsSinceLeafCompaction ?? 0) + 1;
|
|
1775
|
-
const tokensAccumulatedSinceLeafCompaction =
|
|
1776
|
-
params.rawTokensOutsideTail ?? existing?.tokensAccumulatedSinceLeafCompaction ?? 0;
|
|
1777
|
-
const lastActivityBand = this.classifyDynamicLeafActivityBand({
|
|
1778
|
-
lastActivityBand: existing?.lastActivityBand,
|
|
1779
|
-
tokensAccumulatedSinceLeafCompaction,
|
|
1780
|
-
turnsSinceLeafCompaction,
|
|
1781
|
-
floor: bounds.floor,
|
|
1782
|
-
});
|
|
1783
|
-
await this.compactionTelemetryStore.upsertConversationCompactionTelemetry({
|
|
1784
|
-
conversationId: params.conversationId,
|
|
1785
|
-
lastObservedCacheRead: snapshot?.lastObservedCacheRead ?? existing?.lastObservedCacheRead ?? null,
|
|
1786
|
-
lastObservedCacheWrite:
|
|
1787
|
-
snapshot?.lastObservedCacheWrite ?? existing?.lastObservedCacheWrite ?? null,
|
|
1788
|
-
lastObservedCacheHitAt:
|
|
1789
|
-
snapshot?.cacheState === "hot"
|
|
1790
|
-
? now
|
|
1791
|
-
: existing?.lastObservedCacheHitAt ?? null,
|
|
1792
|
-
lastObservedCacheBreakAt:
|
|
1793
|
-
snapshot?.sawExplicitBreak
|
|
1794
|
-
? now
|
|
1795
|
-
: existing?.lastObservedCacheBreakAt ?? null,
|
|
1796
|
-
cacheState: snapshot?.cacheState ?? existing?.cacheState ?? "unknown",
|
|
1797
|
-
retention: snapshot?.retention ?? existing?.retention ?? null,
|
|
1798
|
-
lastLeafCompactionAt: existing?.lastLeafCompactionAt ?? null,
|
|
1799
|
-
turnsSinceLeafCompaction,
|
|
1800
|
-
tokensAccumulatedSinceLeafCompaction,
|
|
1801
|
-
lastActivityBand,
|
|
1802
|
-
});
|
|
1803
|
-
const updated = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
|
|
1804
|
-
params.conversationId,
|
|
1805
|
-
);
|
|
1806
|
-
if (updated) {
|
|
1807
|
-
this.deps.log.debug(
|
|
1808
|
-
`[lcm] compaction telemetry updated: conversation=${params.conversationId} cacheState=${updated.cacheState} cacheRead=${updated.lastObservedCacheRead ?? "null"} cacheWrite=${updated.lastObservedCacheWrite ?? "null"} retention=${updated.retention ?? "null"} turnsSinceLeafCompaction=${updated.turnsSinceLeafCompaction} tokensSinceLeafCompaction=${updated.tokensAccumulatedSinceLeafCompaction} activityBand=${updated.lastActivityBand} rawTokensOutsideTail=${params.rawTokensOutsideTail ?? "null"} tokenBudget=${params.tokenBudget ?? "null"}`,
|
|
1809
|
-
);
|
|
1810
|
-
}
|
|
1811
|
-
return updated;
|
|
1812
|
-
}
|
|
1813
|
-
|
|
1814
|
-
/** Reset refill counters after any successful leaf-producing compaction. */
|
|
1815
|
-
private async markLeafCompactionTelemetrySuccess(params: {
|
|
1816
|
-
conversationId: number;
|
|
1817
|
-
activityBand?: ActivityBand;
|
|
1818
|
-
}): Promise<void> {
|
|
1819
|
-
const existing = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
|
|
1820
|
-
params.conversationId,
|
|
1821
|
-
);
|
|
1822
|
-
await this.compactionTelemetryStore.upsertConversationCompactionTelemetry({
|
|
1823
|
-
conversationId: params.conversationId,
|
|
1824
|
-
lastObservedCacheRead: existing?.lastObservedCacheRead ?? null,
|
|
1825
|
-
lastObservedCacheWrite: existing?.lastObservedCacheWrite ?? null,
|
|
1826
|
-
lastObservedCacheHitAt: existing?.lastObservedCacheHitAt ?? null,
|
|
1827
|
-
lastObservedCacheBreakAt: existing?.lastObservedCacheBreakAt ?? null,
|
|
1828
|
-
cacheState: existing?.cacheState ?? "unknown",
|
|
1829
|
-
retention: existing?.retention ?? null,
|
|
1830
|
-
lastLeafCompactionAt: new Date(),
|
|
1831
|
-
turnsSinceLeafCompaction: 0,
|
|
1832
|
-
tokensAccumulatedSinceLeafCompaction: 0,
|
|
1833
|
-
lastActivityBand: params.activityBand ?? existing?.lastActivityBand ?? "low",
|
|
1834
|
-
});
|
|
1835
|
-
this.deps.log.debug(
|
|
1836
|
-
`[lcm] compaction telemetry reset after leaf compaction: conversation=${params.conversationId} cacheState=${existing?.cacheState ?? "unknown"} activityBand=${params.activityBand ?? existing?.lastActivityBand ?? "low"}`,
|
|
1837
|
-
);
|
|
1838
|
-
}
|
|
1839
|
-
|
|
1840
|
-
/** Emit an operational trace for the incremental compaction policy decision. */
|
|
1841
|
-
private logIncrementalCompactionDecision(params: {
|
|
1842
|
-
conversationId: number;
|
|
1843
|
-
cacheState: CacheState;
|
|
1844
|
-
activityBand: ActivityBand;
|
|
1845
|
-
triggerLeafChunkTokens: number;
|
|
1846
|
-
preferredLeafChunkTokens: number;
|
|
1847
|
-
fallbackLeafChunkTokens: number[];
|
|
1848
|
-
rawTokensOutsideTail: number;
|
|
1849
|
-
threshold: number;
|
|
1850
|
-
shouldCompact: boolean;
|
|
1851
|
-
maxPasses: number;
|
|
1852
|
-
allowCondensedPasses: boolean;
|
|
1853
|
-
reason: string;
|
|
1854
|
-
}): IncrementalCompactionDecision {
|
|
1855
|
-
this.deps.log.info(
|
|
1856
|
-
`[lcm] incremental compaction decision: conversation=${params.conversationId} cacheState=${params.cacheState} activityBand=${params.activityBand} triggerLeafChunkTokens=${params.triggerLeafChunkTokens} preferredLeafChunkTokens=${params.preferredLeafChunkTokens} fallbackLeafChunkTokens=${params.fallbackLeafChunkTokens.join(",")} rawTokensOutsideTail=${params.rawTokensOutsideTail} threshold=${params.threshold} shouldCompact=${params.shouldCompact} maxPasses=${params.maxPasses} allowCondensedPasses=${params.allowCondensedPasses} reason=${params.reason}`,
|
|
1857
|
-
);
|
|
1858
|
-
return {
|
|
1859
|
-
shouldCompact: params.shouldCompact,
|
|
1860
|
-
cacheState: params.cacheState,
|
|
1861
|
-
maxPasses: params.maxPasses,
|
|
1862
|
-
rawTokensOutsideTail: params.rawTokensOutsideTail,
|
|
1863
|
-
threshold: params.threshold,
|
|
1864
|
-
leafChunkTokens: params.preferredLeafChunkTokens,
|
|
1865
|
-
fallbackLeafChunkTokens: params.fallbackLeafChunkTokens,
|
|
1866
|
-
activityBand: params.activityBand,
|
|
1867
|
-
allowCondensedPasses: params.allowCondensedPasses,
|
|
1868
|
-
};
|
|
1869
|
-
}
|
|
1870
|
-
|
|
1871
|
-
/** Resolve the cache-aware incremental-compaction policy for the current session. */
|
|
1872
|
-
private async evaluateIncrementalCompaction(params: {
|
|
1873
|
-
conversationId: number;
|
|
1874
|
-
tokenBudget: number;
|
|
1875
|
-
currentTokenCount?: number;
|
|
1876
|
-
}): Promise<IncrementalCompactionDecision> {
|
|
1877
|
-
const telemetry = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
|
|
1878
|
-
params.conversationId,
|
|
1879
|
-
);
|
|
1880
|
-
const cacheState =
|
|
1881
|
-
this.config.cacheAwareCompaction.enabled
|
|
1882
|
-
? this.resolveCacheAwareState(telemetry)
|
|
1883
|
-
: "unknown";
|
|
1884
|
-
const bounds = this.resolveDynamicLeafChunkBounds(params.tokenBudget);
|
|
1885
|
-
const activityBand =
|
|
1886
|
-
this.config.dynamicLeafChunkTokens.enabled
|
|
1887
|
-
? this.classifyDynamicLeafActivityBand({
|
|
1888
|
-
lastActivityBand: telemetry?.lastActivityBand,
|
|
1889
|
-
tokensAccumulatedSinceLeafCompaction:
|
|
1890
|
-
telemetry?.tokensAccumulatedSinceLeafCompaction ?? 0,
|
|
1891
|
-
turnsSinceLeafCompaction: telemetry?.turnsSinceLeafCompaction ?? 0,
|
|
1892
|
-
floor: bounds.floor,
|
|
1893
|
-
})
|
|
1894
|
-
: "low";
|
|
1895
|
-
const triggerLeafChunkTokens =
|
|
1896
|
-
this.config.dynamicLeafChunkTokens.enabled && cacheState === "hot"
|
|
1897
|
-
? bounds.max
|
|
1898
|
-
: this.config.dynamicLeafChunkTokens.enabled
|
|
1899
|
-
? this.resolveLeafChunkTokensForBand(activityBand, bounds)
|
|
1900
|
-
: bounds.floor;
|
|
1901
|
-
const preferredLeafChunkTokens =
|
|
1902
|
-
this.config.cacheAwareCompaction.enabled && (cacheState === "cold" || cacheState === "hot")
|
|
1903
|
-
? bounds.max
|
|
1904
|
-
: triggerLeafChunkTokens;
|
|
1905
|
-
const fallbackLeafChunkTokens = this.buildLeafChunkFallbacks({
|
|
1906
|
-
preferred: preferredLeafChunkTokens,
|
|
1907
|
-
bounds,
|
|
1908
|
-
});
|
|
1909
|
-
const leafTrigger = await this.compaction.evaluateLeafTrigger(
|
|
1910
|
-
params.conversationId,
|
|
1911
|
-
triggerLeafChunkTokens,
|
|
1912
|
-
);
|
|
1913
|
-
if (!leafTrigger.shouldCompact) {
|
|
1914
|
-
return this.logIncrementalCompactionDecision({
|
|
1915
|
-
conversationId: params.conversationId,
|
|
1916
|
-
cacheState,
|
|
1917
|
-
activityBand,
|
|
1918
|
-
triggerLeafChunkTokens,
|
|
1919
|
-
preferredLeafChunkTokens,
|
|
1920
|
-
fallbackLeafChunkTokens,
|
|
1921
|
-
rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
|
|
1922
|
-
threshold: leafTrigger.threshold,
|
|
1923
|
-
shouldCompact: false,
|
|
1924
|
-
maxPasses: 1,
|
|
1925
|
-
allowCondensedPasses: false,
|
|
1926
|
-
reason: "below-leaf-trigger",
|
|
1927
|
-
});
|
|
1928
|
-
}
|
|
1929
|
-
|
|
1930
|
-
const budgetDecision = await this.compaction.evaluate(
|
|
1931
|
-
params.conversationId,
|
|
1932
|
-
params.tokenBudget,
|
|
1933
|
-
params.currentTokenCount,
|
|
1934
|
-
);
|
|
1935
|
-
if (budgetDecision.shouldCompact) {
|
|
1936
|
-
return this.logIncrementalCompactionDecision({
|
|
1937
|
-
conversationId: params.conversationId,
|
|
1938
|
-
cacheState,
|
|
1939
|
-
activityBand,
|
|
1940
|
-
triggerLeafChunkTokens,
|
|
1941
|
-
preferredLeafChunkTokens,
|
|
1942
|
-
fallbackLeafChunkTokens,
|
|
1943
|
-
rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
|
|
1944
|
-
threshold: leafTrigger.threshold,
|
|
1945
|
-
shouldCompact: true,
|
|
1946
|
-
maxPasses: 1,
|
|
1947
|
-
allowCondensedPasses: true,
|
|
1948
|
-
reason: "budget-trigger",
|
|
1949
|
-
});
|
|
1950
|
-
}
|
|
1951
|
-
|
|
1952
|
-
if (
|
|
1953
|
-
cacheState === "hot"
|
|
1954
|
-
&& this.isComfortablyUnderTokenBudget({
|
|
1955
|
-
currentTokenCount: params.currentTokenCount,
|
|
1956
|
-
tokenBudget: params.tokenBudget,
|
|
1957
|
-
})
|
|
1958
|
-
) {
|
|
1959
|
-
return this.logIncrementalCompactionDecision({
|
|
1960
|
-
conversationId: params.conversationId,
|
|
1961
|
-
cacheState,
|
|
1962
|
-
activityBand,
|
|
1963
|
-
triggerLeafChunkTokens,
|
|
1964
|
-
preferredLeafChunkTokens,
|
|
1965
|
-
fallbackLeafChunkTokens,
|
|
1966
|
-
rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
|
|
1967
|
-
threshold: leafTrigger.threshold,
|
|
1968
|
-
shouldCompact: false,
|
|
1969
|
-
maxPasses: 1,
|
|
1970
|
-
allowCondensedPasses: false,
|
|
1971
|
-
reason: "hot-cache-budget-headroom",
|
|
1972
|
-
});
|
|
1973
|
-
}
|
|
1974
|
-
|
|
1975
|
-
if (
|
|
1976
|
-
cacheState === "hot"
|
|
1977
|
-
&& leafTrigger.rawTokensOutsideTail
|
|
1978
|
-
< Math.floor(
|
|
1979
|
-
leafTrigger.threshold * this.config.cacheAwareCompaction.hotCachePressureFactor,
|
|
1980
|
-
)
|
|
1981
|
-
) {
|
|
1982
|
-
return this.logIncrementalCompactionDecision({
|
|
1983
|
-
conversationId: params.conversationId,
|
|
1984
|
-
cacheState,
|
|
1985
|
-
activityBand,
|
|
1986
|
-
triggerLeafChunkTokens,
|
|
1987
|
-
preferredLeafChunkTokens,
|
|
1988
|
-
fallbackLeafChunkTokens,
|
|
1989
|
-
rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
|
|
1990
|
-
threshold: leafTrigger.threshold,
|
|
1991
|
-
shouldCompact: false,
|
|
1992
|
-
maxPasses: 1,
|
|
1993
|
-
allowCondensedPasses: false,
|
|
1994
|
-
reason: "hot-cache-defer",
|
|
1995
|
-
});
|
|
1996
|
-
}
|
|
1997
|
-
|
|
1998
|
-
const maxPasses =
|
|
1999
|
-
cacheState === "cold"
|
|
2000
|
-
? Math.max(1, this.config.cacheAwareCompaction.maxColdCacheCatchupPasses)
|
|
2001
|
-
: 1;
|
|
2002
|
-
return this.logIncrementalCompactionDecision({
|
|
2003
|
-
conversationId: params.conversationId,
|
|
2004
|
-
cacheState,
|
|
2005
|
-
activityBand,
|
|
2006
|
-
triggerLeafChunkTokens,
|
|
2007
|
-
preferredLeafChunkTokens,
|
|
2008
|
-
fallbackLeafChunkTokens,
|
|
2009
|
-
rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
|
|
2010
|
-
threshold: leafTrigger.threshold,
|
|
2011
|
-
shouldCompact: true,
|
|
2012
|
-
maxPasses,
|
|
2013
|
-
allowCondensedPasses: cacheState !== "hot",
|
|
2014
|
-
reason: cacheState === "cold" ? "cold-cache-catchup" : "leaf-trigger",
|
|
2015
|
-
});
|
|
2016
|
-
}
|
|
2017
|
-
|
|
2018
|
-
/** Resolve an LCM conversation id from a session key via the session store. */
|
|
2019
|
-
private async resolveConversationIdForSessionKey(
|
|
2020
|
-
sessionKey: string,
|
|
2021
|
-
): Promise<number | undefined> {
|
|
2022
|
-
const trimmedKey = sessionKey.trim();
|
|
2023
|
-
if (!trimmedKey) {
|
|
2024
|
-
return undefined;
|
|
2025
|
-
}
|
|
2026
|
-
try {
|
|
2027
|
-
const bySessionKey = await this.conversationStore.getConversationForSession({
|
|
2028
|
-
sessionKey: trimmedKey,
|
|
2029
|
-
});
|
|
2030
|
-
if (bySessionKey) {
|
|
2031
|
-
return bySessionKey.conversationId;
|
|
2032
|
-
}
|
|
2033
|
-
|
|
2034
|
-
const runtimeSessionId = await this.deps.resolveSessionIdFromSessionKey(trimmedKey);
|
|
2035
|
-
if (!runtimeSessionId) {
|
|
2036
|
-
return undefined;
|
|
2037
|
-
}
|
|
2038
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
2039
|
-
sessionId: runtimeSessionId,
|
|
2040
|
-
});
|
|
2041
|
-
return conversation?.conversationId;
|
|
2042
|
-
} catch {
|
|
2043
|
-
return undefined;
|
|
2044
|
-
}
|
|
2045
|
-
}
|
|
2046
|
-
|
|
2047
|
-
/** Format stable session identifiers for LCM diagnostic logs. */
|
|
2048
|
-
private formatSessionLogContext(params: {
|
|
2049
|
-
conversationId: number;
|
|
2050
|
-
sessionId: string;
|
|
2051
|
-
sessionKey?: string;
|
|
2052
|
-
}): string {
|
|
2053
|
-
const parts = [
|
|
2054
|
-
`conversation=${params.conversationId}`,
|
|
2055
|
-
`session=${params.sessionId}`,
|
|
2056
|
-
];
|
|
2057
|
-
const trimmedSessionKey = params.sessionKey?.trim();
|
|
2058
|
-
if (trimmedSessionKey) {
|
|
2059
|
-
parts.push(`sessionKey=${trimmedSessionKey}`);
|
|
2060
|
-
}
|
|
2061
|
-
return parts.join(" ");
|
|
2062
|
-
}
|
|
2063
|
-
|
|
2064
|
-
/** Build a summarize callback with runtime provider fallback handling. */
|
|
2065
|
-
private async resolveSummarize(params: {
|
|
2066
|
-
legacyParams?: Record<string, unknown>;
|
|
2067
|
-
customInstructions?: string;
|
|
2068
|
-
breakerScope: string;
|
|
2069
|
-
}): Promise<{
|
|
2070
|
-
summarize: (text: string, aggressive?: boolean) => Promise<string>;
|
|
2071
|
-
summaryModel: string;
|
|
2072
|
-
breakerKey?: string;
|
|
2073
|
-
}> {
|
|
2074
|
-
const lp = params.legacyParams ?? {};
|
|
2075
|
-
if (typeof lp.summarize === "function") {
|
|
2076
|
-
return {
|
|
2077
|
-
summarize: lp.summarize as (text: string, aggressive?: boolean) => Promise<string>,
|
|
2078
|
-
summaryModel: "unknown",
|
|
2079
|
-
breakerKey: `custom:${params.breakerScope}`,
|
|
2080
|
-
};
|
|
2081
|
-
}
|
|
2082
|
-
try {
|
|
2083
|
-
const customInstructions =
|
|
2084
|
-
params.customInstructions !== undefined
|
|
2085
|
-
? params.customInstructions
|
|
2086
|
-
: (this.config.customInstructions || undefined);
|
|
2087
|
-
const runtimeSummarizer = await createLcmSummarizeFromLegacyParams({
|
|
2088
|
-
deps: this.deps,
|
|
2089
|
-
legacyParams: lp,
|
|
2090
|
-
customInstructions,
|
|
2091
|
-
});
|
|
2092
|
-
if (runtimeSummarizer) {
|
|
2093
|
-
return {
|
|
2094
|
-
summarize: runtimeSummarizer.fn,
|
|
2095
|
-
summaryModel: runtimeSummarizer.model,
|
|
2096
|
-
breakerKey: runtimeSummarizer.breakerKey,
|
|
2097
|
-
};
|
|
2098
|
-
}
|
|
2099
|
-
this.deps.log.error(`[lcm] resolveSummarize: createLcmSummarizeFromLegacyParams returned undefined`);
|
|
2100
|
-
} catch (err) {
|
|
2101
|
-
this.deps.log.error(
|
|
2102
|
-
`[lcm] resolveSummarize failed, using emergency fallback: ${describeLogError(err)}`,
|
|
2103
|
-
);
|
|
2104
|
-
}
|
|
2105
|
-
this.deps.log.error(`[lcm] resolveSummarize: FALLING BACK TO EMERGENCY TRUNCATION`);
|
|
2106
|
-
return { summarize: createEmergencyFallbackSummarize(), summaryModel: "unknown" };
|
|
2107
|
-
}
|
|
2108
|
-
|
|
2109
|
-
/**
|
|
2110
|
-
* Resolve an optional model-backed summarizer for large text file exploration.
|
|
2111
|
-
*
|
|
2112
|
-
* This is opt-in via env so ingest remains deterministic and lightweight when
|
|
2113
|
-
* no summarization model is configured.
|
|
2114
|
-
*/
|
|
2115
|
-
private async resolveLargeFileTextSummarizer(): Promise<
|
|
2116
|
-
((prompt: string) => Promise<string | null>) | undefined
|
|
2117
|
-
> {
|
|
2118
|
-
if (this.largeFileTextSummarizerResolved) {
|
|
2119
|
-
return this.largeFileTextSummarizer;
|
|
2120
|
-
}
|
|
2121
|
-
this.largeFileTextSummarizerResolved = true;
|
|
2122
|
-
|
|
2123
|
-
const provider = this.deps.config.largeFileSummaryProvider;
|
|
2124
|
-
const model = this.deps.config.largeFileSummaryModel;
|
|
2125
|
-
if (!provider || !model) {
|
|
2126
|
-
return undefined;
|
|
2127
|
-
}
|
|
2128
|
-
|
|
2129
|
-
try {
|
|
2130
|
-
const result = await createLcmSummarizeFromLegacyParams({
|
|
2131
|
-
deps: this.deps,
|
|
2132
|
-
legacyParams: { provider, model },
|
|
2133
|
-
customInstructions: this.config.customInstructions || undefined,
|
|
2134
|
-
});
|
|
2135
|
-
if (!result) {
|
|
2136
|
-
return undefined;
|
|
2137
|
-
}
|
|
2138
|
-
|
|
2139
|
-
this.largeFileTextSummarizer = async (prompt: string): Promise<string | null> => {
|
|
2140
|
-
let summary: string;
|
|
2141
|
-
try {
|
|
2142
|
-
summary = await result.fn(prompt, false);
|
|
2143
|
-
} catch (err) {
|
|
2144
|
-
if (err instanceof LcmProviderAuthError) {
|
|
2145
|
-
return null;
|
|
2146
|
-
}
|
|
2147
|
-
throw err;
|
|
2148
|
-
}
|
|
2149
|
-
if (typeof summary !== "string") {
|
|
2150
|
-
return null;
|
|
2151
|
-
}
|
|
2152
|
-
const trimmed = summary.trim();
|
|
2153
|
-
return trimmed.length > 0 ? trimmed : null;
|
|
2154
|
-
};
|
|
2155
|
-
return this.largeFileTextSummarizer;
|
|
2156
|
-
} catch {
|
|
2157
|
-
return undefined;
|
|
2158
|
-
}
|
|
2159
|
-
}
|
|
2160
|
-
|
|
2161
|
-
/** Persist intercepted large-file text payloads to ~/.openclaw/lcm-files. */
|
|
2162
|
-
private async storeLargeFileContent(params: {
|
|
2163
|
-
conversationId: number;
|
|
2164
|
-
fileId: string;
|
|
2165
|
-
extension: string;
|
|
2166
|
-
content: string;
|
|
2167
|
-
}): Promise<string> {
|
|
2168
|
-
const dir = join(homedir(), ".openclaw", "lcm-files", String(params.conversationId));
|
|
2169
|
-
await mkdir(dir, { recursive: true });
|
|
2170
|
-
|
|
2171
|
-
const normalizedExtension = params.extension.replace(/[^a-z0-9]/gi, "").toLowerCase() || "txt";
|
|
2172
|
-
const filePath = join(dir, `${params.fileId}.${normalizedExtension}`);
|
|
2173
|
-
await writeFile(filePath, params.content, "utf8");
|
|
2174
|
-
return filePath;
|
|
2175
|
-
}
|
|
2176
|
-
|
|
2177
|
-
/** Persist a large text payload and return the resulting compact placeholder. */
|
|
2178
|
-
private async externalizeLargeTextPayload(params: {
|
|
2179
|
-
conversationId: number;
|
|
2180
|
-
content: string;
|
|
2181
|
-
fileName?: string;
|
|
2182
|
-
mimeType?: string;
|
|
2183
|
-
formatReference: (input: { fileId: string; byteSize: number; summary: string }) => string;
|
|
2184
|
-
}): Promise<{ fileId: string; byteSize: number; summary: string; reference: string }> {
|
|
2185
|
-
const summarizeText = await this.resolveLargeFileTextSummarizer();
|
|
2186
|
-
const fileId = `file_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
|
|
2187
|
-
const extension = extensionFromNameOrMime(params.fileName, params.mimeType);
|
|
2188
|
-
const storageUri = await this.storeLargeFileContent({
|
|
2189
|
-
conversationId: params.conversationId,
|
|
2190
|
-
fileId,
|
|
2191
|
-
extension,
|
|
2192
|
-
content: params.content,
|
|
2193
|
-
});
|
|
2194
|
-
const byteSize = Buffer.byteLength(params.content, "utf8");
|
|
2195
|
-
const explorationSummary = await generateExplorationSummary({
|
|
2196
|
-
content: params.content,
|
|
2197
|
-
fileName: params.fileName,
|
|
2198
|
-
mimeType: params.mimeType,
|
|
2199
|
-
summarizeText,
|
|
2200
|
-
});
|
|
2201
|
-
|
|
2202
|
-
await this.summaryStore.insertLargeFile({
|
|
2203
|
-
fileId,
|
|
2204
|
-
conversationId: params.conversationId,
|
|
2205
|
-
fileName: params.fileName,
|
|
2206
|
-
mimeType: params.mimeType,
|
|
2207
|
-
byteSize,
|
|
2208
|
-
storageUri,
|
|
2209
|
-
explorationSummary,
|
|
2210
|
-
});
|
|
2211
|
-
|
|
2212
|
-
return {
|
|
2213
|
-
fileId,
|
|
2214
|
-
byteSize,
|
|
2215
|
-
summary: explorationSummary,
|
|
2216
|
-
reference: params.formatReference({
|
|
2217
|
-
fileId,
|
|
2218
|
-
byteSize,
|
|
2219
|
-
summary: explorationSummary,
|
|
2220
|
-
}),
|
|
2221
|
-
};
|
|
2222
|
-
}
|
|
2223
|
-
|
|
2224
|
-
/**
|
|
2225
|
-
* Intercept oversized <file> blocks before persistence and replace them with
|
|
2226
|
-
* compact file references backed by large_files records.
|
|
2227
|
-
*/
|
|
2228
|
-
private async interceptLargeFiles(params: {
|
|
2229
|
-
conversationId: number;
|
|
2230
|
-
content: string;
|
|
2231
|
-
}): Promise<{ rewrittenContent: string; fileIds: string[] } | null> {
|
|
2232
|
-
const blocks = parseFileBlocks(params.content);
|
|
2233
|
-
if (blocks.length === 0) {
|
|
2234
|
-
return null;
|
|
2235
|
-
}
|
|
2236
|
-
|
|
2237
|
-
const threshold = Math.max(1, this.config.largeFileTokenThreshold);
|
|
2238
|
-
const fileIds: string[] = [];
|
|
2239
|
-
const rewrittenSegments: string[] = [];
|
|
2240
|
-
let cursor = 0;
|
|
2241
|
-
let interceptedAny = false;
|
|
2242
|
-
|
|
2243
|
-
for (const block of blocks) {
|
|
2244
|
-
const blockTokens = estimateTokens(block.text);
|
|
2245
|
-
if (blockTokens < threshold) {
|
|
2246
|
-
continue;
|
|
2247
|
-
}
|
|
2248
|
-
|
|
2249
|
-
interceptedAny = true;
|
|
2250
|
-
const externalized = await this.externalizeLargeTextPayload({
|
|
2251
|
-
conversationId: params.conversationId,
|
|
2252
|
-
content: block.text,
|
|
2253
|
-
fileName: block.fileName,
|
|
2254
|
-
mimeType: block.mimeType,
|
|
2255
|
-
formatReference: ({ fileId, byteSize, summary }) =>
|
|
2256
|
-
formatFileReference({
|
|
2257
|
-
fileId,
|
|
2258
|
-
fileName: block.fileName,
|
|
2259
|
-
mimeType: block.mimeType,
|
|
2260
|
-
byteSize,
|
|
2261
|
-
summary,
|
|
2262
|
-
}),
|
|
2263
|
-
});
|
|
2264
|
-
|
|
2265
|
-
rewrittenSegments.push(params.content.slice(cursor, block.start));
|
|
2266
|
-
rewrittenSegments.push(externalized.reference);
|
|
2267
|
-
cursor = block.end;
|
|
2268
|
-
fileIds.push(externalized.fileId);
|
|
2269
|
-
}
|
|
2270
|
-
|
|
2271
|
-
if (!interceptedAny) {
|
|
2272
|
-
return null;
|
|
2273
|
-
}
|
|
2274
|
-
|
|
2275
|
-
rewrittenSegments.push(params.content.slice(cursor));
|
|
2276
|
-
return {
|
|
2277
|
-
rewrittenContent: rewrittenSegments.join(""),
|
|
2278
|
-
fileIds,
|
|
2279
|
-
};
|
|
2280
|
-
}
|
|
2281
|
-
|
|
2282
|
-
/** Externalize oversized textual tool outputs before they are persisted inline. */
|
|
2283
|
-
private async interceptLargeToolResults(params: {
|
|
2284
|
-
conversationId: number;
|
|
2285
|
-
message: AgentMessage;
|
|
2286
|
-
}): Promise<{ rewrittenMessage: AgentMessage; fileIds: string[] } | null> {
|
|
2287
|
-
if (
|
|
2288
|
-
(params.message.role !== "toolResult" && params.message.role !== "tool") ||
|
|
2289
|
-
!("content" in params.message)
|
|
2290
|
-
) {
|
|
2291
|
-
return null;
|
|
2292
|
-
}
|
|
2293
|
-
if (!Array.isArray(params.message.content)) {
|
|
2294
|
-
return null;
|
|
2295
|
-
}
|
|
2296
|
-
|
|
2297
|
-
const threshold = Math.max(1, this.config.largeFileTokenThreshold);
|
|
2298
|
-
const rewrittenContent: unknown[] = [];
|
|
2299
|
-
const fileIds: string[] = [];
|
|
2300
|
-
let interceptedAny = false;
|
|
2301
|
-
const topLevel = params.message as Record<string, unknown>;
|
|
2302
|
-
const topLevelToolCallId =
|
|
2303
|
-
safeString(topLevel.toolCallId) ??
|
|
2304
|
-
safeString(topLevel.tool_call_id) ??
|
|
2305
|
-
safeString(topLevel.toolUseId) ??
|
|
2306
|
-
safeString(topLevel.tool_use_id) ??
|
|
2307
|
-
safeString(topLevel.call_id) ??
|
|
2308
|
-
safeString(topLevel.id);
|
|
2309
|
-
const topLevelToolName =
|
|
2310
|
-
safeString(topLevel.toolName) ??
|
|
2311
|
-
safeString(topLevel.tool_name);
|
|
2312
|
-
const topLevelIsError =
|
|
2313
|
-
safeBoolean(topLevel.isError) ??
|
|
2314
|
-
safeBoolean(topLevel.is_error);
|
|
2315
|
-
|
|
2316
|
-
for (const item of params.message.content) {
|
|
2317
|
-
if (!item || typeof item !== "object" || Array.isArray(item)) {
|
|
2318
|
-
rewrittenContent.push(item);
|
|
2319
|
-
continue;
|
|
2320
|
-
}
|
|
2321
|
-
|
|
2322
|
-
const record = item as Record<string, unknown>;
|
|
2323
|
-
const rawType = safeString(record.type);
|
|
2324
|
-
const isStructuredToolResult =
|
|
2325
|
-
rawType !== "tool_result" &&
|
|
2326
|
-
rawType !== "toolResult" &&
|
|
2327
|
-
rawType !== "function_call_output";
|
|
2328
|
-
const isPlainTextToolResult =
|
|
2329
|
-
rawType === "text" &&
|
|
2330
|
-
typeof record.text === "string";
|
|
2331
|
-
if (isStructuredToolResult && !isPlainTextToolResult) {
|
|
2332
|
-
rewrittenContent.push(item);
|
|
2333
|
-
continue;
|
|
2334
|
-
}
|
|
2335
|
-
|
|
2336
|
-
const textSource =
|
|
2337
|
-
isPlainTextToolResult
|
|
2338
|
-
? record.text
|
|
2339
|
-
: record.output !== undefined
|
|
2340
|
-
? record.output
|
|
2341
|
-
: record.content !== undefined
|
|
2342
|
-
? record.content
|
|
2343
|
-
: record;
|
|
2344
|
-
const extractedText = extractStructuredText(textSource);
|
|
2345
|
-
if (typeof extractedText !== "string" || estimateTokens(extractedText) < threshold) {
|
|
2346
|
-
rewrittenContent.push(item);
|
|
2347
|
-
continue;
|
|
2348
|
-
}
|
|
2349
|
-
|
|
2350
|
-
interceptedAny = true;
|
|
2351
|
-
const toolName =
|
|
2352
|
-
safeString(record.name) ??
|
|
2353
|
-
topLevelToolName ??
|
|
2354
|
-
"tool-result";
|
|
2355
|
-
const externalized = await this.externalizeLargeTextPayload({
|
|
2356
|
-
conversationId: params.conversationId,
|
|
2357
|
-
content: extractedText,
|
|
2358
|
-
fileName: `${toolName}.txt`,
|
|
2359
|
-
mimeType: "text/plain",
|
|
2360
|
-
formatReference: ({ fileId, byteSize, summary }) =>
|
|
2361
|
-
formatToolOutputReference({
|
|
2362
|
-
fileId,
|
|
2363
|
-
toolName,
|
|
2364
|
-
byteSize,
|
|
2365
|
-
summary,
|
|
2366
|
-
}),
|
|
2367
|
-
});
|
|
2368
|
-
|
|
2369
|
-
const normalizedRawType =
|
|
2370
|
-
rawType === "function_call_output" ? "function_call_output" : "tool_result";
|
|
2371
|
-
const compactBlock: Record<string, unknown> = isPlainTextToolResult
|
|
2372
|
-
? {
|
|
2373
|
-
type: "text",
|
|
2374
|
-
text: externalized.reference,
|
|
2375
|
-
rawType: normalizedRawType,
|
|
2376
|
-
externalizedFileId: externalized.fileId,
|
|
2377
|
-
originalByteSize: externalized.byteSize,
|
|
2378
|
-
toolOutputExternalized: true,
|
|
2379
|
-
externalizationReason: "large_tool_result",
|
|
2380
|
-
}
|
|
2381
|
-
: {
|
|
2382
|
-
type: normalizedRawType,
|
|
2383
|
-
output: externalized.reference,
|
|
2384
|
-
externalizedFileId: externalized.fileId,
|
|
2385
|
-
originalByteSize: externalized.byteSize,
|
|
2386
|
-
toolOutputExternalized: true,
|
|
2387
|
-
externalizationReason: "large_tool_result",
|
|
2388
|
-
};
|
|
2389
|
-
const callId =
|
|
2390
|
-
safeString(record.tool_use_id) ??
|
|
2391
|
-
safeString(record.toolUseId) ??
|
|
2392
|
-
safeString(record.tool_call_id) ??
|
|
2393
|
-
safeString(record.toolCallId) ??
|
|
2394
|
-
safeString(record.call_id) ??
|
|
2395
|
-
safeString(record.id) ??
|
|
2396
|
-
topLevelToolCallId;
|
|
2397
|
-
if (callId) {
|
|
2398
|
-
if (normalizedRawType === "function_call_output") {
|
|
2399
|
-
compactBlock.call_id = callId;
|
|
2400
|
-
} else {
|
|
2401
|
-
compactBlock.tool_use_id = callId;
|
|
2402
|
-
}
|
|
2403
|
-
}
|
|
2404
|
-
if (typeof record.is_error === "boolean") {
|
|
2405
|
-
compactBlock.is_error = record.is_error;
|
|
2406
|
-
} else if (typeof record.isError === "boolean") {
|
|
2407
|
-
compactBlock.isError = record.isError;
|
|
2408
|
-
} else if (typeof topLevelIsError === "boolean") {
|
|
2409
|
-
compactBlock.isError = topLevelIsError;
|
|
2410
|
-
}
|
|
2411
|
-
if (toolName) {
|
|
2412
|
-
compactBlock.name = toolName;
|
|
2413
|
-
}
|
|
2414
|
-
|
|
2415
|
-
rewrittenContent.push(compactBlock);
|
|
2416
|
-
fileIds.push(externalized.fileId);
|
|
2417
|
-
}
|
|
2418
|
-
|
|
2419
|
-
if (!interceptedAny) {
|
|
2420
|
-
return null;
|
|
2421
|
-
}
|
|
2422
|
-
|
|
2423
|
-
return {
|
|
2424
|
-
rewrittenMessage: {
|
|
2425
|
-
...params.message,
|
|
2426
|
-
content: rewrittenContent,
|
|
2427
|
-
} as AgentMessage,
|
|
2428
|
-
fileIds,
|
|
2429
|
-
};
|
|
2430
|
-
}
|
|
2431
|
-
|
|
2432
|
-
// ── ContextEngine interface ─────────────────────────────────────────────
|
|
2433
|
-
|
|
2434
|
-
/**
|
|
2435
|
-
* Reconcile session-file history with persisted messages and append only the
|
|
2436
|
-
* tail that is present in JSONL but missing from LCM.
|
|
2437
|
-
*/
|
|
2438
|
-
private async reconcileSessionTail(params: {
|
|
2439
|
-
sessionId: string;
|
|
2440
|
-
sessionKey?: string;
|
|
2441
|
-
conversationId: number;
|
|
2442
|
-
historicalMessages: AgentMessage[];
|
|
2443
|
-
}): Promise<{
|
|
2444
|
-
blockedByImportCap: boolean;
|
|
2445
|
-
importedMessages: number;
|
|
2446
|
-
hasOverlap: boolean;
|
|
2447
|
-
}> {
|
|
2448
|
-
const { sessionId, conversationId, historicalMessages } = params;
|
|
2449
|
-
const startedAt = Date.now();
|
|
2450
|
-
const sessionContext = this.formatSessionLogContext({
|
|
2451
|
-
conversationId,
|
|
2452
|
-
sessionId,
|
|
2453
|
-
sessionKey: params.sessionKey,
|
|
2454
|
-
});
|
|
2455
|
-
if (historicalMessages.length === 0) {
|
|
2456
|
-
this.deps.log.info(
|
|
2457
|
-
`[lcm] reconcileSessionTail: skipped for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=0 reason=empty-history`,
|
|
2458
|
-
);
|
|
2459
|
-
return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
|
|
2460
|
-
}
|
|
2461
|
-
|
|
2462
|
-
const latestDbMessage = await this.conversationStore.getLastMessage(conversationId);
|
|
2463
|
-
if (!latestDbMessage) {
|
|
2464
|
-
this.deps.log.info(
|
|
2465
|
-
`[lcm] reconcileSessionTail: skipped for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} reason=no-db-tail`,
|
|
2466
|
-
);
|
|
2467
|
-
return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
|
|
2468
|
-
}
|
|
2469
|
-
|
|
2470
|
-
const storedHistoricalMessages = historicalMessages.map((message) => toStoredMessage(message));
|
|
2471
|
-
|
|
2472
|
-
// Fast path: one tail comparison for the common in-sync case.
|
|
2473
|
-
const latestHistorical = storedHistoricalMessages[storedHistoricalMessages.length - 1];
|
|
2474
|
-
const latestIdentity = messageIdentity(latestDbMessage.role, latestDbMessage.content);
|
|
2475
|
-
if (latestIdentity === messageIdentity(latestHistorical.role, latestHistorical.content)) {
|
|
2476
|
-
const dbOccurrences = await this.conversationStore.countMessagesByIdentity(
|
|
2477
|
-
conversationId,
|
|
2478
|
-
latestDbMessage.role,
|
|
2479
|
-
latestDbMessage.content,
|
|
2480
|
-
);
|
|
2481
|
-
let historicalOccurrences = 0;
|
|
2482
|
-
for (const stored of storedHistoricalMessages) {
|
|
2483
|
-
if (messageIdentity(stored.role, stored.content) === latestIdentity) {
|
|
2484
|
-
historicalOccurrences += 1;
|
|
2485
|
-
}
|
|
2486
|
-
}
|
|
2487
|
-
if (dbOccurrences === historicalOccurrences) {
|
|
2488
|
-
this.deps.log.info(
|
|
2489
|
-
`[lcm] reconcileSessionTail: fast path for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=true`,
|
|
2490
|
-
);
|
|
2491
|
-
return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
|
|
2492
|
-
}
|
|
2493
|
-
}
|
|
2494
|
-
|
|
2495
|
-
// Slow path: walk backward through JSONL to find the most recent anchor
|
|
2496
|
-
// message that already exists in LCM, then append everything after it.
|
|
2497
|
-
let anchorIndex = -1;
|
|
2498
|
-
const historicalIdentityTotals = new Map<string, number>();
|
|
2499
|
-
for (const stored of storedHistoricalMessages) {
|
|
2500
|
-
const identity = messageIdentity(stored.role, stored.content);
|
|
2501
|
-
historicalIdentityTotals.set(identity, (historicalIdentityTotals.get(identity) ?? 0) + 1);
|
|
2502
|
-
}
|
|
2503
|
-
|
|
2504
|
-
const historicalIdentityCountsAfterIndex = new Map<string, number>();
|
|
2505
|
-
const dbIdentityCounts = new Map<string, number>();
|
|
2506
|
-
for (let index = storedHistoricalMessages.length - 1; index >= 0; index--) {
|
|
2507
|
-
const stored = storedHistoricalMessages[index];
|
|
2508
|
-
const identity = messageIdentity(stored.role, stored.content);
|
|
2509
|
-
const seenAfter = historicalIdentityCountsAfterIndex.get(identity) ?? 0;
|
|
2510
|
-
const total = historicalIdentityTotals.get(identity) ?? 0;
|
|
2511
|
-
const occurrencesThroughIndex = total - seenAfter;
|
|
2512
|
-
const exists = await this.conversationStore.hasMessage(
|
|
2513
|
-
conversationId,
|
|
2514
|
-
stored.role,
|
|
2515
|
-
stored.content,
|
|
2516
|
-
);
|
|
2517
|
-
historicalIdentityCountsAfterIndex.set(identity, seenAfter + 1);
|
|
2518
|
-
if (!exists) {
|
|
2519
|
-
continue;
|
|
2520
|
-
}
|
|
2521
|
-
|
|
2522
|
-
let dbCountForIdentity = dbIdentityCounts.get(identity);
|
|
2523
|
-
if (dbCountForIdentity === undefined) {
|
|
2524
|
-
dbCountForIdentity = await this.conversationStore.countMessagesByIdentity(
|
|
2525
|
-
conversationId,
|
|
2526
|
-
stored.role,
|
|
2527
|
-
stored.content,
|
|
2528
|
-
);
|
|
2529
|
-
dbIdentityCounts.set(identity, dbCountForIdentity);
|
|
2530
|
-
}
|
|
2531
|
-
|
|
2532
|
-
// Match the same occurrence index as the DB tail so repeated empty
|
|
2533
|
-
// tool messages do not anchor against a later, still-missing entry.
|
|
2534
|
-
if (dbCountForIdentity !== occurrencesThroughIndex) {
|
|
2535
|
-
continue;
|
|
2536
|
-
}
|
|
2537
|
-
|
|
2538
|
-
anchorIndex = index;
|
|
2539
|
-
break;
|
|
2540
|
-
}
|
|
2541
|
-
|
|
2542
|
-
if (anchorIndex < 0) {
|
|
2543
|
-
this.deps.log.info(
|
|
2544
|
-
`[lcm] reconcileSessionTail: no anchor for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=false`,
|
|
2545
|
-
);
|
|
2546
|
-
return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
|
|
2547
|
-
}
|
|
2548
|
-
if (anchorIndex >= historicalMessages.length - 1) {
|
|
2549
|
-
this.deps.log.info(
|
|
2550
|
-
`[lcm] reconcileSessionTail: anchor at tip for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=true`,
|
|
2551
|
-
);
|
|
2552
|
-
return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
|
|
2553
|
-
}
|
|
2554
|
-
|
|
2555
|
-
const missingTail = historicalMessages.slice(anchorIndex + 1);
|
|
2556
|
-
|
|
2557
|
-
const existingDbCount = await this.conversationStore.getMessageCount(conversationId);
|
|
2558
|
-
if (existingDbCount > 0 && missingTail.length > Math.max(existingDbCount * 0.2, 50)) {
|
|
2559
|
-
this.deps.log.warn(
|
|
2560
|
-
`[lcm] reconcileSessionTail: import cap exceeded for ${sessionContext} — would import ${missingTail.length} messages (existing: ${existingDbCount}). Aborting to prevent flood.`,
|
|
2561
|
-
);
|
|
2562
|
-
this.deps.log.info(
|
|
2563
|
-
`[lcm] reconcileSessionTail: blocked for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} missingTail=${missingTail.length} existingDbCount=${existingDbCount}`,
|
|
2564
|
-
);
|
|
2565
|
-
return { blockedByImportCap: true, importedMessages: 0, hasOverlap: true };
|
|
2566
|
-
}
|
|
2567
|
-
|
|
2568
|
-
let importedMessages = 0;
|
|
2569
|
-
for (const message of missingTail) {
|
|
2570
|
-
const result = await this.ingestSingle({ sessionId, sessionKey: params.sessionKey, message });
|
|
2571
|
-
if (result.ingested) {
|
|
2572
|
-
importedMessages += 1;
|
|
2573
|
-
}
|
|
2574
|
-
}
|
|
2575
|
-
|
|
2576
|
-
this.deps.log.info(
|
|
2577
|
-
`[lcm] reconcileSessionTail: slow path for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} anchorIndex=${anchorIndex} missingTail=${missingTail.length} importedMessages=${importedMessages}`,
|
|
2578
|
-
);
|
|
2579
|
-
return { blockedByImportCap: false, importedMessages, hasOverlap: true };
|
|
2580
|
-
}
|
|
2581
|
-
|
|
2582
|
-
/**
|
|
2583
|
-
* Persist bootstrap checkpoint metadata anchored to the current DB frontier.
|
|
2584
|
-
*
|
|
2585
|
-
* We intentionally checkpoint the session file's current EOF while hashing the
|
|
2586
|
-
* latest persisted DB message. This keeps append-only recovery aligned with the
|
|
2587
|
-
* canonical LCM frontier even when trailing transcript entries are pruned or
|
|
2588
|
-
* otherwise noncanonical.
|
|
2589
|
-
*/
|
|
2590
|
-
private async refreshBootstrapState(params: {
|
|
2591
|
-
conversationId: number;
|
|
2592
|
-
sessionFile: string;
|
|
2593
|
-
fileStats?: { size: number; mtimeMs: number };
|
|
2594
|
-
}): Promise<void> {
|
|
2595
|
-
const latestDbMessage = await this.conversationStore.getLastMessage(params.conversationId);
|
|
2596
|
-
const fileStats = params.fileStats ?? statSync(params.sessionFile);
|
|
2597
|
-
await this.summaryStore.upsertConversationBootstrapState({
|
|
2598
|
-
conversationId: params.conversationId,
|
|
2599
|
-
sessionFilePath: params.sessionFile,
|
|
2600
|
-
lastSeenSize: fileStats.size,
|
|
2601
|
-
lastSeenMtimeMs: Math.trunc(fileStats.mtimeMs),
|
|
2602
|
-
lastProcessedOffset: fileStats.size,
|
|
2603
|
-
lastProcessedEntryHash: latestDbMessage
|
|
2604
|
-
? createBootstrapEntryHash({
|
|
2605
|
-
role: latestDbMessage.role,
|
|
2606
|
-
content: latestDbMessage.content,
|
|
2607
|
-
tokenCount: latestDbMessage.tokenCount,
|
|
2608
|
-
})
|
|
2609
|
-
: null,
|
|
2610
|
-
});
|
|
2611
|
-
}
|
|
2612
|
-
|
|
2613
|
-
async bootstrap(params: {
|
|
2614
|
-
sessionId: string;
|
|
2615
|
-
sessionFile: string;
|
|
2616
|
-
sessionKey?: string;
|
|
2617
|
-
}): Promise<BootstrapResult> {
|
|
2618
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
2619
|
-
return {
|
|
2620
|
-
bootstrapped: false,
|
|
2621
|
-
importedMessages: 0,
|
|
2622
|
-
reason: "session excluded by pattern",
|
|
2623
|
-
};
|
|
2624
|
-
}
|
|
2625
|
-
if (this.isStatelessSession(params.sessionKey)) {
|
|
2626
|
-
return {
|
|
2627
|
-
bootstrapped: false,
|
|
2628
|
-
importedMessages: 0,
|
|
2629
|
-
reason: "stateless session",
|
|
2630
|
-
};
|
|
2631
|
-
}
|
|
2632
|
-
this.ensureMigrated();
|
|
2633
|
-
const startedAt = Date.now();
|
|
2634
|
-
const sessionLabel = [
|
|
2635
|
-
`session=${params.sessionId}`,
|
|
2636
|
-
...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
|
|
2637
|
-
].join(" ");
|
|
2638
|
-
const sessionFileStats = statSync(params.sessionFile);
|
|
2639
|
-
const sessionFileSize = sessionFileStats.size;
|
|
2640
|
-
const sessionFileMtimeMs = Math.trunc(sessionFileStats.mtimeMs);
|
|
2641
|
-
|
|
2642
|
-
const result = await this.withSessionQueue(
|
|
2643
|
-
this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
2644
|
-
async () =>
|
|
2645
|
-
this.conversationStore.withTransaction(async () => {
|
|
2646
|
-
const persistBootstrapState = async (
|
|
2647
|
-
conversationId: number,
|
|
2648
|
-
): Promise<void> => {
|
|
2649
|
-
await this.refreshBootstrapState({
|
|
2650
|
-
conversationId,
|
|
2651
|
-
sessionFile: params.sessionFile,
|
|
2652
|
-
fileStats: {
|
|
2653
|
-
size: sessionFileSize,
|
|
2654
|
-
mtimeMs: sessionFileMtimeMs,
|
|
2655
|
-
},
|
|
2656
|
-
});
|
|
2657
|
-
};
|
|
2658
|
-
|
|
2659
|
-
const conversation = await this.conversationStore.getOrCreateConversation(params.sessionId, {
|
|
2660
|
-
sessionKey: params.sessionKey,
|
|
2661
|
-
});
|
|
2662
|
-
const conversationId = conversation.conversationId;
|
|
2663
|
-
const existingCount = await this.conversationStore.getMessageCount(conversationId);
|
|
2664
|
-
const bootstrapState =
|
|
2665
|
-
existingCount > 0
|
|
2666
|
-
? await this.summaryStore.getConversationBootstrapState(conversationId)
|
|
2667
|
-
: null;
|
|
2668
|
-
|
|
2669
|
-
// If the transcript file is byte-for-byte unchanged from the last
|
|
2670
|
-
// successful bootstrap checkpoint, skip reopening and reparsing it.
|
|
2671
|
-
if (
|
|
2672
|
-
bootstrapState &&
|
|
2673
|
-
bootstrapState.sessionFilePath === params.sessionFile &&
|
|
2674
|
-
bootstrapState.lastSeenSize === sessionFileSize &&
|
|
2675
|
-
bootstrapState.lastSeenMtimeMs === sessionFileMtimeMs
|
|
2676
|
-
) {
|
|
2677
|
-
if (!conversation.bootstrappedAt) {
|
|
2678
|
-
await this.conversationStore.markConversationBootstrapped(conversationId);
|
|
2679
|
-
}
|
|
2680
|
-
this.deps.log.info(
|
|
2681
|
-
`[lcm] bootstrap: checkpoint hit conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
2682
|
-
);
|
|
2683
|
-
return {
|
|
2684
|
-
bootstrapped: false,
|
|
2685
|
-
importedMessages: 0,
|
|
2686
|
-
reason: conversation.bootstrappedAt ? "already bootstrapped" : "conversation already up to date",
|
|
2687
|
-
};
|
|
2688
|
-
}
|
|
2689
|
-
|
|
2690
|
-
if (
|
|
2691
|
-
existingCount > 0 &&
|
|
2692
|
-
bootstrapState &&
|
|
2693
|
-
bootstrapState.sessionFilePath === params.sessionFile &&
|
|
2694
|
-
sessionFileSize > bootstrapState.lastSeenSize &&
|
|
2695
|
-
sessionFileMtimeMs >= bootstrapState.lastSeenMtimeMs
|
|
2696
|
-
) {
|
|
2697
|
-
const latestDbMessage = await this.conversationStore.getLastMessage(conversationId);
|
|
2698
|
-
const latestDbHash = latestDbMessage
|
|
2699
|
-
? createBootstrapEntryHash({
|
|
2700
|
-
role: latestDbMessage.role,
|
|
2701
|
-
content: latestDbMessage.content,
|
|
2702
|
-
tokenCount: latestDbMessage.tokenCount,
|
|
2703
|
-
})
|
|
2704
|
-
: null;
|
|
2705
|
-
const tailEntryRaw = readLastJsonlEntryBeforeOffset(
|
|
2706
|
-
params.sessionFile,
|
|
2707
|
-
bootstrapState.lastProcessedOffset,
|
|
2708
|
-
true,
|
|
2709
|
-
(message) => createBootstrapEntryHash(toStoredMessage(message)) === latestDbHash,
|
|
2710
|
-
);
|
|
2711
|
-
const tailEntryMessage = readBootstrapMessageFromJsonLine(tailEntryRaw);
|
|
2712
|
-
const tailEntryHash = tailEntryMessage
|
|
2713
|
-
? createBootstrapEntryHash(toStoredMessage(tailEntryMessage))
|
|
2714
|
-
: null;
|
|
2715
|
-
|
|
2716
|
-
if (
|
|
2717
|
-
latestDbHash &&
|
|
2718
|
-
latestDbHash === bootstrapState.lastProcessedEntryHash &&
|
|
2719
|
-
tailEntryHash &&
|
|
2720
|
-
tailEntryHash === bootstrapState.lastProcessedEntryHash
|
|
2721
|
-
) {
|
|
2722
|
-
const appended = readAppendedLeafPathMessages({
|
|
2723
|
-
sessionFile: params.sessionFile,
|
|
2724
|
-
offset: bootstrapState.lastProcessedOffset,
|
|
2725
|
-
});
|
|
2726
|
-
if (appended.canUseAppendOnly) {
|
|
2727
|
-
if (!conversation.bootstrappedAt) {
|
|
2728
|
-
await this.conversationStore.markConversationBootstrapped(conversationId);
|
|
2729
|
-
}
|
|
2730
|
-
|
|
2731
|
-
let importedMessages = 0;
|
|
2732
|
-
for (const message of appended.messages) {
|
|
2733
|
-
const ingestResult = await this.ingestSingle({
|
|
2734
|
-
sessionId: params.sessionId,
|
|
2735
|
-
sessionKey: params.sessionKey,
|
|
2736
|
-
message,
|
|
2737
|
-
});
|
|
2738
|
-
if (ingestResult.ingested) {
|
|
2739
|
-
importedMessages += 1;
|
|
2740
|
-
}
|
|
2741
|
-
}
|
|
2742
|
-
|
|
2743
|
-
await persistBootstrapState(conversationId);
|
|
2744
|
-
this.deps.log.info(
|
|
2745
|
-
`[lcm] bootstrap: append-only conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} appendedMessages=${appended.messages.length} importedMessages=${importedMessages} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
2746
|
-
);
|
|
2747
|
-
|
|
2748
|
-
if (importedMessages > 0) {
|
|
2749
|
-
return {
|
|
2750
|
-
bootstrapped: true,
|
|
2751
|
-
importedMessages,
|
|
2752
|
-
reason: "reconciled missing session messages",
|
|
2753
|
-
};
|
|
2754
|
-
}
|
|
2755
|
-
|
|
2756
|
-
return {
|
|
2757
|
-
bootstrapped: false,
|
|
2758
|
-
importedMessages: 0,
|
|
2759
|
-
reason: conversation.bootstrappedAt ? "already bootstrapped" : "conversation already up to date",
|
|
2760
|
-
};
|
|
2761
|
-
}
|
|
2762
|
-
}
|
|
2763
|
-
}
|
|
2764
|
-
|
|
2765
|
-
const historicalMessages = await readLeafPathMessages(params.sessionFile);
|
|
2766
|
-
this.deps.log.info(
|
|
2767
|
-
`[lcm] bootstrap: full transcript read conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} historicalMessages=${historicalMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
2768
|
-
);
|
|
2769
|
-
|
|
2770
|
-
// First-time import path: no LCM rows yet, so seed directly from the
|
|
2771
|
-
// active leaf context snapshot.
|
|
2772
|
-
if (existingCount === 0) {
|
|
2773
|
-
const bootstrapMessages = trimBootstrapMessagesToBudget(
|
|
2774
|
-
historicalMessages,
|
|
2775
|
-
resolveBootstrapMaxTokens(this.config),
|
|
2776
|
-
);
|
|
2777
|
-
|
|
2778
|
-
if (bootstrapMessages.length === 0) {
|
|
2779
|
-
await this.conversationStore.markConversationBootstrapped(conversationId);
|
|
2780
|
-
await persistBootstrapState(conversationId);
|
|
2781
|
-
return {
|
|
2782
|
-
bootstrapped: false,
|
|
2783
|
-
importedMessages: 0,
|
|
2784
|
-
reason: "no leaf-path messages in session",
|
|
2785
|
-
};
|
|
2786
|
-
}
|
|
2787
|
-
|
|
2788
|
-
const nextSeq = (await this.conversationStore.getMaxSeq(conversationId)) + 1;
|
|
2789
|
-
const bulkInput = bootstrapMessages.map((message, index) => {
|
|
2790
|
-
const stored = toStoredMessage(message);
|
|
2791
|
-
return {
|
|
2792
|
-
conversationId,
|
|
2793
|
-
seq: nextSeq + index,
|
|
2794
|
-
role: stored.role,
|
|
2795
|
-
content: stored.content,
|
|
2796
|
-
tokenCount: stored.tokenCount,
|
|
2797
|
-
};
|
|
2798
|
-
});
|
|
2799
|
-
|
|
2800
|
-
const inserted = await this.conversationStore.createMessagesBulk(bulkInput);
|
|
2801
|
-
await this.summaryStore.appendContextMessages(
|
|
2802
|
-
conversationId,
|
|
2803
|
-
inserted.map((record) => record.messageId),
|
|
2804
|
-
);
|
|
2805
|
-
await this.conversationStore.markConversationBootstrapped(conversationId);
|
|
2806
|
-
|
|
2807
|
-
// Prune HEARTBEAT_OK turns from the freshly imported data
|
|
2808
|
-
if (this.config.pruneHeartbeatOk) {
|
|
2809
|
-
const pruned = await this.pruneHeartbeatOkTurns(conversationId);
|
|
2810
|
-
if (pruned > 0) {
|
|
2811
|
-
this.deps.log.info(
|
|
2812
|
-
`[lcm] bootstrap: pruned ${pruned} HEARTBEAT_OK messages from conversation ${conversationId}`,
|
|
2813
|
-
);
|
|
2814
|
-
}
|
|
2815
|
-
}
|
|
2816
|
-
|
|
2817
|
-
await persistBootstrapState(conversationId);
|
|
2818
|
-
this.deps.log.info(
|
|
2819
|
-
`[lcm] bootstrap: initial import conversation=${conversationId} ${sessionLabel} importedMessages=${inserted.length} sourceMessages=${historicalMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
2820
|
-
);
|
|
2821
|
-
|
|
2822
|
-
return {
|
|
2823
|
-
bootstrapped: true,
|
|
2824
|
-
importedMessages: inserted.length,
|
|
2825
|
-
};
|
|
2826
|
-
}
|
|
2827
|
-
|
|
2828
|
-
// Existing conversation path: reconcile crash gaps by appending JSONL
|
|
2829
|
-
// messages that were never persisted to LCM.
|
|
2830
|
-
const reconcile = await this.reconcileSessionTail({
|
|
2831
|
-
sessionId: params.sessionId,
|
|
2832
|
-
sessionKey: params.sessionKey,
|
|
2833
|
-
conversationId,
|
|
2834
|
-
historicalMessages,
|
|
2835
|
-
});
|
|
2836
|
-
this.deps.log.info(
|
|
2837
|
-
`[lcm] bootstrap: reconcile finished conversation=${conversationId} ${sessionLabel} importedMessages=${reconcile.importedMessages} overlap=${reconcile.hasOverlap} blockedByImportCap=${reconcile.blockedByImportCap} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
2838
|
-
);
|
|
2839
|
-
|
|
2840
|
-
if (reconcile.blockedByImportCap) {
|
|
2841
|
-
return {
|
|
2842
|
-
bootstrapped: false,
|
|
2843
|
-
importedMessages: 0,
|
|
2844
|
-
reason: "reconcile import capped",
|
|
2845
|
-
};
|
|
2846
|
-
}
|
|
2847
|
-
|
|
2848
|
-
if (!conversation.bootstrappedAt) {
|
|
2849
|
-
await this.conversationStore.markConversationBootstrapped(conversationId);
|
|
2850
|
-
}
|
|
2851
|
-
|
|
2852
|
-
if (reconcile.importedMessages > 0) {
|
|
2853
|
-
await persistBootstrapState(conversationId);
|
|
2854
|
-
return {
|
|
2855
|
-
bootstrapped: true,
|
|
2856
|
-
importedMessages: reconcile.importedMessages,
|
|
2857
|
-
reason: "reconciled missing session messages",
|
|
2858
|
-
};
|
|
2859
|
-
}
|
|
2860
|
-
|
|
2861
|
-
if (reconcile.hasOverlap) {
|
|
2862
|
-
await persistBootstrapState(conversationId);
|
|
2863
|
-
}
|
|
2864
|
-
|
|
2865
|
-
if (conversation.bootstrappedAt) {
|
|
2866
|
-
return {
|
|
2867
|
-
bootstrapped: false,
|
|
2868
|
-
importedMessages: 0,
|
|
2869
|
-
reason: "already bootstrapped",
|
|
2870
|
-
};
|
|
2871
|
-
}
|
|
2872
|
-
|
|
2873
|
-
return {
|
|
2874
|
-
bootstrapped: false,
|
|
2875
|
-
importedMessages: 0,
|
|
2876
|
-
reason: reconcile.hasOverlap
|
|
2877
|
-
? "conversation already up to date"
|
|
2878
|
-
: "conversation already has messages",
|
|
2879
|
-
};
|
|
2880
|
-
}),
|
|
2881
|
-
{ operationName: "bootstrap", context: sessionLabel },
|
|
2882
|
-
);
|
|
2883
|
-
|
|
2884
|
-
// Post-bootstrap pruning: clean HEARTBEAT_OK turns that were already
|
|
2885
|
-
// in the DB from prior bootstrap cycles (before pruning was enabled).
|
|
2886
|
-
if (this.config.pruneHeartbeatOk && result.bootstrapped === false) {
|
|
2887
|
-
try {
|
|
2888
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
2889
|
-
sessionId: params.sessionId,
|
|
2890
|
-
sessionKey: params.sessionKey,
|
|
2891
|
-
});
|
|
2892
|
-
if (conversation) {
|
|
2893
|
-
const pruned = await this.pruneHeartbeatOkTurns(conversation.conversationId);
|
|
2894
|
-
if (pruned > 0) {
|
|
2895
|
-
await this.refreshBootstrapState({
|
|
2896
|
-
conversationId: conversation.conversationId,
|
|
2897
|
-
sessionFile: params.sessionFile,
|
|
2898
|
-
});
|
|
2899
|
-
this.deps.log.info(
|
|
2900
|
-
`[lcm] bootstrap: retroactively pruned ${pruned} HEARTBEAT_OK messages from conversation ${conversation.conversationId}`,
|
|
2901
|
-
);
|
|
2902
|
-
}
|
|
2903
|
-
}
|
|
2904
|
-
} catch (err) {
|
|
2905
|
-
this.deps.log.warn(
|
|
2906
|
-
`[lcm] bootstrap: heartbeat pruning failed: ${describeLogError(err)}`,
|
|
2907
|
-
);
|
|
2908
|
-
}
|
|
2909
|
-
}
|
|
2910
|
-
|
|
2911
|
-
this.deps.log.info(
|
|
2912
|
-
`[lcm] bootstrap: done ${sessionLabel} bootstrapped=${result.bootstrapped} importedMessages=${result.importedMessages} reason=${result.reason ?? "none"} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
2913
|
-
);
|
|
2914
|
-
return result;
|
|
2915
|
-
}
|
|
2916
|
-
|
|
2917
|
-
/**
|
|
2918
|
-
* Remove messages from the batch that already exist in the DB for this session.
|
|
2919
|
-
* Conservative replay detection: only strip a prefix when the incoming
|
|
2920
|
-
* batch begins with the entire stored transcript for the session.
|
|
2921
|
-
*
|
|
2922
|
-
* Fixes two issues from #246:
|
|
2923
|
-
* 1. Replaced hasMessage() fast-path with aligned-tail check — the old
|
|
2924
|
-
* approach false-positives on legitimate repeated first messages
|
|
2925
|
-
* 2. Dedup now runs on newMessages only, before autoCompactionSummary
|
|
2926
|
-
* is prepended — synthetic summaries can no longer interfere with
|
|
2927
|
-
* replay detection
|
|
2928
|
-
*/
|
|
2929
|
-
private async deduplicateAfterTurnBatch(
|
|
2930
|
-
sessionId: string,
|
|
2931
|
-
sessionKey: string | undefined,
|
|
2932
|
-
batch: AgentMessage[],
|
|
2933
|
-
): Promise<AgentMessage[]> {
|
|
2934
|
-
if (batch.length === 0) return batch;
|
|
2935
|
-
|
|
2936
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
2937
|
-
sessionId,
|
|
2938
|
-
sessionKey,
|
|
2939
|
-
});
|
|
2940
|
-
if (!conversation) return batch;
|
|
2941
|
-
|
|
2942
|
-
const conversationId = conversation.conversationId;
|
|
2943
|
-
const storedMessageCount = await this.conversationStore.getMessageCount(conversationId);
|
|
2944
|
-
if (storedMessageCount === 0 || storedMessageCount > batch.length) {
|
|
2945
|
-
return batch;
|
|
2946
|
-
}
|
|
2947
|
-
|
|
2948
|
-
// Aligned-tail check: DB's last message must match the message at the
|
|
2949
|
-
// exact replay boundary in the incoming batch. This replaces the
|
|
2950
|
-
// hasMessage() check which could false-positive on any repeated content.
|
|
2951
|
-
const lastDbMessage = await this.conversationStore.getLastMessage(conversationId);
|
|
2952
|
-
if (!lastDbMessage) return batch;
|
|
2953
|
-
|
|
2954
|
-
const storedBatch = batch.map((m) => toStoredMessage(m));
|
|
2955
|
-
const batchAtBoundary = storedBatch[storedMessageCount - 1]!;
|
|
2956
|
-
if (
|
|
2957
|
-
messageIdentity(lastDbMessage.role, lastDbMessage.content) !==
|
|
2958
|
-
messageIdentity(batchAtBoundary.role, batchAtBoundary.content)
|
|
2959
|
-
) {
|
|
2960
|
-
return batch;
|
|
2961
|
-
}
|
|
2962
|
-
|
|
2963
|
-
// Full proof: incoming batch must start with the entire stored transcript
|
|
2964
|
-
// in exact order before we trim anything.
|
|
2965
|
-
const storedMessages = await this.conversationStore.getMessages(conversationId, {
|
|
2966
|
-
limit: storedMessageCount,
|
|
2967
|
-
});
|
|
2968
|
-
if (storedMessages.length !== storedMessageCount) {
|
|
2969
|
-
return batch;
|
|
2970
|
-
}
|
|
2971
|
-
for (let i = 0; i < storedMessageCount; i += 1) {
|
|
2972
|
-
const storedConversationMessage = storedMessages[i]!;
|
|
2973
|
-
const incomingMessage = storedBatch[i]!;
|
|
2974
|
-
if (
|
|
2975
|
-
messageIdentity(storedConversationMessage.role, storedConversationMessage.content) !==
|
|
2976
|
-
messageIdentity(incomingMessage.role, incomingMessage.content)
|
|
2977
|
-
) {
|
|
2978
|
-
return batch;
|
|
2979
|
-
}
|
|
2980
|
-
}
|
|
2981
|
-
|
|
2982
|
-
return batch.slice(storedMessageCount);
|
|
2983
|
-
}
|
|
2984
|
-
/**
|
|
2985
|
-
* Rebuild a compact tool-result message from stored message parts.
|
|
2986
|
-
*
|
|
2987
|
-
* The first transcript-GC pass only rewrites tool results that were already
|
|
2988
|
-
* externalized into large_files during ingest, so the stored placeholder is
|
|
2989
|
-
* the canonical replacement content.
|
|
2990
|
-
*/
|
|
2991
|
-
private async buildTranscriptGcReplacementMessage(
|
|
2992
|
-
messageId: number,
|
|
2993
|
-
): Promise<AgentMessage | null> {
|
|
2994
|
-
const message = await this.conversationStore.getMessageById(messageId);
|
|
2995
|
-
if (!message) {
|
|
2996
|
-
return null;
|
|
2997
|
-
}
|
|
2998
|
-
|
|
2999
|
-
const parts = await this.conversationStore.getMessageParts(messageId);
|
|
3000
|
-
const toolCallId = pickToolCallId(parts);
|
|
3001
|
-
if (!toolCallId) {
|
|
3002
|
-
return null;
|
|
3003
|
-
}
|
|
3004
|
-
|
|
3005
|
-
const content = contentFromParts(parts, "toolResult", message.content);
|
|
3006
|
-
const toolName = pickToolName(parts) ?? "unknown";
|
|
3007
|
-
const isError = pickToolIsError(parts);
|
|
3008
|
-
|
|
3009
|
-
return {
|
|
3010
|
-
role: "toolResult",
|
|
3011
|
-
toolCallId,
|
|
3012
|
-
toolName,
|
|
3013
|
-
content,
|
|
3014
|
-
...(isError !== undefined ? { isError } : {}),
|
|
3015
|
-
} as AgentMessage;
|
|
3016
|
-
}
|
|
3017
|
-
|
|
3018
|
-
/**
|
|
3019
|
-
* Run transcript GC for summarized tool-result messages that already have a
|
|
3020
|
-
* large_files-backed placeholder stored in LCM.
|
|
3021
|
-
*/
|
|
3022
|
-
async maintain(params: {
|
|
3023
|
-
sessionId: string;
|
|
3024
|
-
sessionFile: string;
|
|
3025
|
-
sessionKey?: string;
|
|
3026
|
-
runtimeContext?: ContextEngineMaintenanceRuntimeContext;
|
|
3027
|
-
}): Promise<ContextEngineMaintenanceResult> {
|
|
3028
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
3029
|
-
return {
|
|
3030
|
-
changed: false,
|
|
3031
|
-
bytesFreed: 0,
|
|
3032
|
-
rewrittenEntries: 0,
|
|
3033
|
-
reason: "session excluded by pattern",
|
|
3034
|
-
};
|
|
3035
|
-
}
|
|
3036
|
-
if (this.isStatelessSession(params.sessionKey)) {
|
|
3037
|
-
return {
|
|
3038
|
-
changed: false,
|
|
3039
|
-
bytesFreed: 0,
|
|
3040
|
-
rewrittenEntries: 0,
|
|
3041
|
-
reason: "stateless session",
|
|
3042
|
-
};
|
|
3043
|
-
}
|
|
3044
|
-
if (typeof params.runtimeContext?.rewriteTranscriptEntries !== "function") {
|
|
3045
|
-
return {
|
|
3046
|
-
changed: false,
|
|
3047
|
-
bytesFreed: 0,
|
|
3048
|
-
rewrittenEntries: 0,
|
|
3049
|
-
reason: "runtime rewrite helper unavailable",
|
|
3050
|
-
};
|
|
3051
|
-
}
|
|
3052
|
-
|
|
3053
|
-
const rewriteTranscriptEntries = params.runtimeContext.rewriteTranscriptEntries;
|
|
3054
|
-
const startedAt = Date.now();
|
|
3055
|
-
const sessionLabel = [
|
|
3056
|
-
`session=${params.sessionId}`,
|
|
3057
|
-
...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
|
|
3058
|
-
].join(" ");
|
|
3059
|
-
return this.withSessionQueue(
|
|
3060
|
-
this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
3061
|
-
async () => {
|
|
3062
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
3063
|
-
sessionId: params.sessionId,
|
|
3064
|
-
sessionKey: params.sessionKey,
|
|
3065
|
-
});
|
|
3066
|
-
if (!conversation) {
|
|
3067
|
-
return {
|
|
3068
|
-
changed: false,
|
|
3069
|
-
bytesFreed: 0,
|
|
3070
|
-
rewrittenEntries: 0,
|
|
3071
|
-
reason: "conversation not found",
|
|
3072
|
-
};
|
|
3073
|
-
}
|
|
3074
|
-
|
|
3075
|
-
const candidates = await this.summaryStore.listTranscriptGcCandidates(
|
|
3076
|
-
conversation.conversationId,
|
|
3077
|
-
{ limit: TRANSCRIPT_GC_BATCH_SIZE },
|
|
3078
|
-
);
|
|
3079
|
-
if (candidates.length === 0) {
|
|
3080
|
-
this.deps.log.info(
|
|
3081
|
-
`[lcm] maintain: no transcript GC candidates conversation=${conversation.conversationId} ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3082
|
-
);
|
|
3083
|
-
return {
|
|
3084
|
-
changed: false,
|
|
3085
|
-
bytesFreed: 0,
|
|
3086
|
-
rewrittenEntries: 0,
|
|
3087
|
-
reason: "no transcript GC candidates",
|
|
3088
|
-
};
|
|
3089
|
-
}
|
|
3090
|
-
|
|
3091
|
-
const transcriptEntryIdsByCallId = listTranscriptToolResultEntryIdsByCallId(
|
|
3092
|
-
params.sessionFile,
|
|
3093
|
-
);
|
|
3094
|
-
const replacements: TranscriptRewriteReplacement[] = [];
|
|
3095
|
-
const seenEntryIds = new Set<string>();
|
|
3096
|
-
|
|
3097
|
-
for (const candidate of candidates) {
|
|
3098
|
-
const entryId = transcriptEntryIdsByCallId.get(candidate.toolCallId);
|
|
3099
|
-
if (!entryId || seenEntryIds.has(entryId)) {
|
|
3100
|
-
continue;
|
|
3101
|
-
}
|
|
3102
|
-
|
|
3103
|
-
const replacementMessage = await this.buildTranscriptGcReplacementMessage(
|
|
3104
|
-
candidate.messageId,
|
|
3105
|
-
);
|
|
3106
|
-
if (!replacementMessage) {
|
|
3107
|
-
continue;
|
|
3108
|
-
}
|
|
3109
|
-
|
|
3110
|
-
seenEntryIds.add(entryId);
|
|
3111
|
-
replacements.push({
|
|
3112
|
-
entryId,
|
|
3113
|
-
message: replacementMessage,
|
|
3114
|
-
});
|
|
3115
|
-
}
|
|
3116
|
-
|
|
3117
|
-
if (replacements.length === 0) {
|
|
3118
|
-
this.deps.log.info(
|
|
3119
|
-
`[lcm] maintain: no matching transcript entries conversation=${conversation.conversationId} ${sessionLabel} candidates=${candidates.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3120
|
-
);
|
|
3121
|
-
return {
|
|
3122
|
-
changed: false,
|
|
3123
|
-
bytesFreed: 0,
|
|
3124
|
-
rewrittenEntries: 0,
|
|
3125
|
-
reason: "no matching transcript entries",
|
|
3126
|
-
};
|
|
3127
|
-
}
|
|
3128
|
-
|
|
3129
|
-
const result = await rewriteTranscriptEntries({
|
|
3130
|
-
replacements,
|
|
3131
|
-
});
|
|
3132
|
-
|
|
3133
|
-
if (result.changed) {
|
|
3134
|
-
try {
|
|
3135
|
-
await this.refreshBootstrapState({
|
|
3136
|
-
conversationId: conversation.conversationId,
|
|
3137
|
-
sessionFile: params.sessionFile,
|
|
3138
|
-
});
|
|
3139
|
-
} catch (e) {
|
|
3140
|
-
this.deps.log.warn(
|
|
3141
|
-
`[lcm] Failed to update bootstrap checkpoint after maintain: ${describeLogError(e)}`,
|
|
3142
|
-
);
|
|
3143
|
-
}
|
|
3144
|
-
}
|
|
3145
|
-
|
|
3146
|
-
this.deps.log.info(
|
|
3147
|
-
`[lcm] maintain: done conversation=${conversation.conversationId} ${sessionLabel} candidates=${candidates.length} replacements=${replacements.length} changed=${result.changed} rewrittenEntries=${result.rewrittenEntries} bytesFreed=${result.bytesFreed} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3148
|
-
);
|
|
3149
|
-
return result;
|
|
3150
|
-
},
|
|
3151
|
-
{ operationName: "maintain", context: sessionLabel },
|
|
3152
|
-
);
|
|
3153
|
-
}
|
|
3154
|
-
private async ingestSingle(params: {
|
|
3155
|
-
sessionId: string;
|
|
3156
|
-
sessionKey?: string;
|
|
3157
|
-
message: AgentMessage;
|
|
3158
|
-
isHeartbeat?: boolean;
|
|
3159
|
-
}): Promise<IngestResult> {
|
|
3160
|
-
const { sessionId, sessionKey, message, isHeartbeat } = params;
|
|
3161
|
-
if (isHeartbeat) {
|
|
3162
|
-
return { ingested: false };
|
|
3163
|
-
}
|
|
3164
|
-
|
|
3165
|
-
// Skip assistant messages that failed with an error and have no useful content.
|
|
3166
|
-
// These occur when an API call returns a 500 or similar transient error.
|
|
3167
|
-
// Ingesting them pollutes the LCM database: on retry, the error messages
|
|
3168
|
-
// accumulate and get assembled into context, creating a positive feedback
|
|
3169
|
-
// loop where each retry sends an increasingly large (and malformed) payload
|
|
3170
|
-
// that continues to fail.
|
|
3171
|
-
if (message.role === "assistant") {
|
|
3172
|
-
const topLevel = message as unknown as Record<string, unknown>;
|
|
3173
|
-
const stopReason =
|
|
3174
|
-
typeof topLevel.stopReason === "string"
|
|
3175
|
-
? topLevel.stopReason
|
|
3176
|
-
: typeof topLevel.stop_reason === "string"
|
|
3177
|
-
? topLevel.stop_reason
|
|
3178
|
-
: undefined;
|
|
3179
|
-
if (stopReason === "error" || stopReason === "aborted") {
|
|
3180
|
-
const content = topLevel.content;
|
|
3181
|
-
const isEmpty =
|
|
3182
|
-
content === undefined ||
|
|
3183
|
-
content === null ||
|
|
3184
|
-
content === "" ||
|
|
3185
|
-
(Array.isArray(content) && content.length === 0);
|
|
3186
|
-
if (isEmpty) {
|
|
3187
|
-
return { ingested: false };
|
|
3188
|
-
}
|
|
3189
|
-
}
|
|
3190
|
-
}
|
|
3191
|
-
|
|
3192
|
-
const stored = toStoredMessage(message);
|
|
3193
|
-
|
|
3194
|
-
// Get or create conversation for this session
|
|
3195
|
-
const conversation = await this.conversationStore.getOrCreateConversation(sessionId, {
|
|
3196
|
-
sessionKey,
|
|
3197
|
-
});
|
|
3198
|
-
const conversationId = conversation.conversationId;
|
|
3199
|
-
|
|
3200
|
-
let messageForParts = message;
|
|
3201
|
-
if (stored.role === "user") {
|
|
3202
|
-
const intercepted = await this.interceptLargeFiles({
|
|
3203
|
-
conversationId,
|
|
3204
|
-
content: stored.content,
|
|
3205
|
-
});
|
|
3206
|
-
if (intercepted) {
|
|
3207
|
-
stored.content = intercepted.rewrittenContent;
|
|
3208
|
-
stored.tokenCount = estimateTokens(stored.content);
|
|
3209
|
-
if ("content" in message) {
|
|
3210
|
-
messageForParts = {
|
|
3211
|
-
...message,
|
|
3212
|
-
content: stored.content,
|
|
3213
|
-
} as AgentMessage;
|
|
3214
|
-
}
|
|
3215
|
-
}
|
|
3216
|
-
} else if (stored.role === "tool") {
|
|
3217
|
-
const intercepted = await this.interceptLargeToolResults({
|
|
3218
|
-
conversationId,
|
|
3219
|
-
message,
|
|
3220
|
-
});
|
|
3221
|
-
if (intercepted) {
|
|
3222
|
-
messageForParts = intercepted.rewrittenMessage;
|
|
3223
|
-
const rewrittenStored = toStoredMessage(intercepted.rewrittenMessage);
|
|
3224
|
-
stored.content = rewrittenStored.content;
|
|
3225
|
-
stored.tokenCount = rewrittenStored.tokenCount;
|
|
3226
|
-
}
|
|
3227
|
-
}
|
|
3228
|
-
|
|
3229
|
-
// Determine next sequence number
|
|
3230
|
-
const maxSeq = await this.conversationStore.getMaxSeq(conversationId);
|
|
3231
|
-
const seq = maxSeq + 1;
|
|
3232
|
-
|
|
3233
|
-
// Persist the message
|
|
3234
|
-
const msgRecord = await this.conversationStore.createMessage({
|
|
3235
|
-
conversationId,
|
|
3236
|
-
seq,
|
|
3237
|
-
role: stored.role,
|
|
3238
|
-
content: stored.content,
|
|
3239
|
-
tokenCount: stored.tokenCount,
|
|
3240
|
-
});
|
|
3241
|
-
await this.conversationStore.createMessageParts(
|
|
3242
|
-
msgRecord.messageId,
|
|
3243
|
-
buildMessageParts({
|
|
3244
|
-
sessionId,
|
|
3245
|
-
message: messageForParts,
|
|
3246
|
-
fallbackContent: stored.content,
|
|
3247
|
-
}),
|
|
3248
|
-
);
|
|
3249
|
-
|
|
3250
|
-
// Append to context items so assembler can see it
|
|
3251
|
-
await this.summaryStore.appendContextMessage(conversationId, msgRecord.messageId);
|
|
3252
|
-
|
|
3253
|
-
return { ingested: true };
|
|
3254
|
-
}
|
|
3255
|
-
|
|
3256
|
-
async ingest(params: {
|
|
3257
|
-
sessionId: string;
|
|
3258
|
-
sessionKey?: string;
|
|
3259
|
-
message: AgentMessage;
|
|
3260
|
-
isHeartbeat?: boolean;
|
|
3261
|
-
}): Promise<IngestResult> {
|
|
3262
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
3263
|
-
return { ingested: false };
|
|
3264
|
-
}
|
|
3265
|
-
if (this.isStatelessSession(params.sessionKey)) {
|
|
3266
|
-
return { ingested: false };
|
|
3267
|
-
}
|
|
3268
|
-
this.ensureMigrated();
|
|
3269
|
-
return this.withSessionQueue(
|
|
3270
|
-
this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
3271
|
-
() => this.ingestSingle(params),
|
|
3272
|
-
{
|
|
3273
|
-
operationName: "ingest",
|
|
3274
|
-
context: [
|
|
3275
|
-
`session=${params.sessionId}`,
|
|
3276
|
-
...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
|
|
3277
|
-
].join(" "),
|
|
3278
|
-
},
|
|
3279
|
-
);
|
|
3280
|
-
}
|
|
3281
|
-
|
|
3282
|
-
async ingestBatch(params: {
|
|
3283
|
-
sessionId: string;
|
|
3284
|
-
sessionKey?: string;
|
|
3285
|
-
messages: AgentMessage[];
|
|
3286
|
-
isHeartbeat?: boolean;
|
|
3287
|
-
}): Promise<IngestBatchResult> {
|
|
3288
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
3289
|
-
return { ingestedCount: 0 };
|
|
3290
|
-
}
|
|
3291
|
-
if (this.isStatelessSession(params.sessionKey)) {
|
|
3292
|
-
return { ingestedCount: 0 };
|
|
3293
|
-
}
|
|
3294
|
-
this.ensureMigrated();
|
|
3295
|
-
if (params.messages.length === 0) {
|
|
3296
|
-
return { ingestedCount: 0 };
|
|
3297
|
-
}
|
|
3298
|
-
return this.withSessionQueue(
|
|
3299
|
-
this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
3300
|
-
async () => {
|
|
3301
|
-
let ingestedCount = 0;
|
|
3302
|
-
for (const message of params.messages) {
|
|
3303
|
-
const result = await this.ingestSingle({
|
|
3304
|
-
sessionId: params.sessionId,
|
|
3305
|
-
sessionKey: params.sessionKey,
|
|
3306
|
-
message,
|
|
3307
|
-
isHeartbeat: params.isHeartbeat,
|
|
3308
|
-
});
|
|
3309
|
-
if (result.ingested) {
|
|
3310
|
-
ingestedCount += 1;
|
|
3311
|
-
}
|
|
3312
|
-
}
|
|
3313
|
-
return { ingestedCount };
|
|
3314
|
-
},
|
|
3315
|
-
{
|
|
3316
|
-
operationName: "ingestBatch",
|
|
3317
|
-
context: [
|
|
3318
|
-
`session=${params.sessionId}`,
|
|
3319
|
-
...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
|
|
3320
|
-
`messages=${params.messages.length}`,
|
|
3321
|
-
].join(" "),
|
|
3322
|
-
},
|
|
3323
|
-
);
|
|
3324
|
-
}
|
|
3325
|
-
|
|
3326
|
-
async afterTurn(params: {
|
|
3327
|
-
sessionId: string;
|
|
3328
|
-
sessionKey?: string;
|
|
3329
|
-
sessionFile: string;
|
|
3330
|
-
messages: AgentMessage[];
|
|
3331
|
-
prePromptMessageCount: number;
|
|
3332
|
-
autoCompactionSummary?: string;
|
|
3333
|
-
isHeartbeat?: boolean;
|
|
3334
|
-
tokenBudget?: number;
|
|
3335
|
-
/** OpenClaw runtime param name (preferred). */
|
|
3336
|
-
runtimeContext?: Record<string, unknown>;
|
|
3337
|
-
/** Back-compat param name. */
|
|
3338
|
-
legacyCompactionParams?: Record<string, unknown>;
|
|
3339
|
-
}): Promise<void> {
|
|
3340
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
3341
|
-
return;
|
|
3342
|
-
}
|
|
3343
|
-
if (this.isStatelessSession(params.sessionKey)) {
|
|
3344
|
-
return;
|
|
3345
|
-
}
|
|
3346
|
-
this.ensureMigrated();
|
|
3347
|
-
const startedAt = Date.now();
|
|
3348
|
-
const sessionLabel = [
|
|
3349
|
-
`session=${params.sessionId}`,
|
|
3350
|
-
...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
|
|
3351
|
-
].join(" ");
|
|
3352
|
-
|
|
3353
|
-
// Dedup guard: prevent duplicate ingestion when gateway restart replays
|
|
3354
|
-
// full history. Run on newMessages BEFORE prepending autoCompactionSummary
|
|
3355
|
-
// so synthetic summaries cannot interfere with replay detection.
|
|
3356
|
-
const newMessages = params.messages.slice(params.prePromptMessageCount);
|
|
3357
|
-
const dedupedNewMessages = await this.deduplicateAfterTurnBatch(
|
|
3358
|
-
params.sessionId,
|
|
3359
|
-
params.sessionKey,
|
|
3360
|
-
newMessages,
|
|
3361
|
-
);
|
|
3362
|
-
|
|
3363
|
-
const ingestBatch: AgentMessage[] = [];
|
|
3364
|
-
if (params.autoCompactionSummary) {
|
|
3365
|
-
ingestBatch.push({
|
|
3366
|
-
role: "user",
|
|
3367
|
-
content: params.autoCompactionSummary,
|
|
3368
|
-
} as AgentMessage);
|
|
3369
|
-
}
|
|
3370
|
-
|
|
3371
|
-
ingestBatch.push(...dedupedNewMessages);
|
|
3372
|
-
if (ingestBatch.length === 0) {
|
|
3373
|
-
this.deps.log.info(
|
|
3374
|
-
`[lcm] afterTurn: nothing to ingest ${sessionLabel} newMessages=${newMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3375
|
-
);
|
|
3376
|
-
return;
|
|
3377
|
-
}
|
|
3378
|
-
|
|
3379
|
-
try {
|
|
3380
|
-
await this.ingestBatch({
|
|
3381
|
-
sessionId: params.sessionId,
|
|
3382
|
-
sessionKey: params.sessionKey,
|
|
3383
|
-
messages: ingestBatch,
|
|
3384
|
-
isHeartbeat: params.isHeartbeat === true,
|
|
3385
|
-
});
|
|
3386
|
-
} catch (err) {
|
|
3387
|
-
// Never compact a stale or partially ingested frontier.
|
|
3388
|
-
this.deps.log.error(
|
|
3389
|
-
`[lcm] afterTurn: ingest failed, skipping compaction: ${describeLogError(err)}`,
|
|
3390
|
-
);
|
|
3391
|
-
return;
|
|
3392
|
-
}
|
|
3393
|
-
|
|
3394
|
-
if (batchLooksLikeHeartbeatAckTurn(ingestBatch)) {
|
|
3395
|
-
try {
|
|
3396
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
3397
|
-
sessionId: params.sessionId,
|
|
3398
|
-
sessionKey: params.sessionKey,
|
|
3399
|
-
});
|
|
3400
|
-
if (conversation) {
|
|
3401
|
-
const pruned = await this.pruneHeartbeatOkTurns(conversation.conversationId);
|
|
3402
|
-
if (pruned > 0) {
|
|
3403
|
-
const sessionContext = this.formatSessionLogContext({
|
|
3404
|
-
conversationId: conversation.conversationId,
|
|
3405
|
-
sessionId: params.sessionId,
|
|
3406
|
-
sessionKey: params.sessionKey,
|
|
3407
|
-
});
|
|
3408
|
-
try {
|
|
3409
|
-
await this.refreshBootstrapState({
|
|
3410
|
-
conversationId: conversation.conversationId,
|
|
3411
|
-
sessionFile: params.sessionFile,
|
|
3412
|
-
});
|
|
3413
|
-
} catch (err) {
|
|
3414
|
-
this.deps.log.warn(
|
|
3415
|
-
`[lcm] afterTurn: heartbeat pruning checkpoint refresh failed for ${sessionContext}: ${describeLogError(err)}`,
|
|
3416
|
-
);
|
|
3417
|
-
}
|
|
3418
|
-
this.deps.log.info(
|
|
3419
|
-
`[lcm] afterTurn: pruned ${pruned} heartbeat ack messages for ${sessionContext}`,
|
|
3420
|
-
);
|
|
3421
|
-
return;
|
|
3422
|
-
}
|
|
3423
|
-
}
|
|
3424
|
-
} catch (err) {
|
|
3425
|
-
this.deps.log.warn(
|
|
3426
|
-
`[lcm] afterTurn: heartbeat pruning failed: ${describeLogError(err)}`,
|
|
3427
|
-
);
|
|
3428
|
-
}
|
|
3429
|
-
}
|
|
3430
|
-
|
|
3431
|
-
const legacyParams = asRecord(params.runtimeContext) ?? asRecord(params.legacyCompactionParams);
|
|
3432
|
-
const DEFAULT_AFTER_TURN_TOKEN_BUDGET = 128_000;
|
|
3433
|
-
const resolvedTokenBudget = this.resolveTokenBudget({
|
|
3434
|
-
tokenBudget: params.tokenBudget,
|
|
3435
|
-
runtimeContext: params.runtimeContext,
|
|
3436
|
-
legacyParams,
|
|
3437
|
-
});
|
|
3438
|
-
const tokenBudget = this.applyAssemblyBudgetCap(resolvedTokenBudget ?? DEFAULT_AFTER_TURN_TOKEN_BUDGET);
|
|
3439
|
-
if (resolvedTokenBudget === undefined) {
|
|
3440
|
-
this.deps.log.warn(
|
|
3441
|
-
`[lcm] afterTurn: tokenBudget not provided; using default ${DEFAULT_AFTER_TURN_TOKEN_BUDGET}`,
|
|
3442
|
-
);
|
|
3443
|
-
}
|
|
3444
|
-
|
|
3445
|
-
const liveContextTokens = estimateSessionTokenCountForAfterTurn(params.messages);
|
|
3446
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
3447
|
-
sessionId: params.sessionId,
|
|
3448
|
-
sessionKey: params.sessionKey,
|
|
3449
|
-
});
|
|
3450
|
-
if (!conversation) {
|
|
3451
|
-
this.deps.log.info(
|
|
3452
|
-
`[lcm] afterTurn: conversation lookup missed ${sessionLabel} ingestBatch=${ingestBatch.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3453
|
-
);
|
|
3454
|
-
return;
|
|
3455
|
-
}
|
|
3456
|
-
|
|
3457
|
-
try {
|
|
3458
|
-
const rawLeafTrigger = await this.compaction.evaluateLeafTrigger(conversation.conversationId);
|
|
3459
|
-
await this.updateCompactionTelemetry({
|
|
3460
|
-
conversationId: conversation.conversationId,
|
|
3461
|
-
runtimeContext: asRecord(params.runtimeContext),
|
|
3462
|
-
tokenBudget,
|
|
3463
|
-
rawTokensOutsideTail: rawLeafTrigger.rawTokensOutsideTail,
|
|
3464
|
-
});
|
|
3465
|
-
} catch (err) {
|
|
3466
|
-
this.deps.log.warn(
|
|
3467
|
-
`[lcm] afterTurn: compaction telemetry update failed: ${describeLogError(err)}`,
|
|
3468
|
-
);
|
|
3469
|
-
}
|
|
3470
|
-
|
|
3471
|
-
try {
|
|
3472
|
-
const leafDecision = await this.evaluateIncrementalCompaction({
|
|
3473
|
-
conversationId: conversation.conversationId,
|
|
3474
|
-
tokenBudget,
|
|
3475
|
-
currentTokenCount: liveContextTokens,
|
|
3476
|
-
});
|
|
3477
|
-
if (leafDecision.shouldCompact) {
|
|
3478
|
-
this.compactLeafAsync({
|
|
3479
|
-
sessionId: params.sessionId,
|
|
3480
|
-
sessionKey: params.sessionKey,
|
|
3481
|
-
sessionFile: params.sessionFile,
|
|
3482
|
-
tokenBudget,
|
|
3483
|
-
currentTokenCount: liveContextTokens,
|
|
3484
|
-
legacyParams,
|
|
3485
|
-
maxPasses: leafDecision.maxPasses,
|
|
3486
|
-
leafChunkTokens: leafDecision.leafChunkTokens,
|
|
3487
|
-
fallbackLeafChunkTokens: leafDecision.fallbackLeafChunkTokens,
|
|
3488
|
-
activityBand: leafDecision.activityBand,
|
|
3489
|
-
allowCondensedPasses: leafDecision.allowCondensedPasses,
|
|
3490
|
-
}).catch(() => {
|
|
3491
|
-
// Leaf compaction is best-effort and should not fail the caller.
|
|
3492
|
-
});
|
|
3493
|
-
}
|
|
3494
|
-
} catch {
|
|
3495
|
-
// Leaf trigger checks are best-effort.
|
|
3496
|
-
}
|
|
3497
|
-
|
|
3498
|
-
try {
|
|
3499
|
-
await this.compact({
|
|
3500
|
-
sessionId: params.sessionId,
|
|
3501
|
-
sessionKey: params.sessionKey,
|
|
3502
|
-
sessionFile: params.sessionFile,
|
|
3503
|
-
tokenBudget,
|
|
3504
|
-
currentTokenCount: liveContextTokens,
|
|
3505
|
-
compactionTarget: "threshold",
|
|
3506
|
-
legacyParams,
|
|
3507
|
-
});
|
|
3508
|
-
} catch {
|
|
3509
|
-
// Proactive compaction is best-effort in the post-turn lifecycle.
|
|
3510
|
-
}
|
|
3511
|
-
|
|
3512
|
-
this.deps.log.info(
|
|
3513
|
-
`[lcm] afterTurn: done conversation=${conversation.conversationId} ${sessionLabel} newMessages=${newMessages.length} dedupedMessages=${dedupedNewMessages.length} ingestedMessages=${ingestBatch.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3514
|
-
);
|
|
3515
|
-
}
|
|
3516
|
-
|
|
3517
|
-
async assemble(params: {
|
|
3518
|
-
sessionId: string;
|
|
3519
|
-
sessionKey?: string;
|
|
3520
|
-
messages: AgentMessage[];
|
|
3521
|
-
tokenBudget?: number;
|
|
3522
|
-
/** Optional user query for relevance-based eviction (BM25-lite). When absent or unsearchable, falls back to chronological eviction. */
|
|
3523
|
-
prompt?: string;
|
|
3524
|
-
}): Promise<AssembleResult> {
|
|
3525
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
3526
|
-
return {
|
|
3527
|
-
messages: params.messages,
|
|
3528
|
-
estimatedTokens: 0,
|
|
3529
|
-
};
|
|
3530
|
-
}
|
|
3531
|
-
try {
|
|
3532
|
-
this.ensureMigrated();
|
|
3533
|
-
const startedAt = Date.now();
|
|
3534
|
-
const sessionLabel = [
|
|
3535
|
-
`session=${params.sessionId}`,
|
|
3536
|
-
...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
|
|
3537
|
-
].join(" ");
|
|
3538
|
-
|
|
3539
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
3540
|
-
sessionId: params.sessionId,
|
|
3541
|
-
sessionKey: params.sessionKey,
|
|
3542
|
-
});
|
|
3543
|
-
if (!conversation) {
|
|
3544
|
-
this.deps.log.info(
|
|
3545
|
-
`[lcm] assemble: conversation lookup missed ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3546
|
-
);
|
|
3547
|
-
return {
|
|
3548
|
-
messages: params.messages,
|
|
3549
|
-
estimatedTokens: 0,
|
|
3550
|
-
};
|
|
3551
|
-
}
|
|
3552
|
-
|
|
3553
|
-
const contextItems = await this.summaryStore.getContextItems(conversation.conversationId);
|
|
3554
|
-
if (contextItems.length === 0) {
|
|
3555
|
-
this.deps.log.info(
|
|
3556
|
-
`[lcm] assemble: no context items conversation=${conversation.conversationId} ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3557
|
-
);
|
|
3558
|
-
return {
|
|
3559
|
-
messages: params.messages,
|
|
3560
|
-
estimatedTokens: 0,
|
|
3561
|
-
};
|
|
3562
|
-
}
|
|
3563
|
-
|
|
3564
|
-
// Guard against incomplete bootstrap/coverage: if the DB only has
|
|
3565
|
-
// raw context items and clearly trails the current live history, keep
|
|
3566
|
-
// the live path to avoid dropping prompt context.
|
|
3567
|
-
const hasSummaryItems = contextItems.some((item) => item.itemType === "summary");
|
|
3568
|
-
if (!hasSummaryItems && contextItems.length < params.messages.length) {
|
|
3569
|
-
this.deps.log.info(
|
|
3570
|
-
`[lcm] assemble: falling back to live context conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} liveMessages=${params.messages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3571
|
-
);
|
|
3572
|
-
return {
|
|
3573
|
-
messages: params.messages,
|
|
3574
|
-
estimatedTokens: 0,
|
|
3575
|
-
};
|
|
3576
|
-
}
|
|
3577
|
-
|
|
3578
|
-
const tokenBudget = this.applyAssemblyBudgetCap(
|
|
3579
|
-
typeof params.tokenBudget === "number" &&
|
|
3580
|
-
Number.isFinite(params.tokenBudget) &&
|
|
3581
|
-
params.tokenBudget > 0
|
|
3582
|
-
? Math.floor(params.tokenBudget)
|
|
3583
|
-
: 128_000,
|
|
3584
|
-
);
|
|
3585
|
-
|
|
3586
|
-
const assembled = await this.assembler.assemble({
|
|
3587
|
-
conversationId: conversation.conversationId,
|
|
3588
|
-
tokenBudget,
|
|
3589
|
-
freshTailCount: this.config.freshTailCount,
|
|
3590
|
-
prompt: params.prompt,
|
|
3591
|
-
});
|
|
3592
|
-
|
|
3593
|
-
// If assembly produced no messages for a non-empty live session,
|
|
3594
|
-
// fail safe to the live context.
|
|
3595
|
-
if (assembled.messages.length === 0 && params.messages.length > 0) {
|
|
3596
|
-
this.deps.log.info(
|
|
3597
|
-
`[lcm] assemble: empty assembled output, using live context conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} tokenBudget=${tokenBudget} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3598
|
-
);
|
|
3599
|
-
return {
|
|
3600
|
-
messages: params.messages,
|
|
3601
|
-
estimatedTokens: 0,
|
|
3602
|
-
};
|
|
3603
|
-
}
|
|
3604
|
-
|
|
3605
|
-
this.deps.log.info(
|
|
3606
|
-
`[lcm] assemble: done conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} hasSummaryItems=${hasSummaryItems} inputMessages=${params.messages.length} outputMessages=${assembled.messages.length} tokenBudget=${tokenBudget} estimatedTokens=${assembled.estimatedTokens} duration=${formatDurationMs(Date.now() - startedAt)}`,
|
|
3607
|
-
);
|
|
3608
|
-
|
|
3609
|
-
const result: AssembleResultWithSystemPrompt = {
|
|
3610
|
-
messages: assembled.messages,
|
|
3611
|
-
estimatedTokens: assembled.estimatedTokens,
|
|
3612
|
-
...(assembled.systemPromptAddition
|
|
3613
|
-
? { systemPromptAddition: assembled.systemPromptAddition }
|
|
3614
|
-
: {}),
|
|
3615
|
-
};
|
|
3616
|
-
return result;
|
|
3617
|
-
} catch (err) {
|
|
3618
|
-
this.deps.log.info(
|
|
3619
|
-
`[lcm] assemble: failed for session=${params.sessionId}${params.sessionKey?.trim() ? ` sessionKey=${params.sessionKey.trim()}` : ""} error=${describeLogError(err)}`,
|
|
3620
|
-
);
|
|
3621
|
-
return {
|
|
3622
|
-
messages: params.messages,
|
|
3623
|
-
estimatedTokens: 0,
|
|
3624
|
-
};
|
|
3625
|
-
}
|
|
3626
|
-
}
|
|
3627
|
-
|
|
3628
|
-
/** Evaluate whether incremental leaf compaction should run for a session. */
|
|
3629
|
-
async evaluateLeafTrigger(sessionId: string, sessionKey?: string): Promise<{
|
|
3630
|
-
shouldCompact: boolean;
|
|
3631
|
-
rawTokensOutsideTail: number;
|
|
3632
|
-
threshold: number;
|
|
3633
|
-
}> {
|
|
3634
|
-
this.ensureMigrated();
|
|
3635
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
3636
|
-
sessionId,
|
|
3637
|
-
sessionKey,
|
|
3638
|
-
});
|
|
3639
|
-
if (!conversation) {
|
|
3640
|
-
const fallbackThreshold =
|
|
3641
|
-
typeof this.config.leafChunkTokens === "number" &&
|
|
3642
|
-
Number.isFinite(this.config.leafChunkTokens) &&
|
|
3643
|
-
this.config.leafChunkTokens > 0
|
|
3644
|
-
? Math.floor(this.config.leafChunkTokens)
|
|
3645
|
-
: 20_000;
|
|
3646
|
-
return {
|
|
3647
|
-
shouldCompact: false,
|
|
3648
|
-
rawTokensOutsideTail: 0,
|
|
3649
|
-
threshold: fallbackThreshold,
|
|
3650
|
-
};
|
|
3651
|
-
}
|
|
3652
|
-
return this.compaction.evaluateLeafTrigger(conversation.conversationId);
|
|
3653
|
-
}
|
|
3654
|
-
|
|
3655
|
-
/** Run one or more incremental leaf compaction passes in the per-session queue. */
|
|
3656
|
-
async compactLeafAsync(params: {
|
|
3657
|
-
sessionId: string;
|
|
3658
|
-
sessionKey?: string;
|
|
3659
|
-
sessionFile: string;
|
|
3660
|
-
tokenBudget?: number;
|
|
3661
|
-
currentTokenCount?: number;
|
|
3662
|
-
customInstructions?: string;
|
|
3663
|
-
/** OpenClaw runtime param name (preferred). */
|
|
3664
|
-
runtimeContext?: Record<string, unknown>;
|
|
3665
|
-
/** Back-compat param name. */
|
|
3666
|
-
legacyParams?: Record<string, unknown>;
|
|
3667
|
-
force?: boolean;
|
|
3668
|
-
previousSummaryContent?: string;
|
|
3669
|
-
maxPasses?: number;
|
|
3670
|
-
leafChunkTokens?: number;
|
|
3671
|
-
fallbackLeafChunkTokens?: number[];
|
|
3672
|
-
activityBand?: ActivityBand;
|
|
3673
|
-
allowCondensedPasses?: boolean;
|
|
3674
|
-
}): Promise<CompactResult> {
|
|
3675
|
-
if (this.isStatelessSession(params.sessionKey)) {
|
|
3676
|
-
return {
|
|
3677
|
-
ok: true,
|
|
3678
|
-
compacted: false,
|
|
3679
|
-
reason: "stateless session",
|
|
3680
|
-
};
|
|
3681
|
-
}
|
|
3682
|
-
this.ensureMigrated();
|
|
3683
|
-
return this.withSessionQueue(
|
|
3684
|
-
this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
3685
|
-
async () => {
|
|
3686
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
3687
|
-
sessionId: params.sessionId,
|
|
3688
|
-
sessionKey: params.sessionKey,
|
|
3689
|
-
});
|
|
3690
|
-
if (!conversation) {
|
|
3691
|
-
return {
|
|
3692
|
-
ok: true,
|
|
3693
|
-
compacted: false,
|
|
3694
|
-
reason: "no conversation found for session",
|
|
3695
|
-
};
|
|
3696
|
-
}
|
|
3697
|
-
|
|
3698
|
-
const legacyParams = asRecord(params.runtimeContext) ?? params.legacyParams;
|
|
3699
|
-
const resolvedTokenBudget = this.resolveTokenBudget({
|
|
3700
|
-
tokenBudget: params.tokenBudget,
|
|
3701
|
-
runtimeContext: params.runtimeContext,
|
|
3702
|
-
legacyParams,
|
|
3703
|
-
});
|
|
3704
|
-
const tokenBudget = resolvedTokenBudget
|
|
3705
|
-
? this.applyAssemblyBudgetCap(resolvedTokenBudget)
|
|
3706
|
-
: resolvedTokenBudget;
|
|
3707
|
-
if (!tokenBudget) {
|
|
3708
|
-
return {
|
|
3709
|
-
ok: false,
|
|
3710
|
-
compacted: false,
|
|
3711
|
-
reason: "missing token budget in compact params",
|
|
3712
|
-
};
|
|
3713
|
-
}
|
|
3714
|
-
|
|
3715
|
-
const lp = legacyParams ?? {};
|
|
3716
|
-
const observedTokens = this.normalizeObservedTokenCount(
|
|
3717
|
-
params.currentTokenCount ??
|
|
3718
|
-
(
|
|
3719
|
-
lp as {
|
|
3720
|
-
currentTokenCount?: unknown;
|
|
3721
|
-
}
|
|
3722
|
-
).currentTokenCount,
|
|
3723
|
-
);
|
|
3724
|
-
const { summarize, summaryModel, breakerKey } = await this.resolveSummarize({
|
|
3725
|
-
legacyParams,
|
|
3726
|
-
customInstructions: params.customInstructions,
|
|
3727
|
-
breakerScope: this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
3728
|
-
});
|
|
3729
|
-
if (breakerKey && this.isCircuitBreakerOpen(breakerKey)) {
|
|
3730
|
-
return {
|
|
3731
|
-
ok: true,
|
|
3732
|
-
compacted: false,
|
|
3733
|
-
reason: "circuit breaker open",
|
|
3734
|
-
};
|
|
3735
|
-
}
|
|
3736
|
-
|
|
3737
|
-
const storedTokensBefore = await this.summaryStore.getContextTokenCount(
|
|
3738
|
-
conversation.conversationId,
|
|
3739
|
-
);
|
|
3740
|
-
const maxPasses =
|
|
3741
|
-
typeof params.maxPasses === "number" &&
|
|
3742
|
-
Number.isFinite(params.maxPasses) &&
|
|
3743
|
-
params.maxPasses > 0
|
|
3744
|
-
? Math.floor(params.maxPasses)
|
|
3745
|
-
: 1;
|
|
3746
|
-
const fallbackLeafChunkTokens = Array.isArray(params.fallbackLeafChunkTokens)
|
|
3747
|
-
? [...new Set(params.fallbackLeafChunkTokens
|
|
3748
|
-
.filter((value): value is number => typeof value === "number" && Number.isFinite(value) && value > 0)
|
|
3749
|
-
.map((value) => Math.floor(value)))]
|
|
3750
|
-
.sort((a, b) => b - a)
|
|
3751
|
-
: [];
|
|
3752
|
-
let activeLeafChunkTokens =
|
|
3753
|
-
typeof params.leafChunkTokens === "number" &&
|
|
3754
|
-
Number.isFinite(params.leafChunkTokens) &&
|
|
3755
|
-
params.leafChunkTokens > 0
|
|
3756
|
-
? Math.floor(params.leafChunkTokens)
|
|
3757
|
-
: fallbackLeafChunkTokens[0];
|
|
3758
|
-
this.deps.log.info(
|
|
3759
|
-
`[lcm] compactLeafAsync start: conversation=${conversation.conversationId} session=${params.sessionId} leafChunkTokens=${activeLeafChunkTokens ?? "null"} fallbackLeafChunkTokens=${fallbackLeafChunkTokens.join(",")} maxPasses=${maxPasses} activityBand=${params.activityBand ?? "unknown"} allowCondensedPasses=${params.allowCondensedPasses !== false}`,
|
|
3760
|
-
);
|
|
3761
|
-
|
|
3762
|
-
let rounds = 0;
|
|
3763
|
-
let finalTokens = observedTokens ?? storedTokensBefore;
|
|
3764
|
-
let authFailure = false;
|
|
3765
|
-
|
|
3766
|
-
for (let pass = 0; pass < maxPasses; pass += 1) {
|
|
3767
|
-
let leafResult: Awaited<ReturnType<typeof this.compaction.compactLeaf>> | undefined;
|
|
3768
|
-
while (true) {
|
|
3769
|
-
try {
|
|
3770
|
-
leafResult = await this.compaction.compactLeaf({
|
|
3771
|
-
conversationId: conversation.conversationId,
|
|
3772
|
-
tokenBudget,
|
|
3773
|
-
summarize,
|
|
3774
|
-
...(activeLeafChunkTokens !== undefined ? { leafChunkTokens: activeLeafChunkTokens } : {}),
|
|
3775
|
-
force: params.force,
|
|
3776
|
-
previousSummaryContent: pass === 0 ? params.previousSummaryContent : undefined,
|
|
3777
|
-
summaryModel,
|
|
3778
|
-
allowCondensedPasses: params.allowCondensedPasses,
|
|
3779
|
-
});
|
|
3780
|
-
break;
|
|
3781
|
-
} catch (err) {
|
|
3782
|
-
const nextLeafChunkTokens = fallbackLeafChunkTokens.find(
|
|
3783
|
-
(value) => activeLeafChunkTokens !== undefined && value < activeLeafChunkTokens,
|
|
3784
|
-
);
|
|
3785
|
-
if (!this.isRecoverableLeafChunkOverflowError(err) || nextLeafChunkTokens === undefined) {
|
|
3786
|
-
throw err;
|
|
3787
|
-
}
|
|
3788
|
-
this.deps.log.warn(
|
|
3789
|
-
`[lcm] compactLeafAsync: retrying with smaller leafChunkTokens=${nextLeafChunkTokens} after provider token-limit error: ${err instanceof Error ? err.message : String(err)}`,
|
|
3790
|
-
);
|
|
3791
|
-
activeLeafChunkTokens = nextLeafChunkTokens;
|
|
3792
|
-
}
|
|
3793
|
-
}
|
|
3794
|
-
if (!leafResult) {
|
|
3795
|
-
break;
|
|
3796
|
-
}
|
|
3797
|
-
finalTokens = leafResult.tokensAfter;
|
|
3798
|
-
|
|
3799
|
-
if (leafResult.authFailure) {
|
|
3800
|
-
authFailure = true;
|
|
3801
|
-
break;
|
|
3802
|
-
}
|
|
3803
|
-
if (!leafResult.actionTaken) {
|
|
3804
|
-
break;
|
|
3805
|
-
}
|
|
3806
|
-
rounds += 1;
|
|
3807
|
-
if (leafResult.tokensAfter >= leafResult.tokensBefore) {
|
|
3808
|
-
break;
|
|
3809
|
-
}
|
|
3810
|
-
}
|
|
3811
|
-
|
|
3812
|
-
if (authFailure && breakerKey) {
|
|
3813
|
-
this.recordCompactionAuthFailure(breakerKey);
|
|
3814
|
-
} else if (rounds > 0 && breakerKey) {
|
|
3815
|
-
this.recordCompactionSuccess(breakerKey);
|
|
3816
|
-
}
|
|
3817
|
-
if (rounds > 0) {
|
|
3818
|
-
await this.markLeafCompactionTelemetrySuccess({
|
|
3819
|
-
conversationId: conversation.conversationId,
|
|
3820
|
-
activityBand: params.activityBand,
|
|
3821
|
-
});
|
|
3822
|
-
}
|
|
3823
|
-
|
|
3824
|
-
const tokensBefore = observedTokens ?? storedTokensBefore;
|
|
3825
|
-
this.deps.log.debug(
|
|
3826
|
-
`[lcm] compactLeafAsync result: conversation=${conversation.conversationId} session=${params.sessionId} rounds=${rounds} compacted=${rounds > 0} authFailure=${authFailure} finalLeafChunkTokens=${activeLeafChunkTokens ?? "null"} finalTokens=${finalTokens}`,
|
|
3827
|
-
);
|
|
3828
|
-
|
|
3829
|
-
return {
|
|
3830
|
-
ok: true,
|
|
3831
|
-
compacted: rounds > 0,
|
|
3832
|
-
reason: authFailure
|
|
3833
|
-
? "provider auth failure"
|
|
3834
|
-
: rounds > 0
|
|
3835
|
-
? "compacted"
|
|
3836
|
-
: "below threshold",
|
|
3837
|
-
result: {
|
|
3838
|
-
tokensBefore,
|
|
3839
|
-
tokensAfter: finalTokens,
|
|
3840
|
-
details: {
|
|
3841
|
-
rounds,
|
|
3842
|
-
targetTokens: tokenBudget,
|
|
3843
|
-
mode: "leaf",
|
|
3844
|
-
maxPasses,
|
|
3845
|
-
},
|
|
3846
|
-
},
|
|
3847
|
-
};
|
|
3848
|
-
},
|
|
3849
|
-
);
|
|
3850
|
-
}
|
|
3851
|
-
|
|
3852
|
-
async compact(params: {
|
|
3853
|
-
sessionId: string;
|
|
3854
|
-
sessionKey?: string;
|
|
3855
|
-
sessionFile: string;
|
|
3856
|
-
tokenBudget?: number;
|
|
3857
|
-
currentTokenCount?: number;
|
|
3858
|
-
compactionTarget?: "budget" | "threshold";
|
|
3859
|
-
customInstructions?: string;
|
|
3860
|
-
/** OpenClaw runtime param name (preferred). */
|
|
3861
|
-
runtimeContext?: Record<string, unknown>;
|
|
3862
|
-
/** Back-compat param name. */
|
|
3863
|
-
legacyParams?: Record<string, unknown>;
|
|
3864
|
-
/** Force compaction even if below threshold */
|
|
3865
|
-
force?: boolean;
|
|
3866
|
-
}): Promise<CompactResult> {
|
|
3867
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
3868
|
-
return {
|
|
3869
|
-
ok: true,
|
|
3870
|
-
compacted: false,
|
|
3871
|
-
reason: "session excluded",
|
|
3872
|
-
};
|
|
3873
|
-
}
|
|
3874
|
-
if (this.isStatelessSession(params.sessionKey)) {
|
|
3875
|
-
return {
|
|
3876
|
-
ok: true,
|
|
3877
|
-
compacted: false,
|
|
3878
|
-
reason: "stateless session",
|
|
3879
|
-
};
|
|
3880
|
-
}
|
|
3881
|
-
this.ensureMigrated();
|
|
3882
|
-
return this.withSessionQueue(
|
|
3883
|
-
this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
3884
|
-
async () => {
|
|
3885
|
-
const { sessionId, force = false } = params;
|
|
3886
|
-
|
|
3887
|
-
// Look up conversation
|
|
3888
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
3889
|
-
sessionId,
|
|
3890
|
-
sessionKey: params.sessionKey,
|
|
3891
|
-
});
|
|
3892
|
-
if (!conversation) {
|
|
3893
|
-
return {
|
|
3894
|
-
ok: true,
|
|
3895
|
-
compacted: false,
|
|
3896
|
-
reason: "no conversation found for session",
|
|
3897
|
-
};
|
|
3898
|
-
}
|
|
3899
|
-
|
|
3900
|
-
const conversationId = conversation.conversationId;
|
|
3901
|
-
|
|
3902
|
-
const legacyParams = asRecord(params.runtimeContext) ?? params.legacyParams;
|
|
3903
|
-
const lp = legacyParams ?? {};
|
|
3904
|
-
const manualCompactionRequested =
|
|
3905
|
-
(
|
|
3906
|
-
lp as {
|
|
3907
|
-
manualCompaction?: unknown;
|
|
3908
|
-
}
|
|
3909
|
-
).manualCompaction === true;
|
|
3910
|
-
const forceCompaction = force || manualCompactionRequested;
|
|
3911
|
-
const resolvedTokenBudget = this.resolveTokenBudget({
|
|
3912
|
-
tokenBudget: params.tokenBudget,
|
|
3913
|
-
runtimeContext: params.runtimeContext,
|
|
3914
|
-
legacyParams,
|
|
3915
|
-
});
|
|
3916
|
-
const tokenBudget = resolvedTokenBudget
|
|
3917
|
-
? this.applyAssemblyBudgetCap(resolvedTokenBudget)
|
|
3918
|
-
: resolvedTokenBudget;
|
|
3919
|
-
if (!tokenBudget) {
|
|
3920
|
-
return {
|
|
3921
|
-
ok: false,
|
|
3922
|
-
compacted: false,
|
|
3923
|
-
reason: "missing token budget in compact params",
|
|
3924
|
-
};
|
|
3925
|
-
}
|
|
3926
|
-
|
|
3927
|
-
const { summarize, summaryModel, breakerKey } = await this.resolveSummarize({
|
|
3928
|
-
legacyParams,
|
|
3929
|
-
customInstructions: params.customInstructions,
|
|
3930
|
-
breakerScope: this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
3931
|
-
});
|
|
3932
|
-
if (breakerKey && this.isCircuitBreakerOpen(breakerKey)) {
|
|
3933
|
-
return {
|
|
3934
|
-
ok: true,
|
|
3935
|
-
compacted: false,
|
|
3936
|
-
reason: "circuit breaker open",
|
|
3937
|
-
};
|
|
3938
|
-
}
|
|
3939
|
-
|
|
3940
|
-
// Evaluate whether compaction is needed (unless forced)
|
|
3941
|
-
const observedTokens = this.normalizeObservedTokenCount(
|
|
3942
|
-
params.currentTokenCount ??
|
|
3943
|
-
(
|
|
3944
|
-
lp as {
|
|
3945
|
-
currentTokenCount?: unknown;
|
|
3946
|
-
}
|
|
3947
|
-
).currentTokenCount,
|
|
3948
|
-
);
|
|
3949
|
-
const decision =
|
|
3950
|
-
observedTokens !== undefined
|
|
3951
|
-
? await this.compaction.evaluate(conversationId, tokenBudget, observedTokens)
|
|
3952
|
-
: await this.compaction.evaluate(conversationId, tokenBudget);
|
|
3953
|
-
const targetTokens =
|
|
3954
|
-
params.compactionTarget === "threshold" ? decision.threshold : tokenBudget;
|
|
3955
|
-
const liveContextStillExceedsTarget =
|
|
3956
|
-
observedTokens !== undefined && observedTokens >= targetTokens;
|
|
3957
|
-
|
|
3958
|
-
if (!forceCompaction && !decision.shouldCompact) {
|
|
3959
|
-
return {
|
|
3960
|
-
ok: true,
|
|
3961
|
-
compacted: false,
|
|
3962
|
-
reason: "below threshold",
|
|
3963
|
-
result: {
|
|
3964
|
-
tokensBefore: decision.currentTokens,
|
|
3965
|
-
},
|
|
3966
|
-
};
|
|
3967
|
-
}
|
|
3968
|
-
|
|
3969
|
-
// Forced budget recovery should use the capped convergence loop so live
|
|
3970
|
-
// overflow counts can drive recovery even when persisted context is already small.
|
|
3971
|
-
const useSweep = manualCompactionRequested || params.compactionTarget === "threshold";
|
|
3972
|
-
if (useSweep) {
|
|
3973
|
-
const sweepResult = await this.compaction.compact({
|
|
3974
|
-
conversationId,
|
|
3975
|
-
tokenBudget,
|
|
3976
|
-
summarize,
|
|
3977
|
-
force: forceCompaction,
|
|
3978
|
-
hardTrigger: false,
|
|
3979
|
-
summaryModel,
|
|
3980
|
-
});
|
|
3981
|
-
|
|
3982
|
-
if (sweepResult.authFailure && breakerKey) {
|
|
3983
|
-
this.recordCompactionAuthFailure(breakerKey);
|
|
3984
|
-
} else if (sweepResult.actionTaken && breakerKey) {
|
|
3985
|
-
this.recordCompactionSuccess(breakerKey);
|
|
3986
|
-
}
|
|
3987
|
-
if (sweepResult.actionTaken) {
|
|
3988
|
-
await this.markLeafCompactionTelemetrySuccess({ conversationId });
|
|
3989
|
-
}
|
|
3990
|
-
|
|
3991
|
-
return {
|
|
3992
|
-
ok: !sweepResult.authFailure && (sweepResult.actionTaken || !liveContextStillExceedsTarget),
|
|
3993
|
-
compacted: sweepResult.actionTaken,
|
|
3994
|
-
reason: sweepResult.authFailure
|
|
3995
|
-
? (sweepResult.actionTaken
|
|
3996
|
-
? "provider auth failure after partial compaction"
|
|
3997
|
-
: "provider auth failure")
|
|
3998
|
-
: sweepResult.actionTaken
|
|
3999
|
-
? "compacted"
|
|
4000
|
-
: manualCompactionRequested
|
|
4001
|
-
? "nothing to compact"
|
|
4002
|
-
: liveContextStillExceedsTarget
|
|
4003
|
-
? "live context still exceeds target"
|
|
4004
|
-
: "already under target",
|
|
4005
|
-
result: {
|
|
4006
|
-
tokensBefore: decision.currentTokens,
|
|
4007
|
-
tokensAfter: sweepResult.tokensAfter,
|
|
4008
|
-
details: {
|
|
4009
|
-
rounds: sweepResult.actionTaken ? 1 : 0,
|
|
4010
|
-
targetTokens,
|
|
4011
|
-
},
|
|
4012
|
-
},
|
|
4013
|
-
};
|
|
4014
|
-
}
|
|
4015
|
-
|
|
4016
|
-
// When forced, use the token budget as target
|
|
4017
|
-
const convergenceTargetTokens = forceCompaction
|
|
4018
|
-
? tokenBudget
|
|
4019
|
-
: params.compactionTarget === "threshold"
|
|
4020
|
-
? decision.threshold
|
|
4021
|
-
: tokenBudget;
|
|
4022
|
-
|
|
4023
|
-
// When forced (overflow recovery) and the caller did not supply an
|
|
4024
|
-
// observed token count, assume we are at least at the token budget so
|
|
4025
|
-
// compactUntilUnder does not bail with "already under target" while the
|
|
4026
|
-
// live context is actually overflowing.
|
|
4027
|
-
const effectiveCurrentTokens =
|
|
4028
|
-
observedTokens !== undefined
|
|
4029
|
-
? observedTokens
|
|
4030
|
-
: forceCompaction
|
|
4031
|
-
? tokenBudget
|
|
4032
|
-
: undefined;
|
|
4033
|
-
const compactResult = await this.compaction.compactUntilUnder({
|
|
4034
|
-
conversationId,
|
|
4035
|
-
tokenBudget,
|
|
4036
|
-
targetTokens: convergenceTargetTokens,
|
|
4037
|
-
...(effectiveCurrentTokens !== undefined ? { currentTokens: effectiveCurrentTokens } : {}),
|
|
4038
|
-
summarize,
|
|
4039
|
-
summaryModel,
|
|
4040
|
-
});
|
|
4041
|
-
|
|
4042
|
-
if (compactResult.authFailure && breakerKey) {
|
|
4043
|
-
this.recordCompactionAuthFailure(breakerKey);
|
|
4044
|
-
} else if (compactResult.rounds > 0 && breakerKey) {
|
|
4045
|
-
this.recordCompactionSuccess(breakerKey);
|
|
4046
|
-
}
|
|
4047
|
-
|
|
4048
|
-
const didCompact = compactResult.rounds > 0;
|
|
4049
|
-
if (didCompact) {
|
|
4050
|
-
await this.markLeafCompactionTelemetrySuccess({ conversationId });
|
|
4051
|
-
}
|
|
4052
|
-
|
|
4053
|
-
return {
|
|
4054
|
-
ok: compactResult.success,
|
|
4055
|
-
compacted: didCompact,
|
|
4056
|
-
reason: compactResult.authFailure
|
|
4057
|
-
? (didCompact
|
|
4058
|
-
? "provider auth failure after partial compaction"
|
|
4059
|
-
: "provider auth failure")
|
|
4060
|
-
: compactResult.success
|
|
4061
|
-
? didCompact
|
|
4062
|
-
? "compacted"
|
|
4063
|
-
: "already under target"
|
|
4064
|
-
: "could not reach target",
|
|
4065
|
-
result: {
|
|
4066
|
-
tokensBefore: decision.currentTokens,
|
|
4067
|
-
tokensAfter: compactResult.finalTokens,
|
|
4068
|
-
details: {
|
|
4069
|
-
rounds: compactResult.rounds,
|
|
4070
|
-
targetTokens: convergenceTargetTokens,
|
|
4071
|
-
},
|
|
4072
|
-
},
|
|
4073
|
-
};
|
|
4074
|
-
},
|
|
4075
|
-
);
|
|
4076
|
-
}
|
|
4077
|
-
|
|
4078
|
-
async prepareSubagentSpawn(params: {
|
|
4079
|
-
parentSessionKey: string;
|
|
4080
|
-
childSessionKey: string;
|
|
4081
|
-
ttlMs?: number;
|
|
4082
|
-
}): Promise<SubagentSpawnPreparation | undefined> {
|
|
4083
|
-
if (
|
|
4084
|
-
this.shouldIgnoreSession({ sessionKey: params.parentSessionKey })
|
|
4085
|
-
|| this.shouldIgnoreSession({ sessionKey: params.childSessionKey })
|
|
4086
|
-
|| this.isStatelessSession(params.parentSessionKey)
|
|
4087
|
-
|| this.isStatelessSession(params.childSessionKey)
|
|
4088
|
-
) {
|
|
4089
|
-
return undefined;
|
|
4090
|
-
}
|
|
4091
|
-
this.ensureMigrated();
|
|
4092
|
-
|
|
4093
|
-
const childSessionKey = params.childSessionKey.trim();
|
|
4094
|
-
const parentSessionKey = params.parentSessionKey.trim();
|
|
4095
|
-
if (!childSessionKey || !parentSessionKey) {
|
|
4096
|
-
return undefined;
|
|
4097
|
-
}
|
|
4098
|
-
|
|
4099
|
-
const conversationId = await this.resolveConversationIdForSessionKey(parentSessionKey);
|
|
4100
|
-
if (typeof conversationId !== "number") {
|
|
4101
|
-
return undefined;
|
|
4102
|
-
}
|
|
4103
|
-
|
|
4104
|
-
const ttlMs =
|
|
4105
|
-
typeof params.ttlMs === "number" && Number.isFinite(params.ttlMs) && params.ttlMs > 0
|
|
4106
|
-
? Math.floor(params.ttlMs)
|
|
4107
|
-
: undefined;
|
|
4108
|
-
|
|
4109
|
-
// Inherit scope from parent grant if one exists (prevents privilege escalation)
|
|
4110
|
-
const parentGrantId = resolveDelegatedExpansionGrantId(parentSessionKey);
|
|
4111
|
-
const parentGrant = parentGrantId
|
|
4112
|
-
? getRuntimeExpansionAuthManager().getGrant(parentGrantId)
|
|
4113
|
-
: null;
|
|
4114
|
-
|
|
4115
|
-
const childTokenCap = parentGrant
|
|
4116
|
-
? Math.min(
|
|
4117
|
-
getRuntimeExpansionAuthManager().getRemainingTokenBudget(parentGrantId!) ?? this.config.maxExpandTokens,
|
|
4118
|
-
this.config.maxExpandTokens,
|
|
4119
|
-
)
|
|
4120
|
-
: this.config.maxExpandTokens;
|
|
4121
|
-
|
|
4122
|
-
const childMaxDepth = parentGrant
|
|
4123
|
-
? Math.max(0, parentGrant.maxDepth - 1)
|
|
4124
|
-
: undefined;
|
|
4125
|
-
|
|
4126
|
-
const childAllowedSummaryIds = parentGrant?.allowedSummaryIds.length
|
|
4127
|
-
? parentGrant.allowedSummaryIds
|
|
4128
|
-
: undefined;
|
|
4129
|
-
|
|
4130
|
-
createDelegatedExpansionGrant({
|
|
4131
|
-
delegatedSessionKey: childSessionKey,
|
|
4132
|
-
issuerSessionId: parentSessionKey,
|
|
4133
|
-
allowedConversationIds: [conversationId],
|
|
4134
|
-
allowedSummaryIds: childAllowedSummaryIds,
|
|
4135
|
-
tokenCap: childTokenCap,
|
|
4136
|
-
maxDepth: childMaxDepth,
|
|
4137
|
-
ttlMs,
|
|
4138
|
-
});
|
|
4139
|
-
|
|
4140
|
-
return {
|
|
4141
|
-
rollback: () => {
|
|
4142
|
-
revokeDelegatedExpansionGrantForSession(childSessionKey, { removeBinding: true });
|
|
4143
|
-
},
|
|
4144
|
-
};
|
|
4145
|
-
}
|
|
4146
|
-
|
|
4147
|
-
async onSubagentEnded(params: {
|
|
4148
|
-
childSessionKey: string;
|
|
4149
|
-
reason: SubagentEndReason;
|
|
4150
|
-
}): Promise<void> {
|
|
4151
|
-
if (
|
|
4152
|
-
this.shouldIgnoreSession({ sessionKey: params.childSessionKey })
|
|
4153
|
-
|| this.isStatelessSession(params.childSessionKey)
|
|
4154
|
-
) {
|
|
4155
|
-
return;
|
|
4156
|
-
}
|
|
4157
|
-
const childSessionKey = params.childSessionKey.trim();
|
|
4158
|
-
if (!childSessionKey) {
|
|
4159
|
-
return;
|
|
4160
|
-
}
|
|
4161
|
-
|
|
4162
|
-
switch (params.reason) {
|
|
4163
|
-
case "deleted":
|
|
4164
|
-
revokeDelegatedExpansionGrantForSession(childSessionKey, { removeBinding: true });
|
|
4165
|
-
break;
|
|
4166
|
-
case "completed":
|
|
4167
|
-
revokeDelegatedExpansionGrantForSession(childSessionKey);
|
|
4168
|
-
break;
|
|
4169
|
-
case "released":
|
|
4170
|
-
case "swept":
|
|
4171
|
-
removeDelegatedExpansionGrantForSession(childSessionKey);
|
|
4172
|
-
break;
|
|
4173
|
-
}
|
|
4174
|
-
}
|
|
4175
|
-
|
|
4176
|
-
async dispose(): Promise<void> {
|
|
4177
|
-
// No-op for plugin singleton — the connection is shared across runs.
|
|
4178
|
-
// OpenClaw's runner calls dispose() after every run, but the plugin
|
|
4179
|
-
// registers a single engine instance reused by the factory. Closing
|
|
4180
|
-
// the DB here would break subsequent runs with "database is not open".
|
|
4181
|
-
// The shared connection is managed for the lifetime of the plugin process.
|
|
4182
|
-
}
|
|
4183
|
-
|
|
4184
|
-
/** Detect the empty replacement row created during a prior lifecycle rollover. */
|
|
4185
|
-
private async isFreshLifecycleConversation(conversation: ConversationRecord): Promise<boolean> {
|
|
4186
|
-
const currentMessageCount = await this.conversationStore.getMessageCount(conversation.conversationId);
|
|
4187
|
-
if (currentMessageCount !== 0) {
|
|
4188
|
-
return false;
|
|
4189
|
-
}
|
|
4190
|
-
const currentContextItems = await this.summaryStore.getContextItems(conversation.conversationId);
|
|
4191
|
-
return currentContextItems.length === 0 && !conversation.bootstrappedAt;
|
|
4192
|
-
}
|
|
4193
|
-
|
|
4194
|
-
/**
|
|
4195
|
-
* Archive the current active conversation and optionally create the replacement
|
|
4196
|
-
* row that bootstrap should attach to for the next session transcript.
|
|
4197
|
-
*/
|
|
4198
|
-
private async applySessionReplacement(params: {
|
|
4199
|
-
reason: string;
|
|
4200
|
-
sessionId?: string;
|
|
4201
|
-
sessionKey?: string;
|
|
4202
|
-
nextSessionId?: string;
|
|
4203
|
-
nextSessionKey?: string;
|
|
4204
|
-
createReplacement: boolean;
|
|
4205
|
-
createReplacementWhenMissing?: boolean;
|
|
4206
|
-
}): Promise<void> {
|
|
4207
|
-
const current = await this.conversationStore.getConversationForSession({
|
|
4208
|
-
sessionId: params.sessionId,
|
|
4209
|
-
sessionKey: params.sessionKey,
|
|
4210
|
-
});
|
|
4211
|
-
if (!current && !params.createReplacementWhenMissing) {
|
|
4212
|
-
return;
|
|
4213
|
-
}
|
|
4214
|
-
|
|
4215
|
-
if (current?.active) {
|
|
4216
|
-
if (params.createReplacement && await this.isFreshLifecycleConversation(current)) {
|
|
4217
|
-
this.deps.log.info(
|
|
4218
|
-
`[lcm] ${params.reason} lifecycle no-op for already fresh conversation ${current.conversationId}`,
|
|
4219
|
-
);
|
|
4220
|
-
return;
|
|
4221
|
-
}
|
|
4222
|
-
await this.conversationStore.archiveConversation(current.conversationId);
|
|
4223
|
-
}
|
|
4224
|
-
|
|
4225
|
-
if (!params.createReplacement) {
|
|
4226
|
-
this.deps.log.info(
|
|
4227
|
-
`[lcm] ${params.reason} lifecycle archived conversation ${current?.conversationId ?? "(none)"}`,
|
|
4228
|
-
);
|
|
4229
|
-
return;
|
|
4230
|
-
}
|
|
4231
|
-
|
|
4232
|
-
const nextSessionId = params.nextSessionId?.trim() || params.sessionId?.trim() || current?.sessionId;
|
|
4233
|
-
if (!nextSessionId) {
|
|
4234
|
-
this.deps.log.warn(`[lcm] ${params.reason} lifecycle skipped: no session identity available`);
|
|
4235
|
-
return;
|
|
4236
|
-
}
|
|
4237
|
-
const nextSessionKey = params.nextSessionKey?.trim() || params.sessionKey?.trim() || current?.sessionKey;
|
|
4238
|
-
const freshConversation = await this.conversationStore.createConversation({
|
|
4239
|
-
sessionId: nextSessionId,
|
|
4240
|
-
...(nextSessionKey ? { sessionKey: nextSessionKey } : {}),
|
|
4241
|
-
});
|
|
4242
|
-
this.deps.log.info(
|
|
4243
|
-
`[lcm] ${params.reason} lifecycle archived prior conversation and created ${freshConversation.conversationId}`,
|
|
4244
|
-
);
|
|
4245
|
-
}
|
|
4246
|
-
|
|
4247
|
-
/** Apply LCM lifecycle semantics for OpenClaw's /new and /reset commands. */
|
|
4248
|
-
async handleBeforeReset(params: {
|
|
4249
|
-
reason?: string;
|
|
4250
|
-
sessionId?: string;
|
|
4251
|
-
sessionKey?: string;
|
|
4252
|
-
}): Promise<void> {
|
|
4253
|
-
const reason = params.reason?.trim();
|
|
4254
|
-
if (reason !== "new" && reason !== "reset") {
|
|
4255
|
-
return;
|
|
4256
|
-
}
|
|
4257
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
4258
|
-
return;
|
|
4259
|
-
}
|
|
4260
|
-
if (this.isStatelessSession(params.sessionKey)) {
|
|
4261
|
-
return;
|
|
4262
|
-
}
|
|
4263
|
-
|
|
4264
|
-
this.ensureMigrated();
|
|
4265
|
-
await this.withSessionQueue(
|
|
4266
|
-
this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
|
|
4267
|
-
async () =>
|
|
4268
|
-
this.conversationStore.withTransaction(async () => {
|
|
4269
|
-
if (reason === "new") {
|
|
4270
|
-
const conversation = await this.conversationStore.getConversationForSession({
|
|
4271
|
-
sessionId: params.sessionId,
|
|
4272
|
-
sessionKey: params.sessionKey,
|
|
4273
|
-
});
|
|
4274
|
-
if (!conversation) {
|
|
4275
|
-
return;
|
|
4276
|
-
}
|
|
4277
|
-
|
|
4278
|
-
const retainDepth =
|
|
4279
|
-
typeof this.config.newSessionRetainDepth === "number"
|
|
4280
|
-
&& Number.isFinite(this.config.newSessionRetainDepth)
|
|
4281
|
-
? this.config.newSessionRetainDepth
|
|
4282
|
-
: 2;
|
|
4283
|
-
await this.summaryStore.pruneForNewSession(conversation.conversationId, retainDepth);
|
|
4284
|
-
this.deps.log.info(
|
|
4285
|
-
`[lcm] /new pruned conversation ${conversation.conversationId} to retain depth ${retainDepth}`,
|
|
4286
|
-
);
|
|
4287
|
-
return;
|
|
4288
|
-
}
|
|
4289
|
-
await this.applySessionReplacement({
|
|
4290
|
-
reason: "/reset",
|
|
4291
|
-
sessionId: params.sessionId,
|
|
4292
|
-
sessionKey: params.sessionKey,
|
|
4293
|
-
createReplacement: true,
|
|
4294
|
-
createReplacementWhenMissing: true,
|
|
4295
|
-
});
|
|
4296
|
-
}),
|
|
4297
|
-
);
|
|
4298
|
-
}
|
|
4299
|
-
|
|
4300
|
-
/** Apply generic lifecycle semantics for session rollover and deletion hooks. */
|
|
4301
|
-
async handleSessionEnd(params: {
|
|
4302
|
-
reason?: string;
|
|
4303
|
-
sessionId?: string;
|
|
4304
|
-
sessionKey?: string;
|
|
4305
|
-
nextSessionId?: string;
|
|
4306
|
-
nextSessionKey?: string;
|
|
4307
|
-
}): Promise<void> {
|
|
4308
|
-
const reason = params.reason?.trim();
|
|
4309
|
-
if (!reason || reason === "new" || reason === "unknown") {
|
|
4310
|
-
return;
|
|
4311
|
-
}
|
|
4312
|
-
if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) {
|
|
4313
|
-
return;
|
|
4314
|
-
}
|
|
4315
|
-
if (this.isStatelessSession(params.sessionKey ?? params.nextSessionKey)) {
|
|
4316
|
-
return;
|
|
4317
|
-
}
|
|
4318
|
-
|
|
4319
|
-
const createReplacement = reason !== "deleted";
|
|
4320
|
-
this.ensureMigrated();
|
|
4321
|
-
await this.withSessionQueue(
|
|
4322
|
-
this.resolveSessionQueueKey(params.nextSessionId ?? params.sessionId, params.sessionKey ?? params.nextSessionKey),
|
|
4323
|
-
async () =>
|
|
4324
|
-
this.conversationStore.withTransaction(async () => {
|
|
4325
|
-
await this.applySessionReplacement({
|
|
4326
|
-
reason: `session_end:${reason}`,
|
|
4327
|
-
sessionId: params.sessionId,
|
|
4328
|
-
sessionKey: params.sessionKey ?? params.nextSessionKey,
|
|
4329
|
-
nextSessionId: params.nextSessionId,
|
|
4330
|
-
nextSessionKey: params.nextSessionKey,
|
|
4331
|
-
createReplacement,
|
|
4332
|
-
});
|
|
4333
|
-
}),
|
|
4334
|
-
);
|
|
4335
|
-
}
|
|
4336
|
-
|
|
4337
|
-
// ── Public accessors for retrieval (used by subagent expansion) ─────────
|
|
4338
|
-
|
|
4339
|
-
getRetrieval(): RetrievalEngine {
|
|
4340
|
-
return this.retrieval;
|
|
4341
|
-
}
|
|
4342
|
-
|
|
4343
|
-
getConversationStore(): ConversationStore {
|
|
4344
|
-
return this.conversationStore;
|
|
4345
|
-
}
|
|
4346
|
-
|
|
4347
|
-
getSummaryStore(): SummaryStore {
|
|
4348
|
-
return this.summaryStore;
|
|
4349
|
-
}
|
|
4350
|
-
|
|
4351
|
-
getCompactionTelemetryStore(): CompactionTelemetryStore {
|
|
4352
|
-
return this.compactionTelemetryStore;
|
|
4353
|
-
}
|
|
4354
|
-
|
|
4355
|
-
// ── Heartbeat pruning ──────────────────────────────────────────────────
|
|
4356
|
-
|
|
4357
|
-
/**
|
|
4358
|
-
* Detect HEARTBEAT_OK turn cycles in a conversation and delete them.
|
|
4359
|
-
*
|
|
4360
|
-
* A HEARTBEAT_OK turn is: a user message (the heartbeat prompt), followed by
|
|
4361
|
-
* any tool call/result messages, ending with an assistant message that is a
|
|
4362
|
-
* heartbeat ack. The entire sequence has no durable information value for LCM.
|
|
4363
|
-
*
|
|
4364
|
-
* Detection: assistant content (trimmed, lowercased) starts with "heartbeat_ok"
|
|
4365
|
-
* and any text after is not alphanumeric (matches OpenClaw core's ack detection).
|
|
4366
|
-
* This catches both exact "HEARTBEAT_OK" and chatty variants like
|
|
4367
|
-
* "HEARTBEAT_OK — weekend, no market".
|
|
4368
|
-
*
|
|
4369
|
-
* Returns the number of messages deleted.
|
|
4370
|
-
*/
|
|
4371
|
-
private async pruneHeartbeatOkTurns(conversationId: number): Promise<number> {
|
|
4372
|
-
const allMessages = await this.conversationStore.getMessages(conversationId);
|
|
4373
|
-
if (allMessages.length === 0) {
|
|
4374
|
-
return 0;
|
|
4375
|
-
}
|
|
4376
|
-
|
|
4377
|
-
const toDelete: number[] = [];
|
|
4378
|
-
|
|
4379
|
-
// Walk through messages finding HEARTBEAT_OK assistant replies, then
|
|
4380
|
-
// collect the entire turn (back to the preceding user message).
|
|
4381
|
-
for (let i = 0; i < allMessages.length; i++) {
|
|
4382
|
-
const msg = allMessages[i];
|
|
4383
|
-
if (msg.role !== "assistant") {
|
|
4384
|
-
continue;
|
|
4385
|
-
}
|
|
4386
|
-
if (!isHeartbeatOkContent(msg.content)) {
|
|
4387
|
-
continue;
|
|
4388
|
-
}
|
|
4389
|
-
|
|
4390
|
-
// Found an exact HEARTBEAT_OK reply. Walk backward to find the turn start
|
|
4391
|
-
// (the preceding user message).
|
|
4392
|
-
const turnMessages = [msg];
|
|
4393
|
-
for (let j = i - 1; j >= 0; j--) {
|
|
4394
|
-
const prev = allMessages[j];
|
|
4395
|
-
turnMessages.push(prev);
|
|
4396
|
-
if (prev.role === "user") {
|
|
4397
|
-
break; // Found turn start
|
|
4398
|
-
}
|
|
4399
|
-
}
|
|
4400
|
-
|
|
4401
|
-
if (!turnMessages.some((record) => record.role === "user")) {
|
|
4402
|
-
continue;
|
|
4403
|
-
}
|
|
4404
|
-
if (!turnLooksLikeHeartbeatTurn(turnMessages)) {
|
|
4405
|
-
continue;
|
|
4406
|
-
}
|
|
4407
|
-
|
|
4408
|
-
toDelete.push(...turnMessages.map((record) => record.messageId));
|
|
4409
|
-
}
|
|
4410
|
-
|
|
4411
|
-
if (toDelete.length === 0) {
|
|
4412
|
-
return 0;
|
|
4413
|
-
}
|
|
4414
|
-
|
|
4415
|
-
// Deduplicate (a message could theoretically appear in multiple turns)
|
|
4416
|
-
const uniqueIds = [...new Set(toDelete)];
|
|
4417
|
-
return this.conversationStore.deleteMessages(uniqueIds);
|
|
4418
|
-
}
|
|
4419
|
-
}
|
|
4420
|
-
|
|
4421
|
-
// ── Heartbeat detection ─────────────────────────────────────────────────────
|
|
4422
|
-
|
|
4423
|
-
const HEARTBEAT_OK_TOKEN = "heartbeat_ok";
|
|
4424
|
-
const HEARTBEAT_TURN_MARKER = "heartbeat.md";
|
|
4425
|
-
|
|
4426
|
-
/**
|
|
4427
|
-
* Detect whether an assistant message is a heartbeat ack.
|
|
4428
|
-
*
|
|
4429
|
-
* Only exact (case-insensitive) "HEARTBEAT_OK" acknowledgements are pruned.
|
|
4430
|
-
* Any additional text indicates the heartbeat carried real content and should remain.
|
|
4431
|
-
*/
|
|
4432
|
-
function isHeartbeatOkContent(content: string): boolean {
|
|
4433
|
-
return content.trim().toLowerCase() === HEARTBEAT_OK_TOKEN;
|
|
4434
|
-
}
|
|
4435
|
-
|
|
4436
|
-
function batchLooksLikeHeartbeatAckTurn(messages: AgentMessage[]): boolean {
|
|
4437
|
-
let sawHeartbeatMarker = false;
|
|
4438
|
-
let sawHeartbeatAck = false;
|
|
4439
|
-
|
|
4440
|
-
for (const message of messages) {
|
|
4441
|
-
const stored = toStoredMessage(message);
|
|
4442
|
-
if (!sawHeartbeatMarker && stored.content.toLowerCase().includes(HEARTBEAT_TURN_MARKER)) {
|
|
4443
|
-
sawHeartbeatMarker = true;
|
|
4444
|
-
}
|
|
4445
|
-
if (!sawHeartbeatAck && stored.role === "assistant" && isHeartbeatOkContent(stored.content)) {
|
|
4446
|
-
sawHeartbeatAck = true;
|
|
4447
|
-
}
|
|
4448
|
-
if (sawHeartbeatMarker && sawHeartbeatAck) {
|
|
4449
|
-
return true;
|
|
4450
|
-
}
|
|
4451
|
-
}
|
|
4452
|
-
|
|
4453
|
-
return false;
|
|
4454
|
-
}
|
|
4455
|
-
|
|
4456
|
-
function turnLooksLikeHeartbeatTurn(turnMessages: Array<{ content: string }>): boolean {
|
|
4457
|
-
return turnMessages.some((message) =>
|
|
4458
|
-
message.content.toLowerCase().includes(HEARTBEAT_TURN_MARKER),
|
|
4459
|
-
);
|
|
4460
|
-
}
|
|
4461
|
-
|
|
4462
|
-
// ── Emergency fallback summarization ────────────────────────────────────────
|
|
4463
|
-
|
|
4464
|
-
/**
|
|
4465
|
-
* Creates a deterministic truncation summarizer used only as an emergency
|
|
4466
|
-
* fallback when the model-backed summarizer cannot be created.
|
|
4467
|
-
*
|
|
4468
|
-
* CompactionEngine already escalates normal -> aggressive -> fallback for
|
|
4469
|
-
* convergence. This function simply provides a stable baseline summarize
|
|
4470
|
-
* callback to keep compaction operable when runtime setup is unavailable.
|
|
4471
|
-
*/
|
|
4472
|
-
function createEmergencyFallbackSummarize(): (
|
|
4473
|
-
text: string,
|
|
4474
|
-
aggressive?: boolean,
|
|
4475
|
-
) => Promise<string> {
|
|
4476
|
-
return async (text: string, aggressive?: boolean): Promise<string> => {
|
|
4477
|
-
const maxChars = aggressive ? 600 * 4 : 900 * 4;
|
|
4478
|
-
if (text.length <= maxChars) {
|
|
4479
|
-
return text;
|
|
4480
|
-
}
|
|
4481
|
-
return text.slice(0, maxChars) + "\n[Truncated for context management]";
|
|
4482
|
-
};
|
|
4483
|
-
}
|
|
4484
|
-
|
|
4485
|
-
/** @internal Exposed for unit tests only. */
|
|
4486
|
-
export const __testing = { readLastJsonlEntryBeforeOffset };
|