@martian-engineering/lossless-claw 0.8.0 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +8 -0
  2. package/dist/index.js +971 -0
  3. package/docs/configuration.md +15 -5
  4. package/openclaw.plugin.json +27 -3
  5. package/package.json +7 -6
  6. package/skills/lossless-claw/references/config.md +37 -0
  7. package/index.ts +0 -2
  8. package/src/assembler.ts +0 -1196
  9. package/src/compaction.ts +0 -1753
  10. package/src/db/config.ts +0 -345
  11. package/src/db/connection.ts +0 -151
  12. package/src/db/features.ts +0 -61
  13. package/src/db/migration.ts +0 -868
  14. package/src/engine.ts +0 -4486
  15. package/src/estimate-tokens.ts +0 -80
  16. package/src/expansion-auth.ts +0 -365
  17. package/src/expansion-policy.ts +0 -303
  18. package/src/expansion.ts +0 -383
  19. package/src/integrity.ts +0 -600
  20. package/src/large-files.ts +0 -546
  21. package/src/lcm-log.ts +0 -37
  22. package/src/openclaw-bridge.ts +0 -22
  23. package/src/plugin/index.ts +0 -2037
  24. package/src/plugin/lcm-command.ts +0 -1040
  25. package/src/plugin/lcm-doctor-apply.ts +0 -540
  26. package/src/plugin/lcm-doctor-cleaners.ts +0 -655
  27. package/src/plugin/lcm-doctor-shared.ts +0 -210
  28. package/src/plugin/shared-init.ts +0 -59
  29. package/src/prune.ts +0 -391
  30. package/src/retrieval.ts +0 -360
  31. package/src/session-patterns.ts +0 -23
  32. package/src/startup-banner-log.ts +0 -49
  33. package/src/store/compaction-telemetry-store.ts +0 -156
  34. package/src/store/conversation-store.ts +0 -929
  35. package/src/store/fts5-sanitize.ts +0 -50
  36. package/src/store/full-text-fallback.ts +0 -83
  37. package/src/store/full-text-sort.ts +0 -21
  38. package/src/store/index.ts +0 -39
  39. package/src/store/parse-utc-timestamp.ts +0 -25
  40. package/src/store/summary-store.ts +0 -1519
  41. package/src/summarize.ts +0 -1508
  42. package/src/tools/common.ts +0 -53
  43. package/src/tools/lcm-conversation-scope.ts +0 -127
  44. package/src/tools/lcm-describe-tool.ts +0 -245
  45. package/src/tools/lcm-expand-query-tool.ts +0 -1235
  46. package/src/tools/lcm-expand-tool.delegation.ts +0 -580
  47. package/src/tools/lcm-expand-tool.ts +0 -453
  48. package/src/tools/lcm-expansion-recursion-guard.ts +0 -373
  49. package/src/tools/lcm-grep-tool.ts +0 -228
  50. package/src/transaction-mutex.ts +0 -136
  51. package/src/transcript-repair.ts +0 -301
  52. package/src/types.ts +0 -165
package/src/compaction.ts DELETED
@@ -1,1753 +0,0 @@
1
- import { createHash } from "node:crypto";
2
- import type { ConversationStore, CreateMessagePartInput } from "./store/conversation-store.js";
3
- import type { SummaryStore, SummaryRecord, ContextItemRecord } from "./store/summary-store.js";
4
- import { estimateTokens, truncateTextToEstimatedTokens } from "./estimate-tokens.js";
5
- import { extractFileIdsFromContent } from "./large-files.js";
6
- import { NOOP_LCM_LOGGER, type LcmLogger } from "./lcm-log.js";
7
- import { LcmProviderAuthError } from "./summarize.js";
8
-
9
- // ── Public types ─────────────────────────────────────────────────────────────
10
-
11
- export interface CompactionDecision {
12
- shouldCompact: boolean;
13
- reason: "threshold" | "manual" | "none";
14
- currentTokens: number;
15
- threshold: number;
16
- }
17
-
18
- export interface CompactionResult {
19
- actionTaken: boolean;
20
- /** Tokens before compaction */
21
- tokensBefore: number;
22
- /** Tokens after compaction */
23
- tokensAfter: number;
24
- /** Summary created (if any) */
25
- createdSummaryId?: string;
26
- /** Whether condensation was performed */
27
- condensed: boolean;
28
- /** Escalation level used: "normal" | "aggressive" | "fallback" */
29
- level?: CompactionLevel;
30
- /** Whether compaction was blocked by a provider auth failure */
31
- authFailure?: boolean;
32
- }
33
-
34
- export interface CompactionConfig {
35
- /** Context threshold as fraction of budget (default 0.75) */
36
- contextThreshold: number;
37
- /** Number of fresh tail turns to protect (default 8) */
38
- freshTailCount: number;
39
- /** Minimum number of depth-0 summaries needed for condensation. */
40
- leafMinFanout: number;
41
- /** Minimum number of depth>=1 summaries needed for condensation. */
42
- condensedMinFanout: number;
43
- /** Relaxed minimum fanout for hard-trigger sweeps. */
44
- condensedMinFanoutHard: number;
45
- /** Incremental depth passes to run after each leaf compaction (default 1). */
46
- incrementalMaxDepth: number;
47
- /** Max source tokens to compact per leaf/condensed chunk (default 20000) */
48
- leafChunkTokens?: number;
49
- /** Target tokens for leaf summaries (default 600) */
50
- leafTargetTokens: number;
51
- /** Target tokens for condensed summaries (default 900) */
52
- condensedTargetTokens: number;
53
- /** Maximum compaction rounds (default 10) */
54
- maxRounds: number;
55
- /** IANA timezone for timestamps in summaries (default: UTC) */
56
- timezone?: string;
57
- /** Maximum allowed overage factor for summaries relative to target tokens (default 3). */
58
- summaryMaxOverageFactor: number;
59
- }
60
-
61
- type CompactionLevel = "normal" | "aggressive" | "fallback" | "capped";
62
- type CompactionPass = "leaf" | "condensed";
63
- type CompactionSummarizeOptions = {
64
- previousSummary?: string;
65
- isCondensed?: boolean;
66
- depth?: number;
67
- };
68
- type CompactionSummarizeFn = (
69
- text: string,
70
- aggressive?: boolean,
71
- options?: CompactionSummarizeOptions,
72
- ) => Promise<string>;
73
- type PassResult = {
74
- summaryId: string;
75
- level: CompactionLevel;
76
- /** Token count of source items removed from context. */
77
- removedTokens: number;
78
- /** Token count of the newly created summary. */
79
- addedTokens: number;
80
- };
81
- type LeafChunkSelection = {
82
- items: ContextItemRecord[];
83
- rawTokensOutsideTail: number;
84
- threshold: number;
85
- };
86
- type CondensedChunkSelection = {
87
- items: ContextItemRecord[];
88
- summaryTokens: number;
89
- };
90
- type CondensedPhaseCandidate = {
91
- targetDepth: number;
92
- chunk: CondensedChunkSelection;
93
- };
94
-
95
- // ── Helpers ──────────────────────────────────────────────────────────────────
96
-
97
-
98
- /** Deterministically cap summary text so the persisted output stays within maxTokens. */
99
- function capSummaryText(
100
- content: string,
101
- originalTokens: number,
102
- maxTokens: number,
103
- ): string {
104
- const suffixes = [
105
- `\n[Capped from ${originalTokens} tokens to ~${maxTokens}]`,
106
- `\n[Capped to ~${maxTokens}]`,
107
- "\n[Capped]",
108
- "",
109
- ];
110
-
111
- for (const suffix of suffixes) {
112
- const contentBudget = Math.max(0, maxTokens - estimateTokens(suffix));
113
- const capped = `${truncateTextToEstimatedTokens(content, contentBudget)}${suffix}`;
114
- if (estimateTokens(capped) <= maxTokens) {
115
- return capped;
116
- }
117
- }
118
-
119
- return truncateTextToEstimatedTokens(content, maxTokens);
120
- }
121
-
122
- /** Format a timestamp as `YYYY-MM-DD HH:mm TZ` for prompt source text. */
123
- export function formatTimestamp(value: Date, timezone: string = "UTC"): string {
124
- try {
125
- const fmt = new Intl.DateTimeFormat("en-CA", {
126
- timeZone: timezone,
127
- year: "numeric",
128
- month: "2-digit",
129
- day: "2-digit",
130
- hour: "2-digit",
131
- minute: "2-digit",
132
- hour12: false,
133
- });
134
- const parts = Object.fromEntries(
135
- fmt.formatToParts(value).map((p) => [p.type, p.value]),
136
- );
137
- const tzAbbr = timezone === "UTC" ? "UTC" : shortTzAbbr(value, timezone);
138
- return `${parts.year}-${parts.month}-${parts.day} ${parts.hour}:${parts.minute} ${tzAbbr}`;
139
- } catch {
140
- // Fallback to UTC on invalid timezone
141
- const year = value.getUTCFullYear();
142
- const month = String(value.getUTCMonth() + 1).padStart(2, "0");
143
- const day = String(value.getUTCDate()).padStart(2, "0");
144
- const hours = String(value.getUTCHours()).padStart(2, "0");
145
- const minutes = String(value.getUTCMinutes()).padStart(2, "0");
146
- return `${year}-${month}-${day} ${hours}:${minutes} UTC`;
147
- }
148
- }
149
-
150
- /** Extract short timezone abbreviation (e.g. "PST", "PDT", "EST"). */
151
- function shortTzAbbr(value: Date, timezone: string): string {
152
- try {
153
- const abbr = new Intl.DateTimeFormat("en-US", {
154
- timeZone: timezone,
155
- timeZoneName: "short",
156
- })
157
- .formatToParts(value)
158
- .find((p) => p.type === "timeZoneName")?.value;
159
- return abbr ?? timezone;
160
- } catch {
161
- return timezone;
162
- }
163
- }
164
-
165
- /** Generate a deterministic summary ID from content + timestamp. */
166
- function generateSummaryId(content: string): string {
167
- return (
168
- "sum_" +
169
- createHash("sha256")
170
- .update(content + Date.now().toString())
171
- .digest("hex")
172
- .slice(0, 16)
173
- );
174
- }
175
-
176
- /** Maximum estimated tokens for the deterministic fallback truncation. */
177
- const FALLBACK_MAX_TOKENS = 512;
178
- const DEFAULT_LEAF_CHUNK_TOKENS = 20_000;
179
-
180
- /**
181
- * Pattern matching MEDIA:/... file path references that appear in message content
182
- * when the original message contained only a media attachment (image, file, etc.)
183
- * with no meaningful text.
184
- */
185
- const MEDIA_PATH_RE = /^MEDIA:\/.+$/;
186
- const EMBEDDED_DATA_URL_RE = /data:[^;\s"'`]+;base64,[A-Za-z0-9+/=\s]+/gi;
187
- const MEDIA_ATTACHMENT_PART_TYPES = new Set(["file", "snapshot"]);
188
- const MEDIA_ATTACHMENT_RAW_TYPES = new Set(["file", "image", "snapshot"]);
189
- const STRUCTURED_MEDIA_TEXT_KEYS = ["text", "caption", "alt", "title", "summary"] as const;
190
- const STRUCTURED_MEDIA_NESTED_KEYS = ["content", "parts", "items", "message", "messages"] as const;
191
-
192
- const CONDENSED_MIN_INPUT_RATIO = 0.1;
193
-
194
- function dedupeOrderedIds(ids: Iterable<string>): string[] {
195
- const seen = new Set<string>();
196
- const ordered: string[] = [];
197
- for (const id of ids) {
198
- if (!seen.has(id)) {
199
- seen.add(id);
200
- ordered.push(id);
201
- }
202
- }
203
- return ordered;
204
- }
205
-
206
- /** Parse message-part metadata without throwing on malformed JSON. */
207
- function parseMessagePartMetadata(part: CreateMessagePartInput | { metadata: string | null }): Record<string, unknown> {
208
- if (typeof part.metadata !== "string" || !part.metadata.trim()) {
209
- return {};
210
- }
211
- try {
212
- const parsed = JSON.parse(part.metadata) as unknown;
213
- return parsed && typeof parsed === "object" && !Array.isArray(parsed)
214
- ? (parsed as Record<string, unknown>)
215
- : {};
216
- } catch {
217
- return {};
218
- }
219
- }
220
-
221
- /** Detect whether a string is mostly binary/base64 payload and not meaningful prose. */
222
- function looksLikeBinaryPayload(value: string): boolean {
223
- const trimmed = value.trim();
224
- if (!trimmed) {
225
- return false;
226
- }
227
- if (/^data:[^;\s"'`]+;base64,/i.test(trimmed)) {
228
- return true;
229
- }
230
- const compact = trimmed.replace(/\s+/g, "");
231
- if (compact.length < 256 || compact.length % 4 !== 0) {
232
- return false;
233
- }
234
- if (!/^[A-Za-z0-9+/=]+$/.test(compact)) {
235
- return false;
236
- }
237
- return !/[ .,:;!?()[\]{}]/.test(trimmed);
238
- }
239
-
240
- /** Strip attachment payloads from plain strings before they reach the summarizer. */
241
- function stripEmbeddedMediaPayloads(content: string): string {
242
- const withoutDataUrls = content.replace(EMBEDDED_DATA_URL_RE, "[embedded media omitted]");
243
- const sanitizedLines = withoutDataUrls
244
- .split(/\r?\n/)
245
- .map((line) => line.trimEnd())
246
- .filter((line) => {
247
- const trimmed = line.trim();
248
- if (!trimmed) {
249
- return false;
250
- }
251
- if (MEDIA_PATH_RE.test(trimmed)) {
252
- return false;
253
- }
254
- if (looksLikeBinaryPayload(trimmed)) {
255
- return false;
256
- }
257
- return true;
258
- });
259
- return sanitizedLines.join("\n").trim();
260
- }
261
-
262
- /** Extract human-readable text from structured content while ignoring attachment payload fields. */
263
- function extractSanitizedStructuredText(value: unknown, depth = 0): string[] {
264
- if (depth >= 4 || value == null) {
265
- return [];
266
- }
267
- if (typeof value === "string") {
268
- const sanitized = stripEmbeddedMediaPayloads(value);
269
- return sanitized ? [sanitized] : [];
270
- }
271
- if (Array.isArray(value)) {
272
- return value.flatMap((entry) => extractSanitizedStructuredText(entry, depth + 1));
273
- }
274
- if (typeof value !== "object") {
275
- return [];
276
- }
277
-
278
- const record = value as Record<string, unknown>;
279
- const rawType = typeof record.type === "string" ? record.type.trim().toLowerCase() : "";
280
- const textFragments: string[] = [];
281
-
282
- for (const key of STRUCTURED_MEDIA_TEXT_KEYS) {
283
- const candidate = record[key];
284
- if (typeof candidate !== "string") {
285
- continue;
286
- }
287
- const sanitized = stripEmbeddedMediaPayloads(candidate);
288
- if (sanitized) {
289
- textFragments.push(sanitized);
290
- }
291
- }
292
-
293
- if (MEDIA_ATTACHMENT_RAW_TYPES.has(rawType)) {
294
- return textFragments;
295
- }
296
-
297
- for (const key of STRUCTURED_MEDIA_NESTED_KEYS) {
298
- textFragments.push(...extractSanitizedStructuredText(record[key], depth + 1));
299
- }
300
-
301
- return textFragments;
302
- }
303
-
304
- /** Normalize message content down to human-readable text, excluding binary/media payloads. */
305
- function extractMeaningfulMessageText(content: string): string {
306
- const trimmed = content.trim();
307
- if (!trimmed) {
308
- return "";
309
- }
310
- if ((trimmed.startsWith("[") && trimmed.endsWith("]")) || (trimmed.startsWith("{") && trimmed.endsWith("}"))) {
311
- try {
312
- const parsed = JSON.parse(trimmed) as unknown;
313
- const extracted = extractSanitizedStructuredText(parsed)
314
- .map((fragment) => fragment.trim())
315
- .filter(Boolean);
316
- return extracted.join("\n").trim();
317
- } catch {
318
- // Fall back to plain-text sanitation below.
319
- }
320
- }
321
- return stripEmbeddedMediaPayloads(content);
322
- }
323
-
324
- /** Identify whether a stored message part represents a media attachment. */
325
- function isMediaAttachmentPart(part: CreateMessagePartInput | { partType: string; metadata: string | null }): boolean {
326
- if (MEDIA_ATTACHMENT_PART_TYPES.has(part.partType)) {
327
- return true;
328
- }
329
- const metadata = parseMessagePartMetadata(part);
330
- const rawType =
331
- typeof metadata.rawType === "string"
332
- ? metadata.rawType.trim().toLowerCase()
333
- : metadata.raw && typeof metadata.raw === "object" && !Array.isArray(metadata.raw) &&
334
- typeof (metadata.raw as Record<string, unknown>).type === "string"
335
- ? ((metadata.raw as Record<string, unknown>).type as string).trim().toLowerCase()
336
- : "";
337
- return MEDIA_ATTACHMENT_RAW_TYPES.has(rawType);
338
- }
339
-
340
- // ── CompactionEngine ─────────────────────────────────────────────────────────
341
-
342
- export class CompactionEngine {
343
- /**
344
- * Per-conversation context items cache, active only during compaction
345
- * entry points. null when inactive — external callers (e.g., engine.ts
346
- * evaluateLeafTrigger) get uncached reads.
347
- *
348
- * Uses a reference count so concurrent compactions on different
349
- * conversations don't interfere: each withContextCache increments
350
- * on entry and decrements on exit; the cache is only destroyed
351
- * when all users have exited.
352
- */
353
- private _contextItemsCache: Map<number, ContextItemRecord[]> | null = null;
354
- private _contextItemsCacheRefCount = 0;
355
-
356
- constructor(
357
- private conversationStore: ConversationStore,
358
- private summaryStore: SummaryStore,
359
- private config: CompactionConfig,
360
- private log: LcmLogger = NOOP_LCM_LOGGER,
361
- ) {}
362
-
363
- /** Read context items, using per-phase cache when active. */
364
- private async getContextItemsCached(conversationId: number): Promise<ContextItemRecord[]> {
365
- if (this._contextItemsCache) {
366
- if (this._contextItemsCache.has(conversationId)) {
367
- return this._contextItemsCache.get(conversationId)!;
368
- }
369
- const items = await this.summaryStore.getContextItems(conversationId);
370
- this._contextItemsCache.set(conversationId, items);
371
- return items;
372
- }
373
- return this.summaryStore.getContextItems(conversationId);
374
- }
375
-
376
- /** Invalidate cache for a conversation after context mutation. */
377
- private invalidateContextCache(conversationId: number): void {
378
- this._contextItemsCache?.delete(conversationId);
379
- }
380
-
381
- /** Execute with context cache active. Reference-counted for concurrent use. */
382
- private async withContextCache<T>(fn: () => Promise<T>): Promise<T> {
383
- if (!this._contextItemsCache) this._contextItemsCache = new Map();
384
- this._contextItemsCacheRefCount++;
385
- try {
386
- return await fn();
387
- } finally {
388
- this._contextItemsCacheRefCount--;
389
- if (this._contextItemsCacheRefCount <= 0) {
390
- this._contextItemsCache = null;
391
- this._contextItemsCacheRefCount = 0;
392
- }
393
- }
394
- }
395
-
396
- // ── evaluate ─────────────────────────────────────────────────────────────
397
-
398
- /** Evaluate whether compaction is needed. */
399
- async evaluate(
400
- conversationId: number,
401
- tokenBudget: number,
402
- observedTokenCount?: number,
403
- ): Promise<CompactionDecision> {
404
- const storedTokens = await this.summaryStore.getContextTokenCount(conversationId);
405
- const liveTokens =
406
- typeof observedTokenCount === "number" &&
407
- Number.isFinite(observedTokenCount) &&
408
- observedTokenCount > 0
409
- ? Math.floor(observedTokenCount)
410
- : 0;
411
- const currentTokens = Math.max(storedTokens, liveTokens);
412
- const threshold = Math.floor(this.config.contextThreshold * tokenBudget);
413
-
414
- if (currentTokens > threshold) {
415
- return {
416
- shouldCompact: true,
417
- reason: "threshold",
418
- currentTokens,
419
- threshold,
420
- };
421
- }
422
-
423
- return {
424
- shouldCompact: false,
425
- reason: "none",
426
- currentTokens,
427
- threshold,
428
- };
429
- }
430
-
431
- /**
432
- * Evaluate whether the raw-message leaf trigger is active.
433
- *
434
- * Counts message tokens outside the protected fresh tail and compares against
435
- * `leafChunkTokens`. This lets callers trigger a soft incremental leaf pass
436
- * before the full context threshold is breached.
437
- */
438
- async evaluateLeafTrigger(conversationId: number, leafChunkTokensOverride?: number): Promise<{
439
- shouldCompact: boolean;
440
- rawTokensOutsideTail: number;
441
- threshold: number;
442
- }> {
443
- const rawTokensOutsideTail = await this.countRawTokensOutsideFreshTail(conversationId);
444
- const threshold = this.resolveLeafChunkTokens(leafChunkTokensOverride);
445
- return {
446
- shouldCompact: rawTokensOutsideTail >= threshold,
447
- rawTokensOutsideTail,
448
- threshold,
449
- };
450
- }
451
-
452
- // ── compact ──────────────────────────────────────────────────────────────
453
-
454
- /** Run a full compaction sweep for a conversation. */
455
- async compact(input: {
456
- conversationId: number;
457
- tokenBudget: number;
458
- /** LLM call function for summarization */
459
- summarize: CompactionSummarizeFn;
460
- force?: boolean;
461
- hardTrigger?: boolean;
462
- summaryModel?: string;
463
- }): Promise<CompactionResult> {
464
- return this.withContextCache(() => this.compactFullSweep(input));
465
- }
466
-
467
- /**
468
- * Run a single leaf pass against the oldest compactable raw chunk.
469
- *
470
- * This is the soft-trigger path used for incremental maintenance.
471
- */
472
- async compactLeaf(input: {
473
- conversationId: number;
474
- tokenBudget: number;
475
- summarize: CompactionSummarizeFn;
476
- leafChunkTokens?: number;
477
- force?: boolean;
478
- previousSummaryContent?: string;
479
- summaryModel?: string;
480
- allowCondensedPasses?: boolean;
481
- }): Promise<CompactionResult> {
482
- return this.withContextCache(() => this._compactLeafImpl(input));
483
- }
484
-
485
- private async _compactLeafImpl(input: {
486
- conversationId: number;
487
- tokenBudget: number;
488
- summarize: CompactionSummarizeFn;
489
- leafChunkTokens?: number;
490
- force?: boolean;
491
- previousSummaryContent?: string;
492
- summaryModel?: string;
493
- }): Promise<CompactionResult> {
494
- const { conversationId, tokenBudget, summarize, force } = input;
495
-
496
- const tokensBefore = await this.summaryStore.getContextTokenCount(conversationId);
497
- const threshold = Math.floor(this.config.contextThreshold * tokenBudget);
498
- const leafTrigger = await this.evaluateLeafTrigger(conversationId, input.leafChunkTokens);
499
-
500
- if (!force && tokensBefore <= threshold && !leafTrigger.shouldCompact) {
501
- return {
502
- actionTaken: false,
503
- tokensBefore,
504
- tokensAfter: tokensBefore,
505
- condensed: false,
506
- };
507
- }
508
-
509
- const leafChunk = await this.selectOldestLeafChunk(conversationId, input.leafChunkTokens);
510
- if (leafChunk.items.length === 0) {
511
- return {
512
- actionTaken: false,
513
- tokensBefore,
514
- tokensAfter: tokensBefore,
515
- condensed: false,
516
- };
517
- }
518
-
519
- const previousSummaryContent =
520
- input.previousSummaryContent ??
521
- (await this.resolvePriorLeafSummaryContext(conversationId, leafChunk.items));
522
-
523
- const leafResult = await this.leafPass(
524
- conversationId,
525
- leafChunk.items,
526
- summarize,
527
- previousSummaryContent,
528
- input.summaryModel,
529
- );
530
- if (!leafResult) {
531
- return {
532
- actionTaken: false,
533
- tokensBefore,
534
- tokensAfter: tokensBefore,
535
- condensed: false,
536
- authFailure: true,
537
- };
538
- }
539
- // Delta tracking: compute token change from pass results instead of re-querying DB
540
- const tokensAfterLeaf = tokensBefore - leafResult.removedTokens + leafResult.addedTokens;
541
-
542
- await this.persistCompactionEvents({
543
- conversationId,
544
- tokensBefore,
545
- tokensAfterLeaf,
546
- tokensAfterFinal: tokensAfterLeaf,
547
- leafResult: { summaryId: leafResult.summaryId, level: leafResult.level },
548
- condenseResult: null,
549
- });
550
-
551
- let tokensAfter = tokensAfterLeaf;
552
- let condensed = false;
553
- let createdSummaryId = leafResult.summaryId;
554
- let level = leafResult.level;
555
-
556
- const incrementalMaxDepth = this.resolveIncrementalMaxDepth();
557
- const condensedMinChunkTokens = this.resolveCondensedMinChunkTokens();
558
- let runningTokens = tokensAfterLeaf;
559
- if (incrementalMaxDepth > 0 && input.allowCondensedPasses !== false) {
560
- for (let targetDepth = 0; targetDepth < incrementalMaxDepth; targetDepth++) {
561
- const fanout = this.resolveFanoutForDepth(targetDepth, false);
562
- const chunk = await this.selectOldestChunkAtDepth(conversationId, targetDepth);
563
- if (chunk.items.length < fanout || chunk.summaryTokens < condensedMinChunkTokens) {
564
- break;
565
- }
566
-
567
- const passTokensBefore = runningTokens;
568
- const condenseResult = await this.condensedPass(
569
- conversationId,
570
- chunk.items,
571
- targetDepth,
572
- summarize,
573
- input.summaryModel,
574
- );
575
- if (!condenseResult) {
576
- break;
577
- }
578
- const passTokensAfter = passTokensBefore - condenseResult.removedTokens + condenseResult.addedTokens;
579
- await this.persistCompactionEvents({
580
- conversationId,
581
- tokensBefore: passTokensBefore,
582
- tokensAfterLeaf: passTokensBefore,
583
- tokensAfterFinal: passTokensAfter,
584
- leafResult: null,
585
- condenseResult,
586
- });
587
-
588
- tokensAfter = passTokensAfter;
589
- runningTokens = passTokensAfter;
590
- condensed = true;
591
- createdSummaryId = condenseResult.summaryId;
592
- level = condenseResult.level;
593
-
594
- if (passTokensAfter >= passTokensBefore) {
595
- break;
596
- }
597
- }
598
- }
599
-
600
- return {
601
- actionTaken: true,
602
- tokensBefore,
603
- tokensAfter,
604
- createdSummaryId,
605
- condensed,
606
- level,
607
- };
608
- }
609
-
610
- /**
611
- * Run a hard-trigger sweep:
612
- *
613
- * Phase 1: repeatedly compact raw-message chunks outside the fresh tail.
614
- * Phase 2: repeatedly condense oldest summary chunks while chunk utilization
615
- * remains high enough to be worthwhile.
616
- */
617
- async compactFullSweep(input: {
618
- conversationId: number;
619
- tokenBudget: number;
620
- summarize: CompactionSummarizeFn;
621
- force?: boolean;
622
- hardTrigger?: boolean;
623
- summaryModel?: string;
624
- }): Promise<CompactionResult> {
625
- const { conversationId, tokenBudget, summarize, force, hardTrigger } = input;
626
-
627
- const tokensBefore = await this.summaryStore.getContextTokenCount(conversationId);
628
- const threshold = Math.floor(this.config.contextThreshold * tokenBudget);
629
- const leafTrigger = await this.evaluateLeafTrigger(conversationId);
630
-
631
- if (!force && tokensBefore <= threshold && !leafTrigger.shouldCompact) {
632
- return {
633
- actionTaken: false,
634
- tokensBefore,
635
- tokensAfter: tokensBefore,
636
- condensed: false,
637
- };
638
- }
639
-
640
- const contextItems = await this.getContextItemsCached(conversationId);
641
- if (contextItems.length === 0) {
642
- return {
643
- actionTaken: false,
644
- tokensBefore,
645
- tokensAfter: tokensBefore,
646
- condensed: false,
647
- };
648
- }
649
-
650
- let actionTaken = false;
651
- let condensed = false;
652
- let createdSummaryId: string | undefined;
653
- let level: CompactionLevel | undefined;
654
- let previousSummaryContent: string | undefined;
655
- let previousTokens = tokensBefore;
656
- let hadAuthFailure = false;
657
-
658
- // Phase 1: leaf passes over oldest raw chunks outside the protected tail.
659
- // Delta tracking: maintain a running token count instead of re-querying DB
660
- // after each pass. The arithmetic is exact: tokensAfter = tokensBefore - removed + added.
661
- let runningTokens = tokensBefore;
662
- while (true) {
663
- const leafChunk = await this.selectOldestLeafChunk(conversationId);
664
- if (leafChunk.items.length === 0) {
665
- break;
666
- }
667
-
668
- const passTokensBefore = runningTokens;
669
- const leafResult = await this.leafPass(
670
- conversationId,
671
- leafChunk.items,
672
- summarize,
673
- previousSummaryContent,
674
- input.summaryModel,
675
- );
676
- if (!leafResult) {
677
- hadAuthFailure = true;
678
- break;
679
- }
680
- const passTokensAfter = passTokensBefore - leafResult.removedTokens + leafResult.addedTokens;
681
- await this.persistCompactionEvents({
682
- conversationId,
683
- tokensBefore: passTokensBefore,
684
- tokensAfterLeaf: passTokensAfter,
685
- tokensAfterFinal: passTokensAfter,
686
- leafResult: { summaryId: leafResult.summaryId, level: leafResult.level },
687
- condenseResult: null,
688
- });
689
-
690
- actionTaken = true;
691
- createdSummaryId = leafResult.summaryId;
692
- level = leafResult.level;
693
- previousSummaryContent = leafResult.content;
694
- runningTokens = passTokensAfter;
695
-
696
- if (!force && passTokensAfter <= threshold) {
697
- previousTokens = passTokensAfter;
698
- break;
699
- }
700
- if (passTokensAfter >= passTokensBefore || passTokensAfter >= previousTokens) {
701
- break;
702
- }
703
- previousTokens = passTokensAfter;
704
- }
705
-
706
- // Phase 2: depth-aware condensed passes, always processing shallowest depth first.
707
- while (force || previousTokens > threshold) {
708
- const candidate = await this.selectShallowestCondensationCandidate({
709
- conversationId,
710
- hardTrigger: hardTrigger === true,
711
- });
712
- if (!candidate) {
713
- break;
714
- }
715
-
716
- const passTokensBefore = runningTokens;
717
- const condenseResult = await this.condensedPass(
718
- conversationId,
719
- candidate.chunk.items,
720
- candidate.targetDepth,
721
- summarize,
722
- input.summaryModel,
723
- );
724
- if (!condenseResult) {
725
- hadAuthFailure = true;
726
- break;
727
- }
728
- const passTokensAfter = passTokensBefore - condenseResult.removedTokens + condenseResult.addedTokens;
729
- await this.persistCompactionEvents({
730
- conversationId,
731
- tokensBefore: passTokensBefore,
732
- tokensAfterLeaf: passTokensBefore,
733
- tokensAfterFinal: passTokensAfter,
734
- leafResult: null,
735
- condenseResult,
736
- });
737
-
738
- actionTaken = true;
739
- condensed = true;
740
- createdSummaryId = condenseResult.summaryId;
741
- level = condenseResult.level;
742
- runningTokens = passTokensAfter;
743
-
744
- if (!force && passTokensAfter <= threshold) {
745
- previousTokens = passTokensAfter;
746
- break;
747
- }
748
- if (passTokensAfter >= passTokensBefore || passTokensAfter >= previousTokens) {
749
- break;
750
- }
751
- previousTokens = passTokensAfter;
752
- }
753
-
754
- const tokensAfter = runningTokens;
755
-
756
- return {
757
- actionTaken,
758
- tokensBefore,
759
- tokensAfter,
760
- createdSummaryId,
761
- condensed,
762
- level,
763
- ...(hadAuthFailure ? { authFailure: true } : {}),
764
- };
765
- }
766
-
767
- // ── compactUntilUnder ────────────────────────────────────────────────────
768
-
769
- /** Compact until under the requested target, running up to maxRounds. */
770
- async compactUntilUnder(input: {
771
- conversationId: number;
772
- tokenBudget: number;
773
- targetTokens?: number;
774
- currentTokens?: number;
775
- summarize: CompactionSummarizeFn;
776
- summaryModel?: string;
777
- }): Promise<{ success: boolean; rounds: number; finalTokens: number; authFailure?: boolean }> {
778
- return this.withContextCache(() => this._compactUntilUnderImpl(input));
779
- }
780
-
781
- private async _compactUntilUnderImpl(input: {
782
- conversationId: number;
783
- tokenBudget: number;
784
- targetTokens?: number;
785
- currentTokens?: number;
786
- summarize: CompactionSummarizeFn;
787
- summaryModel?: string;
788
- }): Promise<{ success: boolean; rounds: number; finalTokens: number; authFailure?: boolean }> {
789
- const { conversationId, tokenBudget, summarize } = input;
790
- const targetTokens =
791
- typeof input.targetTokens === "number" &&
792
- Number.isFinite(input.targetTokens) &&
793
- input.targetTokens > 0
794
- ? Math.floor(input.targetTokens)
795
- : tokenBudget;
796
-
797
- const storedTokens = await this.summaryStore.getContextTokenCount(conversationId);
798
- const liveTokens =
799
- typeof input.currentTokens === "number" &&
800
- Number.isFinite(input.currentTokens) &&
801
- input.currentTokens > 0
802
- ? Math.floor(input.currentTokens)
803
- : 0;
804
- let lastTokens = Math.max(storedTokens, liveTokens);
805
-
806
- // For forced overflow recovery, callers may pass an observed count that
807
- // equals the context budget. Treat equality as still needing a compaction
808
- // attempt so we can create headroom for provider-side framing overhead.
809
- if (lastTokens < targetTokens) {
810
- return { success: true, rounds: 0, finalTokens: lastTokens };
811
- }
812
-
813
- for (let round = 1; round <= this.config.maxRounds; round++) {
814
- const result = await this.compact({
815
- conversationId,
816
- tokenBudget,
817
- summarize,
818
- force: true,
819
- summaryModel: input.summaryModel,
820
- });
821
-
822
- if (result.authFailure) {
823
- return {
824
- success: false,
825
- rounds: round,
826
- finalTokens: result.tokensAfter,
827
- authFailure: true,
828
- };
829
- }
830
-
831
- if (result.tokensAfter <= targetTokens) {
832
- return {
833
- success: true,
834
- rounds: round,
835
- finalTokens: result.tokensAfter,
836
- };
837
- }
838
-
839
- // No progress -- bail to avoid infinite loop
840
- if (!result.actionTaken || result.tokensAfter >= lastTokens) {
841
- return {
842
- success: false,
843
- rounds: round,
844
- finalTokens: result.tokensAfter,
845
- };
846
- }
847
-
848
- lastTokens = result.tokensAfter;
849
- }
850
-
851
- // Exhausted all rounds — use the last known token count from compact() result
852
- const finalTokens = lastTokens;
853
- return {
854
- success: finalTokens <= targetTokens,
855
- rounds: this.config.maxRounds,
856
- finalTokens,
857
- };
858
- }
859
-
860
- // ── Private helpers ──────────────────────────────────────────────────────
861
-
862
- /** Normalize configured leaf chunk size to a safe positive integer. */
863
- private resolveLeafChunkTokens(leafChunkTokensOverride?: number): number {
864
- if (
865
- typeof leafChunkTokensOverride === "number" &&
866
- Number.isFinite(leafChunkTokensOverride) &&
867
- leafChunkTokensOverride > 0
868
- ) {
869
- return Math.floor(leafChunkTokensOverride);
870
- }
871
- if (
872
- typeof this.config.leafChunkTokens === "number" &&
873
- Number.isFinite(this.config.leafChunkTokens) &&
874
- this.config.leafChunkTokens > 0
875
- ) {
876
- return Math.floor(this.config.leafChunkTokens);
877
- }
878
- return DEFAULT_LEAF_CHUNK_TOKENS;
879
- }
880
-
881
- /** Normalize configured fresh tail count to a safe non-negative integer. */
882
- private resolveFreshTailCount(): number {
883
- if (
884
- typeof this.config.freshTailCount === "number" &&
885
- Number.isFinite(this.config.freshTailCount) &&
886
- this.config.freshTailCount > 0
887
- ) {
888
- return Math.floor(this.config.freshTailCount);
889
- }
890
- return 0;
891
- }
892
-
893
- /**
894
- * Compute the ordinal boundary for protected fresh messages.
895
- *
896
- * Messages with ordinal >= returned value are preserved as fresh tail.
897
- */
898
- private resolveFreshTailOrdinal(contextItems: ContextItemRecord[]): number {
899
- const freshTailCount = this.resolveFreshTailCount();
900
- if (freshTailCount <= 0) {
901
- return Infinity;
902
- }
903
-
904
- const rawMessageItems = contextItems.filter(
905
- (item) => item.itemType === "message" && item.messageId != null,
906
- );
907
- if (rawMessageItems.length === 0) {
908
- return Infinity;
909
- }
910
-
911
- const tailStartIdx = Math.max(0, rawMessageItems.length - freshTailCount);
912
- return rawMessageItems[tailStartIdx]?.ordinal ?? Infinity;
913
- }
914
-
915
- /** Resolve message token count with a content-length fallback. */
916
- private async getMessageTokenCount(messageId: number): Promise<number> {
917
- const message = await this.conversationStore.getMessageById(messageId);
918
- if (!message) {
919
- return 0;
920
- }
921
- if (
922
- typeof message.tokenCount === "number" &&
923
- Number.isFinite(message.tokenCount) &&
924
- message.tokenCount > 0
925
- ) {
926
- return message.tokenCount;
927
- }
928
- return estimateTokens(message.content);
929
- }
930
-
931
- /** Sum raw message tokens outside the protected fresh tail. */
932
- private async countRawTokensOutsideFreshTail(conversationId: number): Promise<number> {
933
- const contextItems = await this.getContextItemsCached(conversationId);
934
- const freshTailOrdinal = this.resolveFreshTailOrdinal(contextItems);
935
- let rawTokens = 0;
936
-
937
- for (const item of contextItems) {
938
- if (item.ordinal >= freshTailOrdinal) {
939
- break;
940
- }
941
- if (item.itemType !== "message" || item.messageId == null) {
942
- continue;
943
- }
944
- rawTokens += await this.getMessageTokenCount(item.messageId);
945
- }
946
-
947
- return rawTokens;
948
- }
949
-
950
- /**
951
- * Select the oldest contiguous raw-message chunk outside fresh tail.
952
- *
953
- * The selected chunk size is capped by `leafChunkTokens`, but we always pick
954
- * at least one message when any compactable message exists.
955
- */
956
- private async selectOldestLeafChunk(
957
- conversationId: number,
958
- leafChunkTokensOverride?: number,
959
- ): Promise<LeafChunkSelection> {
960
- const contextItems = await this.getContextItemsCached(conversationId);
961
- const freshTailOrdinal = this.resolveFreshTailOrdinal(contextItems);
962
- const threshold = this.resolveLeafChunkTokens(leafChunkTokensOverride);
963
-
964
- let rawTokensOutsideTail = 0;
965
- for (const item of contextItems) {
966
- if (item.ordinal >= freshTailOrdinal) {
967
- break;
968
- }
969
- if (item.itemType !== "message" || item.messageId == null) {
970
- continue;
971
- }
972
- rawTokensOutsideTail += await this.getMessageTokenCount(item.messageId);
973
- }
974
-
975
- const chunk: ContextItemRecord[] = [];
976
- let chunkTokens = 0;
977
- let started = false;
978
- for (const item of contextItems) {
979
- if (item.ordinal >= freshTailOrdinal) {
980
- break;
981
- }
982
-
983
- if (!started) {
984
- if (item.itemType !== "message" || item.messageId == null) {
985
- continue;
986
- }
987
- started = true;
988
- } else if (item.itemType !== "message" || item.messageId == null) {
989
- break;
990
- }
991
-
992
- if (item.messageId == null) {
993
- continue;
994
- }
995
- const messageTokens = await this.getMessageTokenCount(item.messageId);
996
- if (chunk.length > 0 && chunkTokens + messageTokens > threshold) {
997
- break;
998
- }
999
-
1000
- chunk.push(item);
1001
- chunkTokens += messageTokens;
1002
- if (chunkTokens >= threshold) {
1003
- break;
1004
- }
1005
- }
1006
-
1007
- return { items: chunk, rawTokensOutsideTail, threshold };
1008
- }
1009
-
1010
- /**
1011
- * Resolve recent summary continuity for a leaf pass.
1012
- *
1013
- * Collects up to two most recent summary context items that precede the
1014
- * compacted raw-message chunk and returns their combined content.
1015
- */
1016
- private async resolvePriorLeafSummaryContext(
1017
- conversationId: number,
1018
- messageItems: ContextItemRecord[],
1019
- ): Promise<string | undefined> {
1020
- if (messageItems.length === 0) {
1021
- return undefined;
1022
- }
1023
-
1024
- const startOrdinal = Math.min(...messageItems.map((item) => item.ordinal));
1025
- const priorSummaryItems = (await this.getContextItemsCached(conversationId))
1026
- .filter(
1027
- (item) =>
1028
- item.ordinal < startOrdinal &&
1029
- item.itemType === "summary" &&
1030
- typeof item.summaryId === "string",
1031
- )
1032
- .slice(-2);
1033
-
1034
- if (priorSummaryItems.length === 0) {
1035
- return undefined;
1036
- }
1037
-
1038
- const summaryContents: string[] = [];
1039
- for (const item of priorSummaryItems) {
1040
- if (typeof item.summaryId !== "string") {
1041
- continue;
1042
- }
1043
- const summary = await this.summaryStore.getSummary(item.summaryId);
1044
- const content = summary?.content.trim();
1045
- if (content) {
1046
- summaryContents.push(content);
1047
- }
1048
- }
1049
-
1050
- if (summaryContents.length === 0) {
1051
- return undefined;
1052
- }
1053
-
1054
- return summaryContents.join("\n\n");
1055
- }
1056
-
1057
- /** Resolve summary token count with content-length fallback. */
1058
- private resolveSummaryTokenCount(summary: SummaryRecord): number {
1059
- if (
1060
- typeof summary.tokenCount === "number" &&
1061
- Number.isFinite(summary.tokenCount) &&
1062
- summary.tokenCount > 0
1063
- ) {
1064
- return summary.tokenCount;
1065
- }
1066
- return estimateTokens(summary.content);
1067
- }
1068
-
1069
- /** Resolve message token count with content-length fallback. */
1070
- private resolveMessageTokenCount(message: { tokenCount: number; content: string }): number {
1071
- if (
1072
- typeof message.tokenCount === "number" &&
1073
- Number.isFinite(message.tokenCount) &&
1074
- message.tokenCount > 0
1075
- ) {
1076
- return message.tokenCount;
1077
- }
1078
- return estimateTokens(message.content);
1079
- }
1080
-
1081
- private resolveLeafMinFanout(): number {
1082
- if (
1083
- typeof this.config.leafMinFanout === "number" &&
1084
- Number.isFinite(this.config.leafMinFanout) &&
1085
- this.config.leafMinFanout > 0
1086
- ) {
1087
- return Math.floor(this.config.leafMinFanout);
1088
- }
1089
- return 8;
1090
- }
1091
-
1092
- private resolveCondensedMinFanout(): number {
1093
- if (
1094
- typeof this.config.condensedMinFanout === "number" &&
1095
- Number.isFinite(this.config.condensedMinFanout) &&
1096
- this.config.condensedMinFanout > 0
1097
- ) {
1098
- return Math.floor(this.config.condensedMinFanout);
1099
- }
1100
- return 4;
1101
- }
1102
-
1103
- private resolveCondensedMinFanoutHard(): number {
1104
- if (
1105
- typeof this.config.condensedMinFanoutHard === "number" &&
1106
- Number.isFinite(this.config.condensedMinFanoutHard) &&
1107
- this.config.condensedMinFanoutHard > 0
1108
- ) {
1109
- return Math.floor(this.config.condensedMinFanoutHard);
1110
- }
1111
- return 2;
1112
- }
1113
-
1114
- private resolveIncrementalMaxDepth(): number {
1115
- if (
1116
- typeof this.config.incrementalMaxDepth === "number" &&
1117
- Number.isFinite(this.config.incrementalMaxDepth)
1118
- ) {
1119
- if (this.config.incrementalMaxDepth < 0) return Infinity;
1120
- if (this.config.incrementalMaxDepth > 0) return Math.floor(this.config.incrementalMaxDepth);
1121
- }
1122
- return 0;
1123
- }
1124
- private resolveFanoutForDepth(targetDepth: number, hardTrigger: boolean): number {
1125
- if (hardTrigger) {
1126
- return this.resolveCondensedMinFanoutHard();
1127
- }
1128
- if (targetDepth === 0) {
1129
- return this.resolveLeafMinFanout();
1130
- }
1131
- return this.resolveCondensedMinFanout();
1132
- }
1133
-
1134
- /** Minimum condensed input size before we run another condensed pass. */
1135
- private resolveCondensedMinChunkTokens(): number {
1136
- const chunkTarget = this.resolveLeafChunkTokens();
1137
- const ratioFloor = Math.floor(chunkTarget * CONDENSED_MIN_INPUT_RATIO);
1138
- return Math.max(this.config.condensedTargetTokens, ratioFloor);
1139
- }
1140
-
1141
- /**
1142
- * Find the shallowest depth with an eligible same-depth summary chunk.
1143
- */
1144
- private async selectShallowestCondensationCandidate(params: {
1145
- conversationId: number;
1146
- hardTrigger: boolean;
1147
- }): Promise<CondensedPhaseCandidate | null> {
1148
- const { conversationId, hardTrigger } = params;
1149
- const contextItems = await this.getContextItemsCached(conversationId);
1150
- const freshTailOrdinal = this.resolveFreshTailOrdinal(contextItems);
1151
- const minChunkTokens = this.resolveCondensedMinChunkTokens();
1152
- const depthLevels = await this.summaryStore.getDistinctDepthsInContext(conversationId, {
1153
- maxOrdinalExclusive: freshTailOrdinal,
1154
- });
1155
-
1156
- for (const targetDepth of depthLevels) {
1157
- const fanout = this.resolveFanoutForDepth(targetDepth, hardTrigger);
1158
- const chunk = await this.selectOldestChunkAtDepth(
1159
- conversationId,
1160
- targetDepth,
1161
- freshTailOrdinal,
1162
- );
1163
- if (chunk.items.length < fanout) {
1164
- continue;
1165
- }
1166
- if (chunk.summaryTokens < minChunkTokens) {
1167
- continue;
1168
- }
1169
- return { targetDepth, chunk };
1170
- }
1171
-
1172
- return null;
1173
- }
1174
-
1175
- /**
1176
- * Select the oldest contiguous summary chunk at a specific summary depth.
1177
- *
1178
- * Once selection starts, any non-summary item or depth mismatch terminates
1179
- * the chunk to prevent mixed-depth condensation.
1180
- */
1181
- private async selectOldestChunkAtDepth(
1182
- conversationId: number,
1183
- targetDepth: number,
1184
- freshTailOrdinalOverride?: number,
1185
- ): Promise<CondensedChunkSelection> {
1186
- const contextItems = await this.getContextItemsCached(conversationId);
1187
- const freshTailOrdinal =
1188
- typeof freshTailOrdinalOverride === "number"
1189
- ? freshTailOrdinalOverride
1190
- : this.resolveFreshTailOrdinal(contextItems);
1191
- const chunkTokenBudget = this.resolveLeafChunkTokens();
1192
-
1193
- const chunk: ContextItemRecord[] = [];
1194
- let summaryTokens = 0;
1195
- for (const item of contextItems) {
1196
- if (item.ordinal >= freshTailOrdinal) {
1197
- break;
1198
- }
1199
- if (item.itemType !== "summary" || item.summaryId == null) {
1200
- if (chunk.length > 0) {
1201
- break;
1202
- }
1203
- continue;
1204
- }
1205
-
1206
- const summary = await this.summaryStore.getSummary(item.summaryId);
1207
- if (!summary) {
1208
- if (chunk.length > 0) {
1209
- break;
1210
- }
1211
- continue;
1212
- }
1213
- if (summary.depth !== targetDepth) {
1214
- if (chunk.length > 0) {
1215
- break;
1216
- }
1217
- continue;
1218
- }
1219
- const tokenCount = this.resolveSummaryTokenCount(summary);
1220
-
1221
- if (chunk.length > 0 && summaryTokens + tokenCount > chunkTokenBudget) {
1222
- break;
1223
- }
1224
-
1225
- chunk.push(item);
1226
- summaryTokens += tokenCount;
1227
- if (summaryTokens >= chunkTokenBudget) {
1228
- break;
1229
- }
1230
- }
1231
-
1232
- return { items: chunk, summaryTokens };
1233
- }
1234
-
1235
- private async resolvePriorSummaryContextAtDepth(
1236
- conversationId: number,
1237
- summaryItems: ContextItemRecord[],
1238
- targetDepth: number,
1239
- ): Promise<string | undefined> {
1240
- if (summaryItems.length === 0) {
1241
- return undefined;
1242
- }
1243
-
1244
- const startOrdinal = Math.min(...summaryItems.map((item) => item.ordinal));
1245
- const priorSummaryItems = (await this.getContextItemsCached(conversationId))
1246
- .filter(
1247
- (item) =>
1248
- item.ordinal < startOrdinal &&
1249
- item.itemType === "summary" &&
1250
- typeof item.summaryId === "string",
1251
- )
1252
- .slice(-4);
1253
- if (priorSummaryItems.length === 0) {
1254
- return undefined;
1255
- }
1256
-
1257
- const summaryContents: string[] = [];
1258
- for (const item of priorSummaryItems) {
1259
- if (typeof item.summaryId !== "string") {
1260
- continue;
1261
- }
1262
- const summary = await this.summaryStore.getSummary(item.summaryId);
1263
- if (!summary || summary.depth !== targetDepth) {
1264
- continue;
1265
- }
1266
- const content = summary.content.trim();
1267
- if (content) {
1268
- summaryContents.push(content);
1269
- }
1270
- }
1271
-
1272
- if (summaryContents.length === 0) {
1273
- return undefined;
1274
- }
1275
- return summaryContents.slice(-2).join("\n\n");
1276
- }
1277
-
1278
- /**
1279
- * Run three-level summarization escalation:
1280
- * normal -> aggressive -> deterministic fallback.
1281
- *
1282
- * Provider-auth failures are treated as non-compacting skips so we do not
1283
- * persist truncation artifacts into the summary DAG.
1284
- */
1285
- private async summarizeWithEscalation(params: {
1286
- sourceText: string;
1287
- summarize: CompactionSummarizeFn;
1288
- options?: CompactionSummarizeOptions;
1289
- /** Target token count for this summary kind (leaf or condensed). Used for hard-cap enforcement. */
1290
- targetTokens: number;
1291
- }): Promise<{ content: string; level: CompactionLevel } | null> {
1292
- const sourceText = params.sourceText.trim();
1293
- if (!sourceText) {
1294
- return {
1295
- content: "[Truncated from 0 tokens]",
1296
- level: "fallback",
1297
- };
1298
- }
1299
- const inputTokens = Math.max(1, estimateTokens(sourceText));
1300
- const buildDeterministicFallback = (): { content: string; level: CompactionLevel } => {
1301
- const suffix = `\n[Truncated from ${inputTokens} tokens]`;
1302
- const truncated = truncateTextToEstimatedTokens(
1303
- sourceText,
1304
- Math.max(0, FALLBACK_MAX_TOKENS - estimateTokens(suffix)),
1305
- );
1306
- return {
1307
- content: `${truncated}${suffix}`,
1308
- level: "fallback",
1309
- };
1310
- };
1311
- const authFailure = Symbol("authFailure");
1312
-
1313
- const runSummarizer = async (
1314
- aggressiveMode: boolean,
1315
- ): Promise<string | null | typeof authFailure> => {
1316
- let output: string;
1317
- try {
1318
- output = await params.summarize(sourceText, aggressiveMode, params.options);
1319
- } catch (err) {
1320
- if (err instanceof LcmProviderAuthError) {
1321
- return authFailure;
1322
- }
1323
- throw err;
1324
- }
1325
- const trimmed = output.trim();
1326
- return trimmed || null;
1327
- };
1328
-
1329
- const initialSummary = await runSummarizer(false);
1330
- if (initialSummary === authFailure) {
1331
- return null;
1332
- }
1333
- if (initialSummary === null) {
1334
- // Empty provider output should still compact deterministically so a
1335
- // silent no-op does not stall compaction forever.
1336
- return buildDeterministicFallback();
1337
- }
1338
- let summaryText = initialSummary;
1339
- let level: CompactionLevel = "normal";
1340
-
1341
- if (estimateTokens(summaryText) >= inputTokens) {
1342
- const aggressiveSummary = await runSummarizer(true);
1343
- if (aggressiveSummary === authFailure) {
1344
- return null;
1345
- }
1346
- if (aggressiveSummary === null) {
1347
- return buildDeterministicFallback();
1348
- }
1349
- summaryText = aggressiveSummary;
1350
- level = "aggressive";
1351
-
1352
- if (estimateTokens(summaryText) >= inputTokens) {
1353
- return buildDeterministicFallback();
1354
- }
1355
- }
1356
-
1357
- // Hard cap: enforce maximum summary size relative to the kind-appropriate target.
1358
- const summaryTokens = estimateTokens(summaryText);
1359
- const maxTokens = Math.ceil(params.targetTokens * this.config.summaryMaxOverageFactor);
1360
-
1361
- if (summaryTokens > Math.ceil(params.targetTokens * 1.5)) {
1362
- this.log.warn(
1363
- `[lcm] summary exceeds target by ${Math.round((summaryTokens / params.targetTokens - 1) * 100)}%: ${summaryTokens} tokens vs target ${params.targetTokens}`,
1364
- );
1365
- }
1366
-
1367
- if (summaryTokens > maxTokens) {
1368
- summaryText = capSummaryText(summaryText, summaryTokens, maxTokens);
1369
- level = "capped";
1370
- }
1371
-
1372
- return { content: summaryText, level };
1373
- }
1374
-
1375
- // ── Private: Media Annotation ────────────────────────────────────────────
1376
-
1377
- /**
1378
- * Annotate a message's content with media context when it has file/media
1379
- * attachments. This gives the summarizer enough context to produce a
1380
- * meaningful summary instead of trying to compress raw file paths.
1381
- *
1382
- * - Media-only messages: content is replaced with "[Media attachment]".
1383
- * - Media-mostly messages: text is preserved and annotated with
1384
- * " [with media attachment]".
1385
- * - Text-only messages: returned unchanged.
1386
- */
1387
- private async annotateMediaContent(
1388
- messageId: number,
1389
- content: string,
1390
- ): Promise<string> {
1391
- const parts = await this.conversationStore.getMessageParts(messageId);
1392
- const hasMediaParts = parts.some((part) => isMediaAttachmentPart(part));
1393
- if (!hasMediaParts) {
1394
- return content;
1395
- }
1396
-
1397
- const partText = parts
1398
- .filter((part) => !isMediaAttachmentPart(part))
1399
- .map((part) => (typeof part.textContent === "string" ? part.textContent : ""))
1400
- .map((text) => stripEmbeddedMediaPayloads(text))
1401
- .map((text) => text.trim())
1402
- .filter(Boolean)
1403
- .join("\n")
1404
- .trim();
1405
- const fallbackText = extractMeaningfulMessageText(content);
1406
- const meaningfulText = (partText || fallbackText).trim();
1407
-
1408
- if (!meaningfulText) {
1409
- return "[Media attachment]";
1410
- }
1411
- if (meaningfulText.includes("[with media attachment]")) {
1412
- return meaningfulText;
1413
- }
1414
- return `${meaningfulText} [with media attachment]`;
1415
- }
1416
-
1417
- // ── Private: Leaf Pass ───────────────────────────────────────────────────
1418
-
1419
- /**
1420
- * Summarize a chunk of messages into one leaf summary.
1421
- */
1422
- private async leafPass(
1423
- conversationId: number,
1424
- messageItems: ContextItemRecord[],
1425
- summarize: CompactionSummarizeFn,
1426
- previousSummaryContent?: string,
1427
- summaryModel?: string,
1428
- ): Promise<{ summaryId: string; level: CompactionLevel; content: string; removedTokens: number; addedTokens: number } | null> {
1429
- // Fetch full message content for each context item
1430
- const messageContents: { messageId: number; content: string; createdAt: Date; tokenCount: number }[] =
1431
- [];
1432
- for (const item of messageItems) {
1433
- if (item.messageId == null) {
1434
- continue;
1435
- }
1436
- const msg = await this.conversationStore.getMessageById(item.messageId);
1437
- if (msg) {
1438
- const annotatedContent = await this.annotateMediaContent(
1439
- msg.messageId,
1440
- msg.content,
1441
- );
1442
- messageContents.push({
1443
- messageId: msg.messageId,
1444
- content: annotatedContent,
1445
- createdAt: msg.createdAt,
1446
- tokenCount: this.resolveMessageTokenCount(msg),
1447
- });
1448
- }
1449
- }
1450
-
1451
- const concatenated = messageContents
1452
- .map((message) => `[${formatTimestamp(message.createdAt, this.config.timezone)}]\n${message.content}`)
1453
- .join("\n\n");
1454
- const fileIds = dedupeOrderedIds(
1455
- messageContents.flatMap((message) => extractFileIdsFromContent(message.content)),
1456
- );
1457
- const summary = await this.summarizeWithEscalation({
1458
- sourceText: concatenated,
1459
- summarize,
1460
- options: {
1461
- previousSummary: previousSummaryContent,
1462
- isCondensed: false,
1463
- },
1464
- targetTokens: this.config.leafTargetTokens,
1465
- });
1466
- if (!summary) {
1467
- this.log.warn(
1468
- `[lcm] leaf compaction skipped summary write; conversationId=${conversationId}; chunkMessages=${messageContents.length}`,
1469
- );
1470
- return null;
1471
- }
1472
-
1473
- // Persist the leaf summary
1474
- const summaryId = generateSummaryId(summary.content);
1475
- const tokenCount = estimateTokens(summary.content);
1476
- // Note: removedTokens uses resolveMessageTokenCount values (which fall back to
1477
- // estimateTokens for messages with token_count <= 0). This can diverge from
1478
- // getContextTokenCount() which would sum the stored 0. The delta feeds into
1479
- // stopping decisions (threshold checks, progress guards), but the divergence
1480
- // is bounded to empty/corrupt messages (token_count=0) which are rare.
1481
- // For summaries, removedTokens matches the DB exactly (same tokenCount column).
1482
- const removedTokens = messageContents.reduce(
1483
- (sum, message) => sum + Math.max(0, Math.floor(message.tokenCount)),
1484
- 0,
1485
- );
1486
-
1487
- await this.summaryStore.withTransaction(async () => {
1488
- await this.summaryStore.insertSummary({
1489
- summaryId,
1490
- conversationId,
1491
- kind: "leaf",
1492
- depth: 0,
1493
- content: summary.content,
1494
- tokenCount,
1495
- fileIds,
1496
- earliestAt:
1497
- messageContents.length > 0
1498
- ? new Date(Math.min(...messageContents.map((message) => message.createdAt.getTime())))
1499
- : undefined,
1500
- latestAt:
1501
- messageContents.length > 0
1502
- ? new Date(Math.max(...messageContents.map((message) => message.createdAt.getTime())))
1503
- : undefined,
1504
- descendantCount: 0,
1505
- descendantTokenCount: 0,
1506
- sourceMessageTokenCount: removedTokens,
1507
- model: summaryModel,
1508
- });
1509
-
1510
- // Link to source messages before the context swap becomes visible.
1511
- const messageIds = messageContents.map((m) => m.messageId);
1512
- await this.summaryStore.linkSummaryToMessages(summaryId, messageIds);
1513
-
1514
- // Replace the message range in context with the new summary.
1515
- const ordinals = messageItems.map((ci) => ci.ordinal);
1516
- const startOrdinal = Math.min(...ordinals);
1517
- const endOrdinal = Math.max(...ordinals);
1518
-
1519
- await this.summaryStore.replaceContextRangeWithSummary({
1520
- conversationId,
1521
- startOrdinal,
1522
- endOrdinal,
1523
- summaryId,
1524
- });
1525
- });
1526
- this.invalidateContextCache(conversationId);
1527
-
1528
- return { summaryId, level: summary.level, content: summary.content, removedTokens, addedTokens: tokenCount };
1529
- }
1530
-
1531
- // ── Private: Condensed Pass ──────────────────────────────────────────────
1532
-
1533
- /**
1534
- * Condense one ratio-sized summary chunk into a single condensed summary.
1535
- */
1536
- private async condensedPass(
1537
- conversationId: number,
1538
- summaryItems: ContextItemRecord[],
1539
- targetDepth: number,
1540
- summarize: CompactionSummarizeFn,
1541
- summaryModel?: string,
1542
- ): Promise<PassResult | null> {
1543
- // Fetch full summary records
1544
- const summaryRecords: SummaryRecord[] = [];
1545
- for (const item of summaryItems) {
1546
- if (item.summaryId == null) {
1547
- continue;
1548
- }
1549
- const rec = await this.summaryStore.getSummary(item.summaryId);
1550
- if (rec) {
1551
- summaryRecords.push(rec);
1552
- }
1553
- }
1554
-
1555
- const concatenated = summaryRecords
1556
- .map((summary) => {
1557
- const earliestAt = summary.earliestAt ?? summary.createdAt;
1558
- const latestAt = summary.latestAt ?? summary.createdAt;
1559
- const tz = this.config.timezone;
1560
- const header = `[${formatTimestamp(earliestAt, tz)} - ${formatTimestamp(latestAt, tz)}]`;
1561
- return `${header}\n${summary.content}`;
1562
- })
1563
- .join("\n\n");
1564
- const fileIds = dedupeOrderedIds(
1565
- summaryRecords.flatMap((summary) => [
1566
- ...summary.fileIds,
1567
- ...extractFileIdsFromContent(summary.content),
1568
- ]),
1569
- );
1570
- const previousSummaryContent =
1571
- targetDepth === 0
1572
- ? await this.resolvePriorSummaryContextAtDepth(conversationId, summaryItems, targetDepth)
1573
- : undefined;
1574
- const condensed = await this.summarizeWithEscalation({
1575
- sourceText: concatenated,
1576
- summarize,
1577
- options: {
1578
- previousSummary: previousSummaryContent,
1579
- isCondensed: true,
1580
- depth: targetDepth + 1,
1581
- },
1582
- targetTokens: this.config.condensedTargetTokens,
1583
- });
1584
- if (!condensed) {
1585
- this.log.warn(
1586
- `[lcm] condensed compaction skipped summary write; conversationId=${conversationId}; depth=${targetDepth}; chunkSummaries=${summaryRecords.length}`,
1587
- );
1588
- return null;
1589
- }
1590
-
1591
- // Persist the condensed summary
1592
- const summaryId = generateSummaryId(condensed.content);
1593
- const tokenCount = estimateTokens(condensed.content);
1594
-
1595
- await this.summaryStore.withTransaction(async () => {
1596
- await this.summaryStore.insertSummary({
1597
- summaryId,
1598
- conversationId,
1599
- kind: "condensed",
1600
- depth: targetDepth + 1,
1601
- content: condensed.content,
1602
- tokenCount,
1603
- fileIds,
1604
- earliestAt:
1605
- summaryRecords.length > 0
1606
- ? new Date(
1607
- Math.min(
1608
- ...summaryRecords.map((summary) =>
1609
- (summary.earliestAt ?? summary.createdAt).getTime(),
1610
- ),
1611
- ),
1612
- )
1613
- : undefined,
1614
- latestAt:
1615
- summaryRecords.length > 0
1616
- ? new Date(
1617
- Math.max(
1618
- ...summaryRecords.map(
1619
- (summary) => (summary.latestAt ?? summary.createdAt).getTime(),
1620
- ),
1621
- ),
1622
- )
1623
- : undefined,
1624
- descendantCount: summaryRecords.reduce((count, summary) => {
1625
- const childDescendants =
1626
- typeof summary.descendantCount === "number" && Number.isFinite(summary.descendantCount)
1627
- ? Math.max(0, Math.floor(summary.descendantCount))
1628
- : 0;
1629
- return count + childDescendants + 1;
1630
- }, 0),
1631
- descendantTokenCount: summaryRecords.reduce((count, summary) => {
1632
- const childDescendantTokens =
1633
- typeof summary.descendantTokenCount === "number" &&
1634
- Number.isFinite(summary.descendantTokenCount)
1635
- ? Math.max(0, Math.floor(summary.descendantTokenCount))
1636
- : 0;
1637
- return count + Math.max(0, Math.floor(summary.tokenCount)) + childDescendantTokens;
1638
- }, 0),
1639
- sourceMessageTokenCount: summaryRecords.reduce((count, summary) => {
1640
- const sourceTokens =
1641
- typeof summary.sourceMessageTokenCount === "number" &&
1642
- Number.isFinite(summary.sourceMessageTokenCount)
1643
- ? Math.max(0, Math.floor(summary.sourceMessageTokenCount))
1644
- : 0;
1645
- return count + sourceTokens;
1646
- }, 0),
1647
- model: summaryModel,
1648
- });
1649
-
1650
- // Link to parent summaries before the context swap becomes visible.
1651
- const parentSummaryIds = summaryRecords.map((s) => s.summaryId);
1652
- await this.summaryStore.linkSummaryToParents(summaryId, parentSummaryIds);
1653
-
1654
- // Replace all summary items in context with the condensed summary.
1655
- const ordinals = summaryItems.map((ci) => ci.ordinal);
1656
- const startOrdinal = Math.min(...ordinals);
1657
- const endOrdinal = Math.max(...ordinals);
1658
-
1659
- await this.summaryStore.replaceContextRangeWithSummary({
1660
- conversationId,
1661
- startOrdinal,
1662
- endOrdinal,
1663
- summaryId,
1664
- });
1665
- });
1666
- this.invalidateContextCache(conversationId);
1667
-
1668
- const removedTokens = summaryRecords.reduce(
1669
- (sum, s) => sum + Math.max(0, Math.floor(s.tokenCount)),
1670
- 0,
1671
- );
1672
- return { summaryId, level: condensed.level, removedTokens, addedTokens: tokenCount };
1673
- }
1674
-
1675
- /** Emit compaction telemetry without mutating canonical conversation history. */
1676
- private async persistCompactionEvents(input: {
1677
- conversationId: number;
1678
- tokensBefore: number;
1679
- tokensAfterLeaf: number;
1680
- tokensAfterFinal: number;
1681
- leafResult: { summaryId: string; level: CompactionLevel } | null;
1682
- condenseResult: { summaryId: string; level: CompactionLevel } | null;
1683
- }): Promise<void> {
1684
- const {
1685
- conversationId,
1686
- tokensBefore,
1687
- tokensAfterLeaf,
1688
- tokensAfterFinal,
1689
- leafResult,
1690
- condenseResult,
1691
- } = input;
1692
-
1693
- if (!leafResult && !condenseResult) {
1694
- return;
1695
- }
1696
-
1697
- const conversation = await this.conversationStore.getConversation(conversationId);
1698
- if (!conversation) {
1699
- return;
1700
- }
1701
-
1702
- const createdSummaryIds = [leafResult?.summaryId, condenseResult?.summaryId].filter(
1703
- (id): id is string => typeof id === "string" && id.length > 0,
1704
- );
1705
- const condensedPassOccurred = condenseResult !== null;
1706
-
1707
- if (leafResult) {
1708
- await this.persistCompactionEvent({
1709
- conversationId,
1710
- sessionId: conversation.sessionId,
1711
- pass: "leaf",
1712
- level: leafResult.level,
1713
- tokensBefore,
1714
- tokensAfter: tokensAfterLeaf,
1715
- createdSummaryId: leafResult.summaryId,
1716
- createdSummaryIds,
1717
- condensedPassOccurred,
1718
- });
1719
- }
1720
-
1721
- if (condenseResult) {
1722
- await this.persistCompactionEvent({
1723
- conversationId,
1724
- sessionId: conversation.sessionId,
1725
- pass: "condensed",
1726
- level: condenseResult.level,
1727
- tokensBefore: tokensAfterLeaf,
1728
- tokensAfter: tokensAfterFinal,
1729
- createdSummaryId: condenseResult.summaryId,
1730
- createdSummaryIds,
1731
- condensedPassOccurred,
1732
- });
1733
- }
1734
- }
1735
-
1736
- /** Log one compaction event without appending a synthetic chat message. */
1737
- private async persistCompactionEvent(input: {
1738
- conversationId: number;
1739
- sessionId: string;
1740
- pass: CompactionPass;
1741
- level: CompactionLevel;
1742
- tokensBefore: number;
1743
- tokensAfter: number;
1744
- createdSummaryId: string;
1745
- createdSummaryIds: string[];
1746
- condensedPassOccurred: boolean;
1747
- }): Promise<void> {
1748
- const content = `LCM compaction ${input.pass} pass (${input.level}): ${input.tokensBefore} -> ${input.tokensAfter}`;
1749
- this.log.info(
1750
- `[lcm] ${content} conversation=${input.conversationId} summary=${input.createdSummaryId}`,
1751
- );
1752
- }
1753
- }