@animalabs/context-manager 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/src/adaptive/chunker.d.ts +70 -0
  2. package/dist/src/adaptive/chunker.d.ts.map +1 -0
  3. package/dist/src/adaptive/chunker.js +233 -0
  4. package/dist/src/adaptive/chunker.js.map +1 -0
  5. package/dist/src/adaptive/folding-strategy.d.ts +118 -0
  6. package/dist/src/adaptive/folding-strategy.d.ts.map +1 -0
  7. package/dist/src/adaptive/folding-strategy.js +12 -0
  8. package/dist/src/adaptive/folding-strategy.js.map +1 -0
  9. package/dist/src/adaptive/index.d.ts +17 -0
  10. package/dist/src/adaptive/index.d.ts.map +1 -0
  11. package/dist/src/adaptive/index.js +23 -0
  12. package/dist/src/adaptive/index.js.map +1 -0
  13. package/dist/src/adaptive/picker.d.ts +103 -0
  14. package/dist/src/adaptive/picker.d.ts.map +1 -0
  15. package/dist/src/adaptive/picker.js +357 -0
  16. package/dist/src/adaptive/picker.js.map +1 -0
  17. package/dist/src/adaptive/render.d.ts +39 -0
  18. package/dist/src/adaptive/render.d.ts.map +1 -0
  19. package/dist/src/adaptive/render.js +104 -0
  20. package/dist/src/adaptive/render.js.map +1 -0
  21. package/dist/src/adaptive/strategies/flat-profile.d.ts +19 -0
  22. package/dist/src/adaptive/strategies/flat-profile.d.ts.map +1 -0
  23. package/dist/src/adaptive/strategies/flat-profile.js +86 -0
  24. package/dist/src/adaptive/strategies/flat-profile.js.map +1 -0
  25. package/dist/src/adaptive/strategies/oldest-first.d.ts +19 -0
  26. package/dist/src/adaptive/strategies/oldest-first.d.ts.map +1 -0
  27. package/dist/src/adaptive/strategies/oldest-first.js +39 -0
  28. package/dist/src/adaptive/strategies/oldest-first.js.map +1 -0
  29. package/dist/src/context-manager.d.ts +5 -0
  30. package/dist/src/context-manager.d.ts.map +1 -1
  31. package/dist/src/context-manager.js +22 -0
  32. package/dist/src/context-manager.js.map +1 -1
  33. package/dist/src/index.d.ts +1 -1
  34. package/dist/src/index.d.ts.map +1 -1
  35. package/dist/src/index.js.map +1 -1
  36. package/dist/src/message-store.d.ts +36 -1
  37. package/dist/src/message-store.d.ts.map +1 -1
  38. package/dist/src/message-store.js +134 -8
  39. package/dist/src/message-store.js.map +1 -1
  40. package/dist/src/strategies/autobiographical.d.ts +217 -2
  41. package/dist/src/strategies/autobiographical.d.ts.map +1 -1
  42. package/dist/src/strategies/autobiographical.js +1479 -37
  43. package/dist/src/strategies/autobiographical.js.map +1 -1
  44. package/dist/src/strategies/index.d.ts +1 -1
  45. package/dist/src/strategies/index.d.ts.map +1 -1
  46. package/dist/src/strategies/index.js.map +1 -1
  47. package/dist/src/types/message.d.ts +38 -0
  48. package/dist/src/types/message.d.ts.map +1 -1
  49. package/dist/src/types/strategy.d.ts +94 -10
  50. package/dist/src/types/strategy.d.ts.map +1 -1
  51. package/dist/src/types/strategy.js +7 -0
  52. package/dist/src/types/strategy.js.map +1 -1
  53. package/dist/test/adaptive/branching.test.d.ts +20 -0
  54. package/dist/test/adaptive/branching.test.d.ts.map +1 -0
  55. package/dist/test/adaptive/branching.test.js +176 -0
  56. package/dist/test/adaptive/branching.test.js.map +1 -0
  57. package/dist/test/adaptive/chunker.test.d.ts +2 -0
  58. package/dist/test/adaptive/chunker.test.d.ts.map +1 -0
  59. package/dist/test/adaptive/chunker.test.js +120 -0
  60. package/dist/test/adaptive/chunker.test.js.map +1 -0
  61. package/dist/test/adaptive/deep-levels.test.d.ts +10 -0
  62. package/dist/test/adaptive/deep-levels.test.d.ts.map +1 -0
  63. package/dist/test/adaptive/deep-levels.test.js +167 -0
  64. package/dist/test/adaptive/deep-levels.test.js.map +1 -0
  65. package/dist/test/adaptive/doc-plus-chat.test.d.ts +10 -0
  66. package/dist/test/adaptive/doc-plus-chat.test.d.ts.map +1 -0
  67. package/dist/test/adaptive/doc-plus-chat.test.js +194 -0
  68. package/dist/test/adaptive/doc-plus-chat.test.js.map +1 -0
  69. package/dist/test/adaptive/hard-fail.test.d.ts +9 -0
  70. package/dist/test/adaptive/hard-fail.test.d.ts.map +1 -0
  71. package/dist/test/adaptive/hard-fail.test.js +154 -0
  72. package/dist/test/adaptive/hard-fail.test.js.map +1 -0
  73. package/dist/test/adaptive/harness.d.ts +109 -0
  74. package/dist/test/adaptive/harness.d.ts.map +1 -0
  75. package/dist/test/adaptive/harness.js +263 -0
  76. package/dist/test/adaptive/harness.js.map +1 -0
  77. package/dist/test/adaptive/ingestion.test.d.ts +13 -0
  78. package/dist/test/adaptive/ingestion.test.d.ts.map +1 -0
  79. package/dist/test/adaptive/ingestion.test.js +306 -0
  80. package/dist/test/adaptive/ingestion.test.js.map +1 -0
  81. package/dist/test/adaptive/integration.test.d.ts +12 -0
  82. package/dist/test/adaptive/integration.test.d.ts.map +1 -0
  83. package/dist/test/adaptive/integration.test.js +247 -0
  84. package/dist/test/adaptive/integration.test.js.map +1 -0
  85. package/dist/test/adaptive/long-chronicle.test.d.ts +13 -0
  86. package/dist/test/adaptive/long-chronicle.test.d.ts.map +1 -0
  87. package/dist/test/adaptive/long-chronicle.test.js +186 -0
  88. package/dist/test/adaptive/long-chronicle.test.js.map +1 -0
  89. package/dist/test/adaptive/persistence.test.d.ts +11 -0
  90. package/dist/test/adaptive/persistence.test.d.ts.map +1 -0
  91. package/dist/test/adaptive/persistence.test.js +200 -0
  92. package/dist/test/adaptive/persistence.test.js.map +1 -0
  93. package/dist/test/adaptive/picker.test.d.ts +2 -0
  94. package/dist/test/adaptive/picker.test.d.ts.map +1 -0
  95. package/dist/test/adaptive/picker.test.js +333 -0
  96. package/dist/test/adaptive/picker.test.js.map +1 -0
  97. package/dist/test/adaptive/render.test.d.ts +2 -0
  98. package/dist/test/adaptive/render.test.d.ts.map +1 -0
  99. package/dist/test/adaptive/render.test.js +109 -0
  100. package/dist/test/adaptive/render.test.js.map +1 -0
  101. package/dist/test/adaptive/shard-immutability.test.d.ts +14 -0
  102. package/dist/test/adaptive/shard-immutability.test.d.ts.map +1 -0
  103. package/dist/test/adaptive/shard-immutability.test.js +184 -0
  104. package/dist/test/adaptive/shard-immutability.test.js.map +1 -0
  105. package/dist/test/adaptive/strategy-integration.test.d.ts +9 -0
  106. package/dist/test/adaptive/strategy-integration.test.d.ts.map +1 -0
  107. package/dist/test/adaptive/strategy-integration.test.js +293 -0
  108. package/dist/test/adaptive/strategy-integration.test.js.map +1 -0
  109. package/dist/test/message-store-sequence.test.d.ts +25 -0
  110. package/dist/test/message-store-sequence.test.d.ts.map +1 -0
  111. package/dist/test/message-store-sequence.test.js +157 -0
  112. package/dist/test/message-store-sequence.test.js.map +1 -0
  113. package/dist/tsconfig.tsbuildinfo +1 -1
  114. package/package.json +1 -1
  115. package/src/adaptive/chunker.ts +323 -0
  116. package/src/adaptive/folding-strategy.ts +129 -0
  117. package/src/adaptive/index.ts +51 -0
  118. package/src/adaptive/picker.ts +442 -0
  119. package/src/adaptive/render.ts +115 -0
  120. package/src/adaptive/strategies/flat-profile.ts +96 -0
  121. package/src/adaptive/strategies/oldest-first.ts +48 -0
  122. package/src/context-manager.ts +32 -0
  123. package/src/index.ts +1 -1
  124. package/src/message-store.ts +147 -8
  125. package/src/strategies/autobiographical.ts +1586 -38
  126. package/src/strategies/index.ts +1 -1
  127. package/src/types/message.ts +43 -0
  128. package/src/types/strategy.ts +109 -10
@@ -0,0 +1,442 @@
1
+ /**
2
+ * Picker orchestrator for the adaptive-resolution design.
3
+ *
4
+ * The picker is a thin loop: build a FoldingState, call strategy.selectNextFold
5
+ * until it returns null, applying 'raise'/'lower' ops as they come and recording
6
+ * 'produce' ops for the caller to enqueue.
7
+ *
8
+ * The picker is stateless across runs — all persistent state lives on the
9
+ * per-chunk fields. Each run builds a fresh FoldingState from the inputs.
10
+ *
11
+ * See `docs/adaptive-resolution-design.md` §5.
12
+ */
13
+
14
+ import type {
15
+ FoldingStrategy,
16
+ FoldingState,
17
+ FoldingBudget,
18
+ FoldOp,
19
+ ChunkView,
20
+ ChunkId,
21
+ SummaryId,
22
+ ChunkRange,
23
+ } from './folding-strategy.js';
24
+ import type { SummaryEntry } from '../types/strategy.js';
25
+ import { getSummaryParentId } from '../types/strategy.js';
26
+
27
+ /**
28
+ * Error raised by the strategy when the picker has folded everything it can
29
+ * and the resulting context still exceeds the hard token budget.
30
+ *
31
+ * The strategy throws this rather than silently dropping entries. The host
32
+ * application decides how to respond — typical responses: raise the budget,
33
+ * switch to a larger-context model, drop the head/tail windows explicitly,
34
+ * or surface a "context too large" error to the user.
35
+ *
36
+ * See `docs/adaptive-resolution-design.md` §3.10.
37
+ */
38
+ export class OverBudgetError extends Error {
39
+ /** The hard budget the strategy was trying to fit under. */
40
+ readonly budget: number;
41
+ /** The token count the strategy could not reduce below `budget`. */
42
+ readonly actual: number;
43
+ /** Diagnostic snapshot of the picker's final state. */
44
+ readonly diagnostics: {
45
+ headTokens: number;
46
+ tailTokens: number;
47
+ middleTokens: number;
48
+ middleChunkCount: number;
49
+ deepestLevel: number;
50
+ };
51
+
52
+ constructor(opts: {
53
+ budget: number;
54
+ actual: number;
55
+ diagnostics: OverBudgetError['diagnostics'];
56
+ }) {
57
+ super(
58
+ `Adaptive picker exhausted but ${opts.actual} tokens still exceed hard budget ${opts.budget}` +
59
+ ` (head=${opts.diagnostics.headTokens}, tail=${opts.diagnostics.tailTokens},` +
60
+ ` middle=${opts.diagnostics.middleTokens} across ${opts.diagnostics.middleChunkCount} chunks,` +
61
+ ` deepest fold level=L${opts.diagnostics.deepestLevel})`
62
+ );
63
+ // Writable per Error convention so instanceof-by-name works across
64
+ // iframe / vm boundaries; the field stays writable on the prototype.
65
+ this.name = 'OverBudgetError';
66
+ this.budget = opts.budget;
67
+ this.actual = opts.actual;
68
+ this.diagnostics = opts.diagnostics;
69
+ }
70
+ }
71
+
72
+ /**
73
+ * Minimal chunk representation used by the picker. Real callers will adapt
74
+ * their `StoredMessage` instances to this shape.
75
+ */
76
+ export interface PickerChunk {
77
+ id: ChunkId;
78
+ sequence: number;
79
+ rawTokens: number;
80
+ currentResolution: number;
81
+ lockedByAgent: boolean;
82
+ bodyGroupId?: string;
83
+ pinned: boolean;
84
+ /**
85
+ * The L1 summary covering this chunk, if any. Higher levels are derived
86
+ * by walking parentId pointers in the summary tree.
87
+ */
88
+ l1Id?: SummaryId;
89
+ }
90
+
91
+ export interface PickerInputs {
92
+ /** All chunks in source order (oldest first). */
93
+ chunks: PickerChunk[];
94
+ /** All summaries, indexed by id. */
95
+ summaries: ReadonlyMap<SummaryId, SummaryEntry>;
96
+ /**
97
+ * Token count for each summary's recall pair. Keyed by summary id.
98
+ * If missing, falls back to SummaryEntry.tokens.
99
+ */
100
+ recallPairTokens?: ReadonlyMap<SummaryId, number>;
101
+ /** Tokens consumed by the head window (fixed, not foldable). */
102
+ headTokens: number;
103
+ /** Tokens consumed by the tail window (fixed, not foldable). */
104
+ tailTokens: number;
105
+ /** Indices into chunks[] that are inside the head window. */
106
+ headChunkIds: ReadonlySet<ChunkId>;
107
+ /** Indices into chunks[] that are inside the tail window. */
108
+ tailChunkIds: ReadonlySet<ChunkId>;
109
+ }
110
+
111
+ export interface PickerResult {
112
+ /** Final resolution state per chunk after all applied ops. */
113
+ finalResolutions: ReadonlyMap<ChunkId, number>;
114
+ /** Ops applied this run, in order. */
115
+ applied: FoldOp[];
116
+ /** Pending production requests the picker emitted. */
117
+ produced: FoldOp[];
118
+ /** Final token count after applying all ops. */
119
+ finalTokens: number;
120
+ /** True if the picker stopped because budget was met. */
121
+ budgetMet: boolean;
122
+ /** True if the picker stopped because no further folds were possible. */
123
+ exhausted: boolean;
124
+ /** Number of iterations the picker performed. */
125
+ iterations: number;
126
+ }
127
+
128
+ /**
129
+ * Floor for the per-run loop bound. Scales with chronicle size to keep the
130
+ * picker from spuriously declaring non-convergence on large stores. See
131
+ * {@link iterationBound}.
132
+ */
133
+ const MIN_ITERATIONS = 1_000;
134
+
135
+ /**
136
+ * Compute the per-run loop bound from chronicle size. A well-behaved strategy
137
+ * converges in at most ~one op per chunk per level (`chunks.length * log_N`),
138
+ * so `chunks.length * 10` is a generous safety net at any reasonable N, and
139
+ * `MIN_ITERATIONS` ensures small chronicles still get adequate headroom for
140
+ * exploratory ops before the strategy commits to a fold.
141
+ *
142
+ * Returning a numeric bound rather than a constant means a 5K-chunk run no
143
+ * longer trips a fixed 10K ceiling that wasn't sized for it.
144
+ */
145
+ function iterationBound(chunkCount: number): number {
146
+ return Math.max(MIN_ITERATIONS, chunkCount * 10);
147
+ }
148
+
149
+ export class Picker {
150
+ constructor(private readonly strategy: FoldingStrategy) {}
151
+
152
+ run(inputs: PickerInputs, budget: FoldingBudget): PickerResult {
153
+ const state = new MutableFoldingState(inputs);
154
+ const applied: FoldOp[] = [];
155
+ const produced: FoldOp[] = [];
156
+ let iterations = 0;
157
+ const maxIterations = iterationBound(inputs.chunks.length);
158
+
159
+ while (iterations < maxIterations) {
160
+ const op = this.strategy.selectNextFold(state, budget);
161
+ if (!op) break;
162
+ iterations++;
163
+
164
+ if (op.kind === 'raise') {
165
+ state.applyRaise(op.groupRoot);
166
+ applied.push(op);
167
+ } else if (op.kind === 'lower') {
168
+ state.applyLower(op.groupRoot);
169
+ applied.push(op);
170
+ } else {
171
+ // produce — record and stop. Caller enqueues; next compile retries.
172
+ produced.push(op);
173
+ break;
174
+ }
175
+ }
176
+
177
+ if (iterations >= maxIterations) {
178
+ throw new Error(
179
+ `Picker: strategy ${this.strategy.name} exceeded iteration bound (${maxIterations}` +
180
+ ` for ${inputs.chunks.length} chunks); likely non-converging`
181
+ );
182
+ }
183
+
184
+ return {
185
+ finalResolutions: state.snapshotResolutions(),
186
+ applied,
187
+ produced,
188
+ finalTokens: state.tokenCount(),
189
+ budgetMet: state.tokenCount() <= budget.targetBudget,
190
+ exhausted: produced.length === 0 && state.tokenCount() > budget.targetBudget,
191
+ iterations,
192
+ };
193
+ }
194
+ }
195
+
196
+ // ---------------------------------------------------------------------------
197
+
198
+ /**
199
+ * Internal mutable state used during one picker run.
200
+ *
201
+ * Implements FoldingState (the strategy-facing view) and additional methods
202
+ * for applying fold ops.
203
+ */
204
+ class MutableFoldingState implements FoldingState {
205
+ private readonly chunkById = new Map<ChunkId, PickerChunk>();
206
+ private readonly chunkOrder: PickerChunk[];
207
+ private readonly summaries: ReadonlyMap<SummaryId, SummaryEntry>;
208
+ private readonly recallPairTokens: ReadonlyMap<SummaryId, number>;
209
+ private readonly headChunkIds: ReadonlySet<ChunkId>;
210
+ private readonly tailChunkIds: ReadonlySet<ChunkId>;
211
+ private readonly headTokens: number;
212
+ private readonly tailTokens: number;
213
+
214
+ // Cached: chunks that are foldable (middle, not pinned, not locked).
215
+ // Updated lazily when needed.
216
+ private foldableCache: ChunkView[] | null = null;
217
+ private allChunkViews: ChunkView[] | null = null;
218
+ private tokenCacheValid = false;
219
+ private tokenCacheValue = 0;
220
+
221
+ // For each chunk, its current resolution (mirrors PickerChunk.currentResolution).
222
+ private resolutions = new Map<ChunkId, number>();
223
+
224
+ constructor(inputs: PickerInputs) {
225
+ this.chunkOrder = [...inputs.chunks].sort((a, b) => a.sequence - b.sequence);
226
+ for (const c of this.chunkOrder) {
227
+ this.chunkById.set(c.id, c);
228
+ this.resolutions.set(c.id, c.currentResolution);
229
+ }
230
+ this.summaries = inputs.summaries;
231
+ this.recallPairTokens = inputs.recallPairTokens ?? new Map();
232
+ this.headChunkIds = inputs.headChunkIds;
233
+ this.tailChunkIds = inputs.tailChunkIds;
234
+ this.headTokens = inputs.headTokens;
235
+ this.tailTokens = inputs.tailTokens;
236
+ }
237
+
238
+ // ---- FoldingState interface ----
239
+
240
+ chunks(): readonly ChunkView[] {
241
+ if (!this.allChunkViews) {
242
+ this.allChunkViews = this.chunkOrder.map((c) => this.makeView(c));
243
+ }
244
+ return this.allChunkViews;
245
+ }
246
+
247
+ foldableMiddle(): readonly ChunkView[] {
248
+ if (this.foldableCache) return this.foldableCache;
249
+ const out: ChunkView[] = [];
250
+ for (const c of this.chunkOrder) {
251
+ const inHead = this.headChunkIds.has(c.id);
252
+ const inTail = this.tailChunkIds.has(c.id);
253
+ if (inHead || inTail || c.pinned || c.lockedByAgent) continue;
254
+ out.push(this.makeView(c));
255
+ }
256
+ this.foldableCache = out;
257
+ return out;
258
+ }
259
+
260
+ getSummary(id: SummaryId): SummaryEntry | null {
261
+ return this.summaries.get(id) ?? null;
262
+ }
263
+
264
+ leavesUnder(groupRoot: SummaryId): readonly ChunkView[] {
265
+ const summary = this.summaries.get(groupRoot);
266
+ if (!summary) return [];
267
+ const leafIds = this.collectLeafIds(summary);
268
+ const views: ChunkView[] = [];
269
+ for (const id of leafIds) {
270
+ const c = this.chunkById.get(id);
271
+ if (c) views.push(this.makeView(c));
272
+ }
273
+ return views;
274
+ }
275
+
276
+ tokenCount(): number {
277
+ if (this.tokenCacheValid) return this.tokenCacheValue;
278
+ this.tokenCacheValue = this.computeTokens();
279
+ this.tokenCacheValid = true;
280
+ return this.tokenCacheValue;
281
+ }
282
+
283
+ // ---- Mutation methods (picker-internal) ----
284
+
285
+ applyRaise(groupRoot: SummaryId): void {
286
+ const summary = this.summaries.get(groupRoot);
287
+ if (!summary) {
288
+ throw new Error(`Picker: applyRaise: unknown groupRoot ${groupRoot}`);
289
+ }
290
+ const targetLevel = summary.level;
291
+ const leafIds = this.collectLeafIds(summary);
292
+ for (const id of leafIds) {
293
+ const chunk = this.chunkById.get(id);
294
+ if (!chunk) continue;
295
+ // Skip chunks that the picker has no authority over for resolution.
296
+ // Pinned chunks always render raw (handled at render time), but we
297
+ // also avoid setting stale resolution state on them — the resolution
298
+ // field is meaningless while pinned and might be visited later if
299
+ // the pin is removed, which would be surprising.
300
+ if (chunk.pinned) continue;
301
+ if (chunk.lockedByAgent) continue;
302
+ if (this.headChunkIds.has(id) || this.tailChunkIds.has(id)) continue;
303
+ const current = this.resolutions.get(id) ?? 0;
304
+ if (current < targetLevel) {
305
+ this.resolutions.set(id, targetLevel);
306
+ }
307
+ }
308
+ this.invalidateCaches();
309
+ }
310
+
311
+ applyLower(groupRoot: SummaryId): void {
312
+ const summary = this.summaries.get(groupRoot);
313
+ if (!summary) {
314
+ throw new Error(`Picker: applyLower: unknown groupRoot ${groupRoot}`);
315
+ }
316
+ // Lower every leaf under groupRoot by one level (clamped at 0).
317
+ const leafIds = this.collectLeafIds(summary);
318
+ for (const id of leafIds) {
319
+ const chunk = this.chunkById.get(id);
320
+ if (!chunk) continue;
321
+ if (chunk.lockedByAgent) continue;
322
+ const current = this.resolutions.get(id) ?? 0;
323
+ if (current > 0) {
324
+ this.resolutions.set(id, current - 1);
325
+ }
326
+ }
327
+ this.invalidateCaches();
328
+ }
329
+
330
+ snapshotResolutions(): ReadonlyMap<ChunkId, number> {
331
+ return new Map(this.resolutions);
332
+ }
333
+
334
+ // ---- Internals ----
335
+
336
+ private invalidateCaches(): void {
337
+ this.foldableCache = null;
338
+ this.allChunkViews = null;
339
+ this.tokenCacheValid = false;
340
+ }
341
+
342
+ private makeView(c: PickerChunk): ChunkView {
343
+ const self = this;
344
+ return {
345
+ id: c.id,
346
+ sequence: c.sequence,
347
+ rawTokens: c.rawTokens,
348
+ currentResolution: self.resolutions.get(c.id) ?? 0,
349
+ lockedByAgent: c.lockedByAgent,
350
+ bodyGroupId: c.bodyGroupId,
351
+ inHead: self.headChunkIds.has(c.id),
352
+ inTail: self.tailChunkIds.has(c.id),
353
+ pinned: c.pinned,
354
+ ancestorAt(level: number): SummaryEntry | null {
355
+ return self.ancestorAt(c, level);
356
+ },
357
+ maxAvailableLevel(): number {
358
+ return self.maxAvailableLevel(c);
359
+ },
360
+ };
361
+ }
362
+
363
+ private ancestorAt(chunk: PickerChunk, level: number): SummaryEntry | null {
364
+ if (level <= 0) return null;
365
+ if (!chunk.l1Id) return null;
366
+ let current: SummaryEntry | undefined = this.summaries.get(chunk.l1Id);
367
+ while (current && current.level < level) {
368
+ const parentId = getSummaryParentId(current);
369
+ if (!parentId) return null;
370
+ current = this.summaries.get(parentId);
371
+ }
372
+ if (!current || current.level !== level) return null;
373
+ return current;
374
+ }
375
+
376
+ private maxAvailableLevel(chunk: PickerChunk): number {
377
+ if (!chunk.l1Id) return 0;
378
+ let current: SummaryEntry | undefined = this.summaries.get(chunk.l1Id);
379
+ let max = 0;
380
+ while (current) {
381
+ max = current.level;
382
+ const parentId = getSummaryParentId(current);
383
+ if (!parentId) break;
384
+ current = this.summaries.get(parentId);
385
+ }
386
+ return max;
387
+ }
388
+
389
+ private collectLeafIds(summary: SummaryEntry): ChunkId[] {
390
+ // Walk down the summary tree from `summary` to leaf chunks.
391
+ // sourceLevel === 0 means sourceIds are message ids (leaves).
392
+ // Otherwise sourceIds are summary ids and we recurse.
393
+ const out: ChunkId[] = [];
394
+ const visit = (s: SummaryEntry): void => {
395
+ if (s.sourceLevel === 0) {
396
+ for (const mid of s.sourceIds) out.push(mid);
397
+ } else {
398
+ for (const sid of s.sourceIds) {
399
+ const child = this.summaries.get(sid);
400
+ if (child) visit(child);
401
+ }
402
+ }
403
+ };
404
+ visit(summary);
405
+ return out;
406
+ }
407
+
408
+ /**
409
+ * Compute total tokens given current resolutions.
410
+ * Algorithm:
411
+ * - Head + tail are fixed.
412
+ * - For chunks in foldableMiddle: those at L0 contribute rawTokens.
413
+ * - Those at L_k (k>0): emit the recall pair for the chunk's L_k ancestor
414
+ * once per distinct ancestor (sibling chunks under the same ancestor
415
+ * share the same rendered recall pair).
416
+ */
417
+ private computeTokens(): number {
418
+ let total = this.headTokens + this.tailTokens;
419
+ const renderedSummaries = new Set<SummaryId>();
420
+ for (const c of this.chunkOrder) {
421
+ if (this.headChunkIds.has(c.id) || this.tailChunkIds.has(c.id)) continue;
422
+ // Pinned chunks render at L0 regardless of resolution.
423
+ const effectiveResolution = c.pinned ? 0 : this.resolutions.get(c.id) ?? 0;
424
+ if (effectiveResolution === 0) {
425
+ total += c.rawTokens;
426
+ continue;
427
+ }
428
+ const ancestor = this.ancestorAt(c, effectiveResolution);
429
+ if (!ancestor) {
430
+ // Resolution is set but summary doesn't exist — treat as L0 for
431
+ // accounting purposes (the renderer will need to make the same call).
432
+ total += c.rawTokens;
433
+ continue;
434
+ }
435
+ if (renderedSummaries.has(ancestor.id)) continue;
436
+ renderedSummaries.add(ancestor.id);
437
+ const tok = this.recallPairTokens.get(ancestor.id) ?? ancestor.tokens;
438
+ total += tok;
439
+ }
440
+ return total;
441
+ }
442
+ }
@@ -0,0 +1,115 @@
1
+ /**
2
+ * Render helpers for the adaptive-resolution design.
3
+ *
4
+ * The main operation is "group consecutive same-bodyGroupId messages into
5
+ * a single API message via body concatenation," which is how a chunked
6
+ * document or large message is reassembled at the API boundary.
7
+ *
8
+ * See `docs/adaptive-resolution-design.md` §3.6.
9
+ */
10
+
11
+ import type { ContentBlock } from '@animalabs/membrane';
12
+ import type { StoredMessage } from '../types/message.js';
13
+
14
+ /**
15
+ * Group consecutive messages that share a bodyGroupId into composite
16
+ * messages whose body is the byte-faithful concatenation of the shards'
17
+ * text content. Messages with a null/undefined bodyGroupId pass through
18
+ * unchanged.
19
+ *
20
+ * For a shard at non-zero currentResolution, the shard's text in the
21
+ * concatenation is replaced by `getRecallText(shard)`. The renderer is
22
+ * responsible for providing recall content that's appropriate for the
23
+ * level (an L_k recall pair, formatted however the deployment prefers).
24
+ *
25
+ * Properties:
26
+ * - **Byte-faithful** when all shards in a group are at L0 and getRecallText
27
+ * is not called: the concatenated body equals the original message body
28
+ * byte-for-byte. Verified by tests in test/adaptive/render.test.ts.
29
+ * - **One API message per group**, regardless of shard count. No turn
30
+ * markers between shards.
31
+ * - **Order preserved**: shards within a group are sorted by `shardIndex`
32
+ * before concatenation; if shardIndex is missing, the order in the input
33
+ * array is used.
34
+ */
35
+ export function concatBodyGroups(
36
+ messages: readonly StoredMessage[],
37
+ getRecallText: (shard: StoredMessage) => string
38
+ ): StoredMessage[] {
39
+ const out: StoredMessage[] = [];
40
+ let i = 0;
41
+ while (i < messages.length) {
42
+ const m = messages[i];
43
+ if (!m.bodyGroupId) {
44
+ out.push(m);
45
+ i++;
46
+ continue;
47
+ }
48
+ // Collect all consecutive messages with the same bodyGroupId.
49
+ const groupId = m.bodyGroupId;
50
+ const groupStart = i;
51
+ while (i < messages.length && messages[i].bodyGroupId === groupId) {
52
+ i++;
53
+ }
54
+ const group = messages.slice(groupStart, i);
55
+ // Sort by shardIndex if present.
56
+ const sorted = [...group].sort(
57
+ (a, b) => (a.shardIndex ?? 0) - (b.shardIndex ?? 0)
58
+ );
59
+
60
+ const concatenated = sorted.map((shard) => {
61
+ const res = shard.currentResolution ?? 0;
62
+ if (res === 0) {
63
+ return extractTextContent(shard.content);
64
+ }
65
+ return getRecallText(shard);
66
+ }).join('');
67
+
68
+ // Build the composite message. Inherit id/participant/timestamp from
69
+ // the first shard; combine metadata; build a single text content block.
70
+ const composite: StoredMessage = {
71
+ id: sorted[0].id,
72
+ sequence: sorted[0].sequence,
73
+ participant: sorted[0].participant,
74
+ content: [{ type: 'text', text: concatenated } as ContentBlock],
75
+ metadata: {
76
+ ...(sorted[0].metadata ?? {}),
77
+ bodyGroupId: groupId,
78
+ shardCount: sorted.length,
79
+ },
80
+ timestamp: sorted[0].timestamp,
81
+ };
82
+ out.push(composite);
83
+ }
84
+ return out;
85
+ }
86
+
87
+ /**
88
+ * Extract text content from a message's content blocks. For shards we
89
+ * expect (and require) all content to be text — they were produced by
90
+ * the chunker, which only splits strings. If non-text blocks slip through,
91
+ * they're stringified as best-effort.
92
+ */
93
+ function extractTextContent(blocks: ContentBlock[]): string {
94
+ const out: string[] = [];
95
+ for (const b of blocks) {
96
+ if (b.type === 'text') {
97
+ out.push(b.text);
98
+ } else {
99
+ // Non-text content in a sharded body shouldn't happen, but if it
100
+ // does, we serialize as a placeholder rather than crashing.
101
+ out.push(`[non-text content: ${b.type}]`);
102
+ }
103
+ }
104
+ return out.join('');
105
+ }
106
+
107
+ /**
108
+ * Default getRecallText for testing or when the strategy hasn't provided
109
+ * a custom one. Returns a simple "[summary of N tokens]" placeholder so
110
+ * the concat doesn't accidentally include the raw shard text.
111
+ */
112
+ export function placeholderRecallText(shard: StoredMessage): string {
113
+ const level = shard.currentResolution ?? 0;
114
+ return `\n[L${level} recall of chunk ${shard.id}]\n`;
115
+ }
@@ -0,0 +1,96 @@
1
+ /**
2
+ * Default folding strategy: keep visible-item counts roughly equal across levels.
3
+ *
4
+ * On each call: if we're over the soft target, find the level with the most
5
+ * *rendered items* (distinct ancestors among chunks at that resolution),
6
+ * pick the oldest raisable group at that level, raise it. Tiebreak on level:
7
+ * lower level wins (fold finer detail before consolidating coarser regions).
8
+ *
9
+ * Monotonic (never emits 'lower'). Memory-fading by construction:
10
+ * a given chunk's resolution only ever increases.
11
+ *
12
+ * See `docs/adaptive-resolution-design.md` §3.7.
13
+ */
14
+
15
+ import type {
16
+ FoldingStrategy,
17
+ FoldingState,
18
+ FoldingBudget,
19
+ FoldOp,
20
+ ChunkId,
21
+ ChunkView,
22
+ SummaryId,
23
+ } from '../folding-strategy.js';
24
+
25
+ export class FlatProfileStrategy implements FoldingStrategy {
26
+ readonly name = 'flat-profile';
27
+
28
+ selectNextFold(state: FoldingState, budget: FoldingBudget): FoldOp | null {
29
+ if (state.tokenCount() <= budget.targetBudget) return null;
30
+
31
+ const middle = state.foldableMiddle();
32
+ if (middle.length === 0) return null;
33
+
34
+ // Count rendered items at each level.
35
+ // A "rendered item" at level k > 0 is a distinct L_k ancestor.
36
+ // At level 0, each raw chunk is its own item.
37
+ const visibleAtLevel = new Map<number, Set<SummaryId | ChunkId>>();
38
+ for (const chunk of middle) {
39
+ const k = chunk.currentResolution;
40
+ if (!visibleAtLevel.has(k)) visibleAtLevel.set(k, new Set());
41
+ if (k === 0) {
42
+ visibleAtLevel.get(k)!.add(chunk.id);
43
+ } else {
44
+ const ancestor = chunk.ancestorAt(k);
45
+ if (ancestor) visibleAtLevel.get(k)!.add(ancestor.id);
46
+ }
47
+ }
48
+
49
+ // Pick most-populous level; tiebreak: lower wins.
50
+ let bestLevel = -1;
51
+ let bestCount = 0;
52
+ for (const [level, items] of visibleAtLevel) {
53
+ const count = items.size;
54
+ if (count > bestCount || (count === bestCount && (bestLevel < 0 || level < bestLevel))) {
55
+ bestLevel = level;
56
+ bestCount = count;
57
+ }
58
+ }
59
+ if (bestLevel < 0) return null;
60
+
61
+ // Find the oldest raisable group at bestLevel. A "group" at level k is
62
+ // the set of chunks sharing an L_{k+1} ancestor. We want to raise that
63
+ // group all the way to L_{k+1}.
64
+ //
65
+ // The oldest group is the one containing the oldest chunk at level k.
66
+ const oldestChunk = middle.find((c) => c.currentResolution === bestLevel);
67
+ if (!oldestChunk) return null;
68
+
69
+ const parentLevel = bestLevel + 1;
70
+ const parentSummary = oldestChunk.ancestorAt(parentLevel);
71
+ if (!parentSummary) {
72
+ // L_{k+1} not produced for this chunk's group yet — request it.
73
+ //
74
+ // Range semantics differ by level:
75
+ // - bestLevel === 0: the caller is asking for an L1 covering this
76
+ // raw chunk. In the autobio chunker model the chunk-to-L1 mapping
77
+ // is 1:1 — a chunk's message span IS the L1's source range — so
78
+ // a single-chunk range is the correct request. (Strategies that
79
+ // bundle N raw chunks into one L1 would override this.)
80
+ // - bestLevel > 0: the caller is asking for an L_{k+1} merge. We
81
+ // use the chunk's existing L_k ancestor's range as a request
82
+ // hint; the consumer expands it to cover the full sibling set
83
+ // at L_k that should consolidate into the missing L_{k+1}.
84
+ const lkAncestor = bestLevel === 0 ? null : oldestChunk.ancestorAt(bestLevel);
85
+ const range = lkAncestor
86
+ ? {
87
+ firstChunkId: lkAncestor.sourceRange.first,
88
+ lastChunkId: lkAncestor.sourceRange.last,
89
+ }
90
+ : { firstChunkId: oldestChunk.id, lastChunkId: oldestChunk.id };
91
+ return { kind: 'produce', level: parentLevel, range };
92
+ }
93
+
94
+ return { kind: 'raise', groupRoot: parentSummary.id };
95
+ }
96
+ }
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Oldest-first folding strategy.
3
+ *
4
+ * Always raise the chronologically-oldest raisable chunk's group, regardless
5
+ * of level distribution. This is the "memory fades chronologically" policy
6
+ * — the simplest possible monotonic strategy and the one originally proposed
7
+ * in rev 1 of the design doc.
8
+ *
9
+ * Kept for comparison against FlatProfileStrategy and as a fallback during
10
+ * early rollout.
11
+ *
12
+ * See `docs/adaptive-resolution-design.md` §3.5.
13
+ */
14
+
15
+ import type {
16
+ FoldingStrategy,
17
+ FoldingState,
18
+ FoldingBudget,
19
+ FoldOp,
20
+ } from '../folding-strategy.js';
21
+
22
+ export class OldestFirstStrategy implements FoldingStrategy {
23
+ readonly name = 'oldest-first';
24
+
25
+ selectNextFold(state: FoldingState, budget: FoldingBudget): FoldOp | null {
26
+ if (state.tokenCount() <= budget.targetBudget) return null;
27
+
28
+ const middle = state.foldableMiddle();
29
+ if (middle.length === 0) return null;
30
+
31
+ // Find the oldest raisable chunk.
32
+ const oldest = middle[0];
33
+ const targetLevel = oldest.currentResolution + 1;
34
+ const parent = oldest.ancestorAt(targetLevel);
35
+ if (!parent) {
36
+ const lkAncestor = oldest.currentResolution === 0 ? null : oldest.ancestorAt(oldest.currentResolution);
37
+ const range = lkAncestor
38
+ ? {
39
+ firstChunkId: lkAncestor.sourceRange.first,
40
+ lastChunkId: lkAncestor.sourceRange.last,
41
+ }
42
+ : { firstChunkId: oldest.id, lastChunkId: oldest.id };
43
+ return { kind: 'produce', level: targetLevel, range };
44
+ }
45
+
46
+ return { kind: 'raise', groupRoot: parent.id };
47
+ }
48
+ }