dsh-compaction-adaptive 0.0.0-stage → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +170 -2
- package/cordis.patch.yml +17 -0
- package/docs/DETAIL.md +174 -0
- package/icon.svg +18 -0
- package/lib/chunking.js +459 -0
- package/lib/index.js +667 -0
- package/lib/policy.js +315 -0
- package/locale/en.json +7 -0
- package/locale/zh.json +7 -0
- package/package.json +51 -4
package/lib/index.js
ADDED
|
@@ -0,0 +1,667 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adaptive compaction backend: the shipped replay-aware engine, re-planned per
|
|
3
|
+
* routed model and per measured session growth.
|
|
4
|
+
*
|
|
5
|
+
* Why a subclass: the durable compaction protocol (start marker, replay, one
|
|
6
|
+
* summary replacement, lock, retries, overflow recovery) already lives in
|
|
7
|
+
* `BasicCompactionEngine` and must not be re-implemented. Only the numeric
|
|
8
|
+
* policy is replaced: before each automatic decision the effective budget for
|
|
9
|
+
* the routed model is recomputed from its window, the reserved output, and an
|
|
10
|
+
* exponential moving average of how many tokens the last turns added, then
|
|
11
|
+
* installed as that model's policy entry. The base engine then prices the
|
|
12
|
+
* request, picks its range, and summarizes exactly as before.
|
|
13
|
+
*
|
|
14
|
+
* @module dsh-compaction-adaptive
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { BasicCompactionEngine } from '@deepseek-ai/dsh-compaction-basic'
|
|
18
|
+
import { BlockAssembler, LlmError } from '@deepseek-ai/dsh-llm'
|
|
19
|
+
import z from '@deepseek-ai/schemastery'
|
|
20
|
+
import {
|
|
21
|
+
conservativeTokens,
|
|
22
|
+
digestMessages,
|
|
23
|
+
isSizeFailure,
|
|
24
|
+
mapLimit,
|
|
25
|
+
mergeInstruction,
|
|
26
|
+
mergeMessages,
|
|
27
|
+
nextScale,
|
|
28
|
+
packByBudget,
|
|
29
|
+
planChunking,
|
|
30
|
+
projectMessages,
|
|
31
|
+
resolveChunkConfig,
|
|
32
|
+
streamText,
|
|
33
|
+
utf8Bytes,
|
|
34
|
+
} from './chunking.js'
|
|
35
|
+
import {
|
|
36
|
+
ADAPTIVE_KEYS,
|
|
37
|
+
describePlan,
|
|
38
|
+
mergeAdaptive,
|
|
39
|
+
nextGrowth,
|
|
40
|
+
planPolicy,
|
|
41
|
+
resolveAdaptiveConfig,
|
|
42
|
+
resolveRoutes,
|
|
43
|
+
routeKey,
|
|
44
|
+
} from './policy.js'
|
|
45
|
+
|
|
46
|
+
/** Fields the base backend resolves itself; everything else is adaptive-only. */
|
|
47
|
+
const BASE_KEYS = Object.freeze([
|
|
48
|
+
'thresholdRatio',
|
|
49
|
+
'headroomTokens',
|
|
50
|
+
'retainRatio',
|
|
51
|
+
'retainTokens',
|
|
52
|
+
'summarizationProvider',
|
|
53
|
+
'summarizationModel',
|
|
54
|
+
'maxTokens',
|
|
55
|
+
'compactionRetries',
|
|
56
|
+
'maxOverflowRetries',
|
|
57
|
+
'modelPolicies',
|
|
58
|
+
'auto',
|
|
59
|
+
])
|
|
60
|
+
|
|
61
|
+
/** Retained per-session observations; derived cache, never persisted. */
|
|
62
|
+
const EMPTY_STATE = Object.freeze({ growth: 0, last: undefined, cooldown: 0 })
|
|
63
|
+
|
|
64
|
+
/** Keep the per-target policy table bounded on long-lived engines. */
|
|
65
|
+
const MAX_POLICY_ENTRIES = 24
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Split raw plugin configuration into base-backend and adaptive halves.
|
|
69
|
+
* @param {Record<string, any>} config - raw row configuration.
|
|
70
|
+
* @returns {Record<string, any>} the base-backend half.
|
|
71
|
+
*/
|
|
72
|
+
function baseConfigOf(config) {
|
|
73
|
+
const base = {}
|
|
74
|
+
for (const key of BASE_KEYS) {
|
|
75
|
+
if (config[key] !== undefined) base[key] = config[key]
|
|
76
|
+
}
|
|
77
|
+
return base
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Adaptive compaction engine exposed as `ctx.compaction`. */
|
|
81
|
+
export default class AdaptiveCompactionEngine extends BasicCompactionEngine {
|
|
82
|
+
/**
|
|
83
|
+
* Row schema. Every documented field is typed so the settings surface can
|
|
84
|
+
* render it; `resolveAdaptiveConfig` remains the single authority on ranges.
|
|
85
|
+
*/
|
|
86
|
+
static Config = z.object({
|
|
87
|
+
mode: z.string(),
|
|
88
|
+
profile: z.string(),
|
|
89
|
+
turnsAhead: z.number(),
|
|
90
|
+
growthAlpha: z.number(),
|
|
91
|
+
floorRatio: z.number(),
|
|
92
|
+
ceilRatio: z.number(),
|
|
93
|
+
headroomMinRatio: z.number(),
|
|
94
|
+
headroomMaxRatio: z.number(),
|
|
95
|
+
minHeadroomTokens: z.number().step(1).min(0),
|
|
96
|
+
tailRatio: z.number(),
|
|
97
|
+
tailMinTokens: z.number().step(1).min(0),
|
|
98
|
+
tailMaxRatio: z.number(),
|
|
99
|
+
keepTurns: z.number(),
|
|
100
|
+
cooldownSteps: z.number().step(1).min(0),
|
|
101
|
+
thrashMargin: z.number(),
|
|
102
|
+
minGrowthTokens: z.number().step(1).min(0),
|
|
103
|
+
verbose: z.boolean(),
|
|
104
|
+
thresholdRatio: z.number(),
|
|
105
|
+
headroomTokens: z.number().step(1).min(0),
|
|
106
|
+
retainRatio: z.number(),
|
|
107
|
+
retainTokens: z.number().step(1).min(0),
|
|
108
|
+
summarizationProvider: z.string(),
|
|
109
|
+
summarizationModel: z.string(),
|
|
110
|
+
maxTokens: z.number().step(1).min(1),
|
|
111
|
+
compactionRetries: z.number().step(1).min(0),
|
|
112
|
+
maxOverflowRetries: z.number().step(1).min(0),
|
|
113
|
+
modelPolicies: z.array(z.any()),
|
|
114
|
+
routes: z.array(z.any()),
|
|
115
|
+
chunking: z.string(),
|
|
116
|
+
chunkTriggerBytes: z.number().step(1).min(10_000),
|
|
117
|
+
chunkBytes: z.number().step(1).min(10_000),
|
|
118
|
+
chunkConcurrency: z.number().step(1).min(1),
|
|
119
|
+
chunkMaxTokens: z.number().step(1).min(128),
|
|
120
|
+
mergeMaxTokens: z.number().step(1).min(128),
|
|
121
|
+
chunkRetries: z.number().step(1).min(0),
|
|
122
|
+
auto: z.boolean(),
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* @param {import('@deepseek-ai/cordis').Context} ctx - plugin context.
|
|
127
|
+
* @param {Record<string, any>} [config] - row configuration.
|
|
128
|
+
*/
|
|
129
|
+
constructor(ctx, config = {}) {
|
|
130
|
+
super(ctx, baseConfigOf(config))
|
|
131
|
+
/** @type {Readonly<Record<string, any>>} */
|
|
132
|
+
this.adaptive = resolveAdaptiveConfig(config)
|
|
133
|
+
/** @type {ReadonlyMap<string, Readonly<Record<string, any>>>} */
|
|
134
|
+
this.routes = resolveRoutes(config.routes)
|
|
135
|
+
/** @type {Readonly<Record<string, any>>} */
|
|
136
|
+
this.chunk = resolveChunkConfig(config)
|
|
137
|
+
/** @type {Map<string, number>} learned per-summarizer capacity multipliers. */
|
|
138
|
+
this.scales = new Map()
|
|
139
|
+
/** @type {WeakMap<object, { growth: number, last: number | undefined, cooldown: number }>} */
|
|
140
|
+
this.sessions = new WeakMap()
|
|
141
|
+
/** @type {Map<string, string>} */
|
|
142
|
+
this.warned = new Map()
|
|
143
|
+
if (this.adaptive.mode === 'adaptive') this._installRequestHook()
|
|
144
|
+
ctx.logger?.info(
|
|
145
|
+
`[compaction-adaptive] ${this.adaptive.mode} mode, ${this.adaptive.profile} profile, `
|
|
146
|
+
+ `${this.adaptive.turnsAhead} turn(s) of headroom, tail >= ${this.adaptive.tailMinTokens} tokens, `
|
|
147
|
+
+ `chunking ${this.chunk.chunking}/${this.chunk.chunkSizing}`,
|
|
148
|
+
)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Re-plan the proposed route before the base engine prices a model switch.
|
|
153
|
+
* The hook is registered as an ordinary waterfall listener, so it runs inside
|
|
154
|
+
* the routing listeners that the base engine awaits, and never changes the
|
|
155
|
+
* selected configuration.
|
|
156
|
+
*/
|
|
157
|
+
_installRequestHook() {
|
|
158
|
+
this.ctx.on('agent/request', async ({ agent, signal }, next) => {
|
|
159
|
+
const config = await next()
|
|
160
|
+
try {
|
|
161
|
+
if (signal?.aborted) return config
|
|
162
|
+
const provider = config?.provider
|
|
163
|
+
const model = config?.model
|
|
164
|
+
if (typeof provider !== 'string' || typeof model !== 'string' || !provider || !model) {
|
|
165
|
+
return config
|
|
166
|
+
}
|
|
167
|
+
// Only a change of route needs a fresh plan here: an unchanged route is
|
|
168
|
+
// planned by the pre-step path, which also owns growth observation.
|
|
169
|
+
const previous = agent.session?.requestHeader?.()?.config
|
|
170
|
+
if (previous?.provider === provider && previous?.model === model) return config
|
|
171
|
+
await this._observe(agent, agent.session?.requestHeader?.(), { provider, model }, {
|
|
172
|
+
signal,
|
|
173
|
+
reservedOverride: config.maxTokens,
|
|
174
|
+
log: true,
|
|
175
|
+
})
|
|
176
|
+
} catch (error) {
|
|
177
|
+
this._warnOnce('request-hook', error)
|
|
178
|
+
}
|
|
179
|
+
return config
|
|
180
|
+
})
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Automatic compaction entry point: plan, install, then delegate.
|
|
185
|
+
* @param {any} agent - agent requesting the step.
|
|
186
|
+
* @param {'pressure' | 'context-overflow'} trigger - why policy is asking.
|
|
187
|
+
* @param {AbortSignal} signal - turn cancellation.
|
|
188
|
+
* @returns {Promise<any>} the base result, or `null`.
|
|
189
|
+
*/
|
|
190
|
+
async compactIfNeeded(agent, trigger, signal) {
|
|
191
|
+
if (this.adaptive.mode !== 'adaptive') return super.compactIfNeeded(agent, trigger, signal)
|
|
192
|
+
|
|
193
|
+
const header = agent.session?.requestHeader?.()
|
|
194
|
+
const routed = header?.config
|
|
195
|
+
if (!routed || !routed.provider || !routed.model) {
|
|
196
|
+
return super.compactIfNeeded(agent, trigger, signal)
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
let planned = null
|
|
200
|
+
try {
|
|
201
|
+
planned = await this._observe(
|
|
202
|
+
agent,
|
|
203
|
+
header,
|
|
204
|
+
{ provider: routed.provider, model: routed.model },
|
|
205
|
+
{ signal, force: trigger === 'context-overflow', log: true },
|
|
206
|
+
)
|
|
207
|
+
} catch (error) {
|
|
208
|
+
if (signal?.aborted) throw error
|
|
209
|
+
this._warnOnce(`${routed.provider}/${routed.model}`, error)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const state = this._state(agent.session)
|
|
213
|
+
if (planned !== null && trigger === 'pressure' && this._shouldHold(planned, state)) {
|
|
214
|
+
if (this.adaptive.verbose) {
|
|
215
|
+
this.ctx.logger?.info(
|
|
216
|
+
`[compaction-adaptive] holding after a recent compaction: `
|
|
217
|
+
+ `${planned.measurement.totalTokens} < `
|
|
218
|
+
+ `${Math.round(planned.plan.thresholdTokens * this.adaptive.thrashMargin)}`,
|
|
219
|
+
)
|
|
220
|
+
}
|
|
221
|
+
return null
|
|
222
|
+
}
|
|
223
|
+
state.cooldown = 0
|
|
224
|
+
|
|
225
|
+
const result = await super.compactIfNeeded(agent, trigger, signal)
|
|
226
|
+
if (result !== null && result !== undefined) {
|
|
227
|
+
state.cooldown = this.adaptive.cooldownSteps
|
|
228
|
+
state.last = undefined
|
|
229
|
+
this.ctx.logger?.info(
|
|
230
|
+
`[compaction-adaptive] shadowed ${result.shadowedSeqs.length} nodes `
|
|
231
|
+
+ `(~${result.shadowedTokenCount} tokens)`,
|
|
232
|
+
)
|
|
233
|
+
}
|
|
234
|
+
return result
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Measure the surface once, fold growth, plan the routed model's budget, and
|
|
239
|
+
* install it as that model's policy entry.
|
|
240
|
+
* @param {any} agent - agent owning the measured session.
|
|
241
|
+
* @param {any} header - latest durable request header, when known.
|
|
242
|
+
* @param {{provider: string, model: string}} target - routed model.
|
|
243
|
+
* @param {{signal?: AbortSignal, force?: boolean, reservedOverride?: number, log?: boolean}} options - planning options.
|
|
244
|
+
* @returns {Promise<null | {plan: any, measurement: any}>} the plan and measurement.
|
|
245
|
+
*/
|
|
246
|
+
async _observe(agent, header, target, options = {}) {
|
|
247
|
+
const info = await this.ctx.llm.resolveModelInfo(target.provider, target.model, options.signal)
|
|
248
|
+
const contextWindow = info?.context?.contextWindow
|
|
249
|
+
const measurement = this.ctx.tokenMeter.measure(agent.session, header)
|
|
250
|
+
const state = this._state(agent.session)
|
|
251
|
+
|
|
252
|
+
const previous = state.last
|
|
253
|
+
const delta = previous === undefined ? undefined : measurement.totalTokens - previous
|
|
254
|
+
if (delta !== undefined && delta > this.adaptive.minGrowthTokens) {
|
|
255
|
+
state.growth = nextGrowth(state.growth, delta, this.adaptive.growthAlpha)
|
|
256
|
+
}
|
|
257
|
+
state.last = measurement.totalTokens
|
|
258
|
+
|
|
259
|
+
const reservedTokens = options.reservedOverride ?? header?.config?.maxTokens ?? info?.defaultMaxTokens ?? 0
|
|
260
|
+
const adaptive = this._adaptiveFor(target)
|
|
261
|
+
const plan = planPolicy({
|
|
262
|
+
contextWindow,
|
|
263
|
+
reservedTokens,
|
|
264
|
+
growthTokens: state.growth,
|
|
265
|
+
adaptive,
|
|
266
|
+
force: options.force === true,
|
|
267
|
+
})
|
|
268
|
+
if (plan === null) return null
|
|
269
|
+
|
|
270
|
+
this._installPlan(target, plan)
|
|
271
|
+
if (options.log) {
|
|
272
|
+
this.ctx.logger?.info(`[compaction-adaptive] ${describePlan(plan, `${target.provider}/${target.model}`)}`)
|
|
273
|
+
}
|
|
274
|
+
return { plan, measurement, adaptive }
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Effective adaptive settings for one route: the service defaults, with the
|
|
279
|
+
* route's partial overrides applied. Route overrides exist for cost control
|
|
280
|
+
* (an expensive prefill wants a lower ceiling), so they must win over the
|
|
281
|
+
* growth-derived target whenever they are stricter.
|
|
282
|
+
* @param {{provider: string, model: string}} target - routed model.
|
|
283
|
+
* @returns {Record<string, any>} effective adaptive settings.
|
|
284
|
+
*/
|
|
285
|
+
_adaptiveFor(target) {
|
|
286
|
+
const override = this.routes.get(routeKey(target.provider, target.model))
|
|
287
|
+
return mergeAdaptive(this.adaptive, override)
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Summarization hook shared with the base backend.
|
|
292
|
+
*
|
|
293
|
+
* The single-shot request is measured in the currency its consumer cares
|
|
294
|
+
* about — the summary model's own token budget — with the transport byte cap
|
|
295
|
+
* as a second guard. A request that fits keeps the base behavior, including
|
|
296
|
+
* its prefix-cache reuse; one that does not is digested in parts sized from
|
|
297
|
+
* that same budget and folded back into one checkpoint. This replaces the
|
|
298
|
+
* separate stream-interception plugin that used to handle the oversized case.
|
|
299
|
+
*
|
|
300
|
+
* The multi-call result is deliberately *unmarked* (`llmStreamCall` absent):
|
|
301
|
+
* the contract reserves that marker for a summary produced by exactly one
|
|
302
|
+
* `ctx.llm.stream()` call, and the durable record then keeps only the output.
|
|
303
|
+
*
|
|
304
|
+
* @param {any} input - replayed conversation prefix to condense.
|
|
305
|
+
* @param {any} agent - routed model, session, and tool history.
|
|
306
|
+
* @param {AbortSignal} [signal] - cancellation forwarded to every call.
|
|
307
|
+
* @returns {Promise<any>} summary blocks and the call envelope.
|
|
308
|
+
*/
|
|
309
|
+
async summarize(input, agent, signal) {
|
|
310
|
+
if (this.chunk.chunking === 'off') return super.summarize(input, agent, signal)
|
|
311
|
+
let prepared
|
|
312
|
+
try {
|
|
313
|
+
prepared = await this._chunkPlan(input, agent)
|
|
314
|
+
} catch (error) {
|
|
315
|
+
this._warnOnce('chunk-plan', error)
|
|
316
|
+
return super.summarize(input, agent, signal)
|
|
317
|
+
}
|
|
318
|
+
if (!prepared.plan.needed) return super.summarize(input, agent, signal)
|
|
319
|
+
return this._chunkedSummarize(prepared, agent, signal)
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Measure the single-shot request and derive the chunk budgets from the model
|
|
324
|
+
* that will summarize it.
|
|
325
|
+
* @param {any} input - summarization input.
|
|
326
|
+
* @param {any} agent - owning agent.
|
|
327
|
+
* @returns {Promise<{target: any, system: string | null, parts: string[], capacity: any, contentTokens: number, contentBytes: number, plan: any}>} plan.
|
|
328
|
+
*/
|
|
329
|
+
async _chunkPlan(input, agent) {
|
|
330
|
+
const target = this._summarizationTarget(agent)
|
|
331
|
+
const info = await this.ctx.llm.resolveModelInfo(target.provider, target.model)
|
|
332
|
+
const { system, parts } = projectMessages(input.messages ?? [])
|
|
333
|
+
const estimate = (text) => this._estimateText(text)
|
|
334
|
+
const systemTokens = system === null ? 0 : conservativeTokens(system, estimate(system))
|
|
335
|
+
let contentTokens = systemTokens
|
|
336
|
+
let contentBytes = system === null ? 0 : utf8Bytes(system)
|
|
337
|
+
for (const part of parts) {
|
|
338
|
+
contentTokens += conservativeTokens(part, estimate(part))
|
|
339
|
+
contentBytes += utf8Bytes(part) + 1
|
|
340
|
+
}
|
|
341
|
+
const toolsBytes = input.tools === undefined ? 0 : utf8Bytes(JSON.stringify(input.tools))
|
|
342
|
+
const capacity = {
|
|
343
|
+
window: info?.context?.contextWindow ?? 0,
|
|
344
|
+
reservedTokens: target.maxTokens ?? 0,
|
|
345
|
+
overheadTokens: systemTokens + this._estimateText(mergeInstruction(0)) + Math.ceil(toolsBytes / 3),
|
|
346
|
+
}
|
|
347
|
+
const plan = planChunking({
|
|
348
|
+
contentTokens,
|
|
349
|
+
contentBytes,
|
|
350
|
+
capacity,
|
|
351
|
+
config: this.chunk,
|
|
352
|
+
scale: this._scaleFor(target),
|
|
353
|
+
})
|
|
354
|
+
return { target, system, parts, capacity, contentTokens, contentBytes, plan }
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Digest the shadowed content in parts, then fold the digests into one
|
|
359
|
+
* checkpoint — shrinking the parts and retrying when the route rejects a
|
|
360
|
+
* request as too large, and recovering the learned capacity after success.
|
|
361
|
+
* @param {any} prepared - measured plan from {@link _chunkPlan}.
|
|
362
|
+
* @param {any} agent - owning agent.
|
|
363
|
+
* @param {AbortSignal | undefined} signal - cancellation.
|
|
364
|
+
* @returns {Promise<any>} unmarked summary result.
|
|
365
|
+
*/
|
|
366
|
+
async _chunkedSummarize(prepared, agent, signal) {
|
|
367
|
+
const { target, system, parts, capacity, contentTokens, contentBytes } = prepared
|
|
368
|
+
const key = routeKey(target.provider, target.model)
|
|
369
|
+
const stream = (options) => this.ctx.llm.stream(options)
|
|
370
|
+
const base = {
|
|
371
|
+
provider: target.provider,
|
|
372
|
+
model: target.model,
|
|
373
|
+
sessionId: agent.session?.id,
|
|
374
|
+
purpose: 'compaction',
|
|
375
|
+
...signal === undefined ? {} : { signal },
|
|
376
|
+
}
|
|
377
|
+
const estimate = (text) => this._estimateText(text)
|
|
378
|
+
|
|
379
|
+
let attempt = 0
|
|
380
|
+
for (;;) {
|
|
381
|
+
const plan = planChunking({
|
|
382
|
+
contentTokens,
|
|
383
|
+
contentBytes,
|
|
384
|
+
capacity,
|
|
385
|
+
config: this.chunk,
|
|
386
|
+
scale: this._scaleFor(target),
|
|
387
|
+
})
|
|
388
|
+
const chunks = packByBudget(
|
|
389
|
+
parts,
|
|
390
|
+
{ tokenBudget: plan.partTokenBudget, byteBudget: plan.partByteBudget },
|
|
391
|
+
estimate,
|
|
392
|
+
)
|
|
393
|
+
if (chunks.length === 0) return super.summarize(prepared, agent, signal)
|
|
394
|
+
|
|
395
|
+
this.ctx.logger?.info(
|
|
396
|
+
`[compaction-adaptive] chunked summary (${plan.reason}): ${chunks.length} parts `
|
|
397
|
+
+ `<= ${plan.partTokenBudget} tokens / ${plan.partByteBudget} B each, `
|
|
398
|
+
+ `concurrency ${this.chunk.chunkConcurrency}, capacity scale ${(plan.scale * 100).toFixed(0)}%`,
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
try {
|
|
402
|
+
const digests = await mapLimit(chunks, this.chunk.chunkConcurrency, (text, index) => (
|
|
403
|
+
this._withRetries(signal, (label) => this._call(stream, base, text, index, chunks.length, system, label))
|
|
404
|
+
))
|
|
405
|
+
signal?.throwIfAborted()
|
|
406
|
+
const merged = await this._fold(target, system, digests, capacity, signal, base)
|
|
407
|
+
this.scales.set(key, nextScale(this._scaleFor(target), 'recover', this.chunk))
|
|
408
|
+
this.ctx.logger?.info(
|
|
409
|
+
`[compaction-adaptive] chunked summary done: ${chunks.length} digests folded into ${merged.length} chars`,
|
|
410
|
+
)
|
|
411
|
+
const summary = [{ type: 'text', text: merged }]
|
|
412
|
+
return {
|
|
413
|
+
summary,
|
|
414
|
+
rawOutput: summary,
|
|
415
|
+
provider: target.provider,
|
|
416
|
+
model: target.model,
|
|
417
|
+
maxTokens: this.chunk.mergeMaxTokens,
|
|
418
|
+
}
|
|
419
|
+
} catch (error) {
|
|
420
|
+
if (signal?.aborted) throw error
|
|
421
|
+
if (!isSizeFailure(error) || attempt >= this.chunk.chunkShrinkAttempts) throw error
|
|
422
|
+
attempt += 1
|
|
423
|
+
const next = nextScale(this._scaleFor(target), 'shrink', this.chunk)
|
|
424
|
+
this.scales.set(key, next)
|
|
425
|
+
this.ctx.logger?.warn(
|
|
426
|
+
`[compaction-adaptive] ${key} rejected a chunked request (${error instanceof Error ? error.message : String(error)}); `
|
|
427
|
+
+ `shrinking capacity to ${(next * 100).toFixed(0)}% and retrying`,
|
|
428
|
+
)
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* One digest call for a packed part.
|
|
435
|
+
* @param {(options: any) => AsyncIterable<any>} stream - LLM stream factory.
|
|
436
|
+
* @param {Record<string, any>} base - shared generate options.
|
|
437
|
+
* @param {string} text - packed part text.
|
|
438
|
+
* @param {number} index - part index.
|
|
439
|
+
* @param {number} total - part count.
|
|
440
|
+
* @param {string | null} system - replayed system head.
|
|
441
|
+
* @param {string} label - retry label.
|
|
442
|
+
* @returns {Promise<string>} digest text.
|
|
443
|
+
*/
|
|
444
|
+
async _call(stream, base, text, index, total, system, label) {
|
|
445
|
+
try {
|
|
446
|
+
return await streamText(
|
|
447
|
+
stream,
|
|
448
|
+
{ ...base, maxTokens: this.chunk.chunkMaxTokens },
|
|
449
|
+
digestMessages(system, text, index, total),
|
|
450
|
+
{ BlockAssembler, LlmError },
|
|
451
|
+
)
|
|
452
|
+
} catch (error) {
|
|
453
|
+
throw new Error(`${label}: ${error instanceof Error ? error.message : String(error)}`, { cause: error })
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* Fold digests into the structured checkpoint, grouping them first when the
|
|
459
|
+
* digests themselves no longer fit one request.
|
|
460
|
+
* @param {any} target - summarization route.
|
|
461
|
+
* @param {string | null} system - replayed system head.
|
|
462
|
+
* @param {string[]} digests - per-part notes.
|
|
463
|
+
* @param {any} capacity - summarizer capacity.
|
|
464
|
+
* @param {AbortSignal | undefined} signal - cancellation.
|
|
465
|
+
* @param {Record<string, any>} base - shared generate options.
|
|
466
|
+
* @returns {Promise<string>} checkpoint text.
|
|
467
|
+
*/
|
|
468
|
+
async _fold(target, system, digests, capacity, signal, base) {
|
|
469
|
+
const stream = (options) => this.ctx.llm.stream(options)
|
|
470
|
+
const foldBudget = Math.max(
|
|
471
|
+
1_024,
|
|
472
|
+
Math.floor(
|
|
473
|
+
Math.max(0, capacity.window - capacity.reservedTokens - capacity.overheadTokens)
|
|
474
|
+
* (this.chunk.chunkSizing === 'fixed' ? 1 : this.chunk.chunkSingleShotRatio)
|
|
475
|
+
* this._scaleFor(target),
|
|
476
|
+
),
|
|
477
|
+
)
|
|
478
|
+
let notes = digests
|
|
479
|
+
for (let round = 0; round < 3; round += 1) {
|
|
480
|
+
const estimate = (text) => this._estimateText(text)
|
|
481
|
+
const total = notes.reduce((sum, note) => sum + conservativeTokens(note, estimate(note)), 0)
|
|
482
|
+
if (total <= foldBudget) break
|
|
483
|
+
const groups = packByBudget(
|
|
484
|
+
notes,
|
|
485
|
+
{ tokenBudget: Math.max(1_024, Math.floor(foldBudget * 0.5)), byteBudget: this.chunk.chunkBytes },
|
|
486
|
+
estimate,
|
|
487
|
+
)
|
|
488
|
+
if (groups.length >= notes.length) break
|
|
489
|
+
this.ctx.logger?.info(
|
|
490
|
+
`[compaction-adaptive] digests (${total} tokens) exceed the fold budget ${foldBudget}; `
|
|
491
|
+
+ `condensing ${notes.length} → ${groups.length} in round ${round + 1}`,
|
|
492
|
+
)
|
|
493
|
+
notes = await mapLimit(groups, this.chunk.chunkConcurrency, (text, index) => (
|
|
494
|
+
this._withRetries(signal, (label) => this._call(stream, base, text, index, groups.length, system, label))
|
|
495
|
+
))
|
|
496
|
+
signal?.throwIfAborted()
|
|
497
|
+
}
|
|
498
|
+
return this._withRetries(signal, () => streamText(
|
|
499
|
+
stream,
|
|
500
|
+
{ ...base, maxTokens: this.chunk.mergeMaxTokens },
|
|
501
|
+
mergeMessages(system, notes),
|
|
502
|
+
{ BlockAssembler, LlmError },
|
|
503
|
+
))
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* Conservative token estimate for arbitrary text.
|
|
508
|
+
* @param {string} text - measured text.
|
|
509
|
+
* @returns {number} tokens.
|
|
510
|
+
*/
|
|
511
|
+
_estimateText(text) {
|
|
512
|
+
const meter = this.ctx.tokenMeter
|
|
513
|
+
if (typeof meter?.estimateMessage === 'function') {
|
|
514
|
+
try {
|
|
515
|
+
return meter.estimateMessage({ role: 'user', content: [{ type: 'text', text }] })
|
|
516
|
+
} catch {
|
|
517
|
+
// Fall through to the character heuristic below.
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
return Math.ceil(text.length / 3)
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Learned capacity multiplier for a route.
|
|
525
|
+
* @param {{provider: string, model: string}} target - summarization route.
|
|
526
|
+
* @returns {number} multiplier in [chunkMinScale, 1].
|
|
527
|
+
*/
|
|
528
|
+
_scaleFor(target) {
|
|
529
|
+
const scale = this.scales.get(routeKey(target.provider, target.model))
|
|
530
|
+
return Number.isFinite(scale) ? scale : 1
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/**
|
|
534
|
+
* Resolve the summarization route and budget, mirroring the base backend:
|
|
535
|
+
* an explicit per-route or service-level summarization pair wins, otherwise
|
|
536
|
+
* the conversation's own routed model summarizes itself.
|
|
537
|
+
* @param {any} agent - owning agent.
|
|
538
|
+
* @returns {{provider: string, model: string, maxTokens: number}} target.
|
|
539
|
+
*/
|
|
540
|
+
_summarizationTarget(agent) {
|
|
541
|
+
const routed = agent.session?.requestHeader?.()?.config
|
|
542
|
+
const target = routed?.provider && routed?.model
|
|
543
|
+
? { provider: routed.provider, model: routed.model }
|
|
544
|
+
: agent.options?.provider && agent.options?.model
|
|
545
|
+
? { provider: agent.options.provider, model: agent.options.model }
|
|
546
|
+
: undefined
|
|
547
|
+
const policy = target === undefined ? this.config : this._policyFor(target)
|
|
548
|
+
const configured = policy.summarizationProvider
|
|
549
|
+
? { provider: policy.summarizationProvider, model: policy.summarizationModel }
|
|
550
|
+
: undefined
|
|
551
|
+
const resolved = configured ?? target ?? undefined
|
|
552
|
+
if (resolved === undefined) {
|
|
553
|
+
throw new Error('chunked summarization: no provider/model available to summarize with')
|
|
554
|
+
}
|
|
555
|
+
return { provider: resolved.provider, model: resolved.model, maxTokens: this.chunk.mergeMaxTokens }
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Read the effective policy for one routed model (explicit entry over the
|
|
560
|
+
* service defaults), without re-planning it.
|
|
561
|
+
* @param {{provider: string, model: string}} target - routed model.
|
|
562
|
+
* @returns {Record<string, any>} effective policy fields.
|
|
563
|
+
*/
|
|
564
|
+
_policyFor(target) {
|
|
565
|
+
const entry = (this.config.modelPolicies ?? []).find((policy) => (
|
|
566
|
+
policy.provider === target.provider && policy.model === target.model
|
|
567
|
+
))
|
|
568
|
+
return {
|
|
569
|
+
summarizationProvider: entry?.summarizationProvider ?? this.config.summarizationProvider,
|
|
570
|
+
summarizationModel: entry?.summarizationModel ?? this.config.summarizationModel,
|
|
571
|
+
maxTokens: entry?.maxTokens ?? this.config.maxTokens,
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* Retry one call a bounded number of times, never past cancellation.
|
|
577
|
+
* @param {AbortSignal | undefined} signal - cancellation.
|
|
578
|
+
* @param {(label: string) => Promise<any>} run - call to attempt.
|
|
579
|
+
* @returns {Promise<any>} the successful result.
|
|
580
|
+
*/
|
|
581
|
+
async _withRetries(signal, run) {
|
|
582
|
+
let attempt = 0
|
|
583
|
+
for (;;) {
|
|
584
|
+
signal?.throwIfAborted()
|
|
585
|
+
try {
|
|
586
|
+
return await run(attempt === 0 ? 'chunk summarization' : `chunk summarization (retry ${attempt})`)
|
|
587
|
+
} catch (error) {
|
|
588
|
+
signal?.throwIfAborted()
|
|
589
|
+
if (attempt >= this.chunk.chunkRetries) throw error
|
|
590
|
+
attempt += 1
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/**
|
|
596
|
+
* Record one routed model's adaptive budget without discarding the explicit
|
|
597
|
+
* routing a user configured for that model (its summarization target, for
|
|
598
|
+
* instance, which must survive re-planning).
|
|
599
|
+
* @param {{provider: string, model: string}} target - routed model.
|
|
600
|
+
* @param {any} plan - planned budget.
|
|
601
|
+
*/
|
|
602
|
+
_installPlan(target, plan) {
|
|
603
|
+
const current = this.config
|
|
604
|
+
const policies = current.modelPolicies ?? []
|
|
605
|
+
const existing = policies.find((policy) => (
|
|
606
|
+
policy.provider === target.provider && policy.model === target.model
|
|
607
|
+
))
|
|
608
|
+
/** @type {Record<string, any>} */
|
|
609
|
+
const entry = { ...existing, provider: target.provider, model: target.model }
|
|
610
|
+
delete entry.retainRatio
|
|
611
|
+
entry.thresholdRatio = plan.thresholdRatio
|
|
612
|
+
entry.headroomTokens = plan.headroomTokens
|
|
613
|
+
entry.retainTokens = plan.retainTokens
|
|
614
|
+
|
|
615
|
+
const others = policies.filter((policy) => (
|
|
616
|
+
policy.provider !== target.provider || policy.model !== target.model
|
|
617
|
+
))
|
|
618
|
+
const next = [...others, entry].slice(-MAX_POLICY_ENTRIES)
|
|
619
|
+
// The base engine re-reads `config` at every decision and at summary time,
|
|
620
|
+
// so replacing the resolved value is enough and leaves no cross-agent state.
|
|
621
|
+
this.config = { ...current, modelPolicies: next }
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* Skip a step immediately after a compaction unless the surface has climbed
|
|
626
|
+
* back past the threshold with a margin: this is what stops two summarization
|
|
627
|
+
* attempts from trading a few tokens back and forth.
|
|
628
|
+
* @param {{plan: any, measurement: any}} planned - current plan and measurement.
|
|
629
|
+
* @param {{cooldown: number}} state - session observation state.
|
|
630
|
+
* @returns {boolean} whether to hold this step.
|
|
631
|
+
*/
|
|
632
|
+
_shouldHold(planned, state) {
|
|
633
|
+
if (state.cooldown <= 0) return false
|
|
634
|
+
state.cooldown -= 1
|
|
635
|
+
const margin = planned.adaptive?.thrashMargin ?? this.adaptive.thrashMargin
|
|
636
|
+
return planned.measurement.totalTokens < planned.plan.thresholdTokens * margin
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* Read or create the derived per-session observation state.
|
|
641
|
+
* @param {object} session - measured session.
|
|
642
|
+
* @returns {{ growth: number, last: number | undefined, cooldown: number }} state.
|
|
643
|
+
*/
|
|
644
|
+
_state(session) {
|
|
645
|
+
let state = this.sessions.get(session)
|
|
646
|
+
if (state === undefined) {
|
|
647
|
+
state = { ...EMPTY_STATE }
|
|
648
|
+
this.sessions.set(session, state)
|
|
649
|
+
}
|
|
650
|
+
return state
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/**
|
|
654
|
+
* Log one warning per key; a missing window declaration must not repeat on
|
|
655
|
+
* every step of a long turn.
|
|
656
|
+
* @param {string} key - deduplication key.
|
|
657
|
+
* @param {unknown} error - failure to report.
|
|
658
|
+
*/
|
|
659
|
+
_warnOnce(key, error) {
|
|
660
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
661
|
+
if (this.warned.get(key) === message) return
|
|
662
|
+
this.warned.set(key, message)
|
|
663
|
+
this.ctx.logger?.warn(`[compaction-adaptive] ${key}: ${message}`)
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
export { ADAPTIVE_KEYS }
|