@snylonue/dsh-context-policy 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +223 -2
- package/assets/deepseek-v4/LICENSE +21 -0
- package/assets/deepseek-v4/README.md +19 -0
- package/assets/deepseek-v4/tokenizer.json.gz +0 -0
- package/assets/o200k-base/LICENSE +21 -0
- package/assets/o200k-base/README.md +20 -0
- package/assets/o200k-base/tokenizer.json.gz +0 -0
- package/cordis.patch.yml +19 -0
- package/dist/builtin-tokenizers.d.ts +2 -0
- package/dist/builtin-tokenizers.d.ts.map +1 -0
- package/dist/builtin-tokenizers.js +15 -0
- package/dist/builtin-tokenizers.js.map +1 -0
- package/dist/compaction.d.ts +43 -0
- package/dist/compaction.d.ts.map +1 -0
- package/dist/compaction.js +198 -0
- package/dist/compaction.js.map +1 -0
- package/dist/core.d.ts +49 -0
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +213 -0
- package/dist/core.js.map +1 -0
- package/dist/projection.d.ts +103 -0
- package/dist/projection.d.ts.map +1 -0
- package/dist/projection.js +137 -0
- package/dist/projection.js.map +1 -0
- package/dist/token-meter.d.ts +46 -0
- package/dist/token-meter.d.ts.map +1 -0
- package/dist/token-meter.js +55 -0
- package/dist/token-meter.js.map +1 -0
- package/dist/tokenizers.d.ts +6 -0
- package/dist/tokenizers.d.ts.map +1 -0
- package/dist/tokenizers.js +64 -0
- package/dist/tokenizers.js.map +1 -0
- package/dist/types.d.ts +62 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/usage.d.ts +67 -0
- package/dist/usage.d.ts.map +1 -0
- package/dist/usage.js +81 -0
- package/dist/usage.js.map +1 -0
- package/examples/preset-compaction.yml +13 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +91 -3
- package/scripts/audit.ts +105 -0
- package/scripts/check.ts +19 -0
- package/scripts/prepare-o200k.ts +40 -0
- package/src/builtin-tokenizers.ts +24 -0
- package/src/compaction.ts +293 -0
- package/src/core.ts +277 -0
- package/src/projection.ts +173 -0
- package/src/token-meter.ts +81 -0
- package/src/tokenizers.ts +78 -0
- package/src/types.ts +63 -0
- package/src/usage.ts +96 -0
- package/tests/bundle.test.ts +43 -0
- package/tests/deepseek-v4-fixtures.json +125 -0
- package/tests/o200k-base-fixtures.json +184 -0
- package/tests/policy.test.ts +773 -0
- package/tests/tokenizers.test.ts +51 -0
- package/tsconfig.check.json +15 -0
- package/tsconfig.json +17 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { gzipSync } from "node:zlib";
|
|
4
|
+
|
|
5
|
+
// Input: Xenova/gpt-4o tokenizer.json at revision
|
|
6
|
+
// 7956d98f2a83b2751a98ea7136fdf7fe6cf54e69 (see assets/o200k-base/README.md).
|
|
7
|
+
const input = process.argv[2];
|
|
8
|
+
if (!input)
|
|
9
|
+
throw new Error("Usage: node scripts/prepare-o200k.ts <tokenizer.json>");
|
|
10
|
+
const source = readFileSync(input);
|
|
11
|
+
if (
|
|
12
|
+
createHash("sha256").update(source).digest("hex") !==
|
|
13
|
+
"43a3ad4618a6a938f8c2614154ea10c31ad53a62d5683ae0b0e6133575cef07e"
|
|
14
|
+
)
|
|
15
|
+
throw new Error("Unexpected o200k_base tokenizer source checksum");
|
|
16
|
+
const data = JSON.parse(source.toString("utf8"));
|
|
17
|
+
data.added_tokens = [];
|
|
18
|
+
// Tiktoken emits a whole pre-tokenized piece when it exists in the vocabulary.
|
|
19
|
+
data.model.ignore_merges = true;
|
|
20
|
+
// Preserve js-tiktoken's ECMAScript whitespace semantics, including FEFF and
|
|
21
|
+
// excluding U+0085. Hugging Face otherwise translates \s as Oniguruma whitespace.
|
|
22
|
+
const whitespace = String.raw`\u0009-\u000D\u0020\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF`;
|
|
23
|
+
const split = data.pre_tokenizer.pretokenizers[0];
|
|
24
|
+
split.pattern.Regex = split.pattern.Regex.replaceAll(
|
|
25
|
+
String.raw`[^\s\p{L}\p{N}]`,
|
|
26
|
+
String.raw`[^${whitespace}\p{L}\p{N}]`,
|
|
27
|
+
)
|
|
28
|
+
.replaceAll(String.raw`\s`, `[${whitespace}]`)
|
|
29
|
+
.replaceAll(String.raw`\S`, `[^${whitespace}]`);
|
|
30
|
+
const output = new URL("../assets/o200k-base/", import.meta.url);
|
|
31
|
+
mkdirSync(output, { recursive: true });
|
|
32
|
+
const json = Buffer.from(JSON.stringify(data));
|
|
33
|
+
writeFileSync(
|
|
34
|
+
new URL("tokenizer.json.gz", output),
|
|
35
|
+
gzipSync(json, { level: 9 }),
|
|
36
|
+
);
|
|
37
|
+
console.log(
|
|
38
|
+
"o200k_base tokenizer SHA-256:",
|
|
39
|
+
createHash("sha256").update(json).digest("hex"),
|
|
40
|
+
);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { gunzipSync } from "node:zlib";
|
|
3
|
+
import { Tokenizer } from "@huggingface/tokenizers";
|
|
4
|
+
import { util } from "zod";
|
|
5
|
+
|
|
6
|
+
export function createTokenizerCounter(
|
|
7
|
+
asset: string,
|
|
8
|
+
): (text: string) => number {
|
|
9
|
+
const tokenizer = util.cached(() => {
|
|
10
|
+
const data = JSON.parse(
|
|
11
|
+
gunzipSync(
|
|
12
|
+
readFileSync(
|
|
13
|
+
new URL(`../assets/${asset}/tokenizer.json.gz`, import.meta.url),
|
|
14
|
+
),
|
|
15
|
+
).toString("utf8"),
|
|
16
|
+
);
|
|
17
|
+
// Count literal message content as ordinary text. Request framing is
|
|
18
|
+
// accounted for separately by the meter.
|
|
19
|
+
data.added_tokens = [];
|
|
20
|
+
return new Tokenizer(data, {});
|
|
21
|
+
});
|
|
22
|
+
return (text) =>
|
|
23
|
+
tokenizer.value.encode(text, { add_special_tokens: false }).ids.length;
|
|
24
|
+
}
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
// Range selection and lock checks adapted from DeepSeek Harness 0.2.0-rc.2 (MIT).
|
|
2
|
+
import type { Context } from "@deepseek-ai/cordis";
|
|
3
|
+
import Schema from "@deepseek-ai/schemastery";
|
|
4
|
+
import BasicCompactionEngine from "@deepseek-ai/dsh-compaction-basic";
|
|
5
|
+
import type {
|
|
6
|
+
BasicCompactionConfig,
|
|
7
|
+
ResolvedTargetPolicy,
|
|
8
|
+
ResolvedCompactSpec,
|
|
9
|
+
ResolvedRetention,
|
|
10
|
+
} from "@deepseek-ai/dsh-compaction-basic";
|
|
11
|
+
import {
|
|
12
|
+
ManualCompactionError,
|
|
13
|
+
toolPairingBalancedBefore,
|
|
14
|
+
} from "@deepseek-ai/dsh-compaction";
|
|
15
|
+
import type { CompactionResult } from "@deepseek-ai/dsh-compaction";
|
|
16
|
+
import { isSurfaceEvent } from "@deepseek-ai/dsh-session";
|
|
17
|
+
import type { ProjectionDefinition } from "@deepseek-ai/dsh-session-projection";
|
|
18
|
+
import { z } from "zod";
|
|
19
|
+
import type { Session } from "@deepseek-ai/dsh-session";
|
|
20
|
+
import type { TokenMeasurement } from "@deepseek-ai/dsh-token-meter";
|
|
21
|
+
|
|
22
|
+
export interface PercentageCompactionConfig extends BasicCompactionConfig {
|
|
23
|
+
thresholdRatio: number;
|
|
24
|
+
maxTokens: number;
|
|
25
|
+
modelPolicies: NonNullable<BasicCompactionConfig["modelPolicies"]>;
|
|
26
|
+
auto: boolean;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
type Agent = Parameters<BasicCompactionEngine["compactIfNeeded"]>[0];
|
|
30
|
+
type Trigger = Parameters<BasicCompactionEngine["compactIfNeeded"]>[1];
|
|
31
|
+
const fields = {
|
|
32
|
+
thresholdRatio: Schema.number().min(0).max(1),
|
|
33
|
+
headroomTokens: Schema.natural(),
|
|
34
|
+
retainRatio: Schema.number().min(0).max(1),
|
|
35
|
+
retainTokens: Schema.natural(),
|
|
36
|
+
summarizationProvider: Schema.string(),
|
|
37
|
+
summarizationModel: Schema.string(),
|
|
38
|
+
maxTokens: Schema.natural(),
|
|
39
|
+
compactionRetries: Schema.natural(),
|
|
40
|
+
maxOverflowRetries: Schema.natural(),
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/** Both budgets are percentages of the full context window. */
|
|
44
|
+
export function resolveCompactSpec(
|
|
45
|
+
policy: ResolvedTargetPolicy,
|
|
46
|
+
contextWindow: number,
|
|
47
|
+
): ResolvedCompactSpec {
|
|
48
|
+
if (!Number.isSafeInteger(contextWindow) || contextWindow <= 0)
|
|
49
|
+
throw new Error(
|
|
50
|
+
"compaction: contextWindow must be a positive safe integer",
|
|
51
|
+
);
|
|
52
|
+
const thresholdTokens = Math.floor(contextWindow * policy.thresholdRatio);
|
|
53
|
+
const retainTokens =
|
|
54
|
+
policy.retainTokens !== undefined
|
|
55
|
+
? policy.retainTokens
|
|
56
|
+
: Math.floor(contextWindow * policy.retainRatio);
|
|
57
|
+
if (thresholdTokens <= 0 || retainTokens >= thresholdTokens)
|
|
58
|
+
throw new Error(
|
|
59
|
+
"compaction: retain budget must be below the percentage threshold",
|
|
60
|
+
);
|
|
61
|
+
return Object.freeze({
|
|
62
|
+
target: policy.target,
|
|
63
|
+
contextWindow,
|
|
64
|
+
thresholdRatio: policy.thresholdRatio,
|
|
65
|
+
thresholdTokens,
|
|
66
|
+
retainTokens,
|
|
67
|
+
summarizationProvider: policy.summarizationProvider,
|
|
68
|
+
summarizationModel: policy.summarizationModel,
|
|
69
|
+
maxTokens: policy.maxTokens,
|
|
70
|
+
compactionRetries: policy.compactionRetries,
|
|
71
|
+
maxOverflowRetries: policy.maxOverflowRetries,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Preserve system head and whole tool-call/result groups. */
|
|
76
|
+
export function selectCompactableRange(
|
|
77
|
+
session: Session,
|
|
78
|
+
measurement: TokenMeasurement,
|
|
79
|
+
retainTokens: number,
|
|
80
|
+
systemHead: boolean,
|
|
81
|
+
) {
|
|
82
|
+
const nodes = measurement.nodes;
|
|
83
|
+
if (!nodes.length) return null;
|
|
84
|
+
const surface = session.surface.nodes;
|
|
85
|
+
if (
|
|
86
|
+
surface.length !== nodes.length ||
|
|
87
|
+
surface.some((seq, i) => seq !== nodes[i].seq)
|
|
88
|
+
)
|
|
89
|
+
throw new Error(
|
|
90
|
+
"compaction: token-meter surface does not match the session surface",
|
|
91
|
+
);
|
|
92
|
+
const first = systemHead ? 1 : 0;
|
|
93
|
+
let total = 0;
|
|
94
|
+
let keep = nodes.length;
|
|
95
|
+
for (let i = nodes.length - 1; i >= 0; i--) {
|
|
96
|
+
total += nodes[i].tokens;
|
|
97
|
+
keep = i;
|
|
98
|
+
if (total >= retainTokens) break;
|
|
99
|
+
}
|
|
100
|
+
while (keep > first && !toolPairingBalancedBefore(session, surface[keep]))
|
|
101
|
+
keep--;
|
|
102
|
+
return keep <= first
|
|
103
|
+
? null
|
|
104
|
+
: { start: surface[first], end: surface[keep - 1] };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
interface CompactionEntryState {
|
|
108
|
+
active: boolean;
|
|
109
|
+
headSeq: number | null;
|
|
110
|
+
systemHead: boolean;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
declare module "@deepseek-ai/dsh-session-projection" {
|
|
114
|
+
interface SessionProjectionStateMap {
|
|
115
|
+
contextPolicyCompaction: CompactionEntryState;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const compactionEntry: ProjectionDefinition<"contextPolicyCompaction"> = {
|
|
120
|
+
key: "contextPolicyCompaction",
|
|
121
|
+
stateVersion: 1,
|
|
122
|
+
stateSchema: z.object({
|
|
123
|
+
active: z.boolean(),
|
|
124
|
+
headSeq: z.number().int().nonnegative().nullable(),
|
|
125
|
+
systemHead: z.boolean(),
|
|
126
|
+
}),
|
|
127
|
+
init: () => ({ active: false, headSeq: null, systemHead: false }),
|
|
128
|
+
apply: (state, event) => {
|
|
129
|
+
if (event.type === "compaction/start") return { ...state, active: true };
|
|
130
|
+
if (event.type === "compaction/end" || event.type === "session/end-seed")
|
|
131
|
+
return state.active ? { ...state, active: false } : state;
|
|
132
|
+
if (
|
|
133
|
+
isSurfaceEvent(event) &&
|
|
134
|
+
(state.headSeq === null ||
|
|
135
|
+
(event.surfaceOp !== "append" &&
|
|
136
|
+
event.surfaceOp.startSeq === state.headSeq))
|
|
137
|
+
)
|
|
138
|
+
return {
|
|
139
|
+
...state,
|
|
140
|
+
headSeq: event.seq,
|
|
141
|
+
systemHead: event.type === "system/message",
|
|
142
|
+
};
|
|
143
|
+
return state;
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
/** Reuse the official summarizer, transactions, commands and overflow listeners. */
|
|
148
|
+
export class PercentageCompactionEngine extends BasicCompactionEngine {
|
|
149
|
+
static override inject = [
|
|
150
|
+
...BasicCompactionEngine.inject,
|
|
151
|
+
"sessionProjections",
|
|
152
|
+
];
|
|
153
|
+
static override Config: Schema<
|
|
154
|
+
BasicCompactionConfig,
|
|
155
|
+
PercentageCompactionConfig
|
|
156
|
+
> = Schema.object({
|
|
157
|
+
...fields,
|
|
158
|
+
thresholdRatio: Schema.number().min(0).max(1).default(0.8),
|
|
159
|
+
maxTokens: Schema.natural().default(65536),
|
|
160
|
+
auto: Schema.boolean().default(true),
|
|
161
|
+
modelPolicies: Schema.array(
|
|
162
|
+
Schema.object({
|
|
163
|
+
...fields,
|
|
164
|
+
provider: Schema.string().required(),
|
|
165
|
+
model: Schema.string().required(),
|
|
166
|
+
}),
|
|
167
|
+
).default([]),
|
|
168
|
+
});
|
|
169
|
+
constructor(ctx: Context, config: PercentageCompactionConfig) {
|
|
170
|
+
super(ctx, {
|
|
171
|
+
...config,
|
|
172
|
+
headroomTokens: 0,
|
|
173
|
+
modelPolicies: config.modelPolicies.map((p) => ({
|
|
174
|
+
...p,
|
|
175
|
+
headroomTokens: 0,
|
|
176
|
+
maxTokens: p.maxTokens ?? config.maxTokens,
|
|
177
|
+
})),
|
|
178
|
+
});
|
|
179
|
+
ctx.sessionProjections.register(compactionEntry);
|
|
180
|
+
}
|
|
181
|
+
private entryState(session: Session) {
|
|
182
|
+
const state = this.ctx.sessionProjections.stateOf(
|
|
183
|
+
session,
|
|
184
|
+
"contextPolicyCompaction",
|
|
185
|
+
);
|
|
186
|
+
if (!state)
|
|
187
|
+
throw new Error("compaction: entry-state projection is unavailable");
|
|
188
|
+
return state;
|
|
189
|
+
}
|
|
190
|
+
private policy(target: {
|
|
191
|
+
provider: string;
|
|
192
|
+
model: string;
|
|
193
|
+
}): ResolvedTargetPolicy {
|
|
194
|
+
const match = this.config.modelPolicies.find(
|
|
195
|
+
(p) => p.provider === target.provider && p.model === target.model,
|
|
196
|
+
);
|
|
197
|
+
let retention: ResolvedRetention;
|
|
198
|
+
if (match?.retainTokens !== undefined)
|
|
199
|
+
retention = { retainTokens: match.retainTokens };
|
|
200
|
+
else if (match?.retainRatio !== undefined)
|
|
201
|
+
retention = { retainRatio: match.retainRatio };
|
|
202
|
+
else if (this.config.retainTokens !== undefined)
|
|
203
|
+
retention = { retainTokens: this.config.retainTokens };
|
|
204
|
+
else retention = { retainRatio: this.config.retainRatio };
|
|
205
|
+
return {
|
|
206
|
+
thresholdRatio: match?.thresholdRatio ?? this.config.thresholdRatio,
|
|
207
|
+
headroomTokens: 0,
|
|
208
|
+
summarizationProvider:
|
|
209
|
+
match?.summarizationProvider ?? this.config.summarizationProvider,
|
|
210
|
+
summarizationModel:
|
|
211
|
+
match?.summarizationModel ?? this.config.summarizationModel,
|
|
212
|
+
maxTokens: match?.maxTokens ?? this.config.maxTokens,
|
|
213
|
+
compactionRetries:
|
|
214
|
+
match?.compactionRetries ?? this.config.compactionRetries,
|
|
215
|
+
maxOverflowRetries:
|
|
216
|
+
match?.maxOverflowRetries ?? this.config.maxOverflowRetries,
|
|
217
|
+
target,
|
|
218
|
+
...retention,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
override async compactIfNeeded(
|
|
222
|
+
agent: Agent,
|
|
223
|
+
trigger: Trigger,
|
|
224
|
+
signal?: AbortSignal,
|
|
225
|
+
): Promise<CompactionResult | null> {
|
|
226
|
+
const header = agent.session.requestHeader();
|
|
227
|
+
if (!header?.config.provider || !header.config.model) return null;
|
|
228
|
+
const target = {
|
|
229
|
+
provider: header.config.provider,
|
|
230
|
+
model: header.config.model,
|
|
231
|
+
};
|
|
232
|
+
const meter = this.ctx.tokenMeter;
|
|
233
|
+
let measurement = meter.measure(agent.session);
|
|
234
|
+
const prune = this.ctx.get("toolResultPruner");
|
|
235
|
+
if (trigger === "context-overflow") {
|
|
236
|
+
prune?.pruneSession(agent.session);
|
|
237
|
+
measurement = meter.measure(agent.session);
|
|
238
|
+
const range = selectCompactableRange(
|
|
239
|
+
agent.session,
|
|
240
|
+
measurement,
|
|
241
|
+
0,
|
|
242
|
+
this.entryState(agent.session).systemHead,
|
|
243
|
+
);
|
|
244
|
+
return range
|
|
245
|
+
? this.compactRegion(range.start, range.end, agent, signal)
|
|
246
|
+
: null;
|
|
247
|
+
}
|
|
248
|
+
const info = await this.ctx.llm.resolveModelInfo(
|
|
249
|
+
target.provider,
|
|
250
|
+
target.model,
|
|
251
|
+
signal,
|
|
252
|
+
);
|
|
253
|
+
if (this.entryState(agent.session).active)
|
|
254
|
+
throw new ManualCompactionError(
|
|
255
|
+
"busy",
|
|
256
|
+
"automatic pressure compaction: compaction already in progress",
|
|
257
|
+
);
|
|
258
|
+
if (!info.context)
|
|
259
|
+
throw new Error(
|
|
260
|
+
"compaction: configure contextWindow for " +
|
|
261
|
+
target.provider +
|
|
262
|
+
"/" +
|
|
263
|
+
target.model,
|
|
264
|
+
);
|
|
265
|
+
const spec = resolveCompactSpec(
|
|
266
|
+
this.policy(target),
|
|
267
|
+
info.context.contextWindow,
|
|
268
|
+
);
|
|
269
|
+
if (measurement.totalTokens < spec.thresholdTokens) return null;
|
|
270
|
+
prune?.pruneSession(agent.session);
|
|
271
|
+
measurement = meter.measure(agent.session);
|
|
272
|
+
if (measurement.totalTokens < spec.thresholdTokens) return null;
|
|
273
|
+
let result: CompactionResult | null = null;
|
|
274
|
+
for (let attempt = 0; attempt <= spec.compactionRetries; attempt++) {
|
|
275
|
+
const range = selectCompactableRange(
|
|
276
|
+
agent.session,
|
|
277
|
+
measurement,
|
|
278
|
+
spec.retainTokens,
|
|
279
|
+
this.entryState(agent.session).systemHead,
|
|
280
|
+
);
|
|
281
|
+
if (!range) return result;
|
|
282
|
+
result = await this.compactRegion(range.start, range.end, agent, signal);
|
|
283
|
+
measurement = meter.measure(agent.session);
|
|
284
|
+
if (measurement.totalTokens < spec.thresholdTokens) return result;
|
|
285
|
+
}
|
|
286
|
+
throw new Error(
|
|
287
|
+
"compaction still above percentage threshold after " +
|
|
288
|
+
(spec.compactionRetries + 1) +
|
|
289
|
+
" attempts",
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
export default PercentageCompactionEngine;
|
package/src/core.ts
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import type { EpochHeader, SessionEvent } from "@deepseek-ai/dsh-session";
|
|
2
|
+
import type { TokenMeasurementBaseline } from "@deepseek-ai/dsh-token-meter";
|
|
3
|
+
import { SessionLogOffset } from "@deepseek-ai/dsh-session";
|
|
4
|
+
import type {
|
|
5
|
+
default as LlmService,
|
|
6
|
+
Message,
|
|
7
|
+
LlmImageRequestPrice,
|
|
8
|
+
LlmImageRequestPricing,
|
|
9
|
+
TokenUsage,
|
|
10
|
+
} from "@deepseek-ai/dsh-llm";
|
|
11
|
+
import type {
|
|
12
|
+
MeterConfig,
|
|
13
|
+
MeterEvent,
|
|
14
|
+
ReplayState,
|
|
15
|
+
ReplayNode,
|
|
16
|
+
} from "./types.js";
|
|
17
|
+
export interface MeterServices {
|
|
18
|
+
llm?: Pick<LlmService, "imageRequestPricing" | "fileRequestText">;
|
|
19
|
+
}
|
|
20
|
+
interface Attachments {
|
|
21
|
+
imagePricing?: LlmImageRequestPricing;
|
|
22
|
+
fileText?: LlmService["fileRequestText"];
|
|
23
|
+
imagePrices: readonly LlmImageRequestPrice[];
|
|
24
|
+
imageCursor: number;
|
|
25
|
+
unpricedImages: boolean;
|
|
26
|
+
unpricedFiles: boolean;
|
|
27
|
+
}
|
|
28
|
+
import {
|
|
29
|
+
canonicalHeader,
|
|
30
|
+
deriveEventMessage,
|
|
31
|
+
headerEquals,
|
|
32
|
+
isSurfaceEvent,
|
|
33
|
+
} from "@deepseek-ai/dsh-session";
|
|
34
|
+
import { estimateMessage as shadowPrice } from "@deepseek-ai/dsh-token-meter/estimate";
|
|
35
|
+
import { lastAssistantStreamChunk } from "@deepseek-ai/dsh-llm";
|
|
36
|
+
import { deepFreeze } from "@deepseek-ai/dsh-util-values";
|
|
37
|
+
import { countText, resolvePolicy } from "./tokenizers.js";
|
|
38
|
+
|
|
39
|
+
export const initialState = (): ReplayState => ({
|
|
40
|
+
nodes: [],
|
|
41
|
+
header: null,
|
|
42
|
+
anchor: null,
|
|
43
|
+
logRevision: SessionLogOffset(0),
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
export function inputTokens(usage: TokenUsage) {
|
|
47
|
+
return (
|
|
48
|
+
usage.inputTokens +
|
|
49
|
+
(usage.cacheReadTokens ?? 0) +
|
|
50
|
+
(usage.cacheWriteTokens ?? 0)
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function usageOf(
|
|
55
|
+
event: Extract<
|
|
56
|
+
SessionEvent,
|
|
57
|
+
{ type: "assistant/message" | "assistant/attempt" }
|
|
58
|
+
>,
|
|
59
|
+
): TokenUsage | undefined {
|
|
60
|
+
return event.type === "assistant/message" && event.data.usage !== undefined
|
|
61
|
+
? event.data.usage
|
|
62
|
+
: lastAssistantStreamChunk(event.data.stream, "usage")?.usage;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Custom counting state driven by the host projection registry. */
|
|
66
|
+
export function fold(state: ReplayState, event: MeterEvent): ReplayState {
|
|
67
|
+
const next: ReplayState = {
|
|
68
|
+
...state,
|
|
69
|
+
logRevision: SessionLogOffset(event.seq + 1),
|
|
70
|
+
};
|
|
71
|
+
if (event.type === "llm/retry-started") return next;
|
|
72
|
+
if (event.type === "request/header")
|
|
73
|
+
next.header = canonicalHeader(event.data.header);
|
|
74
|
+
// Failed attempts never calibrate occupancy. Zero samples are also ignored.
|
|
75
|
+
if (event.type === "assistant/message") {
|
|
76
|
+
const usage = usageOf(event);
|
|
77
|
+
if (usage && inputTokens(usage) > 0 && next.header)
|
|
78
|
+
next.anchor = {
|
|
79
|
+
header: next.header,
|
|
80
|
+
nodes: state.nodes,
|
|
81
|
+
usage,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
if (event.type === "image/offload") {
|
|
85
|
+
const targets = new Map(
|
|
86
|
+
event.data.targets.map((t) => [t.seq, new Set(t.imageIndexes)]),
|
|
87
|
+
);
|
|
88
|
+
next.nodes = state.nodes.map((node) => {
|
|
89
|
+
const indexes = targets.get(node.seq);
|
|
90
|
+
if (!indexes || !node.message) return node;
|
|
91
|
+
let index = 0;
|
|
92
|
+
const content = node.message.content.map((block) =>
|
|
93
|
+
block.type !== "image"
|
|
94
|
+
? block
|
|
95
|
+
: indexes.has(index++)
|
|
96
|
+
? { ...block, offloaded: true as const }
|
|
97
|
+
: block,
|
|
98
|
+
);
|
|
99
|
+
return { ...node, message: { ...node.message, content } };
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
if (event.type === "image/offload") return next;
|
|
103
|
+
if (isSurfaceEvent(event)) {
|
|
104
|
+
const message = deriveEventMessage(event);
|
|
105
|
+
const node = {
|
|
106
|
+
seq: event.seq,
|
|
107
|
+
message,
|
|
108
|
+
heuristicTokens: message ? shadowPrice(message) : 0,
|
|
109
|
+
};
|
|
110
|
+
if (event.surfaceOp === "append") next.nodes = [...state.nodes, node];
|
|
111
|
+
else {
|
|
112
|
+
const { startSeq, endSeq } = event.surfaceOp;
|
|
113
|
+
const start = state.nodes.findIndex((n) => n.seq === startSeq);
|
|
114
|
+
const end = state.nodes.findIndex((n) => n.seq === endSeq);
|
|
115
|
+
if (start < 0 || end < start)
|
|
116
|
+
throw new Error(`Invalid surface replacement at seq ${event.seq}`);
|
|
117
|
+
next.nodes = [
|
|
118
|
+
...state.nodes.slice(0, start),
|
|
119
|
+
node,
|
|
120
|
+
...state.nodes.slice(end + 1),
|
|
121
|
+
];
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return next;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function contentTokens(
|
|
128
|
+
blocks: Message["content"],
|
|
129
|
+
policy: ReturnType<typeof resolvePolicy>,
|
|
130
|
+
attachments: Attachments,
|
|
131
|
+
) {
|
|
132
|
+
const count = (text: string) => countText(text, policy.tokenizer);
|
|
133
|
+
let tokens = 0;
|
|
134
|
+
for (const block of blocks) {
|
|
135
|
+
switch (block.type) {
|
|
136
|
+
case "text":
|
|
137
|
+
tokens += count(block.text);
|
|
138
|
+
break;
|
|
139
|
+
case "reasoning":
|
|
140
|
+
if (policy.includeReasoning) tokens += count(block.text);
|
|
141
|
+
break;
|
|
142
|
+
case "tool-call":
|
|
143
|
+
tokens += count(block.name) + count(block.arguments);
|
|
144
|
+
if (block.id) tokens += count(block.id);
|
|
145
|
+
break;
|
|
146
|
+
case "image": {
|
|
147
|
+
if (attachments.imagePricing) {
|
|
148
|
+
const price = attachments.imagePrices[attachments.imageCursor++];
|
|
149
|
+
tokens += price.visualTokens + count(price.text);
|
|
150
|
+
} else {
|
|
151
|
+
// Reference tokenization is not a visual-token count; expose this in diagnostics.
|
|
152
|
+
attachments.unpricedImages = true;
|
|
153
|
+
tokens += count(JSON.stringify(block));
|
|
154
|
+
}
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
case "file":
|
|
158
|
+
if (attachments.fileText)
|
|
159
|
+
tokens += count(attachments.fileText(block.attachment));
|
|
160
|
+
else {
|
|
161
|
+
attachments.unpricedFiles = true;
|
|
162
|
+
tokens += count(JSON.stringify(block));
|
|
163
|
+
}
|
|
164
|
+
break;
|
|
165
|
+
default:
|
|
166
|
+
tokens += count(JSON.stringify(block));
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return tokens;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function priceSurface(
|
|
173
|
+
nodes: readonly ReplayNode[],
|
|
174
|
+
header: EpochHeader | null,
|
|
175
|
+
config: MeterConfig,
|
|
176
|
+
services: MeterServices,
|
|
177
|
+
) {
|
|
178
|
+
const policy = resolvePolicy(config, header);
|
|
179
|
+
const route = header?.config;
|
|
180
|
+
const llm = services.llm;
|
|
181
|
+
const attachments: Attachments = {
|
|
182
|
+
imagePrices: [],
|
|
183
|
+
imageCursor: 0,
|
|
184
|
+
imagePricing:
|
|
185
|
+
route && services.llm?.imageRequestPricing(route.provider, route.model),
|
|
186
|
+
fileText: llm ? (ref) => llm.fileRequestText(ref) : undefined,
|
|
187
|
+
unpricedImages: false,
|
|
188
|
+
unpricedFiles: false,
|
|
189
|
+
};
|
|
190
|
+
const images = nodes.flatMap(
|
|
191
|
+
(n) => n.message?.content.filter((b) => b.type === "image") ?? [],
|
|
192
|
+
);
|
|
193
|
+
attachments.imagePrices =
|
|
194
|
+
attachments.imagePricing && images.length
|
|
195
|
+
? attachments.imagePricing.priceImages(images)
|
|
196
|
+
: [];
|
|
197
|
+
if (
|
|
198
|
+
attachments.imagePricing &&
|
|
199
|
+
(attachments.imagePrices.length !== images.length ||
|
|
200
|
+
attachments.imagePrices.some(
|
|
201
|
+
(p) => !Number.isSafeInteger(p.visualTokens) || p.visualTokens < 0,
|
|
202
|
+
))
|
|
203
|
+
)
|
|
204
|
+
throw new Error("Invalid route image pricing");
|
|
205
|
+
const priced = nodes.map((node) => {
|
|
206
|
+
const message = node.message;
|
|
207
|
+
let tokens = 0;
|
|
208
|
+
if (message) {
|
|
209
|
+
const body = contentTokens(message.content, policy, attachments);
|
|
210
|
+
if (
|
|
211
|
+
body > 0 ||
|
|
212
|
+
message.content.some((b) => b.type === "text" || b.type === "tool-call")
|
|
213
|
+
)
|
|
214
|
+
tokens = body + policy.tokensPerMessage;
|
|
215
|
+
}
|
|
216
|
+
return { seq: node.seq, tokens, heuristicTokens: node.heuristicTokens };
|
|
217
|
+
});
|
|
218
|
+
const surfaceTokens = priced.reduce((sum, node) => sum + node.tokens, 0);
|
|
219
|
+
const toolsTokens = header?.tools?.length
|
|
220
|
+
? countText(JSON.stringify(header.tools), policy.tokenizer) +
|
|
221
|
+
policy.toolsOverhead
|
|
222
|
+
: 0;
|
|
223
|
+
const primingTokens = header || nodes.length ? policy.replyPrimingTokens : 0;
|
|
224
|
+
return {
|
|
225
|
+
nodes: priced,
|
|
226
|
+
surfaceTokens,
|
|
227
|
+
toolsTokens,
|
|
228
|
+
primingTokens,
|
|
229
|
+
requestTokens: surfaceTokens + toolsTokens + primingTokens,
|
|
230
|
+
policy,
|
|
231
|
+
unpricedImages: attachments.unpricedImages,
|
|
232
|
+
unpricedFiles: attachments.unpricedFiles,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** Provider input calibrates request framing; retained assistant text is counted locally.
|
|
237
|
+
* Output usage (which may include hidden reasoning) is never blindly added to the next input.
|
|
238
|
+
*/
|
|
239
|
+
export function measureState(
|
|
240
|
+
state: ReplayState,
|
|
241
|
+
config: MeterConfig,
|
|
242
|
+
services: MeterServices = {},
|
|
243
|
+
requestHeader?: EpochHeader,
|
|
244
|
+
) {
|
|
245
|
+
const header =
|
|
246
|
+
requestHeader === undefined ? state.header : canonicalHeader(requestHeader);
|
|
247
|
+
const current = priceSurface(state.nodes, header, config, services);
|
|
248
|
+
const anchor = state.anchor;
|
|
249
|
+
const reusable = anchor && header && headerEquals(anchor.header, header);
|
|
250
|
+
let baseline: TokenMeasurementBaseline;
|
|
251
|
+
let surfaceDeltaTokens = 0;
|
|
252
|
+
if (reusable) {
|
|
253
|
+
const previous = priceSurface(anchor.nodes, header, config, services);
|
|
254
|
+
baseline = {
|
|
255
|
+
kind: "usage",
|
|
256
|
+
tokens: inputTokens(anchor.usage),
|
|
257
|
+
usage: anchor.usage,
|
|
258
|
+
};
|
|
259
|
+
surfaceDeltaTokens = current.requestTokens - previous.requestTokens;
|
|
260
|
+
} else
|
|
261
|
+
baseline =
|
|
262
|
+
!header && state.nodes.length === 0
|
|
263
|
+
? { kind: "none", tokens: 0 }
|
|
264
|
+
: { kind: "estimated", tokens: current.requestTokens };
|
|
265
|
+
return deepFreeze({
|
|
266
|
+
logRevision: state.logRevision,
|
|
267
|
+
baseline,
|
|
268
|
+
surfaceDeltaTokens,
|
|
269
|
+
totalTokens: Math.max(0, baseline.tokens + surfaceDeltaTokens),
|
|
270
|
+
surfaceTokens: current.surfaceTokens,
|
|
271
|
+
toolsTokens: current.toolsTokens,
|
|
272
|
+
nodes: current.nodes,
|
|
273
|
+
tokenizer: current.policy.tokenizer,
|
|
274
|
+
unpricedImages: current.unpricedImages,
|
|
275
|
+
unpricedFiles: current.unpricedFiles,
|
|
276
|
+
});
|
|
277
|
+
}
|