@snylonue/dsh-context-policy 0.2.1 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/types.d.ts CHANGED
@@ -49,15 +49,24 @@ export interface ResolvedPolicy {
49
49
  replyPrimingTokens: number;
50
50
  includeReasoning: boolean;
51
51
  }
52
+ export interface ReplayHeader extends EpochHeader {
53
+ toolsKey?: string;
54
+ toolCounts?: Record<string, number> | null;
55
+ }
52
56
  export interface ReplayNode {
53
57
  seq: SessionSeq;
54
- message: Message | null;
58
+ message: Pick<Message, "content"> | null;
59
+ counts?: {
60
+ text: Record<string, number>;
61
+ reasoning: Record<string, number>;
62
+ envelope: boolean;
63
+ };
55
64
  }
56
65
  export interface ReplayState {
57
66
  nodes: readonly ReplayNode[];
58
- header: EpochHeader | null;
67
+ header: ReplayHeader | null;
59
68
  anchor: {
60
- header: EpochHeader;
69
+ header: ReplayHeader;
61
70
  nodes: readonly ReplayNode[];
62
71
  usage: TokenUsage;
63
72
  } | null;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,WAAW,EACX,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAEhE,6EAA6E;AAC7E,MAAM,MAAM,UAAU,GACnB,YAAY,GACZ;IACA,IAAI,EAAE,eAAe,CAAC;IACtB,GAAG,EAAE,UAAU,CAAC;IAChB,IAAI,EAAE;QAAE,OAAO,EAAE;YAAE,GAAG,EAAE,UAAU,CAAC;YAAC,YAAY,EAAE,MAAM,EAAE,CAAA;SAAE,EAAE,CAAA;KAAE,CAAC;CAChE,GACD;IACA,IAAI,EAAE,mBAAmB,CAAC;IAC1B,GAAG,EAAE,UAAU,CAAC;IAChB,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACpC,CAAC;AAEL,MAAM,WAAW,WAAW;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC3B;AACD,MAAM,WAAW,cAAe,SAAQ,WAAW;IAClD,QAAQ,EAAE,MAAM,CAAC;CACjB;AACD,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACd;AACD,MAAM,WAAW,gBAAgB;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,cAAc,EAAE,CAAC;IACpC,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC;CAC9B;AACD,MAAM,WAAW,WAAW;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,cAAc,EAAE,CAAC;IACpC,aAAa,EAAE,WAAW,EAAE,CAAC;CAC7B;AACD,MAAM,WAAW,cAAc;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;CAC1B;AACD,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,UAAU,CAAC;IAChB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CACxB;AACD,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7B,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE;QACP,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;QAC7B,KAAK,EAAE,UAAU,CAAC;KAClB,GAAG,IAAI,CAAC;IACT,WAAW,EAAE,gBAAgB,CAAC;CAC9B"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,WAAW,EACX,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAEhE,6EAA6E;AAC7E,MAAM,MAAM,UAAU,GACnB,YAAY,GACZ;IACA,IAAI,EAAE,eAAe,CAAC;IACtB,GAAG,EAAE,UAAU,CAAC;IAChB,IAAI,EAAE;QAAE,OAAO,EAAE;YAAE,GAAG,EAAE,UAAU,CAAC;YAAC,YAAY,EAAE,MAAM,EAAE,CAAA;SAAE,EAAE,CAAA;KAAE,CAAC;CAChE,GACD;IACA,IAAI,EAAE,mBAAmB,CAAC;IAC1B,GAAG,EAAE,UAAU,CAAC;IAChB,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACpC,CAAC;AAEL,MAAM,WAAW,WAAW;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC3B;AACD,MAAM,WAAW,cAAe,SAAQ,WAAW;IAClD,QAAQ,EAAE,MAAM,CAAC;CACjB;AACD,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACd;AACD,MAAM,WAAW,gBAAgB;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,cAAc,EAAE,CAAC;IACpC,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC;CAC9B;AACD,MAAM,WAAW,WAAW;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,cAAc,EAAE,CAAC;IACpC,aAAa,EAAE,WAAW,EAAE,CAAC;CAC7B;AACD,MAAM,WAAW,cAAc;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;CAC1B;AACD,MAAM,WAAW,YAAa,SAAQ,WAAW;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;CAC3C;AACD,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,UAAU,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,IAAI,CAAC;IACzC,MAAM,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAClC,QAAQ,EAAE,OAAO,CAAC;KAClB,CAAC;CACF;AACD,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE;QACP,MAAM,EAAE,YAAY,CAAC;QACrB,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;QAC7B,KAAK,EAAE,UAAU,CAAC;KAClB,GAAG,IAAI,CAAC;IACT,WAAW,EAAE,gBAAgB,CAAC;CAC9B"}
package/package.json CHANGED
@@ -1,104 +1,103 @@
1
1
  {
2
- "name": "@snylonue/dsh-context-policy",
3
- "version": "0.2.1",
4
- "description": "Percentage-only compaction and model-configurable GPT and DeepSeek token counting for DSH",
5
- "type": "module",
6
- "license": "MIT",
7
- "keywords": [
8
- "dsh-plugin"
9
- ],
10
- "repository": {
11
- "type": "git",
12
- "url": "github:snylonue/dsh-context-policy"
13
- },
14
- "publishConfig": {
15
- "access": "public"
16
- },
17
- "engines": {
18
- "node": ">=24",
19
- "dsh": "^0.2.0-rc.2 || ^0.2.1-rc.1"
20
- },
21
- "exports": {
22
- ".": {
23
- "types": "./dist/token-meter.d.ts",
24
- "default": "./dist/token-meter.js"
25
- },
26
- "./token-meter": {
27
- "types": "./dist/token-meter.d.ts",
28
- "default": "./dist/token-meter.js"
29
- },
30
- "./compaction": {
31
- "types": "./dist/compaction.d.ts",
32
- "default": "./dist/compaction.js"
33
- },
34
- "./tokenizers": {
35
- "types": "./dist/tokenizers.d.ts",
36
- "default": "./dist/tokenizers.js"
37
- },
38
- "./core": {
39
- "types": "./dist/core.d.ts",
40
- "default": "./dist/core.js"
41
- },
42
- "./locale/*.json": "./locale/*.json",
43
- "./package.json": "./package.json"
44
- },
45
- "files": [
46
- "dist",
47
- "src",
48
- "assets",
49
- "cordis.patch.yml",
50
- "locale/*.json",
51
- "README.md",
52
- "LICENSE",
53
- "UPSTREAM.md",
54
- "examples",
55
- "tsconfig.json",
56
- "scripts",
57
- "tsconfig.check.json",
58
- "tests"
59
- ],
60
- "dependencies": {
61
- "@huggingface/tokenizers": "0.2.0"
62
- },
63
- "peerDependencies": {
64
- "@deepseek-ai/cordis": "~4.0.4",
65
- "@deepseek-ai/dsh-compaction": "^0.2.0-rc.2 || ^0.2.1-rc.1",
66
- "@deepseek-ai/dsh-compaction-basic": "^0.2.0-rc.2 || ^0.2.1-rc.1",
67
- "@deepseek-ai/dsh-llm": "^0.2.0-rc.2 || ^0.2.1-rc.1",
68
- "@deepseek-ai/dsh-session": "^0.2.0-rc.2 || ^0.2.1-rc.1",
69
- "@deepseek-ai/dsh-session-projection": "^0.2.0-rc.2 || ^0.2.1-rc.1",
70
- "@deepseek-ai/dsh-token-meter": "^0.2.0-rc.2 || ^0.2.1-rc.1",
71
- "@deepseek-ai/dsh-util-values": "^0.2.0-rc.2 || ^0.2.1-rc.1",
72
- "@deepseek-ai/schemastery": "~3.18.4",
73
- "zod": "^4.4.3"
74
- },
75
- "dsh": {
76
- "manifestVersion": 1,
77
- "bundle": {
78
- "patch": "./cordis.patch.yml"
79
- }
80
- },
81
- "devDependencies": {
82
- "@biomejs/biome": "2",
83
- "@deepseek-ai/cordis": "4.0.5",
84
- "@deepseek-ai/dsh-app-boot": "0.2.1-rc.1",
85
- "@deepseek-ai/dsh-compaction": "0.2.1-rc.1",
86
- "@deepseek-ai/dsh-compaction-basic": "0.2.1-rc.1",
87
- "@deepseek-ai/dsh-llm": "0.2.1-rc.1",
88
- "@deepseek-ai/dsh-session": "0.2.1-rc.1",
89
- "@deepseek-ai/dsh-session-projection": "0.2.1-rc.1",
90
- "@deepseek-ai/dsh-token-meter": "0.2.1-rc.1",
91
- "@deepseek-ai/dsh-util-values": "0.2.1-rc.1",
92
- "@deepseek-ai/schemastery": "3.18.5",
93
- "@types/node": "24.19.0",
94
- "typescript": "5.9.3",
95
- "zod": "4.4.3"
96
- },
97
- "scripts": {
98
- "build": "tsc -p tsconfig.json",
99
- "test": "node --test tests/*.test.ts",
100
- "check": "node scripts/check.ts",
101
- "format": "biome format --write .",
102
- "lint": "biome lint ."
103
- }
104
- }
2
+ "name": "@snylonue/dsh-context-policy",
3
+ "version": "0.3.1",
4
+ "description": "Percentage-only compaction and model-configurable GPT and DeepSeek token counting for DSH",
5
+ "type": "module",
6
+ "license": "MIT",
7
+ "keywords": [
8
+ "dsh-plugin"
9
+ ],
10
+ "repository": "github:snylonue/dsh-context-policy",
11
+ "packageManager": "pnpm@12.3.4",
12
+ "publishConfig": {
13
+ "access": "public"
14
+ },
15
+ "engines": {
16
+ "node": ">=24",
17
+ "dsh": "0.2.1-rc.2"
18
+ },
19
+ "exports": {
20
+ ".": {
21
+ "types": "./dist/token-meter.d.ts",
22
+ "default": "./dist/token-meter.js"
23
+ },
24
+ "./token-meter": {
25
+ "types": "./dist/token-meter.d.ts",
26
+ "default": "./dist/token-meter.js"
27
+ },
28
+ "./compaction": {
29
+ "types": "./dist/compaction.d.ts",
30
+ "default": "./dist/compaction.js"
31
+ },
32
+ "./tokenizers": {
33
+ "types": "./dist/tokenizers.d.ts",
34
+ "default": "./dist/tokenizers.js"
35
+ },
36
+ "./core": {
37
+ "types": "./dist/core.d.ts",
38
+ "default": "./dist/core.js"
39
+ },
40
+ "./locale/*.json": "./locale/*.json",
41
+ "./package.json": "./package.json"
42
+ },
43
+ "files": [
44
+ "dist",
45
+ "src",
46
+ "assets",
47
+ "cordis.patch.yml",
48
+ "locale/*.json",
49
+ "README.md",
50
+ "LICENSE",
51
+ "UPSTREAM.md",
52
+ "examples",
53
+ "tsconfig.json",
54
+ "scripts",
55
+ "tsconfig.check.json",
56
+ "tests"
57
+ ],
58
+ "scripts": {
59
+ "build": "tsc -p tsconfig.json",
60
+ "test": "node --test tests/*.test.ts",
61
+ "check": "node scripts/check.ts",
62
+ "prepack": "node scripts/check.ts",
63
+ "format": "biome format --write .",
64
+ "lint": "biome lint ."
65
+ },
66
+ "dependencies": {
67
+ "@huggingface/tokenizers": "0.2.0"
68
+ },
69
+ "peerDependencies": {
70
+ "@deepseek-ai/cordis": "~4.0.5",
71
+ "@deepseek-ai/dsh-compaction": "0.2.1-rc.2",
72
+ "@deepseek-ai/dsh-compaction-basic": "0.2.1-rc.2",
73
+ "@deepseek-ai/dsh-llm": "0.2.1-rc.2",
74
+ "@deepseek-ai/dsh-session": "0.2.1-rc.2",
75
+ "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2",
76
+ "@deepseek-ai/dsh-token-meter": "0.2.1-rc.2",
77
+ "@deepseek-ai/dsh-util-values": "0.2.1-rc.2",
78
+ "@deepseek-ai/schemastery": "~3.18.5",
79
+ "zod": "^4.4.3"
80
+ },
81
+ "dsh": {
82
+ "manifestVersion": 1,
83
+ "bundle": {
84
+ "patch": "./cordis.patch.yml"
85
+ }
86
+ },
87
+ "devDependencies": {
88
+ "@biomejs/biome": "2",
89
+ "@deepseek-ai/cordis": "4.0.5",
90
+ "@deepseek-ai/dsh-app-boot": "0.2.1-rc.2",
91
+ "@deepseek-ai/dsh-compaction": "0.2.1-rc.2",
92
+ "@deepseek-ai/dsh-compaction-basic": "0.2.1-rc.2",
93
+ "@deepseek-ai/dsh-llm": "0.2.1-rc.2",
94
+ "@deepseek-ai/dsh-session": "0.2.1-rc.2",
95
+ "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2",
96
+ "@deepseek-ai/dsh-token-meter": "0.2.1-rc.2",
97
+ "@deepseek-ai/dsh-util-values": "0.2.1-rc.2",
98
+ "@deepseek-ai/schemastery": "3.18.5",
99
+ "@types/node": "24.19.0",
100
+ "typescript": "5.9.3",
101
+ "zod": "4.4.3"
102
+ }
103
+ }
package/scripts/check.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { spawnSync } from "node:child_process";
2
2
  import { createRequire } from "node:module";
3
3
  import { fileURLToPath } from "node:url";
4
+
4
5
  const require = createRequire(import.meta.url);
5
6
  const cwd = fileURLToPath(new URL("../", import.meta.url));
6
7
  for (const args of [
@@ -8,7 +9,10 @@ for (const args of [
8
9
  [require.resolve("typescript/bin/tsc"), "-p", "tsconfig.check.json"],
9
10
  [
10
11
  "--test",
12
+ "tests/current-host.test.ts",
11
13
  "tests/policy.test.ts",
14
+ "tests/compact.test.ts",
15
+ "tests/replay.test.ts",
12
16
  "tests/bundle.test.ts",
13
17
  "tests/tokenizers.test.ts",
14
18
  ],
@@ -20,5 +20,5 @@ export function createTokenizerCounter(
20
20
  return new Tokenizer(data, {});
21
21
  });
22
22
  return (text) =>
23
- tokenizer.value.encode(text, { add_special_tokens: false }).ids.length;
23
+ tokenizer.value.tokenize(text, { add_special_tokens: false }).length;
24
24
  }
package/src/compact.ts ADDED
@@ -0,0 +1,189 @@
1
+ import { createHash } from "node:crypto";
2
+ import type { EpochHeader } from "@deepseek-ai/dsh-session";
3
+ import { SessionLogOffset, SessionSeq } from "@deepseek-ai/dsh-session";
4
+ import type { FileBlock, ImageBlock, TokenUsage } from "@deepseek-ai/dsh-llm";
5
+ import { z } from "zod";
6
+ import { contentTokens, fold, headerToolsKey, initialState } from "./core.js";
7
+ import { deepFreeze } from "@deepseek-ai/dsh-util-values";
8
+ import { countText, tokenizerGeneration } from "./tokenizers.js";
9
+ import type {
10
+ MeterConfig,
11
+ MeterEvent,
12
+ ReplayHeader,
13
+ ReplayNode,
14
+ } from "./types.js";
15
+ export { measureState as measureCompact } from "./core.js";
16
+
17
+ const count = z.number().int().nonnegative();
18
+ const counts = z.record(z.string(), count);
19
+ const nodeSchema = z.object({
20
+ seq: count.transform(SessionSeq),
21
+ system: z.boolean(),
22
+ counts: z.object({ text: counts, reasoning: counts, envelope: z.boolean() }),
23
+ // Only durable attachment references survive; never text or tool-call bodies.
24
+ message: z
25
+ .object({ content: z.array(z.custom<ImageBlock | FileBlock>()) })
26
+ .nullable(),
27
+ heuristicTokens: count.optional(),
28
+ });
29
+ const headerSchema = z.object({
30
+ config: z.custom<EpochHeader["config"]>(),
31
+ adapterDefaults: z.custom<EpochHeader["adapterDefaults"]>().optional(),
32
+ toolsKey: z.string(),
33
+ toolCounts: counts.nullable(),
34
+ });
35
+ export const compactSchema = z.object({
36
+ nodes: z.array(nodeSchema),
37
+ header: headerSchema.nullable(),
38
+ anchor: z
39
+ .object({
40
+ header: headerSchema,
41
+ nodes: z.array(nodeSchema),
42
+ usage: z.custom<TokenUsage>(),
43
+ })
44
+ .nullable(),
45
+ logRevision: count.transform(SessionLogOffset),
46
+ });
47
+ export type CompactState = z.infer<typeof compactSchema>;
48
+ export const initialCompactState = () => initialState() as CompactState;
49
+
50
+ /** Counts cannot be reused across schema or policy changes. */
51
+ export function compactVersion(config: MeterConfig) {
52
+ const hash = createHash("sha256")
53
+ .update(JSON.stringify([2, config]))
54
+ .digest("hex");
55
+ return 0x100000000 + Number.parseInt(hash.slice(0, 8), 16);
56
+ }
57
+ function tokenizerNames(config: MeterConfig) {
58
+ return [
59
+ ...new Set([
60
+ config.defaultTokenizer,
61
+ ...[...(config.providerPolicies ?? []), ...config.modelPolicies].flatMap(
62
+ (p) => (p.tokenizer === undefined ? [] : [p.tokenizer]),
63
+ ),
64
+ ]),
65
+ ];
66
+ }
67
+ interface PayloadCounts {
68
+ header?: Pick<NonNullable<CompactState["header"]>, "toolsKey" | "toolCounts">;
69
+ node?: CompactState["nodes"][number]["counts"];
70
+ }
71
+ // The host cold-replays one whole projection at a time. A bounded text cache
72
+ // cannot bridge those passes. Share only immutable numeric payload summaries:
73
+ // never a fold state, body, tool schema, attachment, or strong event reference.
74
+ let payloads = new WeakMap<MeterEvent, Map<string, PayloadCounts>>();
75
+ let payloadGeneration = -1;
76
+ function payloadCounts(event: MeterEvent, config: MeterConfig): PayloadCounts {
77
+ const generation = tokenizerGeneration();
78
+ if (generation !== payloadGeneration) {
79
+ payloads = new WeakMap();
80
+ payloadGeneration = generation;
81
+ }
82
+ let policies = payloads.get(event);
83
+ if (!policies) {
84
+ policies = new Map();
85
+ payloads.set(event, policies);
86
+ }
87
+ // Overheads and route/reasoning selection do not affect these base counts.
88
+ const key = JSON.stringify(tokenizerNames(config));
89
+ let summary = policies.get(key);
90
+ if (!summary) {
91
+ summary = {};
92
+ policies.set(key, summary);
93
+ }
94
+ return summary;
95
+ }
96
+ function compactHeader(
97
+ header: ReplayHeader,
98
+ config: MeterConfig,
99
+ memo: PayloadCounts,
100
+ ) {
101
+ if (!memo.header) {
102
+ const tools = header.tools?.length ? JSON.stringify(header.tools) : null;
103
+ memo.header = deepFreeze({
104
+ toolsKey: headerToolsKey(header),
105
+ toolCounts:
106
+ tools === null
107
+ ? null
108
+ : Object.fromEntries(
109
+ tokenizerNames(config).map((name) => [
110
+ name,
111
+ countText(tools, name),
112
+ ]),
113
+ ),
114
+ });
115
+ }
116
+ return {
117
+ config: header.config,
118
+ ...(header.adapterDefaults
119
+ ? { adapterDefaults: header.adapterDefaults }
120
+ : {}),
121
+ ...memo.header,
122
+ };
123
+ }
124
+ function compactNode(
125
+ node: ReplayNode,
126
+ system: boolean,
127
+ config: MeterConfig,
128
+ memo: PayloadCounts,
129
+ ): CompactState["nodes"][number] {
130
+ const content = node.message?.content ?? [];
131
+ const attachments = content.filter(
132
+ (b): b is ImageBlock | FileBlock => b.type === "image" || b.type === "file",
133
+ );
134
+ if (!memo.node) {
135
+ const text = content.filter((b) => b.type !== "image" && b.type !== "file");
136
+ const names = tokenizerNames(config);
137
+ const prices = (blocks: typeof content, includeReasoning: boolean) =>
138
+ Object.fromEntries(
139
+ names.map((tokenizer) => [
140
+ tokenizer,
141
+ contentTokens(blocks, { tokenizer, includeReasoning }),
142
+ ]),
143
+ );
144
+ memo.node = deepFreeze({
145
+ text: prices(text, false),
146
+ reasoning: prices(
147
+ text.filter((b) => b.type === "reasoning"),
148
+ true,
149
+ ),
150
+ envelope: text.some((b) => b.type === "text" || b.type === "tool-call"),
151
+ });
152
+ }
153
+ return {
154
+ seq: node.seq,
155
+ system,
156
+ counts: memo.node,
157
+ message: attachments.length ? { content: attachments } : null,
158
+ };
159
+ }
160
+
161
+ /** The existing fold owns usage, offload and replacement semantics; only its new payloads are summarized. */
162
+ export function foldCompact(
163
+ state: CompactState,
164
+ event: MeterEvent,
165
+ config: MeterConfig,
166
+ ): CompactState {
167
+ const next = fold(state, event);
168
+ return {
169
+ ...next,
170
+ header:
171
+ next.header === state.header
172
+ ? state.header
173
+ : next.header &&
174
+ compactHeader(next.header, config, payloadCounts(event, config)),
175
+ nodes:
176
+ next.nodes === state.nodes
177
+ ? state.nodes
178
+ : next.nodes.map((node) =>
179
+ node.counts
180
+ ? node
181
+ : compactNode(
182
+ node,
183
+ event.type === "system/message",
184
+ config,
185
+ payloadCounts(event, config),
186
+ ),
187
+ ),
188
+ } as CompactState;
189
+ }
package/src/core.ts CHANGED
@@ -1,18 +1,20 @@
1
- import type { EpochHeader, SessionEvent } from "@deepseek-ai/dsh-session";
2
- import type { TokenMeasurementBaseline } from "@deepseek-ai/dsh-token-meter";
3
- import { SessionLogOffset } from "@deepseek-ai/dsh-session";
1
+ import { createHash } from "node:crypto";
4
2
  import type {
5
- default as LlmService,
6
- Message,
7
3
  LlmImageRequestPrice,
8
4
  LlmImageRequestPricing,
5
+ default as LlmService,
6
+ Message,
9
7
  TokenUsage,
10
8
  } from "@deepseek-ai/dsh-llm";
9
+ import type { EpochHeader, SessionEvent } from "@deepseek-ai/dsh-session";
10
+ import { SessionLogOffset } from "@deepseek-ai/dsh-session";
11
+ import type { TokenMeasurementBaseline } from "@deepseek-ai/dsh-token-meter";
11
12
  import type {
12
13
  MeterConfig,
13
14
  MeterEvent,
14
- ReplayState,
15
+ ReplayHeader,
15
16
  ReplayNode,
17
+ ReplayState,
16
18
  } from "./types.js";
17
19
  export interface MeterServices {
18
20
  llm?: Pick<LlmService, "imageRequestPricing" | "fileRequestText">;
@@ -25,16 +27,30 @@ interface Attachments {
25
27
  unpricedImages: boolean;
26
28
  unpricedFiles: boolean;
27
29
  }
30
+
31
+ import { lastAssistantStreamChunk } from "@deepseek-ai/dsh-llm";
28
32
  import {
29
33
  canonicalHeader,
30
34
  deriveEventMessage,
31
35
  headerEquals,
32
36
  isSurfaceEvent,
33
37
  } from "@deepseek-ai/dsh-session";
34
- import { lastAssistantStreamChunk } from "@deepseek-ai/dsh-llm";
35
38
  import { deepFreeze } from "@deepseek-ai/dsh-util-values";
36
39
  import { countText, resolvePolicy } from "./tokenizers.js";
37
40
 
41
+ /** Tool schemas compare in order, identically for raw and summarized headers. */
42
+ export const headerToolsKey = (header: ReplayHeader) =>
43
+ header.toolsKey ??
44
+ createHash("sha256")
45
+ .update(JSON.stringify(header.tools ?? []))
46
+ .digest("hex");
47
+ function meterHeaderEquals(a: ReplayHeader, b: ReplayHeader) {
48
+ return a.toolsKey || b.toolsKey
49
+ ? headerEquals({ ...a, tools: undefined }, { ...b, tools: undefined }) &&
50
+ headerToolsKey(a) === headerToolsKey(b)
51
+ : headerEquals(a, b);
52
+ }
53
+
38
54
  export const initialState = (): ReplayState => ({
39
55
  nodes: [],
40
56
  header: null,
@@ -122,10 +138,18 @@ export function fold(state: ReplayState, event: MeterEvent): ReplayState {
122
138
  return next;
123
139
  }
124
140
 
125
- function contentTokens(
141
+ export function contentTokens(
126
142
  blocks: Message["content"],
127
- policy: ReturnType<typeof resolvePolicy>,
128
- attachments: Attachments,
143
+ policy: Pick<
144
+ ReturnType<typeof resolvePolicy>,
145
+ "tokenizer" | "includeReasoning"
146
+ >,
147
+ attachments: Attachments = {
148
+ imagePrices: [],
149
+ imageCursor: 0,
150
+ unpricedImages: false,
151
+ unpricedFiles: false,
152
+ },
129
153
  ) {
130
154
  const count = (text: string) => countText(text, policy.tokenizer);
131
155
  let tokens = 0;
@@ -169,7 +193,7 @@ function contentTokens(
169
193
 
170
194
  export function priceSurface(
171
195
  nodes: readonly ReplayNode[],
172
- header: EpochHeader | null,
196
+ header: ReplayHeader | null,
173
197
  config: MeterConfig,
174
198
  services: MeterServices,
175
199
  ) {
@@ -202,23 +226,27 @@ export function priceSurface(
202
226
  throw new Error("Invalid route image pricing");
203
227
  const priced = nodes.map((node) => {
204
228
  const message = node.message;
205
- let tokens = 0;
206
- if (message) {
207
- const body = contentTokens(message.content, policy, attachments);
208
- if (
209
- body > 0 ||
210
- message.content.some((b) => b.type === "text" || b.type === "tool-call")
211
- )
212
- tokens = body + policy.tokensPerMessage;
213
- }
214
- // Upstream compaction reads this compatibility field for shadowedTokenCount.
229
+ const counts = node.counts;
230
+ const body =
231
+ contentTokens(message?.content ?? [], policy, attachments) +
232
+ (counts?.text[policy.tokenizer] ?? 0) +
233
+ (policy.includeReasoning
234
+ ? (counts?.reasoning[policy.tokenizer] ?? 0)
235
+ : 0);
236
+ const envelope =
237
+ counts?.envelope ||
238
+ message?.content.some((b) => b.type === "text" || b.type === "tool-call");
239
+ const tokens = body > 0 || envelope ? body + policy.tokensPerMessage : 0;
240
+ // DSH 0.2.1-rc.2 compaction requires this field for shadowedTokenCount.
215
241
  return { seq: node.seq, tokens, heuristicTokens: tokens };
216
242
  });
217
243
  const surfaceTokens = priced.reduce((sum, node) => sum + node.tokens, 0);
218
- const toolsTokens = header?.tools?.length
219
- ? countText(JSON.stringify(header.tools), policy.tokenizer) +
220
- policy.toolsOverhead
221
- : 0;
244
+ const toolsTokens = header?.toolCounts
245
+ ? header.toolCounts[policy.tokenizer] + policy.toolsOverhead
246
+ : header?.tools?.length
247
+ ? countText(JSON.stringify(header.tools), policy.tokenizer) +
248
+ policy.toolsOverhead
249
+ : 0;
222
250
  const primingTokens = header || nodes.length ? policy.replyPrimingTokens : 0;
223
251
  return {
224
252
  nodes: priced,
@@ -241,11 +269,11 @@ export function measureState(
241
269
  services: MeterServices = {},
242
270
  requestHeader?: EpochHeader,
243
271
  ) {
244
- const header =
272
+ const header: ReplayHeader | null =
245
273
  requestHeader === undefined ? state.header : canonicalHeader(requestHeader);
246
274
  const current = priceSurface(state.nodes, header, config, services);
247
275
  const anchor = state.anchor;
248
- const reusable = anchor && header && headerEquals(anchor.header, header);
276
+ const reusable = anchor && header && meterHeaderEquals(anchor.header, header);
249
277
  let baseline: TokenMeasurementBaseline;
250
278
  let surfaceDeltaTokens = 0;
251
279
  if (reusable) {