@snylonue/dsh-context-policy 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +223 -2
  3. package/assets/deepseek-v4/LICENSE +21 -0
  4. package/assets/deepseek-v4/README.md +19 -0
  5. package/assets/deepseek-v4/tokenizer.json.gz +0 -0
  6. package/assets/o200k-base/LICENSE +21 -0
  7. package/assets/o200k-base/README.md +20 -0
  8. package/assets/o200k-base/tokenizer.json.gz +0 -0
  9. package/cordis.patch.yml +19 -0
  10. package/dist/builtin-tokenizers.d.ts +2 -0
  11. package/dist/builtin-tokenizers.d.ts.map +1 -0
  12. package/dist/builtin-tokenizers.js +15 -0
  13. package/dist/builtin-tokenizers.js.map +1 -0
  14. package/dist/compaction.d.ts +43 -0
  15. package/dist/compaction.d.ts.map +1 -0
  16. package/dist/compaction.js +198 -0
  17. package/dist/compaction.js.map +1 -0
  18. package/dist/core.d.ts +49 -0
  19. package/dist/core.d.ts.map +1 -0
  20. package/dist/core.js +213 -0
  21. package/dist/core.js.map +1 -0
  22. package/dist/projection.d.ts +103 -0
  23. package/dist/projection.d.ts.map +1 -0
  24. package/dist/projection.js +137 -0
  25. package/dist/projection.js.map +1 -0
  26. package/dist/token-meter.d.ts +46 -0
  27. package/dist/token-meter.d.ts.map +1 -0
  28. package/dist/token-meter.js +55 -0
  29. package/dist/token-meter.js.map +1 -0
  30. package/dist/tokenizers.d.ts +6 -0
  31. package/dist/tokenizers.d.ts.map +1 -0
  32. package/dist/tokenizers.js +64 -0
  33. package/dist/tokenizers.js.map +1 -0
  34. package/dist/types.d.ts +62 -0
  35. package/dist/types.d.ts.map +1 -0
  36. package/dist/types.js +2 -0
  37. package/dist/types.js.map +1 -0
  38. package/dist/usage.d.ts +67 -0
  39. package/dist/usage.d.ts.map +1 -0
  40. package/dist/usage.js +81 -0
  41. package/dist/usage.js.map +1 -0
  42. package/examples/preset-compaction.yml +13 -0
  43. package/locale/en.json +6 -0
  44. package/locale/zh.json +6 -0
  45. package/package.json +91 -3
  46. package/scripts/audit.ts +105 -0
  47. package/scripts/check.ts +19 -0
  48. package/scripts/prepare-o200k.ts +40 -0
  49. package/src/builtin-tokenizers.ts +24 -0
  50. package/src/compaction.ts +293 -0
  51. package/src/core.ts +277 -0
  52. package/src/projection.ts +173 -0
  53. package/src/token-meter.ts +81 -0
  54. package/src/tokenizers.ts +78 -0
  55. package/src/types.ts +63 -0
  56. package/src/usage.ts +96 -0
  57. package/tests/bundle.test.ts +43 -0
  58. package/tests/deepseek-v4-fixtures.json +125 -0
  59. package/tests/o200k-base-fixtures.json +184 -0
  60. package/tests/policy.test.ts +773 -0
  61. package/tests/tokenizers.test.ts +51 -0
  62. package/tsconfig.check.json +15 -0
  63. package/tsconfig.json +17 -0
@@ -0,0 +1,55 @@
1
+ import { Service } from "@deepseek-ai/cordis";
2
+ import { SessionSeq } from "@deepseek-ai/dsh-session";
3
+ import "@deepseek-ai/dsh-session-projection";
4
+ import settings from "@deepseek-ai/schemastery";
5
+ import { measureState, priceSurface } from "./core.js";
6
+ import { createDisplayProjections, replayProjection } from "./projection.js";
7
+ import { usageProjection } from "./usage.js";
8
+ export const Config = settings.object({
9
+ defaultTokenizer: settings.string().default("deepseek_v4"),
10
+ providerTokenizers: settings.dict(settings.string()).default({}),
11
+ modelPolicies: settings
12
+ .array(settings.object({
13
+ provider: settings.string(),
14
+ model: settings.string().required(),
15
+ tokenizer: settings.string(),
16
+ tokensPerMessage: settings.number().min(0).step(1),
17
+ toolsOverhead: settings.number().min(0).step(1),
18
+ replyPrimingTokens: settings.number().min(0).step(1),
19
+ includeReasoning: settings.boolean(),
20
+ }))
21
+ .default([]),
22
+ });
23
+ export const inject = ["sessionProjections"];
24
+ export class ModelTokenMeter extends Service {
25
+ constructor(ctx, config) {
26
+ super(ctx, "tokenMeter");
27
+ this.config = structuredClone(config);
28
+ this.services = {
29
+ get llm() {
30
+ return ctx.get("llm");
31
+ },
32
+ };
33
+ const display = createDisplayProjections(this.config, this.services);
34
+ ctx.sessionProjections.register(usageProjection);
35
+ ctx.sessionProjections.register(display.pressure);
36
+ ctx.sessionProjections.register(display.breakdown);
37
+ ctx.sessionProjections.register(replayProjection);
38
+ }
39
+ config;
40
+ services;
41
+ measure(session, requestHeader) {
42
+ const state = this.ctx.sessionProjections.stateOf(session, "contextPolicyMeter");
43
+ if (!state)
44
+ throw new Error("ModelTokenMeter contextPolicyMeter projection is unavailable");
45
+ return measureState(state, this.config, this.services, requestHeader);
46
+ }
47
+ estimateMessage(message) {
48
+ return priceSurface([{ seq: SessionSeq(0), message, heuristicTokens: 0 }], null, this.config, this.services).surfaceTokens;
49
+ }
50
+ }
51
+ export function apply(ctx, config) {
52
+ new ModelTokenMeter(ctx, config);
53
+ }
54
+ export default { Config, inject, apply };
55
+ //# sourceMappingURL=token-meter.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"token-meter.js","sourceRoot":"","sources":["../src/token-meter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAG9C,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAItD,OAAO,qCAAqC,CAAC;AAC7C,OAAO,QAAQ,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAQ7C,MAAM,CAAC,MAAM,MAAM,GAA4C,QAAQ,CAAC,MAAM,CAAC;IAC9E,gBAAgB,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC;IAC1D,kBAAkB,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAChE,aAAa,EAAE,QAAQ;SACrB,KAAK,CACL,QAAQ,CAAC,MAAM,CAAC;QACf,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE;QAC3B,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QACnC,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE;QAC5B,gBAAgB,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClD,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,kBAAkB,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,gBAAgB,EAAE,QAAQ,CAAC,OAAO,EAAE;KACpC,CAAC,CACF;SACA,OAAO,CAAC,EAAE,CAAC;CACb,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,oBAAoB,CAAC,CAAC;AAE7C,MAAM,OAAO,eAAgB,SAAQ,OAAO;IAC3C,YAAY,GAAY,EAAE,MAAmB;QAC5C,KAAK,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,GAAG;YACf,IAAI,GAAG;gBACN,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;SACD,CAAC;QACF,MAAM,OAAO,GAAG,wBAAwB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrE,GAAG,CAAC,kBAAkB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QACjD,GAAG,CAAC,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAClD,GAAG,CAAC,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACnD,GAAG,CAAC,kBAAkB,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,CAAc;IACpB,QAAQ,CAAgB;IACxB,OAAO,CAAC,OAAgB,EAAE,aAA2B;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,OAAO,CAChD,OAAO,EACP,oBAAoB,CACpB,CAAC;QACF,IAAI,CAAC,KAAK;YACT,MAAM,IAAI,KAAK,CACd,8DAA8D,CAC9D,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACvE,CAAC;IACD,eAAe,CAAC,OAAgB;QAC/B,OAAO,YAAY,CAClB,CAAC,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,EACrD,IAAI,EACJ,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,QAAQ,CACb,CAAC,aAAa,CAAC;IACjB,CAAC;CACD;AAED,MAAM,UAAU,KAAK,CAAC,GAAY,EAAE,MAAmB;IACtD,IAAI,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC"}
@@ -0,0 +1,6 @@
1
+ import type { EpochHeader } from "@deepseek-ai/dsh-session";
2
+ import type { MeterConfig, ResolvedPolicy } from "./types.js";
3
+ export declare function registerTokenizer(name: string, count: (text: string) => number): () => void;
4
+ export declare function countText(text: string, tokenizer: string): number;
5
+ export declare function resolvePolicy(config: MeterConfig, header: EpochHeader | null): Readonly<ResolvedPolicy>;
6
+ //# sourceMappingURL=tokenizers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../src/tokenizers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAc9D,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,cAW/B;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAyBjE;AAED,wBAAgB,aAAa,CAC5B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,WAAW,GAAG,IAAI,GACxB,QAAQ,CAAC,cAAc,CAAC,CAiB1B"}
@@ -0,0 +1,64 @@
1
+ import { createTokenizerCounter } from "./builtin-tokenizers.js";
2
+ // Treat special-token-looking user text as ordinary text, rather than
3
+ // rejecting or interpreting it as control.
4
+ const caches = new Map();
5
+ const tokenizers = new Map([
6
+ ["deepseek_v4", createTokenizerCounter("deepseek-v4")],
7
+ ["o200k_base", createTokenizerCounter("o200k-base")],
8
+ ]);
9
+ export function registerTokenizer(name, count) {
10
+ if (tokenizers.has(name))
11
+ throw new Error(`Tokenizer already registered: ${name}`);
12
+ tokenizers.set(name, count);
13
+ return () => {
14
+ if (tokenizers.get(name) === count) {
15
+ tokenizers.delete(name);
16
+ caches.delete(count);
17
+ }
18
+ };
19
+ }
20
+ export function countText(text, tokenizer) {
21
+ const count = tokenizers.get(tokenizer);
22
+ if (!count)
23
+ throw new Error(`Unknown tokenizer: ${tokenizer}`);
24
+ let cache = caches.get(count);
25
+ if (!cache) {
26
+ cache = { values: new Map(), chars: 0 };
27
+ caches.set(count, cache);
28
+ }
29
+ const cached = cache.values.get(text);
30
+ if (cached !== undefined)
31
+ return cached;
32
+ const result = count(text);
33
+ if (!Number.isSafeInteger(result) || result < 0)
34
+ throw new TypeError("Tokenizer must return a non-negative safe integer");
35
+ if (text.length <= 2000000) {
36
+ while (cache.values.size >= 512 || cache.chars + text.length > 2000000) {
37
+ const oldest = cache.values.keys().next();
38
+ if (oldest.done)
39
+ break;
40
+ const key = oldest.value;
41
+ cache.chars -= key.length;
42
+ cache.values.delete(key);
43
+ }
44
+ cache.values.set(text, result);
45
+ cache.chars += text.length;
46
+ }
47
+ return result;
48
+ }
49
+ export function resolvePolicy(config, header) {
50
+ const policies = config.modelPolicies;
51
+ const route = header?.config;
52
+ const match = policies.find((p) => p.provider === route?.provider && p.model === route?.model) ??
53
+ policies.find((p) => p.provider === undefined && p.model === route?.model);
54
+ const providerTokenizer = route && config.providerTokenizers?.[route.provider];
55
+ return Object.freeze({
56
+ tokensPerMessage: 3,
57
+ toolsOverhead: 4,
58
+ replyPrimingTokens: 3,
59
+ includeReasoning: false,
60
+ ...match,
61
+ tokenizer: match?.tokenizer ?? providerTokenizer ?? config.defaultTokenizer,
62
+ });
63
+ }
64
+ //# sourceMappingURL=tokenizers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokenizers.js","sourceRoot":"","sources":["../src/tokenizers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AAEjE,sEAAsE;AACtE,2CAA2C;AAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAGnB,CAAC;AACJ,MAAM,UAAU,GAAG,IAAI,GAAG,CAAmC;IAC5D,CAAC,aAAa,EAAE,sBAAsB,CAAC,aAAa,CAAC,CAAC;IACtD,CAAC,YAAY,EAAE,sBAAsB,CAAC,YAAY,CAAC,CAAC;CACpD,CAAC,CAAC;AAEH,MAAM,UAAU,iBAAiB,CAChC,IAAY,EACZ,KAA+B;IAE/B,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC;IAC1D,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC5B,OAAO,GAAG,EAAE;QACX,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;YACpC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACF,CAAC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,SAAiB;IACxD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC/D,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAC9C,MAAM,IAAI,SAAS,CAAC,mDAAmD,CAAC,CAAC;IAC1E,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAC,MAAM,CAAC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YACxE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC1C,IAAI,MAAM,CAAC,IAAI;gBAAE,MAAM;YACvB,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC;YACzB,KAAK,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC;YAC1B,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC/B,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC5B,MAAmB,EACnB,MAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC;IACtC,MAAM,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC;IAC7B,MAAM,KAAK,GACV,QAAQ,CAAC,IAAI,CACZ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,EAAE,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,KAAK,EAAE,KAAK,CACjE;QACD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,KAAK,EAAE,KAAK,CAAC,CAAC;IAC5E,MAAM,iBAAiB,GAAG,KAAK,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/E,OAAO,MAAM,CAAC,MAAM,CAAC;QACpB,gBAAgB,EAAE,CAAC;QACnB,aAAa,EAAE,CAAC;QAChB,kBAAkB,EAAE,CAAC;QACrB,gBAAgB,EAAE,KAAK;QACvB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,EAAE,SAAS,IAAI,iBAAiB,IAAI,MAAM,CAAC,gBAAgB;KAC3E,CAAC,CAAC;AACJ,CAAC"}
@@ -0,0 +1,62 @@
1
+ import type { EpochHeader, SessionSeq, SessionLogOffset, SessionEvent } from "@deepseek-ai/dsh-session";
2
+ import type { Message, TokenUsage } from "@deepseek-ai/dsh-llm";
3
+ /** Optional domain events; no runtime dependency on their owning plugins. */
4
+ export type MeterEvent = SessionEvent | {
5
+ type: "image/offload";
6
+ seq: SessionSeq;
7
+ data: {
8
+ targets: {
9
+ seq: SessionSeq;
10
+ imageIndexes: number[];
11
+ }[];
12
+ };
13
+ } | {
14
+ type: "llm/retry-started";
15
+ seq: SessionSeq;
16
+ data: {
17
+ turn: number;
18
+ step: number;
19
+ };
20
+ };
21
+ export interface ModelPolicy {
22
+ provider?: string;
23
+ model: string;
24
+ tokenizer?: string;
25
+ tokensPerMessage?: number;
26
+ toolsOverhead?: number;
27
+ replyPrimingTokens?: number;
28
+ includeReasoning?: boolean;
29
+ }
30
+ export interface MeterConfigInput {
31
+ defaultTokenizer?: string;
32
+ providerTokenizers?: Record<string, string>;
33
+ modelPolicies?: ModelPolicy[];
34
+ }
35
+ export interface MeterConfig {
36
+ defaultTokenizer: string;
37
+ providerTokenizers?: Record<string, string>;
38
+ modelPolicies: ModelPolicy[];
39
+ }
40
+ export interface ResolvedPolicy {
41
+ tokenizer: string;
42
+ tokensPerMessage: number;
43
+ toolsOverhead: number;
44
+ replyPrimingTokens: number;
45
+ includeReasoning: boolean;
46
+ }
47
+ export interface ReplayNode {
48
+ seq: SessionSeq;
49
+ message: Message | null;
50
+ heuristicTokens: number;
51
+ }
52
+ export interface ReplayState {
53
+ nodes: readonly ReplayNode[];
54
+ header: EpochHeader | null;
55
+ anchor: {
56
+ header: EpochHeader;
57
+ nodes: readonly ReplayNode[];
58
+ usage: TokenUsage;
59
+ } | null;
60
+ logRevision: SessionLogOffset;
61
+ }
62
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,WAAW,EACX,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAEhE,6EAA6E;AAC7E,MAAM,MAAM,UAAU,GACnB,YAAY,GACZ;IACA,IAAI,EAAE,eAAe,CAAC;IACtB,GAAG,EAAE,UAAU,CAAC;IAChB,IAAI,EAAE;QAAE,OAAO,EAAE;YAAE,GAAG,EAAE,UAAU,CAAC;YAAC,YAAY,EAAE,MAAM,EAAE,CAAA;SAAE,EAAE,CAAA;KAAE,CAAC;CAChE,GACD;IACA,IAAI,EAAE,mBAAmB,CAAC;IAC1B,GAAG,EAAE,UAAU,CAAC;IAChB,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACpC,CAAC;AAEL,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC3B;AACD,MAAM,WAAW,gBAAgB;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC;CAC9B;AACD,MAAM,WAAW,WAAW;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,aAAa,EAAE,WAAW,EAAE,CAAC;CAC7B;AACD,MAAM,WAAW,cAAc;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;CAC1B;AACD,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,UAAU,CAAC;IAChB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;CACxB;AACD,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7B,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE;QACP,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;QAC7B,KAAK,EAAE,UAAU,CAAC;KAClB,GAAG,IAAI,CAAC;IACT,WAAW,EAAE,gBAAgB,CAAC;CAC9B"}
package/dist/types.js ADDED
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
@@ -0,0 +1,67 @@
1
+ import type { SessionProjectionStateMap } from "@deepseek-ai/dsh-session-projection";
2
+ import { z } from "zod";
3
+ import type { MeterEvent } from "./types.js";
4
+ export declare const usageProjection: {
5
+ key: "tokenUsage";
6
+ stateVersion: number;
7
+ stateSchema: z.ZodObject<{
8
+ totals: z.ZodObject<{
9
+ uncachedInputTokens: z.ZodNumber;
10
+ outputTokens: z.ZodNumber;
11
+ cacheReadTokens: z.ZodNumber;
12
+ cacheWriteTokens: z.ZodNumber;
13
+ }, z.core.$strict>;
14
+ last: z.ZodNullable<z.ZodObject<{
15
+ turn: z.ZodNumber;
16
+ step: z.ZodNumber;
17
+ buckets: z.ZodObject<{
18
+ uncachedInputTokens: z.ZodNumber;
19
+ outputTokens: z.ZodNumber;
20
+ cacheReadTokens: z.ZodNumber;
21
+ cacheWriteTokens: z.ZodNumber;
22
+ }, z.core.$strict>;
23
+ }, z.core.$strip>>;
24
+ }, z.core.$strict>;
25
+ init: () => {
26
+ totals: {
27
+ uncachedInputTokens: number;
28
+ outputTokens: number;
29
+ cacheReadTokens: number;
30
+ cacheWriteTokens: number;
31
+ };
32
+ last: null;
33
+ };
34
+ apply: (state: SessionProjectionStateMap["tokenUsage"], event: MeterEvent) => {
35
+ totals: {
36
+ uncachedInputTokens: number;
37
+ outputTokens: number;
38
+ cacheReadTokens: number;
39
+ cacheWriteTokens: number;
40
+ };
41
+ last: {
42
+ turn: number;
43
+ step: number;
44
+ buckets: {
45
+ uncachedInputTokens: number;
46
+ outputTokens: number;
47
+ cacheReadTokens: number;
48
+ cacheWriteTokens: number;
49
+ };
50
+ } | null;
51
+ };
52
+ wire: {
53
+ viewSchema: z.ZodObject<{
54
+ uncachedInputTokens: z.ZodNumber;
55
+ outputTokens: z.ZodNumber;
56
+ cacheReadTokens: z.ZodNumber;
57
+ cacheWriteTokens: z.ZodNumber;
58
+ }, z.core.$strict>;
59
+ view: (state: SessionProjectionStateMap["tokenUsage"]) => {
60
+ uncachedInputTokens: number;
61
+ outputTokens: number;
62
+ cacheReadTokens: number;
63
+ cacheWriteTokens: number;
64
+ };
65
+ };
66
+ };
67
+ //# sourceMappingURL=usage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"usage.d.ts","sourceRoot":"","sources":["../src/usage.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACX,yBAAyB,EAEzB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAY7C,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAqBnB,yBAAyB,CAAC,YAAY,CAAC,SACvC,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;sBAkDH,yBAAyB,CAAC,YAAY,CAAC;;;;;;;CAET,CAAC"}
package/dist/usage.js ADDED
@@ -0,0 +1,81 @@
1
+ // Provider billing fold adapted from DeepSeek Harness 0.2.0-rc.2 (MIT).
2
+ import { deepEqualJson } from "@deepseek-ai/dsh-util-values";
3
+ import { usageOf } from "./core.js";
4
+ import { z } from "zod";
5
+ const count = z.number().int().nonnegative();
6
+ const bucketsSchema = z
7
+ .object({
8
+ uncachedInputTokens: count,
9
+ outputTokens: count,
10
+ cacheReadTokens: count,
11
+ cacheWriteTokens: count,
12
+ })
13
+ .strict();
14
+ export const usageProjection = {
15
+ key: "tokenUsage",
16
+ stateVersion: 2,
17
+ stateSchema: z
18
+ .object({
19
+ totals: bucketsSchema,
20
+ last: z
21
+ .object({ turn: count, step: count, buckets: bucketsSchema })
22
+ .nullable(),
23
+ })
24
+ .strict(),
25
+ init: () => ({
26
+ totals: {
27
+ uncachedInputTokens: 0,
28
+ outputTokens: 0,
29
+ cacheReadTokens: 0,
30
+ cacheWriteTokens: 0,
31
+ },
32
+ last: null,
33
+ }),
34
+ apply: (state, event) => {
35
+ if (event.type === "llm/retry-started")
36
+ return state.last?.turn === event.data.turn &&
37
+ state.last.step === event.data.step
38
+ ? { ...state, last: null }
39
+ : state;
40
+ if (event.type !== "assistant/message" &&
41
+ event.type !== "assistant/attempt")
42
+ return state;
43
+ const usage = usageOf(event);
44
+ if (usage === undefined)
45
+ return state;
46
+ const { turn, step } = event.data;
47
+ const buckets = {
48
+ uncachedInputTokens: usage.inputTokens,
49
+ outputTokens: usage.outputTokens,
50
+ cacheReadTokens: usage.cacheReadTokens ?? 0,
51
+ cacheWriteTokens: usage.cacheWriteTokens ?? 0,
52
+ };
53
+ const previous = state.last?.turn === turn && state.last.step === step
54
+ ? state.last.buckets
55
+ : undefined;
56
+ if (previous && deepEqualJson(previous, buckets))
57
+ return state;
58
+ return {
59
+ totals: {
60
+ uncachedInputTokens: state.totals.uncachedInputTokens -
61
+ (previous?.uncachedInputTokens ?? 0) +
62
+ buckets.uncachedInputTokens,
63
+ outputTokens: state.totals.outputTokens -
64
+ (previous?.outputTokens ?? 0) +
65
+ buckets.outputTokens,
66
+ cacheReadTokens: state.totals.cacheReadTokens -
67
+ (previous?.cacheReadTokens ?? 0) +
68
+ buckets.cacheReadTokens,
69
+ cacheWriteTokens: state.totals.cacheWriteTokens -
70
+ (previous?.cacheWriteTokens ?? 0) +
71
+ buckets.cacheWriteTokens,
72
+ },
73
+ last: { turn, step, buckets },
74
+ };
75
+ },
76
+ wire: {
77
+ viewSchema: bucketsSchema,
78
+ view: (state) => state.totals,
79
+ },
80
+ };
81
+ //# sourceMappingURL=usage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"usage.js","sourceRoot":"","sources":["../src/usage.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;AAC7C,MAAM,aAAa,GAAG,CAAC;KACrB,MAAM,CAAC;IACP,mBAAmB,EAAE,KAAK;IAC1B,YAAY,EAAE,KAAK;IACnB,eAAe,EAAE,KAAK;IACtB,gBAAgB,EAAE,KAAK;CACvB,CAAC;KACD,MAAM,EAAE,CAAC;AAEX,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,GAAG,EAAE,YAAY;IACjB,YAAY,EAAE,CAAC;IACf,WAAW,EAAE,CAAC;SACZ,MAAM,CAAC;QACP,MAAM,EAAE,aAAa;QACrB,IAAI,EAAE,CAAC;aACL,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;aAC5D,QAAQ,EAAE;KACZ,CAAC;SACD,MAAM,EAAE;IACV,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;QACZ,MAAM,EAAE;YACP,mBAAmB,EAAE,CAAC;YACtB,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;SACnB;QACD,IAAI,EAAE,IAAI;KACV,CAAC;IACF,KAAK,EAAE,CACN,KAA8C,EAC9C,KAAiB,EAChB,EAAE;QACH,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB;YACrC,OAAO,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI;gBAC1C,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI;gBACnC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE;gBAC1B,CAAC,CAAC,KAAK,CAAC;QACV,IACC,KAAK,CAAC,IAAI,KAAK,mBAAmB;YAClC,KAAK,CAAC,IAAI,KAAK,mBAAmB;YAElC,OAAO,KAAK,CAAC;QACd,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC;QAClC,MAAM,OAAO,GAAyB;YACrC,mBAAmB,EAAE,KAAK,CAAC,WAAW;YACtC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,CAAC;YAC3C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,CAAC;SAC7C,CAAC;QACF,MAAM,QAAQ,GACb,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI;YACpD,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO;YACpB,CAAC,CAAC,SAAS,CAAC;QACd,IAAI,QAAQ,IAAI,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/D,OAAO;YACN,MAAM,EAAE;gBACP,mBAAmB,EAClB,KAAK,CAAC,MAAM,CAAC,mBAAmB;oBAChC,CAAC,QAAQ,EAAE,mBAAmB,IAAI,CAAC,CAAC;oBACpC,OAAO,CAAC,mBAAmB;gBAC5B,YAAY,EACX,KAAK,CAAC,MAAM,CAAC,YAAY;oBACzB,CAAC,QAAQ,EAAE,YAAY,IAAI,CAAC,CAAC;oBAC7B,OAAO,CAAC,YAAY;gBACrB,eAAe,EACd,KAAK,CAAC,MAAM,CAAC,eAAe;oBAC5B,CAAC,QAAQ,EAAE,eAAe,IAAI,CAAC,CAAC;oBAChC,OAAO,CAAC,eAAe;gBACxB,gBAAgB,EACf,KAAK,CAAC,MAAM,CAAC,gBAAgB;oBAC7B,CAAC,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC;oBACjC,OAAO,CAAC,gBAAgB;aACzB;YACD,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;SAC7B,CAAC;IACH,CAAC;IACD,IAAI,EAAE;QACL,UAAU,EAAE,aAAa;QACzB,IAAI,EAAE,CAAC,KAA8C,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM;KACtE;CAC4C,CAAC"}
@@ -0,0 +1,13 @@
1
+ # Replace the existing compaction group's child in the preset's plugin list.
2
+ # Preserve its group/isolate placement; do not replace a whole preset config
3
+ # with this fragment, because DSH patch config replacement is not a deep merge.
4
+ - group: true
5
+ isolate: [compaction]
6
+ config:
7
+ - name: '@snylonue/dsh-context-policy/compaction'
8
+ config:
9
+ thresholdRatio: 0.8
10
+ retainRatio: 0.16
11
+ maxTokens: 65536
12
+ compactionRetries: 1
13
+ maxOverflowRetries: 1
package/locale/en.json ADDED
@@ -0,0 +1,6 @@
1
+ {
2
+ "meta": {
3
+ "title": "Context Policy",
4
+ "description": "Percentage-based context compaction and model-configurable token counting."
5
+ }
6
+ }
package/locale/zh.json ADDED
@@ -0,0 +1,6 @@
1
+ {
2
+ "meta": {
3
+ "title": "Context Policy",
4
+ "description": "按上下文窗口比例自动压缩,并支持按模型配置的 token 计数。"
5
+ }
6
+ }
package/package.json CHANGED
@@ -1,6 +1,94 @@
1
1
  {
2
2
  "name": "@snylonue/dsh-context-policy",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
3
+ "version": "0.1.1",
4
+ "description": "Percentage-only compaction and model-configurable GPT and DeepSeek token counting for DSH",
5
+ "type": "module",
6
+ "license": "MIT",
7
+ "keywords": [
8
+ "dsh-plugin"
9
+ ],
10
+ "repository": {
11
+ "type": "git",
12
+ "url": "github:snylonue/dsh-context-policy"
13
+ },
14
+ "publishConfig": {
15
+ "access": "public"
16
+ },
17
+ "engines": {
18
+ "node": ">=24",
19
+ "dsh": "0.2.0-rc.2"
20
+ },
21
+ "exports": {
22
+ ".": {
23
+ "types": "./dist/token-meter.d.ts",
24
+ "default": "./dist/token-meter.js"
25
+ },
26
+ "./token-meter": {
27
+ "types": "./dist/token-meter.d.ts",
28
+ "default": "./dist/token-meter.js"
29
+ },
30
+ "./compaction": {
31
+ "types": "./dist/compaction.d.ts",
32
+ "default": "./dist/compaction.js"
33
+ },
34
+ "./tokenizers": {
35
+ "types": "./dist/tokenizers.d.ts",
36
+ "default": "./dist/tokenizers.js"
37
+ },
38
+ "./core": {
39
+ "types": "./dist/core.d.ts",
40
+ "default": "./dist/core.js"
41
+ },
42
+ "./locale/*.json": "./locale/*.json",
43
+ "./package.json": "./package.json"
44
+ },
45
+ "files": [
46
+ "dist",
47
+ "src",
48
+ "assets",
49
+ "cordis.patch.yml",
50
+ "locale/*.json",
51
+ "README.md",
52
+ "LICENSE",
53
+ "UPSTREAM.md",
54
+ "examples",
55
+ "tsconfig.json",
56
+ "scripts",
57
+ "tsconfig.check.json",
58
+ "tests"
59
+ ],
60
+ "dependencies": {
61
+ "@huggingface/tokenizers": "0.2.0"
62
+ },
63
+ "peerDependencies": {
64
+ "@deepseek-ai/cordis": "~4.0.4",
65
+ "@deepseek-ai/dsh-compaction": "0.2.0-rc.2",
66
+ "@deepseek-ai/dsh-compaction-basic": "0.2.0-rc.2",
67
+ "@deepseek-ai/dsh-llm": "0.2.0-rc.2",
68
+ "@deepseek-ai/dsh-session": "0.2.0-rc.2",
69
+ "@deepseek-ai/dsh-session-projection": "0.2.0-rc.2",
70
+ "@deepseek-ai/dsh-token-meter": "0.2.0-rc.2",
71
+ "@deepseek-ai/dsh-util-values": "0.2.0-rc.2",
72
+ "@deepseek-ai/schemastery": "~3.18.4",
73
+ "zod": "^4.4.3"
74
+ },
75
+ "dsh": {
76
+ "manifestVersion": 1,
77
+ "bundle": {
78
+ "patch": "./cordis.patch.yml"
79
+ }
80
+ },
81
+ "devDependencies": {
82
+ "@biomejs/biome": "2",
83
+ "@deepseek-ai/dsh-app-boot": "0.2.0-rc.2",
84
+ "@types/node": "24.19.0",
85
+ "typescript": "5.9.3"
86
+ },
87
+ "scripts": {
88
+ "build": "tsc -p tsconfig.json",
89
+ "test": "node --test tests/*.test.ts",
90
+ "check": "node scripts/check.ts",
91
+ "format": "biome format --write .",
92
+ "lint": "biome lint ."
93
+ }
6
94
  }
@@ -0,0 +1,105 @@
1
+ /** Aggregate local prediction errors; never emits conversation content. */
2
+ import { readFile } from "node:fs/promises";
3
+ import { zstdDecompressSync } from "node:zlib";
4
+ import { basename, dirname } from "node:path";
5
+ import type { SessionEvent } from "@deepseek-ai/dsh-session";
6
+ import {
7
+ initialState,
8
+ fold,
9
+ usageOf,
10
+ inputTokens,
11
+ measureState,
12
+ } from "../dist/core.js";
13
+ const meterConfig = { defaultTokenizer: "deepseek_v4", modelPolicies: [] };
14
+ const files = process.argv.slice(2);
15
+ if (!files.length)
16
+ throw new Error("Pass one or more session.v4.jsonl.zstd paths");
17
+ const percentile = (values: number[], fraction: number) => {
18
+ const sorted = values.toSorted((a, b) => a - b);
19
+ return sorted.length
20
+ ? sorted[Math.floor((sorted.length - 1) * fraction)]
21
+ : null;
22
+ };
23
+ const rows = [];
24
+ for (const file of files) {
25
+ const compressed = await readFile(file);
26
+ const decoded: Buffer[] = [];
27
+ for (let offset = 0; offset < compressed.length; ) {
28
+ const result = zstdDecompressSync(compressed.subarray(offset), {
29
+ info: true,
30
+ }) as unknown as { buffer: Buffer; engine: { bytesWritten: number } };
31
+ if (!result.engine.bytesWritten)
32
+ throw new Error("Invalid empty Zstandard frame");
33
+ decoded.push(result.buffer);
34
+ offset += result.engine.bytesWritten;
35
+ }
36
+ const records = Buffer.concat(decoded)
37
+ .toString("utf8")
38
+ .trim()
39
+ .split("\n")
40
+ .map((line) => JSON.parse(line));
41
+ let state = initialState(),
42
+ zeroUsagePreserved = 0;
43
+ const errors: number[] = [],
44
+ initialErrors: number[] = [];
45
+ const regressions = [];
46
+ for (const raw of records) {
47
+ if (typeof raw.seq !== "number" || typeof raw.type !== "string") continue;
48
+ const event = raw as SessionEvent;
49
+ if (
50
+ event.type === "assistant/message" ||
51
+ event.type === "assistant/attempt"
52
+ ) {
53
+ const before = measureState(state, meterConfig);
54
+ const usage = usageOf(event);
55
+ if (usage && inputTokens(usage) === 0) {
56
+ const after = measureState(fold(state, event), meterConfig);
57
+ if (
58
+ event.type === "assistant/attempt" &&
59
+ before.totalTokens === after.totalTokens
60
+ )
61
+ zeroUsagePreserved++;
62
+ }
63
+ if (
64
+ event.type === "assistant/message" &&
65
+ usage &&
66
+ inputTokens(usage) > 0
67
+ ) {
68
+ const actual = inputTokens(usage),
69
+ error = (before.totalTokens - actual) / actual;
70
+ (before.baseline.kind === "usage" ? errors : initialErrors).push(error);
71
+ if ([30, 35, 83].includes(event.seq))
72
+ regressions.push({
73
+ seq: event.seq,
74
+ predicted: before.totalTokens,
75
+ actual,
76
+ errorPercent: Math.round(error * 10000) / 100,
77
+ });
78
+ }
79
+ }
80
+ state = fold(state, event);
81
+ }
82
+ const absolute = errors.map(Math.abs);
83
+ rows.push({
84
+ session: basename(dirname(file)),
85
+ successfulAnchoredRequests: errors.length,
86
+ unanchoredRequests: initialErrors.length,
87
+ zeroUsagePreserved,
88
+ anchoredMedianAbsoluteErrorPercent: (percentile(absolute, 0.5) ?? 0) * 100,
89
+ anchoredP90AbsoluteErrorPercent: (percentile(absolute, 0.9) ?? 0) * 100,
90
+ anchoredWorstAbsoluteErrorPercent: Math.max(0, ...absolute) * 100,
91
+ regressions,
92
+ });
93
+ }
94
+ console.log(
95
+ JSON.stringify(
96
+ {
97
+ tokenizer: meterConfig.defaultTokenizer,
98
+ adapterPricingAvailable: false,
99
+ note: "Offline estimates omit provider-specific image/file projections and serialization.",
100
+ rows,
101
+ },
102
+ null,
103
+ 2,
104
+ ),
105
+ );
@@ -0,0 +1,19 @@
1
+ import { spawnSync } from "node:child_process";
2
+ import { createRequire } from "node:module";
3
+ import { fileURLToPath } from "node:url";
4
+ const require = createRequire(import.meta.url);
5
+ const cwd = fileURLToPath(new URL("../", import.meta.url));
6
+ for (const args of [
7
+ [require.resolve("typescript/bin/tsc"), "-p", "tsconfig.json"],
8
+ [require.resolve("typescript/bin/tsc"), "-p", "tsconfig.check.json"],
9
+ [
10
+ "--test",
11
+ "tests/policy.test.ts",
12
+ "tests/bundle.test.ts",
13
+ "tests/tokenizers.test.ts",
14
+ ],
15
+ ]) {
16
+ const result = spawnSync(process.execPath, args, { cwd, stdio: "inherit" });
17
+ if (result.error) throw result.error;
18
+ if (result.status !== 0) process.exit(result.status ?? 1);
19
+ }