doppler-gpu 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CHANGELOG.md +76 -0
  2. package/README.md +4 -0
  3. package/package.json +2 -2
  4. package/src/client/inspection.d.ts +27 -0
  5. package/src/client/inspection.js +7 -1
  6. package/src/client/model-host/layer-projection.js +46 -0
  7. package/src/client/model-host/model-session.d.ts +18 -0
  8. package/src/client/model-host/model-session.js +7 -1
  9. package/src/config/inspection/activity-stages.json +29 -1
  10. package/src/config/inspection/observation-policies.json +14 -2
  11. package/src/config/kernels/registry.json +2 -2
  12. package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-candidate.json +4 -4
  13. package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-pooled-q8-common-diagnostic.json +4 -4
  14. package/src/debug/activation-trace.d.ts +1 -0
  15. package/src/debug/activity-trace.d.ts +24 -0
  16. package/src/debug/activity-trace.js +112 -7
  17. package/src/gpu/kernels/moe_shared_slot_combine_f32.wgsl +11 -11
  18. package/src/gpu/kernels/moe_shared_slot_down_f32.wgsl +11 -11
  19. package/src/gpu/kernels/moe_shared_slot_down_pair_combine_f32.wgsl +11 -11
  20. package/src/gpu/kernels/moe_shared_slot_down_pair_combine_residual_f32.wgsl +11 -11
  21. package/src/gpu/kernels/moe_shared_slot_gu_f32.wgsl +11 -11
  22. package/src/gpu/kernels/moe_shared_slot_gu_pair_f32.wgsl +11 -11
  23. package/src/gpu/kernels/sigmoid_gated_add_f32.wgsl +3 -3
  24. package/src/inference/pipelines/shader-scoped-pipeline.js +1 -0
  25. package/src/inference/pipelines/text/generator/decode-runtime.js +3 -1
  26. package/src/inference/pipelines/text/generator/hidden-projection.d.ts +18 -0
  27. package/src/inference/pipelines/text/generator/hidden-projection.js +49 -0
  28. package/src/inference/pipelines/text/generator.js +5 -0
  29. package/src/inference/pipelines/text/partition-execution.js +5 -0
  30. package/src/inference/pipelines/text/pipeline.d.ts +8 -0
  31. package/src/inference/pipelines/text/pipeline.js +4 -0
  32. package/src/inference/pipelines/text/sequence-state.d.ts +2 -0
  33. package/src/inference/pipelines/text/sequence-state.js +7 -0
  34. package/src/memory/sequence-retirement.js +19 -0
  35. package/src/version.js +1 -1
package/CHANGELOG.md CHANGED
@@ -4,6 +4,82 @@ Historical package notes are preserved outside the runtime archive in
4
4
  `docs/status/archive/package-changelog-through-0.6.1.md` and the earlier
5
5
  `package-changelog-through-0.4.15.md` snapshot in that directory.
6
6
 
7
+ ## [0.7.2] - 2026-11-11
8
+
9
+ ### Added
10
+
11
+ - `model.inspect.projectLayerOutputs({ vectors, topK })` projects recorded
12
+ hidden states through the loaded model's final normalization and output head.
13
+ It returns vocabulary-normalized probabilities explicitly labeled
14
+ `projected`. Intermediate projections are an inspection aid, not generated
15
+ tokens or evidence of causation. Hyper-connection and paired-precision plans
16
+ fail explicitly. Calls accept at most 64 vectors and 64 candidates per row.
17
+
18
+ ### Fixed
19
+
20
+ - Projection requests snapshot both vector values and row count before waiting
21
+ for execution. Mutating the caller's vector array cannot change an admitted
22
+ request's shape.
23
+
24
+ ### Limits
25
+
26
+ - Projection reads back a full vocabulary row for each input vector; recording
27
+ and projection costs are separate from uninstrumented inference performance.
28
+ - Flash-Next's 2,187-token full-logit gate remains failing and unqualified.
29
+ This release does not fix or relax that gate.
30
+
31
+ ### Comparison evidence
32
+
33
+ - The release report includes Qwen 3.5 0.8B and 2B on Apple M3 against
34
+ Transformers.js 4.3.2 / ONNX Runtime WebGPU and llama.cpp, plus Qwen3.8 27B
35
+ on Jupiter against direct ONNX Runtime WebGPU and llama.cpp Vulkan.
36
+ It retains exact prompts, artifact identities, outputs, load times and
37
+ complete-request timings. Host variability and output mismatches prevent a
38
+ general speed or quality claim; native llama.cpp remains ahead on the
39
+ retained matched-output workloads. See
40
+ [the release evidence](https://github.com/clocksmith/doppler/tree/main/reports/package-release-0.7.2).
41
+
42
+ ## [0.7.1] - 2026-11-11
43
+
44
+ Patch release. Published bytes for 0.7.0 are unchanged.
45
+
46
+ ### Fixed
47
+
48
+ - **Gate activations agree across fused and unfused paths.** 0.7.0 removed
49
+ the +/-15 sigmoid input clamp from `silu.wgsl`, but
50
+ `sigmoid_gated_add_f32.wgsl` and the `moe_shared_slot_*` kernels kept it.
51
+ For |gate| > 15 the separate and fused paths disagreed, and the Qwen
52
+ shared-expert gate differed from the logistic function: `sigmoid(-20)`
53
+ gave 3.06e-7 instead of 2.06e-9. All of them now use `silu.wgsl`'s
54
+ unclamped form; only the explicit `swigluLimit` output clamp remains. The
55
+ two Qwen 3.6 35B-A3B q4_0 runtime profiles carry the current kernel digests.
56
+ - **Repeated prompts no longer exhaust an explicit device memory budget.**
57
+ With `configureDeviceMemoryBudget`, idle pooled buffers count as live.
58
+ Each new prompt length added buckets on top of earlier ones until a later
59
+ request was rejected. A new generation's first prefill now retires idle
60
+ pooled buffers after confirmed GPU completion, only when a budget is set.
61
+ Owned buffers are never retired. Measured with Qwen 3.5 2B under a
62
+ 6,442,450,944-byte budget: steady memory between requests stays at or
63
+ below 2.6 GB.
64
+
65
+ ### Added
66
+
67
+ - `model.inspect.generate(prompt, { capture })` narrows activity recording by
68
+ kind, layer, token range and budget (selective capture, `e56cf5d6`).
69
+
70
+ ### Known limits
71
+
72
+ - Budgeted hosts should also set
73
+ `inference.session.prefillChunkSubmitMode: "sync"` (or
74
+ `prefillMaxInFlightChunks: 1`) for models whose manifest declares async
75
+ prefill submits. With async submits, warm requests keep up to three layer
76
+ chunks of prompt-sized temporaries alive. Qwen 3.5 2B at 2,000 tokens
77
+ peaked at 4.40 GB with sync, against a failure under the same 6.44 GB
78
+ budget with async, at the same prefill time.
79
+ - Flash-Next (Qwen 3.8) is not qualified. Its 2,187-token prompt still fails
80
+ the full-logit gate against independent CPU references; see
81
+ `reports/flash-next-reference-controls-20261110/`.
82
+
7
83
  ## [0.7.0] - 2026-11-10
8
84
 
9
85
  First npm release since 0.6.1. It also carries the 0.6.27 and 0.6.2 candidate
package/README.md CHANGED
@@ -1,5 +1,9 @@
1
1
  # Doppler
2
2
 
3
+ <a href="assets/readme/doppler-demo-hero.mp4"><img src="assets/readme/doppler-demo-hero.gif" width="100%" alt="Doppler demo: a recorded Qwen 3.5 0.8B answer appears token by token beside per-layer activation changes, then one word's next-token alternatives open with their probabilities." /></a>
4
+
5
+ <sub>Recorded replay from the <a href="https://d4da.com/doppler/">demo</a>, not a live run. <a href="assets/readme/doppler-demo-hero.mp4">Full-quality video</a>.</sub>
6
+
3
7
  A JavaScript library for local model execution through WebGPU: generation,
4
8
  embeddings, reranking, and choice scoring, qualified per model and host.
5
9
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "doppler-gpu",
3
- "version": "0.7.0",
3
+ "version": "0.7.2",
4
4
  "description": "Doppler Rig prepares signed model Capsules; Doppler Run executes them locally through WebGPU for JavaScript",
5
5
  "main": "src/capsule-runtime.js",
6
6
  "types": "src/capsule-runtime.d.ts",
@@ -355,7 +355,7 @@
355
355
  "tools/convert-safetensors-node.js"
356
356
  ],
357
357
  "devDependencies": {
358
- "@huggingface/transformers": "^4.0.1",
358
+ "@huggingface/transformers": "^4.3.2",
359
359
  "@types/node": "^25.6.0",
360
360
  "@webgpu/types": "^0.1.69",
361
361
  "jest": "^30.2.0",
@@ -113,6 +113,33 @@ export declare function buildInspectionTokenRecords(
113
113
  topKSize?: number
114
114
  ): DopplerInspectionTokenRecord[];
115
115
 
116
+ export declare const LAYER_PROJECTION_SCHEMA: 'doppler.layer-projection/v1';
117
+ export declare const LAYER_PROJECTION_MAX_ROWS: 64;
118
+ export declare const LAYER_PROJECTION_MAX_TOP_K: 64;
119
+
120
+ export interface DopplerLayerProjection {
121
+ schema: typeof LAYER_PROJECTION_SCHEMA;
122
+ modelId: string;
123
+ /** What the vectors were passed through. `kind` is always `projected`: these are not tokens the model produced. */
124
+ projection: { through: ['final-normalization', 'output-head']; kind: 'projected' };
125
+ vocabSize: number;
126
+ rows: Array<{ candidates: DopplerTokenCandidate[] }>;
127
+ }
128
+
129
+ export interface DopplerTokenCandidate {
130
+ tokenId: number;
131
+ logit: number;
132
+ text: string;
133
+ probability: number | null;
134
+ }
135
+
136
+ /** The `count` highest logits with their softmax probability over the whole row. */
137
+ export declare function topCandidates(
138
+ logits: ArrayLike<number> | null | undefined,
139
+ tokenizer: { decode(ids: number[], skipSpecialTokens?: boolean, trim?: boolean): string },
140
+ count: number
141
+ ): DopplerTokenCandidate[];
142
+
116
143
  export declare function buildInspectionTokenRecord(
117
144
  tokenId: number,
118
145
  logits: Float32Array | null,
@@ -215,7 +215,13 @@ function selectedProbability(logits, tokenId) {
215
215
  return Math.exp(Number(logits[tokenId]) - maximum) / denominator;
216
216
  }
217
217
 
218
- function topCandidates(logits, tokenizer, count) {
218
+ export const LAYER_PROJECTION_SCHEMA = 'doppler.layer-projection/v1';
219
+ // One request returns a full vocabulary row per vector before the top candidates
220
+ // are kept, so both bounds limit what a single call can hold.
221
+ export const LAYER_PROJECTION_MAX_ROWS = 64;
222
+ export const LAYER_PROJECTION_MAX_TOP_K = 64;
223
+
224
+ export function topCandidates(logits, tokenizer, count) {
219
225
  if (!logits || !Number.isInteger(count) || count <= 0) return [];
220
226
  const candidates = [];
221
227
  for (let tokenId = 0; tokenId < logits.length; tokenId += 1) {
@@ -0,0 +1,46 @@
1
+ import { runPipelineOperation } from '../../inference/pipelines/shader-scoped-pipeline.js';
2
+ import {
3
+ LAYER_PROJECTION_MAX_ROWS,
4
+ LAYER_PROJECTION_MAX_TOP_K,
5
+ LAYER_PROJECTION_SCHEMA,
6
+ topCandidates,
7
+ } from '../inspection.js';
8
+
9
+ // Projects recorded layer outputs through the loaded model's final
10
+ // normalization and output head. The pipeline owns the computation; this
11
+ // validates the request, snapshots it and labels the result. The candidates are
12
+ // a projection of each vector: they are not tokens the model produced there.
13
+ /** @type {typeof import('./layer-projection.js').projectLayerOutputs} */
14
+ export const projectLayerOutputs = async function projectLayerOutputs(pipeline, modelId, request) {
15
+ const vectors = request?.vectors;
16
+ const topK = request?.topK;
17
+ if (!Array.isArray(vectors) || vectors.length === 0 || vectors.length > LAYER_PROJECTION_MAX_ROWS) {
18
+ throw new Error(`Doppler model.inspect.projectLayerOutputs requires 1 to ${LAYER_PROJECTION_MAX_ROWS} vectors.`);
19
+ }
20
+ if (!Number.isInteger(topK) || topK < 1 || topK > LAYER_PROJECTION_MAX_TOP_K) {
21
+ throw new Error(`Doppler model.inspect.projectLayerOutputs requires topK as a whole number from 1 to ${LAYER_PROJECTION_MAX_TOP_K}.`);
22
+ }
23
+ const width = vectors[0]?.length;
24
+ if (!Number.isInteger(width) || width < 1 || vectors.some((vector) => !(vector instanceof Float32Array) || vector.length !== width)) {
25
+ throw new Error('Doppler model.inspect.projectLayerOutputs requires Float32Array vectors of one length.');
26
+ }
27
+ // Snapshot caller-owned vectors before awaiting the pipeline's execution slot.
28
+ const requestedRows = vectors.length;
29
+ const hiddenStates = new Float32Array(requestedRows * width);
30
+ vectors.forEach((vector, row) => hiddenStates.set(vector, row * width));
31
+ return runPipelineOperation(pipeline, async (scoped) => {
32
+ if (typeof scoped.projectHiddenStates !== 'function') {
33
+ throw new Error('The loaded Doppler pipeline cannot project hidden states.');
34
+ }
35
+ const { logits, rowCount, vocabSize } = await scoped.projectHiddenStates(hiddenStates, requestedRows);
36
+ return {
37
+ schema: LAYER_PROJECTION_SCHEMA,
38
+ modelId,
39
+ projection: { through: ['final-normalization', 'output-head'], kind: 'projected' },
40
+ vocabSize,
41
+ rows: Array.from({ length: rowCount }, (_, row) => ({
42
+ candidates: topCandidates(logits.subarray(row * vocabSize, (row + 1) * vocabSize), scoped.tokenizer, topK),
43
+ })),
44
+ };
45
+ });
46
+ };
@@ -231,6 +231,12 @@ export interface DopplerModelHandle {
231
231
  policyId?: string;
232
232
  generation?: DopplerGenerateOptions;
233
233
  topKSize?: number;
234
+ /**
235
+ * Narrows what an activity-recording policy saves: capture kinds, layers,
236
+ * a token range, and smaller budgets. It can only record less than the
237
+ * policy allows; anything outside the policy fails before the run.
238
+ */
239
+ capture?: import('../../debug/activity-trace.js').ActivityCaptureRequest | null;
234
240
  /**
235
241
  * Exact token IDs to prefill instead of tokenizing `prompt`. Generation
236
242
  * starts from cleared attention and recurrent state, so the last ID is
@@ -255,6 +261,18 @@ export interface DopplerModelHandle {
255
261
  }) => void;
256
262
  }
257
263
  ): Promise<DopplerModelInspectionReceipt>;
264
+ /**
265
+ * Projects hidden-state vectors (for example recorded `layer-output` frames)
266
+ * through this model's final normalization and output head, and returns the
267
+ * `topK` candidates per vector. The result is a projection, sometimes called
268
+ * a logit lens: an estimate of what each vector leans toward, not a token the
269
+ * model produced at that layer. Fails for hyper-connection models and for
270
+ * paired-precision execution plans. Does not change sequence state.
271
+ */
272
+ projectLayerOutputs(request: {
273
+ vectors: Float32Array[];
274
+ topK: number;
275
+ }): Promise<import('../inspection.js').DopplerLayerProjection>;
258
276
  };
259
277
  readonly advanced: {
260
278
  tokenizeText(text: string): number[];
@@ -5,6 +5,7 @@ import {
5
5
  } from './model-evidence.js';
6
6
  import { getKernelCapabilities } from '../../gpu/device.js';
7
7
  import { createActivationTrace } from '../../debug/activation-trace.js';
8
+ import { resolveActivityCapture } from '../../debug/activity-trace.js';
8
9
  import { DOPPLER_VERSION } from '../../version.js';
9
10
  import { createGenerationOutputDecoder, decodeGenerationOutput } from '../../inference/generation-output.js';
10
11
  import { formatChatMessages } from '../../inference/pipelines/text/chat-format.js';
@@ -23,6 +24,7 @@ import {
23
24
  listObservationPolicies,
24
25
  resolveObservationPolicy,
25
26
  } from '../inspection.js';
27
+ import { projectLayerOutputs } from './layer-projection.js';
26
28
  import {
27
29
  activateLoRAFromTrainingOutputForPipeline,
28
30
  activatePreparedLoRAForPipeline,
@@ -625,7 +627,8 @@ export function createModelHandle(pipeline, resolved) {
625
627
  let streaming = typeof options.onEvent === 'function';
626
628
  let tokenIndex = 0;
627
629
  const topKSize = Number.isInteger(options.topKSize) ? options.topKSize : 5;
628
- const traceConfig = policy.activationTrace;
630
+ // A request may record less than the policy allows; the receipt keeps what was chosen.
631
+ const traceConfig = resolveActivityCapture(policy.activationTrace, options.capture);
629
632
  const activationTrace = traceConfig ? createActivationTrace(traceConfig, {
630
633
  layers: pipeline.modelConfig.numLayers,
631
634
  width: pipeline.modelConfig.hiddenSize * (pipeline.modelConfig.hyperConnections?.streamCount ?? 1),
@@ -728,6 +731,9 @@ export function createModelHandle(pipeline, resolved) {
728
731
  }
729
732
  return receipt;
730
733
  },
734
+ projectLayerOutputs(request) {
735
+ return projectLayerOutputs(pipeline, resolved.modelId, request);
736
+ },
731
737
  };
732
738
 
733
739
  return handle;
@@ -10,5 +10,33 @@
10
10
  "ffnDown",
11
11
  "down"
12
12
  ]
13
- }
13
+ },
14
+ "roles": [
15
+ "kv-key",
16
+ "kv-value",
17
+ "attention-query",
18
+ "ffn-gate",
19
+ "ffn-up",
20
+ "ffn-product",
21
+ "ffn-down-input",
22
+ "recurrent-memory",
23
+ "convolution-history",
24
+ "linear-a",
25
+ "linear-b",
26
+ "linear-a-log",
27
+ "linear-dt-bias",
28
+ "expert-scores",
29
+ "expert-indices",
30
+ "expert-weights",
31
+ "logits",
32
+ "weight-access"
33
+ ],
34
+ "constantRoles": [
35
+ "linear-a-log",
36
+ "linear-dt-bias"
37
+ ],
38
+ "prefixRoles": [
39
+ "kv-key",
40
+ "kv-value"
41
+ ]
14
42
  }
@@ -151,7 +151,13 @@
151
151
  "weightRawRows": 4,
152
152
  "weightMaxBytes": 16777216,
153
153
  "activityTokenWindow": 4,
154
- "activitySummaryBlock": 256
154
+ "activitySummaryBlock": 256,
155
+ "activitySelection": {
156
+ "roles": null,
157
+ "layers": null,
158
+ "tokens": null
159
+ },
160
+ "activityRoleMaxBytes": {}
155
161
  }
156
162
  },
157
163
  {
@@ -196,7 +202,13 @@
196
202
  "weightRawRows": 4,
197
203
  "weightMaxBytes": 16777216,
198
204
  "activityTokenWindow": 4,
199
- "activitySummaryBlock": 256
205
+ "activitySummaryBlock": 256,
206
+ "activitySelection": {
207
+ "roles": null,
208
+ "layers": null,
209
+ "tokens": null
210
+ },
211
+ "activityRoleMaxBytes": {}
200
212
  }
201
213
  }
202
214
  ]
@@ -11991,8 +11991,8 @@
11991
11991
  },
11992
11992
  "description": "Explicit F32 matrix row projections with subgroup-owned outputs; independent activation rows use grid y and retain the scalar-buffer single-row arithmetic.",
11993
11993
  "reachability": {
11994
- "status": "unused",
11995
- "inlineConfigs": [],
11994
+ "status": "pinned",
11995
+ "inlineConfigs": ["runtime/profiles/flash-next-routed-f32kv-qualification"],
11996
11996
  "ruleChains": [],
11997
11997
  "wgslOnDisk": true
11998
11998
  },
@@ -190,7 +190,7 @@
190
190
  "constants": {
191
191
  "HAS_GATE": true
192
192
  },
193
- "digest": "sha256:c6e28a7e6b2a7cfa805db03c9079fbc90fc2717c621e388bfa55b75ce1737ac0"
193
+ "digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
194
194
  },
195
195
  {
196
196
  "op": "down_proj",
@@ -288,13 +288,13 @@
288
288
  "op": "moe_shared_slot_gu",
289
289
  "kernel": "moe_shared_slot_gu_pair_f32.wgsl",
290
290
  "entry": "main",
291
- "digest": "sha256:1a28e076e93486589a3eace88c3cb230ff1ef4aa982fdd91dcc918d00891649b"
291
+ "digest": "sha256:9bd7442fd04389e0ad901e660aeb18abf14ba4a2f34c5668680da6d35be65be4"
292
292
  },
293
293
  {
294
294
  "op": "moe_shared_slot_down",
295
295
  "kernel": "moe_shared_slot_down_pair_combine_residual_f32.wgsl",
296
296
  "entry": "main",
297
- "digest": "sha256:750bce8b9e6af48729fcb9dc27569590663dc0f85e6625d8de7e3bb3ba3041f2"
297
+ "digest": "sha256:ce5918180213bd0923b90f579564f1f00b4c4f716ace51ad1200075a3b1a78a8"
298
298
  }
299
299
  ]
300
300
  },
@@ -432,7 +432,7 @@
432
432
  "constants": {
433
433
  "HAS_GATE": true
434
434
  },
435
- "digest": "sha256:c6e28a7e6b2a7cfa805db03c9079fbc90fc2717c621e388bfa55b75ce1737ac0"
435
+ "digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
436
436
  },
437
437
  {
438
438
  "op": "down_proj",
@@ -175,7 +175,7 @@
175
175
  "constants": {
176
176
  "HAS_GATE": true
177
177
  },
178
- "digest": "sha256:c6e28a7e6b2a7cfa805db03c9079fbc90fc2717c621e388bfa55b75ce1737ac0"
178
+ "digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
179
179
  },
180
180
  {
181
181
  "op": "down_proj",
@@ -273,13 +273,13 @@
273
273
  "op": "moe_shared_slot_gu",
274
274
  "kernel": "moe_shared_slot_gu_pair_f32.wgsl",
275
275
  "entry": "main",
276
- "digest": "sha256:1a28e076e93486589a3eace88c3cb230ff1ef4aa982fdd91dcc918d00891649b"
276
+ "digest": "sha256:9bd7442fd04389e0ad901e660aeb18abf14ba4a2f34c5668680da6d35be65be4"
277
277
  },
278
278
  {
279
279
  "op": "moe_shared_slot_down",
280
280
  "kernel": "moe_shared_slot_down_pair_combine_residual_f32.wgsl",
281
281
  "entry": "main",
282
- "digest": "sha256:750bce8b9e6af48729fcb9dc27569590663dc0f85e6625d8de7e3bb3ba3041f2"
282
+ "digest": "sha256:ce5918180213bd0923b90f579564f1f00b4c4f716ace51ad1200075a3b1a78a8"
283
283
  }
284
284
  ]
285
285
  },
@@ -412,7 +412,7 @@
412
412
  "constants": {
413
413
  "HAS_GATE": true
414
414
  },
415
- "digest": "sha256:c6e28a7e6b2a7cfa805db03c9079fbc90fc2717c621e388bfa55b75ce1737ac0"
415
+ "digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
416
416
  },
417
417
  {
418
418
  "op": "down_proj",
@@ -1,4 +1,5 @@
1
1
  export interface ActivationTraceLimits {
2
+ activitySelection?: import('./activity-trace.js').ActivitySelection; activityRoleMaxBytes?: Record<string, number>;
2
3
  activityMaxBytes?: number; activityMaxFrameValues?: number; activityMaxStateValues?: number;
3
4
  activityTokenWindow?: number; activitySummaryBlock?: number;
4
5
  weightBlockValues?: number; weightReadValues?: number; weightRowGroupSize?: number; weightRawRows?: number; weightMaxBytes?: number;
@@ -2,7 +2,29 @@ import type { ActivationTraceLimits } from './activation-trace.js';
2
2
  import type { TensorRepresentation } from './tensor.js';
3
3
  import type { CommandRecorder } from '../gpu/command-recorder.js';
4
4
 
5
+ /** Which captures are recorded. null in a field means no restriction. `tokens.to` is exclusive. */
6
+ export interface ActivitySelection {
7
+ roles: string[] | null; layers: number[] | null; tokens: { from: number; to: number | null } | null;
8
+ }
9
+ /** A request's narrowing of its policy: record less, or within smaller budgets. */
10
+ export interface ActivityCaptureRequest {
11
+ roles?: string[] | null; layers?: number[] | null; tokens?: { from: number; to: number | null } | null;
12
+ maxBytes?: number; roleMaxBytes?: Record<string, number>;
13
+ }
14
+ /** What happened to one capture kind over a run. Counts are capture attempts. */
15
+ export interface ActivityCoverage {
16
+ captured: number; capturedBytes: number;
17
+ /** Attempts served by an earlier capture of a fixed per-layer value. */
18
+ reused: number;
19
+ dropped: number; failed: number;
20
+ /** Attempts the selection left out, by the first rule that excluded them. */
21
+ notSelected: { role: number; layer: number; token: number };
22
+ firstDroppedToken: number | null;
23
+ /** This kind's own budget, or null when only the recording budget applies. */
24
+ maxBytes: number | null;
25
+ }
5
26
  export interface ActivityLimits extends ActivationTraceLimits {
27
+ activitySelection: ActivitySelection; activityRoleMaxBytes: Record<string, number>;
6
28
  activitySummaryBlock: number; activityTokenWindow: number; activityMaxBytes: number; activityMaxFrameValues: number; activityMaxStateValues: number;
7
29
  weightBlockValues: number; weightReadValues: number; weightRowGroupSize: number; weightRawRows: number; weightMaxBytes: number;
8
30
  }
@@ -26,6 +48,7 @@ export interface ActivityWeight {
26
48
  export interface ActivityReceipt {
27
49
  schema: 'doppler.model-activity/v1'; frames: ActivityFrame[]; weights: ActivityWeight[];
28
50
  recordedBytes: number; limits: ActivityLimits;
51
+ selection: ActivitySelection; coverage: Record<string, ActivityCoverage>;
29
52
  unavailable: Record<string, string>; model: Record<string, unknown>;
30
53
  }
31
54
  export interface ActivityCaptureOptions extends TensorRepresentation {
@@ -50,3 +73,4 @@ export interface ActivityTrace {
50
73
  logits(logits: Float32Array): Promise<void>;
51
74
  }
52
75
  export function createActivityTrace(limits: ActivityLimits, started: number): ActivityTrace;
76
+ export function resolveActivityCapture<T extends ActivationTraceLimits>(limits: T, request: ActivityCaptureRequest | null | undefined): T;
@@ -1,6 +1,81 @@
1
1
  import { decodeSnapshotData, snapshotElementBytes } from './tensor.js';
2
2
  import stages from '../config/inspection/activity-stages.json' with { type: 'json' };
3
3
 
4
+ const ROLES = new Set(stages.roles);
5
+ // Fixed per-layer parameters: one capture per layer serves every token.
6
+ const CONSTANT_ROLES = new Set(stages.constantRoles);
7
+ // Later tokens read these from every earlier token, so a token range keeps its prefix.
8
+ const PREFIX_ROLES = new Set(stages.prefixRoles);
9
+
10
+ /** @param {string} label @param {unknown} value @param {(item: unknown) => boolean} valid @returns {asserts value is unknown[] | null} */
11
+ function checkList(label, value, valid) {
12
+ if (value === null) return;
13
+ if (!Array.isArray(value) || new Set(value).size !== value.length || !value.every(valid)) {
14
+ throw new Error(`Activity capture ${label} must be null (no restriction) or a list of distinct valid entries; an empty list records none.`);
15
+ }
16
+ }
17
+
18
+ /** @param {unknown} selection @returns {asserts selection is import('./activity-trace.js').ActivitySelection} */
19
+ function checkSelection(selection) {
20
+ const value = /** @type {Record<string, unknown> | null} */ (selection);
21
+ if (!value || typeof value !== 'object' || ['roles', 'layers', 'tokens'].some(key => value[key] === undefined)) {
22
+ throw new Error('Activity capture requires activitySelection with explicit roles, layers and tokens (null for no restriction).');
23
+ }
24
+ checkList('roles', value.roles, role => ROLES.has(/** @type {string} */ (role)));
25
+ checkList('layers', value.layers, layer => Number.isSafeInteger(layer) && Number(layer) >= 0);
26
+ const tokens = /** @type {{ from?: unknown; to?: unknown } | null} */ (value.tokens);
27
+ if (tokens !== null && (!Number.isSafeInteger(tokens?.from) || Number(tokens.from) < 0
28
+ || (tokens.to !== null && (!Number.isSafeInteger(tokens.to) || Number(tokens.to) <= Number(tokens.from))))) {
29
+ throw new Error('Activity capture tokens must be null or { from, to } with to after from (to: null for no end).');
30
+ }
31
+ }
32
+
33
+ /** @param {unknown} budgets @returns {asserts budgets is Record<string, number>} */
34
+ function checkRoleBudgets(budgets) {
35
+ if (!budgets || typeof budgets !== 'object' || Array.isArray(budgets)) throw new Error('Activity capture requires activityRoleMaxBytes (an empty object for no per-kind budgets).');
36
+ for (const [role, bytes] of Object.entries(budgets)) {
37
+ if (!ROLES.has(role) || !Number.isSafeInteger(bytes) || bytes < 1) throw new Error(`Activity capture budget for "${role}" must name a capture kind and a positive byte count.`);
38
+ }
39
+ }
40
+
41
+ // A request records less than its policy allows, or within smaller budgets;
42
+ // it never widens the policy. Anything outside the policy fails here, before a run.
43
+ /** @type {typeof import('./activity-trace.js').resolveActivityCapture} */
44
+ export function resolveActivityCapture(limits, request) {
45
+ if (request === undefined || request === null) return limits;
46
+ if (!limits?.activityMaxBytes) throw new Error('A capture selection requires an observation policy that records model activity.');
47
+ if (typeof request !== 'object' || Array.isArray(request)) throw new Error('A capture selection must be an object.');
48
+ const unknown = Object.keys(request).find(key => !['roles', 'layers', 'tokens', 'maxBytes', 'roleMaxBytes'].includes(key));
49
+ if (unknown) throw new Error(`Unknown capture selection field "${unknown}".`);
50
+ const policy = limits.activitySelection;
51
+ checkSelection(policy);
52
+ const asked = { roles: request.roles ?? null, layers: request.layers ?? null, tokens: request.tokens ?? null };
53
+ checkSelection(asked);
54
+ for (const key of /** @type {const} */ (['roles', 'layers'])) {
55
+ const allowed = /** @type {unknown[] | null} */ (policy[key]);
56
+ const outside = allowed ? (asked[key] ?? []).filter(item => !allowed.includes(item)) : [];
57
+ if (outside.length) throw new Error(`This observation policy does not record ${key === 'roles' ? 'the kind' : 'layer'} "${outside[0]}".`);
58
+ }
59
+ if (policy.tokens && asked.tokens && (asked.tokens.from < policy.tokens.from
60
+ || (policy.tokens.to !== null && (asked.tokens.to === null || asked.tokens.to > policy.tokens.to)))) {
61
+ throw new Error('The requested token range reaches outside the range this observation policy records.');
62
+ }
63
+ const maxBytes = request.maxBytes ?? limits.activityMaxBytes;
64
+ if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > limits.activityMaxBytes) {
65
+ throw new Error(`A capture budget must be between 1 and the policy's ${limits.activityMaxBytes} bytes.`);
66
+ }
67
+ const roleMaxBytes = request.roleMaxBytes ?? {};
68
+ checkRoleBudgets(roleMaxBytes);
69
+ checkRoleBudgets(limits.activityRoleMaxBytes);
70
+ for (const [role, bytes] of Object.entries(roleMaxBytes)) {
71
+ const ceiling = Math.min(limits.activityRoleMaxBytes[role] ?? maxBytes, maxBytes);
72
+ if (bytes > ceiling) throw new Error(`The budget for "${role}" exceeds the ${ceiling} bytes this policy and request allow.`);
73
+ }
74
+ return { ...limits, activityMaxBytes: maxBytes,
75
+ activityRoleMaxBytes: { ...limits.activityRoleMaxBytes, ...roleMaxBytes },
76
+ activitySelection: { roles: asked.roles ?? policy.roles, layers: asked.layers ?? policy.layers, tokens: asked.tokens ?? policy.tokens } };
77
+ }
78
+
4
79
  /** @type {typeof import('./activity-trace.js').createActivityTrace} */
5
80
  export function createActivityTrace(limits, started) {
6
81
  /** @type {Array<'activityMaxBytes' | 'activityMaxFrameValues' | 'activityMaxStateValues' | 'activityTokenWindow' | 'activitySummaryBlock'>} */
@@ -8,6 +83,16 @@ export function createActivityTrace(limits, started) {
8
83
  for (const name of required) {
9
84
  if (!Number.isSafeInteger(limits[name]) || limits[name] < 1) throw new Error(`Activity capture requires ${name}.`);
10
85
  }
86
+ checkSelection(limits.activitySelection);
87
+ checkRoleBudgets(limits.activityRoleMaxBytes);
88
+ const selection = limits.activitySelection;
89
+ const selectedRoles = selection.roles && new Set(selection.roles), selectedLayers = selection.layers && new Set(selection.layers);
90
+ /** @type {Record<string, import('./activity-trace.js').ActivityCoverage>} */
91
+ const coverage = {};
92
+ const cover = (/** @type {string} */ role) => coverage[role] ??= { captured: 0, capturedBytes: 0, reused: 0, dropped: 0, failed: 0,
93
+ notSelected: { role: 0, layer: 0, token: 0 }, firstDroppedToken: null, maxBytes: limits.activityRoleMaxBytes[role] ?? null };
94
+ /** @type {Set<string>} */
95
+ const constants = new Set();
11
96
  /** @type {import('./activity-trace.js').ActivityFrame[]} */
12
97
  const frames = [];
13
98
  /** @type {import('./activity-trace.js').ActivityWeight[]} */
@@ -34,11 +119,25 @@ export function createActivityTrace(limits, started) {
34
119
  async capture(role, buffer, options) {
35
120
  if (role === 'ffn-down-input' && !weights.some(w => w.layer === options.layer && w.columnEnergy.length)) return;
36
121
  if (!tokenIds.length) return;
37
- if (frames.length >= limits.maxFrames) { unavailable['capture-limit'] = 'Activity frame limit reached; later boundaries are unrecorded.'; return; }
122
+ const seen = cover(role), constant = CONSTANT_ROLES.has(role), constantKey = `${role}/${options.layer}`;
123
+ if (constants.has(constantKey)) { seen.reused++; return; }
124
+ // What the selection leaves out is counted, not stored: it takes no frame and no budget.
125
+ if (selectedRoles && !selectedRoles.has(role)) { seen.notSelected.role++; return; }
126
+ if (selectedLayers && options.layer >= 0 && !selectedLayers.has(options.layer)) { seen.notSelected.layer++; return; }
38
127
  const originalShape = [...options.shape];
39
128
  let ids = [...(options.tokenIds ?? tokenIds)], start = options.tokenStart ?? tokenStart;
40
129
  let offset = options.offset ?? 0;
41
130
  const fullShape = [...originalShape];
131
+ if (selection.tokens && !constant) {
132
+ const first = Math.max(start, PREFIX_ROLES.has(role) ? 0 : selection.tokens.from);
133
+ const end = Math.min(start + ids.length, selection.tokens.to ?? Infinity);
134
+ if (first >= end) { seen.notSelected.token++; return; }
135
+ // One row per token: keep only the selected rows. Other shapes are kept whole.
136
+ if (!options.state && fullShape.length === 2 && fullShape[0] === ids.length && end - first < ids.length) {
137
+ offset += (first - start) * fullShape[1]; ids = ids.slice(first - start, end - start); start = first; fullShape[0] = ids.length;
138
+ }
139
+ }
140
+ if (frames.length >= limits.maxFrames) { unavailable['capture-limit'] = 'Activity frame limit reached; later boundaries are unrecorded.'; return; }
42
141
  if (role.startsWith('ffn-') && fullShape.length === 2 && fullShape[0] === ids.length && ids.length > limits.activityTokenWindow) {
43
142
  const skipped = ids.length - limits.activityTokenWindow;
44
143
  offset += skipped * fullShape[1]; start += skipped; ids = ids.slice(skipped); fullShape[0] = ids.length;
@@ -56,10 +155,14 @@ export function createActivityTrace(limits, started) {
56
155
  layer: options.layer, tokenStart: start,
57
156
  tokenIds: ids, shape: [count], fullShape, offset: 0,
58
157
  dtype: options.dtype, values: null, status: 'captured', reason: null,
59
- representation: options.representation ?? 'raw', metadata: { originalShape, precision: options.precision ?? null, ...options.metadata } };
158
+ representation: options.representation ?? 'raw', metadata: { originalShape, precision: options.precision ?? null, ...options.metadata, ...(constant ? { constant: true } : {}) } };
60
159
  frames.push(frame);
61
- if (bytes + payloadBytes > limits.activityMaxBytes) {
62
- frame.status = 'dropped'; frame.reason = 'Activity recording budget reached; inference continues.';
160
+ const overRole = seen.maxBytes !== null && seen.capturedBytes + payloadBytes > seen.maxBytes;
161
+ if (overRole || bytes + payloadBytes > limits.activityMaxBytes) {
162
+ frame.status = 'dropped';
163
+ frame.reason = overRole ? `Activity budget for ${role} reached; inference continues.` : 'Activity recording budget reached; inference continues.';
164
+ frame.metadata.limit = overRole ? 'activityRoleMaxBytes' : 'activityMaxBytes';
165
+ seen.dropped++; seen.firstDroppedToken ??= start;
63
166
  return;
64
167
  }
65
168
  try {
@@ -94,9 +197,10 @@ export function createActivityTrace(limits, started) {
94
197
  }
95
198
  frame.summary = { blockSize: limits.activitySummaryBlock, observedValues: values.length, values: summary };
96
199
  }
97
- bytes += payloadBytes;
200
+ bytes += payloadBytes; seen.captured++; seen.capturedBytes += payloadBytes;
201
+ if (constant) constants.add(constantKey);
98
202
  } catch (error) {
99
- frame.status = 'failed'; frame.reason = String(error);
203
+ frame.status = 'failed'; frame.reason = String(error); seen.failed++;
100
204
  }
101
205
  },
102
206
  weightAccess(layer, stage) {
@@ -116,7 +220,8 @@ export function createActivityTrace(limits, started) {
116
220
  setWeights(value) { weights = value; },
117
221
  snapshot(includeFrames = true) {
118
222
  return { schema: 'doppler.model-activity/v1', frames: includeFrames ? [...frames] : [],
119
- weights, recordedBytes: bytes, limits: { ...limits }, unavailable: { ...unavailable }, model };
223
+ weights, recordedBytes: bytes, limits: { ...limits }, selection: structuredClone(selection), coverage: structuredClone(coverage),
224
+ unavailable: { ...unavailable }, model };
120
225
  },
121
226
  async logits(values) {
122
227
  await api.capture('logits', values, { layer: -1, dtype: 'f32', shape: [1, values.length],
@@ -17,24 +17,24 @@ fn clamp_product(value: f32) -> f32 {
17
17
  return value;
18
18
  }
19
19
  fn sigmoid(x: f32) -> f32 {
20
- let clamped = clamp(x, -15.0, 15.0);
21
- let z = exp_refined(-abs(clamped));
22
- let numerator = select(z, 1.0, clamped >= 0.0);
20
+ // The exponent is nonpositive, so overflow protection needs no input clamp.
21
+ let z = exp_refined(-abs(x));
22
+ let numerator = select(z, 1.0, x >= 0.0);
23
23
  let denominator = 1.0 + z;
24
24
  let inverse = reciprocal_refined(denominator);
25
25
  let quotient = numerator * inverse;
26
26
  return fma(fma(-quotient, denominator, numerator), inverse, quotient);
27
27
  }
28
28
 
29
- // Compensate the rounded numerator and quotient; retain the established gate clamp.
29
+ // Compensate the rounded numerator and quotient without changing the activation.
30
30
  fn silu(x: f32) -> f32 {
31
- let z = exp_refined(-abs(clamp(x, -15.0, 15.0)));
32
- let numerator = select(x * z, x, x >= 0.0);
33
- let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
34
- let den = 1.0 + z;
35
- let inverse = reciprocal_refined(den);
36
- let q = numerator * inverse;
37
- return fma(fma(-q, den, numerator) + error, inverse, q);
31
+ let z = exp_refined(-abs(x));
32
+ let numerator = select(x * z, x, x >= 0.0);
33
+ let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
34
+ let den = 1.0 + z;
35
+ let inverse = reciprocal_refined(den);
36
+ let q = numerator * inverse;
37
+ return fma(fma(-q, den, numerator) + error, inverse, q);
38
38
  }
39
39
 
40
40
  fn exp_refined(x: f32) -> f32 {
@@ -152,24 +152,24 @@ fn clamp_product(value: f32) -> f32 {
152
152
  return value;
153
153
  }
154
154
  fn sigmoid(x: f32) -> f32 {
155
- let clamped = clamp(x, -15.0, 15.0);
156
- let z = exp_refined(-abs(clamped));
157
- let numerator = select(z, 1.0, clamped >= 0.0);
155
+ // The exponent is nonpositive, so overflow protection needs no input clamp.
156
+ let z = exp_refined(-abs(x));
157
+ let numerator = select(z, 1.0, x >= 0.0);
158
158
  let denominator = 1.0 + z;
159
159
  let inverse = reciprocal_refined(denominator);
160
160
  let quotient = numerator * inverse;
161
161
  return fma(fma(-quotient, denominator, numerator), inverse, quotient);
162
162
  }
163
163
 
164
- // Compensate the rounded numerator and quotient; retain the established gate clamp.
164
+ // Compensate the rounded numerator and quotient without changing the activation.
165
165
  fn silu(x: f32) -> f32 {
166
- let z = exp_refined(-abs(clamp(x, -15.0, 15.0)));
167
- let numerator = select(x * z, x, x >= 0.0);
168
- let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
169
- let den = 1.0 + z;
170
- let inverse = reciprocal_refined(den);
171
- let q = numerator * inverse;
172
- return fma(fma(-q, den, numerator) + error, inverse, q);
166
+ let z = exp_refined(-abs(x));
167
+ let numerator = select(x * z, x, x >= 0.0);
168
+ let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
169
+ let den = 1.0 + z;
170
+ let inverse = reciprocal_refined(den);
171
+ let q = numerator * inverse;
172
+ return fma(fma(-q, den, numerator) + error, inverse, q);
173
173
  }
174
174
 
175
175
  fn exp_refined(x: f32) -> f32 {
@@ -152,24 +152,24 @@ fn clamp_product(value: f32) -> f32 {
152
152
  return value;
153
153
  }
154
154
  fn sigmoid(x: f32) -> f32 {
155
- let clamped = clamp(x, -15.0, 15.0);
156
- let z = exp_refined(-abs(clamped));
157
- let numerator = select(z, 1.0, clamped >= 0.0);
155
+ // The exponent is nonpositive, so overflow protection needs no input clamp.
156
+ let z = exp_refined(-abs(x));
157
+ let numerator = select(z, 1.0, x >= 0.0);
158
158
  let denominator = 1.0 + z;
159
159
  let inverse = reciprocal_refined(denominator);
160
160
  let quotient = numerator * inverse;
161
161
  return fma(fma(-quotient, denominator, numerator), inverse, quotient);
162
162
  }
163
163
 
164
- // Compensate the rounded numerator and quotient; retain the established gate clamp.
164
+ // Compensate the rounded numerator and quotient without changing the activation.
165
165
  fn silu(x: f32) -> f32 {
166
- let z = exp_refined(-abs(clamp(x, -15.0, 15.0)));
167
- let numerator = select(x * z, x, x >= 0.0);
168
- let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
169
- let den = 1.0 + z;
170
- let inverse = reciprocal_refined(den);
171
- let q = numerator * inverse;
172
- return fma(fma(-q, den, numerator) + error, inverse, q);
166
+ let z = exp_refined(-abs(x));
167
+ let numerator = select(x * z, x, x >= 0.0);
168
+ let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
169
+ let den = 1.0 + z;
170
+ let inverse = reciprocal_refined(den);
171
+ let q = numerator * inverse;
172
+ return fma(fma(-q, den, numerator) + error, inverse, q);
173
173
  }
174
174
 
175
175
  fn exp_refined(x: f32) -> f32 {
@@ -158,24 +158,24 @@ fn clamp_product(value: f32) -> f32 {
158
158
  return value;
159
159
  }
160
160
  fn sigmoid(x: f32) -> f32 {
161
- let clamped = clamp(x, -15.0, 15.0);
162
- let z = exp_refined(-abs(clamped));
163
- let numerator = select(z, 1.0, clamped >= 0.0);
161
+ // The exponent is nonpositive, so overflow protection needs no input clamp.
162
+ let z = exp_refined(-abs(x));
163
+ let numerator = select(z, 1.0, x >= 0.0);
164
164
  let denominator = 1.0 + z;
165
165
  let inverse = reciprocal_refined(denominator);
166
166
  let quotient = numerator * inverse;
167
167
  return fma(fma(-quotient, denominator, numerator), inverse, quotient);
168
168
  }
169
169
 
170
- // Compensate the rounded numerator and quotient; retain the established gate clamp.
170
+ // Compensate the rounded numerator and quotient without changing the activation.
171
171
  fn silu(x: f32) -> f32 {
172
- let z = exp_refined(-abs(clamp(x, -15.0, 15.0)));
173
- let numerator = select(x * z, x, x >= 0.0);
174
- let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
175
- let den = 1.0 + z;
176
- let inverse = reciprocal_refined(den);
177
- let q = numerator * inverse;
178
- return fma(fma(-q, den, numerator) + error, inverse, q);
172
+ let z = exp_refined(-abs(x));
173
+ let numerator = select(x * z, x, x >= 0.0);
174
+ let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
175
+ let den = 1.0 + z;
176
+ let inverse = reciprocal_refined(den);
177
+ let q = numerator * inverse;
178
+ return fma(fma(-q, den, numerator) + error, inverse, q);
179
179
  }
180
180
 
181
181
  fn exp_refined(x: f32) -> f32 {
@@ -156,24 +156,24 @@ fn clamp_product(value: f32) -> f32 {
156
156
  return value;
157
157
  }
158
158
  fn sigmoid(x: f32) -> f32 {
159
- let clamped = clamp(x, -15.0, 15.0);
160
- let z = exp_refined(-abs(clamped));
161
- let numerator = select(z, 1.0, clamped >= 0.0);
159
+ // The exponent is nonpositive, so overflow protection needs no input clamp.
160
+ let z = exp_refined(-abs(x));
161
+ let numerator = select(z, 1.0, x >= 0.0);
162
162
  let denominator = 1.0 + z;
163
163
  let inverse = reciprocal_refined(denominator);
164
164
  let quotient = numerator * inverse;
165
165
  return fma(fma(-quotient, denominator, numerator), inverse, quotient);
166
166
  }
167
167
 
168
- // Compensate the rounded numerator and quotient; retain the established gate clamp.
168
+ // Compensate the rounded numerator and quotient without changing the activation.
169
169
  fn silu(x: f32) -> f32 {
170
- let z = exp_refined(-abs(clamp(x, -15.0, 15.0)));
171
- let numerator = select(x * z, x, x >= 0.0);
172
- let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
173
- let den = 1.0 + z;
174
- let inverse = reciprocal_refined(den);
175
- let q = numerator * inverse;
176
- return fma(fma(-q, den, numerator) + error, inverse, q);
170
+ let z = exp_refined(-abs(x));
171
+ let numerator = select(x * z, x, x >= 0.0);
172
+ let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
173
+ let den = 1.0 + z;
174
+ let inverse = reciprocal_refined(den);
175
+ let q = numerator * inverse;
176
+ return fma(fma(-q, den, numerator) + error, inverse, q);
177
177
  }
178
178
 
179
179
  fn exp_refined(x: f32) -> f32 {
@@ -158,24 +158,24 @@ fn clamp_product(value: f32) -> f32 {
158
158
  return value;
159
159
  }
160
160
  fn sigmoid(x: f32) -> f32 {
161
- let clamped = clamp(x, -15.0, 15.0);
162
- let z = exp_refined(-abs(clamped));
163
- let numerator = select(z, 1.0, clamped >= 0.0);
161
+ // The exponent is nonpositive, so overflow protection needs no input clamp.
162
+ let z = exp_refined(-abs(x));
163
+ let numerator = select(z, 1.0, x >= 0.0);
164
164
  let denominator = 1.0 + z;
165
165
  let inverse = reciprocal_refined(denominator);
166
166
  let quotient = numerator * inverse;
167
167
  return fma(fma(-quotient, denominator, numerator), inverse, quotient);
168
168
  }
169
169
 
170
- // Compensate the rounded numerator and quotient; retain the established gate clamp.
170
+ // Compensate the rounded numerator and quotient without changing the activation.
171
171
  fn silu(x: f32) -> f32 {
172
- let z = exp_refined(-abs(clamp(x, -15.0, 15.0)));
173
- let numerator = select(x * z, x, x >= 0.0);
174
- let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
175
- let den = 1.0 + z;
176
- let inverse = reciprocal_refined(den);
177
- let q = numerator * inverse;
178
- return fma(fma(-q, den, numerator) + error, inverse, q);
172
+ let z = exp_refined(-abs(x));
173
+ let numerator = select(x * z, x, x >= 0.0);
174
+ let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
175
+ let den = 1.0 + z;
176
+ let inverse = reciprocal_refined(den);
177
+ let q = numerator * inverse;
178
+ return fma(fma(-q, den, numerator) + error, inverse, q);
179
179
  }
180
180
 
181
181
  fn exp_refined(x: f32) -> f32 {
@@ -17,9 +17,9 @@ struct Uniforms {
17
17
  @group(0) @binding(4) var<storage, read_write> output: array<f32>;
18
18
 
19
19
  fn sigmoid(x: f32) -> f32 {
20
- let clamped = clamp(x, -15.0, 15.0);
21
- let z = exp_refined(-abs(clamped));
22
- let numerator = select(z, 1.0, clamped >= 0.0);
20
+ // The exponent is nonpositive, so overflow protection needs no input clamp.
21
+ let z = exp_refined(-abs(x));
22
+ let numerator = select(z, 1.0, x >= 0.0);
23
23
  let denominator = 1.0 + z;
24
24
  let inverse = reciprocal_refined(denominator);
25
25
  let quotient = numerator * inverse;
@@ -31,6 +31,7 @@ export const PIPELINE_OPERATIONS = Object.freeze({
31
31
  advanceWithToken: 'execution', advanceWithTokenAndEmbedding: 'execution',
32
32
  prefillKVOnly: 'execution', prefillForLoRATraining: 'execution',
33
33
  computeDiffusionGemmaCanvasLogits: 'execution', computeDiffusionGemmaCanvasStep: 'execution',
34
+ projectHiddenStates: 'execution',
34
35
  prefillWithEmbedding: 'execution', prefillWithLogits: 'execution',
35
36
  prefillWithTokenLogits: 'execution', prefillWithTokenLogitsFromKV: 'execution',
36
37
  embed: 'execution', embedBatch: Object.freeze({ kind: 'execution', context: 'explicit' }), encodeSequence: 'execution',
@@ -1,5 +1,5 @@
1
1
  import { prepareActivityRecording } from '../activity-weights.js';
2
- import { beginGenerationContext, finishGenerationContext, generationContextInput } from '../sequence-state.js';
2
+ import { beginGenerationContext, beginSequenceMemoryBoundary, finishGenerationContext, generationContextInput } from '../sequence-state.js';
3
3
  import { assertHyperConnectionOperation } from '../hyper-connection/sequence-transaction.js';
4
4
  import { lookupDraft, verifySpeculativeTokens } from './speculative-runtime.js';
5
5
  import { markDiagnosticSequenceStep } from './diagnostic-sequence.js';
@@ -211,6 +211,7 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
211
211
  try {
212
212
  beginGenerationContext(this._state, options.generationContext, opts);
213
213
  await prepareActivityRecording(this._state, opts);
214
+ if (this._state.currentSeqLen === 0) await beginSequenceMemoryBoundary();
214
215
  const prefillStartSeqLen = this._state.currentSeqLen;
215
216
  const prefillStart = performance.now();
216
217
  opts._whileFinalPrefillRuns = wholeTokenAheadVerification(this, opts);
@@ -345,6 +346,7 @@ export async function generateTokenIds(prompt, options = {}) {
345
346
  try {
346
347
  beginGenerationContext(this._state, options.generationContext, opts);
347
348
  await prepareActivityRecording(this._state, opts);
349
+ if (this._state.currentSeqLen === 0) await beginSequenceMemoryBoundary();
348
350
  const prefillStartSeqLen = this._state.currentSeqLen;
349
351
  const prefillStart = performance.now();
350
352
  const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
@@ -0,0 +1,18 @@
1
+ import type { PipelineState } from '../state.js';
2
+
3
+ export interface HiddenProjectionResult {
4
+ /** Row-major logits: `rowCount` rows of `vocabSize` values. */
5
+ logits: Float32Array;
6
+ rowCount: number;
7
+ vocabSize: number;
8
+ }
9
+
10
+ /**
11
+ * Applies the model's final normalization and output head to caller-supplied
12
+ * hidden states. Fails for hyper-connection models and paired-precision plans.
13
+ */
14
+ export declare function projectHiddenStates(
15
+ state: PipelineState,
16
+ hiddenStates: Float32Array,
17
+ rowCount: number
18
+ ): Promise<HiddenProjectionResult>;
@@ -0,0 +1,49 @@
1
+ import { computeLogits } from '../logits/index.js';
2
+ import { getLogitsConfig, getLogitsWeights } from './logits-config.js';
3
+
4
+ // Projects caller-supplied hidden states through this model's own final
5
+ // normalization and output head: the computation every decode step ends with,
6
+ // applied to vectors that did not come from the last layer. The result is an
7
+ // observation aid (a "logit lens"), not something the model produced at that
8
+ // layer. Nothing here touches sequence state or the KV cache.
9
+ /** @type {typeof import('./hidden-projection.js').projectHiddenStates} */
10
+ export const projectHiddenStates = async function projectHiddenStates(state, hiddenStates, rowCount) {
11
+ if (!(hiddenStates instanceof Float32Array)) {
12
+ throw new Error('projectHiddenStates requires hidden states as a Float32Array.');
13
+ }
14
+ if (!Number.isInteger(rowCount) || rowCount < 1) {
15
+ throw new Error('projectHiddenStates requires a positive whole number of rows.');
16
+ }
17
+ const config = getLogitsConfig(state);
18
+ if (config.hyperConnections) {
19
+ // Their trace rows hold several streams and a mixer replaces the final norm.
20
+ throw new Error('projectHiddenStates does not support hyper-connection models: a recorded layer output there is not one hidden state.');
21
+ }
22
+ if (config.activationPrecision !== undefined && config.activationPrecision !== null) {
23
+ throw new Error(`projectHiddenStates requires single-component activations; the active plan uses "${config.activationPrecision}".`);
24
+ }
25
+ if (hiddenStates.length !== rowCount * config.hiddenSize) {
26
+ throw new Error(`projectHiddenStates expected ${rowCount} rows of ${config.hiddenSize} values (${rowCount * config.hiddenSize}), got ${hiddenStates.length}.`);
27
+ }
28
+ for (let index = 0; index < hiddenStates.length; index += 1) {
29
+ if (!Number.isFinite(hiddenStates[index])) {
30
+ throw new Error(`projectHiddenStates received a non-finite value at index ${index}.`);
31
+ }
32
+ }
33
+ const logits = await computeLogits(
34
+ hiddenStates,
35
+ rowCount,
36
+ getLogitsWeights(state),
37
+ config,
38
+ state.useGPU,
39
+ state.debugFlags,
40
+ undefined,
41
+ undefined,
42
+ state.runtimeConfig.shared.debug.probes,
43
+ { lastPositionOnly: false }
44
+ );
45
+ if (!(logits instanceof Float32Array) || logits.length % rowCount !== 0) {
46
+ throw new Error('projectHiddenStates did not receive one logit row per hidden state.');
47
+ }
48
+ return { logits, rowCount, vocabSize: logits.length / rowCount };
49
+ };
@@ -8,6 +8,7 @@ import { releaseBuffer } from '../../../memory/buffer-pool.js';
8
8
  import { sample, applyRepetitionPenalty, applyPresencePenalty } from './sampling.js';
9
9
  import { isStopToken } from './init.js';
10
10
  import { decodeStepLogits } from './generator-steps.js';
11
+ import { projectHiddenStates as projectHiddenStatesImpl } from './generator/hidden-projection.js';
11
12
  import { buildLayerContext, debugCheckBuffer as debugCheckBufferHelper } from './generator/session-context.js';
12
13
  import { getLogitsConfig, getLogitsWeights } from './generator/logits-config.js';
13
14
  import { releaseSharedAttentionState } from './generator/attention-lifecycle.js';
@@ -408,6 +409,10 @@ export class PipelineGenerator {
408
409
  return computeDiffusionGemmaCanvasLogitsImpl.apply(this, arguments);
409
410
  }
410
411
 
412
+ async projectHiddenStates(hiddenStates, rowCount) {
413
+ return projectHiddenStatesImpl(this._state, hiddenStates, rowCount);
414
+ }
415
+
411
416
  async computeDiffusionGemmaCanvasStep(args, options = {}) {
412
417
  return computeDiffusionGemmaCanvasStepImpl.apply(this, arguments);
413
418
  }
@@ -2,6 +2,8 @@ import { createCommandRecorder } from '../../../gpu/command-recorder.js';
2
2
  import { beginImmediateDispatchProfile } from '../../../gpu/kernels/dispatch.js';
3
3
  import { log } from '../../../debug/index.js';
4
4
  import { acquireBuffer, releaseBuffer, discardBuffer, readBuffer, uploadData } from '../../../memory/buffer-pool.js';
5
+ import { retireIdleBuffersUnderDeviceBudget } from '../../../memory/sequence-retirement.js';
6
+ import { getDevice } from '../../../gpu/device.js';
5
7
  import { getWeightDtype, getWeightMetadata, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../gpu/weight-buffer.js';
6
8
  import { selectRuleValue } from '../../../rules/rule-registry.js';
7
9
  import { embed } from './embed.js';
@@ -121,6 +123,9 @@ export async function executePartitionLayers(state, input, signal) {
121
123
  signal.throwIfAborted();
122
124
  const prefill = state.currentSeqLen === 0;
123
125
  const setup = resolvePartitionInput(state, input, prefill);
126
+ // A sequence's first prefill starts from its owned state under a device budget.
127
+ if (prefill) await retireIdleBuffersUnderDeviceBudget(getDevice());
128
+ signal.throwIfAborted();
124
129
  if (setup.config.hyperConnections && !state.hyperConnectionSession) {
125
130
  if (!state.dopplerLoader || !state.kvCache) throw new Error('Resident hyper-connections require the owning loader and attempt cache.');
126
131
  state.hyperConnectionSession = await createHyperConnectionSession(setup.config, state.dopplerLoader,
@@ -114,6 +114,14 @@ export declare class InferencePipeline extends PipelineState {
114
114
  options?: GenerateOptions & { __internalGenerate?: boolean }
115
115
  ): Promise<Float32Array>;
116
116
 
117
+ /**
118
+ * Applies the final normalization and output head to caller-supplied hidden
119
+ * states (`rowCount` rows of `hiddenSize`). An observation aid: the logits
120
+ * are a projection, not what the model produced at an earlier layer. Does
121
+ * not read or change sequence state.
122
+ */
123
+ projectHiddenStates(hiddenStates: Float32Array, rowCount: number): Promise<import('./generator/hidden-projection.js').HiddenProjectionResult>;
124
+
117
125
  computeDiffusionGemmaCanvasStep(
118
126
  args: DiffusionGemmaCanvasStepInput,
119
127
  options?: GenerateOptions & { __internalGenerate?: boolean }
@@ -301,6 +301,10 @@ export class InferencePipeline extends PipelineState {
301
301
  return this.generator.computeDiffusionGemmaCanvasLogits(args, options);
302
302
  }
303
303
 
304
+ projectHiddenStates(hiddenStates, rowCount) {
305
+ return this.generator.projectHiddenStates(hiddenStates, rowCount);
306
+ }
307
+
304
308
  computeDiffusionGemmaCanvasStep(args, options = {}) {
305
309
  assertNotAborted(options?.signal);
306
310
  return this.generator.computeDiffusionGemmaCanvasStep(args, options);
@@ -45,6 +45,8 @@ export interface ContextLease {
45
45
  export function supportsGenerationContext(state: ContextPipelineState): boolean;
46
46
  export function createGenerationContext(state: ContextPipelineState, options: { maxBytes: number }): GenerationContext;
47
47
  export function closeGenerationContexts(state: ContextPipelineState): void;
48
+ /** Retire idle pooled memory before a new generation's first prefill when a device budget is set. */
49
+ export function beginSequenceMemoryBoundary(): Promise<boolean>;
48
50
  export function beginGenerationContext(state: ContextPipelineState, context: GenerationContext | undefined, options: ContextOptions): void;
49
51
  export function generationContextInput(state: ContextPipelineState, inputIds: number[]): number[];
50
52
  export function generationContextCheckpointLength(state: ContextPipelineState, prompt: import('./generator.js').PromptInput,
@@ -1,5 +1,6 @@
1
1
  import { getDevice } from '../../../gpu/device.js';
2
2
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
3
+ import { retireIdleBuffersUnderDeviceBudget } from '../../../memory/sequence-retirement.js';
3
4
  import { createKVCache } from './init.js';
4
5
  import { createLinearAttentionRuntime, resetLinearAttentionRuntime, resetLinearAttentionSequence } from './linear-attention.js';
5
6
  import { isConvLayerType } from './layer-input-execution.js';
@@ -397,6 +398,12 @@ export async function checkpointGenerationContext(state, tokens) {
397
398
  state.wholeTokenProgram?.program.invalidate('generation checkpoint parked');
398
399
  }
399
400
 
401
+ // A new generation's first prefill starts from its owned state under a device budget.
402
+ /** @type {import('./sequence-state.js').beginSequenceMemoryBoundary} */
403
+ export async function beginSequenceMemoryBoundary() {
404
+ return retireIdleBuffersUnderDeviceBudget(getDevice());
405
+ }
406
+
400
407
  /** @type {import('./sequence-state.js').prepareGenerationContextPrefill} */
401
408
  export async function prepareGenerationContextPrefill(generator, prompt, inputIds, opts) {
402
409
  const state = generator._state;
@@ -0,0 +1,19 @@
1
+ import { getBufferPool } from './buffer-pool.js';
2
+ import { getDeviceMemorySnapshot } from './device-budget.js';
3
+ import { isDeviceLost } from './device-residency.js';
4
+
5
+ // Under an explicit device memory budget, idle pooled buffers and pending
6
+ // retirements are charged as live allocations. A sequence boundary retires them
7
+ // after confirmed GPU completion, so one sequence's working set is bounded by its
8
+ // own allocations rather than by buckets left over from earlier prompt lengths.
9
+ // Without a device budget the pool keeps its reuse behavior. Owned (active)
10
+ // buffers are never touched.
11
+ /** @type {typeof import('./sequence-retirement.js').retireIdleBuffersUnderDeviceBudget} */
12
+ export async function retireIdleBuffersUnderDeviceBudget(device) {
13
+ if (!device || isDeviceLost(device)) return false;
14
+ if ((getDeviceMemorySnapshot(device)?.maxBytes ?? null) === null) return false;
15
+ const pool = getBufferPool(device);
16
+ pool.clearPool();
17
+ await pool.settleDeferredDestruction();
18
+ return true;
19
+ }
package/src/version.js CHANGED
@@ -1,2 +1,2 @@
1
- export const DOPPLER_VERSION = '0.7.0';
1
+ export const DOPPLER_VERSION = '0.7.2';
2
2
  export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;