doppler-gpu 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/README.md +4 -0
- package/package.json +2 -2
- package/src/client/inspection.d.ts +27 -0
- package/src/client/inspection.js +7 -1
- package/src/client/model-host/layer-projection.js +46 -0
- package/src/client/model-host/model-session.d.ts +18 -0
- package/src/client/model-host/model-session.js +7 -1
- package/src/config/inspection/activity-stages.json +29 -1
- package/src/config/inspection/observation-policies.json +14 -2
- package/src/config/kernels/registry.json +2 -2
- package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-candidate.json +4 -4
- package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-pooled-q8-common-diagnostic.json +4 -4
- package/src/debug/activation-trace.d.ts +1 -0
- package/src/debug/activity-trace.d.ts +24 -0
- package/src/debug/activity-trace.js +112 -7
- package/src/gpu/kernels/moe_shared_slot_combine_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_down_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_down_pair_combine_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_down_pair_combine_residual_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_gu_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_gu_pair_f32.wgsl +11 -11
- package/src/gpu/kernels/sigmoid_gated_add_f32.wgsl +3 -3
- package/src/inference/pipelines/shader-scoped-pipeline.js +1 -0
- package/src/inference/pipelines/text/generator/decode-runtime.js +3 -1
- package/src/inference/pipelines/text/generator/hidden-projection.d.ts +18 -0
- package/src/inference/pipelines/text/generator/hidden-projection.js +49 -0
- package/src/inference/pipelines/text/generator.js +5 -0
- package/src/inference/pipelines/text/partition-execution.js +5 -0
- package/src/inference/pipelines/text/pipeline.d.ts +8 -0
- package/src/inference/pipelines/text/pipeline.js +4 -0
- package/src/inference/pipelines/text/sequence-state.d.ts +2 -0
- package/src/inference/pipelines/text/sequence-state.js +7 -0
- package/src/memory/sequence-retirement.js +19 -0
- package/src/version.js +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,82 @@ Historical package notes are preserved outside the runtime archive in
|
|
|
4
4
|
`docs/status/archive/package-changelog-through-0.6.1.md` and the earlier
|
|
5
5
|
`package-changelog-through-0.4.15.md` snapshot in that directory.
|
|
6
6
|
|
|
7
|
+
## [0.7.2] - 2026-11-11
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `model.inspect.projectLayerOutputs({ vectors, topK })` projects recorded
|
|
12
|
+
hidden states through the loaded model's final normalization and output head.
|
|
13
|
+
It returns vocabulary-normalized probabilities explicitly labeled
|
|
14
|
+
`projected`. Intermediate projections are an inspection aid, not generated
|
|
15
|
+
tokens or evidence of causation. Hyper-connection and paired-precision plans
|
|
16
|
+
fail explicitly. Calls accept at most 64 vectors and 64 candidates per row.
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- Projection requests snapshot both vector values and row count before waiting
|
|
21
|
+
for execution. Mutating the caller's vector array cannot change an admitted
|
|
22
|
+
request's shape.
|
|
23
|
+
|
|
24
|
+
### Limits
|
|
25
|
+
|
|
26
|
+
- Projection reads back a full vocabulary row for each input vector; recording
|
|
27
|
+
and projection costs are separate from uninstrumented inference performance.
|
|
28
|
+
- Flash-Next's 2,187-token full-logit gate remains failing and unqualified.
|
|
29
|
+
This release does not fix or relax that gate.
|
|
30
|
+
|
|
31
|
+
### Comparison evidence
|
|
32
|
+
|
|
33
|
+
- The release report includes Qwen 3.5 0.8B and 2B on Apple M3 against
|
|
34
|
+
Transformers.js 4.3.2 / ONNX Runtime WebGPU and llama.cpp, plus Qwen3.8 27B
|
|
35
|
+
on Jupiter against direct ONNX Runtime WebGPU and llama.cpp Vulkan.
|
|
36
|
+
It retains exact prompts, artifact identities, outputs, load times and
|
|
37
|
+
complete-request timings. Host variability and output mismatches prevent a
|
|
38
|
+
general speed or quality claim; native llama.cpp remains ahead on the
|
|
39
|
+
retained matched-output workloads. See
|
|
40
|
+
[the release evidence](https://github.com/clocksmith/doppler/tree/main/reports/package-release-0.7.2).
|
|
41
|
+
|
|
42
|
+
## [0.7.1] - 2026-11-11
|
|
43
|
+
|
|
44
|
+
Patch release. Published bytes for 0.7.0 are unchanged.
|
|
45
|
+
|
|
46
|
+
### Fixed
|
|
47
|
+
|
|
48
|
+
- **Gate activations agree across fused and unfused paths.** 0.7.0 removed
|
|
49
|
+
the +/-15 sigmoid input clamp from `silu.wgsl`, but
|
|
50
|
+
`sigmoid_gated_add_f32.wgsl` and the `moe_shared_slot_*` kernels kept it.
|
|
51
|
+
For |gate| > 15 the separate and fused paths disagreed, and the Qwen
|
|
52
|
+
shared-expert gate differed from the logistic function: `sigmoid(-20)`
|
|
53
|
+
gave 3.06e-7 instead of 2.06e-9. All of them now use `silu.wgsl`'s
|
|
54
|
+
unclamped form; only the explicit `swigluLimit` output clamp remains. The
|
|
55
|
+
two Qwen 3.6 35B-A3B q4_0 runtime profiles carry the current kernel digests.
|
|
56
|
+
- **Repeated prompts no longer exhaust an explicit device memory budget.**
|
|
57
|
+
With `configureDeviceMemoryBudget`, idle pooled buffers count as live.
|
|
58
|
+
Each new prompt length added buckets on top of earlier ones until a later
|
|
59
|
+
request was rejected. A new generation's first prefill now retires idle
|
|
60
|
+
pooled buffers after confirmed GPU completion, only when a budget is set.
|
|
61
|
+
Owned buffers are never retired. Measured with Qwen 3.5 2B under a
|
|
62
|
+
6,442,450,944-byte budget: steady memory between requests stays at or
|
|
63
|
+
below 2.6 GB.
|
|
64
|
+
|
|
65
|
+
### Added
|
|
66
|
+
|
|
67
|
+
- `model.inspect.generate(prompt, { capture })` narrows activity recording by
|
|
68
|
+
kind, layer, token range and budget (selective capture, `e56cf5d6`).
|
|
69
|
+
|
|
70
|
+
### Known limits
|
|
71
|
+
|
|
72
|
+
- Budgeted hosts should also set
|
|
73
|
+
`inference.session.prefillChunkSubmitMode: "sync"` (or
|
|
74
|
+
`prefillMaxInFlightChunks: 1`) for models whose manifest declares async
|
|
75
|
+
prefill submits. With async submits, warm requests keep up to three layer
|
|
76
|
+
chunks of prompt-sized temporaries alive. Qwen 3.5 2B at 2,000 tokens
|
|
77
|
+
peaked at 4.40 GB with sync, against a failure under the same 6.44 GB
|
|
78
|
+
budget with async, at the same prefill time.
|
|
79
|
+
- Flash-Next (Qwen 3.8) is not qualified. Its 2,187-token prompt still fails
|
|
80
|
+
the full-logit gate against independent CPU references; see
|
|
81
|
+
`reports/flash-next-reference-controls-20261110/`.
|
|
82
|
+
|
|
7
83
|
## [0.7.0] - 2026-11-10
|
|
8
84
|
|
|
9
85
|
First npm release since 0.6.1. It also carries the 0.6.27 and 0.6.2 candidate
|
package/README.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# Doppler
|
|
2
2
|
|
|
3
|
+
<a href="assets/readme/doppler-demo-hero.mp4"><img src="assets/readme/doppler-demo-hero.gif" width="100%" alt="Doppler demo: a recorded Qwen 3.5 0.8B answer appears token by token beside per-layer activation changes, then one word's next-token alternatives open with their probabilities." /></a>
|
|
4
|
+
|
|
5
|
+
<sub>Recorded replay from the <a href="https://d4da.com/doppler/">demo</a>, not a live run. <a href="assets/readme/doppler-demo-hero.mp4">Full-quality video</a>.</sub>
|
|
6
|
+
|
|
3
7
|
A JavaScript library for local model execution through WebGPU: generation,
|
|
4
8
|
embeddings, reranking, and choice scoring, qualified per model and host.
|
|
5
9
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "doppler-gpu",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.2",
|
|
4
4
|
"description": "Doppler Rig prepares signed model Capsules; Doppler Run executes them locally through WebGPU for JavaScript",
|
|
5
5
|
"main": "src/capsule-runtime.js",
|
|
6
6
|
"types": "src/capsule-runtime.d.ts",
|
|
@@ -355,7 +355,7 @@
|
|
|
355
355
|
"tools/convert-safetensors-node.js"
|
|
356
356
|
],
|
|
357
357
|
"devDependencies": {
|
|
358
|
-
"@huggingface/transformers": "^4.
|
|
358
|
+
"@huggingface/transformers": "^4.3.2",
|
|
359
359
|
"@types/node": "^25.6.0",
|
|
360
360
|
"@webgpu/types": "^0.1.69",
|
|
361
361
|
"jest": "^30.2.0",
|
|
@@ -113,6 +113,33 @@ export declare function buildInspectionTokenRecords(
|
|
|
113
113
|
topKSize?: number
|
|
114
114
|
): DopplerInspectionTokenRecord[];
|
|
115
115
|
|
|
116
|
+
export declare const LAYER_PROJECTION_SCHEMA: 'doppler.layer-projection/v1';
|
|
117
|
+
export declare const LAYER_PROJECTION_MAX_ROWS: 64;
|
|
118
|
+
export declare const LAYER_PROJECTION_MAX_TOP_K: 64;
|
|
119
|
+
|
|
120
|
+
export interface DopplerLayerProjection {
|
|
121
|
+
schema: typeof LAYER_PROJECTION_SCHEMA;
|
|
122
|
+
modelId: string;
|
|
123
|
+
/** What the vectors were passed through. `kind` is always `projected`: these are not tokens the model produced. */
|
|
124
|
+
projection: { through: ['final-normalization', 'output-head']; kind: 'projected' };
|
|
125
|
+
vocabSize: number;
|
|
126
|
+
rows: Array<{ candidates: DopplerTokenCandidate[] }>;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export interface DopplerTokenCandidate {
|
|
130
|
+
tokenId: number;
|
|
131
|
+
logit: number;
|
|
132
|
+
text: string;
|
|
133
|
+
probability: number | null;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The `count` highest logits with their softmax probability over the whole row. */
|
|
137
|
+
export declare function topCandidates(
|
|
138
|
+
logits: ArrayLike<number> | null | undefined,
|
|
139
|
+
tokenizer: { decode(ids: number[], skipSpecialTokens?: boolean, trim?: boolean): string },
|
|
140
|
+
count: number
|
|
141
|
+
): DopplerTokenCandidate[];
|
|
142
|
+
|
|
116
143
|
export declare function buildInspectionTokenRecord(
|
|
117
144
|
tokenId: number,
|
|
118
145
|
logits: Float32Array | null,
|
package/src/client/inspection.js
CHANGED
|
@@ -215,7 +215,13 @@ function selectedProbability(logits, tokenId) {
|
|
|
215
215
|
return Math.exp(Number(logits[tokenId]) - maximum) / denominator;
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
-
|
|
218
|
+
export const LAYER_PROJECTION_SCHEMA = 'doppler.layer-projection/v1';
|
|
219
|
+
// One request returns a full vocabulary row per vector before the top candidates
|
|
220
|
+
// are kept, so both bounds limit what a single call can hold.
|
|
221
|
+
export const LAYER_PROJECTION_MAX_ROWS = 64;
|
|
222
|
+
export const LAYER_PROJECTION_MAX_TOP_K = 64;
|
|
223
|
+
|
|
224
|
+
export function topCandidates(logits, tokenizer, count) {
|
|
219
225
|
if (!logits || !Number.isInteger(count) || count <= 0) return [];
|
|
220
226
|
const candidates = [];
|
|
221
227
|
for (let tokenId = 0; tokenId < logits.length; tokenId += 1) {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { runPipelineOperation } from '../../inference/pipelines/shader-scoped-pipeline.js';
|
|
2
|
+
import {
|
|
3
|
+
LAYER_PROJECTION_MAX_ROWS,
|
|
4
|
+
LAYER_PROJECTION_MAX_TOP_K,
|
|
5
|
+
LAYER_PROJECTION_SCHEMA,
|
|
6
|
+
topCandidates,
|
|
7
|
+
} from '../inspection.js';
|
|
8
|
+
|
|
9
|
+
// Projects recorded layer outputs through the loaded model's final
|
|
10
|
+
// normalization and output head. The pipeline owns the computation; this
|
|
11
|
+
// validates the request, snapshots it and labels the result. The candidates are
|
|
12
|
+
// a projection of each vector: they are not tokens the model produced there.
|
|
13
|
+
/** @type {typeof import('./layer-projection.js').projectLayerOutputs} */
|
|
14
|
+
export const projectLayerOutputs = async function projectLayerOutputs(pipeline, modelId, request) {
|
|
15
|
+
const vectors = request?.vectors;
|
|
16
|
+
const topK = request?.topK;
|
|
17
|
+
if (!Array.isArray(vectors) || vectors.length === 0 || vectors.length > LAYER_PROJECTION_MAX_ROWS) {
|
|
18
|
+
throw new Error(`Doppler model.inspect.projectLayerOutputs requires 1 to ${LAYER_PROJECTION_MAX_ROWS} vectors.`);
|
|
19
|
+
}
|
|
20
|
+
if (!Number.isInteger(topK) || topK < 1 || topK > LAYER_PROJECTION_MAX_TOP_K) {
|
|
21
|
+
throw new Error(`Doppler model.inspect.projectLayerOutputs requires topK as a whole number from 1 to ${LAYER_PROJECTION_MAX_TOP_K}.`);
|
|
22
|
+
}
|
|
23
|
+
const width = vectors[0]?.length;
|
|
24
|
+
if (!Number.isInteger(width) || width < 1 || vectors.some((vector) => !(vector instanceof Float32Array) || vector.length !== width)) {
|
|
25
|
+
throw new Error('Doppler model.inspect.projectLayerOutputs requires Float32Array vectors of one length.');
|
|
26
|
+
}
|
|
27
|
+
// Snapshot caller-owned vectors before awaiting the pipeline's execution slot.
|
|
28
|
+
const requestedRows = vectors.length;
|
|
29
|
+
const hiddenStates = new Float32Array(requestedRows * width);
|
|
30
|
+
vectors.forEach((vector, row) => hiddenStates.set(vector, row * width));
|
|
31
|
+
return runPipelineOperation(pipeline, async (scoped) => {
|
|
32
|
+
if (typeof scoped.projectHiddenStates !== 'function') {
|
|
33
|
+
throw new Error('The loaded Doppler pipeline cannot project hidden states.');
|
|
34
|
+
}
|
|
35
|
+
const { logits, rowCount, vocabSize } = await scoped.projectHiddenStates(hiddenStates, requestedRows);
|
|
36
|
+
return {
|
|
37
|
+
schema: LAYER_PROJECTION_SCHEMA,
|
|
38
|
+
modelId,
|
|
39
|
+
projection: { through: ['final-normalization', 'output-head'], kind: 'projected' },
|
|
40
|
+
vocabSize,
|
|
41
|
+
rows: Array.from({ length: rowCount }, (_, row) => ({
|
|
42
|
+
candidates: topCandidates(logits.subarray(row * vocabSize, (row + 1) * vocabSize), scoped.tokenizer, topK),
|
|
43
|
+
})),
|
|
44
|
+
};
|
|
45
|
+
});
|
|
46
|
+
};
|
|
@@ -231,6 +231,12 @@ export interface DopplerModelHandle {
|
|
|
231
231
|
policyId?: string;
|
|
232
232
|
generation?: DopplerGenerateOptions;
|
|
233
233
|
topKSize?: number;
|
|
234
|
+
/**
|
|
235
|
+
* Narrows what an activity-recording policy saves: capture kinds, layers,
|
|
236
|
+
* a token range, and smaller budgets. It can only record less than the
|
|
237
|
+
* policy allows; anything outside the policy fails before the run.
|
|
238
|
+
*/
|
|
239
|
+
capture?: import('../../debug/activity-trace.js').ActivityCaptureRequest | null;
|
|
234
240
|
/**
|
|
235
241
|
* Exact token IDs to prefill instead of tokenizing `prompt`. Generation
|
|
236
242
|
* starts from cleared attention and recurrent state, so the last ID is
|
|
@@ -255,6 +261,18 @@ export interface DopplerModelHandle {
|
|
|
255
261
|
}) => void;
|
|
256
262
|
}
|
|
257
263
|
): Promise<DopplerModelInspectionReceipt>;
|
|
264
|
+
/**
|
|
265
|
+
* Projects hidden-state vectors (for example recorded `layer-output` frames)
|
|
266
|
+
* through this model's final normalization and output head, and returns the
|
|
267
|
+
* `topK` candidates per vector. The result is a projection, sometimes called
|
|
268
|
+
* a logit lens: an estimate of what each vector leans toward, not a token the
|
|
269
|
+
* model produced at that layer. Fails for hyper-connection models and for
|
|
270
|
+
* paired-precision execution plans. Does not change sequence state.
|
|
271
|
+
*/
|
|
272
|
+
projectLayerOutputs(request: {
|
|
273
|
+
vectors: Float32Array[];
|
|
274
|
+
topK: number;
|
|
275
|
+
}): Promise<import('../inspection.js').DopplerLayerProjection>;
|
|
258
276
|
};
|
|
259
277
|
readonly advanced: {
|
|
260
278
|
tokenizeText(text: string): number[];
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
} from './model-evidence.js';
|
|
6
6
|
import { getKernelCapabilities } from '../../gpu/device.js';
|
|
7
7
|
import { createActivationTrace } from '../../debug/activation-trace.js';
|
|
8
|
+
import { resolveActivityCapture } from '../../debug/activity-trace.js';
|
|
8
9
|
import { DOPPLER_VERSION } from '../../version.js';
|
|
9
10
|
import { createGenerationOutputDecoder, decodeGenerationOutput } from '../../inference/generation-output.js';
|
|
10
11
|
import { formatChatMessages } from '../../inference/pipelines/text/chat-format.js';
|
|
@@ -23,6 +24,7 @@ import {
|
|
|
23
24
|
listObservationPolicies,
|
|
24
25
|
resolveObservationPolicy,
|
|
25
26
|
} from '../inspection.js';
|
|
27
|
+
import { projectLayerOutputs } from './layer-projection.js';
|
|
26
28
|
import {
|
|
27
29
|
activateLoRAFromTrainingOutputForPipeline,
|
|
28
30
|
activatePreparedLoRAForPipeline,
|
|
@@ -625,7 +627,8 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
625
627
|
let streaming = typeof options.onEvent === 'function';
|
|
626
628
|
let tokenIndex = 0;
|
|
627
629
|
const topKSize = Number.isInteger(options.topKSize) ? options.topKSize : 5;
|
|
628
|
-
|
|
630
|
+
// A request may record less than the policy allows; the receipt keeps what was chosen.
|
|
631
|
+
const traceConfig = resolveActivityCapture(policy.activationTrace, options.capture);
|
|
629
632
|
const activationTrace = traceConfig ? createActivationTrace(traceConfig, {
|
|
630
633
|
layers: pipeline.modelConfig.numLayers,
|
|
631
634
|
width: pipeline.modelConfig.hiddenSize * (pipeline.modelConfig.hyperConnections?.streamCount ?? 1),
|
|
@@ -728,6 +731,9 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
728
731
|
}
|
|
729
732
|
return receipt;
|
|
730
733
|
},
|
|
734
|
+
projectLayerOutputs(request) {
|
|
735
|
+
return projectLayerOutputs(pipeline, resolved.modelId, request);
|
|
736
|
+
},
|
|
731
737
|
};
|
|
732
738
|
|
|
733
739
|
return handle;
|
|
@@ -10,5 +10,33 @@
|
|
|
10
10
|
"ffnDown",
|
|
11
11
|
"down"
|
|
12
12
|
]
|
|
13
|
-
}
|
|
13
|
+
},
|
|
14
|
+
"roles": [
|
|
15
|
+
"kv-key",
|
|
16
|
+
"kv-value",
|
|
17
|
+
"attention-query",
|
|
18
|
+
"ffn-gate",
|
|
19
|
+
"ffn-up",
|
|
20
|
+
"ffn-product",
|
|
21
|
+
"ffn-down-input",
|
|
22
|
+
"recurrent-memory",
|
|
23
|
+
"convolution-history",
|
|
24
|
+
"linear-a",
|
|
25
|
+
"linear-b",
|
|
26
|
+
"linear-a-log",
|
|
27
|
+
"linear-dt-bias",
|
|
28
|
+
"expert-scores",
|
|
29
|
+
"expert-indices",
|
|
30
|
+
"expert-weights",
|
|
31
|
+
"logits",
|
|
32
|
+
"weight-access"
|
|
33
|
+
],
|
|
34
|
+
"constantRoles": [
|
|
35
|
+
"linear-a-log",
|
|
36
|
+
"linear-dt-bias"
|
|
37
|
+
],
|
|
38
|
+
"prefixRoles": [
|
|
39
|
+
"kv-key",
|
|
40
|
+
"kv-value"
|
|
41
|
+
]
|
|
14
42
|
}
|
|
@@ -151,7 +151,13 @@
|
|
|
151
151
|
"weightRawRows": 4,
|
|
152
152
|
"weightMaxBytes": 16777216,
|
|
153
153
|
"activityTokenWindow": 4,
|
|
154
|
-
"activitySummaryBlock": 256
|
|
154
|
+
"activitySummaryBlock": 256,
|
|
155
|
+
"activitySelection": {
|
|
156
|
+
"roles": null,
|
|
157
|
+
"layers": null,
|
|
158
|
+
"tokens": null
|
|
159
|
+
},
|
|
160
|
+
"activityRoleMaxBytes": {}
|
|
155
161
|
}
|
|
156
162
|
},
|
|
157
163
|
{
|
|
@@ -196,7 +202,13 @@
|
|
|
196
202
|
"weightRawRows": 4,
|
|
197
203
|
"weightMaxBytes": 16777216,
|
|
198
204
|
"activityTokenWindow": 4,
|
|
199
|
-
"activitySummaryBlock": 256
|
|
205
|
+
"activitySummaryBlock": 256,
|
|
206
|
+
"activitySelection": {
|
|
207
|
+
"roles": null,
|
|
208
|
+
"layers": null,
|
|
209
|
+
"tokens": null
|
|
210
|
+
},
|
|
211
|
+
"activityRoleMaxBytes": {}
|
|
200
212
|
}
|
|
201
213
|
}
|
|
202
214
|
]
|
|
@@ -11991,8 +11991,8 @@
|
|
|
11991
11991
|
},
|
|
11992
11992
|
"description": "Explicit F32 matrix row projections with subgroup-owned outputs; independent activation rows use grid y and retain the scalar-buffer single-row arithmetic.",
|
|
11993
11993
|
"reachability": {
|
|
11994
|
-
"status": "
|
|
11995
|
-
"inlineConfigs": [],
|
|
11994
|
+
"status": "pinned",
|
|
11995
|
+
"inlineConfigs": ["runtime/profiles/flash-next-routed-f32kv-qualification"],
|
|
11996
11996
|
"ruleChains": [],
|
|
11997
11997
|
"wgslOnDisk": true
|
|
11998
11998
|
},
|
|
@@ -190,7 +190,7 @@
|
|
|
190
190
|
"constants": {
|
|
191
191
|
"HAS_GATE": true
|
|
192
192
|
},
|
|
193
|
-
"digest": "sha256:
|
|
193
|
+
"digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
|
|
194
194
|
},
|
|
195
195
|
{
|
|
196
196
|
"op": "down_proj",
|
|
@@ -288,13 +288,13 @@
|
|
|
288
288
|
"op": "moe_shared_slot_gu",
|
|
289
289
|
"kernel": "moe_shared_slot_gu_pair_f32.wgsl",
|
|
290
290
|
"entry": "main",
|
|
291
|
-
"digest": "sha256:
|
|
291
|
+
"digest": "sha256:9bd7442fd04389e0ad901e660aeb18abf14ba4a2f34c5668680da6d35be65be4"
|
|
292
292
|
},
|
|
293
293
|
{
|
|
294
294
|
"op": "moe_shared_slot_down",
|
|
295
295
|
"kernel": "moe_shared_slot_down_pair_combine_residual_f32.wgsl",
|
|
296
296
|
"entry": "main",
|
|
297
|
-
"digest": "sha256:
|
|
297
|
+
"digest": "sha256:ce5918180213bd0923b90f579564f1f00b4c4f716ace51ad1200075a3b1a78a8"
|
|
298
298
|
}
|
|
299
299
|
]
|
|
300
300
|
},
|
|
@@ -432,7 +432,7 @@
|
|
|
432
432
|
"constants": {
|
|
433
433
|
"HAS_GATE": true
|
|
434
434
|
},
|
|
435
|
-
"digest": "sha256:
|
|
435
|
+
"digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
|
|
436
436
|
},
|
|
437
437
|
{
|
|
438
438
|
"op": "down_proj",
|
|
@@ -175,7 +175,7 @@
|
|
|
175
175
|
"constants": {
|
|
176
176
|
"HAS_GATE": true
|
|
177
177
|
},
|
|
178
|
-
"digest": "sha256:
|
|
178
|
+
"digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
|
|
179
179
|
},
|
|
180
180
|
{
|
|
181
181
|
"op": "down_proj",
|
|
@@ -273,13 +273,13 @@
|
|
|
273
273
|
"op": "moe_shared_slot_gu",
|
|
274
274
|
"kernel": "moe_shared_slot_gu_pair_f32.wgsl",
|
|
275
275
|
"entry": "main",
|
|
276
|
-
"digest": "sha256:
|
|
276
|
+
"digest": "sha256:9bd7442fd04389e0ad901e660aeb18abf14ba4a2f34c5668680da6d35be65be4"
|
|
277
277
|
},
|
|
278
278
|
{
|
|
279
279
|
"op": "moe_shared_slot_down",
|
|
280
280
|
"kernel": "moe_shared_slot_down_pair_combine_residual_f32.wgsl",
|
|
281
281
|
"entry": "main",
|
|
282
|
-
"digest": "sha256:
|
|
282
|
+
"digest": "sha256:ce5918180213bd0923b90f579564f1f00b4c4f716ace51ad1200075a3b1a78a8"
|
|
283
283
|
}
|
|
284
284
|
]
|
|
285
285
|
},
|
|
@@ -412,7 +412,7 @@
|
|
|
412
412
|
"constants": {
|
|
413
413
|
"HAS_GATE": true
|
|
414
414
|
},
|
|
415
|
-
"digest": "sha256:
|
|
415
|
+
"digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
|
|
416
416
|
},
|
|
417
417
|
{
|
|
418
418
|
"op": "down_proj",
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export interface ActivationTraceLimits {
|
|
2
|
+
activitySelection?: import('./activity-trace.js').ActivitySelection; activityRoleMaxBytes?: Record<string, number>;
|
|
2
3
|
activityMaxBytes?: number; activityMaxFrameValues?: number; activityMaxStateValues?: number;
|
|
3
4
|
activityTokenWindow?: number; activitySummaryBlock?: number;
|
|
4
5
|
weightBlockValues?: number; weightReadValues?: number; weightRowGroupSize?: number; weightRawRows?: number; weightMaxBytes?: number;
|
|
@@ -2,7 +2,29 @@ import type { ActivationTraceLimits } from './activation-trace.js';
|
|
|
2
2
|
import type { TensorRepresentation } from './tensor.js';
|
|
3
3
|
import type { CommandRecorder } from '../gpu/command-recorder.js';
|
|
4
4
|
|
|
5
|
+
/** Which captures are recorded. null in a field means no restriction. `tokens.to` is exclusive. */
|
|
6
|
+
export interface ActivitySelection {
|
|
7
|
+
roles: string[] | null; layers: number[] | null; tokens: { from: number; to: number | null } | null;
|
|
8
|
+
}
|
|
9
|
+
/** A request's narrowing of its policy: record less, or within smaller budgets. */
|
|
10
|
+
export interface ActivityCaptureRequest {
|
|
11
|
+
roles?: string[] | null; layers?: number[] | null; tokens?: { from: number; to: number | null } | null;
|
|
12
|
+
maxBytes?: number; roleMaxBytes?: Record<string, number>;
|
|
13
|
+
}
|
|
14
|
+
/** What happened to one capture kind over a run. Counts are capture attempts. */
|
|
15
|
+
export interface ActivityCoverage {
|
|
16
|
+
captured: number; capturedBytes: number;
|
|
17
|
+
/** Attempts served by an earlier capture of a fixed per-layer value. */
|
|
18
|
+
reused: number;
|
|
19
|
+
dropped: number; failed: number;
|
|
20
|
+
/** Attempts the selection left out, by the first rule that excluded them. */
|
|
21
|
+
notSelected: { role: number; layer: number; token: number };
|
|
22
|
+
firstDroppedToken: number | null;
|
|
23
|
+
/** This kind's own budget, or null when only the recording budget applies. */
|
|
24
|
+
maxBytes: number | null;
|
|
25
|
+
}
|
|
5
26
|
export interface ActivityLimits extends ActivationTraceLimits {
|
|
27
|
+
activitySelection: ActivitySelection; activityRoleMaxBytes: Record<string, number>;
|
|
6
28
|
activitySummaryBlock: number; activityTokenWindow: number; activityMaxBytes: number; activityMaxFrameValues: number; activityMaxStateValues: number;
|
|
7
29
|
weightBlockValues: number; weightReadValues: number; weightRowGroupSize: number; weightRawRows: number; weightMaxBytes: number;
|
|
8
30
|
}
|
|
@@ -26,6 +48,7 @@ export interface ActivityWeight {
|
|
|
26
48
|
export interface ActivityReceipt {
|
|
27
49
|
schema: 'doppler.model-activity/v1'; frames: ActivityFrame[]; weights: ActivityWeight[];
|
|
28
50
|
recordedBytes: number; limits: ActivityLimits;
|
|
51
|
+
selection: ActivitySelection; coverage: Record<string, ActivityCoverage>;
|
|
29
52
|
unavailable: Record<string, string>; model: Record<string, unknown>;
|
|
30
53
|
}
|
|
31
54
|
export interface ActivityCaptureOptions extends TensorRepresentation {
|
|
@@ -50,3 +73,4 @@ export interface ActivityTrace {
|
|
|
50
73
|
logits(logits: Float32Array): Promise<void>;
|
|
51
74
|
}
|
|
52
75
|
export function createActivityTrace(limits: ActivityLimits, started: number): ActivityTrace;
|
|
76
|
+
export function resolveActivityCapture<T extends ActivationTraceLimits>(limits: T, request: ActivityCaptureRequest | null | undefined): T;
|
|
@@ -1,6 +1,81 @@
|
|
|
1
1
|
import { decodeSnapshotData, snapshotElementBytes } from './tensor.js';
|
|
2
2
|
import stages from '../config/inspection/activity-stages.json' with { type: 'json' };
|
|
3
3
|
|
|
4
|
+
const ROLES = new Set(stages.roles);
|
|
5
|
+
// Fixed per-layer parameters: one capture per layer serves every token.
|
|
6
|
+
const CONSTANT_ROLES = new Set(stages.constantRoles);
|
|
7
|
+
// Later tokens read these from every earlier token, so a token range keeps its prefix.
|
|
8
|
+
const PREFIX_ROLES = new Set(stages.prefixRoles);
|
|
9
|
+
|
|
10
|
+
/** @param {string} label @param {unknown} value @param {(item: unknown) => boolean} valid @returns {asserts value is unknown[] | null} */
|
|
11
|
+
function checkList(label, value, valid) {
|
|
12
|
+
if (value === null) return;
|
|
13
|
+
if (!Array.isArray(value) || new Set(value).size !== value.length || !value.every(valid)) {
|
|
14
|
+
throw new Error(`Activity capture ${label} must be null (no restriction) or a list of distinct valid entries; an empty list records none.`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** @param {unknown} selection @returns {asserts selection is import('./activity-trace.js').ActivitySelection} */
|
|
19
|
+
function checkSelection(selection) {
|
|
20
|
+
const value = /** @type {Record<string, unknown> | null} */ (selection);
|
|
21
|
+
if (!value || typeof value !== 'object' || ['roles', 'layers', 'tokens'].some(key => value[key] === undefined)) {
|
|
22
|
+
throw new Error('Activity capture requires activitySelection with explicit roles, layers and tokens (null for no restriction).');
|
|
23
|
+
}
|
|
24
|
+
checkList('roles', value.roles, role => ROLES.has(/** @type {string} */ (role)));
|
|
25
|
+
checkList('layers', value.layers, layer => Number.isSafeInteger(layer) && Number(layer) >= 0);
|
|
26
|
+
const tokens = /** @type {{ from?: unknown; to?: unknown } | null} */ (value.tokens);
|
|
27
|
+
if (tokens !== null && (!Number.isSafeInteger(tokens?.from) || Number(tokens.from) < 0
|
|
28
|
+
|| (tokens.to !== null && (!Number.isSafeInteger(tokens.to) || Number(tokens.to) <= Number(tokens.from))))) {
|
|
29
|
+
throw new Error('Activity capture tokens must be null or { from, to } with to after from (to: null for no end).');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** @param {unknown} budgets @returns {asserts budgets is Record<string, number>} */
|
|
34
|
+
function checkRoleBudgets(budgets) {
|
|
35
|
+
if (!budgets || typeof budgets !== 'object' || Array.isArray(budgets)) throw new Error('Activity capture requires activityRoleMaxBytes (an empty object for no per-kind budgets).');
|
|
36
|
+
for (const [role, bytes] of Object.entries(budgets)) {
|
|
37
|
+
if (!ROLES.has(role) || !Number.isSafeInteger(bytes) || bytes < 1) throw new Error(`Activity capture budget for "${role}" must name a capture kind and a positive byte count.`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// A request records less than its policy allows, or within smaller budgets;
|
|
42
|
+
// it never widens the policy. Anything outside the policy fails here, before a run.
|
|
43
|
+
/** @type {typeof import('./activity-trace.js').resolveActivityCapture} */
|
|
44
|
+
export function resolveActivityCapture(limits, request) {
|
|
45
|
+
if (request === undefined || request === null) return limits;
|
|
46
|
+
if (!limits?.activityMaxBytes) throw new Error('A capture selection requires an observation policy that records model activity.');
|
|
47
|
+
if (typeof request !== 'object' || Array.isArray(request)) throw new Error('A capture selection must be an object.');
|
|
48
|
+
const unknown = Object.keys(request).find(key => !['roles', 'layers', 'tokens', 'maxBytes', 'roleMaxBytes'].includes(key));
|
|
49
|
+
if (unknown) throw new Error(`Unknown capture selection field "${unknown}".`);
|
|
50
|
+
const policy = limits.activitySelection;
|
|
51
|
+
checkSelection(policy);
|
|
52
|
+
const asked = { roles: request.roles ?? null, layers: request.layers ?? null, tokens: request.tokens ?? null };
|
|
53
|
+
checkSelection(asked);
|
|
54
|
+
for (const key of /** @type {const} */ (['roles', 'layers'])) {
|
|
55
|
+
const allowed = /** @type {unknown[] | null} */ (policy[key]);
|
|
56
|
+
const outside = allowed ? (asked[key] ?? []).filter(item => !allowed.includes(item)) : [];
|
|
57
|
+
if (outside.length) throw new Error(`This observation policy does not record ${key === 'roles' ? 'the kind' : 'layer'} "${outside[0]}".`);
|
|
58
|
+
}
|
|
59
|
+
if (policy.tokens && asked.tokens && (asked.tokens.from < policy.tokens.from
|
|
60
|
+
|| (policy.tokens.to !== null && (asked.tokens.to === null || asked.tokens.to > policy.tokens.to)))) {
|
|
61
|
+
throw new Error('The requested token range reaches outside the range this observation policy records.');
|
|
62
|
+
}
|
|
63
|
+
const maxBytes = request.maxBytes ?? limits.activityMaxBytes;
|
|
64
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > limits.activityMaxBytes) {
|
|
65
|
+
throw new Error(`A capture budget must be between 1 and the policy's ${limits.activityMaxBytes} bytes.`);
|
|
66
|
+
}
|
|
67
|
+
const roleMaxBytes = request.roleMaxBytes ?? {};
|
|
68
|
+
checkRoleBudgets(roleMaxBytes);
|
|
69
|
+
checkRoleBudgets(limits.activityRoleMaxBytes);
|
|
70
|
+
for (const [role, bytes] of Object.entries(roleMaxBytes)) {
|
|
71
|
+
const ceiling = Math.min(limits.activityRoleMaxBytes[role] ?? maxBytes, maxBytes);
|
|
72
|
+
if (bytes > ceiling) throw new Error(`The budget for "${role}" exceeds the ${ceiling} bytes this policy and request allow.`);
|
|
73
|
+
}
|
|
74
|
+
return { ...limits, activityMaxBytes: maxBytes,
|
|
75
|
+
activityRoleMaxBytes: { ...limits.activityRoleMaxBytes, ...roleMaxBytes },
|
|
76
|
+
activitySelection: { roles: asked.roles ?? policy.roles, layers: asked.layers ?? policy.layers, tokens: asked.tokens ?? policy.tokens } };
|
|
77
|
+
}
|
|
78
|
+
|
|
4
79
|
/** @type {typeof import('./activity-trace.js').createActivityTrace} */
|
|
5
80
|
export function createActivityTrace(limits, started) {
|
|
6
81
|
/** @type {Array<'activityMaxBytes' | 'activityMaxFrameValues' | 'activityMaxStateValues' | 'activityTokenWindow' | 'activitySummaryBlock'>} */
|
|
@@ -8,6 +83,16 @@ export function createActivityTrace(limits, started) {
|
|
|
8
83
|
for (const name of required) {
|
|
9
84
|
if (!Number.isSafeInteger(limits[name]) || limits[name] < 1) throw new Error(`Activity capture requires ${name}.`);
|
|
10
85
|
}
|
|
86
|
+
checkSelection(limits.activitySelection);
|
|
87
|
+
checkRoleBudgets(limits.activityRoleMaxBytes);
|
|
88
|
+
const selection = limits.activitySelection;
|
|
89
|
+
const selectedRoles = selection.roles && new Set(selection.roles), selectedLayers = selection.layers && new Set(selection.layers);
|
|
90
|
+
/** @type {Record<string, import('./activity-trace.js').ActivityCoverage>} */
|
|
91
|
+
const coverage = {};
|
|
92
|
+
const cover = (/** @type {string} */ role) => coverage[role] ??= { captured: 0, capturedBytes: 0, reused: 0, dropped: 0, failed: 0,
|
|
93
|
+
notSelected: { role: 0, layer: 0, token: 0 }, firstDroppedToken: null, maxBytes: limits.activityRoleMaxBytes[role] ?? null };
|
|
94
|
+
/** @type {Set<string>} */
|
|
95
|
+
const constants = new Set();
|
|
11
96
|
/** @type {import('./activity-trace.js').ActivityFrame[]} */
|
|
12
97
|
const frames = [];
|
|
13
98
|
/** @type {import('./activity-trace.js').ActivityWeight[]} */
|
|
@@ -34,11 +119,25 @@ export function createActivityTrace(limits, started) {
|
|
|
34
119
|
async capture(role, buffer, options) {
|
|
35
120
|
if (role === 'ffn-down-input' && !weights.some(w => w.layer === options.layer && w.columnEnergy.length)) return;
|
|
36
121
|
if (!tokenIds.length) return;
|
|
37
|
-
|
|
122
|
+
const seen = cover(role), constant = CONSTANT_ROLES.has(role), constantKey = `${role}/${options.layer}`;
|
|
123
|
+
if (constants.has(constantKey)) { seen.reused++; return; }
|
|
124
|
+
// What the selection leaves out is counted, not stored: it takes no frame and no budget.
|
|
125
|
+
if (selectedRoles && !selectedRoles.has(role)) { seen.notSelected.role++; return; }
|
|
126
|
+
if (selectedLayers && options.layer >= 0 && !selectedLayers.has(options.layer)) { seen.notSelected.layer++; return; }
|
|
38
127
|
const originalShape = [...options.shape];
|
|
39
128
|
let ids = [...(options.tokenIds ?? tokenIds)], start = options.tokenStart ?? tokenStart;
|
|
40
129
|
let offset = options.offset ?? 0;
|
|
41
130
|
const fullShape = [...originalShape];
|
|
131
|
+
if (selection.tokens && !constant) {
|
|
132
|
+
const first = Math.max(start, PREFIX_ROLES.has(role) ? 0 : selection.tokens.from);
|
|
133
|
+
const end = Math.min(start + ids.length, selection.tokens.to ?? Infinity);
|
|
134
|
+
if (first >= end) { seen.notSelected.token++; return; }
|
|
135
|
+
// One row per token: keep only the selected rows. Other shapes are kept whole.
|
|
136
|
+
if (!options.state && fullShape.length === 2 && fullShape[0] === ids.length && end - first < ids.length) {
|
|
137
|
+
offset += (first - start) * fullShape[1]; ids = ids.slice(first - start, end - start); start = first; fullShape[0] = ids.length;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (frames.length >= limits.maxFrames) { unavailable['capture-limit'] = 'Activity frame limit reached; later boundaries are unrecorded.'; return; }
|
|
42
141
|
if (role.startsWith('ffn-') && fullShape.length === 2 && fullShape[0] === ids.length && ids.length > limits.activityTokenWindow) {
|
|
43
142
|
const skipped = ids.length - limits.activityTokenWindow;
|
|
44
143
|
offset += skipped * fullShape[1]; start += skipped; ids = ids.slice(skipped); fullShape[0] = ids.length;
|
|
@@ -56,10 +155,14 @@ export function createActivityTrace(limits, started) {
|
|
|
56
155
|
layer: options.layer, tokenStart: start,
|
|
57
156
|
tokenIds: ids, shape: [count], fullShape, offset: 0,
|
|
58
157
|
dtype: options.dtype, values: null, status: 'captured', reason: null,
|
|
59
|
-
representation: options.representation ?? 'raw', metadata: { originalShape, precision: options.precision ?? null, ...options.metadata } };
|
|
158
|
+
representation: options.representation ?? 'raw', metadata: { originalShape, precision: options.precision ?? null, ...options.metadata, ...(constant ? { constant: true } : {}) } };
|
|
60
159
|
frames.push(frame);
|
|
61
|
-
|
|
62
|
-
|
|
160
|
+
const overRole = seen.maxBytes !== null && seen.capturedBytes + payloadBytes > seen.maxBytes;
|
|
161
|
+
if (overRole || bytes + payloadBytes > limits.activityMaxBytes) {
|
|
162
|
+
frame.status = 'dropped';
|
|
163
|
+
frame.reason = overRole ? `Activity budget for ${role} reached; inference continues.` : 'Activity recording budget reached; inference continues.';
|
|
164
|
+
frame.metadata.limit = overRole ? 'activityRoleMaxBytes' : 'activityMaxBytes';
|
|
165
|
+
seen.dropped++; seen.firstDroppedToken ??= start;
|
|
63
166
|
return;
|
|
64
167
|
}
|
|
65
168
|
try {
|
|
@@ -94,9 +197,10 @@ export function createActivityTrace(limits, started) {
|
|
|
94
197
|
}
|
|
95
198
|
frame.summary = { blockSize: limits.activitySummaryBlock, observedValues: values.length, values: summary };
|
|
96
199
|
}
|
|
97
|
-
bytes += payloadBytes;
|
|
200
|
+
bytes += payloadBytes; seen.captured++; seen.capturedBytes += payloadBytes;
|
|
201
|
+
if (constant) constants.add(constantKey);
|
|
98
202
|
} catch (error) {
|
|
99
|
-
frame.status = 'failed'; frame.reason = String(error);
|
|
203
|
+
frame.status = 'failed'; frame.reason = String(error); seen.failed++;
|
|
100
204
|
}
|
|
101
205
|
},
|
|
102
206
|
weightAccess(layer, stage) {
|
|
@@ -116,7 +220,8 @@ export function createActivityTrace(limits, started) {
|
|
|
116
220
|
setWeights(value) { weights = value; },
|
|
117
221
|
snapshot(includeFrames = true) {
|
|
118
222
|
return { schema: 'doppler.model-activity/v1', frames: includeFrames ? [...frames] : [],
|
|
119
|
-
weights, recordedBytes: bytes, limits: { ...limits },
|
|
223
|
+
weights, recordedBytes: bytes, limits: { ...limits }, selection: structuredClone(selection), coverage: structuredClone(coverage),
|
|
224
|
+
unavailable: { ...unavailable }, model };
|
|
120
225
|
},
|
|
121
226
|
async logits(values) {
|
|
122
227
|
await api.capture('logits', values, { layer: -1, dtype: 'f32', shape: [1, values.length],
|
|
@@ -17,24 +17,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
17
17
|
return value;
|
|
18
18
|
}
|
|
19
19
|
fn sigmoid(x: f32) -> f32 {
|
|
20
|
-
|
|
21
|
-
let z = exp_refined(-abs(
|
|
22
|
-
let numerator = select(z, 1.0,
|
|
20
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
21
|
+
let z = exp_refined(-abs(x));
|
|
22
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
23
23
|
let denominator = 1.0 + z;
|
|
24
24
|
let inverse = reciprocal_refined(denominator);
|
|
25
25
|
let quotient = numerator * inverse;
|
|
26
26
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
// Compensate the rounded numerator and quotient
|
|
29
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
30
30
|
fn silu(x: f32) -> f32 {
|
|
31
|
-
let z = exp_refined(-abs(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
31
|
+
let z = exp_refined(-abs(x));
|
|
32
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
33
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
34
|
+
let den = 1.0 + z;
|
|
35
|
+
let inverse = reciprocal_refined(den);
|
|
36
|
+
let q = numerator * inverse;
|
|
37
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -152,24 +152,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
152
152
|
return value;
|
|
153
153
|
}
|
|
154
154
|
fn sigmoid(x: f32) -> f32 {
|
|
155
|
-
|
|
156
|
-
let z = exp_refined(-abs(
|
|
157
|
-
let numerator = select(z, 1.0,
|
|
155
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
156
|
+
let z = exp_refined(-abs(x));
|
|
157
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
158
158
|
let denominator = 1.0 + z;
|
|
159
159
|
let inverse = reciprocal_refined(denominator);
|
|
160
160
|
let quotient = numerator * inverse;
|
|
161
161
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
-
// Compensate the rounded numerator and quotient
|
|
164
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
165
165
|
fn silu(x: f32) -> f32 {
|
|
166
|
-
let z = exp_refined(-abs(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
166
|
+
let z = exp_refined(-abs(x));
|
|
167
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
168
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
169
|
+
let den = 1.0 + z;
|
|
170
|
+
let inverse = reciprocal_refined(den);
|
|
171
|
+
let q = numerator * inverse;
|
|
172
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -152,24 +152,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
152
152
|
return value;
|
|
153
153
|
}
|
|
154
154
|
fn sigmoid(x: f32) -> f32 {
|
|
155
|
-
|
|
156
|
-
let z = exp_refined(-abs(
|
|
157
|
-
let numerator = select(z, 1.0,
|
|
155
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
156
|
+
let z = exp_refined(-abs(x));
|
|
157
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
158
158
|
let denominator = 1.0 + z;
|
|
159
159
|
let inverse = reciprocal_refined(denominator);
|
|
160
160
|
let quotient = numerator * inverse;
|
|
161
161
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
-
// Compensate the rounded numerator and quotient
|
|
164
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
165
165
|
fn silu(x: f32) -> f32 {
|
|
166
|
-
let z = exp_refined(-abs(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
166
|
+
let z = exp_refined(-abs(x));
|
|
167
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
168
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
169
|
+
let den = 1.0 + z;
|
|
170
|
+
let inverse = reciprocal_refined(den);
|
|
171
|
+
let q = numerator * inverse;
|
|
172
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -158,24 +158,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
158
158
|
return value;
|
|
159
159
|
}
|
|
160
160
|
fn sigmoid(x: f32) -> f32 {
|
|
161
|
-
|
|
162
|
-
let z = exp_refined(-abs(
|
|
163
|
-
let numerator = select(z, 1.0,
|
|
161
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
162
|
+
let z = exp_refined(-abs(x));
|
|
163
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
164
164
|
let denominator = 1.0 + z;
|
|
165
165
|
let inverse = reciprocal_refined(denominator);
|
|
166
166
|
let quotient = numerator * inverse;
|
|
167
167
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
-
// Compensate the rounded numerator and quotient
|
|
170
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
171
171
|
fn silu(x: f32) -> f32 {
|
|
172
|
-
let z = exp_refined(-abs(
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
172
|
+
let z = exp_refined(-abs(x));
|
|
173
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
174
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
175
|
+
let den = 1.0 + z;
|
|
176
|
+
let inverse = reciprocal_refined(den);
|
|
177
|
+
let q = numerator * inverse;
|
|
178
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
179
179
|
}
|
|
180
180
|
|
|
181
181
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -156,24 +156,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
156
156
|
return value;
|
|
157
157
|
}
|
|
158
158
|
fn sigmoid(x: f32) -> f32 {
|
|
159
|
-
|
|
160
|
-
let z = exp_refined(-abs(
|
|
161
|
-
let numerator = select(z, 1.0,
|
|
159
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
160
|
+
let z = exp_refined(-abs(x));
|
|
161
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
162
162
|
let denominator = 1.0 + z;
|
|
163
163
|
let inverse = reciprocal_refined(denominator);
|
|
164
164
|
let quotient = numerator * inverse;
|
|
165
165
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
-
// Compensate the rounded numerator and quotient
|
|
168
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
169
169
|
fn silu(x: f32) -> f32 {
|
|
170
|
-
let z = exp_refined(-abs(
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
170
|
+
let z = exp_refined(-abs(x));
|
|
171
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
172
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
173
|
+
let den = 1.0 + z;
|
|
174
|
+
let inverse = reciprocal_refined(den);
|
|
175
|
+
let q = numerator * inverse;
|
|
176
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
177
177
|
}
|
|
178
178
|
|
|
179
179
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -158,24 +158,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
158
158
|
return value;
|
|
159
159
|
}
|
|
160
160
|
fn sigmoid(x: f32) -> f32 {
|
|
161
|
-
|
|
162
|
-
let z = exp_refined(-abs(
|
|
163
|
-
let numerator = select(z, 1.0,
|
|
161
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
162
|
+
let z = exp_refined(-abs(x));
|
|
163
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
164
164
|
let denominator = 1.0 + z;
|
|
165
165
|
let inverse = reciprocal_refined(denominator);
|
|
166
166
|
let quotient = numerator * inverse;
|
|
167
167
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
-
// Compensate the rounded numerator and quotient
|
|
170
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
171
171
|
fn silu(x: f32) -> f32 {
|
|
172
|
-
let z = exp_refined(-abs(
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
172
|
+
let z = exp_refined(-abs(x));
|
|
173
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
174
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
175
|
+
let den = 1.0 + z;
|
|
176
|
+
let inverse = reciprocal_refined(den);
|
|
177
|
+
let q = numerator * inverse;
|
|
178
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
179
179
|
}
|
|
180
180
|
|
|
181
181
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -17,9 +17,9 @@ struct Uniforms {
|
|
|
17
17
|
@group(0) @binding(4) var<storage, read_write> output: array<f32>;
|
|
18
18
|
|
|
19
19
|
fn sigmoid(x: f32) -> f32 {
|
|
20
|
-
|
|
21
|
-
let z = exp_refined(-abs(
|
|
22
|
-
let numerator = select(z, 1.0,
|
|
20
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
21
|
+
let z = exp_refined(-abs(x));
|
|
22
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
23
23
|
let denominator = 1.0 + z;
|
|
24
24
|
let inverse = reciprocal_refined(denominator);
|
|
25
25
|
let quotient = numerator * inverse;
|
|
@@ -31,6 +31,7 @@ export const PIPELINE_OPERATIONS = Object.freeze({
|
|
|
31
31
|
advanceWithToken: 'execution', advanceWithTokenAndEmbedding: 'execution',
|
|
32
32
|
prefillKVOnly: 'execution', prefillForLoRATraining: 'execution',
|
|
33
33
|
computeDiffusionGemmaCanvasLogits: 'execution', computeDiffusionGemmaCanvasStep: 'execution',
|
|
34
|
+
projectHiddenStates: 'execution',
|
|
34
35
|
prefillWithEmbedding: 'execution', prefillWithLogits: 'execution',
|
|
35
36
|
prefillWithTokenLogits: 'execution', prefillWithTokenLogitsFromKV: 'execution',
|
|
36
37
|
embed: 'execution', embedBatch: Object.freeze({ kind: 'execution', context: 'explicit' }), encodeSequence: 'execution',
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { prepareActivityRecording } from '../activity-weights.js';
|
|
2
|
-
import { beginGenerationContext, finishGenerationContext, generationContextInput } from '../sequence-state.js';
|
|
2
|
+
import { beginGenerationContext, beginSequenceMemoryBoundary, finishGenerationContext, generationContextInput } from '../sequence-state.js';
|
|
3
3
|
import { assertHyperConnectionOperation } from '../hyper-connection/sequence-transaction.js';
|
|
4
4
|
import { lookupDraft, verifySpeculativeTokens } from './speculative-runtime.js';
|
|
5
5
|
import { markDiagnosticSequenceStep } from './diagnostic-sequence.js';
|
|
@@ -211,6 +211,7 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
|
|
|
211
211
|
try {
|
|
212
212
|
beginGenerationContext(this._state, options.generationContext, opts);
|
|
213
213
|
await prepareActivityRecording(this._state, opts);
|
|
214
|
+
if (this._state.currentSeqLen === 0) await beginSequenceMemoryBoundary();
|
|
214
215
|
const prefillStartSeqLen = this._state.currentSeqLen;
|
|
215
216
|
const prefillStart = performance.now();
|
|
216
217
|
opts._whileFinalPrefillRuns = wholeTokenAheadVerification(this, opts);
|
|
@@ -345,6 +346,7 @@ export async function generateTokenIds(prompt, options = {}) {
|
|
|
345
346
|
try {
|
|
346
347
|
beginGenerationContext(this._state, options.generationContext, opts);
|
|
347
348
|
await prepareActivityRecording(this._state, opts);
|
|
349
|
+
if (this._state.currentSeqLen === 0) await beginSequenceMemoryBoundary();
|
|
348
350
|
const prefillStartSeqLen = this._state.currentSeqLen;
|
|
349
351
|
const prefillStart = performance.now();
|
|
350
352
|
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { PipelineState } from '../state.js';
|
|
2
|
+
|
|
3
|
+
export interface HiddenProjectionResult {
|
|
4
|
+
/** Row-major logits: `rowCount` rows of `vocabSize` values. */
|
|
5
|
+
logits: Float32Array;
|
|
6
|
+
rowCount: number;
|
|
7
|
+
vocabSize: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Applies the model's final normalization and output head to caller-supplied
|
|
12
|
+
* hidden states. Fails for hyper-connection models and paired-precision plans.
|
|
13
|
+
*/
|
|
14
|
+
export declare function projectHiddenStates(
|
|
15
|
+
state: PipelineState,
|
|
16
|
+
hiddenStates: Float32Array,
|
|
17
|
+
rowCount: number
|
|
18
|
+
): Promise<HiddenProjectionResult>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { computeLogits } from '../logits/index.js';
|
|
2
|
+
import { getLogitsConfig, getLogitsWeights } from './logits-config.js';
|
|
3
|
+
|
|
4
|
+
// Projects caller-supplied hidden states through this model's own final
|
|
5
|
+
// normalization and output head: the computation every decode step ends with,
|
|
6
|
+
// applied to vectors that did not come from the last layer. The result is an
|
|
7
|
+
// observation aid (a "logit lens"), not something the model produced at that
|
|
8
|
+
// layer. Nothing here touches sequence state or the KV cache.
|
|
9
|
+
/** @type {typeof import('./hidden-projection.js').projectHiddenStates} */
|
|
10
|
+
export const projectHiddenStates = async function projectHiddenStates(state, hiddenStates, rowCount) {
|
|
11
|
+
if (!(hiddenStates instanceof Float32Array)) {
|
|
12
|
+
throw new Error('projectHiddenStates requires hidden states as a Float32Array.');
|
|
13
|
+
}
|
|
14
|
+
if (!Number.isInteger(rowCount) || rowCount < 1) {
|
|
15
|
+
throw new Error('projectHiddenStates requires a positive whole number of rows.');
|
|
16
|
+
}
|
|
17
|
+
const config = getLogitsConfig(state);
|
|
18
|
+
if (config.hyperConnections) {
|
|
19
|
+
// Their trace rows hold several streams and a mixer replaces the final norm.
|
|
20
|
+
throw new Error('projectHiddenStates does not support hyper-connection models: a recorded layer output there is not one hidden state.');
|
|
21
|
+
}
|
|
22
|
+
if (config.activationPrecision !== undefined && config.activationPrecision !== null) {
|
|
23
|
+
throw new Error(`projectHiddenStates requires single-component activations; the active plan uses "${config.activationPrecision}".`);
|
|
24
|
+
}
|
|
25
|
+
if (hiddenStates.length !== rowCount * config.hiddenSize) {
|
|
26
|
+
throw new Error(`projectHiddenStates expected ${rowCount} rows of ${config.hiddenSize} values (${rowCount * config.hiddenSize}), got ${hiddenStates.length}.`);
|
|
27
|
+
}
|
|
28
|
+
for (let index = 0; index < hiddenStates.length; index += 1) {
|
|
29
|
+
if (!Number.isFinite(hiddenStates[index])) {
|
|
30
|
+
throw new Error(`projectHiddenStates received a non-finite value at index ${index}.`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const logits = await computeLogits(
|
|
34
|
+
hiddenStates,
|
|
35
|
+
rowCount,
|
|
36
|
+
getLogitsWeights(state),
|
|
37
|
+
config,
|
|
38
|
+
state.useGPU,
|
|
39
|
+
state.debugFlags,
|
|
40
|
+
undefined,
|
|
41
|
+
undefined,
|
|
42
|
+
state.runtimeConfig.shared.debug.probes,
|
|
43
|
+
{ lastPositionOnly: false }
|
|
44
|
+
);
|
|
45
|
+
if (!(logits instanceof Float32Array) || logits.length % rowCount !== 0) {
|
|
46
|
+
throw new Error('projectHiddenStates did not receive one logit row per hidden state.');
|
|
47
|
+
}
|
|
48
|
+
return { logits, rowCount, vocabSize: logits.length / rowCount };
|
|
49
|
+
};
|
|
@@ -8,6 +8,7 @@ import { releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
|
8
8
|
import { sample, applyRepetitionPenalty, applyPresencePenalty } from './sampling.js';
|
|
9
9
|
import { isStopToken } from './init.js';
|
|
10
10
|
import { decodeStepLogits } from './generator-steps.js';
|
|
11
|
+
import { projectHiddenStates as projectHiddenStatesImpl } from './generator/hidden-projection.js';
|
|
11
12
|
import { buildLayerContext, debugCheckBuffer as debugCheckBufferHelper } from './generator/session-context.js';
|
|
12
13
|
import { getLogitsConfig, getLogitsWeights } from './generator/logits-config.js';
|
|
13
14
|
import { releaseSharedAttentionState } from './generator/attention-lifecycle.js';
|
|
@@ -408,6 +409,10 @@ export class PipelineGenerator {
|
|
|
408
409
|
return computeDiffusionGemmaCanvasLogitsImpl.apply(this, arguments);
|
|
409
410
|
}
|
|
410
411
|
|
|
412
|
+
async projectHiddenStates(hiddenStates, rowCount) {
|
|
413
|
+
return projectHiddenStatesImpl(this._state, hiddenStates, rowCount);
|
|
414
|
+
}
|
|
415
|
+
|
|
411
416
|
async computeDiffusionGemmaCanvasStep(args, options = {}) {
|
|
412
417
|
return computeDiffusionGemmaCanvasStepImpl.apply(this, arguments);
|
|
413
418
|
}
|
|
@@ -2,6 +2,8 @@ import { createCommandRecorder } from '../../../gpu/command-recorder.js';
|
|
|
2
2
|
import { beginImmediateDispatchProfile } from '../../../gpu/kernels/dispatch.js';
|
|
3
3
|
import { log } from '../../../debug/index.js';
|
|
4
4
|
import { acquireBuffer, releaseBuffer, discardBuffer, readBuffer, uploadData } from '../../../memory/buffer-pool.js';
|
|
5
|
+
import { retireIdleBuffersUnderDeviceBudget } from '../../../memory/sequence-retirement.js';
|
|
6
|
+
import { getDevice } from '../../../gpu/device.js';
|
|
5
7
|
import { getWeightDtype, getWeightMetadata, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
6
8
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
7
9
|
import { embed } from './embed.js';
|
|
@@ -121,6 +123,9 @@ export async function executePartitionLayers(state, input, signal) {
|
|
|
121
123
|
signal.throwIfAborted();
|
|
122
124
|
const prefill = state.currentSeqLen === 0;
|
|
123
125
|
const setup = resolvePartitionInput(state, input, prefill);
|
|
126
|
+
// A sequence's first prefill starts from its owned state under a device budget.
|
|
127
|
+
if (prefill) await retireIdleBuffersUnderDeviceBudget(getDevice());
|
|
128
|
+
signal.throwIfAborted();
|
|
124
129
|
if (setup.config.hyperConnections && !state.hyperConnectionSession) {
|
|
125
130
|
if (!state.dopplerLoader || !state.kvCache) throw new Error('Resident hyper-connections require the owning loader and attempt cache.');
|
|
126
131
|
state.hyperConnectionSession = await createHyperConnectionSession(setup.config, state.dopplerLoader,
|
|
@@ -114,6 +114,14 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
114
114
|
options?: GenerateOptions & { __internalGenerate?: boolean }
|
|
115
115
|
): Promise<Float32Array>;
|
|
116
116
|
|
|
117
|
+
/**
|
|
118
|
+
* Applies the final normalization and output head to caller-supplied hidden
|
|
119
|
+
* states (`rowCount` rows of `hiddenSize`). An observation aid: the logits
|
|
120
|
+
* are a projection, not what the model produced at an earlier layer. Does
|
|
121
|
+
* not read or change sequence state.
|
|
122
|
+
*/
|
|
123
|
+
projectHiddenStates(hiddenStates: Float32Array, rowCount: number): Promise<import('./generator/hidden-projection.js').HiddenProjectionResult>;
|
|
124
|
+
|
|
117
125
|
computeDiffusionGemmaCanvasStep(
|
|
118
126
|
args: DiffusionGemmaCanvasStepInput,
|
|
119
127
|
options?: GenerateOptions & { __internalGenerate?: boolean }
|
|
@@ -301,6 +301,10 @@ export class InferencePipeline extends PipelineState {
|
|
|
301
301
|
return this.generator.computeDiffusionGemmaCanvasLogits(args, options);
|
|
302
302
|
}
|
|
303
303
|
|
|
304
|
+
projectHiddenStates(hiddenStates, rowCount) {
|
|
305
|
+
return this.generator.projectHiddenStates(hiddenStates, rowCount);
|
|
306
|
+
}
|
|
307
|
+
|
|
304
308
|
computeDiffusionGemmaCanvasStep(args, options = {}) {
|
|
305
309
|
assertNotAborted(options?.signal);
|
|
306
310
|
return this.generator.computeDiffusionGemmaCanvasStep(args, options);
|
|
@@ -45,6 +45,8 @@ export interface ContextLease {
|
|
|
45
45
|
export function supportsGenerationContext(state: ContextPipelineState): boolean;
|
|
46
46
|
export function createGenerationContext(state: ContextPipelineState, options: { maxBytes: number }): GenerationContext;
|
|
47
47
|
export function closeGenerationContexts(state: ContextPipelineState): void;
|
|
48
|
+
/** Retire idle pooled memory before a new generation's first prefill when a device budget is set. */
|
|
49
|
+
export function beginSequenceMemoryBoundary(): Promise<boolean>;
|
|
48
50
|
export function beginGenerationContext(state: ContextPipelineState, context: GenerationContext | undefined, options: ContextOptions): void;
|
|
49
51
|
export function generationContextInput(state: ContextPipelineState, inputIds: number[]): number[];
|
|
50
52
|
export function generationContextCheckpointLength(state: ContextPipelineState, prompt: import('./generator.js').PromptInput,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { getDevice } from '../../../gpu/device.js';
|
|
2
2
|
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
3
|
+
import { retireIdleBuffersUnderDeviceBudget } from '../../../memory/sequence-retirement.js';
|
|
3
4
|
import { createKVCache } from './init.js';
|
|
4
5
|
import { createLinearAttentionRuntime, resetLinearAttentionRuntime, resetLinearAttentionSequence } from './linear-attention.js';
|
|
5
6
|
import { isConvLayerType } from './layer-input-execution.js';
|
|
@@ -397,6 +398,12 @@ export async function checkpointGenerationContext(state, tokens) {
|
|
|
397
398
|
state.wholeTokenProgram?.program.invalidate('generation checkpoint parked');
|
|
398
399
|
}
|
|
399
400
|
|
|
401
|
+
// A new generation's first prefill starts from its owned state under a device budget.
|
|
402
|
+
/** @type {import('./sequence-state.js').beginSequenceMemoryBoundary} */
|
|
403
|
+
export async function beginSequenceMemoryBoundary() {
|
|
404
|
+
return retireIdleBuffersUnderDeviceBudget(getDevice());
|
|
405
|
+
}
|
|
406
|
+
|
|
400
407
|
/** @type {import('./sequence-state.js').prepareGenerationContextPrefill} */
|
|
401
408
|
export async function prepareGenerationContextPrefill(generator, prompt, inputIds, opts) {
|
|
402
409
|
const state = generator._state;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { getBufferPool } from './buffer-pool.js';
|
|
2
|
+
import { getDeviceMemorySnapshot } from './device-budget.js';
|
|
3
|
+
import { isDeviceLost } from './device-residency.js';
|
|
4
|
+
|
|
5
|
+
// Under an explicit device memory budget, idle pooled buffers and pending
|
|
6
|
+
// retirements are charged as live allocations. A sequence boundary retires them
|
|
7
|
+
// after confirmed GPU completion, so one sequence's working set is bounded by its
|
|
8
|
+
// own allocations rather than by buckets left over from earlier prompt lengths.
|
|
9
|
+
// Without a device budget the pool keeps its reuse behavior. Owned (active)
|
|
10
|
+
// buffers are never touched.
|
|
11
|
+
/** @type {typeof import('./sequence-retirement.js').retireIdleBuffersUnderDeviceBudget} */
|
|
12
|
+
export async function retireIdleBuffersUnderDeviceBudget(device) {
|
|
13
|
+
if (!device || isDeviceLost(device)) return false;
|
|
14
|
+
if ((getDeviceMemorySnapshot(device)?.maxBytes ?? null) === null) return false;
|
|
15
|
+
const pool = getBufferPool(device);
|
|
16
|
+
pool.clearPool();
|
|
17
|
+
await pool.settleDeferredDestruction();
|
|
18
|
+
return true;
|
|
19
|
+
}
|
package/src/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const DOPPLER_VERSION = '0.7.
|
|
1
|
+
export const DOPPLER_VERSION = '0.7.2';
|
|
2
2
|
export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;
|