doppler-gpu 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/package.json +1 -1
- package/src/client/model-host/model-session.d.ts +6 -0
- package/src/client/model-host/model-session.js +3 -1
- package/src/config/inspection/activity-stages.json +29 -1
- package/src/config/inspection/observation-policies.json +14 -2
- package/src/config/kernels/registry.json +2 -2
- package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-candidate.json +4 -4
- package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-pooled-q8-common-diagnostic.json +4 -4
- package/src/debug/activation-trace.d.ts +1 -0
- package/src/debug/activity-trace.d.ts +24 -0
- package/src/debug/activity-trace.js +112 -7
- package/src/gpu/kernels/moe_shared_slot_combine_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_down_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_down_pair_combine_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_down_pair_combine_residual_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_gu_f32.wgsl +11 -11
- package/src/gpu/kernels/moe_shared_slot_gu_pair_f32.wgsl +11 -11
- package/src/gpu/kernels/sigmoid_gated_add_f32.wgsl +3 -3
- package/src/inference/pipelines/text/generator/decode-runtime.js +3 -1
- package/src/inference/pipelines/text/partition-execution.js +5 -0
- package/src/inference/pipelines/text/sequence-state.d.ts +2 -0
- package/src/inference/pipelines/text/sequence-state.js +7 -0
- package/src/memory/sequence-retirement.js +19 -0
- package/src/version.js +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,47 @@ Historical package notes are preserved outside the runtime archive in
|
|
|
4
4
|
`docs/status/archive/package-changelog-through-0.6.1.md` and the earlier
|
|
5
5
|
`package-changelog-through-0.4.15.md` snapshot in that directory.
|
|
6
6
|
|
|
7
|
+
## [0.7.1] - 2026-11-11
|
|
8
|
+
|
|
9
|
+
Patch release. Published bytes for 0.7.0 are unchanged.
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- **Gate activations agree across fused and unfused paths.** 0.7.0 removed
|
|
14
|
+
the +/-15 sigmoid input clamp from `silu.wgsl`, but
|
|
15
|
+
`sigmoid_gated_add_f32.wgsl` and the `moe_shared_slot_*` kernels kept it.
|
|
16
|
+
For |gate| > 15 the separate and fused paths disagreed, and the Qwen
|
|
17
|
+
shared-expert gate differed from the logistic function: `sigmoid(-20)`
|
|
18
|
+
gave 3.06e-7 instead of 2.06e-9. All of them now use `silu.wgsl`'s
|
|
19
|
+
unclamped form; only the explicit `swigluLimit` output clamp remains. The
|
|
20
|
+
two Qwen 3.6 35B-A3B q4_0 runtime profiles carry the current kernel digests.
|
|
21
|
+
- **Repeated prompts no longer exhaust an explicit device memory budget.**
|
|
22
|
+
With `configureDeviceMemoryBudget`, idle pooled buffers count as live.
|
|
23
|
+
Each new prompt length added buckets on top of earlier ones until a later
|
|
24
|
+
request was rejected. A new generation's first prefill now retires idle
|
|
25
|
+
pooled buffers after confirmed GPU completion, only when a budget is set.
|
|
26
|
+
Owned buffers are never retired. Measured with Qwen 3.5 2B under a
|
|
27
|
+
6,442,450,944-byte budget: steady memory between requests stays at or
|
|
28
|
+
below 2.6 GB.
|
|
29
|
+
|
|
30
|
+
### Added
|
|
31
|
+
|
|
32
|
+
- `model.inspect.generate(prompt, { capture })` narrows activity recording by
|
|
33
|
+
kind, layer, token range and budget (selective capture, `e56cf5d6`).
|
|
34
|
+
|
|
35
|
+
### Known limits
|
|
36
|
+
|
|
37
|
+
- Budgeted hosts should also set
|
|
38
|
+
`inference.session.prefillChunkSubmitMode: "sync"` (or
|
|
39
|
+
`prefillMaxInFlightChunks: 1`) for models whose manifest declares async
|
|
40
|
+
prefill submits. With async submits, warm requests keep up to three layer
|
|
41
|
+
chunks of prompt-sized temporaries alive. Qwen 3.5 2B at 2,000 tokens
|
|
42
|
+
peaked at 4.40 GB with sync, against a failure under the same 6.44 GB
|
|
43
|
+
budget with async, at the same prefill time.
|
|
44
|
+
- Flash-Next (Qwen 3.8) is not qualified. Its 2,187-token prompt still fails
|
|
45
|
+
the full-logit gate against independent CPU references; see
|
|
46
|
+
`reports/flash-next-reference-controls-20261110/`.
|
|
47
|
+
|
|
7
48
|
## [0.7.0] - 2026-11-10
|
|
8
49
|
|
|
9
50
|
First npm release since 0.6.1. It also carries the 0.6.27 and 0.6.2 candidate
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "doppler-gpu",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.1",
|
|
4
4
|
"description": "Doppler Rig prepares signed model Capsules; Doppler Run executes them locally through WebGPU for JavaScript",
|
|
5
5
|
"main": "src/capsule-runtime.js",
|
|
6
6
|
"types": "src/capsule-runtime.d.ts",
|
|
@@ -231,6 +231,12 @@ export interface DopplerModelHandle {
|
|
|
231
231
|
policyId?: string;
|
|
232
232
|
generation?: DopplerGenerateOptions;
|
|
233
233
|
topKSize?: number;
|
|
234
|
+
/**
|
|
235
|
+
* Narrows what an activity-recording policy saves: capture kinds, layers,
|
|
236
|
+
* a token range, and smaller budgets. It can only record less than the
|
|
237
|
+
* policy allows; anything outside the policy fails before the run.
|
|
238
|
+
*/
|
|
239
|
+
capture?: import('../../debug/activity-trace.js').ActivityCaptureRequest | null;
|
|
234
240
|
/**
|
|
235
241
|
* Exact token IDs to prefill instead of tokenizing `prompt`. Generation
|
|
236
242
|
* starts from cleared attention and recurrent state, so the last ID is
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
} from './model-evidence.js';
|
|
6
6
|
import { getKernelCapabilities } from '../../gpu/device.js';
|
|
7
7
|
import { createActivationTrace } from '../../debug/activation-trace.js';
|
|
8
|
+
import { resolveActivityCapture } from '../../debug/activity-trace.js';
|
|
8
9
|
import { DOPPLER_VERSION } from '../../version.js';
|
|
9
10
|
import { createGenerationOutputDecoder, decodeGenerationOutput } from '../../inference/generation-output.js';
|
|
10
11
|
import { formatChatMessages } from '../../inference/pipelines/text/chat-format.js';
|
|
@@ -625,7 +626,8 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
625
626
|
let streaming = typeof options.onEvent === 'function';
|
|
626
627
|
let tokenIndex = 0;
|
|
627
628
|
const topKSize = Number.isInteger(options.topKSize) ? options.topKSize : 5;
|
|
628
|
-
|
|
629
|
+
// A request may record less than the policy allows; the receipt keeps what was chosen.
|
|
630
|
+
const traceConfig = resolveActivityCapture(policy.activationTrace, options.capture);
|
|
629
631
|
const activationTrace = traceConfig ? createActivationTrace(traceConfig, {
|
|
630
632
|
layers: pipeline.modelConfig.numLayers,
|
|
631
633
|
width: pipeline.modelConfig.hiddenSize * (pipeline.modelConfig.hyperConnections?.streamCount ?? 1),
|
|
@@ -10,5 +10,33 @@
|
|
|
10
10
|
"ffnDown",
|
|
11
11
|
"down"
|
|
12
12
|
]
|
|
13
|
-
}
|
|
13
|
+
},
|
|
14
|
+
"roles": [
|
|
15
|
+
"kv-key",
|
|
16
|
+
"kv-value",
|
|
17
|
+
"attention-query",
|
|
18
|
+
"ffn-gate",
|
|
19
|
+
"ffn-up",
|
|
20
|
+
"ffn-product",
|
|
21
|
+
"ffn-down-input",
|
|
22
|
+
"recurrent-memory",
|
|
23
|
+
"convolution-history",
|
|
24
|
+
"linear-a",
|
|
25
|
+
"linear-b",
|
|
26
|
+
"linear-a-log",
|
|
27
|
+
"linear-dt-bias",
|
|
28
|
+
"expert-scores",
|
|
29
|
+
"expert-indices",
|
|
30
|
+
"expert-weights",
|
|
31
|
+
"logits",
|
|
32
|
+
"weight-access"
|
|
33
|
+
],
|
|
34
|
+
"constantRoles": [
|
|
35
|
+
"linear-a-log",
|
|
36
|
+
"linear-dt-bias"
|
|
37
|
+
],
|
|
38
|
+
"prefixRoles": [
|
|
39
|
+
"kv-key",
|
|
40
|
+
"kv-value"
|
|
41
|
+
]
|
|
14
42
|
}
|
|
@@ -151,7 +151,13 @@
|
|
|
151
151
|
"weightRawRows": 4,
|
|
152
152
|
"weightMaxBytes": 16777216,
|
|
153
153
|
"activityTokenWindow": 4,
|
|
154
|
-
"activitySummaryBlock": 256
|
|
154
|
+
"activitySummaryBlock": 256,
|
|
155
|
+
"activitySelection": {
|
|
156
|
+
"roles": null,
|
|
157
|
+
"layers": null,
|
|
158
|
+
"tokens": null
|
|
159
|
+
},
|
|
160
|
+
"activityRoleMaxBytes": {}
|
|
155
161
|
}
|
|
156
162
|
},
|
|
157
163
|
{
|
|
@@ -196,7 +202,13 @@
|
|
|
196
202
|
"weightRawRows": 4,
|
|
197
203
|
"weightMaxBytes": 16777216,
|
|
198
204
|
"activityTokenWindow": 4,
|
|
199
|
-
"activitySummaryBlock": 256
|
|
205
|
+
"activitySummaryBlock": 256,
|
|
206
|
+
"activitySelection": {
|
|
207
|
+
"roles": null,
|
|
208
|
+
"layers": null,
|
|
209
|
+
"tokens": null
|
|
210
|
+
},
|
|
211
|
+
"activityRoleMaxBytes": {}
|
|
200
212
|
}
|
|
201
213
|
}
|
|
202
214
|
]
|
|
@@ -11991,8 +11991,8 @@
|
|
|
11991
11991
|
},
|
|
11992
11992
|
"description": "Explicit F32 matrix row projections with subgroup-owned outputs; independent activation rows use grid y and retain the scalar-buffer single-row arithmetic.",
|
|
11993
11993
|
"reachability": {
|
|
11994
|
-
"status": "
|
|
11995
|
-
"inlineConfigs": [],
|
|
11994
|
+
"status": "pinned",
|
|
11995
|
+
"inlineConfigs": ["runtime/profiles/flash-next-routed-f32kv-qualification"],
|
|
11996
11996
|
"ruleChains": [],
|
|
11997
11997
|
"wgslOnDisk": true
|
|
11998
11998
|
},
|
|
@@ -190,7 +190,7 @@
|
|
|
190
190
|
"constants": {
|
|
191
191
|
"HAS_GATE": true
|
|
192
192
|
},
|
|
193
|
-
"digest": "sha256:
|
|
193
|
+
"digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
|
|
194
194
|
},
|
|
195
195
|
{
|
|
196
196
|
"op": "down_proj",
|
|
@@ -288,13 +288,13 @@
|
|
|
288
288
|
"op": "moe_shared_slot_gu",
|
|
289
289
|
"kernel": "moe_shared_slot_gu_pair_f32.wgsl",
|
|
290
290
|
"entry": "main",
|
|
291
|
-
"digest": "sha256:
|
|
291
|
+
"digest": "sha256:9bd7442fd04389e0ad901e660aeb18abf14ba4a2f34c5668680da6d35be65be4"
|
|
292
292
|
},
|
|
293
293
|
{
|
|
294
294
|
"op": "moe_shared_slot_down",
|
|
295
295
|
"kernel": "moe_shared_slot_down_pair_combine_residual_f32.wgsl",
|
|
296
296
|
"entry": "main",
|
|
297
|
-
"digest": "sha256:
|
|
297
|
+
"digest": "sha256:ce5918180213bd0923b90f579564f1f00b4c4f716ace51ad1200075a3b1a78a8"
|
|
298
298
|
}
|
|
299
299
|
]
|
|
300
300
|
},
|
|
@@ -432,7 +432,7 @@
|
|
|
432
432
|
"constants": {
|
|
433
433
|
"HAS_GATE": true
|
|
434
434
|
},
|
|
435
|
-
"digest": "sha256:
|
|
435
|
+
"digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
|
|
436
436
|
},
|
|
437
437
|
{
|
|
438
438
|
"op": "down_proj",
|
|
@@ -175,7 +175,7 @@
|
|
|
175
175
|
"constants": {
|
|
176
176
|
"HAS_GATE": true
|
|
177
177
|
},
|
|
178
|
-
"digest": "sha256:
|
|
178
|
+
"digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
|
|
179
179
|
},
|
|
180
180
|
{
|
|
181
181
|
"op": "down_proj",
|
|
@@ -273,13 +273,13 @@
|
|
|
273
273
|
"op": "moe_shared_slot_gu",
|
|
274
274
|
"kernel": "moe_shared_slot_gu_pair_f32.wgsl",
|
|
275
275
|
"entry": "main",
|
|
276
|
-
"digest": "sha256:
|
|
276
|
+
"digest": "sha256:9bd7442fd04389e0ad901e660aeb18abf14ba4a2f34c5668680da6d35be65be4"
|
|
277
277
|
},
|
|
278
278
|
{
|
|
279
279
|
"op": "moe_shared_slot_down",
|
|
280
280
|
"kernel": "moe_shared_slot_down_pair_combine_residual_f32.wgsl",
|
|
281
281
|
"entry": "main",
|
|
282
|
-
"digest": "sha256:
|
|
282
|
+
"digest": "sha256:ce5918180213bd0923b90f579564f1f00b4c4f716ace51ad1200075a3b1a78a8"
|
|
283
283
|
}
|
|
284
284
|
]
|
|
285
285
|
},
|
|
@@ -412,7 +412,7 @@
|
|
|
412
412
|
"constants": {
|
|
413
413
|
"HAS_GATE": true
|
|
414
414
|
},
|
|
415
|
-
"digest": "sha256:
|
|
415
|
+
"digest": "sha256:712787a59b032453b09f147be0d0c073266c47a7a12c3b746984aaf28075bedb"
|
|
416
416
|
},
|
|
417
417
|
{
|
|
418
418
|
"op": "down_proj",
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export interface ActivationTraceLimits {
|
|
2
|
+
activitySelection?: import('./activity-trace.js').ActivitySelection; activityRoleMaxBytes?: Record<string, number>;
|
|
2
3
|
activityMaxBytes?: number; activityMaxFrameValues?: number; activityMaxStateValues?: number;
|
|
3
4
|
activityTokenWindow?: number; activitySummaryBlock?: number;
|
|
4
5
|
weightBlockValues?: number; weightReadValues?: number; weightRowGroupSize?: number; weightRawRows?: number; weightMaxBytes?: number;
|
|
@@ -2,7 +2,29 @@ import type { ActivationTraceLimits } from './activation-trace.js';
|
|
|
2
2
|
import type { TensorRepresentation } from './tensor.js';
|
|
3
3
|
import type { CommandRecorder } from '../gpu/command-recorder.js';
|
|
4
4
|
|
|
5
|
+
/** Which captures are recorded. null in a field means no restriction. `tokens.to` is exclusive. */
|
|
6
|
+
export interface ActivitySelection {
|
|
7
|
+
roles: string[] | null; layers: number[] | null; tokens: { from: number; to: number | null } | null;
|
|
8
|
+
}
|
|
9
|
+
/** A request's narrowing of its policy: record less, or within smaller budgets. */
|
|
10
|
+
export interface ActivityCaptureRequest {
|
|
11
|
+
roles?: string[] | null; layers?: number[] | null; tokens?: { from: number; to: number | null } | null;
|
|
12
|
+
maxBytes?: number; roleMaxBytes?: Record<string, number>;
|
|
13
|
+
}
|
|
14
|
+
/** What happened to one capture kind over a run. Counts are capture attempts. */
|
|
15
|
+
export interface ActivityCoverage {
|
|
16
|
+
captured: number; capturedBytes: number;
|
|
17
|
+
/** Attempts served by an earlier capture of a fixed per-layer value. */
|
|
18
|
+
reused: number;
|
|
19
|
+
dropped: number; failed: number;
|
|
20
|
+
/** Attempts the selection left out, by the first rule that excluded them. */
|
|
21
|
+
notSelected: { role: number; layer: number; token: number };
|
|
22
|
+
firstDroppedToken: number | null;
|
|
23
|
+
/** This kind's own budget, or null when only the recording budget applies. */
|
|
24
|
+
maxBytes: number | null;
|
|
25
|
+
}
|
|
5
26
|
export interface ActivityLimits extends ActivationTraceLimits {
|
|
27
|
+
activitySelection: ActivitySelection; activityRoleMaxBytes: Record<string, number>;
|
|
6
28
|
activitySummaryBlock: number; activityTokenWindow: number; activityMaxBytes: number; activityMaxFrameValues: number; activityMaxStateValues: number;
|
|
7
29
|
weightBlockValues: number; weightReadValues: number; weightRowGroupSize: number; weightRawRows: number; weightMaxBytes: number;
|
|
8
30
|
}
|
|
@@ -26,6 +48,7 @@ export interface ActivityWeight {
|
|
|
26
48
|
export interface ActivityReceipt {
|
|
27
49
|
schema: 'doppler.model-activity/v1'; frames: ActivityFrame[]; weights: ActivityWeight[];
|
|
28
50
|
recordedBytes: number; limits: ActivityLimits;
|
|
51
|
+
selection: ActivitySelection; coverage: Record<string, ActivityCoverage>;
|
|
29
52
|
unavailable: Record<string, string>; model: Record<string, unknown>;
|
|
30
53
|
}
|
|
31
54
|
export interface ActivityCaptureOptions extends TensorRepresentation {
|
|
@@ -50,3 +73,4 @@ export interface ActivityTrace {
|
|
|
50
73
|
logits(logits: Float32Array): Promise<void>;
|
|
51
74
|
}
|
|
52
75
|
export function createActivityTrace(limits: ActivityLimits, started: number): ActivityTrace;
|
|
76
|
+
export function resolveActivityCapture<T extends ActivationTraceLimits>(limits: T, request: ActivityCaptureRequest | null | undefined): T;
|
|
@@ -1,6 +1,81 @@
|
|
|
1
1
|
import { decodeSnapshotData, snapshotElementBytes } from './tensor.js';
|
|
2
2
|
import stages from '../config/inspection/activity-stages.json' with { type: 'json' };
|
|
3
3
|
|
|
4
|
+
const ROLES = new Set(stages.roles);
|
|
5
|
+
// Fixed per-layer parameters: one capture per layer serves every token.
|
|
6
|
+
const CONSTANT_ROLES = new Set(stages.constantRoles);
|
|
7
|
+
// Later tokens read these from every earlier token, so a token range keeps its prefix.
|
|
8
|
+
const PREFIX_ROLES = new Set(stages.prefixRoles);
|
|
9
|
+
|
|
10
|
+
/** @param {string} label @param {unknown} value @param {(item: unknown) => boolean} valid @returns {asserts value is unknown[] | null} */
|
|
11
|
+
function checkList(label, value, valid) {
|
|
12
|
+
if (value === null) return;
|
|
13
|
+
if (!Array.isArray(value) || new Set(value).size !== value.length || !value.every(valid)) {
|
|
14
|
+
throw new Error(`Activity capture ${label} must be null (no restriction) or a list of distinct valid entries; an empty list records none.`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** @param {unknown} selection @returns {asserts selection is import('./activity-trace.js').ActivitySelection} */
|
|
19
|
+
function checkSelection(selection) {
|
|
20
|
+
const value = /** @type {Record<string, unknown> | null} */ (selection);
|
|
21
|
+
if (!value || typeof value !== 'object' || ['roles', 'layers', 'tokens'].some(key => value[key] === undefined)) {
|
|
22
|
+
throw new Error('Activity capture requires activitySelection with explicit roles, layers and tokens (null for no restriction).');
|
|
23
|
+
}
|
|
24
|
+
checkList('roles', value.roles, role => ROLES.has(/** @type {string} */ (role)));
|
|
25
|
+
checkList('layers', value.layers, layer => Number.isSafeInteger(layer) && Number(layer) >= 0);
|
|
26
|
+
const tokens = /** @type {{ from?: unknown; to?: unknown } | null} */ (value.tokens);
|
|
27
|
+
if (tokens !== null && (!Number.isSafeInteger(tokens?.from) || Number(tokens.from) < 0
|
|
28
|
+
|| (tokens.to !== null && (!Number.isSafeInteger(tokens.to) || Number(tokens.to) <= Number(tokens.from))))) {
|
|
29
|
+
throw new Error('Activity capture tokens must be null or { from, to } with to after from (to: null for no end).');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** @param {unknown} budgets @returns {asserts budgets is Record<string, number>} */
|
|
34
|
+
function checkRoleBudgets(budgets) {
|
|
35
|
+
if (!budgets || typeof budgets !== 'object' || Array.isArray(budgets)) throw new Error('Activity capture requires activityRoleMaxBytes (an empty object for no per-kind budgets).');
|
|
36
|
+
for (const [role, bytes] of Object.entries(budgets)) {
|
|
37
|
+
if (!ROLES.has(role) || !Number.isSafeInteger(bytes) || bytes < 1) throw new Error(`Activity capture budget for "${role}" must name a capture kind and a positive byte count.`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// A request records less than its policy allows, or within smaller budgets;
|
|
42
|
+
// it never widens the policy. Anything outside the policy fails here, before a run.
|
|
43
|
+
/** @type {typeof import('./activity-trace.js').resolveActivityCapture} */
|
|
44
|
+
export function resolveActivityCapture(limits, request) {
|
|
45
|
+
if (request === undefined || request === null) return limits;
|
|
46
|
+
if (!limits?.activityMaxBytes) throw new Error('A capture selection requires an observation policy that records model activity.');
|
|
47
|
+
if (typeof request !== 'object' || Array.isArray(request)) throw new Error('A capture selection must be an object.');
|
|
48
|
+
const unknown = Object.keys(request).find(key => !['roles', 'layers', 'tokens', 'maxBytes', 'roleMaxBytes'].includes(key));
|
|
49
|
+
if (unknown) throw new Error(`Unknown capture selection field "${unknown}".`);
|
|
50
|
+
const policy = limits.activitySelection;
|
|
51
|
+
checkSelection(policy);
|
|
52
|
+
const asked = { roles: request.roles ?? null, layers: request.layers ?? null, tokens: request.tokens ?? null };
|
|
53
|
+
checkSelection(asked);
|
|
54
|
+
for (const key of /** @type {const} */ (['roles', 'layers'])) {
|
|
55
|
+
const allowed = /** @type {unknown[] | null} */ (policy[key]);
|
|
56
|
+
const outside = allowed ? (asked[key] ?? []).filter(item => !allowed.includes(item)) : [];
|
|
57
|
+
if (outside.length) throw new Error(`This observation policy does not record ${key === 'roles' ? 'the kind' : 'layer'} "${outside[0]}".`);
|
|
58
|
+
}
|
|
59
|
+
if (policy.tokens && asked.tokens && (asked.tokens.from < policy.tokens.from
|
|
60
|
+
|| (policy.tokens.to !== null && (asked.tokens.to === null || asked.tokens.to > policy.tokens.to)))) {
|
|
61
|
+
throw new Error('The requested token range reaches outside the range this observation policy records.');
|
|
62
|
+
}
|
|
63
|
+
const maxBytes = request.maxBytes ?? limits.activityMaxBytes;
|
|
64
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > limits.activityMaxBytes) {
|
|
65
|
+
throw new Error(`A capture budget must be between 1 and the policy's ${limits.activityMaxBytes} bytes.`);
|
|
66
|
+
}
|
|
67
|
+
const roleMaxBytes = request.roleMaxBytes ?? {};
|
|
68
|
+
checkRoleBudgets(roleMaxBytes);
|
|
69
|
+
checkRoleBudgets(limits.activityRoleMaxBytes);
|
|
70
|
+
for (const [role, bytes] of Object.entries(roleMaxBytes)) {
|
|
71
|
+
const ceiling = Math.min(limits.activityRoleMaxBytes[role] ?? maxBytes, maxBytes);
|
|
72
|
+
if (bytes > ceiling) throw new Error(`The budget for "${role}" exceeds the ${ceiling} bytes this policy and request allow.`);
|
|
73
|
+
}
|
|
74
|
+
return { ...limits, activityMaxBytes: maxBytes,
|
|
75
|
+
activityRoleMaxBytes: { ...limits.activityRoleMaxBytes, ...roleMaxBytes },
|
|
76
|
+
activitySelection: { roles: asked.roles ?? policy.roles, layers: asked.layers ?? policy.layers, tokens: asked.tokens ?? policy.tokens } };
|
|
77
|
+
}
|
|
78
|
+
|
|
4
79
|
/** @type {typeof import('./activity-trace.js').createActivityTrace} */
|
|
5
80
|
export function createActivityTrace(limits, started) {
|
|
6
81
|
/** @type {Array<'activityMaxBytes' | 'activityMaxFrameValues' | 'activityMaxStateValues' | 'activityTokenWindow' | 'activitySummaryBlock'>} */
|
|
@@ -8,6 +83,16 @@ export function createActivityTrace(limits, started) {
|
|
|
8
83
|
for (const name of required) {
|
|
9
84
|
if (!Number.isSafeInteger(limits[name]) || limits[name] < 1) throw new Error(`Activity capture requires ${name}.`);
|
|
10
85
|
}
|
|
86
|
+
checkSelection(limits.activitySelection);
|
|
87
|
+
checkRoleBudgets(limits.activityRoleMaxBytes);
|
|
88
|
+
const selection = limits.activitySelection;
|
|
89
|
+
const selectedRoles = selection.roles && new Set(selection.roles), selectedLayers = selection.layers && new Set(selection.layers);
|
|
90
|
+
/** @type {Record<string, import('./activity-trace.js').ActivityCoverage>} */
|
|
91
|
+
const coverage = {};
|
|
92
|
+
const cover = (/** @type {string} */ role) => coverage[role] ??= { captured: 0, capturedBytes: 0, reused: 0, dropped: 0, failed: 0,
|
|
93
|
+
notSelected: { role: 0, layer: 0, token: 0 }, firstDroppedToken: null, maxBytes: limits.activityRoleMaxBytes[role] ?? null };
|
|
94
|
+
/** @type {Set<string>} */
|
|
95
|
+
const constants = new Set();
|
|
11
96
|
/** @type {import('./activity-trace.js').ActivityFrame[]} */
|
|
12
97
|
const frames = [];
|
|
13
98
|
/** @type {import('./activity-trace.js').ActivityWeight[]} */
|
|
@@ -34,11 +119,25 @@ export function createActivityTrace(limits, started) {
|
|
|
34
119
|
async capture(role, buffer, options) {
|
|
35
120
|
if (role === 'ffn-down-input' && !weights.some(w => w.layer === options.layer && w.columnEnergy.length)) return;
|
|
36
121
|
if (!tokenIds.length) return;
|
|
37
|
-
|
|
122
|
+
const seen = cover(role), constant = CONSTANT_ROLES.has(role), constantKey = `${role}/${options.layer}`;
|
|
123
|
+
if (constants.has(constantKey)) { seen.reused++; return; }
|
|
124
|
+
// What the selection leaves out is counted, not stored: it takes no frame and no budget.
|
|
125
|
+
if (selectedRoles && !selectedRoles.has(role)) { seen.notSelected.role++; return; }
|
|
126
|
+
if (selectedLayers && options.layer >= 0 && !selectedLayers.has(options.layer)) { seen.notSelected.layer++; return; }
|
|
38
127
|
const originalShape = [...options.shape];
|
|
39
128
|
let ids = [...(options.tokenIds ?? tokenIds)], start = options.tokenStart ?? tokenStart;
|
|
40
129
|
let offset = options.offset ?? 0;
|
|
41
130
|
const fullShape = [...originalShape];
|
|
131
|
+
if (selection.tokens && !constant) {
|
|
132
|
+
const first = Math.max(start, PREFIX_ROLES.has(role) ? 0 : selection.tokens.from);
|
|
133
|
+
const end = Math.min(start + ids.length, selection.tokens.to ?? Infinity);
|
|
134
|
+
if (first >= end) { seen.notSelected.token++; return; }
|
|
135
|
+
// One row per token: keep only the selected rows. Other shapes are kept whole.
|
|
136
|
+
if (!options.state && fullShape.length === 2 && fullShape[0] === ids.length && end - first < ids.length) {
|
|
137
|
+
offset += (first - start) * fullShape[1]; ids = ids.slice(first - start, end - start); start = first; fullShape[0] = ids.length;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (frames.length >= limits.maxFrames) { unavailable['capture-limit'] = 'Activity frame limit reached; later boundaries are unrecorded.'; return; }
|
|
42
141
|
if (role.startsWith('ffn-') && fullShape.length === 2 && fullShape[0] === ids.length && ids.length > limits.activityTokenWindow) {
|
|
43
142
|
const skipped = ids.length - limits.activityTokenWindow;
|
|
44
143
|
offset += skipped * fullShape[1]; start += skipped; ids = ids.slice(skipped); fullShape[0] = ids.length;
|
|
@@ -56,10 +155,14 @@ export function createActivityTrace(limits, started) {
|
|
|
56
155
|
layer: options.layer, tokenStart: start,
|
|
57
156
|
tokenIds: ids, shape: [count], fullShape, offset: 0,
|
|
58
157
|
dtype: options.dtype, values: null, status: 'captured', reason: null,
|
|
59
|
-
representation: options.representation ?? 'raw', metadata: { originalShape, precision: options.precision ?? null, ...options.metadata } };
|
|
158
|
+
representation: options.representation ?? 'raw', metadata: { originalShape, precision: options.precision ?? null, ...options.metadata, ...(constant ? { constant: true } : {}) } };
|
|
60
159
|
frames.push(frame);
|
|
61
|
-
|
|
62
|
-
|
|
160
|
+
const overRole = seen.maxBytes !== null && seen.capturedBytes + payloadBytes > seen.maxBytes;
|
|
161
|
+
if (overRole || bytes + payloadBytes > limits.activityMaxBytes) {
|
|
162
|
+
frame.status = 'dropped';
|
|
163
|
+
frame.reason = overRole ? `Activity budget for ${role} reached; inference continues.` : 'Activity recording budget reached; inference continues.';
|
|
164
|
+
frame.metadata.limit = overRole ? 'activityRoleMaxBytes' : 'activityMaxBytes';
|
|
165
|
+
seen.dropped++; seen.firstDroppedToken ??= start;
|
|
63
166
|
return;
|
|
64
167
|
}
|
|
65
168
|
try {
|
|
@@ -94,9 +197,10 @@ export function createActivityTrace(limits, started) {
|
|
|
94
197
|
}
|
|
95
198
|
frame.summary = { blockSize: limits.activitySummaryBlock, observedValues: values.length, values: summary };
|
|
96
199
|
}
|
|
97
|
-
bytes += payloadBytes;
|
|
200
|
+
bytes += payloadBytes; seen.captured++; seen.capturedBytes += payloadBytes;
|
|
201
|
+
if (constant) constants.add(constantKey);
|
|
98
202
|
} catch (error) {
|
|
99
|
-
frame.status = 'failed'; frame.reason = String(error);
|
|
203
|
+
frame.status = 'failed'; frame.reason = String(error); seen.failed++;
|
|
100
204
|
}
|
|
101
205
|
},
|
|
102
206
|
weightAccess(layer, stage) {
|
|
@@ -116,7 +220,8 @@ export function createActivityTrace(limits, started) {
|
|
|
116
220
|
setWeights(value) { weights = value; },
|
|
117
221
|
snapshot(includeFrames = true) {
|
|
118
222
|
return { schema: 'doppler.model-activity/v1', frames: includeFrames ? [...frames] : [],
|
|
119
|
-
weights, recordedBytes: bytes, limits: { ...limits },
|
|
223
|
+
weights, recordedBytes: bytes, limits: { ...limits }, selection: structuredClone(selection), coverage: structuredClone(coverage),
|
|
224
|
+
unavailable: { ...unavailable }, model };
|
|
120
225
|
},
|
|
121
226
|
async logits(values) {
|
|
122
227
|
await api.capture('logits', values, { layer: -1, dtype: 'f32', shape: [1, values.length],
|
|
@@ -17,24 +17,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
17
17
|
return value;
|
|
18
18
|
}
|
|
19
19
|
fn sigmoid(x: f32) -> f32 {
|
|
20
|
-
|
|
21
|
-
let z = exp_refined(-abs(
|
|
22
|
-
let numerator = select(z, 1.0,
|
|
20
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
21
|
+
let z = exp_refined(-abs(x));
|
|
22
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
23
23
|
let denominator = 1.0 + z;
|
|
24
24
|
let inverse = reciprocal_refined(denominator);
|
|
25
25
|
let quotient = numerator * inverse;
|
|
26
26
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
// Compensate the rounded numerator and quotient
|
|
29
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
30
30
|
fn silu(x: f32) -> f32 {
|
|
31
|
-
let z = exp_refined(-abs(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
31
|
+
let z = exp_refined(-abs(x));
|
|
32
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
33
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
34
|
+
let den = 1.0 + z;
|
|
35
|
+
let inverse = reciprocal_refined(den);
|
|
36
|
+
let q = numerator * inverse;
|
|
37
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -152,24 +152,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
152
152
|
return value;
|
|
153
153
|
}
|
|
154
154
|
fn sigmoid(x: f32) -> f32 {
|
|
155
|
-
|
|
156
|
-
let z = exp_refined(-abs(
|
|
157
|
-
let numerator = select(z, 1.0,
|
|
155
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
156
|
+
let z = exp_refined(-abs(x));
|
|
157
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
158
158
|
let denominator = 1.0 + z;
|
|
159
159
|
let inverse = reciprocal_refined(denominator);
|
|
160
160
|
let quotient = numerator * inverse;
|
|
161
161
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
-
// Compensate the rounded numerator and quotient
|
|
164
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
165
165
|
fn silu(x: f32) -> f32 {
|
|
166
|
-
let z = exp_refined(-abs(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
166
|
+
let z = exp_refined(-abs(x));
|
|
167
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
168
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
169
|
+
let den = 1.0 + z;
|
|
170
|
+
let inverse = reciprocal_refined(den);
|
|
171
|
+
let q = numerator * inverse;
|
|
172
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -152,24 +152,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
152
152
|
return value;
|
|
153
153
|
}
|
|
154
154
|
fn sigmoid(x: f32) -> f32 {
|
|
155
|
-
|
|
156
|
-
let z = exp_refined(-abs(
|
|
157
|
-
let numerator = select(z, 1.0,
|
|
155
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
156
|
+
let z = exp_refined(-abs(x));
|
|
157
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
158
158
|
let denominator = 1.0 + z;
|
|
159
159
|
let inverse = reciprocal_refined(denominator);
|
|
160
160
|
let quotient = numerator * inverse;
|
|
161
161
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
-
// Compensate the rounded numerator and quotient
|
|
164
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
165
165
|
fn silu(x: f32) -> f32 {
|
|
166
|
-
let z = exp_refined(-abs(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
166
|
+
let z = exp_refined(-abs(x));
|
|
167
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
168
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
169
|
+
let den = 1.0 + z;
|
|
170
|
+
let inverse = reciprocal_refined(den);
|
|
171
|
+
let q = numerator * inverse;
|
|
172
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -158,24 +158,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
158
158
|
return value;
|
|
159
159
|
}
|
|
160
160
|
fn sigmoid(x: f32) -> f32 {
|
|
161
|
-
|
|
162
|
-
let z = exp_refined(-abs(
|
|
163
|
-
let numerator = select(z, 1.0,
|
|
161
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
162
|
+
let z = exp_refined(-abs(x));
|
|
163
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
164
164
|
let denominator = 1.0 + z;
|
|
165
165
|
let inverse = reciprocal_refined(denominator);
|
|
166
166
|
let quotient = numerator * inverse;
|
|
167
167
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
-
// Compensate the rounded numerator and quotient
|
|
170
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
171
171
|
fn silu(x: f32) -> f32 {
|
|
172
|
-
let z = exp_refined(-abs(
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
172
|
+
let z = exp_refined(-abs(x));
|
|
173
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
174
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
175
|
+
let den = 1.0 + z;
|
|
176
|
+
let inverse = reciprocal_refined(den);
|
|
177
|
+
let q = numerator * inverse;
|
|
178
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
179
179
|
}
|
|
180
180
|
|
|
181
181
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -156,24 +156,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
156
156
|
return value;
|
|
157
157
|
}
|
|
158
158
|
fn sigmoid(x: f32) -> f32 {
|
|
159
|
-
|
|
160
|
-
let z = exp_refined(-abs(
|
|
161
|
-
let numerator = select(z, 1.0,
|
|
159
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
160
|
+
let z = exp_refined(-abs(x));
|
|
161
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
162
162
|
let denominator = 1.0 + z;
|
|
163
163
|
let inverse = reciprocal_refined(denominator);
|
|
164
164
|
let quotient = numerator * inverse;
|
|
165
165
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
-
// Compensate the rounded numerator and quotient
|
|
168
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
169
169
|
fn silu(x: f32) -> f32 {
|
|
170
|
-
let z = exp_refined(-abs(
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
170
|
+
let z = exp_refined(-abs(x));
|
|
171
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
172
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
173
|
+
let den = 1.0 + z;
|
|
174
|
+
let inverse = reciprocal_refined(den);
|
|
175
|
+
let q = numerator * inverse;
|
|
176
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
177
177
|
}
|
|
178
178
|
|
|
179
179
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -158,24 +158,24 @@ fn clamp_product(value: f32) -> f32 {
|
|
|
158
158
|
return value;
|
|
159
159
|
}
|
|
160
160
|
fn sigmoid(x: f32) -> f32 {
|
|
161
|
-
|
|
162
|
-
let z = exp_refined(-abs(
|
|
163
|
-
let numerator = select(z, 1.0,
|
|
161
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
162
|
+
let z = exp_refined(-abs(x));
|
|
163
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
164
164
|
let denominator = 1.0 + z;
|
|
165
165
|
let inverse = reciprocal_refined(denominator);
|
|
166
166
|
let quotient = numerator * inverse;
|
|
167
167
|
return fma(fma(-quotient, denominator, numerator), inverse, quotient);
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
-
// Compensate the rounded numerator and quotient
|
|
170
|
+
// Compensate the rounded numerator and quotient without changing the activation.
|
|
171
171
|
fn silu(x: f32) -> f32 {
|
|
172
|
-
let z = exp_refined(-abs(
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
172
|
+
let z = exp_refined(-abs(x));
|
|
173
|
+
let numerator = select(x * z, x, x >= 0.0);
|
|
174
|
+
let error = select(fma(x, z, -numerator), 0.0, x >= 0.0);
|
|
175
|
+
let den = 1.0 + z;
|
|
176
|
+
let inverse = reciprocal_refined(den);
|
|
177
|
+
let q = numerator * inverse;
|
|
178
|
+
return fma(fma(-q, den, numerator) + error, inverse, q);
|
|
179
179
|
}
|
|
180
180
|
|
|
181
181
|
fn exp_refined(x: f32) -> f32 {
|
|
@@ -17,9 +17,9 @@ struct Uniforms {
|
|
|
17
17
|
@group(0) @binding(4) var<storage, read_write> output: array<f32>;
|
|
18
18
|
|
|
19
19
|
fn sigmoid(x: f32) -> f32 {
|
|
20
|
-
|
|
21
|
-
let z = exp_refined(-abs(
|
|
22
|
-
let numerator = select(z, 1.0,
|
|
20
|
+
// The exponent is nonpositive, so overflow protection needs no input clamp.
|
|
21
|
+
let z = exp_refined(-abs(x));
|
|
22
|
+
let numerator = select(z, 1.0, x >= 0.0);
|
|
23
23
|
let denominator = 1.0 + z;
|
|
24
24
|
let inverse = reciprocal_refined(denominator);
|
|
25
25
|
let quotient = numerator * inverse;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { prepareActivityRecording } from '../activity-weights.js';
|
|
2
|
-
import { beginGenerationContext, finishGenerationContext, generationContextInput } from '../sequence-state.js';
|
|
2
|
+
import { beginGenerationContext, beginSequenceMemoryBoundary, finishGenerationContext, generationContextInput } from '../sequence-state.js';
|
|
3
3
|
import { assertHyperConnectionOperation } from '../hyper-connection/sequence-transaction.js';
|
|
4
4
|
import { lookupDraft, verifySpeculativeTokens } from './speculative-runtime.js';
|
|
5
5
|
import { markDiagnosticSequenceStep } from './diagnostic-sequence.js';
|
|
@@ -211,6 +211,7 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
|
|
|
211
211
|
try {
|
|
212
212
|
beginGenerationContext(this._state, options.generationContext, opts);
|
|
213
213
|
await prepareActivityRecording(this._state, opts);
|
|
214
|
+
if (this._state.currentSeqLen === 0) await beginSequenceMemoryBoundary();
|
|
214
215
|
const prefillStartSeqLen = this._state.currentSeqLen;
|
|
215
216
|
const prefillStart = performance.now();
|
|
216
217
|
opts._whileFinalPrefillRuns = wholeTokenAheadVerification(this, opts);
|
|
@@ -345,6 +346,7 @@ export async function generateTokenIds(prompt, options = {}) {
|
|
|
345
346
|
try {
|
|
346
347
|
beginGenerationContext(this._state, options.generationContext, opts);
|
|
347
348
|
await prepareActivityRecording(this._state, opts);
|
|
349
|
+
if (this._state.currentSeqLen === 0) await beginSequenceMemoryBoundary();
|
|
348
350
|
const prefillStartSeqLen = this._state.currentSeqLen;
|
|
349
351
|
const prefillStart = performance.now();
|
|
350
352
|
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
|
|
@@ -2,6 +2,8 @@ import { createCommandRecorder } from '../../../gpu/command-recorder.js';
|
|
|
2
2
|
import { beginImmediateDispatchProfile } from '../../../gpu/kernels/dispatch.js';
|
|
3
3
|
import { log } from '../../../debug/index.js';
|
|
4
4
|
import { acquireBuffer, releaseBuffer, discardBuffer, readBuffer, uploadData } from '../../../memory/buffer-pool.js';
|
|
5
|
+
import { retireIdleBuffersUnderDeviceBudget } from '../../../memory/sequence-retirement.js';
|
|
6
|
+
import { getDevice } from '../../../gpu/device.js';
|
|
5
7
|
import { getWeightDtype, getWeightMetadata, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
6
8
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
7
9
|
import { embed } from './embed.js';
|
|
@@ -121,6 +123,9 @@ export async function executePartitionLayers(state, input, signal) {
|
|
|
121
123
|
signal.throwIfAborted();
|
|
122
124
|
const prefill = state.currentSeqLen === 0;
|
|
123
125
|
const setup = resolvePartitionInput(state, input, prefill);
|
|
126
|
+
// A sequence's first prefill starts from its owned state under a device budget.
|
|
127
|
+
if (prefill) await retireIdleBuffersUnderDeviceBudget(getDevice());
|
|
128
|
+
signal.throwIfAborted();
|
|
124
129
|
if (setup.config.hyperConnections && !state.hyperConnectionSession) {
|
|
125
130
|
if (!state.dopplerLoader || !state.kvCache) throw new Error('Resident hyper-connections require the owning loader and attempt cache.');
|
|
126
131
|
state.hyperConnectionSession = await createHyperConnectionSession(setup.config, state.dopplerLoader,
|
|
@@ -45,6 +45,8 @@ export interface ContextLease {
|
|
|
45
45
|
export function supportsGenerationContext(state: ContextPipelineState): boolean;
|
|
46
46
|
export function createGenerationContext(state: ContextPipelineState, options: { maxBytes: number }): GenerationContext;
|
|
47
47
|
export function closeGenerationContexts(state: ContextPipelineState): void;
|
|
48
|
+
/** Retire idle pooled memory before a new generation's first prefill when a device budget is set. */
|
|
49
|
+
export function beginSequenceMemoryBoundary(): Promise<boolean>;
|
|
48
50
|
export function beginGenerationContext(state: ContextPipelineState, context: GenerationContext | undefined, options: ContextOptions): void;
|
|
49
51
|
export function generationContextInput(state: ContextPipelineState, inputIds: number[]): number[];
|
|
50
52
|
export function generationContextCheckpointLength(state: ContextPipelineState, prompt: import('./generator.js').PromptInput,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { getDevice } from '../../../gpu/device.js';
|
|
2
2
|
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
3
|
+
import { retireIdleBuffersUnderDeviceBudget } from '../../../memory/sequence-retirement.js';
|
|
3
4
|
import { createKVCache } from './init.js';
|
|
4
5
|
import { createLinearAttentionRuntime, resetLinearAttentionRuntime, resetLinearAttentionSequence } from './linear-attention.js';
|
|
5
6
|
import { isConvLayerType } from './layer-input-execution.js';
|
|
@@ -397,6 +398,12 @@ export async function checkpointGenerationContext(state, tokens) {
|
|
|
397
398
|
state.wholeTokenProgram?.program.invalidate('generation checkpoint parked');
|
|
398
399
|
}
|
|
399
400
|
|
|
401
|
+
// A new generation's first prefill starts from its owned state under a device budget.
|
|
402
|
+
/** @type {import('./sequence-state.js').beginSequenceMemoryBoundary} */
|
|
403
|
+
export async function beginSequenceMemoryBoundary() {
|
|
404
|
+
return retireIdleBuffersUnderDeviceBudget(getDevice());
|
|
405
|
+
}
|
|
406
|
+
|
|
400
407
|
/** @type {import('./sequence-state.js').prepareGenerationContextPrefill} */
|
|
401
408
|
export async function prepareGenerationContextPrefill(generator, prompt, inputIds, opts) {
|
|
402
409
|
const state = generator._state;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { getBufferPool } from './buffer-pool.js';
|
|
2
|
+
import { getDeviceMemorySnapshot } from './device-budget.js';
|
|
3
|
+
import { isDeviceLost } from './device-residency.js';
|
|
4
|
+
|
|
5
|
+
// Under an explicit device memory budget, idle pooled buffers and pending
|
|
6
|
+
// retirements are charged as live allocations. A sequence boundary retires them
|
|
7
|
+
// after confirmed GPU completion, so one sequence's working set is bounded by its
|
|
8
|
+
// own allocations rather than by buckets left over from earlier prompt lengths.
|
|
9
|
+
// Without a device budget the pool keeps its reuse behavior. Owned (active)
|
|
10
|
+
// buffers are never touched.
|
|
11
|
+
/** @type {typeof import('./sequence-retirement.js').retireIdleBuffersUnderDeviceBudget} */
|
|
12
|
+
export async function retireIdleBuffersUnderDeviceBudget(device) {
|
|
13
|
+
if (!device || isDeviceLost(device)) return false;
|
|
14
|
+
if ((getDeviceMemorySnapshot(device)?.maxBytes ?? null) === null) return false;
|
|
15
|
+
const pool = getBufferPool(device);
|
|
16
|
+
pool.clearPool();
|
|
17
|
+
await pool.settleDeferredDestruction();
|
|
18
|
+
return true;
|
|
19
|
+
}
|
package/src/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const DOPPLER_VERSION = '0.7.
|
|
1
|
+
export const DOPPLER_VERSION = '0.7.1';
|
|
2
2
|
export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;
|