doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import type { Tensor } from '../../../../gpu/tensor.js';
|
|
11
|
-
import type { CpuWeightBuffer } from '../../../../gpu/weight-buffer.js';
|
|
11
|
+
import type { CpuWeightBuffer, CpuTensorRangeSource } from '../../../../gpu/weight-buffer.js';
|
|
12
12
|
import type { CommandRecorder } from '../../../../gpu/command-recorder.js';
|
|
13
13
|
import type { LargeWeightConfigSchema, ProbeConfigSchema, KernelPathSchema } from '../../../../config/schema/index.js';
|
|
14
14
|
import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
@@ -32,13 +32,14 @@ export function resolveLmHeadChunkRows(
|
|
|
32
32
|
* Extract a chunk of the LM head weight matrix.
|
|
33
33
|
*/
|
|
34
34
|
export function extractLmHeadChunk(
|
|
35
|
-
data: Float32Array,
|
|
35
|
+
data: Float32Array | Uint16Array | CpuTensorRangeSource,
|
|
36
36
|
layout: 'row' | 'column',
|
|
37
37
|
hiddenSize: number,
|
|
38
38
|
vocabSize: number,
|
|
39
39
|
rowOffset: number,
|
|
40
|
-
rowCount: number
|
|
41
|
-
|
|
40
|
+
rowCount: number,
|
|
41
|
+
sourceDtype?: 'f16' | 'f32' | 'bf16'
|
|
42
|
+
): Promise<Float32Array>;
|
|
42
43
|
|
|
43
44
|
/**
|
|
44
45
|
* Write chunk logits to the full logits buffer.
|
|
@@ -21,6 +21,7 @@ import { runProbes } from '../probes.js';
|
|
|
21
21
|
import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
|
|
22
22
|
import { f16BufferToF32 } from './cpu.js';
|
|
23
23
|
import { readBufferWithCleanup } from './utils.js';
|
|
24
|
+
import { f16ToF32 } from '../../../../loader/dtype-utils.js';
|
|
24
25
|
|
|
25
26
|
function shouldForceStableF32Logits(config, inputDtype) {
|
|
26
27
|
if (inputDtype !== 'f16') {
|
|
@@ -140,6 +141,101 @@ function createStableF32LogitsKernelPath(kernelPath) {
|
|
|
140
141
|
};
|
|
141
142
|
}
|
|
142
143
|
|
|
144
|
+
const bf16ScratchU32 = new Uint32Array(1);
|
|
145
|
+
const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
|
|
146
|
+
|
|
147
|
+
function bf16ToF32(value) {
|
|
148
|
+
bf16ScratchU32[0] = (value & 0xffff) << 16;
|
|
149
|
+
return bf16ScratchF32[0];
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function isRangeBackedCpuWeightSource(value) {
|
|
153
|
+
return (
|
|
154
|
+
typeof value === 'object'
|
|
155
|
+
&& value !== null
|
|
156
|
+
&& value.kind === 'tensor_range_source'
|
|
157
|
+
&& typeof value.loadRange === 'function'
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function normalizeRangeBytes(value, label) {
|
|
162
|
+
if (value instanceof Uint8Array) return value;
|
|
163
|
+
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
164
|
+
if (ArrayBuffer.isView(value)) {
|
|
165
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
166
|
+
}
|
|
167
|
+
throw new Error(
|
|
168
|
+
`[Logits] ${label} returned unsupported byte payload type "${value?.constructor?.name ?? typeof value}".`
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function decodeChunkIntoOutput(bytes, sourceDtype, output, dstOffset, valueCount) {
|
|
173
|
+
if (sourceDtype === 'f16') {
|
|
174
|
+
const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
|
|
175
|
+
for (let index = 0; index < valueCount; index += 1) {
|
|
176
|
+
output[dstOffset + index] = f16ToF32(values[index]);
|
|
177
|
+
}
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
if (sourceDtype === 'bf16') {
|
|
181
|
+
const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
|
|
182
|
+
for (let index = 0; index < valueCount; index += 1) {
|
|
183
|
+
output[dstOffset + index] = bf16ToF32(values[index]);
|
|
184
|
+
}
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
if (((bytes.byteOffset % 4) === 0) && ((bytes.byteLength % 4) === 0)) {
|
|
188
|
+
output.set(new Float32Array(bytes.buffer, bytes.byteOffset, valueCount), dstOffset);
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
192
|
+
for (let index = 0; index < valueCount; index += 1) {
|
|
193
|
+
output[dstOffset + index] = view.getFloat32(index * 4, true);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function readTypedLmHeadChunk(data, layout, hiddenSize, vocabSize, rowOffset, rowCount, sourceDtype) {
|
|
198
|
+
if (data instanceof Float32Array) {
|
|
199
|
+
if (layout === 'row') {
|
|
200
|
+
const start = rowOffset * hiddenSize;
|
|
201
|
+
return data.subarray(start, start + rowCount * hiddenSize);
|
|
202
|
+
}
|
|
203
|
+
const chunk = new Float32Array(hiddenSize * rowCount);
|
|
204
|
+
for (let k = 0; k < hiddenSize; k++) {
|
|
205
|
+
const srcOffset = k * vocabSize + rowOffset;
|
|
206
|
+
const dstOffset = k * rowCount;
|
|
207
|
+
chunk.set(data.subarray(srcOffset, srcOffset + rowCount), dstOffset);
|
|
208
|
+
}
|
|
209
|
+
return chunk;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (!(data instanceof Uint16Array)) {
|
|
213
|
+
throw new Error(
|
|
214
|
+
`[Logits] Unsupported CPU LM head chunk source type "${data?.constructor?.name ?? typeof data}".`
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const chunk = new Float32Array(hiddenSize * rowCount);
|
|
219
|
+
if (layout === 'row') {
|
|
220
|
+
const start = rowOffset * hiddenSize;
|
|
221
|
+
for (let index = 0; index < rowCount * hiddenSize; index += 1) {
|
|
222
|
+
const raw = data[start + index];
|
|
223
|
+
chunk[index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
|
|
224
|
+
}
|
|
225
|
+
return chunk;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
for (let k = 0; k < hiddenSize; k += 1) {
|
|
229
|
+
const srcOffset = k * vocabSize + rowOffset;
|
|
230
|
+
const dstOffset = k * rowCount;
|
|
231
|
+
for (let index = 0; index < rowCount; index += 1) {
|
|
232
|
+
const raw = data[srcOffset + index];
|
|
233
|
+
chunk[dstOffset + index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return chunk;
|
|
237
|
+
}
|
|
238
|
+
|
|
143
239
|
|
|
144
240
|
export function resolveCpuWeightDims(lmHead) {
|
|
145
241
|
if (lmHead.shape.length !== 2) {
|
|
@@ -184,24 +280,62 @@ export function resolveLmHeadChunkRows(
|
|
|
184
280
|
}
|
|
185
281
|
|
|
186
282
|
|
|
187
|
-
export function extractLmHeadChunk(
|
|
283
|
+
export async function extractLmHeadChunk(
|
|
188
284
|
data,
|
|
189
285
|
layout,
|
|
190
286
|
hiddenSize,
|
|
191
287
|
vocabSize,
|
|
192
288
|
rowOffset,
|
|
193
|
-
rowCount
|
|
289
|
+
rowCount,
|
|
290
|
+
sourceDtype = 'f32'
|
|
194
291
|
) {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
292
|
+
const normalizedSourceDtype = String(sourceDtype ?? 'f32').toLowerCase();
|
|
293
|
+
if (normalizedSourceDtype !== 'f32' && normalizedSourceDtype !== 'f16' && normalizedSourceDtype !== 'bf16') {
|
|
294
|
+
throw new Error(`[Logits] Unsupported CPU LM head source dtype "${sourceDtype}".`);
|
|
295
|
+
}
|
|
296
|
+
if (!isRangeBackedCpuWeightSource(data)) {
|
|
297
|
+
return readTypedLmHeadChunk(
|
|
298
|
+
data,
|
|
299
|
+
layout,
|
|
300
|
+
hiddenSize,
|
|
301
|
+
vocabSize,
|
|
302
|
+
rowOffset,
|
|
303
|
+
rowCount,
|
|
304
|
+
normalizedSourceDtype
|
|
305
|
+
);
|
|
198
306
|
}
|
|
199
307
|
|
|
308
|
+
const bytesPerElement = normalizedSourceDtype === 'f32' ? 4 : 2;
|
|
200
309
|
const chunk = new Float32Array(hiddenSize * rowCount);
|
|
201
|
-
|
|
202
|
-
const
|
|
203
|
-
const
|
|
204
|
-
|
|
310
|
+
if (layout === 'row') {
|
|
311
|
+
const byteOffset = rowOffset * hiddenSize * bytesPerElement;
|
|
312
|
+
const byteLength = rowCount * hiddenSize * bytesPerElement;
|
|
313
|
+
const bytes = normalizeRangeBytes(
|
|
314
|
+
await data.loadRange(byteOffset, byteLength),
|
|
315
|
+
'CPU LM head range source'
|
|
316
|
+
);
|
|
317
|
+
if (bytes.byteLength !== byteLength) {
|
|
318
|
+
throw new Error(
|
|
319
|
+
`[Logits] CPU LM head range source returned ${bytes.byteLength} bytes, expected ${byteLength}.`
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, 0, rowCount * hiddenSize);
|
|
323
|
+
return chunk;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
for (let k = 0; k < hiddenSize; k += 1) {
|
|
327
|
+
const byteOffset = (k * vocabSize + rowOffset) * bytesPerElement;
|
|
328
|
+
const byteLength = rowCount * bytesPerElement;
|
|
329
|
+
const bytes = normalizeRangeBytes(
|
|
330
|
+
await data.loadRange(byteOffset, byteLength),
|
|
331
|
+
`CPU LM head range source column ${k}`
|
|
332
|
+
);
|
|
333
|
+
if (bytes.byteLength !== byteLength) {
|
|
334
|
+
throw new Error(
|
|
335
|
+
`[Logits] CPU LM head range source returned ${bytes.byteLength} bytes for column ${k}, expected ${byteLength}.`
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, k * rowCount, rowCount);
|
|
205
339
|
}
|
|
206
340
|
return chunk;
|
|
207
341
|
}
|
|
@@ -271,13 +405,14 @@ export async function computeChunkedLogitsGPU(
|
|
|
271
405
|
|
|
272
406
|
for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
|
|
273
407
|
const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
|
|
274
|
-
const chunkData = extractLmHeadChunk(
|
|
408
|
+
const chunkData = await extractLmHeadChunk(
|
|
275
409
|
lmHead.data,
|
|
276
410
|
lmHead.layout,
|
|
277
411
|
hiddenSize,
|
|
278
412
|
weightVocabSize,
|
|
279
413
|
rowOffset,
|
|
280
|
-
rowCount
|
|
414
|
+
rowCount,
|
|
415
|
+
lmHead.dtype
|
|
281
416
|
);
|
|
282
417
|
|
|
283
418
|
const f32Buffer = acquireBuffer(chunkData.byteLength, undefined, 'lm_head_chunk_f32');
|