doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -64,6 +64,16 @@ function bytesToHex(bytes) {
|
|
|
64
64
|
.join('');
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
function getNodeFileCacheKey(fileRef) {
|
|
68
|
+
if (typeof fileRef === 'string') {
|
|
69
|
+
return fileRef;
|
|
70
|
+
}
|
|
71
|
+
if (fileRef instanceof URL) {
|
|
72
|
+
return fileRef.href;
|
|
73
|
+
}
|
|
74
|
+
return String(fileRef);
|
|
75
|
+
}
|
|
76
|
+
|
|
67
77
|
function getSourceRuntimeMetadata(manifest) {
|
|
68
78
|
const metadata = manifest?.metadata?.sourceRuntime;
|
|
69
79
|
if (!metadata || typeof metadata !== 'object') {
|
|
@@ -158,6 +168,9 @@ function createPathBackedStorageContext(options) {
|
|
|
158
168
|
const readBinary = typeof options?.readBinary === 'function'
|
|
159
169
|
? options.readBinary
|
|
160
170
|
: null;
|
|
171
|
+
const close = typeof options?.close === 'function'
|
|
172
|
+
? options.close
|
|
173
|
+
: null;
|
|
161
174
|
const tokenizerJsonPath = options?.tokenizerJsonPath ?? null;
|
|
162
175
|
const tokenizerModelPath = options?.tokenizerModelPath ?? null;
|
|
163
176
|
const verifyHashes = options?.verifyHashes === true;
|
|
@@ -323,6 +336,7 @@ function createPathBackedStorageContext(options) {
|
|
|
323
336
|
}
|
|
324
337
|
: null,
|
|
325
338
|
verifyHashes,
|
|
339
|
+
close,
|
|
326
340
|
};
|
|
327
341
|
}
|
|
328
342
|
|
|
@@ -358,6 +372,7 @@ export function createArtifactStorageContext(options = {}) {
|
|
|
358
372
|
streamRange: options.streamRange,
|
|
359
373
|
readText: options.readText,
|
|
360
374
|
readBinary: options.readBinary,
|
|
375
|
+
close: options.close,
|
|
361
376
|
tokenizerJsonPath: options.tokenizerJsonPath ?? tokenizerPaths.jsonPath,
|
|
362
377
|
tokenizerModelPath: options.tokenizerModelPath ?? tokenizerPaths.modelPath,
|
|
363
378
|
verifyHashes: options.verifyHashes === true,
|
|
@@ -413,32 +428,96 @@ async function resolveArtifactFileReference(root, relativePath) {
|
|
|
413
428
|
throw new Error(`Unsupported local artifact root "${root}".`);
|
|
414
429
|
}
|
|
415
430
|
|
|
416
|
-
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
const
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
? fileSize
|
|
425
|
-
: Math.min(fileSize, start + Math.max(0, Math.floor(Number(length) || 0)));
|
|
426
|
-
if (end <= start) {
|
|
427
|
-
return new ArrayBuffer(0);
|
|
431
|
+
function createNodeFileAccess() {
|
|
432
|
+
const readers = new Map();
|
|
433
|
+
|
|
434
|
+
const getReader = (fileRef) => {
|
|
435
|
+
const cacheKey = getNodeFileCacheKey(fileRef);
|
|
436
|
+
let reader = readers.get(cacheKey);
|
|
437
|
+
if (reader) {
|
|
438
|
+
return reader;
|
|
428
439
|
}
|
|
429
|
-
|
|
430
|
-
let
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
if (
|
|
434
|
-
|
|
440
|
+
let handlePromise = null;
|
|
441
|
+
let sizePromise = null;
|
|
442
|
+
let closed = false;
|
|
443
|
+
const ensureHandle = async () => {
|
|
444
|
+
if (closed) {
|
|
445
|
+
throw new Error(`artifact storage context: file reader already closed for "${cacheKey}".`);
|
|
435
446
|
}
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
447
|
+
if (!handlePromise) {
|
|
448
|
+
handlePromise = import('node:fs/promises')
|
|
449
|
+
.then((fs) => fs.open(fileRef, 'r'))
|
|
450
|
+
.catch((error) => {
|
|
451
|
+
handlePromise = null;
|
|
452
|
+
throw error;
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
return handlePromise;
|
|
456
|
+
};
|
|
457
|
+
const getSize = async () => {
|
|
458
|
+
if (!sizePromise) {
|
|
459
|
+
sizePromise = (async () => {
|
|
460
|
+
const handle = await ensureHandle();
|
|
461
|
+
const stats = await handle.stat();
|
|
462
|
+
return Number(stats.size);
|
|
463
|
+
})().catch((error) => {
|
|
464
|
+
sizePromise = null;
|
|
465
|
+
throw error;
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
return sizePromise;
|
|
469
|
+
};
|
|
470
|
+
reader = {
|
|
471
|
+
async readRange(offset = 0, length = null) {
|
|
472
|
+
const start = Math.max(0, Math.floor(Number(offset) || 0));
|
|
473
|
+
const fileSize = await getSize();
|
|
474
|
+
const end = length == null
|
|
475
|
+
? fileSize
|
|
476
|
+
: Math.min(fileSize, start + Math.max(0, Math.floor(Number(length) || 0)));
|
|
477
|
+
if (end <= start) {
|
|
478
|
+
return new ArrayBuffer(0);
|
|
479
|
+
}
|
|
480
|
+
const handle = await ensureHandle();
|
|
481
|
+
const out = Buffer.allocUnsafe(end - start);
|
|
482
|
+
let position = 0;
|
|
483
|
+
while (position < out.length) {
|
|
484
|
+
const { bytesRead } = await handle.read(out, position, out.length - position, start + position);
|
|
485
|
+
if (bytesRead === 0) {
|
|
486
|
+
break;
|
|
487
|
+
}
|
|
488
|
+
position += bytesRead;
|
|
489
|
+
}
|
|
490
|
+
return out.buffer.slice(out.byteOffset, out.byteOffset + position);
|
|
491
|
+
},
|
|
492
|
+
async getSize() {
|
|
493
|
+
return getSize();
|
|
494
|
+
},
|
|
495
|
+
async close() {
|
|
496
|
+
closed = true;
|
|
497
|
+
const handle = await handlePromise;
|
|
498
|
+
handlePromise = null;
|
|
499
|
+
if (handle) {
|
|
500
|
+
await handle.close();
|
|
501
|
+
}
|
|
502
|
+
},
|
|
503
|
+
};
|
|
504
|
+
readers.set(cacheKey, reader);
|
|
505
|
+
return reader;
|
|
506
|
+
};
|
|
507
|
+
|
|
508
|
+
return {
|
|
509
|
+
async readRange(fileRef, offset = 0, length = null) {
|
|
510
|
+
return getReader(fileRef).readRange(offset, length);
|
|
511
|
+
},
|
|
512
|
+
async getSize(fileRef) {
|
|
513
|
+
return getReader(fileRef).getSize();
|
|
514
|
+
},
|
|
515
|
+
async close() {
|
|
516
|
+
const pending = Array.from(readers.values(), (reader) => reader.close());
|
|
517
|
+
readers.clear();
|
|
518
|
+
await Promise.all(pending);
|
|
519
|
+
},
|
|
520
|
+
};
|
|
442
521
|
}
|
|
443
522
|
|
|
444
523
|
export function createNodeFileArtifactStorageContext(baseUrl, manifest) {
|
|
@@ -452,55 +531,62 @@ export function createNodeFileArtifactStorageContext(baseUrl, manifest) {
|
|
|
452
531
|
return null;
|
|
453
532
|
}
|
|
454
533
|
|
|
455
|
-
const
|
|
456
|
-
const fileRef = await resolveArtifactFileReference(root, relativePath);
|
|
457
|
-
return readNodeFileRange(fileRef, offset, length);
|
|
458
|
-
};
|
|
459
|
-
const streamRange = async function* (relativePath, offset = 0, length = null, options = {}) {
|
|
460
|
-
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
461
|
-
return;
|
|
462
|
-
}
|
|
463
|
-
const fileRef = await resolveArtifactFileReference(root, relativePath);
|
|
464
|
-
const fs = await import('node:fs/promises');
|
|
465
|
-
const { createReadStream } = await import('node:fs');
|
|
466
|
-
const stats = await fs.stat(fileRef);
|
|
467
|
-
const start = Math.max(0, Math.floor(offset));
|
|
468
|
-
const end = Math.min(Number(stats.size), start + Math.floor(length));
|
|
469
|
-
if (end <= start) {
|
|
470
|
-
return;
|
|
471
|
-
}
|
|
472
|
-
const chunkBytesRaw = Number(options?.chunkBytes);
|
|
473
|
-
const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
|
|
474
|
-
? Math.floor(chunkBytesRaw)
|
|
475
|
-
: SOURCE_VERIFY_CHUNK_BYTES;
|
|
476
|
-
const stream = createReadStream(fileRef, {
|
|
477
|
-
start,
|
|
478
|
-
end: end - 1,
|
|
479
|
-
highWaterMark,
|
|
480
|
-
});
|
|
481
|
-
for await (const chunk of stream) {
|
|
482
|
-
yield chunk;
|
|
483
|
-
}
|
|
484
|
-
};
|
|
485
|
-
const readText = async (relativePath) => {
|
|
486
|
-
const fileRef = await resolveArtifactFileReference(root, relativePath);
|
|
487
|
-
const fs = await import('node:fs/promises');
|
|
488
|
-
return fs.readFile(fileRef, 'utf8');
|
|
489
|
-
};
|
|
490
|
-
const readBinary = async (relativePath) => {
|
|
491
|
-
const fileRef = await resolveArtifactFileReference(root, relativePath);
|
|
492
|
-
const fs = await import('node:fs/promises');
|
|
493
|
-
return fs.readFile(fileRef);
|
|
494
|
-
};
|
|
534
|
+
const fileAccess = createNodeFileAccess();
|
|
495
535
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
536
|
+
try {
|
|
537
|
+
const readRange = async (relativePath, offset = 0, length = null) => {
|
|
538
|
+
const fileRef = await resolveArtifactFileReference(root, relativePath);
|
|
539
|
+
return fileAccess.readRange(fileRef, offset, length);
|
|
540
|
+
};
|
|
541
|
+
const streamRange = async function* (relativePath, offset = 0, length = null, options = {}) {
|
|
542
|
+
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
const fileRef = await resolveArtifactFileReference(root, relativePath);
|
|
546
|
+
const { createReadStream } = await import('node:fs');
|
|
547
|
+
const fileSize = await fileAccess.getSize(fileRef);
|
|
548
|
+
const start = Math.max(0, Math.floor(offset));
|
|
549
|
+
const end = Math.min(fileSize, start + Math.floor(length));
|
|
550
|
+
if (end <= start) {
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
const chunkBytesRaw = Number(options?.chunkBytes);
|
|
554
|
+
const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
|
|
555
|
+
? Math.floor(chunkBytesRaw)
|
|
556
|
+
: SOURCE_VERIFY_CHUNK_BYTES;
|
|
557
|
+
const stream = createReadStream(fileRef, {
|
|
558
|
+
start,
|
|
559
|
+
end: end - 1,
|
|
560
|
+
highWaterMark,
|
|
561
|
+
});
|
|
562
|
+
for await (const chunk of stream) {
|
|
563
|
+
yield chunk;
|
|
564
|
+
}
|
|
565
|
+
};
|
|
566
|
+
const readText = async (relativePath) => {
|
|
567
|
+
const fileRef = await resolveArtifactFileReference(root, relativePath);
|
|
568
|
+
const fs = await import('node:fs/promises');
|
|
569
|
+
return fs.readFile(fileRef, 'utf8');
|
|
570
|
+
};
|
|
571
|
+
const readBinary = async (relativePath) => {
|
|
572
|
+
const fileRef = await resolveArtifactFileReference(root, relativePath);
|
|
573
|
+
const fs = await import('node:fs/promises');
|
|
574
|
+
return fs.readFile(fileRef);
|
|
575
|
+
};
|
|
576
|
+
|
|
577
|
+
return createArtifactStorageContext({
|
|
578
|
+
manifest,
|
|
579
|
+
readRange,
|
|
580
|
+
streamRange,
|
|
581
|
+
readText,
|
|
582
|
+
readBinary,
|
|
583
|
+
close: fileAccess.close,
|
|
584
|
+
verifyHashes: false,
|
|
585
|
+
});
|
|
586
|
+
} catch (error) {
|
|
587
|
+
fileAccess.close().catch(() => {});
|
|
588
|
+
throw error;
|
|
589
|
+
}
|
|
504
590
|
}
|
|
505
591
|
|
|
506
592
|
async function fetchBytes(url, offset = null, length = null) {
|
package/src/storage/preflight.js
CHANGED
|
@@ -21,6 +21,16 @@ export const GEMMA_1B_REQUIREMENTS = {
|
|
|
21
21
|
architecture: 'Gemma3ForCausalLM',
|
|
22
22
|
};
|
|
23
23
|
|
|
24
|
+
const GEMMA_4_E2B_REQUIREMENTS = {
|
|
25
|
+
modelId: 'gemma-4-e2b-it-q4k-ehf16-af32',
|
|
26
|
+
displayName: 'Gemma 4 E2B IT (Q4K)',
|
|
27
|
+
downloadSize: 7094076955,
|
|
28
|
+
vramRequired: 12 * GB,
|
|
29
|
+
paramCount: 'E2B',
|
|
30
|
+
quantization: 'Q4K',
|
|
31
|
+
architecture: 'Gemma4ForConditionalGeneration',
|
|
32
|
+
};
|
|
33
|
+
|
|
24
34
|
const GEMMA_270M_REQUIREMENTS = {
|
|
25
35
|
modelId: 'gemma-3-270m-it-q4k-ehf16-af32',
|
|
26
36
|
displayName: 'Gemma 3 270M IT (Q4K)',
|
|
@@ -66,6 +76,7 @@ export const MODEL_REQUIREMENTS = {
|
|
|
66
76
|
'gemma-3-270m-it-q4k-ehf16-af32': GEMMA_270M_REQUIREMENTS,
|
|
67
77
|
'google-embeddinggemma-300m-q4k-ehf16-af32': EMBEDDINGGEMMA_300M_REQUIREMENTS,
|
|
68
78
|
'gemma-3-1b-it-q4k-ehf16-af32': GEMMA_1B_REQUIREMENTS,
|
|
79
|
+
'gemma-4-e2b-it-q4k-ehf16-af32': GEMMA_4_E2B_REQUIREMENTS,
|
|
69
80
|
'qwen-3-5-0-8b-q4k-ehaf16': QWEN_08B_REQUIREMENTS,
|
|
70
81
|
'qwen-3-5-2b-q4k-ehaf16': QWEN_2B_REQUIREMENTS,
|
|
71
82
|
};
|
|
@@ -34,7 +34,7 @@ export const QUICKSTART_MODELS = {
|
|
|
34
34
|
baseUrl: null,
|
|
35
35
|
hf: {
|
|
36
36
|
repoId: 'Clocksmith/rdrr',
|
|
37
|
-
revision: '
|
|
37
|
+
revision: '732b2bee08cfe315e5c0df5af30e7ad795721b48',
|
|
38
38
|
path: 'models/gemma-3-270m-it-q4k-ehf16-af32',
|
|
39
39
|
},
|
|
40
40
|
requirements: MODEL_REQUIREMENTS['gemma-3-270m-it-q4k-ehf16-af32'],
|
|
@@ -56,18 +56,29 @@ export const QUICKSTART_MODELS = {
|
|
|
56
56
|
baseUrl: null,
|
|
57
57
|
hf: {
|
|
58
58
|
repoId: 'Clocksmith/rdrr',
|
|
59
|
-
revision: '
|
|
59
|
+
revision: 'a6651ea48c5ebc420ebc1c6972979049b72db96e',
|
|
60
60
|
path: 'models/gemma-3-1b-it-q4k-ehf16-af32',
|
|
61
61
|
},
|
|
62
62
|
requirements: MODEL_REQUIREMENTS['gemma-3-1b-it-q4k-ehf16-af32'],
|
|
63
63
|
},
|
|
64
|
+
'gemma-4-e2b-it-q4k-ehf16-af32': {
|
|
65
|
+
modelId: 'gemma-4-e2b-it-q4k-ehf16-af32',
|
|
66
|
+
displayName: 'Gemma 4 E2B IT (Q4K)',
|
|
67
|
+
baseUrl: null,
|
|
68
|
+
hf: {
|
|
69
|
+
repoId: 'Clocksmith/rdrr',
|
|
70
|
+
revision: '88fa42770918e0c61bfcd67c6a54952e079b5d36',
|
|
71
|
+
path: 'models/gemma-4-e2b-it-q4k-ehf16-af32',
|
|
72
|
+
},
|
|
73
|
+
requirements: MODEL_REQUIREMENTS['gemma-4-e2b-it-q4k-ehf16-af32'],
|
|
74
|
+
},
|
|
64
75
|
'qwen-3-5-0-8b-q4k-ehaf16': {
|
|
65
76
|
modelId: 'qwen-3-5-0-8b-q4k-ehaf16',
|
|
66
77
|
displayName: 'Qwen 3.5 0.8B (Q4K)',
|
|
67
78
|
baseUrl: null,
|
|
68
79
|
hf: {
|
|
69
80
|
repoId: 'Clocksmith/rdrr',
|
|
70
|
-
revision: '
|
|
81
|
+
revision: '0b7e01c70500b78cbe69c66cb5601dd00941f9c9',
|
|
71
82
|
path: 'models/qwen-3-5-0-8b-q4k-ehaf16',
|
|
72
83
|
},
|
|
73
84
|
requirements: MODEL_REQUIREMENTS['qwen-3-5-0-8b-q4k-ehaf16'],
|
|
@@ -78,7 +89,7 @@ export const QUICKSTART_MODELS = {
|
|
|
78
89
|
baseUrl: null,
|
|
79
90
|
hf: {
|
|
80
91
|
repoId: 'Clocksmith/rdrr',
|
|
81
|
-
revision: '
|
|
92
|
+
revision: '1e14085a6d8e14035ec64966d6cd44250be3df83',
|
|
82
93
|
path: 'models/qwen-3-5-2b-q4k-ehaf16',
|
|
83
94
|
},
|
|
84
95
|
requirements: MODEL_REQUIREMENTS['qwen-3-5-2b-q4k-ehaf16'],
|