doppler-gpu 0.2.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/README.md +9 -1
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +6 -5
  5. package/src/cli/doppler-quickstart.js +16 -2
  6. package/src/cli/doppler-serve.js +0 -0
  7. package/src/client/doppler-registry.json +17 -1
  8. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  9. package/src/config/kernels/registry.json +73 -0
  10. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  11. package/src/config/schema/benchmark.schema.d.ts +2 -0
  12. package/src/config/schema/benchmark.schema.js +1 -0
  13. package/src/config/schema/doppler.schema.d.ts +9 -1
  14. package/src/config/schema/doppler.schema.js +8 -1
  15. package/src/config/schema/index.d.ts +2 -0
  16. package/src/config/schema/index.js +1 -0
  17. package/src/config/schema/manifest.schema.d.ts +5 -0
  18. package/src/converter/quantizer.js +5 -4
  19. package/src/formats/rdrr/source-transform-contract.js +390 -0
  20. package/src/formats/rdrr/types.d.ts +8 -1
  21. package/src/formats/tflite/types.d.ts +8 -0
  22. package/src/formats/tflite/types.js +45 -0
  23. package/src/gpu/command-recorder.d.ts +7 -1
  24. package/src/gpu/command-recorder.js +41 -2
  25. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  26. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  27. package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
  28. package/src/gpu/kernels/index.d.ts +7 -0
  29. package/src/gpu/kernels/index.js +6 -0
  30. package/src/inference/browser-harness-text-helpers.d.ts +1 -0
  31. package/src/inference/browser-harness-text-helpers.js +44 -4
  32. package/src/inference/browser-harness.js +7 -0
  33. package/src/inference/pipelines/context.js +4 -2
  34. package/src/inference/pipelines/text/attention/types.js +9 -4
  35. package/src/inference/pipelines/text/embed.d.ts +1 -0
  36. package/src/inference/pipelines/text/embed.js +10 -4
  37. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
  38. package/src/inference/pipelines/text/generator-helpers.js +20 -1
  39. package/src/inference/pipelines/text/generator-steps.js +17 -8
  40. package/src/inference/pipelines/text/generator.js +51 -3
  41. package/src/inference/pipelines/text/init.d.ts +3 -0
  42. package/src/inference/pipelines/text/init.js +24 -3
  43. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  44. package/src/inference/pipelines/text/logits/gpu.js +146 -11
  45. package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
  46. package/src/inference/pipelines/text/probes.js +79 -1
  47. package/src/inference/pipelines/text/stage-names.js +9 -0
  48. package/src/inference/pipelines/text.js +10 -2
  49. package/src/loader/embedding-loader.js +20 -5
  50. package/src/loader/loader-types.d.ts +2 -0
  51. package/src/loader/manifest-config.d.ts +2 -7
  52. package/src/loader/per-layer-input-loader.js +114 -1
  53. package/src/loader/tensors/source-transform.d.ts +15 -0
  54. package/src/loader/tensors/source-transform.js +312 -34
  55. package/src/loader/tensors/tensor-reader.js +28 -2
  56. package/src/storage/artifact-storage-context.d.ts +1 -0
  57. package/src/storage/artifact-storage-context.js +158 -72
  58. package/src/storage/preflight.js +11 -0
  59. package/src/storage/quickstart-downloader.js +15 -4
  60. package/src/tooling/litert-package-runtime.js +369 -14
  61. package/src/tooling/node-source-runtime.js +256 -140
  62. package/src/tooling/source-runtime-bundle.d.ts +25 -0
  63. package/src/tooling/source-runtime-bundle.js +53 -1
  64. package/src/tooling/source-runtime-materializer.js +3 -1
  65. package/src/version.js +1 -1
@@ -64,6 +64,16 @@ function bytesToHex(bytes) {
64
64
  .join('');
65
65
  }
66
66
 
67
+ function getNodeFileCacheKey(fileRef) {
68
+ if (typeof fileRef === 'string') {
69
+ return fileRef;
70
+ }
71
+ if (fileRef instanceof URL) {
72
+ return fileRef.href;
73
+ }
74
+ return String(fileRef);
75
+ }
76
+
67
77
  function getSourceRuntimeMetadata(manifest) {
68
78
  const metadata = manifest?.metadata?.sourceRuntime;
69
79
  if (!metadata || typeof metadata !== 'object') {
@@ -158,6 +168,9 @@ function createPathBackedStorageContext(options) {
158
168
  const readBinary = typeof options?.readBinary === 'function'
159
169
  ? options.readBinary
160
170
  : null;
171
+ const close = typeof options?.close === 'function'
172
+ ? options.close
173
+ : null;
161
174
  const tokenizerJsonPath = options?.tokenizerJsonPath ?? null;
162
175
  const tokenizerModelPath = options?.tokenizerModelPath ?? null;
163
176
  const verifyHashes = options?.verifyHashes === true;
@@ -323,6 +336,7 @@ function createPathBackedStorageContext(options) {
323
336
  }
324
337
  : null,
325
338
  verifyHashes,
339
+ close,
326
340
  };
327
341
  }
328
342
 
@@ -358,6 +372,7 @@ export function createArtifactStorageContext(options = {}) {
358
372
  streamRange: options.streamRange,
359
373
  readText: options.readText,
360
374
  readBinary: options.readBinary,
375
+ close: options.close,
361
376
  tokenizerJsonPath: options.tokenizerJsonPath ?? tokenizerPaths.jsonPath,
362
377
  tokenizerModelPath: options.tokenizerModelPath ?? tokenizerPaths.modelPath,
363
378
  verifyHashes: options.verifyHashes === true,
@@ -413,32 +428,96 @@ async function resolveArtifactFileReference(root, relativePath) {
413
428
  throw new Error(`Unsupported local artifact root "${root}".`);
414
429
  }
415
430
 
416
- async function readNodeFileRange(fileRef, offset = 0, length = null) {
417
- const fs = await import('node:fs/promises');
418
- const handle = await fs.open(fileRef, 'r');
419
- try {
420
- const stats = await handle.stat();
421
- const start = Math.max(0, Math.floor(Number(offset) || 0));
422
- const fileSize = Number(stats.size);
423
- const end = length == null
424
- ? fileSize
425
- : Math.min(fileSize, start + Math.max(0, Math.floor(Number(length) || 0)));
426
- if (end <= start) {
427
- return new ArrayBuffer(0);
431
+ function createNodeFileAccess() {
432
+ const readers = new Map();
433
+
434
+ const getReader = (fileRef) => {
435
+ const cacheKey = getNodeFileCacheKey(fileRef);
436
+ let reader = readers.get(cacheKey);
437
+ if (reader) {
438
+ return reader;
428
439
  }
429
- const out = Buffer.allocUnsafe(end - start);
430
- let position = 0;
431
- while (position < out.length) {
432
- const { bytesRead } = await handle.read(out, position, out.length - position, start + position);
433
- if (bytesRead === 0) {
434
- break;
440
+ let handlePromise = null;
441
+ let sizePromise = null;
442
+ let closed = false;
443
+ const ensureHandle = async () => {
444
+ if (closed) {
445
+ throw new Error(`artifact storage context: file reader already closed for "${cacheKey}".`);
435
446
  }
436
- position += bytesRead;
437
- }
438
- return out.buffer.slice(out.byteOffset, out.byteOffset + position);
439
- } finally {
440
- await handle.close();
441
- }
447
+ if (!handlePromise) {
448
+ handlePromise = import('node:fs/promises')
449
+ .then((fs) => fs.open(fileRef, 'r'))
450
+ .catch((error) => {
451
+ handlePromise = null;
452
+ throw error;
453
+ });
454
+ }
455
+ return handlePromise;
456
+ };
457
+ const getSize = async () => {
458
+ if (!sizePromise) {
459
+ sizePromise = (async () => {
460
+ const handle = await ensureHandle();
461
+ const stats = await handle.stat();
462
+ return Number(stats.size);
463
+ })().catch((error) => {
464
+ sizePromise = null;
465
+ throw error;
466
+ });
467
+ }
468
+ return sizePromise;
469
+ };
470
+ reader = {
471
+ async readRange(offset = 0, length = null) {
472
+ const start = Math.max(0, Math.floor(Number(offset) || 0));
473
+ const fileSize = await getSize();
474
+ const end = length == null
475
+ ? fileSize
476
+ : Math.min(fileSize, start + Math.max(0, Math.floor(Number(length) || 0)));
477
+ if (end <= start) {
478
+ return new ArrayBuffer(0);
479
+ }
480
+ const handle = await ensureHandle();
481
+ const out = Buffer.allocUnsafe(end - start);
482
+ let position = 0;
483
+ while (position < out.length) {
484
+ const { bytesRead } = await handle.read(out, position, out.length - position, start + position);
485
+ if (bytesRead === 0) {
486
+ break;
487
+ }
488
+ position += bytesRead;
489
+ }
490
+ return out.buffer.slice(out.byteOffset, out.byteOffset + position);
491
+ },
492
+ async getSize() {
493
+ return getSize();
494
+ },
495
+ async close() {
496
+ closed = true;
497
+ const handle = await handlePromise;
498
+ handlePromise = null;
499
+ if (handle) {
500
+ await handle.close();
501
+ }
502
+ },
503
+ };
504
+ readers.set(cacheKey, reader);
505
+ return reader;
506
+ };
507
+
508
+ return {
509
+ async readRange(fileRef, offset = 0, length = null) {
510
+ return getReader(fileRef).readRange(offset, length);
511
+ },
512
+ async getSize(fileRef) {
513
+ return getReader(fileRef).getSize();
514
+ },
515
+ async close() {
516
+ const pending = Array.from(readers.values(), (reader) => reader.close());
517
+ readers.clear();
518
+ await Promise.all(pending);
519
+ },
520
+ };
442
521
  }
443
522
 
444
523
  export function createNodeFileArtifactStorageContext(baseUrl, manifest) {
@@ -452,55 +531,62 @@ export function createNodeFileArtifactStorageContext(baseUrl, manifest) {
452
531
  return null;
453
532
  }
454
533
 
455
- const readRange = async (relativePath, offset = 0, length = null) => {
456
- const fileRef = await resolveArtifactFileReference(root, relativePath);
457
- return readNodeFileRange(fileRef, offset, length);
458
- };
459
- const streamRange = async function* (relativePath, offset = 0, length = null, options = {}) {
460
- if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
461
- return;
462
- }
463
- const fileRef = await resolveArtifactFileReference(root, relativePath);
464
- const fs = await import('node:fs/promises');
465
- const { createReadStream } = await import('node:fs');
466
- const stats = await fs.stat(fileRef);
467
- const start = Math.max(0, Math.floor(offset));
468
- const end = Math.min(Number(stats.size), start + Math.floor(length));
469
- if (end <= start) {
470
- return;
471
- }
472
- const chunkBytesRaw = Number(options?.chunkBytes);
473
- const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
474
- ? Math.floor(chunkBytesRaw)
475
- : SOURCE_VERIFY_CHUNK_BYTES;
476
- const stream = createReadStream(fileRef, {
477
- start,
478
- end: end - 1,
479
- highWaterMark,
480
- });
481
- for await (const chunk of stream) {
482
- yield chunk;
483
- }
484
- };
485
- const readText = async (relativePath) => {
486
- const fileRef = await resolveArtifactFileReference(root, relativePath);
487
- const fs = await import('node:fs/promises');
488
- return fs.readFile(fileRef, 'utf8');
489
- };
490
- const readBinary = async (relativePath) => {
491
- const fileRef = await resolveArtifactFileReference(root, relativePath);
492
- const fs = await import('node:fs/promises');
493
- return fs.readFile(fileRef);
494
- };
534
+ const fileAccess = createNodeFileAccess();
495
535
 
496
- return createArtifactStorageContext({
497
- manifest,
498
- readRange,
499
- streamRange,
500
- readText,
501
- readBinary,
502
- verifyHashes: false,
503
- });
536
+ try {
537
+ const readRange = async (relativePath, offset = 0, length = null) => {
538
+ const fileRef = await resolveArtifactFileReference(root, relativePath);
539
+ return fileAccess.readRange(fileRef, offset, length);
540
+ };
541
+ const streamRange = async function* (relativePath, offset = 0, length = null, options = {}) {
542
+ if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
543
+ return;
544
+ }
545
+ const fileRef = await resolveArtifactFileReference(root, relativePath);
546
+ const { createReadStream } = await import('node:fs');
547
+ const fileSize = await fileAccess.getSize(fileRef);
548
+ const start = Math.max(0, Math.floor(offset));
549
+ const end = Math.min(fileSize, start + Math.floor(length));
550
+ if (end <= start) {
551
+ return;
552
+ }
553
+ const chunkBytesRaw = Number(options?.chunkBytes);
554
+ const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
555
+ ? Math.floor(chunkBytesRaw)
556
+ : SOURCE_VERIFY_CHUNK_BYTES;
557
+ const stream = createReadStream(fileRef, {
558
+ start,
559
+ end: end - 1,
560
+ highWaterMark,
561
+ });
562
+ for await (const chunk of stream) {
563
+ yield chunk;
564
+ }
565
+ };
566
+ const readText = async (relativePath) => {
567
+ const fileRef = await resolveArtifactFileReference(root, relativePath);
568
+ const fs = await import('node:fs/promises');
569
+ return fs.readFile(fileRef, 'utf8');
570
+ };
571
+ const readBinary = async (relativePath) => {
572
+ const fileRef = await resolveArtifactFileReference(root, relativePath);
573
+ const fs = await import('node:fs/promises');
574
+ return fs.readFile(fileRef);
575
+ };
576
+
577
+ return createArtifactStorageContext({
578
+ manifest,
579
+ readRange,
580
+ streamRange,
581
+ readText,
582
+ readBinary,
583
+ close: fileAccess.close,
584
+ verifyHashes: false,
585
+ });
586
+ } catch (error) {
587
+ fileAccess.close().catch(() => {});
588
+ throw error;
589
+ }
504
590
  }
505
591
 
506
592
  async function fetchBytes(url, offset = null, length = null) {
@@ -21,6 +21,16 @@ export const GEMMA_1B_REQUIREMENTS = {
21
21
  architecture: 'Gemma3ForCausalLM',
22
22
  };
23
23
 
24
+ const GEMMA_4_E2B_REQUIREMENTS = {
25
+ modelId: 'gemma-4-e2b-it-q4k-ehf16-af32',
26
+ displayName: 'Gemma 4 E2B IT (Q4K)',
27
+ downloadSize: 7094076955,
28
+ vramRequired: 12 * GB,
29
+ paramCount: 'E2B',
30
+ quantization: 'Q4K',
31
+ architecture: 'Gemma4ForConditionalGeneration',
32
+ };
33
+
24
34
  const GEMMA_270M_REQUIREMENTS = {
25
35
  modelId: 'gemma-3-270m-it-q4k-ehf16-af32',
26
36
  displayName: 'Gemma 3 270M IT (Q4K)',
@@ -66,6 +76,7 @@ export const MODEL_REQUIREMENTS = {
66
76
  'gemma-3-270m-it-q4k-ehf16-af32': GEMMA_270M_REQUIREMENTS,
67
77
  'google-embeddinggemma-300m-q4k-ehf16-af32': EMBEDDINGGEMMA_300M_REQUIREMENTS,
68
78
  'gemma-3-1b-it-q4k-ehf16-af32': GEMMA_1B_REQUIREMENTS,
79
+ 'gemma-4-e2b-it-q4k-ehf16-af32': GEMMA_4_E2B_REQUIREMENTS,
69
80
  'qwen-3-5-0-8b-q4k-ehaf16': QWEN_08B_REQUIREMENTS,
70
81
  'qwen-3-5-2b-q4k-ehaf16': QWEN_2B_REQUIREMENTS,
71
82
  };
@@ -34,7 +34,7 @@ export const QUICKSTART_MODELS = {
34
34
  baseUrl: null,
35
35
  hf: {
36
36
  repoId: 'Clocksmith/rdrr',
37
- revision: 'f242e44c9a35e72186aa61e4cc0c9873f0596741',
37
+ revision: '732b2bee08cfe315e5c0df5af30e7ad795721b48',
38
38
  path: 'models/gemma-3-270m-it-q4k-ehf16-af32',
39
39
  },
40
40
  requirements: MODEL_REQUIREMENTS['gemma-3-270m-it-q4k-ehf16-af32'],
@@ -56,18 +56,29 @@ export const QUICKSTART_MODELS = {
56
56
  baseUrl: null,
57
57
  hf: {
58
58
  repoId: 'Clocksmith/rdrr',
59
- revision: 'f242e44c9a35e72186aa61e4cc0c9873f0596741',
59
+ revision: 'a6651ea48c5ebc420ebc1c6972979049b72db96e',
60
60
  path: 'models/gemma-3-1b-it-q4k-ehf16-af32',
61
61
  },
62
62
  requirements: MODEL_REQUIREMENTS['gemma-3-1b-it-q4k-ehf16-af32'],
63
63
  },
64
+ 'gemma-4-e2b-it-q4k-ehf16-af32': {
65
+ modelId: 'gemma-4-e2b-it-q4k-ehf16-af32',
66
+ displayName: 'Gemma 4 E2B IT (Q4K)',
67
+ baseUrl: null,
68
+ hf: {
69
+ repoId: 'Clocksmith/rdrr',
70
+ revision: '88fa42770918e0c61bfcd67c6a54952e079b5d36',
71
+ path: 'models/gemma-4-e2b-it-q4k-ehf16-af32',
72
+ },
73
+ requirements: MODEL_REQUIREMENTS['gemma-4-e2b-it-q4k-ehf16-af32'],
74
+ },
64
75
  'qwen-3-5-0-8b-q4k-ehaf16': {
65
76
  modelId: 'qwen-3-5-0-8b-q4k-ehaf16',
66
77
  displayName: 'Qwen 3.5 0.8B (Q4K)',
67
78
  baseUrl: null,
68
79
  hf: {
69
80
  repoId: 'Clocksmith/rdrr',
70
- revision: 'f242e44c9a35e72186aa61e4cc0c9873f0596741',
81
+ revision: '0b7e01c70500b78cbe69c66cb5601dd00941f9c9',
71
82
  path: 'models/qwen-3-5-0-8b-q4k-ehaf16',
72
83
  },
73
84
  requirements: MODEL_REQUIREMENTS['qwen-3-5-0-8b-q4k-ehaf16'],
@@ -78,7 +89,7 @@ export const QUICKSTART_MODELS = {
78
89
  baseUrl: null,
79
90
  hf: {
80
91
  repoId: 'Clocksmith/rdrr',
81
- revision: 'f242e44c9a35e72186aa61e4cc0c9873f0596741',
92
+ revision: '1e14085a6d8e14035ec64966d6cd44250be3df83',
82
93
  path: 'models/qwen-3-5-2b-q4k-ehaf16',
83
94
  },
84
95
  requirements: MODEL_REQUIREMENTS['qwen-3-5-2b-q4k-ehaf16'],