doppler-gpu 0.2.2 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +8 -0
  2. package/assets/doppler.svg +1601 -0
  3. package/package.json +6 -5
  4. package/src/cli/doppler-quickstart.js +16 -2
  5. package/src/cli/doppler-serve.js +0 -0
  6. package/src/client/doppler-registry.json +17 -1
  7. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  8. package/src/config/kernels/registry.json +73 -0
  9. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  10. package/src/config/schema/benchmark.schema.d.ts +2 -0
  11. package/src/config/schema/benchmark.schema.js +1 -0
  12. package/src/config/schema/doppler.schema.d.ts +9 -1
  13. package/src/config/schema/doppler.schema.js +8 -1
  14. package/src/config/schema/index.d.ts +2 -0
  15. package/src/config/schema/index.js +1 -0
  16. package/src/config/schema/manifest.schema.d.ts +5 -0
  17. package/src/converter/quantizer.js +5 -4
  18. package/src/formats/rdrr/source-transform-contract.js +390 -0
  19. package/src/formats/rdrr/types.d.ts +8 -1
  20. package/src/formats/tflite/types.d.ts +8 -0
  21. package/src/formats/tflite/types.js +45 -0
  22. package/src/gpu/command-recorder.d.ts +7 -1
  23. package/src/gpu/command-recorder.js +41 -2
  24. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  25. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  26. package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
  27. package/src/gpu/kernels/index.d.ts +7 -0
  28. package/src/gpu/kernels/index.js +6 -0
  29. package/src/inference/browser-harness-text-helpers.d.ts +1 -0
  30. package/src/inference/browser-harness-text-helpers.js +44 -4
  31. package/src/inference/browser-harness.js +7 -0
  32. package/src/inference/pipelines/context.js +4 -2
  33. package/src/inference/pipelines/text/attention/types.js +9 -4
  34. package/src/inference/pipelines/text/embed.d.ts +1 -0
  35. package/src/inference/pipelines/text/embed.js +10 -4
  36. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
  37. package/src/inference/pipelines/text/generator-helpers.js +20 -1
  38. package/src/inference/pipelines/text/generator-steps.js +17 -8
  39. package/src/inference/pipelines/text/generator.js +51 -3
  40. package/src/inference/pipelines/text/init.d.ts +3 -0
  41. package/src/inference/pipelines/text/init.js +24 -3
  42. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  43. package/src/inference/pipelines/text/logits/gpu.js +146 -11
  44. package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
  45. package/src/inference/pipelines/text/probes.js +79 -1
  46. package/src/inference/pipelines/text/stage-names.js +9 -0
  47. package/src/inference/pipelines/text.js +10 -2
  48. package/src/loader/embedding-loader.js +20 -5
  49. package/src/loader/loader-types.d.ts +2 -0
  50. package/src/loader/manifest-config.d.ts +2 -7
  51. package/src/loader/per-layer-input-loader.js +114 -1
  52. package/src/loader/tensors/source-transform.d.ts +15 -0
  53. package/src/loader/tensors/source-transform.js +312 -34
  54. package/src/loader/tensors/tensor-reader.js +28 -2
  55. package/src/storage/artifact-storage-context.d.ts +1 -0
  56. package/src/storage/artifact-storage-context.js +158 -72
  57. package/src/storage/preflight.js +11 -0
  58. package/src/storage/quickstart-downloader.js +15 -4
  59. package/src/tooling/litert-package-runtime.js +369 -14
  60. package/src/tooling/node-source-runtime.js +256 -140
  61. package/src/tooling/source-runtime-bundle.d.ts +25 -0
  62. package/src/tooling/source-runtime-bundle.js +53 -1
  63. package/src/tooling/source-runtime-materializer.js +3 -1
@@ -121,14 +121,107 @@ async function readRange(filePath, offset, length) {
121
121
  }
122
122
  }
123
123
 
124
- async function readSafetensorsHeaderFromFile(filePath) {
125
- const headerPrefixBuffer = await readRange(filePath, 0, 8);
124
+ function createNodeFileAccess() {
125
+ const readers = new Map();
126
+
127
+ const getReader = (filePath) => {
128
+ const normalizedPath = normalizePath(filePath);
129
+ if (!normalizedPath) {
130
+ throw new Error('node source runtime: filePath is required.');
131
+ }
132
+ let reader = readers.get(normalizedPath);
133
+ if (reader) {
134
+ return reader;
135
+ }
136
+ let handlePromise = null;
137
+ let sizePromise = null;
138
+ let closed = false;
139
+ const ensureHandle = async () => {
140
+ if (closed) {
141
+ throw new Error(`node source runtime: file reader already closed for "${normalizedPath}".`);
142
+ }
143
+ if (!handlePromise) {
144
+ handlePromise = fs.open(normalizedPath, 'r').catch((error) => {
145
+ handlePromise = null;
146
+ throw error;
147
+ });
148
+ }
149
+ return handlePromise;
150
+ };
151
+ const getSize = async () => {
152
+ if (!sizePromise) {
153
+ sizePromise = (async () => {
154
+ const handle = await ensureHandle();
155
+ const stats = await handle.stat();
156
+ return Number(stats.size);
157
+ })().catch((error) => {
158
+ sizePromise = null;
159
+ throw error;
160
+ });
161
+ }
162
+ return sizePromise;
163
+ };
164
+ reader = {
165
+ async readRange(offset, length) {
166
+ if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
167
+ return new ArrayBuffer(0);
168
+ }
169
+ const start = Math.max(0, Math.floor(offset));
170
+ const fileSize = await getSize();
171
+ const end = Math.min(fileSize, start + Math.floor(length));
172
+ if (end <= start) {
173
+ return new ArrayBuffer(0);
174
+ }
175
+ const handle = await ensureHandle();
176
+ const out = Buffer.allocUnsafe(end - start);
177
+ let pos = 0;
178
+ while (pos < out.length) {
179
+ const nextChunkBytes = Math.min(out.length - pos, MAX_NODE_READ_BYTES);
180
+ const { bytesRead } = await handle.read(out, pos, nextChunkBytes, start + pos);
181
+ if (bytesRead === 0) break;
182
+ pos += bytesRead;
183
+ }
184
+ return out.buffer.slice(out.byteOffset, out.byteOffset + out.byteLength);
185
+ },
186
+ async getSize() {
187
+ return getSize();
188
+ },
189
+ async close() {
190
+ closed = true;
191
+ const handle = await handlePromise;
192
+ handlePromise = null;
193
+ if (handle) {
194
+ await handle.close();
195
+ }
196
+ },
197
+ };
198
+ readers.set(normalizedPath, reader);
199
+ return reader;
200
+ };
201
+
202
+ return {
203
+ async readRange(filePath, offset, length) {
204
+ return getReader(filePath).readRange(offset, length);
205
+ },
206
+ async getSize(filePath) {
207
+ return getReader(filePath).getSize();
208
+ },
209
+ async close() {
210
+ const pending = Array.from(readers.values(), (reader) => reader.close());
211
+ readers.clear();
212
+ await Promise.all(pending);
213
+ },
214
+ };
215
+ }
216
+
217
+ async function readSafetensorsHeaderFromFile(filePath, fileAccess) {
218
+ const headerPrefixBuffer = await fileAccess.readRange(filePath, 0, 8);
126
219
  const prefixBytes = new Uint8Array(headerPrefixBuffer);
127
220
  if (prefixBytes.byteLength < 8) {
128
221
  throw new Error(`Invalid safetensors header prefix for "${filePath}"`);
129
222
  }
130
223
  const headerSize = Number(new DataView(headerPrefixBuffer).getBigUint64(0, true));
131
- const headerBuffer = await readRange(filePath, 8, headerSize);
224
+ const headerBuffer = await fileAccess.readRange(filePath, 8, headerSize);
132
225
  const fullHeader = new Uint8Array(8 + headerSize);
133
226
  fullHeader.set(prefixBytes, 0);
134
227
  fullHeader.set(new Uint8Array(headerBuffer), 8);
@@ -237,7 +330,7 @@ function applyPackageTokenizerAssets(parsedArtifact, tokenizerAssets) {
237
330
  return next;
238
331
  }
239
332
 
240
- async function parseSafetensorsInput(inputDir) {
333
+ async function parseSafetensorsInput(inputDir, fileAccess) {
241
334
  const configPath = path.join(inputDir, 'config.json');
242
335
  if (!(await fileExists(configPath))) {
243
336
  return null;
@@ -257,7 +350,7 @@ async function parseSafetensorsInput(inputDir) {
257
350
  },
258
351
  async loadSingleSafetensors(suffix) {
259
352
  const filePath = path.join(inputDir, suffix);
260
- const parsed = await readSafetensorsHeaderFromFile(filePath);
353
+ const parsed = await readSafetensorsHeaderFromFile(filePath, fileAccess);
261
354
  return parsed.tensors.map((tensor) => ({
262
355
  ...tensor,
263
356
  sourcePath: filePath,
@@ -268,7 +361,7 @@ async function parseSafetensorsInput(inputDir) {
268
361
  const tensors = [];
269
362
  for (const shardFile of shardFiles) {
270
363
  const shardPath = path.join(inputDir, shardFile);
271
- const parsed = await readSafetensorsHeaderFromFile(shardPath);
364
+ const parsed = await readSafetensorsHeaderFromFile(shardPath, fileAccess);
272
365
  for (const tensor of parsed.tensors) {
273
366
  tensors.push({
274
367
  ...tensor,
@@ -328,7 +421,7 @@ async function parseSafetensorsInput(inputDir) {
328
421
  };
329
422
  }
330
423
 
331
- async function parseTfliteInput(tflitePath) {
424
+ async function parseTfliteInput(tflitePath, fileAccess) {
332
425
  const tfliteStats = await getPathStats(tflitePath, 'TFLite file');
333
426
  const inputDir = path.dirname(tflitePath);
334
427
  const configPath = path.join(inputDir, 'config.json');
@@ -343,7 +436,7 @@ async function parseTfliteInput(tflitePath) {
343
436
  name: path.basename(tflitePath),
344
437
  size: Number(tfliteStats.size),
345
438
  async readRange(offset, length) {
346
- return readRange(tflitePath, offset, length);
439
+ return fileAccess.readRange(tflitePath, offset, length);
347
440
  },
348
441
  });
349
442
  const tokenizerAssets = await collectTokenizerAssets(inputDir);
@@ -379,7 +472,7 @@ async function parseTfliteInput(tflitePath) {
379
472
  };
380
473
  }
381
474
 
382
- function createNodeVirtualFileReaders(packagePath, virtualFiles) {
475
+ function createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess) {
383
476
  const virtualFileMap = new Map(
384
477
  (Array.isArray(virtualFiles) ? virtualFiles : []).map((entry) => [entry.path, entry])
385
478
  );
@@ -396,7 +489,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
396
489
  const readRangeFromVirtualFile = async (virtualPath, offset, length) => {
397
490
  const entry = resolveVirtualFile(virtualPath);
398
491
  if (entry.externalPath) {
399
- return readRange(entry.externalPath, offset, length);
492
+ return fileAccess.readRange(entry.externalPath, offset, length);
400
493
  }
401
494
  const start = Math.max(0, Math.floor(Number(offset) || 0));
402
495
  const requested = Math.max(0, Math.floor(Number(length) || 0));
@@ -405,7 +498,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
405
498
  if (readLength <= 0) {
406
499
  return new ArrayBuffer(0);
407
500
  }
408
- return readRange(packagePath, entry.offset + start, readLength);
501
+ return fileAccess.readRange(packagePath, entry.offset + start, readLength);
409
502
  };
410
503
 
411
504
  const streamRange = async function* (virtualPath, offset, length, options = {}) {
@@ -458,7 +551,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
458
551
  const entry = resolveVirtualFile(virtualPath);
459
552
  const bytes = entry.externalPath
460
553
  ? await fs.readFile(entry.externalPath)
461
- : await readRange(packagePath, entry.offset, entry.size);
554
+ : await fileAccess.readRange(packagePath, entry.offset, entry.size);
462
555
  return new TextDecoder().decode(bytes);
463
556
  };
464
557
 
@@ -466,7 +559,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
466
559
  const entry = resolveVirtualFile(virtualPath);
467
560
  return entry.externalPath
468
561
  ? fs.readFile(entry.externalPath)
469
- : readRange(packagePath, entry.offset, entry.size);
562
+ : fileAccess.readRange(packagePath, entry.offset, entry.size);
470
563
  };
471
564
 
472
565
  return {
@@ -475,11 +568,12 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
475
568
  streamRange,
476
569
  readText,
477
570
  readBinary,
571
+ close: fileAccess.close,
478
572
  };
479
573
  }
480
574
 
481
- async function addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm) {
482
- const readers = createNodeVirtualFileReaders(packagePath, virtualFiles);
575
+ async function addHashesToVirtualEntries(packagePath, virtualFiles, fileAccess, entries, hashAlgorithm) {
576
+ const readers = createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess);
483
577
  const hashedEntries = [];
484
578
  for (const entry of Array.isArray(entries) ? entries : []) {
485
579
  const virtualPath = normalizePath(entry?.path);
@@ -505,7 +599,7 @@ async function addHashesToVirtualEntries(packagePath, virtualFiles, entries, has
505
599
  return hashedEntries;
506
600
  }
507
601
 
508
- async function parseLiteRTPackageInput(packagePath, sourceKind) {
602
+ async function parseLiteRTPackageInput(packagePath, sourceKind, fileAccess) {
509
603
  const stats = await getPathStats(packagePath, `LiteRT package (${sourceKind})`);
510
604
  const resolved = await resolveLiteRTPackageParsedArtifact({
511
605
  sourceKind,
@@ -514,7 +608,7 @@ async function parseLiteRTPackageInput(packagePath, sourceKind) {
514
608
  name: path.basename(packagePath),
515
609
  size: Number(stats.size),
516
610
  async readRange(offset, length) {
517
- return readRange(packagePath, offset, length);
611
+ return fileAccess.readRange(packagePath, offset, length);
518
612
  },
519
613
  },
520
614
  });
@@ -527,14 +621,14 @@ async function parseLiteRTPackageInput(packagePath, sourceKind) {
527
621
  return {
528
622
  ...parsedArtifact,
529
623
  sourceRoot: packagePath,
530
- storageReaders: createNodeVirtualFileReaders(packagePath, virtualFiles),
624
+ storageReaders: createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess),
531
625
  async hashFileEntries(entries, hashAlgorithm) {
532
- return addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm);
626
+ return addHashesToVirtualEntries(packagePath, virtualFiles, fileAccess, entries, hashAlgorithm);
533
627
  },
534
628
  };
535
629
  }
536
630
 
537
- async function parseGgufInput(ggufPath) {
631
+ async function parseGgufInput(ggufPath, fileAccess) {
538
632
  const ggufStats = await getPathStats(ggufPath, 'GGUF file');
539
633
  const fileSize = Number(ggufStats.size);
540
634
  const ggufSource = {
@@ -546,7 +640,7 @@ async function parseGgufInput(ggufPath) {
546
640
  size: fileSize,
547
641
  },
548
642
  async readRange(offset, length) {
549
- return readRange(ggufPath, offset, length);
643
+ return fileAccess.readRange(ggufPath, offset, length);
550
644
  },
551
645
  };
552
646
 
@@ -601,15 +695,15 @@ async function parseGgufInput(ggufPath) {
601
695
  };
602
696
  }
603
697
 
604
- function buildNodeFileReaders() {
605
- const readRangeFromFile = async (filePath, offset, length) => readRange(filePath, offset, length);
698
+ function buildNodeFileReaders(fileAccess) {
699
+ const readRangeFromFile = async (filePath, offset, length) => fileAccess.readRange(filePath, offset, length);
606
700
  const streamRange = async function* (filePath, offset, length, options = {}) {
607
701
  if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
608
702
  return;
609
703
  }
610
- const stats = await getPathStats(filePath, `stream source asset (${filePath})`);
704
+ const fileSize = await fileAccess.getSize(filePath);
611
705
  const start = Math.max(0, Math.floor(offset));
612
- const end = Math.min(Number(stats.size), start + Math.floor(length));
706
+ const end = Math.min(fileSize, start + Math.floor(length));
613
707
  if (end <= start) {
614
708
  return;
615
709
  }
@@ -645,6 +739,7 @@ function buildNodeFileReaders() {
645
739
  streamRange,
646
740
  readText,
647
741
  readBinary,
742
+ close: fileAccess.close,
648
743
  };
649
744
  }
650
745
 
@@ -800,138 +895,159 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
800
895
  const verifyHashes = options.verifyHashes === true;
801
896
  const resolvedInputPath = path.resolve(inputPath);
802
897
  const stats = await getPathStats(resolvedInputPath, 'inputPath');
898
+ const fileAccess = createNodeFileAccess();
803
899
 
804
- let parsed = null;
805
- if (stats.isFile()) {
806
- if (isTflitePath(resolvedInputPath)) {
807
- parsed = await parseTfliteInput(resolvedInputPath);
808
- }
809
- if (!parsed && isLiteRTTaskPath(resolvedInputPath)) {
810
- parsed = await parseLiteRTPackageInput(resolvedInputPath, LITERT_PACKAGE_SOURCE_KIND_TASK);
811
- }
812
- if (!parsed && isLiteRTLMPath(resolvedInputPath)) {
813
- parsed = await parseLiteRTPackageInput(resolvedInputPath, LITERT_PACKAGE_SOURCE_KIND_LITERTLM);
814
- }
815
- if (!parsed && !isGgufPath(resolvedInputPath)) {
816
- return null;
817
- }
818
- if (!parsed) {
819
- parsed = await parseGgufInput(resolvedInputPath);
820
- }
821
- } else if (stats.isDirectory()) {
822
- if (await fileExists(path.join(resolvedInputPath, 'manifest.json'))) {
823
- return null;
824
- }
825
- parsed = await parseSafetensorsInput(resolvedInputPath);
826
- if (!parsed) {
827
- const entries = await fs.readdir(resolvedInputPath, { withFileTypes: true });
828
- const ggufFiles = entries
829
- .filter((entry) => entry.isFile() && isGgufPath(entry.name))
830
- .map((entry) => entry.name)
831
- .sort((left, right) => left.localeCompare(right));
832
- if (ggufFiles.length === 1) {
833
- parsed = await parseGgufInput(path.join(resolvedInputPath, ggufFiles[0]));
834
- } else if (ggufFiles.length > 1) {
835
- throw new Error(
836
- `node source runtime: multiple GGUF files found in "${resolvedInputPath}": ${ggufFiles.join(', ')}.`
900
+ try {
901
+ let parsed = null;
902
+ if (stats.isFile()) {
903
+ if (isTflitePath(resolvedInputPath)) {
904
+ parsed = await parseTfliteInput(resolvedInputPath, fileAccess);
905
+ }
906
+ if (!parsed && isLiteRTTaskPath(resolvedInputPath)) {
907
+ parsed = await parseLiteRTPackageInput(
908
+ resolvedInputPath,
909
+ LITERT_PACKAGE_SOURCE_KIND_TASK,
910
+ fileAccess
837
911
  );
838
912
  }
839
- if (!parsed) {
840
- const tfliteFiles = entries
841
- .filter((entry) => entry.isFile() && isTflitePath(entry.name))
842
- .map((entry) => entry.name)
843
- .sort((left, right) => left.localeCompare(right));
844
- if (tfliteFiles.length === 1) {
845
- parsed = await parseTfliteInput(path.join(resolvedInputPath, tfliteFiles[0]));
846
- } else if (tfliteFiles.length > 1) {
847
- throw new Error(
848
- `node source runtime: multiple TFLite files found in "${resolvedInputPath}": ${tfliteFiles.join(', ')}.`
849
- );
850
- }
913
+ if (!parsed && isLiteRTLMPath(resolvedInputPath)) {
914
+ parsed = await parseLiteRTPackageInput(
915
+ resolvedInputPath,
916
+ LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
917
+ fileAccess
918
+ );
919
+ }
920
+ if (!parsed && !isGgufPath(resolvedInputPath)) {
921
+ await fileAccess.close();
922
+ return null;
851
923
  }
852
924
  if (!parsed) {
853
- const taskFiles = entries
854
- .filter((entry) => entry.isFile() && isLiteRTTaskPath(entry.name))
855
- .map((entry) => entry.name)
856
- .sort((left, right) => left.localeCompare(right));
857
- if (taskFiles.length === 1) {
858
- parsed = await parseLiteRTPackageInput(
859
- path.join(resolvedInputPath, taskFiles[0]),
860
- LITERT_PACKAGE_SOURCE_KIND_TASK
861
- );
862
- } else if (taskFiles.length > 1) {
863
- throw new Error(
864
- `node source runtime: multiple LiteRT task files found in "${resolvedInputPath}": ${taskFiles.join(', ')}.`
865
- );
866
- }
925
+ parsed = await parseGgufInput(resolvedInputPath, fileAccess);
926
+ }
927
+ } else if (stats.isDirectory()) {
928
+ if (await fileExists(path.join(resolvedInputPath, 'manifest.json'))) {
929
+ await fileAccess.close();
930
+ return null;
867
931
  }
932
+ parsed = await parseSafetensorsInput(resolvedInputPath, fileAccess);
868
933
  if (!parsed) {
869
- const litertLmFiles = entries
870
- .filter((entry) => entry.isFile() && isLiteRTLMPath(entry.name))
934
+ const entries = await fs.readdir(resolvedInputPath, { withFileTypes: true });
935
+ const ggufFiles = entries
936
+ .filter((entry) => entry.isFile() && isGgufPath(entry.name))
871
937
  .map((entry) => entry.name)
872
938
  .sort((left, right) => left.localeCompare(right));
873
- if (litertLmFiles.length === 1) {
874
- parsed = await parseLiteRTPackageInput(
875
- path.join(resolvedInputPath, litertLmFiles[0]),
876
- LITERT_PACKAGE_SOURCE_KIND_LITERTLM
877
- );
878
- } else if (litertLmFiles.length > 1) {
939
+ if (ggufFiles.length === 1) {
940
+ parsed = await parseGgufInput(path.join(resolvedInputPath, ggufFiles[0]), fileAccess);
941
+ } else if (ggufFiles.length > 1) {
879
942
  throw new Error(
880
- `node source runtime: multiple LiteRT-LM files found in "${resolvedInputPath}": ${litertLmFiles.join(', ')}.`
943
+ `node source runtime: multiple GGUF files found in "${resolvedInputPath}": ${ggufFiles.join(', ')}.`
881
944
  );
882
945
  }
946
+ if (!parsed) {
947
+ const tfliteFiles = entries
948
+ .filter((entry) => entry.isFile() && isTflitePath(entry.name))
949
+ .map((entry) => entry.name)
950
+ .sort((left, right) => left.localeCompare(right));
951
+ if (tfliteFiles.length === 1) {
952
+ parsed = await parseTfliteInput(path.join(resolvedInputPath, tfliteFiles[0]), fileAccess);
953
+ } else if (tfliteFiles.length > 1) {
954
+ throw new Error(
955
+ `node source runtime: multiple TFLite files found in "${resolvedInputPath}": ${tfliteFiles.join(', ')}.`
956
+ );
957
+ }
958
+ }
959
+ if (!parsed) {
960
+ const taskFiles = entries
961
+ .filter((entry) => entry.isFile() && isLiteRTTaskPath(entry.name))
962
+ .map((entry) => entry.name)
963
+ .sort((left, right) => left.localeCompare(right));
964
+ if (taskFiles.length === 1) {
965
+ parsed = await parseLiteRTPackageInput(
966
+ path.join(resolvedInputPath, taskFiles[0]),
967
+ LITERT_PACKAGE_SOURCE_KIND_TASK,
968
+ fileAccess
969
+ );
970
+ } else if (taskFiles.length > 1) {
971
+ throw new Error(
972
+ `node source runtime: multiple LiteRT task files found in "${resolvedInputPath}": ${taskFiles.join(', ')}.`
973
+ );
974
+ }
975
+ }
976
+ if (!parsed) {
977
+ const litertLmFiles = entries
978
+ .filter((entry) => entry.isFile() && isLiteRTLMPath(entry.name))
979
+ .map((entry) => entry.name)
980
+ .sort((left, right) => left.localeCompare(right));
981
+ if (litertLmFiles.length === 1) {
982
+ parsed = await parseLiteRTPackageInput(
983
+ path.join(resolvedInputPath, litertLmFiles[0]),
984
+ LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
985
+ fileAccess
986
+ );
987
+ } else if (litertLmFiles.length > 1) {
988
+ throw new Error(
989
+ `node source runtime: multiple LiteRT-LM files found in "${resolvedInputPath}": ${litertLmFiles.join(', ')}.`
990
+ );
991
+ }
992
+ }
883
993
  }
994
+ } else {
995
+ await fileAccess.close();
996
+ return null;
884
997
  }
885
- } else {
886
- return null;
887
- }
888
998
 
889
- if (!parsed) {
890
- return null;
891
- }
999
+ if (!parsed) {
1000
+ await fileAccess.close();
1001
+ return null;
1002
+ }
892
1003
 
893
- const loadingConfig = resolveLoadingConfig(options.runtimeConfig ?? null);
894
- const resolvedMemoryBudgetBytes = resolveResidentBudgetBytes(loadingConfig);
895
- assertSourceRuntimeFitsResidentBudget(parsed, loadingConfig, resolvedMemoryBudgetBytes);
896
- const {
897
- model,
898
- shardSources,
899
- sourceKind,
900
- } = await resolveSourceRuntimeBundleFromParsedArtifact({
901
- parsedArtifact: parsed,
902
- requestedModelId: options.modelId || null,
903
- runtimeLabel: 'node source runtime',
904
- logCategory: 'NodeSourceRuntime',
905
- hashFileEntries: typeof parsed?.hashFileEntries === 'function'
906
- ? (entries, hashAlgorithm) => parsed.hashFileEntries(entries, hashAlgorithm)
907
- : addHashesToFileEntries,
908
- });
1004
+ const loadingConfig = resolveLoadingConfig(options.runtimeConfig ?? null);
1005
+ const resolvedMemoryBudgetBytes = resolveResidentBudgetBytes(loadingConfig);
1006
+ assertSourceRuntimeFitsResidentBudget(parsed, loadingConfig, resolvedMemoryBudgetBytes);
1007
+ const {
1008
+ model,
1009
+ shardSources,
1010
+ sourceKind,
1011
+ } = await resolveSourceRuntimeBundleFromParsedArtifact({
1012
+ parsedArtifact: parsed,
1013
+ requestedModelId: options.modelId || null,
1014
+ runtimeLabel: 'node source runtime',
1015
+ logCategory: 'NodeSourceRuntime',
1016
+ hashFileEntries: typeof parsed?.hashFileEntries === 'function'
1017
+ ? (entries, hashAlgorithm) => parsed.hashFileEntries(entries, hashAlgorithm)
1018
+ : addHashesToFileEntries,
1019
+ });
909
1020
 
910
- const readers = parsed?.storageReaders ?? buildNodeFileReaders();
911
- const storageContext = createSourceStorageContext({
912
- model,
913
- shardSources,
914
- readRange: readers.readRange,
915
- streamRange: readers.streamRange,
916
- readText: readers.readText,
917
- readBinary: readers.readBinary,
918
- tokenizerJsonPath: parsed.tokenizerJsonPath,
919
- tokenizerModelPath: parsed.tokenizerModelPath,
920
- verifyHashes,
921
- sourceHashesTrusted: true,
922
- });
1021
+ const readers = parsed?.storageReaders ?? buildNodeFileReaders(fileAccess);
1022
+ const storageContext = createSourceStorageContext({
1023
+ model,
1024
+ shardSources,
1025
+ readRange: readers.readRange,
1026
+ streamRange: readers.streamRange,
1027
+ readText: readers.readText,
1028
+ readBinary: readers.readBinary,
1029
+ close: readers.close,
1030
+ tokenizerJsonPath: parsed.tokenizerJsonPath,
1031
+ tokenizerModelPath: parsed.tokenizerModelPath,
1032
+ verifyHashes,
1033
+ sourceHashesTrusted: true,
1034
+ });
923
1035
 
924
- log.info(
925
- 'NodeSourceRuntime',
926
- `Source runtime ready: ${model.modelId} (${sourceKind}, ${parsed.tensors.length} tensors)`
927
- );
1036
+ log.info(
1037
+ 'NodeSourceRuntime',
1038
+ `Source runtime ready: ${model.modelId} (${sourceKind}, ${parsed.tensors.length} tensors)`
1039
+ );
928
1040
 
929
- return {
930
- model,
931
- manifest: model,
932
- storageContext,
933
- sourceKind,
934
- sourceRoot: parsed.sourceRoot,
935
- resolvedMemoryBudgetBytes,
936
- };
1041
+ return {
1042
+ model,
1043
+ manifest: model,
1044
+ storageContext,
1045
+ sourceKind,
1046
+ sourceRoot: parsed.sourceRoot,
1047
+ resolvedMemoryBudgetBytes,
1048
+ };
1049
+ } catch (error) {
1050
+ await fileAccess.close();
1051
+ throw error;
1052
+ }
937
1053
  }
@@ -43,6 +43,29 @@ export interface SourceRuntimeTensor {
43
43
  scaleDivisor?: number;
44
44
  storageShape: [number, number];
45
45
  quantAxis: 0 | 1;
46
+ scaleCompanionDtype?: 'UINT8';
47
+ scaleCompanionDequant?: {
48
+ scale: number;
49
+ zeroPoint: number;
50
+ };
51
+ scaleSourcePath: string;
52
+ scaleOffset: number;
53
+ scaleSize: number;
54
+ sumSourcePath?: string;
55
+ sumOffset?: number;
56
+ sumSize?: number;
57
+ } | {
58
+ kind: 'litert_axis_blocked_dequant';
59
+ scheme: 'per_axis_affine';
60
+ sourceDtype: 'INT8' | 'UINT8' | 'INT4' | 'INT2';
61
+ targetDtype: 'F16';
62
+ storageEncoding: 'signed' | 'offset_binary';
63
+ scaleSemantics: 'step' | 'qmax_abs';
64
+ scaleDivisor?: number;
65
+ storageShape: [number, number];
66
+ quantAxis: 0;
67
+ storageBlockSize: number;
68
+ storageLaneOrder: number[];
46
69
  scaleSourcePath: string;
47
70
  scaleOffset: number;
48
71
  scaleSize: number;
@@ -151,6 +174,7 @@ export interface CreateSourceStorageContextOptions {
151
174
  ) => AsyncIterable<ArrayBuffer | Uint8Array>;
152
175
  readText?: (path: string) => Promise<string | Record<string, unknown> | null | undefined>;
153
176
  readBinary?: (path: string) => Promise<ArrayBuffer | Uint8Array>;
177
+ close?: (() => Promise<void>) | null;
154
178
  tokenizerJsonPath?: string | null;
155
179
  tokenizerConfigPath?: string | null;
156
180
  tokenizerModelPath?: string | null;
@@ -174,6 +198,7 @@ export interface SourceStorageContext {
174
198
  loadTokenizerJson: (() => Promise<Record<string, unknown> | null>) | null;
175
199
  loadTokenizerModel: ((pathHint?: string) => Promise<ArrayBuffer | null>) | null;
176
200
  verifyHashes: boolean;
201
+ close?: (() => Promise<void>) | null;
177
202
  }
178
203
 
179
204
  export declare function createSourceStorageContext(