doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -121,14 +121,107 @@ async function readRange(filePath, offset, length) {
|
|
|
121
121
|
}
|
|
122
122
|
}
|
|
123
123
|
|
|
124
|
-
|
|
125
|
-
const
|
|
124
|
+
function createNodeFileAccess() {
|
|
125
|
+
const readers = new Map();
|
|
126
|
+
|
|
127
|
+
const getReader = (filePath) => {
|
|
128
|
+
const normalizedPath = normalizePath(filePath);
|
|
129
|
+
if (!normalizedPath) {
|
|
130
|
+
throw new Error('node source runtime: filePath is required.');
|
|
131
|
+
}
|
|
132
|
+
let reader = readers.get(normalizedPath);
|
|
133
|
+
if (reader) {
|
|
134
|
+
return reader;
|
|
135
|
+
}
|
|
136
|
+
let handlePromise = null;
|
|
137
|
+
let sizePromise = null;
|
|
138
|
+
let closed = false;
|
|
139
|
+
const ensureHandle = async () => {
|
|
140
|
+
if (closed) {
|
|
141
|
+
throw new Error(`node source runtime: file reader already closed for "${normalizedPath}".`);
|
|
142
|
+
}
|
|
143
|
+
if (!handlePromise) {
|
|
144
|
+
handlePromise = fs.open(normalizedPath, 'r').catch((error) => {
|
|
145
|
+
handlePromise = null;
|
|
146
|
+
throw error;
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
return handlePromise;
|
|
150
|
+
};
|
|
151
|
+
const getSize = async () => {
|
|
152
|
+
if (!sizePromise) {
|
|
153
|
+
sizePromise = (async () => {
|
|
154
|
+
const handle = await ensureHandle();
|
|
155
|
+
const stats = await handle.stat();
|
|
156
|
+
return Number(stats.size);
|
|
157
|
+
})().catch((error) => {
|
|
158
|
+
sizePromise = null;
|
|
159
|
+
throw error;
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
return sizePromise;
|
|
163
|
+
};
|
|
164
|
+
reader = {
|
|
165
|
+
async readRange(offset, length) {
|
|
166
|
+
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
167
|
+
return new ArrayBuffer(0);
|
|
168
|
+
}
|
|
169
|
+
const start = Math.max(0, Math.floor(offset));
|
|
170
|
+
const fileSize = await getSize();
|
|
171
|
+
const end = Math.min(fileSize, start + Math.floor(length));
|
|
172
|
+
if (end <= start) {
|
|
173
|
+
return new ArrayBuffer(0);
|
|
174
|
+
}
|
|
175
|
+
const handle = await ensureHandle();
|
|
176
|
+
const out = Buffer.allocUnsafe(end - start);
|
|
177
|
+
let pos = 0;
|
|
178
|
+
while (pos < out.length) {
|
|
179
|
+
const nextChunkBytes = Math.min(out.length - pos, MAX_NODE_READ_BYTES);
|
|
180
|
+
const { bytesRead } = await handle.read(out, pos, nextChunkBytes, start + pos);
|
|
181
|
+
if (bytesRead === 0) break;
|
|
182
|
+
pos += bytesRead;
|
|
183
|
+
}
|
|
184
|
+
return out.buffer.slice(out.byteOffset, out.byteOffset + out.byteLength);
|
|
185
|
+
},
|
|
186
|
+
async getSize() {
|
|
187
|
+
return getSize();
|
|
188
|
+
},
|
|
189
|
+
async close() {
|
|
190
|
+
closed = true;
|
|
191
|
+
const handle = await handlePromise;
|
|
192
|
+
handlePromise = null;
|
|
193
|
+
if (handle) {
|
|
194
|
+
await handle.close();
|
|
195
|
+
}
|
|
196
|
+
},
|
|
197
|
+
};
|
|
198
|
+
readers.set(normalizedPath, reader);
|
|
199
|
+
return reader;
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
async readRange(filePath, offset, length) {
|
|
204
|
+
return getReader(filePath).readRange(offset, length);
|
|
205
|
+
},
|
|
206
|
+
async getSize(filePath) {
|
|
207
|
+
return getReader(filePath).getSize();
|
|
208
|
+
},
|
|
209
|
+
async close() {
|
|
210
|
+
const pending = Array.from(readers.values(), (reader) => reader.close());
|
|
211
|
+
readers.clear();
|
|
212
|
+
await Promise.all(pending);
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async function readSafetensorsHeaderFromFile(filePath, fileAccess) {
|
|
218
|
+
const headerPrefixBuffer = await fileAccess.readRange(filePath, 0, 8);
|
|
126
219
|
const prefixBytes = new Uint8Array(headerPrefixBuffer);
|
|
127
220
|
if (prefixBytes.byteLength < 8) {
|
|
128
221
|
throw new Error(`Invalid safetensors header prefix for "${filePath}"`);
|
|
129
222
|
}
|
|
130
223
|
const headerSize = Number(new DataView(headerPrefixBuffer).getBigUint64(0, true));
|
|
131
|
-
const headerBuffer = await readRange(filePath, 8, headerSize);
|
|
224
|
+
const headerBuffer = await fileAccess.readRange(filePath, 8, headerSize);
|
|
132
225
|
const fullHeader = new Uint8Array(8 + headerSize);
|
|
133
226
|
fullHeader.set(prefixBytes, 0);
|
|
134
227
|
fullHeader.set(new Uint8Array(headerBuffer), 8);
|
|
@@ -237,7 +330,7 @@ function applyPackageTokenizerAssets(parsedArtifact, tokenizerAssets) {
|
|
|
237
330
|
return next;
|
|
238
331
|
}
|
|
239
332
|
|
|
240
|
-
async function parseSafetensorsInput(inputDir) {
|
|
333
|
+
async function parseSafetensorsInput(inputDir, fileAccess) {
|
|
241
334
|
const configPath = path.join(inputDir, 'config.json');
|
|
242
335
|
if (!(await fileExists(configPath))) {
|
|
243
336
|
return null;
|
|
@@ -257,7 +350,7 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
257
350
|
},
|
|
258
351
|
async loadSingleSafetensors(suffix) {
|
|
259
352
|
const filePath = path.join(inputDir, suffix);
|
|
260
|
-
const parsed = await readSafetensorsHeaderFromFile(filePath);
|
|
353
|
+
const parsed = await readSafetensorsHeaderFromFile(filePath, fileAccess);
|
|
261
354
|
return parsed.tensors.map((tensor) => ({
|
|
262
355
|
...tensor,
|
|
263
356
|
sourcePath: filePath,
|
|
@@ -268,7 +361,7 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
268
361
|
const tensors = [];
|
|
269
362
|
for (const shardFile of shardFiles) {
|
|
270
363
|
const shardPath = path.join(inputDir, shardFile);
|
|
271
|
-
const parsed = await readSafetensorsHeaderFromFile(shardPath);
|
|
364
|
+
const parsed = await readSafetensorsHeaderFromFile(shardPath, fileAccess);
|
|
272
365
|
for (const tensor of parsed.tensors) {
|
|
273
366
|
tensors.push({
|
|
274
367
|
...tensor,
|
|
@@ -328,7 +421,7 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
328
421
|
};
|
|
329
422
|
}
|
|
330
423
|
|
|
331
|
-
async function parseTfliteInput(tflitePath) {
|
|
424
|
+
async function parseTfliteInput(tflitePath, fileAccess) {
|
|
332
425
|
const tfliteStats = await getPathStats(tflitePath, 'TFLite file');
|
|
333
426
|
const inputDir = path.dirname(tflitePath);
|
|
334
427
|
const configPath = path.join(inputDir, 'config.json');
|
|
@@ -343,7 +436,7 @@ async function parseTfliteInput(tflitePath) {
|
|
|
343
436
|
name: path.basename(tflitePath),
|
|
344
437
|
size: Number(tfliteStats.size),
|
|
345
438
|
async readRange(offset, length) {
|
|
346
|
-
return readRange(tflitePath, offset, length);
|
|
439
|
+
return fileAccess.readRange(tflitePath, offset, length);
|
|
347
440
|
},
|
|
348
441
|
});
|
|
349
442
|
const tokenizerAssets = await collectTokenizerAssets(inputDir);
|
|
@@ -379,7 +472,7 @@ async function parseTfliteInput(tflitePath) {
|
|
|
379
472
|
};
|
|
380
473
|
}
|
|
381
474
|
|
|
382
|
-
function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
475
|
+
function createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess) {
|
|
383
476
|
const virtualFileMap = new Map(
|
|
384
477
|
(Array.isArray(virtualFiles) ? virtualFiles : []).map((entry) => [entry.path, entry])
|
|
385
478
|
);
|
|
@@ -396,7 +489,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
396
489
|
const readRangeFromVirtualFile = async (virtualPath, offset, length) => {
|
|
397
490
|
const entry = resolveVirtualFile(virtualPath);
|
|
398
491
|
if (entry.externalPath) {
|
|
399
|
-
return readRange(entry.externalPath, offset, length);
|
|
492
|
+
return fileAccess.readRange(entry.externalPath, offset, length);
|
|
400
493
|
}
|
|
401
494
|
const start = Math.max(0, Math.floor(Number(offset) || 0));
|
|
402
495
|
const requested = Math.max(0, Math.floor(Number(length) || 0));
|
|
@@ -405,7 +498,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
405
498
|
if (readLength <= 0) {
|
|
406
499
|
return new ArrayBuffer(0);
|
|
407
500
|
}
|
|
408
|
-
return readRange(packagePath, entry.offset + start, readLength);
|
|
501
|
+
return fileAccess.readRange(packagePath, entry.offset + start, readLength);
|
|
409
502
|
};
|
|
410
503
|
|
|
411
504
|
const streamRange = async function* (virtualPath, offset, length, options = {}) {
|
|
@@ -458,7 +551,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
458
551
|
const entry = resolveVirtualFile(virtualPath);
|
|
459
552
|
const bytes = entry.externalPath
|
|
460
553
|
? await fs.readFile(entry.externalPath)
|
|
461
|
-
: await readRange(packagePath, entry.offset, entry.size);
|
|
554
|
+
: await fileAccess.readRange(packagePath, entry.offset, entry.size);
|
|
462
555
|
return new TextDecoder().decode(bytes);
|
|
463
556
|
};
|
|
464
557
|
|
|
@@ -466,7 +559,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
466
559
|
const entry = resolveVirtualFile(virtualPath);
|
|
467
560
|
return entry.externalPath
|
|
468
561
|
? fs.readFile(entry.externalPath)
|
|
469
|
-
: readRange(packagePath, entry.offset, entry.size);
|
|
562
|
+
: fileAccess.readRange(packagePath, entry.offset, entry.size);
|
|
470
563
|
};
|
|
471
564
|
|
|
472
565
|
return {
|
|
@@ -475,11 +568,12 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
475
568
|
streamRange,
|
|
476
569
|
readText,
|
|
477
570
|
readBinary,
|
|
571
|
+
close: fileAccess.close,
|
|
478
572
|
};
|
|
479
573
|
}
|
|
480
574
|
|
|
481
|
-
async function addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm) {
|
|
482
|
-
const readers = createNodeVirtualFileReaders(packagePath, virtualFiles);
|
|
575
|
+
async function addHashesToVirtualEntries(packagePath, virtualFiles, fileAccess, entries, hashAlgorithm) {
|
|
576
|
+
const readers = createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess);
|
|
483
577
|
const hashedEntries = [];
|
|
484
578
|
for (const entry of Array.isArray(entries) ? entries : []) {
|
|
485
579
|
const virtualPath = normalizePath(entry?.path);
|
|
@@ -505,7 +599,7 @@ async function addHashesToVirtualEntries(packagePath, virtualFiles, entries, has
|
|
|
505
599
|
return hashedEntries;
|
|
506
600
|
}
|
|
507
601
|
|
|
508
|
-
async function parseLiteRTPackageInput(packagePath, sourceKind) {
|
|
602
|
+
async function parseLiteRTPackageInput(packagePath, sourceKind, fileAccess) {
|
|
509
603
|
const stats = await getPathStats(packagePath, `LiteRT package (${sourceKind})`);
|
|
510
604
|
const resolved = await resolveLiteRTPackageParsedArtifact({
|
|
511
605
|
sourceKind,
|
|
@@ -514,7 +608,7 @@ async function parseLiteRTPackageInput(packagePath, sourceKind) {
|
|
|
514
608
|
name: path.basename(packagePath),
|
|
515
609
|
size: Number(stats.size),
|
|
516
610
|
async readRange(offset, length) {
|
|
517
|
-
return readRange(packagePath, offset, length);
|
|
611
|
+
return fileAccess.readRange(packagePath, offset, length);
|
|
518
612
|
},
|
|
519
613
|
},
|
|
520
614
|
});
|
|
@@ -527,14 +621,14 @@ async function parseLiteRTPackageInput(packagePath, sourceKind) {
|
|
|
527
621
|
return {
|
|
528
622
|
...parsedArtifact,
|
|
529
623
|
sourceRoot: packagePath,
|
|
530
|
-
storageReaders: createNodeVirtualFileReaders(packagePath, virtualFiles),
|
|
624
|
+
storageReaders: createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess),
|
|
531
625
|
async hashFileEntries(entries, hashAlgorithm) {
|
|
532
|
-
return addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm);
|
|
626
|
+
return addHashesToVirtualEntries(packagePath, virtualFiles, fileAccess, entries, hashAlgorithm);
|
|
533
627
|
},
|
|
534
628
|
};
|
|
535
629
|
}
|
|
536
630
|
|
|
537
|
-
async function parseGgufInput(ggufPath) {
|
|
631
|
+
async function parseGgufInput(ggufPath, fileAccess) {
|
|
538
632
|
const ggufStats = await getPathStats(ggufPath, 'GGUF file');
|
|
539
633
|
const fileSize = Number(ggufStats.size);
|
|
540
634
|
const ggufSource = {
|
|
@@ -546,7 +640,7 @@ async function parseGgufInput(ggufPath) {
|
|
|
546
640
|
size: fileSize,
|
|
547
641
|
},
|
|
548
642
|
async readRange(offset, length) {
|
|
549
|
-
return readRange(ggufPath, offset, length);
|
|
643
|
+
return fileAccess.readRange(ggufPath, offset, length);
|
|
550
644
|
},
|
|
551
645
|
};
|
|
552
646
|
|
|
@@ -601,15 +695,15 @@ async function parseGgufInput(ggufPath) {
|
|
|
601
695
|
};
|
|
602
696
|
}
|
|
603
697
|
|
|
604
|
-
function buildNodeFileReaders() {
|
|
605
|
-
const readRangeFromFile = async (filePath, offset, length) => readRange(filePath, offset, length);
|
|
698
|
+
function buildNodeFileReaders(fileAccess) {
|
|
699
|
+
const readRangeFromFile = async (filePath, offset, length) => fileAccess.readRange(filePath, offset, length);
|
|
606
700
|
const streamRange = async function* (filePath, offset, length, options = {}) {
|
|
607
701
|
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
608
702
|
return;
|
|
609
703
|
}
|
|
610
|
-
const
|
|
704
|
+
const fileSize = await fileAccess.getSize(filePath);
|
|
611
705
|
const start = Math.max(0, Math.floor(offset));
|
|
612
|
-
const end = Math.min(
|
|
706
|
+
const end = Math.min(fileSize, start + Math.floor(length));
|
|
613
707
|
if (end <= start) {
|
|
614
708
|
return;
|
|
615
709
|
}
|
|
@@ -645,6 +739,7 @@ function buildNodeFileReaders() {
|
|
|
645
739
|
streamRange,
|
|
646
740
|
readText,
|
|
647
741
|
readBinary,
|
|
742
|
+
close: fileAccess.close,
|
|
648
743
|
};
|
|
649
744
|
}
|
|
650
745
|
|
|
@@ -800,138 +895,159 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
|
|
|
800
895
|
const verifyHashes = options.verifyHashes === true;
|
|
801
896
|
const resolvedInputPath = path.resolve(inputPath);
|
|
802
897
|
const stats = await getPathStats(resolvedInputPath, 'inputPath');
|
|
898
|
+
const fileAccess = createNodeFileAccess();
|
|
803
899
|
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
if (
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
parsed
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
if (!parsed && !isGgufPath(resolvedInputPath)) {
|
|
816
|
-
return null;
|
|
817
|
-
}
|
|
818
|
-
if (!parsed) {
|
|
819
|
-
parsed = await parseGgufInput(resolvedInputPath);
|
|
820
|
-
}
|
|
821
|
-
} else if (stats.isDirectory()) {
|
|
822
|
-
if (await fileExists(path.join(resolvedInputPath, 'manifest.json'))) {
|
|
823
|
-
return null;
|
|
824
|
-
}
|
|
825
|
-
parsed = await parseSafetensorsInput(resolvedInputPath);
|
|
826
|
-
if (!parsed) {
|
|
827
|
-
const entries = await fs.readdir(resolvedInputPath, { withFileTypes: true });
|
|
828
|
-
const ggufFiles = entries
|
|
829
|
-
.filter((entry) => entry.isFile() && isGgufPath(entry.name))
|
|
830
|
-
.map((entry) => entry.name)
|
|
831
|
-
.sort((left, right) => left.localeCompare(right));
|
|
832
|
-
if (ggufFiles.length === 1) {
|
|
833
|
-
parsed = await parseGgufInput(path.join(resolvedInputPath, ggufFiles[0]));
|
|
834
|
-
} else if (ggufFiles.length > 1) {
|
|
835
|
-
throw new Error(
|
|
836
|
-
`node source runtime: multiple GGUF files found in "${resolvedInputPath}": ${ggufFiles.join(', ')}.`
|
|
900
|
+
try {
|
|
901
|
+
let parsed = null;
|
|
902
|
+
if (stats.isFile()) {
|
|
903
|
+
if (isTflitePath(resolvedInputPath)) {
|
|
904
|
+
parsed = await parseTfliteInput(resolvedInputPath, fileAccess);
|
|
905
|
+
}
|
|
906
|
+
if (!parsed && isLiteRTTaskPath(resolvedInputPath)) {
|
|
907
|
+
parsed = await parseLiteRTPackageInput(
|
|
908
|
+
resolvedInputPath,
|
|
909
|
+
LITERT_PACKAGE_SOURCE_KIND_TASK,
|
|
910
|
+
fileAccess
|
|
837
911
|
);
|
|
838
912
|
}
|
|
839
|
-
if (!parsed) {
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
);
|
|
850
|
-
}
|
|
913
|
+
if (!parsed && isLiteRTLMPath(resolvedInputPath)) {
|
|
914
|
+
parsed = await parseLiteRTPackageInput(
|
|
915
|
+
resolvedInputPath,
|
|
916
|
+
LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
|
|
917
|
+
fileAccess
|
|
918
|
+
);
|
|
919
|
+
}
|
|
920
|
+
if (!parsed && !isGgufPath(resolvedInputPath)) {
|
|
921
|
+
await fileAccess.close();
|
|
922
|
+
return null;
|
|
851
923
|
}
|
|
852
924
|
if (!parsed) {
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
path.join(resolvedInputPath, taskFiles[0]),
|
|
860
|
-
LITERT_PACKAGE_SOURCE_KIND_TASK
|
|
861
|
-
);
|
|
862
|
-
} else if (taskFiles.length > 1) {
|
|
863
|
-
throw new Error(
|
|
864
|
-
`node source runtime: multiple LiteRT task files found in "${resolvedInputPath}": ${taskFiles.join(', ')}.`
|
|
865
|
-
);
|
|
866
|
-
}
|
|
925
|
+
parsed = await parseGgufInput(resolvedInputPath, fileAccess);
|
|
926
|
+
}
|
|
927
|
+
} else if (stats.isDirectory()) {
|
|
928
|
+
if (await fileExists(path.join(resolvedInputPath, 'manifest.json'))) {
|
|
929
|
+
await fileAccess.close();
|
|
930
|
+
return null;
|
|
867
931
|
}
|
|
932
|
+
parsed = await parseSafetensorsInput(resolvedInputPath, fileAccess);
|
|
868
933
|
if (!parsed) {
|
|
869
|
-
const
|
|
870
|
-
|
|
934
|
+
const entries = await fs.readdir(resolvedInputPath, { withFileTypes: true });
|
|
935
|
+
const ggufFiles = entries
|
|
936
|
+
.filter((entry) => entry.isFile() && isGgufPath(entry.name))
|
|
871
937
|
.map((entry) => entry.name)
|
|
872
938
|
.sort((left, right) => left.localeCompare(right));
|
|
873
|
-
if (
|
|
874
|
-
parsed = await
|
|
875
|
-
|
|
876
|
-
LITERT_PACKAGE_SOURCE_KIND_LITERTLM
|
|
877
|
-
);
|
|
878
|
-
} else if (litertLmFiles.length > 1) {
|
|
939
|
+
if (ggufFiles.length === 1) {
|
|
940
|
+
parsed = await parseGgufInput(path.join(resolvedInputPath, ggufFiles[0]), fileAccess);
|
|
941
|
+
} else if (ggufFiles.length > 1) {
|
|
879
942
|
throw new Error(
|
|
880
|
-
`node source runtime: multiple
|
|
943
|
+
`node source runtime: multiple GGUF files found in "${resolvedInputPath}": ${ggufFiles.join(', ')}.`
|
|
881
944
|
);
|
|
882
945
|
}
|
|
946
|
+
if (!parsed) {
|
|
947
|
+
const tfliteFiles = entries
|
|
948
|
+
.filter((entry) => entry.isFile() && isTflitePath(entry.name))
|
|
949
|
+
.map((entry) => entry.name)
|
|
950
|
+
.sort((left, right) => left.localeCompare(right));
|
|
951
|
+
if (tfliteFiles.length === 1) {
|
|
952
|
+
parsed = await parseTfliteInput(path.join(resolvedInputPath, tfliteFiles[0]), fileAccess);
|
|
953
|
+
} else if (tfliteFiles.length > 1) {
|
|
954
|
+
throw new Error(
|
|
955
|
+
`node source runtime: multiple TFLite files found in "${resolvedInputPath}": ${tfliteFiles.join(', ')}.`
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
if (!parsed) {
|
|
960
|
+
const taskFiles = entries
|
|
961
|
+
.filter((entry) => entry.isFile() && isLiteRTTaskPath(entry.name))
|
|
962
|
+
.map((entry) => entry.name)
|
|
963
|
+
.sort((left, right) => left.localeCompare(right));
|
|
964
|
+
if (taskFiles.length === 1) {
|
|
965
|
+
parsed = await parseLiteRTPackageInput(
|
|
966
|
+
path.join(resolvedInputPath, taskFiles[0]),
|
|
967
|
+
LITERT_PACKAGE_SOURCE_KIND_TASK,
|
|
968
|
+
fileAccess
|
|
969
|
+
);
|
|
970
|
+
} else if (taskFiles.length > 1) {
|
|
971
|
+
throw new Error(
|
|
972
|
+
`node source runtime: multiple LiteRT task files found in "${resolvedInputPath}": ${taskFiles.join(', ')}.`
|
|
973
|
+
);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
if (!parsed) {
|
|
977
|
+
const litertLmFiles = entries
|
|
978
|
+
.filter((entry) => entry.isFile() && isLiteRTLMPath(entry.name))
|
|
979
|
+
.map((entry) => entry.name)
|
|
980
|
+
.sort((left, right) => left.localeCompare(right));
|
|
981
|
+
if (litertLmFiles.length === 1) {
|
|
982
|
+
parsed = await parseLiteRTPackageInput(
|
|
983
|
+
path.join(resolvedInputPath, litertLmFiles[0]),
|
|
984
|
+
LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
|
|
985
|
+
fileAccess
|
|
986
|
+
);
|
|
987
|
+
} else if (litertLmFiles.length > 1) {
|
|
988
|
+
throw new Error(
|
|
989
|
+
`node source runtime: multiple LiteRT-LM files found in "${resolvedInputPath}": ${litertLmFiles.join(', ')}.`
|
|
990
|
+
);
|
|
991
|
+
}
|
|
992
|
+
}
|
|
883
993
|
}
|
|
994
|
+
} else {
|
|
995
|
+
await fileAccess.close();
|
|
996
|
+
return null;
|
|
884
997
|
}
|
|
885
|
-
} else {
|
|
886
|
-
return null;
|
|
887
|
-
}
|
|
888
998
|
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
999
|
+
if (!parsed) {
|
|
1000
|
+
await fileAccess.close();
|
|
1001
|
+
return null;
|
|
1002
|
+
}
|
|
892
1003
|
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
1004
|
+
const loadingConfig = resolveLoadingConfig(options.runtimeConfig ?? null);
|
|
1005
|
+
const resolvedMemoryBudgetBytes = resolveResidentBudgetBytes(loadingConfig);
|
|
1006
|
+
assertSourceRuntimeFitsResidentBudget(parsed, loadingConfig, resolvedMemoryBudgetBytes);
|
|
1007
|
+
const {
|
|
1008
|
+
model,
|
|
1009
|
+
shardSources,
|
|
1010
|
+
sourceKind,
|
|
1011
|
+
} = await resolveSourceRuntimeBundleFromParsedArtifact({
|
|
1012
|
+
parsedArtifact: parsed,
|
|
1013
|
+
requestedModelId: options.modelId || null,
|
|
1014
|
+
runtimeLabel: 'node source runtime',
|
|
1015
|
+
logCategory: 'NodeSourceRuntime',
|
|
1016
|
+
hashFileEntries: typeof parsed?.hashFileEntries === 'function'
|
|
1017
|
+
? (entries, hashAlgorithm) => parsed.hashFileEntries(entries, hashAlgorithm)
|
|
1018
|
+
: addHashesToFileEntries,
|
|
1019
|
+
});
|
|
909
1020
|
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1021
|
+
const readers = parsed?.storageReaders ?? buildNodeFileReaders(fileAccess);
|
|
1022
|
+
const storageContext = createSourceStorageContext({
|
|
1023
|
+
model,
|
|
1024
|
+
shardSources,
|
|
1025
|
+
readRange: readers.readRange,
|
|
1026
|
+
streamRange: readers.streamRange,
|
|
1027
|
+
readText: readers.readText,
|
|
1028
|
+
readBinary: readers.readBinary,
|
|
1029
|
+
close: readers.close,
|
|
1030
|
+
tokenizerJsonPath: parsed.tokenizerJsonPath,
|
|
1031
|
+
tokenizerModelPath: parsed.tokenizerModelPath,
|
|
1032
|
+
verifyHashes,
|
|
1033
|
+
sourceHashesTrusted: true,
|
|
1034
|
+
});
|
|
923
1035
|
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1036
|
+
log.info(
|
|
1037
|
+
'NodeSourceRuntime',
|
|
1038
|
+
`Source runtime ready: ${model.modelId} (${sourceKind}, ${parsed.tensors.length} tensors)`
|
|
1039
|
+
);
|
|
928
1040
|
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
1041
|
+
return {
|
|
1042
|
+
model,
|
|
1043
|
+
manifest: model,
|
|
1044
|
+
storageContext,
|
|
1045
|
+
sourceKind,
|
|
1046
|
+
sourceRoot: parsed.sourceRoot,
|
|
1047
|
+
resolvedMemoryBudgetBytes,
|
|
1048
|
+
};
|
|
1049
|
+
} catch (error) {
|
|
1050
|
+
await fileAccess.close();
|
|
1051
|
+
throw error;
|
|
1052
|
+
}
|
|
937
1053
|
}
|
|
@@ -43,6 +43,29 @@ export interface SourceRuntimeTensor {
|
|
|
43
43
|
scaleDivisor?: number;
|
|
44
44
|
storageShape: [number, number];
|
|
45
45
|
quantAxis: 0 | 1;
|
|
46
|
+
scaleCompanionDtype?: 'UINT8';
|
|
47
|
+
scaleCompanionDequant?: {
|
|
48
|
+
scale: number;
|
|
49
|
+
zeroPoint: number;
|
|
50
|
+
};
|
|
51
|
+
scaleSourcePath: string;
|
|
52
|
+
scaleOffset: number;
|
|
53
|
+
scaleSize: number;
|
|
54
|
+
sumSourcePath?: string;
|
|
55
|
+
sumOffset?: number;
|
|
56
|
+
sumSize?: number;
|
|
57
|
+
} | {
|
|
58
|
+
kind: 'litert_axis_blocked_dequant';
|
|
59
|
+
scheme: 'per_axis_affine';
|
|
60
|
+
sourceDtype: 'INT8' | 'UINT8' | 'INT4' | 'INT2';
|
|
61
|
+
targetDtype: 'F16';
|
|
62
|
+
storageEncoding: 'signed' | 'offset_binary';
|
|
63
|
+
scaleSemantics: 'step' | 'qmax_abs';
|
|
64
|
+
scaleDivisor?: number;
|
|
65
|
+
storageShape: [number, number];
|
|
66
|
+
quantAxis: 0;
|
|
67
|
+
storageBlockSize: number;
|
|
68
|
+
storageLaneOrder: number[];
|
|
46
69
|
scaleSourcePath: string;
|
|
47
70
|
scaleOffset: number;
|
|
48
71
|
scaleSize: number;
|
|
@@ -151,6 +174,7 @@ export interface CreateSourceStorageContextOptions {
|
|
|
151
174
|
) => AsyncIterable<ArrayBuffer | Uint8Array>;
|
|
152
175
|
readText?: (path: string) => Promise<string | Record<string, unknown> | null | undefined>;
|
|
153
176
|
readBinary?: (path: string) => Promise<ArrayBuffer | Uint8Array>;
|
|
177
|
+
close?: (() => Promise<void>) | null;
|
|
154
178
|
tokenizerJsonPath?: string | null;
|
|
155
179
|
tokenizerConfigPath?: string | null;
|
|
156
180
|
tokenizerModelPath?: string | null;
|
|
@@ -174,6 +198,7 @@ export interface SourceStorageContext {
|
|
|
174
198
|
loadTokenizerJson: (() => Promise<Record<string, unknown> | null>) | null;
|
|
175
199
|
loadTokenizerModel: ((pathHint?: string) => Promise<ArrayBuffer | null>) | null;
|
|
176
200
|
verifyHashes: boolean;
|
|
201
|
+
close?: (() => Promise<void>) | null;
|
|
177
202
|
}
|
|
178
203
|
|
|
179
204
|
export declare function createSourceStorageContext(
|