doppler-gpu 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +27 -0
- package/models/catalog.json +612 -7
- package/package.json +13 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +126 -4
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +64 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/kernels/registry.json +52 -9
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/lora-loader.js +15 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +189 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.js +90 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/ops.js +32 -0
- package/src/inference/pipelines/text/types.d.ts +38 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +4 -2
- package/src/inference/pipelines/text.js +112 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/version.js +1 -1
|
@@ -2,7 +2,13 @@ import { getDevice } from '../../../gpu/device.js';
|
|
|
2
2
|
import { Q4K_BLOCK_BYTES, q4kBlockCount } from '../../../config/schema/index.js';
|
|
3
3
|
import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
|
|
4
4
|
import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
createWeightBuffer,
|
|
7
|
+
getWeightDtype,
|
|
8
|
+
isGpuBufferInstance,
|
|
9
|
+
isWeightBuffer,
|
|
10
|
+
tagBufferDtype,
|
|
11
|
+
} from '../../../gpu/weight-buffer.js';
|
|
6
12
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
7
13
|
import { log } from '../../../debug/index.js';
|
|
8
14
|
|
|
@@ -112,6 +118,26 @@ function formatsMatch(a, b) {
|
|
|
112
118
|
&& a?.bytesPerRow === b?.bytesPerRow;
|
|
113
119
|
}
|
|
114
120
|
|
|
121
|
+
function resolveBiasStorageFormat(weight, expectedElements) {
|
|
122
|
+
if (!weight || !Number.isFinite(expectedElements) || expectedElements <= 0) {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
const buffer = isWeightBuffer(weight) ? weight.buffer : weight;
|
|
126
|
+
if (!isGpuBufferInstance(buffer)) {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
const dtype = String(getWeightDtype(weight) ?? getWeightDtype(buffer) ?? '').toLowerCase();
|
|
130
|
+
if (dtype !== 'f16' && dtype !== 'f32') {
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
const bytesPerElement = dtype === 'f16' ? 2 : 4;
|
|
134
|
+
const byteLength = expectedElements * bytesPerElement;
|
|
135
|
+
if (buffer.size < byteLength) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
return { buffer, dtype, bytesPerElement, byteLength };
|
|
139
|
+
}
|
|
140
|
+
|
|
115
141
|
export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, options = {}) {
|
|
116
142
|
const device = getDevice();
|
|
117
143
|
if (!device) {
|
|
@@ -174,6 +200,17 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
174
200
|
continue;
|
|
175
201
|
}
|
|
176
202
|
|
|
203
|
+
const projectionBiases = [weights.qProjBias, weights.kProjBias, weights.vProjBias];
|
|
204
|
+
const presentProjectionBiases = projectionBiases.filter(Boolean).length;
|
|
205
|
+
if (presentProjectionBiases !== 0 && presentProjectionBiases !== projectionBiases.length) {
|
|
206
|
+
log.debug('QKV Fusion', `Layer ${l}: partial Q/K/V projection biases require split projections`);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
if (hasAttentionOutputGate && presentProjectionBiases > 0) {
|
|
210
|
+
log.debug('QKV Fusion', `Layer ${l}: gated Q projection biases require split projections`);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
|
|
177
214
|
const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
|
|
178
215
|
const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
|
|
179
216
|
const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
|
|
@@ -228,6 +265,32 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
228
265
|
})
|
|
229
266
|
: null;
|
|
230
267
|
|
|
268
|
+
let qBiasFormat = null;
|
|
269
|
+
let kBiasFormat = null;
|
|
270
|
+
let vBiasFormat = null;
|
|
271
|
+
let qkvBiasBuffer = null;
|
|
272
|
+
if (presentProjectionBiases === projectionBiases.length) {
|
|
273
|
+
qBiasFormat = resolveBiasStorageFormat(weights.qProjBias, qSize);
|
|
274
|
+
kBiasFormat = resolveBiasStorageFormat(weights.kProjBias, kSize);
|
|
275
|
+
vBiasFormat = resolveBiasStorageFormat(weights.vProjBias, vSize);
|
|
276
|
+
const biasFormatsMatch = qBiasFormat
|
|
277
|
+
&& kBiasFormat
|
|
278
|
+
&& vBiasFormat
|
|
279
|
+
&& qBiasFormat.dtype === kBiasFormat.dtype
|
|
280
|
+
&& qBiasFormat.dtype === vBiasFormat.dtype;
|
|
281
|
+
if (!biasFormatsMatch) {
|
|
282
|
+
log.debug('QKV Fusion', `Layer ${l}: inconsistent Q/K/V projection bias formats, skipping`);
|
|
283
|
+
qkvBuffer.destroy();
|
|
284
|
+
qGateBuffer?.destroy();
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
qkvBiasBuffer = device.createBuffer({
|
|
288
|
+
label: `layer_${l}_qkv_proj_bias`,
|
|
289
|
+
size: qBiasFormat.byteLength + kBiasFormat.byteLength + vBiasFormat.byteLength,
|
|
290
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
|
|
231
294
|
const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
|
|
232
295
|
if (hasAttentionOutputGate) {
|
|
233
296
|
for (let head = 0; head < numHeads; head++) {
|
|
@@ -266,6 +329,23 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
266
329
|
qkvBuffer, qBytes + kFormat.byteLength,
|
|
267
330
|
vFormat.byteLength
|
|
268
331
|
);
|
|
332
|
+
if (qkvBiasBuffer) {
|
|
333
|
+
encoder.copyBufferToBuffer(qBiasFormat.buffer, 0, qkvBiasBuffer, 0, qBiasFormat.byteLength);
|
|
334
|
+
encoder.copyBufferToBuffer(
|
|
335
|
+
kBiasFormat.buffer,
|
|
336
|
+
0,
|
|
337
|
+
qkvBiasBuffer,
|
|
338
|
+
qBiasFormat.byteLength,
|
|
339
|
+
kBiasFormat.byteLength
|
|
340
|
+
);
|
|
341
|
+
encoder.copyBufferToBuffer(
|
|
342
|
+
vBiasFormat.buffer,
|
|
343
|
+
0,
|
|
344
|
+
qkvBiasBuffer,
|
|
345
|
+
qBiasFormat.byteLength + kBiasFormat.byteLength,
|
|
346
|
+
vBiasFormat.byteLength
|
|
347
|
+
);
|
|
348
|
+
}
|
|
269
349
|
device.queue.submit([encoder.finish()]);
|
|
270
350
|
|
|
271
351
|
// Store fused buffer, sizes, and dtype
|
|
@@ -278,6 +358,10 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
278
358
|
);
|
|
279
359
|
weights.qkvSizes = [qSize, kSize, vSize];
|
|
280
360
|
weights.qkvDtype = dtype;
|
|
361
|
+
if (qkvBiasBuffer) {
|
|
362
|
+
tagBufferDtype(qkvBiasBuffer, qBiasFormat.dtype);
|
|
363
|
+
weights.qkvProjBias = qkvBiasBuffer;
|
|
364
|
+
}
|
|
281
365
|
if (qGateBuffer) {
|
|
282
366
|
weights.qGateProj = createWeightBuffer(
|
|
283
367
|
qGateBuffer,
|
|
@@ -237,7 +237,9 @@ export interface WeightLoadResult {
|
|
|
237
237
|
layerWeights: Map<string, LayerWeights>;
|
|
238
238
|
embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
239
239
|
lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
240
|
+
lmHeadBias: Float32Array | null;
|
|
240
241
|
finalNorm: GPUBuffer | Float32Array | null;
|
|
242
|
+
finalNormBias: Float32Array | null;
|
|
241
243
|
embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
|
|
242
244
|
diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
|
|
243
245
|
perLayerInputWeights: PerLayerInputWeights | null;
|
|
@@ -1020,7 +1020,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1020
1020
|
layerWeights,
|
|
1021
1021
|
embeddings: dopplerLoader.embeddings,
|
|
1022
1022
|
lmHead: dopplerLoader.lmHead,
|
|
1023
|
+
lmHeadBias: dopplerLoader.lmHeadBias,
|
|
1023
1024
|
finalNorm: dopplerLoader.finalNorm,
|
|
1025
|
+
finalNormBias: dopplerLoader.finalNormBias,
|
|
1024
1026
|
embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
|
|
1025
1027
|
diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
|
|
1026
1028
|
perLayerInputWeights: dopplerLoader.perLayerInputWeights,
|
|
@@ -57,6 +57,7 @@ function shouldUseSandwichRMSNormPairFusion({
|
|
|
57
57
|
attnOutput,
|
|
58
58
|
inputTensor,
|
|
59
59
|
}) {
|
|
60
|
+
if (context.config?.normalizationType === 'layernorm') return false;
|
|
60
61
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
61
62
|
if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
|
|
62
63
|
return false;
|
|
@@ -123,6 +124,7 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
|
|
|
123
124
|
activationDtype,
|
|
124
125
|
layerScalar,
|
|
125
126
|
}) {
|
|
127
|
+
if (config.normalizationType === 'layernorm') return false;
|
|
126
128
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
127
129
|
if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
|
|
128
130
|
return false;
|
|
@@ -176,6 +178,7 @@ function shouldUseStandardPostFfnNextInputRMSNormPairFusion({
|
|
|
176
178
|
layerScalar,
|
|
177
179
|
residualBranchScale,
|
|
178
180
|
}) {
|
|
181
|
+
if (config.normalizationType === 'layernorm') return false;
|
|
179
182
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
180
183
|
if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
|
|
181
184
|
return false;
|
|
@@ -851,6 +854,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
851
854
|
attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
|
|
852
855
|
queryPreAttnScalar: config.queryPreAttnScalar,
|
|
853
856
|
queryKeyNorm,
|
|
857
|
+
queryKeyNormType: config.queryKeyNormType,
|
|
858
|
+
queryKeyNormAxis: config.queryKeyNormAxis,
|
|
854
859
|
queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
|
|
855
860
|
valueNorm: config.valueNorm,
|
|
856
861
|
attentionOutputGate: config.attentionOutputGate,
|
|
@@ -860,6 +865,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
860
865
|
? (context.multimodalBidirectionalSpan ?? null)
|
|
861
866
|
: null,
|
|
862
867
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
868
|
+
normalizationType: config.normalizationType,
|
|
863
869
|
ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
|
|
864
870
|
ropeFrequencyBaseDim: resolveAttentionFrequencyBaseDim(config, layerType),
|
|
865
871
|
ropeInterleaved: config.ropeInterleaved,
|
|
@@ -25,6 +25,12 @@ export function rmsNormCPU(
|
|
|
25
25
|
eps?: number,
|
|
26
26
|
rmsNormWeightOffset?: boolean
|
|
27
27
|
): Float32Array;
|
|
28
|
+
export function layerNormCPU(
|
|
29
|
+
x: Float32Array,
|
|
30
|
+
weight: Float32Array,
|
|
31
|
+
bias: Float32Array,
|
|
32
|
+
eps?: number
|
|
33
|
+
): Float32Array;
|
|
28
34
|
|
|
29
35
|
/**
|
|
30
36
|
* Convert a single float16 value to float32.
|
|
@@ -25,6 +25,28 @@ export function rmsNormCPU(
|
|
|
25
25
|
return result;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
export function layerNormCPU(x, weight, bias, eps = 1e-5) {
|
|
29
|
+
if (weight.length !== x.length || bias.length !== x.length) {
|
|
30
|
+
throw new Error(
|
|
31
|
+
`LayerNorm parameter length mismatch: input=${x.length}, weight=${weight.length}, bias=${bias.length}.`
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
let sum = 0;
|
|
35
|
+
for (let i = 0; i < x.length; i++) sum += x[i];
|
|
36
|
+
const mean = sum / x.length;
|
|
37
|
+
let squaredDeviation = 0;
|
|
38
|
+
for (let i = 0; i < x.length; i++) {
|
|
39
|
+
const centered = x[i] - mean;
|
|
40
|
+
squaredDeviation += centered * centered;
|
|
41
|
+
}
|
|
42
|
+
const invStd = 1 / Math.sqrt(squaredDeviation / x.length + eps);
|
|
43
|
+
const result = new Float32Array(x.length);
|
|
44
|
+
for (let i = 0; i < x.length; i++) {
|
|
45
|
+
result[i] = (x[i] - mean) * invStd * weight[i] + bias[i];
|
|
46
|
+
}
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
|
|
28
50
|
|
|
29
51
|
export function f16BufferToF32(data) {
|
|
30
52
|
const u16 = new Uint16Array(data);
|
|
@@ -17,7 +17,7 @@ import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
|
17
17
|
export type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
18
18
|
|
|
19
19
|
// Re-export CPU functions
|
|
20
|
-
export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
20
|
+
export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
21
21
|
|
|
22
22
|
// Re-export GPU functions
|
|
23
23
|
export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
// Re-export CPU functions
|
|
4
|
-
export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
4
|
+
export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
5
5
|
|
|
6
6
|
// Re-export GPU functions
|
|
7
7
|
export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
|
|
|
12
12
|
// Imports for computeLogits orchestrator
|
|
13
13
|
import { getDevice } from '../../../../gpu/device.js';
|
|
14
14
|
import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
15
|
-
import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
|
|
15
|
+
import { runLayerNorm, runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
|
|
16
16
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
17
17
|
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
|
|
18
18
|
import { kernelTrace, traceStep } from '../kernel-trace.js';
|
|
@@ -72,6 +72,45 @@ function normalizeSelectedLogitTokenIds(value, vocabSize) {
|
|
|
72
72
|
return tokenIds;
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
+
function applyLmHeadBias(logits, bias, rowCount, rowSize, selectedTokenIds = null) {
|
|
76
|
+
if (bias == null) {
|
|
77
|
+
return logits;
|
|
78
|
+
}
|
|
79
|
+
if (!(bias instanceof Float32Array)) {
|
|
80
|
+
throw new Error('[Logits] LM head bias must be a Float32Array.');
|
|
81
|
+
}
|
|
82
|
+
if (selectedTokenIds !== null) {
|
|
83
|
+
if (logits.length !== selectedTokenIds.length) {
|
|
84
|
+
throw new Error('[Logits] Selected-logit bias application length mismatch.');
|
|
85
|
+
}
|
|
86
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
87
|
+
const tokenId = selectedTokenIds[index];
|
|
88
|
+
if (tokenId >= bias.length) {
|
|
89
|
+
throw new Error(`[Logits] LM head bias is missing token ${tokenId}.`);
|
|
90
|
+
}
|
|
91
|
+
logits[index] += bias[tokenId];
|
|
92
|
+
}
|
|
93
|
+
return logits;
|
|
94
|
+
}
|
|
95
|
+
if (bias.length < rowSize) {
|
|
96
|
+
throw new Error(
|
|
97
|
+
`[Logits] LM head bias length mismatch: expected at least ${rowSize}, got ${bias.length}.`
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
if (logits.length !== rowCount * rowSize) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
`[Logits] LM head bias matrix mismatch: expected ${rowCount * rowSize} logits, got ${logits.length}.`
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
for (let row = 0; row < rowCount; row += 1) {
|
|
106
|
+
const offset = row * rowSize;
|
|
107
|
+
for (let column = 0; column < rowSize; column += 1) {
|
|
108
|
+
logits[offset + column] += bias[column];
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return logits;
|
|
112
|
+
}
|
|
113
|
+
|
|
75
114
|
async function coerceTensorDtype(tensor, targetDtype, options = {}) {
|
|
76
115
|
if (!targetDtype || tensor.dtype === targetDtype) {
|
|
77
116
|
return tensor;
|
|
@@ -164,7 +203,7 @@ export async function computeLogits(
|
|
|
164
203
|
activationDtype: activationDtypeOverride,
|
|
165
204
|
} = config;
|
|
166
205
|
const activationDtype = activationDtypeOverride ?? getRuntimeConfig().inference.compute.activationDtype;
|
|
167
|
-
const { finalNorm, lmHead } = weights;
|
|
206
|
+
const { finalNorm, finalNormBias = null, lmHead, lmHeadBias = null } = weights;
|
|
168
207
|
const device = getDevice();
|
|
169
208
|
|
|
170
209
|
// Consistency check: warn if lmHead weight dtype disagrees with layer activation dtype.
|
|
@@ -269,6 +308,7 @@ export async function computeLogits(
|
|
|
269
308
|
cpuWeightLayout === 'column' ? cpuWeightVocabSize : null
|
|
270
309
|
)
|
|
271
310
|
: matmulCPU(normed, (lmHead), numTokens, matmulVocabSize, hiddenSize);
|
|
311
|
+
applyLmHeadBias(rawLogits, lmHeadBias, numTokens, matmulVocabSize);
|
|
272
312
|
return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
|
|
273
313
|
}
|
|
274
314
|
|
|
@@ -293,7 +333,7 @@ export async function computeLogits(
|
|
|
293
333
|
dtype: inputDtype,
|
|
294
334
|
});
|
|
295
335
|
|
|
296
|
-
// 2. Apply final
|
|
336
|
+
// 2. Apply the manifest-owned final normalization.
|
|
297
337
|
|
|
298
338
|
let normWeightBuffer;
|
|
299
339
|
let normWeightBufferOwned = false;
|
|
@@ -319,7 +359,10 @@ export async function computeLogits(
|
|
|
319
359
|
const kernelPath = config.kernelPath ?? null;
|
|
320
360
|
const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
|
|
321
361
|
const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
|
|
322
|
-
const forceStableF32Logits = !hasExplicitFinalNormPrecision &&
|
|
362
|
+
const forceStableF32Logits = !hasExplicitFinalNormPrecision && (
|
|
363
|
+
config.normalizationType === 'layernorm'
|
|
364
|
+
|| shouldForceStableF32Logits(config, inputDtype)
|
|
365
|
+
);
|
|
323
366
|
const stableKernelPath = forceStableF32Logits
|
|
324
367
|
? createStableF32LogitsKernelPath(kernelPath)
|
|
325
368
|
: kernelPath;
|
|
@@ -346,11 +389,39 @@ export async function computeLogits(
|
|
|
346
389
|
: inputTensor;
|
|
347
390
|
normInputBufferOwned = normInputTensor !== inputTensor;
|
|
348
391
|
}
|
|
349
|
-
let
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
392
|
+
let finalNormBiasBuffer = null;
|
|
393
|
+
let normedTensor;
|
|
394
|
+
if (config.normalizationType === 'layernorm') {
|
|
395
|
+
if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
|
|
396
|
+
throw new Error(
|
|
397
|
+
`[Logits] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
const effectiveBias = finalNormBias instanceof Float32Array
|
|
401
|
+
? finalNormBias
|
|
402
|
+
: new Float32Array(hiddenSize);
|
|
403
|
+
if (effectiveBias.length !== hiddenSize) {
|
|
404
|
+
throw new Error(
|
|
405
|
+
`[Logits] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveBias.length}.`
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
finalNormBiasBuffer = acquireBuffer(effectiveBias.byteLength, undefined, 'final_norm_bias');
|
|
409
|
+
device.queue.writeBuffer(finalNormBiasBuffer, 0, effectiveBias);
|
|
410
|
+
normedTensor = await runLayerNorm(
|
|
411
|
+
normInputTensor,
|
|
412
|
+
normWeightBuffer,
|
|
413
|
+
finalNormBiasBuffer,
|
|
414
|
+
rmsNormEps,
|
|
415
|
+
{ batchSize: numTokens, hiddenSize }
|
|
416
|
+
);
|
|
417
|
+
} else {
|
|
418
|
+
normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
|
|
419
|
+
batchSize: numTokens,
|
|
420
|
+
hiddenSize,
|
|
421
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
if (finalNormBiasBuffer) releaseBuffer(finalNormBiasBuffer);
|
|
354
425
|
if (normInputBufferOwned) {
|
|
355
426
|
releaseBuffer(normInputTensor.buffer);
|
|
356
427
|
}
|
|
@@ -386,7 +457,7 @@ export async function computeLogits(
|
|
|
386
457
|
|
|
387
458
|
// Trace final norm output
|
|
388
459
|
if (kernelTrace.enabled) {
|
|
389
|
-
await traceStep(
|
|
460
|
+
await traceStep(config.normalizationType, 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
|
|
390
461
|
}
|
|
391
462
|
|
|
392
463
|
// Debug: Check hidden state after final norm
|
|
@@ -461,6 +532,7 @@ export async function computeLogits(
|
|
|
461
532
|
if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
|
|
462
533
|
if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
|
|
463
534
|
|
|
535
|
+
applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
|
|
464
536
|
return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
|
|
465
537
|
}
|
|
466
538
|
|
|
@@ -533,6 +605,7 @@ export async function computeLogits(
|
|
|
533
605
|
if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
|
|
534
606
|
});
|
|
535
607
|
const selectedLogits = new Float32Array(selectedData);
|
|
608
|
+
applyLmHeadBias(selectedLogits, lmHeadBias, 1, selectedTokenIds.length, selectedTokenIds);
|
|
536
609
|
await runProbes('logits_final', selectedLogits, {
|
|
537
610
|
numTokens: 1,
|
|
538
611
|
hiddenSize: selectedTokenIds.length,
|
|
@@ -579,6 +652,7 @@ export async function computeLogits(
|
|
|
579
652
|
const rawLogits = logitsTensor.dtype === 'f16'
|
|
580
653
|
? f16BufferToF32(logitsData)
|
|
581
654
|
: new Float32Array(logitsData);
|
|
655
|
+
applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
|
|
582
656
|
if (isTraceEnabled('logits')) {
|
|
583
657
|
trace.logits('LM_HEAD_RAW_LOGITS_HEALTH', getLogitsHealth(rawLogits));
|
|
584
658
|
}
|
|
@@ -17,6 +17,8 @@ import type { WeightBuffer, CpuWeightBuffer, SplitWeightBuffer } from '../../../
|
|
|
17
17
|
export interface LogitsConfig {
|
|
18
18
|
hiddenSize: number;
|
|
19
19
|
vocabSize: number;
|
|
20
|
+
normalizationType: 'rmsnorm' | 'layernorm';
|
|
21
|
+
finalNormBiasTensor: string | null;
|
|
20
22
|
rmsNormEps: number;
|
|
21
23
|
useTiedEmbeddings: boolean;
|
|
22
24
|
embeddingVocabSize: number | null;
|
|
@@ -38,7 +40,9 @@ export interface LogitsConfig {
|
|
|
38
40
|
*/
|
|
39
41
|
export interface LogitsWeights {
|
|
40
42
|
finalNorm: GPUBuffer | Float32Array;
|
|
43
|
+
finalNormBias?: Float32Array | null;
|
|
41
44
|
lmHead: GPUBuffer | Float32Array | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer;
|
|
45
|
+
lmHeadBias?: Float32Array | null;
|
|
42
46
|
}
|
|
43
47
|
|
|
44
48
|
/**
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
4
|
runRMSNorm, runResidualAdd, runMatmul, runSiLU, runGeLU,
|
|
5
|
+
runLayerNorm, runBiasAdd,
|
|
5
6
|
recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
|
|
7
|
+
recordLayerNorm, recordBiasAdd,
|
|
6
8
|
runSiLURowSplit, recordSiLURowSplit,
|
|
7
9
|
runMatmulRMSNormFused, recordMatmulRMSNormFused,
|
|
8
10
|
runRMSNormStats, recordRMSNormStats,
|
|
@@ -60,6 +62,36 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
|
|
|
60
62
|
return result;
|
|
61
63
|
}
|
|
62
64
|
|
|
65
|
+
export async function doLayerNorm(input, weight, bias, eps, options, recorder) {
|
|
66
|
+
const result = recorder
|
|
67
|
+
? await recordLayerNorm(recorder, input, weight, bias, eps, options)
|
|
68
|
+
: await runLayerNorm(input, weight, bias, eps, options);
|
|
69
|
+
|
|
70
|
+
if (kernelTrace.enabled && !recorder) {
|
|
71
|
+
const layer = options.layerIdx ?? -1;
|
|
72
|
+
const label = options.label ?? 'layernorm';
|
|
73
|
+
await traceStep('layernorm', label, layer, result.buffer, [options.batchSize, options.hiddenSize]);
|
|
74
|
+
}
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export async function doBiasAdd(data, bias, numTokens, dim, options, recorder) {
|
|
79
|
+
const result = recorder
|
|
80
|
+
? await recordBiasAdd(recorder, data, bias, numTokens, dim, options)
|
|
81
|
+
: await runBiasAdd(data, bias, numTokens, dim, options);
|
|
82
|
+
|
|
83
|
+
if (kernelTrace.enabled && !recorder) {
|
|
84
|
+
await traceStep(
|
|
85
|
+
'bias_add',
|
|
86
|
+
options?.label ?? 'bias_add',
|
|
87
|
+
options?.layerIdx ?? -1,
|
|
88
|
+
result.buffer,
|
|
89
|
+
[numTokens, dim]
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
return result;
|
|
93
|
+
}
|
|
94
|
+
|
|
63
95
|
export async function doRMSNormStats(input, residual, eps, options, recorder) {
|
|
64
96
|
return recorder
|
|
65
97
|
? await recordRMSNormStats(recorder, input, residual, eps, options)
|
|
@@ -390,6 +390,12 @@ export interface GenerateOptions {
|
|
|
390
390
|
offset: number;
|
|
391
391
|
length: number;
|
|
392
392
|
} | null;
|
|
393
|
+
|
|
394
|
+
/** @internal Return final-normalized embeddings for every input token. */
|
|
395
|
+
__returnTokenEmbeddings?: boolean;
|
|
396
|
+
|
|
397
|
+
/** @internal Return full per-token vocabulary logits. */
|
|
398
|
+
__returnSequenceLogits?: boolean;
|
|
393
399
|
}
|
|
394
400
|
|
|
395
401
|
/**
|
|
@@ -438,6 +444,30 @@ export interface PrefillEmbeddingResult extends KVCacheSnapshot {
|
|
|
438
444
|
/** Pooling mode used to construct embedding */
|
|
439
445
|
embeddingMode: 'last' | 'mean';
|
|
440
446
|
|
|
447
|
+
/** Final-normalized token matrix when explicitly requested. */
|
|
448
|
+
tokenEmbeddings?: Float32Array | null;
|
|
449
|
+
|
|
450
|
+
/** Full token-by-vocabulary logits when explicitly requested. */
|
|
451
|
+
logits?: Float32Array | null;
|
|
452
|
+
|
|
453
|
+
phase?: WorkloadPhaseTiming | null;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
export interface SequenceEncodeOptions extends GenerateOptions {
|
|
457
|
+
includeTokenEmbeddings?: boolean;
|
|
458
|
+
includeLogits?: boolean;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
export interface SequenceEncodeResult {
|
|
462
|
+
alphabet: 'amino_acid' | 'nucleotide';
|
|
463
|
+
tokens: number[];
|
|
464
|
+
tokenMask: Uint8Array;
|
|
465
|
+
includedTokenCount: number;
|
|
466
|
+
tokenEmbeddings: Float32Array | null;
|
|
467
|
+
pooledEmbedding: Float32Array | null;
|
|
468
|
+
logits: Float32Array | null;
|
|
469
|
+
embeddingDim: number;
|
|
470
|
+
vocabSize: number;
|
|
441
471
|
phase?: WorkloadPhaseTiming | null;
|
|
442
472
|
}
|
|
443
473
|
|
|
@@ -495,16 +525,23 @@ export type LayerWeightBuffer = GPUBuffer | WeightBuffer | Float32Array | CpuWei
|
|
|
495
525
|
export interface LayerWeights {
|
|
496
526
|
// Attention
|
|
497
527
|
inputNorm: GPUBuffer | Float32Array;
|
|
528
|
+
inputNormBias?: GPUBuffer | Float32Array | null;
|
|
498
529
|
qProj: LayerWeightBuffer;
|
|
530
|
+
qProjBias?: GPUBuffer | Float32Array | null;
|
|
499
531
|
kProj: LayerWeightBuffer;
|
|
532
|
+
kProjBias?: GPUBuffer | Float32Array | null;
|
|
500
533
|
vProj?: LayerWeightBuffer;
|
|
534
|
+
vProjBias?: GPUBuffer | Float32Array | null;
|
|
501
535
|
oProj: LayerWeightBuffer;
|
|
536
|
+
oProjBias?: GPUBuffer | Float32Array | null;
|
|
502
537
|
qGateProj?: LayerWeightBuffer | null;
|
|
503
538
|
convInProj?: LayerWeightBuffer;
|
|
504
539
|
convKernel?: LayerWeightBuffer;
|
|
505
540
|
convOutProj?: LayerWeightBuffer;
|
|
506
541
|
/** Fused Q/K/V projection (runtime-generated for 3->1 matmul optimization) */
|
|
507
542
|
qkvProj?: GPUBuffer | WeightBuffer | null;
|
|
543
|
+
/** Concatenated Q/K/V projection bias matching qkvSizes. */
|
|
544
|
+
qkvProjBias?: GPUBuffer | Float32Array | null;
|
|
508
545
|
/** Sizes for splitting fused QKV output: [qSize, kSize, vSize] in elements */
|
|
509
546
|
qkvSizes?: [number, number, number];
|
|
510
547
|
/** Data type of fused QKV weights (f16, f32, or q4k) */
|
|
@@ -516,6 +553,7 @@ export interface LayerWeights {
|
|
|
516
553
|
|
|
517
554
|
// FFN (dense layers)
|
|
518
555
|
postAttentionNorm?: GPUBuffer | Float32Array;
|
|
556
|
+
postAttentionNormBias?: GPUBuffer | Float32Array | null;
|
|
519
557
|
postAttnNorm?: GPUBuffer | Float32Array; // LLaMA-style pre-FFN norm
|
|
520
558
|
gate?: LayerWeightBuffer;
|
|
521
559
|
up?: LayerWeightBuffer;
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import type { LayerWeights } from './types.js';
|
|
14
14
|
import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
15
|
+
import type { Tensor } from '../../../gpu/tensor.js';
|
|
15
16
|
|
|
16
17
|
// ============================================================================
|
|
17
18
|
// Types
|
|
@@ -60,3 +61,12 @@ export function getNormWeightBuffer(
|
|
|
60
61
|
debugFlags?: WeightDebugFlags,
|
|
61
62
|
deviceOverride?: GPUDevice | null
|
|
62
63
|
): GPUBuffer;
|
|
64
|
+
|
|
65
|
+
export function getVectorTensor(
|
|
66
|
+
weight: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | { buffer: ArrayBuffer; byteOffset: number; byteLength: number } | CpuWeightBuffer,
|
|
67
|
+
label: string,
|
|
68
|
+
length: number,
|
|
69
|
+
config: WeightBufferConfig,
|
|
70
|
+
debugFlags?: WeightDebugFlags,
|
|
71
|
+
deviceOverride?: GPUDevice | null
|
|
72
|
+
): { tensor: Tensor; owned: boolean };
|
|
@@ -3,7 +3,14 @@
|
|
|
3
3
|
import { getDevice } from '../../../gpu/device.js';
|
|
4
4
|
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
5
5
|
import { log } from '../../../debug/index.js';
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
getBufferDtype,
|
|
8
|
+
isWeightBuffer,
|
|
9
|
+
isCpuWeightBuffer,
|
|
10
|
+
isGpuBufferInstance,
|
|
11
|
+
tagBufferDtype,
|
|
12
|
+
} from '../../../gpu/weight-buffer.js';
|
|
13
|
+
import { createTensor } from '../../../gpu/tensor.js';
|
|
7
14
|
|
|
8
15
|
// ============================================================================
|
|
9
16
|
// Type Guards
|
|
@@ -116,6 +123,31 @@ export function getNormWeightBuffer(weight, label, config, debugFlags, deviceOve
|
|
|
116
123
|
}
|
|
117
124
|
}
|
|
118
125
|
|
|
126
|
+
export function getVectorTensor(weight, label, length, config, debugFlags, deviceOverride = null) {
|
|
127
|
+
if (!Number.isInteger(length) || length <= 0) {
|
|
128
|
+
throw new Error(`Vector tensor "${label}" requires a positive integer length.`);
|
|
129
|
+
}
|
|
130
|
+
const buffer = getNormWeightBuffer(weight, label, config, debugFlags, deviceOverride);
|
|
131
|
+
let dtype = getBufferDtype(buffer);
|
|
132
|
+
if (dtype == null) {
|
|
133
|
+
const bytesPerElement = buffer.size / length;
|
|
134
|
+
if (bytesPerElement === 2) dtype = 'f16';
|
|
135
|
+
else if (bytesPerElement === 4) dtype = 'f32';
|
|
136
|
+
else {
|
|
137
|
+
throw new Error(
|
|
138
|
+
`Vector tensor "${label}" cannot infer dtype from ${buffer.size} bytes for ${length} elements.`
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (dtype !== 'f16' && dtype !== 'f32') {
|
|
143
|
+
throw new Error(`Vector tensor "${label}" has unsupported dtype "${String(dtype)}".`);
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
tensor: createTensor(buffer, dtype, [length], label),
|
|
147
|
+
owned: !isGpuBufferInstance(weight) && !isWeightBuffer(weight),
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
119
151
|
|
|
120
152
|
function getGPUWeightBuffer(weight, label) {
|
|
121
153
|
// Handle WeightBuffer by extracting underlying GPUBuffer
|
|
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
|
|
|
15
15
|
import { PipelineGenerator } from './text/generator.js';
|
|
16
16
|
import type { Manifest } from './text/config.js';
|
|
17
17
|
import type { WeightLoadResult, PipelineContexts } from './text/init.js';
|
|
18
|
-
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
|
|
18
|
+
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
|
|
19
19
|
import type { ChatMessage } from './text/chat-format.js';
|
|
20
20
|
import type { LoRAAdapter } from './text/lora.js';
|
|
21
21
|
import type { DiffusionPipeline } from './diffusion/pipeline.js';
|
|
@@ -31,7 +31,7 @@ import type {
|
|
|
31
31
|
} from './text/generator.js';
|
|
32
32
|
|
|
33
33
|
// Re-export types for external use
|
|
34
|
-
export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
|
|
34
|
+
export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
|
|
35
35
|
export type { PipelineContexts };
|
|
36
36
|
|
|
37
37
|
export interface ChatRequestInput {
|
|
@@ -130,6 +130,8 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
130
130
|
embeddingMode: string;
|
|
131
131
|
}>>;
|
|
132
132
|
|
|
133
|
+
encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise<SequenceEncodeResult>;
|
|
134
|
+
|
|
133
135
|
prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
|
|
134
136
|
|
|
135
137
|
prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
|