doppler-gpu 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/capsule-runtime.d.ts +2 -0
- package/src/capsule-runtime.js +1 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +2 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.d.ts +2 -0
- package/src/client/doppler-api.js +1 -0
- package/src/client/model-host/model-session.d.ts +1 -0
- package/src/client/model-host/model-session.js +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +15 -14
- package/src/client/runtime/capsule-operation-executor.js +3 -2
- package/src/client/runtime/capsule-program-adapter.js +2 -0
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/composition-root.d.ts +6 -3
- package/src/client/runtime/composition-root.js +13 -5
- package/src/client/runtime/session-controller.d.ts +6 -17
- package/src/client/runtime/session-controller.js +42 -120
- package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
- package/src/config/capsule-loading.d.ts +1 -1
- package/src/config/capsule-loading.schema.json +3 -2
- package/src/config/capsule-operation.js +6 -1
- package/src/config/capsule-operations.json +1 -2
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/kernel-path-loader.d.ts +7 -0
- package/src/config/kernel-path-loader.js +19 -0
- package/src/config/kernels/kernel-ref-digests.js +16 -16
- package/src/config/kernels/registry.json +35 -10
- package/src/config/schema/doppler.schema.js +1 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/converter/forge-qualification.js +2 -1
- package/src/converter/forge-stages.js +6 -4
- package/src/converter/quantizer.js +18 -4
- package/src/gpu/device-state.js +8 -0
- package/src/gpu/device.js +6 -1
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +11 -1
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +31 -16
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/index.d.ts +2 -0
- package/src/index.js +1 -0
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
- package/src/inference/pipelines/text/ffn/dense.js +2 -2
- package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
- package/src/inference/pipelines/text/generator/decode.js +18 -6
- package/src/inference/pipelines/text/generator/training.js +3 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.js +18 -19
- package/src/inference/pipelines/text/generator-steps.js +15 -8
- package/src/inference/pipelines/text/generator.js +16 -10
- package/src/inference/pipelines/text/linear-attention.js +24 -0
- package/src/inference/pipelines/text/logits/output-transform.js +10 -3
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/types.d.ts +3 -3
- package/src/inference/token-sampling.js +319 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/tooling/node-browser/static-server.js +1 -0
- package/src/tooling-exports/storage.d.ts +2 -0
- package/src/tooling-exports/storage.js +1 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/test-page.js +1 -0
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import { computeCanonicalSha256, hashBytesSha256 } from '../../formats/canonical-hash.js';
|
|
2
2
|
import { assertCapsuleLoadActive, waitForCapsuleRead } from './capsule-acquisition.js';
|
|
3
|
+
import { normalizeCapsuleLoadingPolicy } from '../../config/capsule-loading.js';
|
|
3
4
|
|
|
4
5
|
export function createVerifiedCapsuleArtifactStore(capsule, source, options = {}) {
|
|
5
6
|
if (typeof source?.readArtifact !== 'function') throw new Error('Capsule execution requires artifactStore.readArtifact().');
|
|
7
|
+
const { maxRetainedArtifactBytes } = normalizeCapsuleLoadingPolicy(options);
|
|
6
8
|
const artifacts = new Map(capsule.artifacts.map((artifact) => [artifact.artifactId, Object.freeze(structuredClone(artifact))]));
|
|
7
9
|
const verified = new Map();
|
|
10
|
+
const pending = new Map();
|
|
8
11
|
let closed = false;
|
|
9
|
-
const metrics = { sourceBytes: 0, hashedBytes: 0, copiedBytes: 0, retainedBytes: 0, peakRetainedBytes: 0, returnedBytes: 0
|
|
12
|
+
const metrics = { sourceBytes: 0, hashedBytes: 0, copiedBytes: 0, retainedBytes: 0, peakRetainedBytes: 0, returnedBytes: 0,
|
|
13
|
+
evictions: 0, sourceReadMs: 0, hashingMs: 0, copyingMs: 0 };
|
|
10
14
|
function assertActive() {
|
|
11
15
|
if (closed) throw new Error('Verified Capsule artifact store is closed.');
|
|
12
16
|
assertCapsuleLoadActive(options.signal);
|
|
@@ -18,27 +22,49 @@ export function createVerifiedCapsuleArtifactStore(capsule, source, options = {}
|
|
|
18
22
|
return declared;
|
|
19
23
|
}
|
|
20
24
|
async function verifiedBytes(declared) {
|
|
21
|
-
|
|
25
|
+
const retained = verified.get(declared.hash);
|
|
26
|
+
if (retained) {
|
|
27
|
+
if (retained.byteLength !== declared.sizeBytes) throw new Error('Capsule artifact size disagrees with shared content.');
|
|
28
|
+
verified.delete(declared.hash); verified.set(declared.hash, retained);
|
|
29
|
+
return retained;
|
|
30
|
+
}
|
|
31
|
+
let task = pending.get(declared.hash);
|
|
22
32
|
if (!task) {
|
|
23
33
|
task = (async () => {
|
|
34
|
+
let started = performance.now();
|
|
24
35
|
const payload = await waitForCapsuleRead(source.readArtifact(declared, options), options.signal);
|
|
36
|
+
metrics.sourceReadMs += performance.now() - started;
|
|
25
37
|
assertActive();
|
|
26
38
|
if (!(payload instanceof Uint8Array) && !(payload instanceof ArrayBuffer)) throw new Error('Capsule artifact source must return bytes.');
|
|
27
39
|
// Buffer.slice() aliases its source; always take an owned Uint8Array copy.
|
|
40
|
+
started = performance.now();
|
|
28
41
|
const bytes = payload instanceof Uint8Array ? Uint8Array.from(payload) : new Uint8Array(payload.slice(0));
|
|
42
|
+
metrics.copyingMs += performance.now() - started;
|
|
29
43
|
metrics.sourceBytes += bytes.byteLength;
|
|
30
44
|
metrics.copiedBytes += bytes.byteLength;
|
|
31
45
|
if (bytes.byteLength !== declared.sizeBytes) throw new Error(`Capsule artifact hash or size mismatch for "${declared.path}".`);
|
|
32
46
|
metrics.hashedBytes += bytes.byteLength;
|
|
33
|
-
|
|
47
|
+
started = performance.now();
|
|
48
|
+
const hash = hashBytesSha256(bytes);
|
|
49
|
+
metrics.hashingMs += performance.now() - started;
|
|
50
|
+
if (hash !== declared.hash) {
|
|
34
51
|
throw new Error(`Capsule artifact hash or size mismatch for "${declared.path}".`);
|
|
35
52
|
}
|
|
36
|
-
|
|
37
|
-
|
|
53
|
+
assertActive();
|
|
54
|
+
if (maxRetainedArtifactBytes === null || bytes.byteLength <= maxRetainedArtifactBytes) {
|
|
55
|
+
while (maxRetainedArtifactBytes !== null && metrics.retainedBytes + bytes.byteLength > maxRetainedArtifactBytes) {
|
|
56
|
+
const [hash, evicted] = verified.entries().next().value;
|
|
57
|
+
verified.delete(hash); metrics.retainedBytes -= evicted.byteLength; metrics.evictions += 1;
|
|
58
|
+
}
|
|
59
|
+
verified.set(declared.hash, bytes);
|
|
60
|
+
metrics.retainedBytes += bytes.byteLength;
|
|
61
|
+
metrics.peakRetainedBytes = Math.max(metrics.peakRetainedBytes, metrics.retainedBytes);
|
|
62
|
+
}
|
|
38
63
|
return bytes;
|
|
39
64
|
})();
|
|
40
|
-
|
|
41
|
-
|
|
65
|
+
pending.set(declared.hash, task);
|
|
66
|
+
const remove = () => { if (pending.get(declared.hash) === task) pending.delete(declared.hash); };
|
|
67
|
+
task.then(remove, remove);
|
|
42
68
|
}
|
|
43
69
|
const bytes = await task;
|
|
44
70
|
assertActive();
|
|
@@ -55,7 +81,10 @@ export function createVerifiedCapsuleArtifactStore(capsule, source, options = {}
|
|
|
55
81
|
assertActive();
|
|
56
82
|
metrics.copiedBytes += length;
|
|
57
83
|
metrics.returnedBytes += length;
|
|
58
|
-
|
|
84
|
+
const started = performance.now();
|
|
85
|
+
const result = bytes.slice(offset, offset + length);
|
|
86
|
+
metrics.copyingMs += performance.now() - started;
|
|
87
|
+
return result;
|
|
59
88
|
}
|
|
60
89
|
return {
|
|
61
90
|
readArtifact(artifact) { return readArtifactRange(artifact, 0, resolveArtifact(artifact).sizeBytes); },
|
|
@@ -67,6 +96,6 @@ export function createVerifiedCapsuleArtifactStore(capsule, source, options = {}
|
|
|
67
96
|
return { hash: declared.hash, sizeBytes: bytes.byteLength };
|
|
68
97
|
},
|
|
69
98
|
getMetrics() { return Object.freeze({ ...metrics }); },
|
|
70
|
-
close() { closed = true; verified.clear(); metrics.retainedBytes = 0; },
|
|
99
|
+
close() { closed = true; verified.clear(); pending.clear(); metrics.retainedBytes = 0; },
|
|
71
100
|
};
|
|
72
101
|
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export interface CapsuleLoadingPolicy { loadTimeoutMs: number | null; maxMetadataBytes: number }
|
|
1
|
+
export interface CapsuleLoadingPolicy { loadTimeoutMs: number | null; maxMetadataBytes: number; maxRetainedArtifactBytes: number | null }
|
|
2
2
|
export declare function normalizeCapsuleLoadingPolicy(options: Partial<CapsuleLoadingPolicy>): Readonly<CapsuleLoadingPolicy>;
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
"additionalProperties": false,
|
|
6
6
|
"properties": {
|
|
7
7
|
"loadTimeoutMs": { "type": ["integer", "null"], "minimum": 1, "maximum": 2147483647, "default": null },
|
|
8
|
-
"maxMetadataBytes": { "type": "integer", "minimum": 1, "maximum": 2147483647, "default": 16777216 }
|
|
8
|
+
"maxMetadataBytes": { "type": "integer", "minimum": 1, "maximum": 2147483647, "default": 16777216 },
|
|
9
|
+
"maxRetainedArtifactBytes": { "type": ["integer", "null"], "minimum": 0, "maximum": 9007199254740991, "default": null }
|
|
9
10
|
},
|
|
10
|
-
"required": ["loadTimeoutMs", "maxMetadataBytes"]
|
|
11
|
+
"required": ["loadTimeoutMs", "maxMetadataBytes", "maxRetainedArtifactBytes"]
|
|
11
12
|
}
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import catalog from './capsule-operations.json' with { type: 'json' };
|
|
2
2
|
import { computeCanonicalSha256 } from '../formats/canonical-hash.js';
|
|
3
3
|
import { freezeCapsuleV2 } from './capsule-v2.js';
|
|
4
|
+
import { GENERATION_CONTRACT } from './generation-contract.js';
|
|
4
5
|
|
|
5
6
|
export const CAPSULE_OPERATION_REQUEST_SCHEMA = 'doppler.capsule-operation-request/v1';
|
|
6
7
|
export const CAPSULE_OPERATION_RECEIPT_SCHEMA = 'doppler.capsule-operation-receipt/v1';
|
|
7
8
|
export const CAPSULE_OPERATION_EVENT_SCHEMA = 'doppler.capsule-operation-event/v1';
|
|
8
|
-
export const CAPSULE_OPERATIONS = freezeCapsuleV2(catalog.operations
|
|
9
|
+
export const CAPSULE_OPERATIONS = freezeCapsuleV2({ ...catalog.operations, generate: {
|
|
10
|
+
...catalog.operations.generate,
|
|
11
|
+
inputFields: Object.keys(GENERATION_CONTRACT.input),
|
|
12
|
+
optionFields: Object.keys(GENERATION_CONTRACT.options),
|
|
13
|
+
} });
|
|
9
14
|
|
|
10
15
|
// Receipt serialization only: no tensor arithmetic or model-policy selection.
|
|
11
16
|
export function normalizeCapsuleObservation(value, depth = 0, ancestors = new Set()) {
|
|
@@ -4,8 +4,7 @@
|
|
|
4
4
|
"operations": {
|
|
5
5
|
"generate": {
|
|
6
6
|
"version": 1,
|
|
7
|
-
"
|
|
8
|
-
"optionFields": ["maxTokens", "maxSeqLen", "temperature", "topP", "topK", "repetitionPenalty", "repetitionPenaltyWindow", "useChatTemplate", "seed", "suppressTokenIds", "stopSequences"]
|
|
7
|
+
"contract": "doppler.generation-contract/v1"
|
|
9
8
|
},
|
|
10
9
|
"embed": { "version": 1, "inputFields": ["texts", "application"], "optionFields": [] },
|
|
11
10
|
"rerank": { "version": 1, "inputFields": ["query", "documents", "application"], "optionFields": [] },
|
|
@@ -252,7 +252,7 @@
|
|
|
252
252
|
"q4_decode_gemv": {
|
|
253
253
|
"kernel": "fused_matmul_q4.wgsl",
|
|
254
254
|
"entry": "main_gemv",
|
|
255
|
-
"digest": "sha256:
|
|
255
|
+
"digest": "sha256:02029c2254c8ddb4c2051c4322e6b11ddbcf555e27d233e7fe73277edfa0f3f7",
|
|
256
256
|
"precision": {
|
|
257
257
|
"inputDtype": "f32",
|
|
258
258
|
"outputDtype": "f32"
|
|
@@ -279,7 +279,7 @@
|
|
|
279
279
|
"gelu": {
|
|
280
280
|
"kernel": "gelu.wgsl",
|
|
281
281
|
"entry": "main",
|
|
282
|
-
"digest": "sha256:
|
|
282
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
283
283
|
"constants": {
|
|
284
284
|
"HAS_GATE": true
|
|
285
285
|
}
|
|
@@ -379,7 +379,7 @@
|
|
|
379
379
|
"sample": {
|
|
380
380
|
"kernel": "sample.wgsl",
|
|
381
381
|
"entry": "sample_single_pass",
|
|
382
|
-
"digest": "sha256:
|
|
382
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
383
383
|
}
|
|
384
384
|
},
|
|
385
385
|
"preLayer": [
|
|
@@ -210,7 +210,7 @@
|
|
|
210
210
|
"gelu": {
|
|
211
211
|
"kernel": "gelu.wgsl",
|
|
212
212
|
"entry": "main",
|
|
213
|
-
"digest": "sha256:
|
|
213
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
214
214
|
"constants": {
|
|
215
215
|
"HAS_GATE": true
|
|
216
216
|
}
|
|
@@ -229,7 +229,7 @@
|
|
|
229
229
|
},
|
|
230
230
|
"normalization": {
|
|
231
231
|
"type": "layernorm",
|
|
232
|
-
"rmsNormEps":
|
|
232
|
+
"rmsNormEps": 1e-05,
|
|
233
233
|
"rmsNormWeightOffset": false,
|
|
234
234
|
"postAttentionNorm": true,
|
|
235
235
|
"preFeedforwardNorm": false,
|
|
@@ -370,8 +370,16 @@
|
|
|
370
370
|
"execution": {
|
|
371
371
|
"inlineKernelPath": true,
|
|
372
372
|
"mechanismKernels": [
|
|
373
|
-
"rope_precompute",
|
|
374
|
-
"
|
|
373
|
+
"rope_precompute",
|
|
374
|
+
"gather_vec4",
|
|
375
|
+
"scale",
|
|
376
|
+
"scale_inplace",
|
|
377
|
+
"bias_add",
|
|
378
|
+
"split_qkv",
|
|
379
|
+
"rope_qk",
|
|
380
|
+
"attention_general",
|
|
381
|
+
"residual_vec4",
|
|
382
|
+
"embedding_pool"
|
|
375
383
|
],
|
|
376
384
|
"policies": {
|
|
377
385
|
"unsupportedPrecision": "error",
|
|
@@ -380,39 +388,48 @@
|
|
|
380
388
|
},
|
|
381
389
|
"kernels": {
|
|
382
390
|
"rope_precompute": {
|
|
383
|
-
"kernel": "rope_precompute.wgsl",
|
|
391
|
+
"kernel": "rope_precompute.wgsl",
|
|
392
|
+
"entry": "main",
|
|
384
393
|
"digest": "sha256:0ccd038aa7e69bf064d599e86f93100735cc32fba20a942cb1deed5da5a692e8"
|
|
385
394
|
},
|
|
386
395
|
"gather_vec4": {
|
|
387
|
-
"kernel": "gather_vec4.wgsl",
|
|
396
|
+
"kernel": "gather_vec4.wgsl",
|
|
397
|
+
"entry": "gather_vec4",
|
|
388
398
|
"digest": "sha256:e8d90b826350a9feb96476fc4a5e568aebfc29948cf07a2d7db0dee4a4216b6a"
|
|
389
399
|
},
|
|
390
400
|
"scale": {
|
|
391
|
-
"kernel": "scale.wgsl",
|
|
401
|
+
"kernel": "scale.wgsl",
|
|
402
|
+
"entry": "main",
|
|
392
403
|
"digest": "sha256:8070894b1488c76bdf6685b99243e0fadd79a2511b52b22b91ba1ead9990831c"
|
|
393
404
|
},
|
|
394
405
|
"scale_inplace": {
|
|
395
|
-
"kernel": "scale.wgsl",
|
|
406
|
+
"kernel": "scale.wgsl",
|
|
407
|
+
"entry": "main_inplace",
|
|
396
408
|
"digest": "sha256:c1dcc2a3c04fe3836457724ccc45dbbee9423ad3710febcf91107f3a23c46cb2"
|
|
397
409
|
},
|
|
398
410
|
"split_qkv": {
|
|
399
|
-
"kernel": "split_qkv.wgsl",
|
|
411
|
+
"kernel": "split_qkv.wgsl",
|
|
412
|
+
"entry": "main",
|
|
400
413
|
"digest": "sha256:57d1e55354e019598c7c39b98b9a48c99269a59506cca3a376fd690b96e15d54"
|
|
401
414
|
},
|
|
402
415
|
"rope_qk": {
|
|
403
|
-
"kernel": "rope_qk.wgsl",
|
|
416
|
+
"kernel": "rope_qk.wgsl",
|
|
417
|
+
"entry": "main",
|
|
404
418
|
"digest": "sha256:379186c35451a1bf08f3d157db52e48f4865c92caee230656fd55b7fa500351c"
|
|
405
419
|
},
|
|
406
420
|
"attention_general": {
|
|
407
|
-
"kernel": "attention.wgsl",
|
|
421
|
+
"kernel": "attention.wgsl",
|
|
422
|
+
"entry": "main",
|
|
408
423
|
"digest": "sha256:9e066a9a201a8b10707995a4b64322c68641873afb017e8f96d5249edc6caf55"
|
|
409
424
|
},
|
|
410
425
|
"residual_vec4": {
|
|
411
|
-
"kernel": "residual_vec4.wgsl",
|
|
426
|
+
"kernel": "residual_vec4.wgsl",
|
|
427
|
+
"entry": "add_vec4",
|
|
412
428
|
"digest": "sha256:99e8217beb9ebc661ae8da6dde06196c0746e884421fe803b1a8bcea22e753b1"
|
|
413
429
|
},
|
|
414
430
|
"embedding_pool": {
|
|
415
|
-
"kernel": "embedding_pool_f32.wgsl",
|
|
431
|
+
"kernel": "embedding_pool_f32.wgsl",
|
|
432
|
+
"entry": "main",
|
|
416
433
|
"digest": "sha256:0e5769590d038ee8b9ebea1600114de9b5044d7f26cd323d54ab14f0536e9c10"
|
|
417
434
|
},
|
|
418
435
|
"embed": {
|
|
@@ -456,9 +473,10 @@
|
|
|
456
473
|
"gelu": {
|
|
457
474
|
"kernel": "gelu.wgsl",
|
|
458
475
|
"entry": "main",
|
|
459
|
-
"digest": "sha256:
|
|
476
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
460
477
|
"constants": {
|
|
461
|
-
"HAS_GATE": false
|
|
478
|
+
"HAS_GATE": false,
|
|
479
|
+
"GELU_ERF": true
|
|
462
480
|
}
|
|
463
481
|
}
|
|
464
482
|
},
|
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
"gelu": {
|
|
187
187
|
"kernel": "gelu.wgsl",
|
|
188
188
|
"entry": "main",
|
|
189
|
-
"digest": "sha256:
|
|
189
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
190
190
|
"constants": {
|
|
191
191
|
"HAS_GATE": true
|
|
192
192
|
}
|
|
@@ -224,7 +224,7 @@
|
|
|
224
224
|
"sample": {
|
|
225
225
|
"kernel": "sample.wgsl",
|
|
226
226
|
"entry": "sample_single_pass",
|
|
227
|
-
"digest": "sha256:
|
|
227
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
228
228
|
}
|
|
229
229
|
},
|
|
230
230
|
"preLayer": [
|
|
@@ -192,7 +192,7 @@
|
|
|
192
192
|
"gelu": {
|
|
193
193
|
"kernel": "gelu.wgsl",
|
|
194
194
|
"entry": "main",
|
|
195
|
-
"digest": "sha256:
|
|
195
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
196
196
|
"constants": {
|
|
197
197
|
"HAS_GATE": true
|
|
198
198
|
}
|
|
@@ -235,7 +235,7 @@
|
|
|
235
235
|
"sample": {
|
|
236
236
|
"kernel": "sample.wgsl",
|
|
237
237
|
"entry": "sample_single_pass",
|
|
238
|
-
"digest": "sha256:
|
|
238
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
239
239
|
}
|
|
240
240
|
},
|
|
241
241
|
"preLayer": [
|
|
@@ -191,7 +191,7 @@
|
|
|
191
191
|
"gelu": {
|
|
192
192
|
"kernel": "gelu.wgsl",
|
|
193
193
|
"entry": "main",
|
|
194
|
-
"digest": "sha256:
|
|
194
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
195
195
|
"constants": {
|
|
196
196
|
"HAS_GATE": true
|
|
197
197
|
}
|
|
@@ -229,7 +229,7 @@
|
|
|
229
229
|
"sample": {
|
|
230
230
|
"kernel": "sample.wgsl",
|
|
231
231
|
"entry": "sample_single_pass",
|
|
232
|
-
"digest": "sha256:
|
|
232
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
233
233
|
}
|
|
234
234
|
},
|
|
235
235
|
"preLayer": [
|
|
@@ -201,7 +201,7 @@
|
|
|
201
201
|
"gelu": {
|
|
202
202
|
"kernel": "gelu.wgsl",
|
|
203
203
|
"entry": "main",
|
|
204
|
-
"digest": "sha256:
|
|
204
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
205
205
|
"constants": {
|
|
206
206
|
"HAS_GATE": true
|
|
207
207
|
}
|
|
@@ -261,7 +261,7 @@
|
|
|
261
261
|
"sample": {
|
|
262
262
|
"kernel": "sample.wgsl",
|
|
263
263
|
"entry": "sample_single_pass",
|
|
264
|
-
"digest": "sha256:
|
|
264
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
265
265
|
}
|
|
266
266
|
},
|
|
267
267
|
"preLayer": [
|
|
@@ -192,7 +192,7 @@
|
|
|
192
192
|
"gelu": {
|
|
193
193
|
"kernel": "gelu.wgsl",
|
|
194
194
|
"entry": "main",
|
|
195
|
-
"digest": "sha256:
|
|
195
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
196
196
|
"constants": {
|
|
197
197
|
"HAS_GATE": true
|
|
198
198
|
}
|
|
@@ -230,7 +230,7 @@
|
|
|
230
230
|
"sample": {
|
|
231
231
|
"kernel": "sample.wgsl",
|
|
232
232
|
"entry": "sample_single_pass",
|
|
233
|
-
"digest": "sha256:
|
|
233
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
234
234
|
}
|
|
235
235
|
},
|
|
236
236
|
"preLayer": [
|
|
@@ -193,7 +193,7 @@
|
|
|
193
193
|
"gelu": {
|
|
194
194
|
"kernel": "gelu.wgsl",
|
|
195
195
|
"entry": "main",
|
|
196
|
-
"digest": "sha256:
|
|
196
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
197
197
|
"constants": {
|
|
198
198
|
"HAS_GATE": true
|
|
199
199
|
}
|
|
@@ -223,7 +223,7 @@
|
|
|
223
223
|
"sample": {
|
|
224
224
|
"kernel": "sample.wgsl",
|
|
225
225
|
"entry": "sample_single_pass",
|
|
226
|
-
"digest": "sha256:
|
|
226
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
227
227
|
}
|
|
228
228
|
},
|
|
229
229
|
"preLayer": [
|
|
@@ -193,7 +193,7 @@
|
|
|
193
193
|
"gelu": {
|
|
194
194
|
"kernel": "gelu.wgsl",
|
|
195
195
|
"entry": "main",
|
|
196
|
-
"digest": "sha256:
|
|
196
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
197
197
|
"constants": {
|
|
198
198
|
"HAS_GATE": true
|
|
199
199
|
}
|
|
@@ -228,7 +228,7 @@
|
|
|
228
228
|
"sample": {
|
|
229
229
|
"kernel": "sample.wgsl",
|
|
230
230
|
"entry": "sample_single_pass",
|
|
231
|
-
"digest": "sha256:
|
|
231
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
232
232
|
}
|
|
233
233
|
},
|
|
234
234
|
"preLayer": [
|
|
@@ -341,7 +341,7 @@
|
|
|
341
341
|
"q4_decode_gemv": {
|
|
342
342
|
"kernel": "fused_matmul_q4.wgsl",
|
|
343
343
|
"entry": "main_gemv",
|
|
344
|
-
"digest": "sha256:
|
|
344
|
+
"digest": "sha256:02029c2254c8ddb4c2051c4322e6b11ddbcf555e27d233e7fe73277edfa0f3f7",
|
|
345
345
|
"precision": {
|
|
346
346
|
"inputDtype": "f32",
|
|
347
347
|
"outputDtype": "f32"
|
|
@@ -368,7 +368,7 @@
|
|
|
368
368
|
"gelu": {
|
|
369
369
|
"kernel": "gelu.wgsl",
|
|
370
370
|
"entry": "main",
|
|
371
|
-
"digest": "sha256:
|
|
371
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
372
372
|
"constants": {
|
|
373
373
|
"HAS_GATE": true
|
|
374
374
|
}
|
|
@@ -458,7 +458,7 @@
|
|
|
458
458
|
"sample": {
|
|
459
459
|
"kernel": "sample.wgsl",
|
|
460
460
|
"entry": "sample_single_pass",
|
|
461
|
-
"digest": "sha256:
|
|
461
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
462
462
|
}
|
|
463
463
|
},
|
|
464
464
|
"preLayer": [
|
|
@@ -334,7 +334,7 @@
|
|
|
334
334
|
"q4_decode_gemv": {
|
|
335
335
|
"kernel": "fused_matmul_q4.wgsl",
|
|
336
336
|
"entry": "main_gemv",
|
|
337
|
-
"digest": "sha256:
|
|
337
|
+
"digest": "sha256:02029c2254c8ddb4c2051c4322e6b11ddbcf555e27d233e7fe73277edfa0f3f7",
|
|
338
338
|
"precision": {
|
|
339
339
|
"inputDtype": "f32",
|
|
340
340
|
"outputDtype": "f32"
|
|
@@ -361,7 +361,7 @@
|
|
|
361
361
|
"gelu": {
|
|
362
362
|
"kernel": "gelu.wgsl",
|
|
363
363
|
"entry": "main",
|
|
364
|
-
"digest": "sha256:
|
|
364
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
365
365
|
"constants": {
|
|
366
366
|
"HAS_GATE": true
|
|
367
367
|
}
|
|
@@ -451,7 +451,7 @@
|
|
|
451
451
|
"sample": {
|
|
452
452
|
"kernel": "sample.wgsl",
|
|
453
453
|
"entry": "sample_single_pass",
|
|
454
|
-
"digest": "sha256:
|
|
454
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
455
455
|
}
|
|
456
456
|
},
|
|
457
457
|
"preLayer": [
|
|
@@ -365,7 +365,7 @@
|
|
|
365
365
|
"gelu": {
|
|
366
366
|
"kernel": "gelu_f16.wgsl",
|
|
367
367
|
"entry": "main",
|
|
368
|
-
"digest": "sha256:
|
|
368
|
+
"digest": "sha256:1f713a30bf6777270e9386792bde2a13824d37284abd0ece1abb1de7e2807fd2",
|
|
369
369
|
"constants": {
|
|
370
370
|
"HAS_GATE": true
|
|
371
371
|
}
|
|
@@ -461,12 +461,12 @@
|
|
|
461
461
|
"sample": {
|
|
462
462
|
"kernel": "sample.wgsl",
|
|
463
463
|
"entry": "sample_single_pass",
|
|
464
|
-
"digest": "sha256:
|
|
464
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
465
465
|
},
|
|
466
466
|
"lm_head_q4": {
|
|
467
467
|
"kernel": "fused_matmul_q4.wgsl",
|
|
468
468
|
"entry": "main_gemv",
|
|
469
|
-
"digest": "sha256:
|
|
469
|
+
"digest": "sha256:02029c2254c8ddb4c2051c4322e6b11ddbcf555e27d233e7fe73277edfa0f3f7",
|
|
470
470
|
"constants": {
|
|
471
471
|
"COLS_PER_WG": 64,
|
|
472
472
|
"THREADS_PER_COL_GEMV": 4
|
|
@@ -475,7 +475,7 @@
|
|
|
475
475
|
"q4_decode_gemv_stable": {
|
|
476
476
|
"kernel": "fused_matmul_q4.wgsl",
|
|
477
477
|
"entry": "main_gemv",
|
|
478
|
-
"digest": "sha256:
|
|
478
|
+
"digest": "sha256:02029c2254c8ddb4c2051c4322e6b11ddbcf555e27d233e7fe73277edfa0f3f7",
|
|
479
479
|
"precision": {
|
|
480
480
|
"inputDtype": "f32",
|
|
481
481
|
"outputDtype": "f32"
|
|
@@ -512,7 +512,7 @@
|
|
|
512
512
|
"fused_ffn_q4k_f16": {
|
|
513
513
|
"kernel": "fused_ffn_q4k_f16.wgsl",
|
|
514
514
|
"entry": "main",
|
|
515
|
-
"digest": "sha256:
|
|
515
|
+
"digest": "sha256:be4b8f38c80d23e560ab321b777eb6fd39135177dc06c330327d037c8c40cc88",
|
|
516
516
|
"precision": {
|
|
517
517
|
"inputDtype": "f16",
|
|
518
518
|
"outputDtype": "f16"
|
|
@@ -357,7 +357,7 @@
|
|
|
357
357
|
"gelu": {
|
|
358
358
|
"kernel": "gelu.wgsl",
|
|
359
359
|
"entry": "main",
|
|
360
|
-
"digest": "sha256:
|
|
360
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
361
361
|
"constants": {
|
|
362
362
|
"HAS_GATE": true
|
|
363
363
|
}
|
|
@@ -437,7 +437,7 @@
|
|
|
437
437
|
"sample": {
|
|
438
438
|
"kernel": "sample.wgsl",
|
|
439
439
|
"entry": "sample_single_pass",
|
|
440
|
-
"digest": "sha256:
|
|
440
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
441
441
|
}
|
|
442
442
|
},
|
|
443
443
|
"preLayer": [
|
|
@@ -311,7 +311,7 @@
|
|
|
311
311
|
"q4_decode_gemv": {
|
|
312
312
|
"kernel": "fused_matmul_q4.wgsl",
|
|
313
313
|
"entry": "main_gemv",
|
|
314
|
-
"digest": "sha256:
|
|
314
|
+
"digest": "sha256:02029c2254c8ddb4c2051c4322e6b11ddbcf555e27d233e7fe73277edfa0f3f7",
|
|
315
315
|
"precision": {
|
|
316
316
|
"inputDtype": "f32",
|
|
317
317
|
"outputDtype": "f32"
|
|
@@ -338,7 +338,7 @@
|
|
|
338
338
|
"gelu": {
|
|
339
339
|
"kernel": "gelu.wgsl",
|
|
340
340
|
"entry": "main",
|
|
341
|
-
"digest": "sha256:
|
|
341
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
342
342
|
"constants": {
|
|
343
343
|
"HAS_GATE": true
|
|
344
344
|
}
|
|
@@ -428,7 +428,7 @@
|
|
|
428
428
|
"sample": {
|
|
429
429
|
"kernel": "sample.wgsl",
|
|
430
430
|
"entry": "sample_single_pass",
|
|
431
|
-
"digest": "sha256:
|
|
431
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
432
432
|
}
|
|
433
433
|
},
|
|
434
434
|
"preLayer": [
|
|
@@ -307,7 +307,7 @@
|
|
|
307
307
|
"q4_decode_gemv": {
|
|
308
308
|
"kernel": "fused_matmul_q4.wgsl",
|
|
309
309
|
"entry": "main_gemv",
|
|
310
|
-
"digest": "sha256:
|
|
310
|
+
"digest": "sha256:02029c2254c8ddb4c2051c4322e6b11ddbcf555e27d233e7fe73277edfa0f3f7",
|
|
311
311
|
"precision": {
|
|
312
312
|
"inputDtype": "f32",
|
|
313
313
|
"outputDtype": "f32"
|
|
@@ -334,7 +334,7 @@
|
|
|
334
334
|
"gelu": {
|
|
335
335
|
"kernel": "gelu.wgsl",
|
|
336
336
|
"entry": "main",
|
|
337
|
-
"digest": "sha256:
|
|
337
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
338
338
|
"constants": {
|
|
339
339
|
"HAS_GATE": true
|
|
340
340
|
}
|
|
@@ -424,7 +424,7 @@
|
|
|
424
424
|
"sample": {
|
|
425
425
|
"kernel": "sample.wgsl",
|
|
426
426
|
"entry": "sample_single_pass",
|
|
427
|
-
"digest": "sha256:
|
|
427
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
428
428
|
}
|
|
429
429
|
},
|
|
430
430
|
"preLayer": [
|
|
@@ -253,7 +253,7 @@
|
|
|
253
253
|
"q4_decode_gemv": {
|
|
254
254
|
"kernel": "fused_matmul_q4.wgsl",
|
|
255
255
|
"entry": "main_gemv",
|
|
256
|
-
"digest": "sha256:
|
|
256
|
+
"digest": "sha256:02029c2254c8ddb4c2051c4322e6b11ddbcf555e27d233e7fe73277edfa0f3f7",
|
|
257
257
|
"precision": {
|
|
258
258
|
"inputDtype": "f32",
|
|
259
259
|
"outputDtype": "f32"
|
|
@@ -280,7 +280,7 @@
|
|
|
280
280
|
"gelu": {
|
|
281
281
|
"kernel": "gelu.wgsl",
|
|
282
282
|
"entry": "main",
|
|
283
|
-
"digest": "sha256:
|
|
283
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
284
284
|
"constants": {
|
|
285
285
|
"HAS_GATE": true
|
|
286
286
|
}
|
|
@@ -370,7 +370,7 @@
|
|
|
370
370
|
"sample": {
|
|
371
371
|
"kernel": "sample.wgsl",
|
|
372
372
|
"entry": "sample_single_pass",
|
|
373
|
-
"digest": "sha256:
|
|
373
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
374
374
|
}
|
|
375
375
|
},
|
|
376
376
|
"preLayer": [
|
|
@@ -264,7 +264,7 @@
|
|
|
264
264
|
"gelu": {
|
|
265
265
|
"kernel": "gelu.wgsl",
|
|
266
266
|
"entry": "main",
|
|
267
|
-
"digest": "sha256:
|
|
267
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
268
268
|
"constants": {
|
|
269
269
|
"HAS_GATE": true
|
|
270
270
|
}
|
|
@@ -345,7 +345,7 @@
|
|
|
345
345
|
"sample": {
|
|
346
346
|
"kernel": "sample.wgsl",
|
|
347
347
|
"entry": "sample_single_pass",
|
|
348
|
-
"digest": "sha256:
|
|
348
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
349
349
|
}
|
|
350
350
|
},
|
|
351
351
|
"preLayer": [
|