doppler-gpu 0.4.14 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +83 -52
- package/package.json +25 -2
- package/src/cli/doppler-cli.js +253 -94
- package/src/cli/runtime-profiles.js +3 -7
- package/src/client/doppler-api.d.ts +28 -0
- package/src/client/doppler-api.js +2 -0
- package/src/client/inspection.d.ts +122 -0
- package/src/client/inspection.js +343 -0
- package/src/client/runtime/index.d.ts +54 -0
- package/src/client/runtime/index.js +190 -18
- package/src/client/runtime/model-manager.d.ts +0 -4
- package/src/client/runtime/model-manager.js +0 -28
- package/src/client/runtime/model-session.d.ts +116 -1
- package/src/client/runtime/model-session.js +351 -1
- package/src/client/runtime/model-source.d.ts +10 -1
- package/src/client/runtime/scoped-session.d.ts +117 -0
- package/src/client/runtime/scoped-session.js +257 -0
- package/src/client/runtime/types.d.ts +0 -1
- package/src/client/runtime/types.js +0 -1
- package/src/config/evidence/boundary-evidence-policy.json +54 -0
- package/src/config/evidence/source-intake-policy.json +122 -0
- package/src/config/inspection/observation-policies.json +80 -0
- package/src/config/param-validator.js +21 -36
- package/src/config/runtime/compute/f16-activations.json +1 -0
- package/src/config/runtime/compute/f16-batched.json +1 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/diagnostics/debug-logits.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
- package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
- package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
- package/src/config/runtime/model/gemma2-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline.json +1 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
- package/src/config/runtime/optimization-candidates/registry.json +5 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
- package/src/config/runtime/profiles/default.json +1 -0
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/production.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/rerank-stability.json +1 -3
- package/src/config/runtime/profiles/rerank-throughput.json +1 -3
- package/src/config/runtime/profiles/simulation.json +1 -3
- package/src/config/runtime/profiles/throughput.json +1 -3
- package/src/config/runtime/profiles/trace-layers.json +1 -3
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
- package/src/config/runtime/profiles/turboquant.json +1 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -3
- package/src/config/runtime/profiles/vector-throughput.json +1 -0
- package/src/config/runtime/profiles/verbose-trace.json +1 -3
- package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
- package/src/config/schema/benchmark.schema.d.ts +7 -0
- package/src/config/schema/benchmark.schema.js +5 -0
- package/src/config/schema/index.d.ts +8 -16
- package/src/config/schema/index.js +5 -8
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
- package/src/config/schema/kernel-thresholds.schema.js +0 -12
- package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
- package/src/config/schema/kernel-warmup.schema.js +0 -1
- package/src/config/schema/runtime-profile.schema.d.ts +24 -0
- package/src/config/schema/runtime-profile.schema.js +92 -0
- package/src/config/schema/shared-runtime.schema.d.ts +1 -4
- package/src/config/schema/shared-runtime.schema.js +0 -2
- package/src/config/schema/tooling.schema.d.ts +8 -5
- package/src/config/schema/tooling.schema.js +3 -2
- package/src/debug/tensor.d.ts +2 -0
- package/src/debug/tensor.js +21 -2
- package/src/gpu/command-recorder.d.ts +5 -0
- package/src/gpu/command-recorder.js +17 -0
- package/src/gpu/kernel-runtime.js +2 -10
- package/src/gpu/kernels/index.d.ts +0 -2
- package/src/gpu/kernels/index.js +0 -2
- package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
- package/src/gpu/kernels/kernel-prewarm.js +60 -0
- package/src/gpu/kernels/utils.d.ts +2 -4
- package/src/gpu/kernels/utils.js +2 -4
- package/src/index.d.ts +13 -1
- package/src/index.js +7 -1
- package/src/inference/browser-harness-model-helpers.js +4 -0
- package/src/inference/browser-harness-runtime-helpers.js +25 -10
- package/src/inference/browser-harness-text-helpers.js +4 -1
- package/src/inference/browser-harness.d.ts +4 -0
- package/src/inference/browser-harness.js +91 -4
- package/src/inference/observation-context.d.ts +22 -0
- package/src/inference/observation-context.js +41 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
- package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
- package/src/inference/pipelines/text/attention/index.js +7 -0
- package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
- package/src/inference/pipelines/text/attention/plan.js +328 -0
- package/src/inference/pipelines/text/attention/receipt.js +60 -0
- package/src/inference/pipelines/text/attention/record.js +8 -1278
- package/src/inference/pipelines/text/attention/run.js +8 -1386
- package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-steps.js +32 -15
- package/src/inference/pipelines/text/generator.js +10 -5
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
- package/src/inference/pipelines/text/layer.js +2 -0
- package/src/inference/pipelines/text/model-load.d.ts +0 -1
- package/src/inference/pipelines/text/model-load.js +2 -11
- package/src/inference/pipelines/text/probes.js +10 -0
- package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
- package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
- package/src/inference/pipelines/text/state.d.ts +6 -0
- package/src/inference/pipelines/text/state.js +3 -0
- package/src/inference/pipelines/text/types.d.ts +6 -0
- package/src/inference/pipelines/text.js +16 -1
- package/src/inference/refactor-receipt.d.ts +49 -0
- package/src/inference/refactor-receipt.js +188 -0
- package/src/inference/resource-scope.d.ts +33 -0
- package/src/inference/resource-scope.js +168 -0
- package/src/inference/test-harness.d.ts +3 -0
- package/src/inference/test-harness.js +1 -0
- package/src/storage/model-cache.d.ts +53 -0
- package/src/storage/model-cache.js +550 -0
- package/src/tooling/boundary-evidence.d.ts +51 -0
- package/src/tooling/boundary-evidence.js +408 -0
- package/src/tooling/browser-command-runner.js +4 -4
- package/src/tooling/command-context.d.ts +25 -0
- package/src/tooling/command-context.js +53 -0
- package/src/tooling/command-runner-shared.js +22 -5
- package/src/tooling/demo-receipts.d.ts +56 -0
- package/src/tooling/demo-receipts.js +113 -0
- package/src/tooling/diagnose-runner.js +2 -2
- package/src/tooling/execution-cost-ledger.d.ts +21 -0
- package/src/tooling/execution-cost-ledger.js +199 -0
- package/src/tooling/node-command-runner.js +2 -2
- package/src/tooling/opfs-cache.d.ts +11 -49
- package/src/tooling/opfs-cache.js +4 -510
- package/src/tooling/precision-replay-math.d.ts +42 -0
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-parity.d.ts +3 -0
- package/src/tooling/program-bundle-parity.js +8 -0
- package/src/tooling/program-bundle.d.ts +1 -52
- package/src/tooling/program-bundle.js +0 -139
- package/src/tooling/registered-variant-calibration.d.ts +24 -0
- package/src/tooling/registered-variant-calibration.js +286 -0
- package/src/tooling/runtime-input-composition.js +105 -53
- package/src/tooling/runtime-intent.js +41 -0
- package/src/tooling/runtime-optimization-index.d.ts +6 -0
- package/src/tooling/runtime-optimization-index.js +72 -0
- package/src/tooling/runtime-optimization.d.ts +47 -6
- package/src/tooling/runtime-optimization.js +559 -40
- package/src/tooling/source-intake.d.ts +49 -0
- package/src/tooling/source-intake.js +414 -0
- package/src/tooling-exports/evidence.d.ts +34 -0
- package/src/tooling-exports/evidence.js +34 -0
- package/src/tooling-exports/runtime.d.ts +7 -0
- package/src/tooling-exports/runtime.js +7 -0
- package/src/tooling-exports.d.ts +11 -0
- package/src/tooling-exports.js +6 -3
- package/src/tooling-exports.shared.d.ts +32 -0
- package/src/tooling-exports.shared.js +31 -0
- package/src/version.js +1 -1
- package/src/config/schema/tuner.schema.d.ts +0 -39
- package/src/config/schema/tuner.schema.js +0 -13
- package/src/gpu/kernel-tuner/benchmarks.js +0 -902
- package/src/gpu/kernel-tuner/cache.js +0 -145
- package/src/gpu/kernel-tuner/index.js +0 -38
- package/src/gpu/kernel-tuner/tuner.js +0 -247
- package/src/gpu/kernel-tuner.js +0 -12
- package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
- package/src/gpu/kernels/kernel-tuning.js +0 -158
- package/src/gpu/profiler.js +0 -382
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
|
@@ -14,10 +14,21 @@ import {
|
|
|
14
14
|
createHttpArtifactStorageContext,
|
|
15
15
|
createNodeFileArtifactStorageContext,
|
|
16
16
|
} from '../../storage/artifact-storage-context.js';
|
|
17
|
+
import {
|
|
18
|
+
ensureModelCachedSource,
|
|
19
|
+
loadPersistentModelSource,
|
|
20
|
+
} from '../../storage/model-cache.js';
|
|
21
|
+
import { deleteModel } from '../../storage/shard-manager.js';
|
|
22
|
+
import {
|
|
23
|
+
listRegisteredModels,
|
|
24
|
+
registerModel,
|
|
25
|
+
removeRegisteredModel,
|
|
26
|
+
} from '../../storage/registry.js';
|
|
17
27
|
import { isNodeRuntime } from '../../utils/runtime-env.js';
|
|
18
28
|
import { resolveManifestGpuResidentEmbeddingLimitError } from '../../loader/embedding-limit-preflight.js';
|
|
19
29
|
import { createDopplerLoader } from '../../loader/doppler-loader.js';
|
|
20
30
|
import { initDevice } from '../../gpu/device.js';
|
|
31
|
+
import { createScopedModelSession } from './scoped-session.js';
|
|
21
32
|
|
|
22
33
|
function emitLoadProgress(callback, phase, percent, message) {
|
|
23
34
|
if (typeof callback !== 'function') return;
|
|
@@ -32,6 +43,7 @@ function assertDopplerOptions(options, apiName) {
|
|
|
32
43
|
options.runtimeConfig !== undefined
|
|
33
44
|
|| options.runtimeProfile !== undefined
|
|
34
45
|
|| options.runtimeConfigUrl !== undefined
|
|
46
|
+
|| options.cache !== undefined
|
|
35
47
|
) {
|
|
36
48
|
throw new Error(
|
|
37
49
|
`${apiName} does not accept load-affecting options. Use doppler.load(model, options) instead.`
|
|
@@ -39,6 +51,72 @@ function assertDopplerOptions(options, apiName) {
|
|
|
39
51
|
}
|
|
40
52
|
}
|
|
41
53
|
|
|
54
|
+
function assertPersistentCacheMode(cache) {
|
|
55
|
+
if (cache === undefined || cache === false || cache === 'opfs') {
|
|
56
|
+
if (cache === 'opfs' && isNodeRuntime()) {
|
|
57
|
+
throw new Error('doppler.load options.cache="opfs" is browser-only.');
|
|
58
|
+
}
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
throw new Error('doppler.load options.cache must be false or "opfs".');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function emitPersistentCacheProgress(callback, event) {
|
|
65
|
+
const percent = Number.isFinite(event?.percent)
|
|
66
|
+
? Math.max(0, Math.min(100, Number(event.percent)))
|
|
67
|
+
: 0;
|
|
68
|
+
emitLoadProgress(
|
|
69
|
+
callback,
|
|
70
|
+
'cache',
|
|
71
|
+
15 + Math.round(percent * 0.1),
|
|
72
|
+
event?.message || 'Preparing persistent model cache'
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export async function resolvePersistentBrowserLoadSource(
|
|
77
|
+
loadSource,
|
|
78
|
+
cache,
|
|
79
|
+
onProgress = null,
|
|
80
|
+
cacheSource = ensureModelCachedSource
|
|
81
|
+
) {
|
|
82
|
+
assertPersistentCacheMode(cache);
|
|
83
|
+
if (cache !== 'opfs') {
|
|
84
|
+
return loadSource;
|
|
85
|
+
}
|
|
86
|
+
if (isNodeRuntime()) {
|
|
87
|
+
throw new Error('doppler.load options.cache="opfs" is browser-only.');
|
|
88
|
+
}
|
|
89
|
+
const storageBaseUrl = loadSource?.storageBaseUrl ?? loadSource?.baseUrl;
|
|
90
|
+
const storageManifest = loadSource?.storageManifest ?? loadSource?.manifest;
|
|
91
|
+
const storageManifestText = loadSource?.storageManifestText
|
|
92
|
+
?? (storageManifest ? JSON.stringify(storageManifest) : null);
|
|
93
|
+
const storageModelId = storageManifest?.modelId;
|
|
94
|
+
if (!storageBaseUrl || !storageManifest || !storageManifestText || !storageModelId) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
'doppler.load options.cache="opfs" requires a URL-backed artifact with a manifest modelId.'
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
const expectedManifestHash = await sha256ManifestText(storageManifestText);
|
|
100
|
+
const cached = await cacheSource(
|
|
101
|
+
storageModelId,
|
|
102
|
+
storageBaseUrl,
|
|
103
|
+
(event) => emitPersistentCacheProgress(onProgress, event),
|
|
104
|
+
{ expectedManifestHash }
|
|
105
|
+
);
|
|
106
|
+
return {
|
|
107
|
+
...loadSource,
|
|
108
|
+
storageContext: cached.storageContext,
|
|
109
|
+
storage: cached.storageContext,
|
|
110
|
+
persistentCache: {
|
|
111
|
+
backend: cached.storageBackend,
|
|
112
|
+
state: cached.cacheState,
|
|
113
|
+
fromCache: cached.fromCache,
|
|
114
|
+
manifestHash: cached.manifestHash,
|
|
115
|
+
totalBytes: cached.totalBytes,
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
42
120
|
async function resolveCachedNodeQuickstartSource(resolved, manifestPayload, onProgress) {
|
|
43
121
|
if (!isNodeRuntime()) {
|
|
44
122
|
return null;
|
|
@@ -91,6 +169,7 @@ export function createDopplerRuntimeService({
|
|
|
91
169
|
}
|
|
92
170
|
|
|
93
171
|
async function load(model, options = {}) {
|
|
172
|
+
assertPersistentCacheMode(options.cache);
|
|
94
173
|
const { userProgress, pipelineProgress } = resolveLoadProgressHandlers(options, defaultLoadProgressLogger);
|
|
95
174
|
|
|
96
175
|
emitLoadProgress(userProgress, 'resolve', 5, 'Resolving model');
|
|
@@ -98,7 +177,16 @@ export function createDopplerRuntimeService({
|
|
|
98
177
|
await ensureWebGPUAvailable();
|
|
99
178
|
|
|
100
179
|
emitLoadProgress(userProgress, 'manifest', 15, 'Fetching manifest');
|
|
101
|
-
const
|
|
180
|
+
const persistentSource = options.cache === 'opfs' && !isNodeRuntime()
|
|
181
|
+
? await loadPersistentModelSource(resolved.modelId)
|
|
182
|
+
: null;
|
|
183
|
+
const manifestPayload = persistentSource
|
|
184
|
+
? {
|
|
185
|
+
text: persistentSource.manifestText,
|
|
186
|
+
manifest: persistentSource.manifest,
|
|
187
|
+
manifestHash: persistentSource.manifestHash,
|
|
188
|
+
}
|
|
189
|
+
: resolved.manifest
|
|
102
190
|
? await (async () => {
|
|
103
191
|
const text = resolved.manifestText ?? JSON.stringify(resolved.manifest);
|
|
104
192
|
const manifestHash = await sha256ManifestText(text);
|
|
@@ -110,7 +198,21 @@ export function createDopplerRuntimeService({
|
|
|
110
198
|
return { text, manifest: resolved.manifest, manifestHash };
|
|
111
199
|
})()
|
|
112
200
|
: await fetchManifestPayloadFromBaseUrl(resolved.baseUrl);
|
|
113
|
-
const resolvedArtifactSource =
|
|
201
|
+
const resolvedArtifactSource = persistentSource
|
|
202
|
+
? {
|
|
203
|
+
...resolved,
|
|
204
|
+
manifest: persistentSource.manifest,
|
|
205
|
+
manifestText: persistentSource.manifestText,
|
|
206
|
+
storageContext: persistentSource.storageContext,
|
|
207
|
+
persistentCache: {
|
|
208
|
+
backend: 'opfs',
|
|
209
|
+
state: 'verified-hit',
|
|
210
|
+
fromCache: true,
|
|
211
|
+
manifestHash: persistentSource.manifestHash,
|
|
212
|
+
totalBytes: persistentSource.totalBytes,
|
|
213
|
+
},
|
|
214
|
+
}
|
|
215
|
+
: await resolveManifestArtifactSource(resolved, manifestPayload);
|
|
114
216
|
await initDevice();
|
|
115
217
|
const embeddingLimitError = resolveManifestGpuResidentEmbeddingLimitError(
|
|
116
218
|
resolvedArtifactSource.manifest,
|
|
@@ -129,7 +231,18 @@ export function createDopplerRuntimeService({
|
|
|
129
231
|
userProgress
|
|
130
232
|
)
|
|
131
233
|
: null;
|
|
132
|
-
const loadSource =
|
|
234
|
+
const loadSource = await resolvePersistentBrowserLoadSource(
|
|
235
|
+
cachedResolved ?? resolvedArtifactSource,
|
|
236
|
+
persistentSource ? false : options.cache,
|
|
237
|
+
userProgress
|
|
238
|
+
);
|
|
239
|
+
if (options.cache === 'opfs' && !isNodeRuntime()) {
|
|
240
|
+
await registerModel({
|
|
241
|
+
modelId: resolved.modelId,
|
|
242
|
+
manifestHash: manifestPayload.manifestHash,
|
|
243
|
+
backend: 'opfs',
|
|
244
|
+
});
|
|
245
|
+
}
|
|
133
246
|
const nodeStorageContext = await resolveNodeArtifactStorageContext(loadSource);
|
|
134
247
|
const storageContext = nodeStorageContext ?? resolveArtifactStorageContext(loadSource);
|
|
135
248
|
await storageContext?.preflight?.();
|
|
@@ -139,26 +252,37 @@ export function createDopplerRuntimeService({
|
|
|
139
252
|
? createDopplerLoader(options.runtimeConfig?.loading)
|
|
140
253
|
: null;
|
|
141
254
|
emitLoadProgress(userProgress, 'load', 25, 'Loading weights');
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
255
|
+
let pipeline;
|
|
256
|
+
try {
|
|
257
|
+
pipeline = await createPipeline(loadSource.manifest, {
|
|
258
|
+
baseUrl: effectiveBaseUrl ?? undefined,
|
|
259
|
+
storage: storageContext ?? undefined,
|
|
260
|
+
runtimeConfig: options.runtimeConfig,
|
|
261
|
+
loader: isolatedLoader ?? undefined,
|
|
262
|
+
ownsLoader: Boolean(isolatedLoader),
|
|
263
|
+
onProgress: pipelineProgress
|
|
264
|
+
? (progress) => emitLoadProgress(
|
|
265
|
+
pipelineProgress,
|
|
266
|
+
'load',
|
|
267
|
+
Math.max(25, Math.min(99, Math.round(progress.percent))),
|
|
268
|
+
progress.message || 'Loading weights'
|
|
269
|
+
)
|
|
270
|
+
: undefined,
|
|
271
|
+
});
|
|
272
|
+
} catch (error) {
|
|
273
|
+
try {
|
|
274
|
+
await storageContext?.close?.();
|
|
275
|
+
} catch {
|
|
276
|
+
// Preserve the original pipeline construction error.
|
|
277
|
+
}
|
|
278
|
+
throw error;
|
|
279
|
+
}
|
|
157
280
|
|
|
158
281
|
emitLoadProgress(userProgress, 'ready', 100, 'Model ready');
|
|
159
282
|
return createModelHandle(pipeline, {
|
|
160
283
|
...resolved,
|
|
161
284
|
manifestHash: manifestPayload.manifestHash,
|
|
285
|
+
persistentCache: loadSource.persistentCache ?? null,
|
|
162
286
|
});
|
|
163
287
|
}
|
|
164
288
|
|
|
@@ -185,6 +309,31 @@ export function createDopplerRuntimeService({
|
|
|
185
309
|
return inFlightLoadCache.get(cacheKey);
|
|
186
310
|
}
|
|
187
311
|
|
|
312
|
+
async function open(model, options = {}) {
|
|
313
|
+
return createScopedModelSession(await load(model, options));
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
async function generate(model, input, options = {}) {
|
|
317
|
+
const {
|
|
318
|
+
cache,
|
|
319
|
+
isolatedLoader,
|
|
320
|
+
onProgress,
|
|
321
|
+
runtimeConfig,
|
|
322
|
+
...generationOptions
|
|
323
|
+
} = options;
|
|
324
|
+
const session = await open(model, {
|
|
325
|
+
cache,
|
|
326
|
+
isolatedLoader,
|
|
327
|
+
onProgress,
|
|
328
|
+
runtimeConfig,
|
|
329
|
+
});
|
|
330
|
+
try {
|
|
331
|
+
return await session.generate(input, generationOptions);
|
|
332
|
+
} finally {
|
|
333
|
+
await session.close();
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
188
337
|
async function* dopplerGenerate(prompt, options = {}) {
|
|
189
338
|
if (!options || typeof options !== 'object' || options.model == null) {
|
|
190
339
|
throw new Error('doppler() requires options.model.');
|
|
@@ -200,6 +349,8 @@ export function createDopplerRuntimeService({
|
|
|
200
349
|
}
|
|
201
350
|
|
|
202
351
|
doppler.load = load;
|
|
352
|
+
doppler.open = open;
|
|
353
|
+
doppler.generate = generate;
|
|
203
354
|
doppler.text = async function text(prompt, options = {}) {
|
|
204
355
|
if (!options || typeof options !== 'object' || options.model == null) {
|
|
205
356
|
throw new Error('doppler.text() requires options.model.');
|
|
@@ -248,10 +399,31 @@ export function createDopplerRuntimeService({
|
|
|
248
399
|
const models = await listQuickstartModels();
|
|
249
400
|
return models.map((entry) => entry.modelId);
|
|
250
401
|
};
|
|
402
|
+
doppler.listModelDetails = async function listModelDetails() {
|
|
403
|
+
return listQuickstartModels();
|
|
404
|
+
};
|
|
405
|
+
doppler.listPersistentModels = async function listPersistentModels() {
|
|
406
|
+
if (isNodeRuntime()) {
|
|
407
|
+
return [];
|
|
408
|
+
}
|
|
409
|
+
return listRegisteredModels();
|
|
410
|
+
};
|
|
411
|
+
doppler.removePersistentModel = async function removePersistentModel(model) {
|
|
412
|
+
if (isNodeRuntime()) {
|
|
413
|
+
throw new Error('doppler.removePersistentModel() is browser-only.');
|
|
414
|
+
}
|
|
415
|
+
const resolved = await resolveModelSource(model);
|
|
416
|
+
await doppler.evict(resolved.modelId);
|
|
417
|
+
const removed = await deleteModel(resolved.modelId);
|
|
418
|
+
await removeRegisteredModel(resolved.modelId);
|
|
419
|
+
return removed;
|
|
420
|
+
};
|
|
251
421
|
|
|
252
422
|
return {
|
|
253
423
|
doppler,
|
|
254
424
|
load,
|
|
425
|
+
open,
|
|
426
|
+
generate,
|
|
255
427
|
clearModelCache,
|
|
256
428
|
resolveLoadProgressHandlers(options = {}) {
|
|
257
429
|
return resolveLoadProgressHandlers(options, defaultLoadProgressLogger);
|
|
@@ -16,10 +16,6 @@ export declare function verifyExplicitModelUrlMatch(
|
|
|
16
16
|
fetchRemoteManifest?: (modelUrl: string) => Promise<RDRRManifest | Record<string, unknown> | null>
|
|
17
17
|
): Promise<void>;
|
|
18
18
|
|
|
19
|
-
export declare function shouldAutoTuneKernels(
|
|
20
|
-
runtimeConfig?: Record<string, unknown> | null
|
|
21
|
-
): boolean;
|
|
22
|
-
|
|
23
19
|
export declare function extractTextModelConfig(manifest: RDRRManifest): TextModelConfig;
|
|
24
20
|
|
|
25
21
|
export declare function readOPFSFile(path: string): Promise<ArrayBuffer>;
|
|
@@ -102,10 +102,6 @@ export async function verifyExplicitModelUrlMatch(
|
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
export function shouldAutoTuneKernels(runtimeConfig = getRuntimeConfig()) {
|
|
106
|
-
return runtimeConfig?.shared?.kernelWarmup?.autoTune === true;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
105
|
export function getPipeline() {
|
|
110
106
|
return pipeline;
|
|
111
107
|
}
|
|
@@ -451,30 +447,6 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
451
447
|
DopplerCapabilities.kernelsWarmed = true;
|
|
452
448
|
}
|
|
453
449
|
|
|
454
|
-
if (
|
|
455
|
-
!DopplerCapabilities.kernelsTuned
|
|
456
|
-
&& shouldAutoTuneKernels()
|
|
457
|
-
&& typeof setTimeout !== 'undefined'
|
|
458
|
-
) {
|
|
459
|
-
DopplerCapabilities.kernelsTuned = true;
|
|
460
|
-
const tuneConfig = extractTextModelConfig(runtimeModel);
|
|
461
|
-
setTimeout(() => {
|
|
462
|
-
prepareKernelRuntime({
|
|
463
|
-
prewarm: false,
|
|
464
|
-
autoTune: true,
|
|
465
|
-
modelConfig: {
|
|
466
|
-
hiddenSize: tuneConfig.hiddenSize,
|
|
467
|
-
intermediateSize: tuneConfig.intermediateSize,
|
|
468
|
-
numHeads: tuneConfig.numHeads,
|
|
469
|
-
numKVHeads: tuneConfig.numKVHeads,
|
|
470
|
-
headDim: tuneConfig.headDim,
|
|
471
|
-
},
|
|
472
|
-
}).catch((e) => {
|
|
473
|
-
log.warn('DopplerProvider', 'Kernel auto-tune failed', e);
|
|
474
|
-
});
|
|
475
|
-
}, 0);
|
|
476
|
-
}
|
|
477
|
-
|
|
478
450
|
const gpuCaps = getKernelCapabilities();
|
|
479
451
|
const memCaps = await getMemoryCapabilities();
|
|
480
452
|
|
|
@@ -2,12 +2,86 @@ import type { InferencePipeline, KVCacheSnapshot, PromptInput } from '../../infe
|
|
|
2
2
|
import type { ChatMessage } from '../../inference/pipelines/text/chat-format.js';
|
|
3
3
|
import type { GenerateOptions } from '../../generation/index.js';
|
|
4
4
|
import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
5
|
-
import type { LogitsStepResult, PrefillResult, SequenceEncodeOptions, SequenceEncodeResult } from '../../inference/pipelines/text/types.d.ts';
|
|
5
|
+
import type { LogitsStepResult, PipelineStats, PrefillResult, SequenceEncodeOptions, SequenceEncodeResult } from '../../inference/pipelines/text/types.d.ts';
|
|
6
6
|
import type { LoRAManifest } from './types.js';
|
|
7
7
|
import type { LoRALoadOptions } from './lora.js';
|
|
8
|
+
import type { DopplerPersistentCacheReceipt } from './model-source.js';
|
|
9
|
+
import type {
|
|
10
|
+
DopplerComparisonFingerprint,
|
|
11
|
+
DopplerInspectionTokenRecord,
|
|
12
|
+
DopplerObservationPolicy,
|
|
13
|
+
DopplerWordQualityRecord,
|
|
14
|
+
} from '../inspection.js';
|
|
8
15
|
|
|
9
16
|
export type DopplerGenerateOptions = Omit<GenerateOptions, 'stopTokens'>;
|
|
10
17
|
|
|
18
|
+
export interface DopplerGenerationConfigEvidence {
|
|
19
|
+
maxTokens: number;
|
|
20
|
+
temperature: number;
|
|
21
|
+
topP: number;
|
|
22
|
+
topK: number;
|
|
23
|
+
repetitionPenalty: number;
|
|
24
|
+
repetitionPenaltyWindow: number;
|
|
25
|
+
greedyThreshold: number;
|
|
26
|
+
suppressSpecialTokens: boolean;
|
|
27
|
+
suppressSpecialLikeTokens: boolean;
|
|
28
|
+
suppressTokenIds: number[];
|
|
29
|
+
stopSequences: string[];
|
|
30
|
+
useChatTemplate: boolean;
|
|
31
|
+
useSpeculative: boolean | null;
|
|
32
|
+
seed: number | null;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface DopplerGenerationBackendIdentity {
|
|
36
|
+
backend: 'webgpu';
|
|
37
|
+
adapter: {
|
|
38
|
+
vendor: string | null;
|
|
39
|
+
architecture: string | null;
|
|
40
|
+
device: string | null;
|
|
41
|
+
description: string | null;
|
|
42
|
+
};
|
|
43
|
+
hasF16: boolean;
|
|
44
|
+
hasSubgroups: boolean;
|
|
45
|
+
maxBufferSize: number;
|
|
46
|
+
deviceEpoch: number;
|
|
47
|
+
kernelPathId: string | null;
|
|
48
|
+
kernelPathSource: string | null;
|
|
49
|
+
executionPlanId: string | null;
|
|
50
|
+
activationDtype: string | null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface DopplerGenerationEvidence {
|
|
54
|
+
schema: 'doppler_generation_evidence/v1';
|
|
55
|
+
outputText: string;
|
|
56
|
+
tokenIds: number[];
|
|
57
|
+
transcript: {
|
|
58
|
+
schema: 'doppler_generation_transcript/v1';
|
|
59
|
+
outputText: string;
|
|
60
|
+
tokenIds: number[];
|
|
61
|
+
};
|
|
62
|
+
transcriptHash: string;
|
|
63
|
+
generationConfig: DopplerGenerationConfigEvidence;
|
|
64
|
+
generationConfigHash: string;
|
|
65
|
+
runtimeProfile: {
|
|
66
|
+
schema: 'doppler_runtime_profile/v1';
|
|
67
|
+
runtime: {
|
|
68
|
+
package: 'doppler-gpu';
|
|
69
|
+
version: string;
|
|
70
|
+
surface: 'node' | 'browser';
|
|
71
|
+
};
|
|
72
|
+
model: {
|
|
73
|
+
modelId: string | null;
|
|
74
|
+
manifestHash: string | null;
|
|
75
|
+
activeAdapter: string | null;
|
|
76
|
+
};
|
|
77
|
+
backendIdentity: DopplerGenerationBackendIdentity;
|
|
78
|
+
};
|
|
79
|
+
runtimeProfileHash: string;
|
|
80
|
+
backendIdentity: DopplerGenerationBackendIdentity;
|
|
81
|
+
backendIdentityHash: string;
|
|
82
|
+
stats: PipelineStats | null;
|
|
83
|
+
}
|
|
84
|
+
|
|
11
85
|
export interface DopplerChatResponse {
|
|
12
86
|
content: string;
|
|
13
87
|
usage: {
|
|
@@ -20,8 +94,11 @@ export interface DopplerChatResponse {
|
|
|
20
94
|
export interface DopplerModelHandle {
|
|
21
95
|
generate(prompt: string, options?: DopplerGenerateOptions): AsyncGenerator<string, void, void>;
|
|
22
96
|
generateText(prompt: string, options?: DopplerGenerateOptions): Promise<string>;
|
|
97
|
+
generateWithEvidence(prompt: string, options?: DopplerGenerateOptions): Promise<DopplerGenerationEvidence>;
|
|
23
98
|
chat(messages: ChatMessage[], options?: DopplerGenerateOptions): AsyncGenerator<string, void, void>;
|
|
24
99
|
chatText(messages: ChatMessage[], options?: DopplerGenerateOptions): Promise<DopplerChatResponse>;
|
|
100
|
+
embed(prompt: string, options?: Record<string, unknown>): Promise<unknown>;
|
|
101
|
+
embedBatch(prompts: string[], options?: Record<string, unknown>): Promise<unknown>;
|
|
25
102
|
encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise<SequenceEncodeResult>;
|
|
26
103
|
resetGenerationState(): void;
|
|
27
104
|
loadLoRA(adapter: LoRAManifest | RDRRManifest | string, loadOptions?: LoRALoadOptions): Promise<void>;
|
|
@@ -49,9 +126,28 @@ export interface DopplerModelHandle {
|
|
|
49
126
|
readonly loaded: boolean;
|
|
50
127
|
readonly modelId: string;
|
|
51
128
|
readonly manifestHash: string | null;
|
|
129
|
+
readonly persistentCache: DopplerPersistentCacheReceipt | null;
|
|
52
130
|
readonly manifest: unknown;
|
|
53
131
|
readonly deviceInfo: Record<string, unknown> | null;
|
|
54
132
|
readonly supportsSequence: boolean;
|
|
133
|
+
readonly supportsEmbedding: boolean;
|
|
134
|
+
readonly supportsTranscription: boolean;
|
|
135
|
+
readonly supportsVision: boolean;
|
|
136
|
+
readonly inspect: {
|
|
137
|
+
listPolicies(): DopplerObservationPolicy[];
|
|
138
|
+
generate(
|
|
139
|
+
prompt: string,
|
|
140
|
+
options?: {
|
|
141
|
+
policyId?: string;
|
|
142
|
+
generation?: DopplerGenerateOptions;
|
|
143
|
+
topKSize?: number;
|
|
144
|
+
onEvent?: (event: {
|
|
145
|
+
type: 'inspection-complete';
|
|
146
|
+
receipt: DopplerModelInspectionReceipt;
|
|
147
|
+
}) => void;
|
|
148
|
+
}
|
|
149
|
+
): Promise<DopplerModelInspectionReceipt>;
|
|
150
|
+
};
|
|
55
151
|
readonly advanced: {
|
|
56
152
|
tokenizeText(text: string): number[];
|
|
57
153
|
prefillKV(prompt: string, options?: DopplerGenerateOptions): Promise<KVCacheSnapshot>;
|
|
@@ -95,6 +191,24 @@ export interface DopplerModelHandle {
|
|
|
95
191
|
};
|
|
96
192
|
}
|
|
97
193
|
|
|
194
|
+
export interface DopplerModelInspectionReceipt {
|
|
195
|
+
schema: 'doppler.model-inspection-receipt/v1';
|
|
196
|
+
policy: DopplerObservationPolicy;
|
|
197
|
+
fingerprint: DopplerComparisonFingerprint;
|
|
198
|
+
outputText: string;
|
|
199
|
+
generatedTokenIds: number[];
|
|
200
|
+
wallTimingMs: number;
|
|
201
|
+
performanceRepresentative: boolean;
|
|
202
|
+
tokens: DopplerInspectionTokenRecord[];
|
|
203
|
+
quality: null | {
|
|
204
|
+
wordSegmentation: 'doppler.word-segmentation/unicode-whitespace-v1';
|
|
205
|
+
aggregation: 'doppler.perplexity/summed-word-surprisal-v1';
|
|
206
|
+
rollingWindow: { unit: 'words' | 'tokens'; size: number };
|
|
207
|
+
words: DopplerWordQualityRecord[];
|
|
208
|
+
};
|
|
209
|
+
generationEvidence: DopplerGenerationEvidence;
|
|
210
|
+
}
|
|
211
|
+
|
|
98
212
|
export declare function assertSupportedGenerationOptions(options?: Record<string, unknown>): void;
|
|
99
213
|
|
|
100
214
|
export declare function createModelHandle(
|
|
@@ -102,5 +216,6 @@ export declare function createModelHandle(
|
|
|
102
216
|
resolved: {
|
|
103
217
|
modelId: string;
|
|
104
218
|
manifestHash?: string | null;
|
|
219
|
+
persistentCache?: DopplerPersistentCacheReceipt | null;
|
|
105
220
|
}
|
|
106
221
|
): DopplerModelHandle;
|