doppler-gpu 0.4.15 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +25 -2
- package/src/cli/doppler-cli.js +253 -94
- package/src/cli/runtime-profiles.js +3 -7
- package/src/client/doppler-api.d.ts +28 -0
- package/src/client/doppler-api.js +2 -0
- package/src/client/inspection.d.ts +122 -0
- package/src/client/inspection.js +343 -0
- package/src/client/runtime/index.d.ts +54 -0
- package/src/client/runtime/index.js +190 -18
- package/src/client/runtime/model-manager.d.ts +0 -4
- package/src/client/runtime/model-manager.js +0 -28
- package/src/client/runtime/model-session.d.ts +48 -1
- package/src/client/runtime/model-session.js +149 -30
- package/src/client/runtime/model-source.d.ts +10 -1
- package/src/client/runtime/scoped-session.d.ts +117 -0
- package/src/client/runtime/scoped-session.js +257 -0
- package/src/client/runtime/types.d.ts +0 -1
- package/src/client/runtime/types.js +0 -1
- package/src/config/evidence/boundary-evidence-policy.json +54 -0
- package/src/config/evidence/source-intake-policy.json +122 -0
- package/src/config/inspection/observation-policies.json +80 -0
- package/src/config/param-validator.js +21 -36
- package/src/config/runtime/compute/f16-activations.json +1 -0
- package/src/config/runtime/compute/f16-batched.json +1 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/diagnostics/debug-logits.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
- package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
- package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
- package/src/config/runtime/model/gemma2-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline.json +1 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
- package/src/config/runtime/optimization-candidates/registry.json +5 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
- package/src/config/runtime/profiles/default.json +1 -0
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/production.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/rerank-stability.json +1 -3
- package/src/config/runtime/profiles/rerank-throughput.json +1 -3
- package/src/config/runtime/profiles/simulation.json +1 -3
- package/src/config/runtime/profiles/throughput.json +1 -3
- package/src/config/runtime/profiles/trace-layers.json +1 -3
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
- package/src/config/runtime/profiles/turboquant.json +1 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -3
- package/src/config/runtime/profiles/vector-throughput.json +1 -0
- package/src/config/runtime/profiles/verbose-trace.json +1 -3
- package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
- package/src/config/schema/benchmark.schema.d.ts +7 -0
- package/src/config/schema/benchmark.schema.js +5 -0
- package/src/config/schema/index.d.ts +8 -16
- package/src/config/schema/index.js +5 -8
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
- package/src/config/schema/kernel-thresholds.schema.js +0 -12
- package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
- package/src/config/schema/kernel-warmup.schema.js +0 -1
- package/src/config/schema/runtime-profile.schema.d.ts +24 -0
- package/src/config/schema/runtime-profile.schema.js +92 -0
- package/src/config/schema/shared-runtime.schema.d.ts +1 -4
- package/src/config/schema/shared-runtime.schema.js +0 -2
- package/src/config/schema/tooling.schema.d.ts +8 -5
- package/src/config/schema/tooling.schema.js +3 -2
- package/src/debug/tensor.d.ts +2 -0
- package/src/debug/tensor.js +21 -2
- package/src/gpu/command-recorder.d.ts +5 -0
- package/src/gpu/command-recorder.js +17 -0
- package/src/gpu/kernel-runtime.js +2 -10
- package/src/gpu/kernels/index.d.ts +0 -2
- package/src/gpu/kernels/index.js +0 -2
- package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
- package/src/gpu/kernels/kernel-prewarm.js +60 -0
- package/src/gpu/kernels/utils.d.ts +2 -4
- package/src/gpu/kernels/utils.js +2 -4
- package/src/index.d.ts +13 -1
- package/src/index.js +7 -1
- package/src/inference/browser-harness-model-helpers.js +4 -0
- package/src/inference/browser-harness-runtime-helpers.js +25 -10
- package/src/inference/browser-harness-text-helpers.js +4 -1
- package/src/inference/browser-harness.d.ts +4 -0
- package/src/inference/browser-harness.js +91 -4
- package/src/inference/observation-context.d.ts +22 -0
- package/src/inference/observation-context.js +41 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
- package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
- package/src/inference/pipelines/text/attention/index.js +7 -0
- package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
- package/src/inference/pipelines/text/attention/plan.js +328 -0
- package/src/inference/pipelines/text/attention/receipt.js +60 -0
- package/src/inference/pipelines/text/attention/record.js +8 -1278
- package/src/inference/pipelines/text/attention/run.js +8 -1386
- package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-steps.js +32 -15
- package/src/inference/pipelines/text/generator.js +10 -5
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
- package/src/inference/pipelines/text/layer.js +2 -0
- package/src/inference/pipelines/text/model-load.d.ts +0 -1
- package/src/inference/pipelines/text/model-load.js +2 -11
- package/src/inference/pipelines/text/probes.js +10 -0
- package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
- package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
- package/src/inference/pipelines/text/state.d.ts +6 -0
- package/src/inference/pipelines/text/state.js +3 -0
- package/src/inference/pipelines/text/types.d.ts +6 -0
- package/src/inference/pipelines/text.js +16 -1
- package/src/inference/refactor-receipt.d.ts +49 -0
- package/src/inference/refactor-receipt.js +188 -0
- package/src/inference/resource-scope.d.ts +33 -0
- package/src/inference/resource-scope.js +168 -0
- package/src/inference/test-harness.d.ts +3 -0
- package/src/inference/test-harness.js +1 -0
- package/src/storage/model-cache.d.ts +53 -0
- package/src/storage/model-cache.js +550 -0
- package/src/tooling/boundary-evidence.d.ts +51 -0
- package/src/tooling/boundary-evidence.js +408 -0
- package/src/tooling/browser-command-runner.js +4 -4
- package/src/tooling/command-context.d.ts +25 -0
- package/src/tooling/command-context.js +53 -0
- package/src/tooling/command-runner-shared.js +22 -5
- package/src/tooling/demo-receipts.d.ts +56 -0
- package/src/tooling/demo-receipts.js +113 -0
- package/src/tooling/diagnose-runner.js +2 -2
- package/src/tooling/execution-cost-ledger.d.ts +21 -0
- package/src/tooling/execution-cost-ledger.js +199 -0
- package/src/tooling/node-command-runner.js +2 -2
- package/src/tooling/opfs-cache.d.ts +11 -49
- package/src/tooling/opfs-cache.js +4 -510
- package/src/tooling/precision-replay-math.d.ts +42 -0
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-parity.d.ts +3 -0
- package/src/tooling/program-bundle-parity.js +8 -0
- package/src/tooling/program-bundle.d.ts +1 -52
- package/src/tooling/program-bundle.js +0 -139
- package/src/tooling/registered-variant-calibration.d.ts +24 -0
- package/src/tooling/registered-variant-calibration.js +286 -0
- package/src/tooling/runtime-input-composition.js +105 -53
- package/src/tooling/runtime-intent.js +41 -0
- package/src/tooling/runtime-optimization-index.d.ts +6 -0
- package/src/tooling/runtime-optimization-index.js +72 -0
- package/src/tooling/runtime-optimization.d.ts +47 -6
- package/src/tooling/runtime-optimization.js +559 -40
- package/src/tooling/source-intake.d.ts +49 -0
- package/src/tooling/source-intake.js +414 -0
- package/src/tooling-exports/evidence.d.ts +34 -0
- package/src/tooling-exports/evidence.js +34 -0
- package/src/tooling-exports/runtime.d.ts +7 -0
- package/src/tooling-exports/runtime.js +7 -0
- package/src/tooling-exports.d.ts +11 -0
- package/src/tooling-exports.js +6 -3
- package/src/tooling-exports.shared.d.ts +32 -0
- package/src/tooling-exports.shared.js +31 -0
- package/src/version.js +1 -1
- package/src/config/schema/tuner.schema.d.ts +0 -39
- package/src/config/schema/tuner.schema.js +0 -13
- package/src/gpu/kernel-tuner/benchmarks.js +0 -902
- package/src/gpu/kernel-tuner/cache.js +0 -145
- package/src/gpu/kernel-tuner/index.js +0 -38
- package/src/gpu/kernel-tuner/tuner.js +0 -247
- package/src/gpu/kernel-tuner.js +0 -12
- package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
- package/src/gpu/kernels/kernel-tuning.js +0 -158
- package/src/gpu/profiler.js +0 -382
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
|
@@ -14,10 +14,21 @@ import {
|
|
|
14
14
|
createHttpArtifactStorageContext,
|
|
15
15
|
createNodeFileArtifactStorageContext,
|
|
16
16
|
} from '../../storage/artifact-storage-context.js';
|
|
17
|
+
import {
|
|
18
|
+
ensureModelCachedSource,
|
|
19
|
+
loadPersistentModelSource,
|
|
20
|
+
} from '../../storage/model-cache.js';
|
|
21
|
+
import { deleteModel } from '../../storage/shard-manager.js';
|
|
22
|
+
import {
|
|
23
|
+
listRegisteredModels,
|
|
24
|
+
registerModel,
|
|
25
|
+
removeRegisteredModel,
|
|
26
|
+
} from '../../storage/registry.js';
|
|
17
27
|
import { isNodeRuntime } from '../../utils/runtime-env.js';
|
|
18
28
|
import { resolveManifestGpuResidentEmbeddingLimitError } from '../../loader/embedding-limit-preflight.js';
|
|
19
29
|
import { createDopplerLoader } from '../../loader/doppler-loader.js';
|
|
20
30
|
import { initDevice } from '../../gpu/device.js';
|
|
31
|
+
import { createScopedModelSession } from './scoped-session.js';
|
|
21
32
|
|
|
22
33
|
function emitLoadProgress(callback, phase, percent, message) {
|
|
23
34
|
if (typeof callback !== 'function') return;
|
|
@@ -32,6 +43,7 @@ function assertDopplerOptions(options, apiName) {
|
|
|
32
43
|
options.runtimeConfig !== undefined
|
|
33
44
|
|| options.runtimeProfile !== undefined
|
|
34
45
|
|| options.runtimeConfigUrl !== undefined
|
|
46
|
+
|| options.cache !== undefined
|
|
35
47
|
) {
|
|
36
48
|
throw new Error(
|
|
37
49
|
`${apiName} does not accept load-affecting options. Use doppler.load(model, options) instead.`
|
|
@@ -39,6 +51,72 @@ function assertDopplerOptions(options, apiName) {
|
|
|
39
51
|
}
|
|
40
52
|
}
|
|
41
53
|
|
|
54
|
+
function assertPersistentCacheMode(cache) {
|
|
55
|
+
if (cache === undefined || cache === false || cache === 'opfs') {
|
|
56
|
+
if (cache === 'opfs' && isNodeRuntime()) {
|
|
57
|
+
throw new Error('doppler.load options.cache="opfs" is browser-only.');
|
|
58
|
+
}
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
throw new Error('doppler.load options.cache must be false or "opfs".');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function emitPersistentCacheProgress(callback, event) {
|
|
65
|
+
const percent = Number.isFinite(event?.percent)
|
|
66
|
+
? Math.max(0, Math.min(100, Number(event.percent)))
|
|
67
|
+
: 0;
|
|
68
|
+
emitLoadProgress(
|
|
69
|
+
callback,
|
|
70
|
+
'cache',
|
|
71
|
+
15 + Math.round(percent * 0.1),
|
|
72
|
+
event?.message || 'Preparing persistent model cache'
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export async function resolvePersistentBrowserLoadSource(
|
|
77
|
+
loadSource,
|
|
78
|
+
cache,
|
|
79
|
+
onProgress = null,
|
|
80
|
+
cacheSource = ensureModelCachedSource
|
|
81
|
+
) {
|
|
82
|
+
assertPersistentCacheMode(cache);
|
|
83
|
+
if (cache !== 'opfs') {
|
|
84
|
+
return loadSource;
|
|
85
|
+
}
|
|
86
|
+
if (isNodeRuntime()) {
|
|
87
|
+
throw new Error('doppler.load options.cache="opfs" is browser-only.');
|
|
88
|
+
}
|
|
89
|
+
const storageBaseUrl = loadSource?.storageBaseUrl ?? loadSource?.baseUrl;
|
|
90
|
+
const storageManifest = loadSource?.storageManifest ?? loadSource?.manifest;
|
|
91
|
+
const storageManifestText = loadSource?.storageManifestText
|
|
92
|
+
?? (storageManifest ? JSON.stringify(storageManifest) : null);
|
|
93
|
+
const storageModelId = storageManifest?.modelId;
|
|
94
|
+
if (!storageBaseUrl || !storageManifest || !storageManifestText || !storageModelId) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
'doppler.load options.cache="opfs" requires a URL-backed artifact with a manifest modelId.'
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
const expectedManifestHash = await sha256ManifestText(storageManifestText);
|
|
100
|
+
const cached = await cacheSource(
|
|
101
|
+
storageModelId,
|
|
102
|
+
storageBaseUrl,
|
|
103
|
+
(event) => emitPersistentCacheProgress(onProgress, event),
|
|
104
|
+
{ expectedManifestHash }
|
|
105
|
+
);
|
|
106
|
+
return {
|
|
107
|
+
...loadSource,
|
|
108
|
+
storageContext: cached.storageContext,
|
|
109
|
+
storage: cached.storageContext,
|
|
110
|
+
persistentCache: {
|
|
111
|
+
backend: cached.storageBackend,
|
|
112
|
+
state: cached.cacheState,
|
|
113
|
+
fromCache: cached.fromCache,
|
|
114
|
+
manifestHash: cached.manifestHash,
|
|
115
|
+
totalBytes: cached.totalBytes,
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
42
120
|
async function resolveCachedNodeQuickstartSource(resolved, manifestPayload, onProgress) {
|
|
43
121
|
if (!isNodeRuntime()) {
|
|
44
122
|
return null;
|
|
@@ -91,6 +169,7 @@ export function createDopplerRuntimeService({
|
|
|
91
169
|
}
|
|
92
170
|
|
|
93
171
|
async function load(model, options = {}) {
|
|
172
|
+
assertPersistentCacheMode(options.cache);
|
|
94
173
|
const { userProgress, pipelineProgress } = resolveLoadProgressHandlers(options, defaultLoadProgressLogger);
|
|
95
174
|
|
|
96
175
|
emitLoadProgress(userProgress, 'resolve', 5, 'Resolving model');
|
|
@@ -98,7 +177,16 @@ export function createDopplerRuntimeService({
|
|
|
98
177
|
await ensureWebGPUAvailable();
|
|
99
178
|
|
|
100
179
|
emitLoadProgress(userProgress, 'manifest', 15, 'Fetching manifest');
|
|
101
|
-
const
|
|
180
|
+
const persistentSource = options.cache === 'opfs' && !isNodeRuntime()
|
|
181
|
+
? await loadPersistentModelSource(resolved.modelId)
|
|
182
|
+
: null;
|
|
183
|
+
const manifestPayload = persistentSource
|
|
184
|
+
? {
|
|
185
|
+
text: persistentSource.manifestText,
|
|
186
|
+
manifest: persistentSource.manifest,
|
|
187
|
+
manifestHash: persistentSource.manifestHash,
|
|
188
|
+
}
|
|
189
|
+
: resolved.manifest
|
|
102
190
|
? await (async () => {
|
|
103
191
|
const text = resolved.manifestText ?? JSON.stringify(resolved.manifest);
|
|
104
192
|
const manifestHash = await sha256ManifestText(text);
|
|
@@ -110,7 +198,21 @@ export function createDopplerRuntimeService({
|
|
|
110
198
|
return { text, manifest: resolved.manifest, manifestHash };
|
|
111
199
|
})()
|
|
112
200
|
: await fetchManifestPayloadFromBaseUrl(resolved.baseUrl);
|
|
113
|
-
const resolvedArtifactSource =
|
|
201
|
+
const resolvedArtifactSource = persistentSource
|
|
202
|
+
? {
|
|
203
|
+
...resolved,
|
|
204
|
+
manifest: persistentSource.manifest,
|
|
205
|
+
manifestText: persistentSource.manifestText,
|
|
206
|
+
storageContext: persistentSource.storageContext,
|
|
207
|
+
persistentCache: {
|
|
208
|
+
backend: 'opfs',
|
|
209
|
+
state: 'verified-hit',
|
|
210
|
+
fromCache: true,
|
|
211
|
+
manifestHash: persistentSource.manifestHash,
|
|
212
|
+
totalBytes: persistentSource.totalBytes,
|
|
213
|
+
},
|
|
214
|
+
}
|
|
215
|
+
: await resolveManifestArtifactSource(resolved, manifestPayload);
|
|
114
216
|
await initDevice();
|
|
115
217
|
const embeddingLimitError = resolveManifestGpuResidentEmbeddingLimitError(
|
|
116
218
|
resolvedArtifactSource.manifest,
|
|
@@ -129,7 +231,18 @@ export function createDopplerRuntimeService({
|
|
|
129
231
|
userProgress
|
|
130
232
|
)
|
|
131
233
|
: null;
|
|
132
|
-
const loadSource =
|
|
234
|
+
const loadSource = await resolvePersistentBrowserLoadSource(
|
|
235
|
+
cachedResolved ?? resolvedArtifactSource,
|
|
236
|
+
persistentSource ? false : options.cache,
|
|
237
|
+
userProgress
|
|
238
|
+
);
|
|
239
|
+
if (options.cache === 'opfs' && !isNodeRuntime()) {
|
|
240
|
+
await registerModel({
|
|
241
|
+
modelId: resolved.modelId,
|
|
242
|
+
manifestHash: manifestPayload.manifestHash,
|
|
243
|
+
backend: 'opfs',
|
|
244
|
+
});
|
|
245
|
+
}
|
|
133
246
|
const nodeStorageContext = await resolveNodeArtifactStorageContext(loadSource);
|
|
134
247
|
const storageContext = nodeStorageContext ?? resolveArtifactStorageContext(loadSource);
|
|
135
248
|
await storageContext?.preflight?.();
|
|
@@ -139,26 +252,37 @@ export function createDopplerRuntimeService({
|
|
|
139
252
|
? createDopplerLoader(options.runtimeConfig?.loading)
|
|
140
253
|
: null;
|
|
141
254
|
emitLoadProgress(userProgress, 'load', 25, 'Loading weights');
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
255
|
+
let pipeline;
|
|
256
|
+
try {
|
|
257
|
+
pipeline = await createPipeline(loadSource.manifest, {
|
|
258
|
+
baseUrl: effectiveBaseUrl ?? undefined,
|
|
259
|
+
storage: storageContext ?? undefined,
|
|
260
|
+
runtimeConfig: options.runtimeConfig,
|
|
261
|
+
loader: isolatedLoader ?? undefined,
|
|
262
|
+
ownsLoader: Boolean(isolatedLoader),
|
|
263
|
+
onProgress: pipelineProgress
|
|
264
|
+
? (progress) => emitLoadProgress(
|
|
265
|
+
pipelineProgress,
|
|
266
|
+
'load',
|
|
267
|
+
Math.max(25, Math.min(99, Math.round(progress.percent))),
|
|
268
|
+
progress.message || 'Loading weights'
|
|
269
|
+
)
|
|
270
|
+
: undefined,
|
|
271
|
+
});
|
|
272
|
+
} catch (error) {
|
|
273
|
+
try {
|
|
274
|
+
await storageContext?.close?.();
|
|
275
|
+
} catch {
|
|
276
|
+
// Preserve the original pipeline construction error.
|
|
277
|
+
}
|
|
278
|
+
throw error;
|
|
279
|
+
}
|
|
157
280
|
|
|
158
281
|
emitLoadProgress(userProgress, 'ready', 100, 'Model ready');
|
|
159
282
|
return createModelHandle(pipeline, {
|
|
160
283
|
...resolved,
|
|
161
284
|
manifestHash: manifestPayload.manifestHash,
|
|
285
|
+
persistentCache: loadSource.persistentCache ?? null,
|
|
162
286
|
});
|
|
163
287
|
}
|
|
164
288
|
|
|
@@ -185,6 +309,31 @@ export function createDopplerRuntimeService({
|
|
|
185
309
|
return inFlightLoadCache.get(cacheKey);
|
|
186
310
|
}
|
|
187
311
|
|
|
312
|
+
async function open(model, options = {}) {
|
|
313
|
+
return createScopedModelSession(await load(model, options));
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
async function generate(model, input, options = {}) {
|
|
317
|
+
const {
|
|
318
|
+
cache,
|
|
319
|
+
isolatedLoader,
|
|
320
|
+
onProgress,
|
|
321
|
+
runtimeConfig,
|
|
322
|
+
...generationOptions
|
|
323
|
+
} = options;
|
|
324
|
+
const session = await open(model, {
|
|
325
|
+
cache,
|
|
326
|
+
isolatedLoader,
|
|
327
|
+
onProgress,
|
|
328
|
+
runtimeConfig,
|
|
329
|
+
});
|
|
330
|
+
try {
|
|
331
|
+
return await session.generate(input, generationOptions);
|
|
332
|
+
} finally {
|
|
333
|
+
await session.close();
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
188
337
|
async function* dopplerGenerate(prompt, options = {}) {
|
|
189
338
|
if (!options || typeof options !== 'object' || options.model == null) {
|
|
190
339
|
throw new Error('doppler() requires options.model.');
|
|
@@ -200,6 +349,8 @@ export function createDopplerRuntimeService({
|
|
|
200
349
|
}
|
|
201
350
|
|
|
202
351
|
doppler.load = load;
|
|
352
|
+
doppler.open = open;
|
|
353
|
+
doppler.generate = generate;
|
|
203
354
|
doppler.text = async function text(prompt, options = {}) {
|
|
204
355
|
if (!options || typeof options !== 'object' || options.model == null) {
|
|
205
356
|
throw new Error('doppler.text() requires options.model.');
|
|
@@ -248,10 +399,31 @@ export function createDopplerRuntimeService({
|
|
|
248
399
|
const models = await listQuickstartModels();
|
|
249
400
|
return models.map((entry) => entry.modelId);
|
|
250
401
|
};
|
|
402
|
+
doppler.listModelDetails = async function listModelDetails() {
|
|
403
|
+
return listQuickstartModels();
|
|
404
|
+
};
|
|
405
|
+
doppler.listPersistentModels = async function listPersistentModels() {
|
|
406
|
+
if (isNodeRuntime()) {
|
|
407
|
+
return [];
|
|
408
|
+
}
|
|
409
|
+
return listRegisteredModels();
|
|
410
|
+
};
|
|
411
|
+
doppler.removePersistentModel = async function removePersistentModel(model) {
|
|
412
|
+
if (isNodeRuntime()) {
|
|
413
|
+
throw new Error('doppler.removePersistentModel() is browser-only.');
|
|
414
|
+
}
|
|
415
|
+
const resolved = await resolveModelSource(model);
|
|
416
|
+
await doppler.evict(resolved.modelId);
|
|
417
|
+
const removed = await deleteModel(resolved.modelId);
|
|
418
|
+
await removeRegisteredModel(resolved.modelId);
|
|
419
|
+
return removed;
|
|
420
|
+
};
|
|
251
421
|
|
|
252
422
|
return {
|
|
253
423
|
doppler,
|
|
254
424
|
load,
|
|
425
|
+
open,
|
|
426
|
+
generate,
|
|
255
427
|
clearModelCache,
|
|
256
428
|
resolveLoadProgressHandlers(options = {}) {
|
|
257
429
|
return resolveLoadProgressHandlers(options, defaultLoadProgressLogger);
|
|
@@ -16,10 +16,6 @@ export declare function verifyExplicitModelUrlMatch(
|
|
|
16
16
|
fetchRemoteManifest?: (modelUrl: string) => Promise<RDRRManifest | Record<string, unknown> | null>
|
|
17
17
|
): Promise<void>;
|
|
18
18
|
|
|
19
|
-
export declare function shouldAutoTuneKernels(
|
|
20
|
-
runtimeConfig?: Record<string, unknown> | null
|
|
21
|
-
): boolean;
|
|
22
|
-
|
|
23
19
|
export declare function extractTextModelConfig(manifest: RDRRManifest): TextModelConfig;
|
|
24
20
|
|
|
25
21
|
export declare function readOPFSFile(path: string): Promise<ArrayBuffer>;
|
|
@@ -102,10 +102,6 @@ export async function verifyExplicitModelUrlMatch(
|
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
export function shouldAutoTuneKernels(runtimeConfig = getRuntimeConfig()) {
|
|
106
|
-
return runtimeConfig?.shared?.kernelWarmup?.autoTune === true;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
105
|
export function getPipeline() {
|
|
110
106
|
return pipeline;
|
|
111
107
|
}
|
|
@@ -451,30 +447,6 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
451
447
|
DopplerCapabilities.kernelsWarmed = true;
|
|
452
448
|
}
|
|
453
449
|
|
|
454
|
-
if (
|
|
455
|
-
!DopplerCapabilities.kernelsTuned
|
|
456
|
-
&& shouldAutoTuneKernels()
|
|
457
|
-
&& typeof setTimeout !== 'undefined'
|
|
458
|
-
) {
|
|
459
|
-
DopplerCapabilities.kernelsTuned = true;
|
|
460
|
-
const tuneConfig = extractTextModelConfig(runtimeModel);
|
|
461
|
-
setTimeout(() => {
|
|
462
|
-
prepareKernelRuntime({
|
|
463
|
-
prewarm: false,
|
|
464
|
-
autoTune: true,
|
|
465
|
-
modelConfig: {
|
|
466
|
-
hiddenSize: tuneConfig.hiddenSize,
|
|
467
|
-
intermediateSize: tuneConfig.intermediateSize,
|
|
468
|
-
numHeads: tuneConfig.numHeads,
|
|
469
|
-
numKVHeads: tuneConfig.numKVHeads,
|
|
470
|
-
headDim: tuneConfig.headDim,
|
|
471
|
-
},
|
|
472
|
-
}).catch((e) => {
|
|
473
|
-
log.warn('DopplerProvider', 'Kernel auto-tune failed', e);
|
|
474
|
-
});
|
|
475
|
-
}, 0);
|
|
476
|
-
}
|
|
477
|
-
|
|
478
450
|
const gpuCaps = getKernelCapabilities();
|
|
479
451
|
const memCaps = await getMemoryCapabilities();
|
|
480
452
|
|
|
@@ -5,6 +5,13 @@ import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
|
5
5
|
import type { LogitsStepResult, PipelineStats, PrefillResult, SequenceEncodeOptions, SequenceEncodeResult } from '../../inference/pipelines/text/types.d.ts';
|
|
6
6
|
import type { LoRAManifest } from './types.js';
|
|
7
7
|
import type { LoRALoadOptions } from './lora.js';
|
|
8
|
+
import type { DopplerPersistentCacheReceipt } from './model-source.js';
|
|
9
|
+
import type {
|
|
10
|
+
DopplerComparisonFingerprint,
|
|
11
|
+
DopplerInspectionTokenRecord,
|
|
12
|
+
DopplerObservationPolicy,
|
|
13
|
+
DopplerWordQualityRecord,
|
|
14
|
+
} from '../inspection.js';
|
|
8
15
|
|
|
9
16
|
export type DopplerGenerateOptions = Omit<GenerateOptions, 'stopTokens'>;
|
|
10
17
|
|
|
@@ -21,7 +28,7 @@ export interface DopplerGenerationConfigEvidence {
|
|
|
21
28
|
suppressTokenIds: number[];
|
|
22
29
|
stopSequences: string[];
|
|
23
30
|
useChatTemplate: boolean;
|
|
24
|
-
useSpeculative: boolean;
|
|
31
|
+
useSpeculative: boolean | null;
|
|
25
32
|
seed: number | null;
|
|
26
33
|
}
|
|
27
34
|
|
|
@@ -90,6 +97,8 @@ export interface DopplerModelHandle {
|
|
|
90
97
|
generateWithEvidence(prompt: string, options?: DopplerGenerateOptions): Promise<DopplerGenerationEvidence>;
|
|
91
98
|
chat(messages: ChatMessage[], options?: DopplerGenerateOptions): AsyncGenerator<string, void, void>;
|
|
92
99
|
chatText(messages: ChatMessage[], options?: DopplerGenerateOptions): Promise<DopplerChatResponse>;
|
|
100
|
+
embed(prompt: string, options?: Record<string, unknown>): Promise<unknown>;
|
|
101
|
+
embedBatch(prompts: string[], options?: Record<string, unknown>): Promise<unknown>;
|
|
93
102
|
encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise<SequenceEncodeResult>;
|
|
94
103
|
resetGenerationState(): void;
|
|
95
104
|
loadLoRA(adapter: LoRAManifest | RDRRManifest | string, loadOptions?: LoRALoadOptions): Promise<void>;
|
|
@@ -117,9 +126,28 @@ export interface DopplerModelHandle {
|
|
|
117
126
|
readonly loaded: boolean;
|
|
118
127
|
readonly modelId: string;
|
|
119
128
|
readonly manifestHash: string | null;
|
|
129
|
+
readonly persistentCache: DopplerPersistentCacheReceipt | null;
|
|
120
130
|
readonly manifest: unknown;
|
|
121
131
|
readonly deviceInfo: Record<string, unknown> | null;
|
|
122
132
|
readonly supportsSequence: boolean;
|
|
133
|
+
readonly supportsEmbedding: boolean;
|
|
134
|
+
readonly supportsTranscription: boolean;
|
|
135
|
+
readonly supportsVision: boolean;
|
|
136
|
+
readonly inspect: {
|
|
137
|
+
listPolicies(): DopplerObservationPolicy[];
|
|
138
|
+
generate(
|
|
139
|
+
prompt: string,
|
|
140
|
+
options?: {
|
|
141
|
+
policyId?: string;
|
|
142
|
+
generation?: DopplerGenerateOptions;
|
|
143
|
+
topKSize?: number;
|
|
144
|
+
onEvent?: (event: {
|
|
145
|
+
type: 'inspection-complete';
|
|
146
|
+
receipt: DopplerModelInspectionReceipt;
|
|
147
|
+
}) => void;
|
|
148
|
+
}
|
|
149
|
+
): Promise<DopplerModelInspectionReceipt>;
|
|
150
|
+
};
|
|
123
151
|
readonly advanced: {
|
|
124
152
|
tokenizeText(text: string): number[];
|
|
125
153
|
prefillKV(prompt: string, options?: DopplerGenerateOptions): Promise<KVCacheSnapshot>;
|
|
@@ -163,6 +191,24 @@ export interface DopplerModelHandle {
|
|
|
163
191
|
};
|
|
164
192
|
}
|
|
165
193
|
|
|
194
|
+
export interface DopplerModelInspectionReceipt {
|
|
195
|
+
schema: 'doppler.model-inspection-receipt/v1';
|
|
196
|
+
policy: DopplerObservationPolicy;
|
|
197
|
+
fingerprint: DopplerComparisonFingerprint;
|
|
198
|
+
outputText: string;
|
|
199
|
+
generatedTokenIds: number[];
|
|
200
|
+
wallTimingMs: number;
|
|
201
|
+
performanceRepresentative: boolean;
|
|
202
|
+
tokens: DopplerInspectionTokenRecord[];
|
|
203
|
+
quality: null | {
|
|
204
|
+
wordSegmentation: 'doppler.word-segmentation/unicode-whitespace-v1';
|
|
205
|
+
aggregation: 'doppler.perplexity/summed-word-surprisal-v1';
|
|
206
|
+
rollingWindow: { unit: 'words' | 'tokens'; size: number };
|
|
207
|
+
words: DopplerWordQualityRecord[];
|
|
208
|
+
};
|
|
209
|
+
generationEvidence: DopplerGenerationEvidence;
|
|
210
|
+
}
|
|
211
|
+
|
|
166
212
|
export declare function assertSupportedGenerationOptions(options?: Record<string, unknown>): void;
|
|
167
213
|
|
|
168
214
|
export declare function createModelHandle(
|
|
@@ -170,5 +216,6 @@ export declare function createModelHandle(
|
|
|
170
216
|
resolved: {
|
|
171
217
|
modelId: string;
|
|
172
218
|
manifestHash?: string | null;
|
|
219
|
+
persistentCache?: DopplerPersistentCacheReceipt | null;
|
|
173
220
|
}
|
|
174
221
|
): DopplerModelHandle;
|
|
@@ -3,6 +3,14 @@ import { formatChatMessages } from '../../inference/pipelines/text/chat-format.j
|
|
|
3
3
|
import { resolveSamplingConfig } from '../../inference/pipelines/text/sampling-config.js';
|
|
4
4
|
import { DOPPLER_VERSION } from '../../version.js';
|
|
5
5
|
import { isNodeRuntime } from '../../utils/runtime-env.js';
|
|
6
|
+
import {
|
|
7
|
+
MODEL_INSPECTION_RECEIPT_SCHEMA,
|
|
8
|
+
aggregateWordPerplexity,
|
|
9
|
+
buildComparisonFingerprint,
|
|
10
|
+
buildInspectionTokenRecords,
|
|
11
|
+
listObservationPolicies,
|
|
12
|
+
resolveObservationPolicy,
|
|
13
|
+
} from '../inspection.js';
|
|
6
14
|
import {
|
|
7
15
|
activateLoRAFromTrainingOutputForPipeline,
|
|
8
16
|
getActiveLoRAForPipeline,
|
|
@@ -230,9 +238,9 @@ function resolveGenerationConfigEvidence(pipeline, options) {
|
|
|
230
238
|
if (!Array.isArray(stopSequences) || stopSequences.some((value) => typeof value !== 'string')) {
|
|
231
239
|
throw new Error('Resolved Doppler stopSequences must be an array of strings.');
|
|
232
240
|
}
|
|
233
|
-
const useSpeculative = options.useSpeculative ?? generation.useSpeculative;
|
|
234
|
-
if (typeof useSpeculative !== 'boolean') {
|
|
235
|
-
throw new Error('Resolved Doppler useSpeculative must be a boolean.');
|
|
241
|
+
const useSpeculative = options.useSpeculative ?? generation.useSpeculative ?? null;
|
|
242
|
+
if (useSpeculative !== null && typeof useSpeculative !== 'boolean') {
|
|
243
|
+
throw new Error('Resolved Doppler useSpeculative must be a boolean or null.');
|
|
236
244
|
}
|
|
237
245
|
const seed = options.seed ?? null;
|
|
238
246
|
if (seed !== null && (!Number.isFinite(seed) || seed < 0)) {
|
|
@@ -263,8 +271,72 @@ function decodeGeneratedTokens(pipeline, tokenIds) {
|
|
|
263
271
|
return String(pipeline.tokenizer.decode(tokenIds, true, false));
|
|
264
272
|
}
|
|
265
273
|
|
|
266
|
-
|
|
274
|
+
function resolveInspectionBrowserIdentity() {
|
|
275
|
+
const navigatorValue = globalThis.navigator;
|
|
267
276
|
return {
|
|
277
|
+
userAgent: navigatorValue?.userAgent ?? '',
|
|
278
|
+
platform: navigatorValue?.platform ?? '',
|
|
279
|
+
language: navigatorValue?.language ?? '',
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function resolveTokenizerContract(pipeline) {
|
|
284
|
+
const contract = pipeline?.manifest?.tokenizer;
|
|
285
|
+
if (!contract || typeof contract !== 'object' || Array.isArray(contract)) {
|
|
286
|
+
throw new Error('Loaded Doppler manifest does not expose tokenizer identity.');
|
|
287
|
+
}
|
|
288
|
+
return contract;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function resolveInspectionGenerationOptions(options, policy) {
|
|
292
|
+
const generation = options?.generation ?? {};
|
|
293
|
+
if (!generation || typeof generation !== 'object' || Array.isArray(generation)) {
|
|
294
|
+
throw new Error('Doppler inspection generation options must be an object.');
|
|
295
|
+
}
|
|
296
|
+
for (const field of ['onToken', 'onLogits', 'profile', 'disableCommandBatching']) {
|
|
297
|
+
if (Object.prototype.hasOwnProperty.call(generation, field)) {
|
|
298
|
+
throw new Error(`Doppler inspection owns generation.${field} through its observation policy.`);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
const resolved = { ...generation };
|
|
302
|
+
if (policy.modifiesExecution) {
|
|
303
|
+
resolved.disableCommandBatching = true;
|
|
304
|
+
}
|
|
305
|
+
if (policy.gpuTimestampQueries) {
|
|
306
|
+
resolved.profile = true;
|
|
307
|
+
}
|
|
308
|
+
return resolved;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
export function createModelHandle(pipeline, resolved) {
|
|
312
|
+
async function generateWithEvidence(prompt, options = {}) {
|
|
313
|
+
assertSupportedGenerationOptions(options);
|
|
314
|
+
const generationConfig = resolveGenerationConfigEvidence(pipeline, options);
|
|
315
|
+
const result = await pipeline.generateTokenIds(prompt, options);
|
|
316
|
+
const tokenIds = Array.from(result?.tokenIds || [], Number);
|
|
317
|
+
const outputText = decodeGeneratedTokens(pipeline, tokenIds);
|
|
318
|
+
const stats = result?.stats || pipeline.getStats?.() || null;
|
|
319
|
+
const kernelCapabilities = typeof pipeline.getKernelCapabilities === 'function'
|
|
320
|
+
? pipeline.getKernelCapabilities()
|
|
321
|
+
: getKernelCapabilities();
|
|
322
|
+
const backendIdentity = buildGenerationBackendIdentity({
|
|
323
|
+
deviceInfo: kernelCapabilities?.adapterInfo || null,
|
|
324
|
+
kernelCapabilities,
|
|
325
|
+
stats,
|
|
326
|
+
});
|
|
327
|
+
return buildGenerationEvidence({
|
|
328
|
+
outputText,
|
|
329
|
+
tokenIds,
|
|
330
|
+
generationConfig,
|
|
331
|
+
modelId: resolved.modelId,
|
|
332
|
+
manifestHash: resolved.manifestHash || null,
|
|
333
|
+
activeAdapter: getActiveLoRAForPipeline(pipeline),
|
|
334
|
+
backendIdentity,
|
|
335
|
+
stats,
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
const handle = {
|
|
268
340
|
generate(prompt, options = {}) {
|
|
269
341
|
assertSupportedGenerationOptions(options);
|
|
270
342
|
return pipeline.generate(prompt, options);
|
|
@@ -273,32 +345,7 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
273
345
|
assertSupportedGenerationOptions(options);
|
|
274
346
|
return collectText(pipeline.generate(prompt, options));
|
|
275
347
|
},
|
|
276
|
-
|
|
277
|
-
assertSupportedGenerationOptions(options);
|
|
278
|
-
const generationConfig = resolveGenerationConfigEvidence(pipeline, options);
|
|
279
|
-
const result = await pipeline.generateTokenIds(prompt, options);
|
|
280
|
-
const tokenIds = Array.from(result?.tokenIds || [], Number);
|
|
281
|
-
const outputText = decodeGeneratedTokens(pipeline, tokenIds);
|
|
282
|
-
const stats = result?.stats || pipeline.getStats?.() || null;
|
|
283
|
-
const kernelCapabilities = typeof pipeline.getKernelCapabilities === 'function'
|
|
284
|
-
? pipeline.getKernelCapabilities()
|
|
285
|
-
: getKernelCapabilities();
|
|
286
|
-
const backendIdentity = buildGenerationBackendIdentity({
|
|
287
|
-
deviceInfo: kernelCapabilities?.adapterInfo || null,
|
|
288
|
-
kernelCapabilities,
|
|
289
|
-
stats,
|
|
290
|
-
});
|
|
291
|
-
return buildGenerationEvidence({
|
|
292
|
-
outputText,
|
|
293
|
-
tokenIds,
|
|
294
|
-
generationConfig,
|
|
295
|
-
modelId: resolved.modelId,
|
|
296
|
-
manifestHash: resolved.manifestHash || null,
|
|
297
|
-
activeAdapter: getActiveLoRAForPipeline(pipeline),
|
|
298
|
-
backendIdentity,
|
|
299
|
-
stats,
|
|
300
|
-
});
|
|
301
|
-
},
|
|
348
|
+
generateWithEvidence,
|
|
302
349
|
chat(messages, options = {}) {
|
|
303
350
|
assertSupportedGenerationOptions(options);
|
|
304
351
|
return pipeline.generate(messages, options);
|
|
@@ -390,6 +437,9 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
390
437
|
get manifestHash() {
|
|
391
438
|
return resolved.manifestHash || null;
|
|
392
439
|
},
|
|
440
|
+
get persistentCache() {
|
|
441
|
+
return resolved.persistentCache || null;
|
|
442
|
+
},
|
|
393
443
|
get manifest() {
|
|
394
444
|
return pipeline.manifest;
|
|
395
445
|
},
|
|
@@ -429,4 +479,73 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
429
479
|
},
|
|
430
480
|
},
|
|
431
481
|
};
|
|
482
|
+
|
|
483
|
+
handle.inspect = {
|
|
484
|
+
listPolicies() {
|
|
485
|
+
return listObservationPolicies();
|
|
486
|
+
},
|
|
487
|
+
async generate(prompt, options = {}) {
|
|
488
|
+
if (typeof prompt !== 'string' || !prompt.trim()) {
|
|
489
|
+
throw new Error('Doppler model.inspect.generate requires a non-empty string prompt.');
|
|
490
|
+
}
|
|
491
|
+
const policy = resolveObservationPolicy(options.policyId);
|
|
492
|
+
const generationOptions = resolveInspectionGenerationOptions(options, policy);
|
|
493
|
+
assertSupportedGenerationOptions(generationOptions);
|
|
494
|
+
const logitsByStep = [];
|
|
495
|
+
if (policy.requiredCaptures.includes('selected-token-probabilities')) {
|
|
496
|
+
generationOptions.onLogits = (logits) => {
|
|
497
|
+
logitsByStep.push(Float32Array.from(logits));
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
const startedAt = performance.now();
|
|
501
|
+
const evidence = await generateWithEvidence(prompt, generationOptions);
|
|
502
|
+
const completedAt = performance.now();
|
|
503
|
+
const promptTokenIds = tokenizeText(pipeline, prompt);
|
|
504
|
+
const tokenRecords = policy.perplexity
|
|
505
|
+
? buildInspectionTokenRecords(
|
|
506
|
+
evidence.tokenIds,
|
|
507
|
+
logitsByStep,
|
|
508
|
+
pipeline.tokenizer,
|
|
509
|
+
Number.isInteger(options.topKSize) ? options.topKSize : 5
|
|
510
|
+
)
|
|
511
|
+
: [];
|
|
512
|
+
const quality = policy.perplexity
|
|
513
|
+
? aggregateWordPerplexity(tokenRecords, {
|
|
514
|
+
windowUnit: policy.perplexity.rollingWindow.unit,
|
|
515
|
+
windowSize: policy.perplexity.rollingWindow.size,
|
|
516
|
+
})
|
|
517
|
+
: null;
|
|
518
|
+
const fingerprint = buildComparisonFingerprint({
|
|
519
|
+
artifact: {
|
|
520
|
+
modelId: resolved.modelId,
|
|
521
|
+
manifestHash: resolved.manifestHash,
|
|
522
|
+
},
|
|
523
|
+
tokenizer: resolveTokenizerContract(pipeline),
|
|
524
|
+
promptTokenIds,
|
|
525
|
+
sampling: evidence.generationConfig,
|
|
526
|
+
observationPolicyId: policy.id,
|
|
527
|
+
execution: evidence.backendIdentity,
|
|
528
|
+
browser: resolveInspectionBrowserIdentity(),
|
|
529
|
+
adapter: evidence.backendIdentity.adapter,
|
|
530
|
+
});
|
|
531
|
+
const receipt = {
|
|
532
|
+
schema: MODEL_INSPECTION_RECEIPT_SCHEMA,
|
|
533
|
+
policy,
|
|
534
|
+
fingerprint,
|
|
535
|
+
outputText: evidence.outputText,
|
|
536
|
+
generatedTokenIds: [...evidence.tokenIds],
|
|
537
|
+
wallTimingMs: completedAt - startedAt,
|
|
538
|
+
performanceRepresentative: policy.performanceRepresentative,
|
|
539
|
+
tokens: tokenRecords,
|
|
540
|
+
quality,
|
|
541
|
+
generationEvidence: evidence,
|
|
542
|
+
};
|
|
543
|
+
if (typeof options.onEvent === 'function') {
|
|
544
|
+
options.onEvent({ type: 'inspection-complete', receipt });
|
|
545
|
+
}
|
|
546
|
+
return receipt;
|
|
547
|
+
},
|
|
548
|
+
};
|
|
549
|
+
|
|
550
|
+
return handle;
|
|
432
551
|
}
|
|
@@ -2,11 +2,19 @@ import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
|
2
2
|
import type { SourceStorageContext } from '../../tooling/source-runtime-bundle.js';
|
|
3
3
|
|
|
4
4
|
export interface DopplerLoadProgress {
|
|
5
|
-
phase: 'resolve' | 'manifest' | 'load' | 'ready';
|
|
5
|
+
phase: 'resolve' | 'manifest' | 'cache' | 'load' | 'ready';
|
|
6
6
|
percent: number;
|
|
7
7
|
message: string;
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
+
export interface DopplerPersistentCacheReceipt {
|
|
11
|
+
backend: 'opfs';
|
|
12
|
+
state: 'hit' | 'verified-hit' | 'manifest-refresh' | 'imported';
|
|
13
|
+
fromCache: boolean;
|
|
14
|
+
manifestHash: string;
|
|
15
|
+
totalBytes: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
10
18
|
export interface DopplerModelSourceResolution {
|
|
11
19
|
modelId: string;
|
|
12
20
|
baseUrl: string | null;
|
|
@@ -46,6 +54,7 @@ export interface DopplerLoadOptions {
|
|
|
46
54
|
onProgress?: (event: DopplerLoadProgress) => void;
|
|
47
55
|
runtimeConfig?: Record<string, unknown>;
|
|
48
56
|
isolatedLoader?: boolean;
|
|
57
|
+
cache?: false | 'opfs';
|
|
49
58
|
}
|
|
50
59
|
|
|
51
60
|
export declare function createDefaultNodeLoadProgressLogger(): (event: DopplerLoadProgress) => void;
|