doppler-gpu 0.4.14 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +83 -52
- package/package.json +25 -2
- package/src/cli/doppler-cli.js +253 -94
- package/src/cli/runtime-profiles.js +3 -7
- package/src/client/doppler-api.d.ts +28 -0
- package/src/client/doppler-api.js +2 -0
- package/src/client/inspection.d.ts +122 -0
- package/src/client/inspection.js +343 -0
- package/src/client/runtime/index.d.ts +54 -0
- package/src/client/runtime/index.js +190 -18
- package/src/client/runtime/model-manager.d.ts +0 -4
- package/src/client/runtime/model-manager.js +0 -28
- package/src/client/runtime/model-session.d.ts +116 -1
- package/src/client/runtime/model-session.js +351 -1
- package/src/client/runtime/model-source.d.ts +10 -1
- package/src/client/runtime/scoped-session.d.ts +117 -0
- package/src/client/runtime/scoped-session.js +257 -0
- package/src/client/runtime/types.d.ts +0 -1
- package/src/client/runtime/types.js +0 -1
- package/src/config/evidence/boundary-evidence-policy.json +54 -0
- package/src/config/evidence/source-intake-policy.json +122 -0
- package/src/config/inspection/observation-policies.json +80 -0
- package/src/config/param-validator.js +21 -36
- package/src/config/runtime/compute/f16-activations.json +1 -0
- package/src/config/runtime/compute/f16-batched.json +1 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/diagnostics/debug-logits.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
- package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
- package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
- package/src/config/runtime/model/gemma2-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline.json +1 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
- package/src/config/runtime/optimization-candidates/registry.json +5 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
- package/src/config/runtime/profiles/default.json +1 -0
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/production.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/rerank-stability.json +1 -3
- package/src/config/runtime/profiles/rerank-throughput.json +1 -3
- package/src/config/runtime/profiles/simulation.json +1 -3
- package/src/config/runtime/profiles/throughput.json +1 -3
- package/src/config/runtime/profiles/trace-layers.json +1 -3
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
- package/src/config/runtime/profiles/turboquant.json +1 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -3
- package/src/config/runtime/profiles/vector-throughput.json +1 -0
- package/src/config/runtime/profiles/verbose-trace.json +1 -3
- package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
- package/src/config/schema/benchmark.schema.d.ts +7 -0
- package/src/config/schema/benchmark.schema.js +5 -0
- package/src/config/schema/index.d.ts +8 -16
- package/src/config/schema/index.js +5 -8
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
- package/src/config/schema/kernel-thresholds.schema.js +0 -12
- package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
- package/src/config/schema/kernel-warmup.schema.js +0 -1
- package/src/config/schema/runtime-profile.schema.d.ts +24 -0
- package/src/config/schema/runtime-profile.schema.js +92 -0
- package/src/config/schema/shared-runtime.schema.d.ts +1 -4
- package/src/config/schema/shared-runtime.schema.js +0 -2
- package/src/config/schema/tooling.schema.d.ts +8 -5
- package/src/config/schema/tooling.schema.js +3 -2
- package/src/debug/tensor.d.ts +2 -0
- package/src/debug/tensor.js +21 -2
- package/src/gpu/command-recorder.d.ts +5 -0
- package/src/gpu/command-recorder.js +17 -0
- package/src/gpu/kernel-runtime.js +2 -10
- package/src/gpu/kernels/index.d.ts +0 -2
- package/src/gpu/kernels/index.js +0 -2
- package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
- package/src/gpu/kernels/kernel-prewarm.js +60 -0
- package/src/gpu/kernels/utils.d.ts +2 -4
- package/src/gpu/kernels/utils.js +2 -4
- package/src/index.d.ts +13 -1
- package/src/index.js +7 -1
- package/src/inference/browser-harness-model-helpers.js +4 -0
- package/src/inference/browser-harness-runtime-helpers.js +25 -10
- package/src/inference/browser-harness-text-helpers.js +4 -1
- package/src/inference/browser-harness.d.ts +4 -0
- package/src/inference/browser-harness.js +91 -4
- package/src/inference/observation-context.d.ts +22 -0
- package/src/inference/observation-context.js +41 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
- package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
- package/src/inference/pipelines/text/attention/index.js +7 -0
- package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
- package/src/inference/pipelines/text/attention/plan.js +328 -0
- package/src/inference/pipelines/text/attention/receipt.js +60 -0
- package/src/inference/pipelines/text/attention/record.js +8 -1278
- package/src/inference/pipelines/text/attention/run.js +8 -1386
- package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-steps.js +32 -15
- package/src/inference/pipelines/text/generator.js +10 -5
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
- package/src/inference/pipelines/text/layer.js +2 -0
- package/src/inference/pipelines/text/model-load.d.ts +0 -1
- package/src/inference/pipelines/text/model-load.js +2 -11
- package/src/inference/pipelines/text/probes.js +10 -0
- package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
- package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
- package/src/inference/pipelines/text/state.d.ts +6 -0
- package/src/inference/pipelines/text/state.js +3 -0
- package/src/inference/pipelines/text/types.d.ts +6 -0
- package/src/inference/pipelines/text.js +16 -1
- package/src/inference/refactor-receipt.d.ts +49 -0
- package/src/inference/refactor-receipt.js +188 -0
- package/src/inference/resource-scope.d.ts +33 -0
- package/src/inference/resource-scope.js +168 -0
- package/src/inference/test-harness.d.ts +3 -0
- package/src/inference/test-harness.js +1 -0
- package/src/storage/model-cache.d.ts +53 -0
- package/src/storage/model-cache.js +550 -0
- package/src/tooling/boundary-evidence.d.ts +51 -0
- package/src/tooling/boundary-evidence.js +408 -0
- package/src/tooling/browser-command-runner.js +4 -4
- package/src/tooling/command-context.d.ts +25 -0
- package/src/tooling/command-context.js +53 -0
- package/src/tooling/command-runner-shared.js +22 -5
- package/src/tooling/demo-receipts.d.ts +56 -0
- package/src/tooling/demo-receipts.js +113 -0
- package/src/tooling/diagnose-runner.js +2 -2
- package/src/tooling/execution-cost-ledger.d.ts +21 -0
- package/src/tooling/execution-cost-ledger.js +199 -0
- package/src/tooling/node-command-runner.js +2 -2
- package/src/tooling/opfs-cache.d.ts +11 -49
- package/src/tooling/opfs-cache.js +4 -510
- package/src/tooling/precision-replay-math.d.ts +42 -0
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-parity.d.ts +3 -0
- package/src/tooling/program-bundle-parity.js +8 -0
- package/src/tooling/program-bundle.d.ts +1 -52
- package/src/tooling/program-bundle.js +0 -139
- package/src/tooling/registered-variant-calibration.d.ts +24 -0
- package/src/tooling/registered-variant-calibration.js +286 -0
- package/src/tooling/runtime-input-composition.js +105 -53
- package/src/tooling/runtime-intent.js +41 -0
- package/src/tooling/runtime-optimization-index.d.ts +6 -0
- package/src/tooling/runtime-optimization-index.js +72 -0
- package/src/tooling/runtime-optimization.d.ts +47 -6
- package/src/tooling/runtime-optimization.js +559 -40
- package/src/tooling/source-intake.d.ts +49 -0
- package/src/tooling/source-intake.js +414 -0
- package/src/tooling-exports/evidence.d.ts +34 -0
- package/src/tooling-exports/evidence.js +34 -0
- package/src/tooling-exports/runtime.d.ts +7 -0
- package/src/tooling-exports/runtime.js +7 -0
- package/src/tooling-exports.d.ts +11 -0
- package/src/tooling-exports.js +6 -3
- package/src/tooling-exports.shared.d.ts +32 -0
- package/src/tooling-exports.shared.js +31 -0
- package/src/version.js +1 -1
- package/src/config/schema/tuner.schema.d.ts +0 -39
- package/src/config/schema/tuner.schema.js +0 -13
- package/src/gpu/kernel-tuner/benchmarks.js +0 -902
- package/src/gpu/kernel-tuner/cache.js +0 -145
- package/src/gpu/kernel-tuner/index.js +0 -38
- package/src/gpu/kernel-tuner/tuner.js +0 -247
- package/src/gpu/kernel-tuner.js +0 -12
- package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
- package/src/gpu/kernels/kernel-tuning.js +0 -158
- package/src/gpu/profiler.js +0 -382
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
|
@@ -79,7 +79,10 @@ export async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
export function recordPrefillProfileStep(state, entry) {
|
|
82
|
-
if (
|
|
82
|
+
if (
|
|
83
|
+
(!entry?.timings || Object.keys(entry.timings).length === 0)
|
|
84
|
+
&& !entry?.recorderStats
|
|
85
|
+
) return;
|
|
83
86
|
if (!state.stats.prefillProfileSteps) {
|
|
84
87
|
state.stats.prefillProfileSteps = [];
|
|
85
88
|
}
|
|
@@ -153,6 +153,8 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
|
|
|
153
153
|
runtimeComputeConfig: effectiveComputeConfig,
|
|
154
154
|
activationDtype: effectiveActivationDtype,
|
|
155
155
|
kernelPath: activeKernelPath,
|
|
156
|
+
resolvedRuntimeSession: state.resolvedRuntimeSession,
|
|
157
|
+
observationContext: state.observationContext,
|
|
156
158
|
executionPolicies: state.executionV1State?.policies ?? null,
|
|
157
159
|
debugLayers: resolvedDebugLayers,
|
|
158
160
|
finitenessBuffer: state.finitenessBuffer,
|
|
@@ -176,7 +176,7 @@ function shouldLogProfileStep(state, step) {
|
|
|
176
176
|
}
|
|
177
177
|
|
|
178
178
|
function recordDecodeProfileStep(state, entry) {
|
|
179
|
-
if (!entry || !entry.timings) return;
|
|
179
|
+
if (!entry || (!entry.timings && !entry.recorderStats)) return;
|
|
180
180
|
if (!state.stats.decodeProfileSteps) {
|
|
181
181
|
state.stats.decodeProfileSteps = [];
|
|
182
182
|
}
|
|
@@ -555,18 +555,25 @@ async function submitDecodeRecorderProfile(state, opts, recorder, profileLabel)
|
|
|
555
555
|
}
|
|
556
556
|
await recorder.submitAndWait();
|
|
557
557
|
|
|
558
|
-
if (!opts.profile
|
|
558
|
+
if (!opts.profile) {
|
|
559
559
|
return;
|
|
560
560
|
}
|
|
561
561
|
|
|
562
|
-
const timings =
|
|
562
|
+
const timings = recorder.isProfilingEnabled()
|
|
563
|
+
? await recorder.resolveProfileTimings()
|
|
564
|
+
: null;
|
|
563
565
|
const total = sumProfileTimings(timings);
|
|
564
566
|
if (total !== null) {
|
|
565
567
|
state.stats.gpuTimeDecodeMs = (state.stats.gpuTimeDecodeMs ?? 0) + total;
|
|
566
568
|
}
|
|
567
|
-
if (timings) {
|
|
568
|
-
recordDecodeProfileStep(state, {
|
|
569
|
-
|
|
569
|
+
if (timings || opts.executionObserver) {
|
|
570
|
+
recordDecodeProfileStep(state, {
|
|
571
|
+
step: state.decodeStepCount,
|
|
572
|
+
timings,
|
|
573
|
+
totalMs: total ?? undefined,
|
|
574
|
+
recorderStats: recorder.getStats(),
|
|
575
|
+
});
|
|
576
|
+
if (!opts.executionObserver && shouldLogProfileStep(state, state.decodeStepCount)) {
|
|
570
577
|
log.warn('Profile', `Decode step ${state.decodeStepCount}${profileLabel}:`);
|
|
571
578
|
log.warn('Profile', CommandRecorder.formatProfileReport(timings));
|
|
572
579
|
}
|
|
@@ -873,15 +880,22 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
873
880
|
setTrackSubmits(false);
|
|
874
881
|
}
|
|
875
882
|
|
|
876
|
-
if (opts.profile
|
|
877
|
-
const timings =
|
|
883
|
+
if (opts.profile) {
|
|
884
|
+
const timings = recorder.isProfilingEnabled()
|
|
885
|
+
? await recorder.resolveProfileTimings()
|
|
886
|
+
: null;
|
|
878
887
|
const total = sumProfileTimings(timings);
|
|
879
888
|
if (total !== null) {
|
|
880
889
|
state.stats.gpuTimeDecodeMs = (state.stats.gpuTimeDecodeMs ?? 0) + total;
|
|
881
890
|
}
|
|
882
|
-
if (timings) {
|
|
883
|
-
recordDecodeProfileStep(state, {
|
|
884
|
-
|
|
891
|
+
if (timings || opts.executionObserver) {
|
|
892
|
+
recordDecodeProfileStep(state, {
|
|
893
|
+
step: state.decodeStepCount,
|
|
894
|
+
timings,
|
|
895
|
+
totalMs: total ?? undefined,
|
|
896
|
+
recorderStats: recorder.getStats(),
|
|
897
|
+
});
|
|
898
|
+
if (!opts.executionObserver && shouldLogProfileStep(state, state.decodeStepCount)) {
|
|
885
899
|
log.warn('Profile', `Decode step ${state.decodeStepCount}:`);
|
|
886
900
|
log.warn('Profile', CommandRecorder.formatProfileReport(timings));
|
|
887
901
|
}
|
|
@@ -1792,22 +1806,25 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1792
1806
|
state.batchingStats.executedBatchTokens = (state.batchingStats.executedBatchTokens ?? 0) + N;
|
|
1793
1807
|
state.batchingStats.resolvedBatchTokens = (state.batchingStats.resolvedBatchTokens ?? 0) + actualCount;
|
|
1794
1808
|
|
|
1795
|
-
if (opts.profile
|
|
1796
|
-
const timings =
|
|
1809
|
+
if (opts.profile) {
|
|
1810
|
+
const timings = recorder.isProfilingEnabled()
|
|
1811
|
+
? await recorder.resolveProfileTimings()
|
|
1812
|
+
: null;
|
|
1797
1813
|
const total = sumProfileTimings(timings);
|
|
1798
1814
|
if (total !== null) {
|
|
1799
1815
|
state.stats.gpuTimeDecodeMs = (state.stats.gpuTimeDecodeMs ?? 0) + total;
|
|
1800
1816
|
}
|
|
1801
|
-
if (timings) {
|
|
1817
|
+
if (timings || opts.executionObserver) {
|
|
1802
1818
|
recordDecodeProfileStep(state, {
|
|
1803
1819
|
batch: true,
|
|
1804
1820
|
stepStart: state.decodeStepCount + 1,
|
|
1805
1821
|
stepCount: actualCount,
|
|
1806
1822
|
timings,
|
|
1807
1823
|
totalMs: total ?? undefined,
|
|
1824
|
+
recorderStats: recorder.getStats(),
|
|
1808
1825
|
});
|
|
1809
1826
|
const stepStart = state.decodeStepCount + 1;
|
|
1810
|
-
if (shouldLogProfileStep(state, stepStart)) {
|
|
1827
|
+
if (!opts.executionObserver && shouldLogProfileStep(state, stepStart)) {
|
|
1811
1828
|
log.warn('Profile', `Batch decode (N=${N}):`);
|
|
1812
1829
|
log.warn('Profile', CommandRecorder.formatProfileReport(timings));
|
|
1813
1830
|
}
|
|
@@ -3299,21 +3299,26 @@ export class PipelineGenerator {
|
|
|
3299
3299
|
let gpuTimePrefillMs = 0;
|
|
3300
3300
|
let hasGpuTimePrefill = false;
|
|
3301
3301
|
const recordProfile = async (rec) => {
|
|
3302
|
-
if (!opts.profile || !rec
|
|
3303
|
-
const timings =
|
|
3302
|
+
if (!opts.profile || !rec) return;
|
|
3303
|
+
const timings = rec.isProfilingEnabled()
|
|
3304
|
+
? await rec.resolveProfileTimings()
|
|
3305
|
+
: null;
|
|
3304
3306
|
const total = sumProfileTimings(timings);
|
|
3305
3307
|
if (total !== null) {
|
|
3306
3308
|
gpuTimePrefillMs += total;
|
|
3307
3309
|
hasGpuTimePrefill = true;
|
|
3308
3310
|
}
|
|
3309
|
-
if (timings) {
|
|
3311
|
+
if (timings || opts.executionObserver) {
|
|
3310
3312
|
recordPrefillProfileStep(this.#state, {
|
|
3311
3313
|
label: rec.label,
|
|
3312
3314
|
timings,
|
|
3313
3315
|
totalMs: total ?? undefined,
|
|
3316
|
+
recorderStats: rec.getStats(),
|
|
3314
3317
|
});
|
|
3315
|
-
|
|
3316
|
-
|
|
3318
|
+
if (!opts.executionObserver) {
|
|
3319
|
+
log.warn('Profile', `Prefill (${rec.label}):`);
|
|
3320
|
+
log.warn('Profile', CommandRecorder.formatProfileReport(timings));
|
|
3321
|
+
}
|
|
3317
3322
|
}
|
|
3318
3323
|
};
|
|
3319
3324
|
|
|
@@ -33,6 +33,7 @@ import { Tokenizer, type ModelManifest as TokenizerManifest } from '../../tokeni
|
|
|
33
33
|
import { MoERouter } from '../../moe-router.js';
|
|
34
34
|
import { SpeculativeDecoder } from '../../speculative.js';
|
|
35
35
|
import type { LayerWeights, RouterWeights } from './types.js';
|
|
36
|
+
import type { CommandContext } from '../../../tooling/command-context.js';
|
|
36
37
|
import type {
|
|
37
38
|
KVCacheConfigSchema,
|
|
38
39
|
RuntimeConfigSchema,
|
|
@@ -86,6 +87,8 @@ export function createNodeFileShardStorageContext(
|
|
|
86
87
|
* External contexts that can be injected into the pipeline.
|
|
87
88
|
*/
|
|
88
89
|
export interface PipelineContexts {
|
|
90
|
+
/** Immutable normalized command identity; observation-only, never numeric policy */
|
|
91
|
+
commandContext?: CommandContext | null;
|
|
89
92
|
/** GPU context (device, capabilities) */
|
|
90
93
|
gpu?: { device?: GPUDevice; capabilities?: KernelCapabilities };
|
|
91
94
|
/** Memory context for allocation */
|
|
@@ -101,6 +101,8 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
101
101
|
sharedAttentionState: context.sharedAttentionState ?? null,
|
|
102
102
|
kvCache: ((kvCache)),
|
|
103
103
|
linearRuntime: context.linearAttentionRuntime ?? null,
|
|
104
|
+
resolvedRuntimeSession: context.resolvedRuntimeSession,
|
|
105
|
+
observationContext: context.observationContext ?? null,
|
|
104
106
|
operatorDiagnostics: context.operatorDiagnostics,
|
|
105
107
|
executionPolicies: context.executionPolicies ?? null,
|
|
106
108
|
};
|
|
@@ -894,6 +894,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
894
894
|
debugProbes: context.debugProbes,
|
|
895
895
|
operatorDiagnostics: context.operatorDiagnostics,
|
|
896
896
|
linearRuntime: context.linearAttentionRuntime ?? null,
|
|
897
|
+
resolvedRuntimeSession: context.resolvedRuntimeSession,
|
|
898
|
+
observationContext: context.observationContext ?? null,
|
|
897
899
|
executionPolicies: context.executionPolicies ?? null,
|
|
898
900
|
skipKVCacheWrites: context.skipKVCacheWrites === true,
|
|
899
901
|
};
|
|
@@ -11,7 +11,6 @@ import type { PipelineStorageContext } from './init.js';
|
|
|
11
11
|
export interface KernelWarmupOptions {
|
|
12
12
|
useGPU: boolean;
|
|
13
13
|
kernelWarmup?: KernelWarmupConfigSchema | null;
|
|
14
|
-
modelConfig: ParsedModelConfig;
|
|
15
14
|
}
|
|
16
15
|
|
|
17
16
|
export interface KernelPathResolutionOptions {
|
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
getKernelPathKVDtype,
|
|
10
10
|
setActiveKernelPath,
|
|
11
11
|
} from '../../../config/kernel-path-loader.js';
|
|
12
|
-
import {
|
|
12
|
+
import { prewarmKernels } from '../../../gpu/kernels/index.js';
|
|
13
13
|
import { KERNEL_CONFIGS } from '../../../gpu/kernels/kernel-configs.js';
|
|
14
14
|
import { initTokenizer } from './init.js';
|
|
15
15
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
@@ -312,7 +312,7 @@ export function applyModelBatchingRuntimeDefaults(runtimeConfig, manifest, model
|
|
|
312
312
|
}
|
|
313
313
|
|
|
314
314
|
export async function runKernelWarmup(options) {
|
|
315
|
-
const { useGPU, kernelWarmup
|
|
315
|
+
const { useGPU, kernelWarmup } = options;
|
|
316
316
|
if (!useGPU || !kernelWarmup) {
|
|
317
317
|
return;
|
|
318
318
|
}
|
|
@@ -327,15 +327,6 @@ export async function runKernelWarmup(options) {
|
|
|
327
327
|
log.warn('Pipeline', `Kernel prewarm failed: ${ (e).message}`);
|
|
328
328
|
}
|
|
329
329
|
}
|
|
330
|
-
if (kernelWarmup.autoTune) {
|
|
331
|
-
log.info('Pipeline', 'Kernel auto-tune enabled');
|
|
332
|
-
try {
|
|
333
|
-
await autoTuneKernels(modelConfig);
|
|
334
|
-
markKernelCacheWarmed();
|
|
335
|
-
} catch (e) {
|
|
336
|
-
log.warn('Pipeline', `Kernel auto-tune failed: ${ (e).message}`);
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
330
|
}
|
|
340
331
|
|
|
341
332
|
function normalizeKernelPathSourceHint(value) {
|
|
@@ -340,6 +340,10 @@ async function buildDiagnosticCapture(level, buffer, options) {
|
|
|
340
340
|
shape: snapshot.shape,
|
|
341
341
|
dtype: snapshot.dtype,
|
|
342
342
|
sample: Array.isArray(snapshot.sample) ? snapshot.sample : null,
|
|
343
|
+
sampleCoordinates: Array.isArray(snapshot.sampleCoordinates)
|
|
344
|
+
? snapshot.sampleCoordinates
|
|
345
|
+
: null,
|
|
346
|
+
fullTensorDigest: snapshot.fullTensorDigest ?? null,
|
|
343
347
|
stats: snapshot.stats ?? null,
|
|
344
348
|
hasNaN: snapshot.hasNaN === true,
|
|
345
349
|
hasInf: snapshot.hasInf === true,
|
|
@@ -366,6 +370,8 @@ function createDeferredDiagnosticCapture(level, buffer, options) {
|
|
|
366
370
|
shape,
|
|
367
371
|
dtype,
|
|
368
372
|
sample: null,
|
|
373
|
+
sampleCoordinates: null,
|
|
374
|
+
fullTensorDigest: null,
|
|
369
375
|
stats: null,
|
|
370
376
|
hasNaN: false,
|
|
371
377
|
hasInf: false,
|
|
@@ -385,6 +391,10 @@ function createDeferredDiagnosticCapture(level, buffer, options) {
|
|
|
385
391
|
capture.shape = snapshot.shape;
|
|
386
392
|
capture.dtype = snapshot.dtype;
|
|
387
393
|
capture.sample = Array.isArray(snapshot.sample) ? snapshot.sample : null;
|
|
394
|
+
capture.sampleCoordinates = Array.isArray(snapshot.sampleCoordinates)
|
|
395
|
+
? snapshot.sampleCoordinates
|
|
396
|
+
: null;
|
|
397
|
+
capture.fullTensorDigest = snapshot.fullTensorDigest ?? null;
|
|
388
398
|
capture.stats = snapshot.stats ?? null;
|
|
389
399
|
capture.hasNaN = snapshot.hasNaN === true;
|
|
390
400
|
capture.hasInf = snapshot.hasInf === true;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { RuntimeConfigSchema, KernelPathSchema } from '../../../config/schema/index.js';
|
|
2
|
+
import type { Manifest, ParsedModelConfig } from './config.js';
|
|
3
|
+
import type { ExecutionPlanState } from './execution-plan.js';
|
|
4
|
+
import type { ExecutionV1CompiledState } from './execution-v1.js';
|
|
5
|
+
|
|
6
|
+
export const RESOLVED_RUNTIME_SESSION_SCHEMA: 'doppler.resolved-runtime-session/v1';
|
|
7
|
+
|
|
8
|
+
export interface ResolvedRuntimeSession {
|
|
9
|
+
readonly schema: typeof RESOLVED_RUNTIME_SESSION_SCHEMA;
|
|
10
|
+
readonly id: string;
|
|
11
|
+
readonly model: {
|
|
12
|
+
readonly id: string;
|
|
13
|
+
readonly type: string;
|
|
14
|
+
readonly architecture: string;
|
|
15
|
+
readonly numLayers: number;
|
|
16
|
+
readonly hiddenSize: number;
|
|
17
|
+
readonly numHeads: number;
|
|
18
|
+
readonly numKVHeads: number;
|
|
19
|
+
readonly headDim: number;
|
|
20
|
+
};
|
|
21
|
+
readonly manifestInference: Readonly<Record<string, unknown>> | null;
|
|
22
|
+
readonly runtime: {
|
|
23
|
+
readonly session: Readonly<Record<string, unknown>>;
|
|
24
|
+
readonly compute: Readonly<Record<string, unknown>>;
|
|
25
|
+
};
|
|
26
|
+
readonly kernelPath: {
|
|
27
|
+
readonly id: string | null;
|
|
28
|
+
readonly source: string;
|
|
29
|
+
readonly hash: string | null;
|
|
30
|
+
readonly definition: Readonly<Record<string, unknown>> | null;
|
|
31
|
+
};
|
|
32
|
+
readonly capabilityPolicy: Readonly<Record<string, unknown>> | null;
|
|
33
|
+
readonly laneIntegrity: Readonly<Record<string, unknown>> | null;
|
|
34
|
+
readonly execution: Readonly<Record<string, unknown>>;
|
|
35
|
+
readonly dtypes: {
|
|
36
|
+
readonly activation: string;
|
|
37
|
+
readonly output: string;
|
|
38
|
+
readonly kv: string;
|
|
39
|
+
readonly math: string | null;
|
|
40
|
+
readonly accumulation: string | null;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function createResolvedRuntimeSession(options: {
|
|
45
|
+
manifest: Manifest;
|
|
46
|
+
modelConfig: ParsedModelConfig;
|
|
47
|
+
runtimeConfig: RuntimeConfigSchema;
|
|
48
|
+
resolvedKernelPath?: KernelPathSchema | null;
|
|
49
|
+
kernelPathSource?: string;
|
|
50
|
+
executionV1State?: ExecutionV1CompiledState | null;
|
|
51
|
+
executionPlanState: ExecutionPlanState;
|
|
52
|
+
}): ResolvedRuntimeSession;
|
|
53
|
+
|
|
54
|
+
export function resolveAttentionRuntimeSession(state: {
|
|
55
|
+
resolvedRuntimeSession?: ResolvedRuntimeSession | null;
|
|
56
|
+
}): Readonly<Record<string, unknown>>;
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { hashRefactorReceiptValue } from '../../refactor-receipt.js';
|
|
2
|
+
|
|
3
|
+
export const RESOLVED_RUNTIME_SESSION_SCHEMA = 'doppler.resolved-runtime-session/v1';
|
|
4
|
+
|
|
5
|
+
function cloneJsonRecord(value, label) {
|
|
6
|
+
if (value == null) {
|
|
7
|
+
return null;
|
|
8
|
+
}
|
|
9
|
+
let encoded;
|
|
10
|
+
try {
|
|
11
|
+
encoded = JSON.stringify(value);
|
|
12
|
+
} catch (error) {
|
|
13
|
+
throw new Error(
|
|
14
|
+
`[ResolvedRuntimeSession] ${label} must be JSON-safe: ${error?.message ?? String(error)}`
|
|
15
|
+
);
|
|
16
|
+
}
|
|
17
|
+
if (encoded === undefined) {
|
|
18
|
+
throw new Error(`[ResolvedRuntimeSession] ${label} must be JSON-safe.`);
|
|
19
|
+
}
|
|
20
|
+
return JSON.parse(encoded);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function deepFreeze(value) {
|
|
24
|
+
if (!value || typeof value !== 'object' || Object.isFrozen(value)) {
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
Object.freeze(value);
|
|
28
|
+
for (const entry of Object.values(value)) {
|
|
29
|
+
deepFreeze(entry);
|
|
30
|
+
}
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function requirePositiveInteger(value, label) {
|
|
35
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
36
|
+
throw new Error(`[ResolvedRuntimeSession] ${label} must be a positive integer.`);
|
|
37
|
+
}
|
|
38
|
+
return value;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function requireModelId(manifest) {
|
|
42
|
+
const modelId = String(manifest?.modelId ?? '').trim();
|
|
43
|
+
if (!modelId) {
|
|
44
|
+
throw new Error('[ResolvedRuntimeSession] manifest.modelId is required.');
|
|
45
|
+
}
|
|
46
|
+
return modelId;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function summarizeExecutionPlan(plan) {
|
|
50
|
+
if (!plan) {
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
id: plan.id,
|
|
55
|
+
source: plan.source,
|
|
56
|
+
activationDtype: plan.activationDtype,
|
|
57
|
+
kernelPathId: plan.kernelPathId ?? null,
|
|
58
|
+
kernelPathSource: plan.kernelPathSource ?? 'none',
|
|
59
|
+
kernelPathHash: plan.kernelPath
|
|
60
|
+
? hashRefactorReceiptValue(cloneJsonRecord(plan.kernelPath, 'executionPlan.kernelPath'))
|
|
61
|
+
: null,
|
|
62
|
+
finitenessGuardEnabled: plan.finitenessGuardEnabled === true,
|
|
63
|
+
finitenessOnTrigger: plan.finitenessOnTrigger ?? null,
|
|
64
|
+
defaultDisableCommandBatching: plan.defaultDisableCommandBatching === true,
|
|
65
|
+
defaultDisableMultiTokenDecode: plan.defaultDisableMultiTokenDecode === true,
|
|
66
|
+
defaultBatchSize: plan.defaultBatchSize,
|
|
67
|
+
defaultStopCheckMode: plan.defaultStopCheckMode,
|
|
68
|
+
readbackInterval: plan.readbackInterval ?? null,
|
|
69
|
+
readbackMode: plan.readbackMode ?? null,
|
|
70
|
+
maxBatchDecodeTokens: plan.maxBatchDecodeTokens ?? null,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function createResolvedRuntimeSession(options = {}) {
|
|
75
|
+
const {
|
|
76
|
+
manifest,
|
|
77
|
+
modelConfig,
|
|
78
|
+
runtimeConfig,
|
|
79
|
+
resolvedKernelPath = null,
|
|
80
|
+
kernelPathSource = 'none',
|
|
81
|
+
executionV1State = null,
|
|
82
|
+
executionPlanState,
|
|
83
|
+
} = options;
|
|
84
|
+
if (!runtimeConfig?.inference?.session) {
|
|
85
|
+
throw new Error('[ResolvedRuntimeSession] runtimeConfig.inference.session is required.');
|
|
86
|
+
}
|
|
87
|
+
if (!runtimeConfig.inference.compute) {
|
|
88
|
+
throw new Error('[ResolvedRuntimeSession] runtimeConfig.inference.compute is required.');
|
|
89
|
+
}
|
|
90
|
+
if (!executionPlanState?.primaryPlan) {
|
|
91
|
+
throw new Error('[ResolvedRuntimeSession] a compiled primary execution plan is required.');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const session = cloneJsonRecord(runtimeConfig.inference.session, 'runtime session');
|
|
95
|
+
const compute = cloneJsonRecord(runtimeConfig.inference.compute, 'compute policy');
|
|
96
|
+
const manifestInference = cloneJsonRecord(manifest?.inference ?? null, 'manifest inference contract');
|
|
97
|
+
const kernelPath = cloneJsonRecord(resolvedKernelPath, 'resolved kernel path');
|
|
98
|
+
const resolvedSteps = cloneJsonRecord(
|
|
99
|
+
executionV1State?.resolvedSteps ?? null,
|
|
100
|
+
'execution-v1 resolved steps'
|
|
101
|
+
);
|
|
102
|
+
const laneIntegrity = cloneJsonRecord(
|
|
103
|
+
executionV1State?.laneIntegrity ?? null,
|
|
104
|
+
'execution-v1 lane integrity'
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
const core = {
|
|
108
|
+
model: {
|
|
109
|
+
id: requireModelId(manifest),
|
|
110
|
+
type: String(manifest?.modelType ?? ''),
|
|
111
|
+
architecture: typeof manifest?.architecture === 'string'
|
|
112
|
+
? manifest.architecture
|
|
113
|
+
: String(manifest?.architecture?.type ?? manifest?.modelType ?? ''),
|
|
114
|
+
numLayers: requirePositiveInteger(modelConfig?.numLayers, 'modelConfig.numLayers'),
|
|
115
|
+
hiddenSize: requirePositiveInteger(modelConfig?.hiddenSize, 'modelConfig.hiddenSize'),
|
|
116
|
+
numHeads: requirePositiveInteger(modelConfig?.numHeads, 'modelConfig.numHeads'),
|
|
117
|
+
numKVHeads: requirePositiveInteger(modelConfig?.numKVHeads, 'modelConfig.numKVHeads'),
|
|
118
|
+
headDim: requirePositiveInteger(modelConfig?.headDim, 'modelConfig.headDim'),
|
|
119
|
+
},
|
|
120
|
+
manifestInference,
|
|
121
|
+
runtime: {
|
|
122
|
+
session,
|
|
123
|
+
compute,
|
|
124
|
+
},
|
|
125
|
+
kernelPath: {
|
|
126
|
+
id: kernelPath?.id ?? null,
|
|
127
|
+
source: kernelPathSource,
|
|
128
|
+
hash: kernelPath ? hashRefactorReceiptValue(kernelPath) : null,
|
|
129
|
+
definition: kernelPath,
|
|
130
|
+
},
|
|
131
|
+
capabilityPolicy: laneIntegrity?.policy ?? null,
|
|
132
|
+
laneIntegrity,
|
|
133
|
+
execution: {
|
|
134
|
+
primary: summarizeExecutionPlan(executionPlanState.primaryPlan),
|
|
135
|
+
fallback: summarizeExecutionPlan(executionPlanState.fallbackPlan),
|
|
136
|
+
resolvedSteps,
|
|
137
|
+
resolvedStepsHash: resolvedSteps ? hashRefactorReceiptValue(resolvedSteps) : null,
|
|
138
|
+
appliedTransforms: cloneJsonRecord(
|
|
139
|
+
executionV1State?.appliedTransforms ?? [],
|
|
140
|
+
'execution-v1 applied transforms'
|
|
141
|
+
),
|
|
142
|
+
},
|
|
143
|
+
dtypes: {
|
|
144
|
+
activation: executionPlanState.primaryPlan.activationDtype,
|
|
145
|
+
output: session?.compute?.defaults?.outputDtype
|
|
146
|
+
?? compute?.outputDtype
|
|
147
|
+
?? executionPlanState.primaryPlan.activationDtype,
|
|
148
|
+
kv: session?.kvcache?.kvDtype
|
|
149
|
+
?? session?.compute?.defaults?.activationDtype
|
|
150
|
+
?? executionPlanState.primaryPlan.activationDtype,
|
|
151
|
+
math: session?.compute?.defaults?.mathDtype ?? null,
|
|
152
|
+
accumulation: session?.compute?.defaults?.accumDtype ?? null,
|
|
153
|
+
},
|
|
154
|
+
};
|
|
155
|
+
const resolved = {
|
|
156
|
+
schema: RESOLVED_RUNTIME_SESSION_SCHEMA,
|
|
157
|
+
id: hashRefactorReceiptValue(core),
|
|
158
|
+
...core,
|
|
159
|
+
};
|
|
160
|
+
return deepFreeze(resolved);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function resolveAttentionRuntimeSession(state) {
|
|
164
|
+
const resolved = state?.resolvedRuntimeSession;
|
|
165
|
+
if (!resolved || resolved.schema !== RESOLVED_RUNTIME_SESSION_SCHEMA) {
|
|
166
|
+
throw new Error(
|
|
167
|
+
'[ResolvedRuntimeSession] attention requires a resolved runtime session. ' +
|
|
168
|
+
'Construct the pipeline session before executing layers.'
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
if (!resolved.runtime?.session) {
|
|
172
|
+
throw new Error('[ResolvedRuntimeSession] resolved runtime session is missing runtime.session.');
|
|
173
|
+
}
|
|
174
|
+
return resolved.runtime.session;
|
|
175
|
+
}
|
|
@@ -37,6 +37,9 @@ import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
|
|
|
37
37
|
import type { EmulationContext, PipelineStorageContext } from './init.js';
|
|
38
38
|
import type { LinearAttentionRuntime } from './linear-attention.js';
|
|
39
39
|
import type { PerLayerInputWeights } from '../../../loader/loader-types.js';
|
|
40
|
+
import type { CommandContext } from '../../../tooling/command-context.js';
|
|
41
|
+
import type { ObservationContext } from '../../observation-context.js';
|
|
42
|
+
import type { ResolvedRuntimeSession } from './resolved-runtime-session.js';
|
|
40
43
|
|
|
41
44
|
export class PipelineState {
|
|
42
45
|
// Components
|
|
@@ -60,6 +63,7 @@ export class PipelineState {
|
|
|
60
63
|
kernelPathSource: KernelPathSource;
|
|
61
64
|
executionPlanState: ExecutionPlanState | null;
|
|
62
65
|
executionV1State: ExecutionV1CompiledState | null;
|
|
66
|
+
resolvedRuntimeSession: ResolvedRuntimeSession | null;
|
|
63
67
|
disableRecordedLogits: boolean;
|
|
64
68
|
disableFusedDecode: boolean;
|
|
65
69
|
|
|
@@ -75,6 +79,8 @@ export class PipelineState {
|
|
|
75
79
|
currentSeqLen: number;
|
|
76
80
|
currentTokenIds: number[] | null;
|
|
77
81
|
runtimeConfig: RuntimeConfigSchema;
|
|
82
|
+
commandContext: CommandContext | null;
|
|
83
|
+
observationContext: ObservationContext | null;
|
|
78
84
|
|
|
79
85
|
// DopplerLoader instance
|
|
80
86
|
dopplerLoader: DopplerLoader | null;
|
|
@@ -44,6 +44,7 @@ export class PipelineState {
|
|
|
44
44
|
this.kernelPathSource = 'none';
|
|
45
45
|
|
|
46
46
|
this.executionPlanState = null;
|
|
47
|
+
this.resolvedRuntimeSession = null;
|
|
47
48
|
|
|
48
49
|
this.disableRecordedLogits = false;
|
|
49
50
|
|
|
@@ -70,6 +71,8 @@ export class PipelineState {
|
|
|
70
71
|
this.currentTokenIds = null;
|
|
71
72
|
|
|
72
73
|
this.runtimeConfig = getRuntimeConfig();
|
|
74
|
+
this.commandContext = null;
|
|
75
|
+
this.observationContext = null;
|
|
73
76
|
this.operatorDiagnostics = null;
|
|
74
77
|
|
|
75
78
|
// DopplerLoader instance
|
|
@@ -767,6 +767,12 @@ export interface PipelineStats {
|
|
|
767
767
|
} | null;
|
|
768
768
|
kernelPathId?: string | null;
|
|
769
769
|
kernelPathSource?: string | null;
|
|
770
|
+
/** Immutable semantic attention plans captured by observation policy */
|
|
771
|
+
attentionPlans?: Array<import('./attention/plan.js').SemanticAttentionPlan>;
|
|
772
|
+
/** Tagged attention ownership/lifetime events */
|
|
773
|
+
attentionResourceEvents?: Array<import('../../resource-scope.js').ResourceEvent>;
|
|
774
|
+
/** Canonical behavior receipts captured by observation policy */
|
|
775
|
+
refactorReceipts?: Array<import('../../refactor-receipt.js').RefactorReceipt>;
|
|
770
776
|
prefillProfileSteps?: Array<{
|
|
771
777
|
label?: string;
|
|
772
778
|
timings: Record<string, number>;
|
|
@@ -49,6 +49,8 @@ import {
|
|
|
49
49
|
import { getDopplerLoader } from '../../loader/doppler-loader.js';
|
|
50
50
|
import { registerPipeline, getPipelineFactory } from './registry.js';
|
|
51
51
|
import { selectRuleValue } from '../../rules/rule-registry.js';
|
|
52
|
+
import { createObservationContext } from '../observation-context.js';
|
|
53
|
+
import { createResolvedRuntimeSession } from './text/resolved-runtime-session.js';
|
|
52
54
|
|
|
53
55
|
// AbortSignal contract: every public inference primitive on this pipeline
|
|
54
56
|
// accepts `options.signal` (or `args.signal`). When the signal aborts the
|
|
@@ -340,6 +342,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
340
342
|
assignStorageContext: true,
|
|
341
343
|
});
|
|
342
344
|
this.runtimeConfig = runtimeConfig;
|
|
345
|
+
this.commandContext = contexts.commandContext ?? null;
|
|
346
|
+
this.observationContext = createObservationContext({
|
|
347
|
+
runtimeConfig,
|
|
348
|
+
commandContext: this.commandContext,
|
|
349
|
+
});
|
|
343
350
|
this.dopplerLoader = contexts.loader || null;
|
|
344
351
|
this.ownsDopplerLoader = contexts.ownsLoader === true;
|
|
345
352
|
this.runtimeOverrides = contexts.runtimeConfig == null
|
|
@@ -528,7 +535,6 @@ export class InferencePipeline extends PipelineState {
|
|
|
528
535
|
() => runKernelWarmup({
|
|
529
536
|
useGPU: this.useGPU,
|
|
530
537
|
kernelWarmup: this.runtimeConfig.shared?.kernelWarmup,
|
|
531
|
-
modelConfig: this.modelConfig,
|
|
532
538
|
})
|
|
533
539
|
);
|
|
534
540
|
|
|
@@ -619,6 +625,15 @@ export class InferencePipeline extends PipelineState {
|
|
|
619
625
|
`but kernel path "${this.resolvedKernelPath.id}" declares activationDtype="${kpActivation}".`
|
|
620
626
|
);
|
|
621
627
|
}
|
|
628
|
+
this.resolvedRuntimeSession = createResolvedRuntimeSession({
|
|
629
|
+
manifest,
|
|
630
|
+
modelConfig: this.modelConfig,
|
|
631
|
+
runtimeConfig: this.runtimeConfig,
|
|
632
|
+
resolvedKernelPath: this.resolvedKernelPath,
|
|
633
|
+
kernelPathSource: this.kernelPathSource,
|
|
634
|
+
executionV1State: this.executionV1State,
|
|
635
|
+
executionPlanState: this.executionPlanState,
|
|
636
|
+
});
|
|
622
637
|
|
|
623
638
|
// Initialize MoE router if needed
|
|
624
639
|
if (this.modelConfig.useMoE) {
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export const REFACTOR_RECEIPT_SCHEMA: 'doppler.refactor-receipt/v1';
|
|
2
|
+
|
|
3
|
+
export type RefactorReceiptValue =
|
|
4
|
+
| null
|
|
5
|
+
| string
|
|
6
|
+
| number
|
|
7
|
+
| boolean
|
|
8
|
+
| RefactorReceiptValue[]
|
|
9
|
+
| { [key: string]: RefactorReceiptValue };
|
|
10
|
+
|
|
11
|
+
export interface RefactorReceiptInput {
|
|
12
|
+
commandContext?: Record<string, RefactorReceiptValue> | null;
|
|
13
|
+
resolvedSession?: Record<string, RefactorReceiptValue> | null;
|
|
14
|
+
observationContext?: Record<string, RefactorReceiptValue> | null;
|
|
15
|
+
operationPlan?: Record<string, RefactorReceiptValue> | null;
|
|
16
|
+
operations?: Array<Record<string, RefactorReceiptValue>> | null;
|
|
17
|
+
dtypeTransitions?: Array<Record<string, RefactorReceiptValue>> | null;
|
|
18
|
+
resourceEvents?: Array<Record<string, RefactorReceiptValue>> | null;
|
|
19
|
+
failure?: Record<string, RefactorReceiptValue> | null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface RefactorReceipt {
|
|
23
|
+
readonly schema: typeof REFACTOR_RECEIPT_SCHEMA;
|
|
24
|
+
readonly commandContext: Record<string, RefactorReceiptValue> | null;
|
|
25
|
+
readonly resolvedSession: Record<string, RefactorReceiptValue> | null;
|
|
26
|
+
readonly observationContext: Record<string, RefactorReceiptValue> | null;
|
|
27
|
+
readonly operationPlan: Record<string, RefactorReceiptValue> | null;
|
|
28
|
+
readonly operations: ReadonlyArray<Record<string, RefactorReceiptValue>>;
|
|
29
|
+
readonly dtypeTransitions: ReadonlyArray<Record<string, RefactorReceiptValue>>;
|
|
30
|
+
readonly resourceEvents: ReadonlyArray<Record<string, RefactorReceiptValue>>;
|
|
31
|
+
readonly failure: Record<string, RefactorReceiptValue> | null;
|
|
32
|
+
readonly semanticHash: string;
|
|
33
|
+
readonly executionHash: string;
|
|
34
|
+
readonly receiptHash: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function canonicalizeRefactorReceiptValue<T extends RefactorReceiptValue>(value: T): T;
|
|
38
|
+
export function hashRefactorReceiptValue(value: RefactorReceiptValue): string;
|
|
39
|
+
export function createRefactorReceipt(input?: RefactorReceiptInput): RefactorReceipt;
|
|
40
|
+
export function verifyRefactorReceipt(receipt: RefactorReceipt): true;
|
|
41
|
+
export function compareRefactorReceipts(
|
|
42
|
+
expected: RefactorReceipt,
|
|
43
|
+
actual: RefactorReceipt
|
|
44
|
+
): {
|
|
45
|
+
readonly matches: boolean;
|
|
46
|
+
readonly semanticMatches: boolean;
|
|
47
|
+
readonly executionMatches: boolean;
|
|
48
|
+
readonly differences: readonly string[];
|
|
49
|
+
};
|