doppler-gpu 0.4.15 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +25 -2
- package/src/cli/doppler-cli.js +253 -94
- package/src/cli/runtime-profiles.js +3 -7
- package/src/client/doppler-api.d.ts +28 -0
- package/src/client/doppler-api.js +2 -0
- package/src/client/inspection.d.ts +122 -0
- package/src/client/inspection.js +343 -0
- package/src/client/runtime/index.d.ts +54 -0
- package/src/client/runtime/index.js +190 -18
- package/src/client/runtime/model-manager.d.ts +0 -4
- package/src/client/runtime/model-manager.js +0 -28
- package/src/client/runtime/model-session.d.ts +48 -1
- package/src/client/runtime/model-session.js +149 -30
- package/src/client/runtime/model-source.d.ts +10 -1
- package/src/client/runtime/scoped-session.d.ts +117 -0
- package/src/client/runtime/scoped-session.js +257 -0
- package/src/client/runtime/types.d.ts +0 -1
- package/src/client/runtime/types.js +0 -1
- package/src/config/evidence/boundary-evidence-policy.json +54 -0
- package/src/config/evidence/source-intake-policy.json +122 -0
- package/src/config/inspection/observation-policies.json +80 -0
- package/src/config/param-validator.js +21 -36
- package/src/config/runtime/compute/f16-activations.json +1 -0
- package/src/config/runtime/compute/f16-batched.json +1 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/diagnostics/debug-logits.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
- package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
- package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
- package/src/config/runtime/model/gemma2-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline.json +1 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
- package/src/config/runtime/optimization-candidates/registry.json +5 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
- package/src/config/runtime/profiles/default.json +1 -0
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/production.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/rerank-stability.json +1 -3
- package/src/config/runtime/profiles/rerank-throughput.json +1 -3
- package/src/config/runtime/profiles/simulation.json +1 -3
- package/src/config/runtime/profiles/throughput.json +1 -3
- package/src/config/runtime/profiles/trace-layers.json +1 -3
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
- package/src/config/runtime/profiles/turboquant.json +1 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -3
- package/src/config/runtime/profiles/vector-throughput.json +1 -0
- package/src/config/runtime/profiles/verbose-trace.json +1 -3
- package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
- package/src/config/schema/benchmark.schema.d.ts +7 -0
- package/src/config/schema/benchmark.schema.js +5 -0
- package/src/config/schema/index.d.ts +8 -16
- package/src/config/schema/index.js +5 -8
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
- package/src/config/schema/kernel-thresholds.schema.js +0 -12
- package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
- package/src/config/schema/kernel-warmup.schema.js +0 -1
- package/src/config/schema/runtime-profile.schema.d.ts +24 -0
- package/src/config/schema/runtime-profile.schema.js +92 -0
- package/src/config/schema/shared-runtime.schema.d.ts +1 -4
- package/src/config/schema/shared-runtime.schema.js +0 -2
- package/src/config/schema/tooling.schema.d.ts +8 -5
- package/src/config/schema/tooling.schema.js +3 -2
- package/src/debug/tensor.d.ts +2 -0
- package/src/debug/tensor.js +21 -2
- package/src/gpu/command-recorder.d.ts +5 -0
- package/src/gpu/command-recorder.js +17 -0
- package/src/gpu/kernel-runtime.js +2 -10
- package/src/gpu/kernels/index.d.ts +0 -2
- package/src/gpu/kernels/index.js +0 -2
- package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
- package/src/gpu/kernels/kernel-prewarm.js +60 -0
- package/src/gpu/kernels/utils.d.ts +2 -4
- package/src/gpu/kernels/utils.js +2 -4
- package/src/index.d.ts +13 -1
- package/src/index.js +7 -1
- package/src/inference/browser-harness-model-helpers.js +4 -0
- package/src/inference/browser-harness-runtime-helpers.js +25 -10
- package/src/inference/browser-harness-text-helpers.js +4 -1
- package/src/inference/browser-harness.d.ts +4 -0
- package/src/inference/browser-harness.js +91 -4
- package/src/inference/observation-context.d.ts +22 -0
- package/src/inference/observation-context.js +41 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
- package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
- package/src/inference/pipelines/text/attention/index.js +7 -0
- package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
- package/src/inference/pipelines/text/attention/plan.js +328 -0
- package/src/inference/pipelines/text/attention/receipt.js +60 -0
- package/src/inference/pipelines/text/attention/record.js +8 -1278
- package/src/inference/pipelines/text/attention/run.js +8 -1386
- package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-steps.js +32 -15
- package/src/inference/pipelines/text/generator.js +10 -5
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
- package/src/inference/pipelines/text/layer.js +2 -0
- package/src/inference/pipelines/text/model-load.d.ts +0 -1
- package/src/inference/pipelines/text/model-load.js +2 -11
- package/src/inference/pipelines/text/probes.js +10 -0
- package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
- package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
- package/src/inference/pipelines/text/state.d.ts +6 -0
- package/src/inference/pipelines/text/state.js +3 -0
- package/src/inference/pipelines/text/types.d.ts +6 -0
- package/src/inference/pipelines/text.js +16 -1
- package/src/inference/refactor-receipt.d.ts +49 -0
- package/src/inference/refactor-receipt.js +188 -0
- package/src/inference/resource-scope.d.ts +33 -0
- package/src/inference/resource-scope.js +168 -0
- package/src/inference/test-harness.d.ts +3 -0
- package/src/inference/test-harness.js +1 -0
- package/src/storage/model-cache.d.ts +53 -0
- package/src/storage/model-cache.js +550 -0
- package/src/tooling/boundary-evidence.d.ts +51 -0
- package/src/tooling/boundary-evidence.js +408 -0
- package/src/tooling/browser-command-runner.js +4 -4
- package/src/tooling/command-context.d.ts +25 -0
- package/src/tooling/command-context.js +53 -0
- package/src/tooling/command-runner-shared.js +22 -5
- package/src/tooling/demo-receipts.d.ts +56 -0
- package/src/tooling/demo-receipts.js +113 -0
- package/src/tooling/diagnose-runner.js +2 -2
- package/src/tooling/execution-cost-ledger.d.ts +21 -0
- package/src/tooling/execution-cost-ledger.js +199 -0
- package/src/tooling/node-command-runner.js +2 -2
- package/src/tooling/opfs-cache.d.ts +11 -49
- package/src/tooling/opfs-cache.js +4 -510
- package/src/tooling/precision-replay-math.d.ts +42 -0
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-parity.d.ts +3 -0
- package/src/tooling/program-bundle-parity.js +8 -0
- package/src/tooling/program-bundle.d.ts +1 -52
- package/src/tooling/program-bundle.js +0 -139
- package/src/tooling/registered-variant-calibration.d.ts +24 -0
- package/src/tooling/registered-variant-calibration.js +286 -0
- package/src/tooling/runtime-input-composition.js +105 -53
- package/src/tooling/runtime-intent.js +41 -0
- package/src/tooling/runtime-optimization-index.d.ts +6 -0
- package/src/tooling/runtime-optimization-index.js +72 -0
- package/src/tooling/runtime-optimization.d.ts +47 -6
- package/src/tooling/runtime-optimization.js +559 -40
- package/src/tooling/source-intake.d.ts +49 -0
- package/src/tooling/source-intake.js +414 -0
- package/src/tooling-exports/evidence.d.ts +34 -0
- package/src/tooling-exports/evidence.js +34 -0
- package/src/tooling-exports/runtime.d.ts +7 -0
- package/src/tooling-exports/runtime.js +7 -0
- package/src/tooling-exports.d.ts +11 -0
- package/src/tooling-exports.js +6 -3
- package/src/tooling-exports.shared.d.ts +32 -0
- package/src/tooling-exports.shared.js +31 -0
- package/src/version.js +1 -1
- package/src/config/schema/tuner.schema.d.ts +0 -39
- package/src/config/schema/tuner.schema.js +0 -13
- package/src/gpu/kernel-tuner/benchmarks.js +0 -902
- package/src/gpu/kernel-tuner/cache.js +0 -145
- package/src/gpu/kernel-tuner/index.js +0 -38
- package/src/gpu/kernel-tuner/tuner.js +0 -247
- package/src/gpu/kernel-tuner.js +0 -12
- package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
- package/src/gpu/kernels/kernel-tuning.js +0 -158
- package/src/gpu/profiler.js +0 -382
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
|
@@ -6,9 +6,16 @@ import { runBrowserCommand } from './browser-command-runner.js';
|
|
|
6
6
|
export const RUNTIME_OPTIMIZATION_CONTRACT_SCHEMA = 'doppler.runtime-optimization-contract/v1';
|
|
7
7
|
export const RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA = 'doppler.runtime-optimization-candidate/v1';
|
|
8
8
|
export const RUNTIME_OPTIMIZATION_RECEIPT_SCHEMA = 'doppler.runtime-optimization-receipt/v1';
|
|
9
|
+
export const RUNTIME_OPTIMIZATION_CANDIDATE_REGISTRY_SCHEMA =
|
|
10
|
+
'doppler.runtime-optimization-candidate-registry/v1';
|
|
9
11
|
|
|
10
12
|
const WORKLOADS = new Set(['inference', 'embedding', 'rerank']);
|
|
11
13
|
const DIRECTIONS = new Set(['maximize', 'minimize']);
|
|
14
|
+
const CANDIDATE_KINDS = new Set([
|
|
15
|
+
'runtime-profile',
|
|
16
|
+
'registered-kernel-variant',
|
|
17
|
+
'registered-execution-graph-patch',
|
|
18
|
+
]);
|
|
12
19
|
const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
13
20
|
const SAFE_MUTATION_PREFIXES = Object.freeze([
|
|
14
21
|
'/loading/shardCache',
|
|
@@ -111,6 +118,28 @@ function assertJsonValue(value, label) {
|
|
|
111
118
|
return value;
|
|
112
119
|
}
|
|
113
120
|
|
|
121
|
+
function normalizeCandidateKind(value) {
|
|
122
|
+
const normalized = value === 'runtime_profile' ? 'runtime-profile' : value;
|
|
123
|
+
if (!CANDIDATE_KINDS.has(normalized)) {
|
|
124
|
+
throw new Error(
|
|
125
|
+
'runtime optimization: contract.kind must be runtime-profile, ' +
|
|
126
|
+
'registered-kernel-variant, or registered-execution-graph-patch.'
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
return normalized;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function validateRegisteredReference(reference, label) {
|
|
133
|
+
assertObject(reference, label);
|
|
134
|
+
assertExactKeys(reference, ['registryId', 'digest'], label);
|
|
135
|
+
assertString(reference.registryId, `${label}.registryId`);
|
|
136
|
+
assertDigestOrNull(reference.digest, `${label}.digest`);
|
|
137
|
+
if (reference.digest === null) {
|
|
138
|
+
throw new Error(`runtime optimization: ${label}.digest must not be null.`);
|
|
139
|
+
}
|
|
140
|
+
return reference;
|
|
141
|
+
}
|
|
142
|
+
|
|
114
143
|
function pointerMatchesPrefix(path, prefix) {
|
|
115
144
|
return path === prefix || path.startsWith(`${prefix}/`);
|
|
116
145
|
}
|
|
@@ -197,15 +226,13 @@ export function validateRuntimeOptimizationContract(input) {
|
|
|
197
226
|
const contract = cloneJsonValue(assertObject(input, 'contract'));
|
|
198
227
|
assertExactKeys(contract, [
|
|
199
228
|
'schema', 'contractId', 'kind', 'model', 'baseline', 'workload',
|
|
200
|
-
'mutationPolicy', 'verification', 'measurement',
|
|
229
|
+
'mutationPolicy', 'verification', 'measurement', 'neighboringWorkloads',
|
|
201
230
|
], 'contract');
|
|
202
231
|
if (contract.schema !== RUNTIME_OPTIMIZATION_CONTRACT_SCHEMA) {
|
|
203
232
|
throw new Error(`runtime optimization: contract.schema must be "${RUNTIME_OPTIMIZATION_CONTRACT_SCHEMA}".`);
|
|
204
233
|
}
|
|
205
234
|
assertString(contract.contractId, 'contract.contractId');
|
|
206
|
-
|
|
207
|
-
throw new Error('runtime optimization: contract.kind must be "runtime_profile".');
|
|
208
|
-
}
|
|
235
|
+
const candidateKind = normalizeCandidateKind(contract.kind);
|
|
209
236
|
|
|
210
237
|
assertObject(contract.model, 'contract.model');
|
|
211
238
|
assertExactKeys(contract.model, ['modelId', 'modelUrl', 'expectedExecutionContractHash'], 'contract.model');
|
|
@@ -238,28 +265,48 @@ export function validateRuntimeOptimizationContract(input) {
|
|
|
238
265
|
validateWorkloadRequest(contract.workload.request, contract.workload.type);
|
|
239
266
|
|
|
240
267
|
assertObject(contract.mutationPolicy, 'contract.mutationPolicy');
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
268
|
+
if (candidateKind === 'runtime-profile') {
|
|
269
|
+
assertExactKeys(contract.mutationPolicy, ['dimensions', 'maxCandidates'], 'contract.mutationPolicy');
|
|
270
|
+
if (!Array.isArray(contract.mutationPolicy.dimensions) || contract.mutationPolicy.dimensions.length === 0) {
|
|
271
|
+
throw new Error('runtime optimization: mutationPolicy.dimensions must be non-empty.');
|
|
272
|
+
}
|
|
273
|
+
const seenPaths = new Set();
|
|
274
|
+
contract.mutationPolicy.dimensions.forEach((dimension, index) => (
|
|
275
|
+
validateDimension(dimension, index, seenPaths)
|
|
276
|
+
));
|
|
277
|
+
const maxCandidates = assertIntegerRange(
|
|
278
|
+
contract.mutationPolicy.maxCandidates,
|
|
279
|
+
'contract.mutationPolicy.maxCandidates',
|
|
280
|
+
1,
|
|
281
|
+
256
|
|
282
|
+
);
|
|
283
|
+
const candidateCount = contract.mutationPolicy.dimensions.reduce(
|
|
284
|
+
(count, dimension) => count * dimension.values.length,
|
|
285
|
+
1
|
|
286
|
+
);
|
|
287
|
+
if (candidateCount > maxCandidates) {
|
|
288
|
+
throw new Error(
|
|
289
|
+
`runtime optimization: search grid has ${candidateCount} candidates, exceeding maxCandidates=${maxCandidates}.`
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
} else {
|
|
293
|
+
assertExactKeys(contract.mutationPolicy, ['references', 'maxCandidates'], 'contract.mutationPolicy');
|
|
294
|
+
if (!Array.isArray(contract.mutationPolicy.references)
|
|
295
|
+
|| contract.mutationPolicy.references.length === 0) {
|
|
296
|
+
throw new Error('runtime optimization: mutationPolicy.references must be non-empty.');
|
|
297
|
+
}
|
|
298
|
+
contract.mutationPolicy.references.forEach((reference, index) => (
|
|
299
|
+
validateRegisteredReference(reference, `mutationPolicy.references[${index}]`)
|
|
300
|
+
));
|
|
301
|
+
const maxCandidates = assertIntegerRange(
|
|
302
|
+
contract.mutationPolicy.maxCandidates,
|
|
303
|
+
'contract.mutationPolicy.maxCandidates',
|
|
304
|
+
1,
|
|
305
|
+
256
|
|
262
306
|
);
|
|
307
|
+
if (contract.mutationPolicy.references.length > maxCandidates) {
|
|
308
|
+
throw new Error('runtime optimization: registered references exceed maxCandidates.');
|
|
309
|
+
}
|
|
263
310
|
}
|
|
264
311
|
|
|
265
312
|
assertObject(contract.verification, 'contract.verification');
|
|
@@ -287,6 +334,7 @@ export function validateRuntimeOptimizationContract(input) {
|
|
|
287
334
|
assertExactKeys(contract.measurement, [
|
|
288
335
|
'metricPath', 'direction', 'pairCount', 'minValidPairs',
|
|
289
336
|
'minImprovementPercent', 'requirePositiveConfidence', 'maxRelativeStdDevPercent',
|
|
337
|
+
'orderPolicy', 'sequentialDecision',
|
|
290
338
|
], 'contract.measurement');
|
|
291
339
|
if (!SAFE_METRIC_PATHS.has(contract.measurement.metricPath)) {
|
|
292
340
|
throw new Error(`runtime optimization: unsupported metric path "${contract.measurement.metricPath}".`);
|
|
@@ -317,6 +365,92 @@ export function validateRuntimeOptimizationContract(input) {
|
|
|
317
365
|
throw new Error('runtime optimization: measurement.maxRelativeStdDevPercent must be non-negative or null.');
|
|
318
366
|
}
|
|
319
367
|
}
|
|
368
|
+
if (contract.measurement.orderPolicy !== undefined) {
|
|
369
|
+
const order = assertObject(contract.measurement.orderPolicy, 'measurement.orderPolicy');
|
|
370
|
+
assertExactKeys(order, ['kind', 'seed', 'blockSize'], 'measurement.orderPolicy');
|
|
371
|
+
if (order.kind !== 'randomized-blocks') {
|
|
372
|
+
throw new Error('runtime optimization: measurement.orderPolicy.kind must be "randomized-blocks".');
|
|
373
|
+
}
|
|
374
|
+
assertIntegerRange(order.seed, 'measurement.orderPolicy.seed', 0, 0xffffffff);
|
|
375
|
+
if (order.blockSize !== 2) {
|
|
376
|
+
throw new Error('runtime optimization: measurement.orderPolicy.blockSize must be 2.');
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
if (contract.measurement.sequentialDecision !== undefined) {
|
|
380
|
+
const sequential = assertObject(
|
|
381
|
+
contract.measurement.sequentialDecision,
|
|
382
|
+
'measurement.sequentialDecision'
|
|
383
|
+
);
|
|
384
|
+
assertExactKeys(
|
|
385
|
+
sequential,
|
|
386
|
+
['kind', 'lookEveryPairs', 'minimumPairs', 'maximumLooks', 'alpha'],
|
|
387
|
+
'measurement.sequentialDecision'
|
|
388
|
+
);
|
|
389
|
+
if (sequential.kind !== 'bonferroni-fixed-looks') {
|
|
390
|
+
throw new Error(
|
|
391
|
+
'runtime optimization: measurement.sequentialDecision.kind must be "bonferroni-fixed-looks".'
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
const lookEveryPairs = assertIntegerRange(
|
|
395
|
+
sequential.lookEveryPairs,
|
|
396
|
+
'measurement.sequentialDecision.lookEveryPairs',
|
|
397
|
+
1,
|
|
398
|
+
pairCount
|
|
399
|
+
);
|
|
400
|
+
assertIntegerRange(
|
|
401
|
+
sequential.minimumPairs,
|
|
402
|
+
'measurement.sequentialDecision.minimumPairs',
|
|
403
|
+
minValidPairs,
|
|
404
|
+
pairCount
|
|
405
|
+
);
|
|
406
|
+
const expectedLooks = Math.ceil(pairCount / lookEveryPairs);
|
|
407
|
+
if (sequential.maximumLooks !== expectedLooks) {
|
|
408
|
+
throw new Error(
|
|
409
|
+
`runtime optimization: measurement.sequentialDecision.maximumLooks must be ${expectedLooks}.`
|
|
410
|
+
);
|
|
411
|
+
}
|
|
412
|
+
if (!Number.isFinite(sequential.alpha) || sequential.alpha <= 0 || sequential.alpha >= 0.5) {
|
|
413
|
+
throw new Error('runtime optimization: measurement.sequentialDecision.alpha must be in (0, 0.5).');
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
if (contract.neighboringWorkloads !== undefined) {
|
|
417
|
+
if (!Array.isArray(contract.neighboringWorkloads)) {
|
|
418
|
+
throw new Error('runtime optimization: neighboringWorkloads must be an array.');
|
|
419
|
+
}
|
|
420
|
+
const ids = new Set();
|
|
421
|
+
contract.neighboringWorkloads.forEach((guard, index) => {
|
|
422
|
+
const label = `neighboringWorkloads[${index}]`;
|
|
423
|
+
assertObject(guard, label);
|
|
424
|
+
assertExactKeys(
|
|
425
|
+
guard,
|
|
426
|
+
['guardId', 'workload', 'metricPath', 'direction', 'maxRegressionPercent', 'pairCount'],
|
|
427
|
+
label
|
|
428
|
+
);
|
|
429
|
+
const guardId = assertString(guard.guardId, `${label}.guardId`);
|
|
430
|
+
if (ids.has(guardId)) throw new Error(`runtime optimization: duplicate guardId "${guardId}".`);
|
|
431
|
+
ids.add(guardId);
|
|
432
|
+
assertObject(guard.workload, `${label}.workload`);
|
|
433
|
+
assertExactKeys(guard.workload, ['type', 'request'], `${label}.workload`);
|
|
434
|
+
if (!WORKLOADS.has(guard.workload.type)) {
|
|
435
|
+
throw new Error(`runtime optimization: unsupported neighboring workload "${guard.workload.type}".`);
|
|
436
|
+
}
|
|
437
|
+
validateWorkloadRequest(guard.workload.request, guard.workload.type);
|
|
438
|
+
if (!SAFE_METRIC_PATHS.has(guard.metricPath)) {
|
|
439
|
+
throw new Error(`runtime optimization: unsupported neighboring metric "${guard.metricPath}".`);
|
|
440
|
+
}
|
|
441
|
+
if (!DIRECTIONS.has(guard.direction)) {
|
|
442
|
+
throw new Error(`runtime optimization: ${label}.direction is invalid.`);
|
|
443
|
+
}
|
|
444
|
+
const maxRegression = assertFiniteNumber(
|
|
445
|
+
guard.maxRegressionPercent,
|
|
446
|
+
`${label}.maxRegressionPercent`
|
|
447
|
+
);
|
|
448
|
+
if (maxRegression < 0) {
|
|
449
|
+
throw new Error(`runtime optimization: ${label}.maxRegressionPercent must be non-negative.`);
|
|
450
|
+
}
|
|
451
|
+
assertIntegerRange(guard.pairCount, `${label}.pairCount`, 1, 16);
|
|
452
|
+
});
|
|
453
|
+
}
|
|
320
454
|
return contract;
|
|
321
455
|
}
|
|
322
456
|
|
|
@@ -331,26 +465,36 @@ function buildParentHash(contract) {
|
|
|
331
465
|
});
|
|
332
466
|
}
|
|
333
467
|
|
|
334
|
-
function buildCandidate(contract, patch) {
|
|
468
|
+
function buildCandidate(contract, patch, registeredReference = null) {
|
|
335
469
|
const contractHash = computeCanonicalSha256(contract);
|
|
336
470
|
const parentHash = buildParentHash(contract);
|
|
471
|
+
const kind = normalizeCandidateKind(contract.kind);
|
|
337
472
|
const identity = computeCanonicalSha256({
|
|
338
473
|
schema: RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA,
|
|
339
474
|
contractHash,
|
|
340
475
|
parentHash,
|
|
476
|
+
kind,
|
|
341
477
|
patch,
|
|
478
|
+
registeredReference,
|
|
342
479
|
});
|
|
343
480
|
return {
|
|
344
481
|
schema: RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA,
|
|
345
482
|
candidateId: `candidate-${identity.slice('sha256:'.length, 'sha256:'.length + 12)}`,
|
|
346
483
|
contractHash,
|
|
347
484
|
parentHash,
|
|
485
|
+
kind,
|
|
348
486
|
patch,
|
|
487
|
+
...(registeredReference ? { registeredReference } : {}),
|
|
349
488
|
};
|
|
350
489
|
}
|
|
351
490
|
|
|
352
491
|
export function enumerateRuntimeOptimizationCandidates(input) {
|
|
353
492
|
const contract = validateRuntimeOptimizationContract(input);
|
|
493
|
+
if (normalizeCandidateKind(contract.kind) !== 'runtime-profile') {
|
|
494
|
+
return contract.mutationPolicy.references.map((reference) => (
|
|
495
|
+
buildCandidate(contract, [], cloneJsonValue(reference))
|
|
496
|
+
));
|
|
497
|
+
}
|
|
354
498
|
let patches = [[]];
|
|
355
499
|
for (const dimension of contract.mutationPolicy.dimensions) {
|
|
356
500
|
const next = [];
|
|
@@ -374,7 +518,10 @@ function findDimension(contract, path) {
|
|
|
374
518
|
export function validateRuntimeOptimizationCandidate(candidateInput, contractInput) {
|
|
375
519
|
const contract = validateRuntimeOptimizationContract(contractInput);
|
|
376
520
|
const candidate = cloneJsonValue(assertObject(candidateInput, 'candidate'));
|
|
377
|
-
assertExactKeys(candidate, [
|
|
521
|
+
assertExactKeys(candidate, [
|
|
522
|
+
'schema', 'candidateId', 'contractHash', 'parentHash', 'kind', 'patch',
|
|
523
|
+
'registeredReference',
|
|
524
|
+
], 'candidate');
|
|
378
525
|
if (candidate.schema !== RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA) {
|
|
379
526
|
throw new Error(`runtime optimization: candidate.schema must be "${RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA}".`);
|
|
380
527
|
}
|
|
@@ -387,6 +534,28 @@ export function validateRuntimeOptimizationCandidate(candidateInput, contractInp
|
|
|
387
534
|
if (candidate.parentHash !== expectedParentHash) {
|
|
388
535
|
throw new Error('runtime optimization: candidate.parentHash does not match the baseline runtime inputs.');
|
|
389
536
|
}
|
|
537
|
+
const contractKind = normalizeCandidateKind(contract.kind);
|
|
538
|
+
const candidateKind = candidate.kind === undefined
|
|
539
|
+
? 'runtime-profile'
|
|
540
|
+
: normalizeCandidateKind(candidate.kind);
|
|
541
|
+
if (candidateKind !== contractKind) {
|
|
542
|
+
throw new Error('runtime optimization: candidate.kind does not match contract.kind.');
|
|
543
|
+
}
|
|
544
|
+
if (contractKind !== 'runtime-profile') {
|
|
545
|
+
if (!Array.isArray(candidate.patch) || candidate.patch.length !== 0) {
|
|
546
|
+
throw new Error('runtime optimization: registered candidates must not contain inline patches.');
|
|
547
|
+
}
|
|
548
|
+
validateRegisteredReference(candidate.registeredReference, 'candidate.registeredReference');
|
|
549
|
+
const referenceMatch = contract.mutationPolicy.references.some((reference) => (
|
|
550
|
+
reference.registryId === candidate.registeredReference.registryId
|
|
551
|
+
&& reference.digest === candidate.registeredReference.digest
|
|
552
|
+
));
|
|
553
|
+
if (!referenceMatch) {
|
|
554
|
+
throw new Error('runtime optimization: candidate reference is outside the frozen registry domain.');
|
|
555
|
+
}
|
|
556
|
+
candidate.kind = candidateKind;
|
|
557
|
+
return candidate;
|
|
558
|
+
}
|
|
390
559
|
if (!Array.isArray(candidate.patch) || candidate.patch.length !== contract.mutationPolicy.dimensions.length) {
|
|
391
560
|
throw new Error('runtime optimization: candidate.patch must set every frozen mutation dimension exactly once.');
|
|
392
561
|
}
|
|
@@ -411,6 +580,7 @@ export function validateRuntimeOptimizationCandidate(candidateInput, contractInp
|
|
|
411
580
|
throw new Error(`runtime optimization: candidate value for "${path}" is outside the frozen domain.`);
|
|
412
581
|
}
|
|
413
582
|
});
|
|
583
|
+
candidate.kind = candidateKind;
|
|
414
584
|
return candidate;
|
|
415
585
|
}
|
|
416
586
|
|
|
@@ -430,9 +600,104 @@ function setPointerValue(target, path, value) {
|
|
|
430
600
|
cursor[segments.at(-1)] = cloneJsonValue(value);
|
|
431
601
|
}
|
|
432
602
|
|
|
433
|
-
export function
|
|
603
|
+
export function validateRuntimeOptimizationCandidateRegistry(registryInput) {
|
|
604
|
+
const registry = cloneJsonValue(assertObject(registryInput, 'candidate registry'));
|
|
605
|
+
assertExactKeys(registry, ['$schema', 'schema', 'entries'], 'candidate registry');
|
|
606
|
+
if (
|
|
607
|
+
registry.$schema !== undefined
|
|
608
|
+
&& registry.$schema !== RUNTIME_OPTIMIZATION_CANDIDATE_REGISTRY_SCHEMA
|
|
609
|
+
) {
|
|
610
|
+
throw new Error(
|
|
611
|
+
`runtime optimization: candidate registry $schema must be ` +
|
|
612
|
+
`"${RUNTIME_OPTIMIZATION_CANDIDATE_REGISTRY_SCHEMA}".`
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
if (registry.schema !== RUNTIME_OPTIMIZATION_CANDIDATE_REGISTRY_SCHEMA) {
|
|
616
|
+
throw new Error(
|
|
617
|
+
`runtime optimization: candidate registry schema must be ` +
|
|
618
|
+
`"${RUNTIME_OPTIMIZATION_CANDIDATE_REGISTRY_SCHEMA}".`
|
|
619
|
+
);
|
|
620
|
+
}
|
|
621
|
+
assertObject(registry.entries, 'candidate registry.entries');
|
|
622
|
+
for (const [registryId, entry] of Object.entries(registry.entries)) {
|
|
623
|
+
const label = `candidate registry.entries.${registryId}`;
|
|
624
|
+
assertObject(entry, label);
|
|
625
|
+
assertExactKeys(
|
|
626
|
+
entry,
|
|
627
|
+
['registryId', 'kind', 'digest', 'runtimeInputs', 'evidenceScope', 'checkedInPath'],
|
|
628
|
+
label
|
|
629
|
+
);
|
|
630
|
+
if (entry.registryId !== registryId) {
|
|
631
|
+
throw new Error(`runtime optimization: ${label}.registryId must match its registry key.`);
|
|
632
|
+
}
|
|
633
|
+
const kind = normalizeCandidateKind(entry.kind);
|
|
634
|
+
if (kind === 'runtime-profile') {
|
|
635
|
+
throw new Error(`runtime optimization: ${label}.kind must be a registered candidate kind.`);
|
|
636
|
+
}
|
|
637
|
+
assertObject(entry.runtimeInputs, `${label}.runtimeInputs`);
|
|
638
|
+
assertExactKeys(entry.runtimeInputs, ['runtimeProfile', 'runtimeConfig'], `${label}.runtimeInputs`);
|
|
639
|
+
if (entry.runtimeInputs.runtimeProfile !== null) {
|
|
640
|
+
throw new Error(`runtime optimization: ${label}.runtimeInputs.runtimeProfile must be null.`);
|
|
641
|
+
}
|
|
642
|
+
assertJsonValue(
|
|
643
|
+
assertObject(entry.runtimeInputs.runtimeConfig, `${label}.runtimeInputs.runtimeConfig`),
|
|
644
|
+
`${label}.runtimeInputs.runtimeConfig`
|
|
645
|
+
);
|
|
646
|
+
assertJsonValue(
|
|
647
|
+
assertObject(entry.evidenceScope, `${label}.evidenceScope`),
|
|
648
|
+
`${label}.evidenceScope`
|
|
649
|
+
);
|
|
650
|
+
const checkedInPath = assertString(entry.checkedInPath, `${label}.checkedInPath`);
|
|
651
|
+
if (
|
|
652
|
+
!checkedInPath.startsWith('src/config/runtime/optimization-candidates/')
|
|
653
|
+
|| checkedInPath.includes('..')
|
|
654
|
+
) {
|
|
655
|
+
throw new Error(
|
|
656
|
+
`runtime optimization: ${label}.checkedInPath must be under ` +
|
|
657
|
+
'src/config/runtime/optimization-candidates/.'
|
|
658
|
+
);
|
|
659
|
+
}
|
|
660
|
+
assertDigestOrNull(entry.digest, `${label}.digest`);
|
|
661
|
+
const expectedDigest = computeCanonicalSha256({
|
|
662
|
+
registryId,
|
|
663
|
+
kind,
|
|
664
|
+
runtimeInputs: entry.runtimeInputs,
|
|
665
|
+
evidenceScope: entry.evidenceScope,
|
|
666
|
+
checkedInPath,
|
|
667
|
+
});
|
|
668
|
+
if (entry.digest !== expectedDigest) {
|
|
669
|
+
throw new Error(`runtime optimization: ${label}.digest does not match its frozen payload.`);
|
|
670
|
+
}
|
|
671
|
+
entry.kind = kind;
|
|
672
|
+
}
|
|
673
|
+
return registry;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
function materializeRegisteredCandidate(contract, candidate, registryInput) {
|
|
677
|
+
if (!registryInput) {
|
|
678
|
+
throw new Error('runtime optimization: registered candidates require candidateRegistry.');
|
|
679
|
+
}
|
|
680
|
+
const registry = validateRuntimeOptimizationCandidateRegistry(registryInput);
|
|
681
|
+
const entry = registry.entries[candidate.registeredReference.registryId];
|
|
682
|
+
if (!entry || entry.digest !== candidate.registeredReference.digest) {
|
|
683
|
+
throw new Error('runtime optimization: registered candidate is missing or has a digest mismatch.');
|
|
684
|
+
}
|
|
685
|
+
if (entry.kind !== normalizeCandidateKind(contract.kind)) {
|
|
686
|
+
throw new Error('runtime optimization: registered candidate kind does not match the contract.');
|
|
687
|
+
}
|
|
688
|
+
return cloneJsonValue(entry.runtimeInputs);
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
export function materializeRuntimeOptimizationCandidate(
|
|
692
|
+
contractInput,
|
|
693
|
+
candidateInput,
|
|
694
|
+
options = {}
|
|
695
|
+
) {
|
|
434
696
|
const contract = validateRuntimeOptimizationContract(contractInput);
|
|
435
697
|
const candidate = validateRuntimeOptimizationCandidate(candidateInput, contract);
|
|
698
|
+
if (normalizeCandidateKind(contract.kind) !== 'runtime-profile') {
|
|
699
|
+
return materializeRegisteredCandidate(contract, candidate, options.candidateRegistry);
|
|
700
|
+
}
|
|
436
701
|
const runtimeConfig = cloneJsonValue(contract.baseline.runtimeConfig);
|
|
437
702
|
for (const operation of candidate.patch) {
|
|
438
703
|
setPointerValue(runtimeConfig, operation.path, operation.value);
|
|
@@ -498,11 +763,11 @@ function summarizeRun(envelope, metricPath = null) {
|
|
|
498
763
|
};
|
|
499
764
|
}
|
|
500
765
|
|
|
501
|
-
function buildCommandRequest(contract, runtimeInputs, command) {
|
|
502
|
-
const request =
|
|
766
|
+
function buildCommandRequest(contract, runtimeInputs, command, workload = contract.workload) {
|
|
767
|
+
const request = workload.request;
|
|
503
768
|
return {
|
|
504
769
|
command,
|
|
505
|
-
workload:
|
|
770
|
+
workload: workload.type,
|
|
506
771
|
modelId: contract.model.modelId,
|
|
507
772
|
...(contract.model.modelUrl === null ? {} : { modelUrl: contract.model.modelUrl }),
|
|
508
773
|
...(request.inferenceInput == null ? {} : { inferenceInput: cloneJsonValue(request.inferenceInput) }),
|
|
@@ -561,11 +826,50 @@ function median(values) {
|
|
|
561
826
|
: sorted[midpoint];
|
|
562
827
|
}
|
|
563
828
|
|
|
564
|
-
function
|
|
829
|
+
function inverseNormalCdf(probability) {
|
|
830
|
+
const a = [-39.6968302866538, 220.946098424521, -275.928510446969,
|
|
831
|
+
138.357751867269, -30.6647980661472, 2.50662827745924];
|
|
832
|
+
const b = [-54.4760987982241, 161.585836858041, -155.698979859887,
|
|
833
|
+
66.8013118877197, -13.2806815528857];
|
|
834
|
+
const c = [-0.00778489400243029, -0.322396458041136, -2.40075827716184,
|
|
835
|
+
-2.54973253934373, 4.37466414146497, 2.93816398269878];
|
|
836
|
+
const d = [0.00778469570904146, 0.32246712907004, 2.445134137143,
|
|
837
|
+
3.75440866190742];
|
|
838
|
+
const low = 0.02425;
|
|
839
|
+
const high = 1 - low;
|
|
840
|
+
if (probability < low) {
|
|
841
|
+
const q = Math.sqrt(-2 * Math.log(probability));
|
|
842
|
+
return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5])
|
|
843
|
+
/ ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
|
|
844
|
+
}
|
|
845
|
+
if (probability > high) {
|
|
846
|
+
const q = Math.sqrt(-2 * Math.log(1 - probability));
|
|
847
|
+
return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5])
|
|
848
|
+
/ ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
|
|
849
|
+
}
|
|
850
|
+
const q = probability - 0.5;
|
|
851
|
+
const r = q * q;
|
|
852
|
+
return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q
|
|
853
|
+
/ (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
function studentTCritical(confidenceLevel, degrees) {
|
|
857
|
+
if (confidenceLevel === 0.95) return STUDENT_T_95[degrees] ?? 1.96;
|
|
858
|
+
const z = inverseNormalCdf(0.5 + confidenceLevel / 2);
|
|
859
|
+
const z2 = z * z;
|
|
860
|
+
const first = (z ** 3 + z) / (4 * degrees);
|
|
861
|
+
const second = (5 * z ** 5 + 16 * z ** 3 + 3 * z) / (96 * degrees ** 2);
|
|
862
|
+
const third = (3 * z ** 7 + 19 * z ** 5 + 17 * z ** 3 - 15 * z)
|
|
863
|
+
/ (384 * degrees ** 3);
|
|
864
|
+
return z + first + second + third;
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
function sampleStats(values, confidenceLevel = 0.95) {
|
|
565
868
|
if (values.length === 0) {
|
|
566
869
|
return {
|
|
567
870
|
count: 0, min: null, max: null, mean: null, median: null,
|
|
568
871
|
stdDev: null, relativeStdDevPercent: null, confidence95: null,
|
|
872
|
+
decisionConfidence: null,
|
|
569
873
|
};
|
|
570
874
|
}
|
|
571
875
|
const count = values.length;
|
|
@@ -576,11 +880,19 @@ function sampleStats(values) {
|
|
|
576
880
|
const stdDev = Math.sqrt(variance);
|
|
577
881
|
const relativeStdDevPercent = mean === 0 ? null : Math.abs((stdDev / mean) * 100);
|
|
578
882
|
let confidence95 = null;
|
|
883
|
+
let decisionConfidence = null;
|
|
579
884
|
if (count > 1) {
|
|
580
885
|
const degrees = count - 1;
|
|
581
|
-
const critical =
|
|
886
|
+
const critical = studentTCritical(0.95, degrees);
|
|
582
887
|
const halfWidth = critical * stdDev / Math.sqrt(count);
|
|
583
888
|
confidence95 = { low: mean - halfWidth, high: mean + halfWidth };
|
|
889
|
+
const decisionCritical = studentTCritical(confidenceLevel, degrees);
|
|
890
|
+
const decisionHalfWidth = decisionCritical * stdDev / Math.sqrt(count);
|
|
891
|
+
decisionConfidence = {
|
|
892
|
+
level: confidenceLevel,
|
|
893
|
+
low: mean - decisionHalfWidth,
|
|
894
|
+
high: mean + decisionHalfWidth,
|
|
895
|
+
};
|
|
584
896
|
}
|
|
585
897
|
return {
|
|
586
898
|
count,
|
|
@@ -591,9 +903,20 @@ function sampleStats(values) {
|
|
|
591
903
|
stdDev,
|
|
592
904
|
relativeStdDevPercent,
|
|
593
905
|
confidence95,
|
|
906
|
+
decisionConfidence,
|
|
594
907
|
};
|
|
595
908
|
}
|
|
596
909
|
|
|
910
|
+
function randomizedBlockOrder(pairIndex, seed, candidateHash) {
|
|
911
|
+
const blockIndex = Math.floor(pairIndex / 2);
|
|
912
|
+
const seedHash = computeCanonicalSha256({ seed, candidateHash, blockIndex });
|
|
913
|
+
const firstBaseline = Number.parseInt(seedHash.slice(7, 15), 16) % 2 === 0;
|
|
914
|
+
const baselineFirst = pairIndex % 2 === 0 ? firstBaseline : !firstBaseline;
|
|
915
|
+
return baselineFirst
|
|
916
|
+
? ['baseline', 'candidate']
|
|
917
|
+
: ['candidate', 'baseline'];
|
|
918
|
+
}
|
|
919
|
+
|
|
597
920
|
function computeImprovementPercent(baseline, candidate, direction) {
|
|
598
921
|
if (!Number.isFinite(baseline) || !Number.isFinite(candidate) || baseline <= 0 || candidate <= 0) {
|
|
599
922
|
return null;
|
|
@@ -638,6 +961,8 @@ function baseReceipt(contract, candidate, runtimeInputs) {
|
|
|
638
961
|
contractHash: candidate.contractHash,
|
|
639
962
|
candidateId: candidate.candidateId,
|
|
640
963
|
candidateHash: computeCanonicalSha256(candidate),
|
|
964
|
+
candidateKind: candidate.kind,
|
|
965
|
+
registeredReference: cloneJsonValue(candidate.registeredReference ?? null),
|
|
641
966
|
parentHash: candidate.parentHash,
|
|
642
967
|
model: cloneJsonValue(contract.model),
|
|
643
968
|
runtimeInputs: {
|
|
@@ -668,6 +993,121 @@ function rejectedReceipt(base, verification, measurement, reasons, status = 'rej
|
|
|
668
993
|
});
|
|
669
994
|
}
|
|
670
995
|
|
|
996
|
+
async function evaluateNeighboringWorkloadGuards({
|
|
997
|
+
contract,
|
|
998
|
+
baselineInputs,
|
|
999
|
+
runtimeInputs,
|
|
1000
|
+
runCommand,
|
|
1001
|
+
options,
|
|
1002
|
+
}) {
|
|
1003
|
+
const results = [];
|
|
1004
|
+
for (const guard of contract.neighboringWorkloads ?? []) {
|
|
1005
|
+
const baselineVerify = await runCommandSafely(
|
|
1006
|
+
runCommand,
|
|
1007
|
+
buildCommandRequest(contract, baselineInputs, 'verify', guard.workload),
|
|
1008
|
+
options,
|
|
1009
|
+
`neighbor:${guard.guardId}:verification:baseline`
|
|
1010
|
+
);
|
|
1011
|
+
const candidateVerify = await runCommandSafely(
|
|
1012
|
+
runCommand,
|
|
1013
|
+
buildCommandRequest(contract, runtimeInputs, 'verify', guard.workload),
|
|
1014
|
+
options,
|
|
1015
|
+
`neighbor:${guard.guardId}:verification:candidate`
|
|
1016
|
+
);
|
|
1017
|
+
let verification = null;
|
|
1018
|
+
try {
|
|
1019
|
+
if (!baselineVerify.ok || !candidateVerify.ok) {
|
|
1020
|
+
throw new Error('neighbor verification command failed');
|
|
1021
|
+
}
|
|
1022
|
+
assertRunEnvelope(baselineVerify.envelope, 'neighbor baseline verification', contract.model.modelId);
|
|
1023
|
+
assertRunEnvelope(candidateVerify.envelope, 'neighbor candidate verification', contract.model.modelId);
|
|
1024
|
+
verification = compareVerificationRuns(
|
|
1025
|
+
contract,
|
|
1026
|
+
baselineVerify.envelope,
|
|
1027
|
+
candidateVerify.envelope
|
|
1028
|
+
);
|
|
1029
|
+
} catch (error) {
|
|
1030
|
+
results.push({
|
|
1031
|
+
guardId: guard.guardId,
|
|
1032
|
+
passed: false,
|
|
1033
|
+
verification: { passed: false, error: summarizeError(error) },
|
|
1034
|
+
pairs: [],
|
|
1035
|
+
reason: 'neighbor_verification_failed',
|
|
1036
|
+
});
|
|
1037
|
+
continue;
|
|
1038
|
+
}
|
|
1039
|
+
if (!verification.passed) {
|
|
1040
|
+
results.push({
|
|
1041
|
+
guardId: guard.guardId,
|
|
1042
|
+
passed: false,
|
|
1043
|
+
verification,
|
|
1044
|
+
pairs: [],
|
|
1045
|
+
reason: 'neighbor_parity_failed',
|
|
1046
|
+
});
|
|
1047
|
+
continue;
|
|
1048
|
+
}
|
|
1049
|
+
const improvements = [];
|
|
1050
|
+
const pairs = [];
|
|
1051
|
+
for (let pairIndex = 0; pairIndex < guard.pairCount; pairIndex += 1) {
|
|
1052
|
+
const order = pairIndex % 2 === 0
|
|
1053
|
+
? ['baseline', 'candidate']
|
|
1054
|
+
: ['candidate', 'baseline'];
|
|
1055
|
+
const runs = {};
|
|
1056
|
+
for (const role of order) {
|
|
1057
|
+
runs[role] = await runCommandSafely(
|
|
1058
|
+
runCommand,
|
|
1059
|
+
buildCommandRequest(
|
|
1060
|
+
contract,
|
|
1061
|
+
role === 'baseline' ? baselineInputs : runtimeInputs,
|
|
1062
|
+
'bench',
|
|
1063
|
+
guard.workload
|
|
1064
|
+
),
|
|
1065
|
+
options,
|
|
1066
|
+
`neighbor:${guard.guardId}:measurement:${pairIndex}:${role}`
|
|
1067
|
+
);
|
|
1068
|
+
}
|
|
1069
|
+
const pair = { index: pairIndex, order, valid: false };
|
|
1070
|
+
try {
|
|
1071
|
+
if (!runs.baseline.ok || !runs.candidate.ok) {
|
|
1072
|
+
throw new Error('neighbor paired command failed');
|
|
1073
|
+
}
|
|
1074
|
+
assertRunEnvelope(runs.baseline.envelope, 'neighbor baseline benchmark', contract.model.modelId);
|
|
1075
|
+
assertRunEnvelope(runs.candidate.envelope, 'neighbor candidate benchmark', contract.model.modelId);
|
|
1076
|
+
const baselineValue = valueAtPath(runs.baseline.envelope, guard.metricPath);
|
|
1077
|
+
const candidateValue = valueAtPath(runs.candidate.envelope, guard.metricPath);
|
|
1078
|
+
const improvement = computeImprovementPercent(
|
|
1079
|
+
baselineValue,
|
|
1080
|
+
candidateValue,
|
|
1081
|
+
guard.direction
|
|
1082
|
+
);
|
|
1083
|
+
if (improvement === null) throw new Error('neighbor metric is invalid');
|
|
1084
|
+
pair.valid = true;
|
|
1085
|
+
pair.improvementPercent = improvement;
|
|
1086
|
+
improvements.push(improvement);
|
|
1087
|
+
} catch (error) {
|
|
1088
|
+
pair.error = summarizeError(error);
|
|
1089
|
+
}
|
|
1090
|
+
pairs.push(pair);
|
|
1091
|
+
}
|
|
1092
|
+
const stats = sampleStats(improvements);
|
|
1093
|
+
const passed = improvements.length === guard.pairCount
|
|
1094
|
+
&& stats.median >= -guard.maxRegressionPercent;
|
|
1095
|
+
results.push({
|
|
1096
|
+
guardId: guard.guardId,
|
|
1097
|
+
passed,
|
|
1098
|
+
verification,
|
|
1099
|
+
pairs,
|
|
1100
|
+
improvementPercent: stats,
|
|
1101
|
+
maxRegressionPercent: guard.maxRegressionPercent,
|
|
1102
|
+
reason: passed ? null : 'neighbor_regression_exceeded',
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1105
|
+
return {
|
|
1106
|
+
passed: results.every((result) => result.passed),
|
|
1107
|
+
results,
|
|
1108
|
+
};
|
|
1109
|
+
}
|
|
1110
|
+
|
|
671
1111
|
export async function evaluateBrowserRuntimeOptimizationCandidate(
|
|
672
1112
|
contractInput,
|
|
673
1113
|
candidateInput,
|
|
@@ -675,7 +1115,9 @@ export async function evaluateBrowserRuntimeOptimizationCandidate(
|
|
|
675
1115
|
) {
|
|
676
1116
|
const contract = validateRuntimeOptimizationContract(contractInput);
|
|
677
1117
|
const candidate = validateRuntimeOptimizationCandidate(candidateInput, contract);
|
|
678
|
-
const runtimeInputs = materializeRuntimeOptimizationCandidate(contract, candidate
|
|
1118
|
+
const runtimeInputs = materializeRuntimeOptimizationCandidate(contract, candidate, {
|
|
1119
|
+
candidateRegistry: options.candidateRegistry,
|
|
1120
|
+
});
|
|
679
1121
|
const baselineInputs = cloneJsonValue(contract.baseline);
|
|
680
1122
|
const runCommand = options.runCommand ?? runBrowserCommand;
|
|
681
1123
|
if (typeof runCommand !== 'function') {
|
|
@@ -757,10 +1199,20 @@ export async function evaluateBrowserRuntimeOptimizationCandidate(
|
|
|
757
1199
|
const baselineValues = [];
|
|
758
1200
|
const candidateValues = [];
|
|
759
1201
|
const improvements = [];
|
|
1202
|
+
const sequentialLooks = [];
|
|
1203
|
+
let sequentialStop = null;
|
|
760
1204
|
for (let pairIndex = 0; pairIndex < contract.measurement.pairCount; pairIndex += 1) {
|
|
761
|
-
const order =
|
|
762
|
-
?
|
|
763
|
-
|
|
1205
|
+
const order = contract.measurement.orderPolicy
|
|
1206
|
+
? randomizedBlockOrder(
|
|
1207
|
+
pairIndex,
|
|
1208
|
+
contract.measurement.orderPolicy.seed,
|
|
1209
|
+
base.candidateHash
|
|
1210
|
+
)
|
|
1211
|
+
: (
|
|
1212
|
+
pairIndex % 2 === 0
|
|
1213
|
+
? ['baseline', 'candidate']
|
|
1214
|
+
: ['candidate', 'baseline']
|
|
1215
|
+
);
|
|
764
1216
|
const pairRuns = {};
|
|
765
1217
|
for (const role of order) {
|
|
766
1218
|
const request = role === 'baseline' ? baselineBenchRequest : candidateBenchRequest;
|
|
@@ -810,11 +1262,53 @@ export async function evaluateBrowserRuntimeOptimizationCandidate(
|
|
|
810
1262
|
}
|
|
811
1263
|
pairs.push(pair);
|
|
812
1264
|
options.onEvent?.({ type: 'measurement:pair', candidateId: candidate.candidateId, pair });
|
|
1265
|
+
const sequential = contract.measurement.sequentialDecision;
|
|
1266
|
+
const isScheduledLook = sequential
|
|
1267
|
+
&& (
|
|
1268
|
+
(pairIndex + 1) % sequential.lookEveryPairs === 0
|
|
1269
|
+
|| pairIndex + 1 === contract.measurement.pairCount
|
|
1270
|
+
);
|
|
1271
|
+
if (isScheduledLook && improvements.length >= sequential.minimumPairs) {
|
|
1272
|
+
const adjustedAlpha = sequential.alpha / sequential.maximumLooks;
|
|
1273
|
+
const lookStats = sampleStats(improvements, 1 - adjustedAlpha);
|
|
1274
|
+
const look = {
|
|
1275
|
+
look: sequentialLooks.length + 1,
|
|
1276
|
+
completedPairs: pairIndex + 1,
|
|
1277
|
+
validPairs: improvements.length,
|
|
1278
|
+
adjustedAlpha,
|
|
1279
|
+
confidence: lookStats.decisionConfidence,
|
|
1280
|
+
decision: 'continue',
|
|
1281
|
+
};
|
|
1282
|
+
if (
|
|
1283
|
+
lookStats.decisionConfidence
|
|
1284
|
+
&& lookStats.decisionConfidence.low >= contract.measurement.minImprovementPercent
|
|
1285
|
+
) {
|
|
1286
|
+
look.decision = 'accept';
|
|
1287
|
+
sequentialStop = 'accept';
|
|
1288
|
+
} else if (
|
|
1289
|
+
lookStats.decisionConfidence
|
|
1290
|
+
&& lookStats.decisionConfidence.high < contract.measurement.minImprovementPercent
|
|
1291
|
+
) {
|
|
1292
|
+
look.decision = 'reject';
|
|
1293
|
+
sequentialStop = 'reject';
|
|
1294
|
+
}
|
|
1295
|
+
sequentialLooks.push(look);
|
|
1296
|
+
options.onEvent?.({
|
|
1297
|
+
type: 'measurement:sequential-look',
|
|
1298
|
+
candidateId: candidate.candidateId,
|
|
1299
|
+
look,
|
|
1300
|
+
});
|
|
1301
|
+
if (sequentialStop) break;
|
|
1302
|
+
}
|
|
813
1303
|
}
|
|
814
1304
|
|
|
815
1305
|
const baselineStats = sampleStats(baselineValues);
|
|
816
1306
|
const candidateStats = sampleStats(candidateValues);
|
|
817
|
-
const
|
|
1307
|
+
const sequential = contract.measurement.sequentialDecision;
|
|
1308
|
+
const decisionConfidenceLevel = sequential
|
|
1309
|
+
? 1 - (sequential.alpha / sequential.maximumLooks)
|
|
1310
|
+
: 0.95;
|
|
1311
|
+
const improvementStats = sampleStats(improvements, decisionConfidenceLevel);
|
|
818
1312
|
const measurement = {
|
|
819
1313
|
metricPath: contract.measurement.metricPath,
|
|
820
1314
|
direction: contract.measurement.direction,
|
|
@@ -824,6 +1318,15 @@ export async function evaluateBrowserRuntimeOptimizationCandidate(
|
|
|
824
1318
|
baseline: baselineStats,
|
|
825
1319
|
candidate: candidateStats,
|
|
826
1320
|
improvementPercent: improvementStats,
|
|
1321
|
+
orderPolicy: contract.measurement.orderPolicy ?? { kind: 'alternating' },
|
|
1322
|
+
sequentialDecision: sequential
|
|
1323
|
+
? {
|
|
1324
|
+
policy: cloneJsonValue(sequential),
|
|
1325
|
+
looks: sequentialLooks,
|
|
1326
|
+
stoppedEarly: pairs.length < contract.measurement.pairCount,
|
|
1327
|
+
stopDecision: sequentialStop,
|
|
1328
|
+
}
|
|
1329
|
+
: null,
|
|
827
1330
|
};
|
|
828
1331
|
const reasons = [];
|
|
829
1332
|
if (improvements.length < contract.measurement.minValidPairs) {
|
|
@@ -837,12 +1340,15 @@ export async function evaluateBrowserRuntimeOptimizationCandidate(
|
|
|
837
1340
|
}
|
|
838
1341
|
if (contract.measurement.requirePositiveConfidence) {
|
|
839
1342
|
if (
|
|
840
|
-
improvementStats.
|
|
841
|
-
|| improvementStats.
|
|
1343
|
+
improvementStats.decisionConfidence === null
|
|
1344
|
+
|| improvementStats.decisionConfidence.low < contract.measurement.minImprovementPercent
|
|
842
1345
|
) {
|
|
843
1346
|
reasons.push('confidence_interval_below_threshold');
|
|
844
1347
|
}
|
|
845
1348
|
}
|
|
1349
|
+
if (sequentialStop === 'reject') {
|
|
1350
|
+
reasons.push('sequential_evidence_below_threshold');
|
|
1351
|
+
}
|
|
846
1352
|
if (
|
|
847
1353
|
contract.measurement.maxRelativeStdDevPercent !== null
|
|
848
1354
|
&& (
|
|
@@ -852,10 +1358,23 @@ export async function evaluateBrowserRuntimeOptimizationCandidate(
|
|
|
852
1358
|
) {
|
|
853
1359
|
reasons.push('candidate_variance_above_threshold');
|
|
854
1360
|
}
|
|
1361
|
+
const neighboringWorkloadGuards = reasons.length === 0
|
|
1362
|
+
? await evaluateNeighboringWorkloadGuards({
|
|
1363
|
+
contract,
|
|
1364
|
+
baselineInputs,
|
|
1365
|
+
runtimeInputs,
|
|
1366
|
+
runCommand,
|
|
1367
|
+
options,
|
|
1368
|
+
})
|
|
1369
|
+
: { passed: true, results: [], skipped: 'primary_candidate_rejected' };
|
|
1370
|
+
if (!neighboringWorkloadGuards.passed) {
|
|
1371
|
+
reasons.push('neighboring_workload_guard_failed');
|
|
1372
|
+
}
|
|
855
1373
|
const receipt = finalizeReceipt({
|
|
856
1374
|
...base,
|
|
857
1375
|
verification,
|
|
858
1376
|
measurement,
|
|
1377
|
+
neighboringWorkloadGuards,
|
|
859
1378
|
decision: {
|
|
860
1379
|
accepted: reasons.length === 0,
|
|
861
1380
|
status: reasons.length === 0 ? 'accepted' : 'rejected',
|