doppler-gpu 0.4.9 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,872 @@
1
+ import { cloneJsonValue } from '../utils/clone-json.js';
2
+ import { computeCanonicalSha256, canonicalizeJson } from '../utils/canonical-hash.js';
3
+ import { isPlainObject } from '../utils/plain-object.js';
4
+ import { runBrowserCommand } from './browser-command-runner.js';
5
+
6
+ export const RUNTIME_OPTIMIZATION_CONTRACT_SCHEMA = 'doppler.runtime-optimization-contract/v1';
7
+ export const RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA = 'doppler.runtime-optimization-candidate/v1';
8
+ export const RUNTIME_OPTIMIZATION_RECEIPT_SCHEMA = 'doppler.runtime-optimization-receipt/v1';
9
+
10
+ const WORKLOADS = new Set(['inference', 'embedding', 'rerank']);
11
+ const DIRECTIONS = new Set(['maximize', 'minimize']);
12
+ const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/;
13
+ const SAFE_MUTATION_PREFIXES = Object.freeze([
14
+ '/loading/shardCache',
15
+ '/loading/memoryManagement',
16
+ '/loading/prefetch',
17
+ '/loading/expertCache',
18
+ '/inference/batching',
19
+ '/inference/generation',
20
+ '/inference/session',
21
+ '/shared/bufferPool',
22
+ '/shared/gpuCache',
23
+ '/shared/kernelWarmup',
24
+ '/shared/memory',
25
+ ]);
26
+ const FORBIDDEN_MUTATION_PREFIXES = Object.freeze([
27
+ '/inference/executionPatch',
28
+ '/inference/kernelPath',
29
+ '/inference/kernelPathPolicy',
30
+ '/shared/benchmark',
31
+ '/shared/debug',
32
+ '/shared/harness',
33
+ '/shared/kernelRegistry',
34
+ '/shared/platform',
35
+ '/shared/tooling',
36
+ ]);
37
+ const SAFE_COMPARISON_PATHS = new Set([
38
+ 'result.output',
39
+ 'result.metrics.referenceTranscript.tokens.generatedTokenIdsHash',
40
+ 'result.metrics.referenceTranscript.output.textHash',
41
+ ]);
42
+ const SAFE_METRIC_PATHS = new Set([
43
+ 'result.metrics.decodeTokensPerSec',
44
+ 'result.metrics.embeddingMs',
45
+ 'result.metrics.rerankMs',
46
+ 'result.timing.decodeTokensPerSec',
47
+ 'result.timing.totalRunMs',
48
+ ]);
49
+ const STUDENT_T_95 = Object.freeze({
50
+ 1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571,
51
+ 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228,
52
+ 11: 2.201, 12: 2.179, 13: 2.160, 14: 2.145, 15: 2.131,
53
+ 16: 2.120, 17: 2.110, 18: 2.101, 19: 2.093, 20: 2.086,
54
+ 21: 2.080, 22: 2.074, 23: 2.069, 24: 2.064, 25: 2.060,
55
+ 26: 2.056, 27: 2.052, 28: 2.048, 29: 2.045, 30: 2.042,
56
+ });
57
+
58
+ function assertObject(value, label) {
59
+ if (!isPlainObject(value)) {
60
+ throw new Error(`runtime optimization: ${label} must be an object.`);
61
+ }
62
+ return value;
63
+ }
64
+
65
+ function assertExactKeys(value, keys, label) {
66
+ const allowed = new Set(keys);
67
+ for (const key of Object.keys(value)) {
68
+ if (!allowed.has(key)) {
69
+ throw new Error(`runtime optimization: ${label}.${key} is not supported.`);
70
+ }
71
+ }
72
+ }
73
+
74
+ function assertString(value, label) {
75
+ if (typeof value !== 'string' || !value.trim()) {
76
+ throw new Error(`runtime optimization: ${label} must be a non-empty string.`);
77
+ }
78
+ return value.trim();
79
+ }
80
+
81
+ function assertDigestOrNull(value, label) {
82
+ if (value === null) return null;
83
+ if (typeof value !== 'string' || !DIGEST_PATTERN.test(value)) {
84
+ throw new Error(`runtime optimization: ${label} must be sha256:<64 lowercase hex> or null.`);
85
+ }
86
+ return value;
87
+ }
88
+
89
+ function assertIntegerRange(value, label, minimum, maximum) {
90
+ if (!Number.isInteger(value) || value < minimum || value > maximum) {
91
+ throw new Error(
92
+ `runtime optimization: ${label} must be an integer in [${minimum}, ${maximum}].`
93
+ );
94
+ }
95
+ return value;
96
+ }
97
+
98
+ function assertFiniteNumber(value, label) {
99
+ if (typeof value !== 'number' || !Number.isFinite(value)) {
100
+ throw new Error(`runtime optimization: ${label} must be a finite number.`);
101
+ }
102
+ return value;
103
+ }
104
+
105
+ function assertJsonValue(value, label) {
106
+ try {
107
+ canonicalizeJson(value);
108
+ } catch (error) {
109
+ throw new Error(`runtime optimization: ${label} must be canonical JSON: ${error.message}`);
110
+ }
111
+ return value;
112
+ }
113
+
114
+ function pointerMatchesPrefix(path, prefix) {
115
+ return path === prefix || path.startsWith(`${prefix}/`);
116
+ }
117
+
118
+ function assertSafeMutationPath(path, label) {
119
+ const normalized = assertString(path, label);
120
+ if (!normalized.startsWith('/') || normalized.endsWith('/')) {
121
+ throw new Error(`runtime optimization: ${label} must be a canonical JSON pointer.`);
122
+ }
123
+ if (FORBIDDEN_MUTATION_PREFIXES.some((prefix) => pointerMatchesPrefix(normalized, prefix))) {
124
+ throw new Error(`runtime optimization: ${label} targets evaluator or manifest-owned policy: ${normalized}.`);
125
+ }
126
+ if (!SAFE_MUTATION_PREFIXES.some((prefix) => pointerMatchesPrefix(normalized, prefix))) {
127
+ throw new Error(`runtime optimization: ${label} is outside the runtime-owned allowlist: ${normalized}.`);
128
+ }
129
+ decodeJsonPointer(normalized);
130
+ return normalized;
131
+ }
132
+
133
+ function decodeJsonPointer(path) {
134
+ if (path === '') return [];
135
+ if (!path.startsWith('/')) {
136
+ throw new Error(`runtime optimization: invalid JSON pointer "${path}".`);
137
+ }
138
+ return path.slice(1).split('/').map((segment) => {
139
+ if (/~(?:[^01]|$)/.test(segment)) {
140
+ throw new Error(`runtime optimization: invalid JSON pointer escape in "${path}".`);
141
+ }
142
+ const decoded = segment.replace(/~1/g, '/').replace(/~0/g, '~');
143
+ if (!decoded || /^\d+$/.test(decoded)) {
144
+ throw new Error(`runtime optimization: JSON pointer segments must name object fields: "${path}".`);
145
+ }
146
+ return decoded;
147
+ });
148
+ }
149
+
150
+ function validateWorkloadRequest(value, workload) {
151
+ const request = assertObject(value, 'workload.request');
152
+ assertExactKeys(request, ['inferenceInput', 'cacheMode', 'loadMode'], 'workload.request');
153
+ if (request.inferenceInput !== undefined && request.inferenceInput !== null) {
154
+ if (workload !== 'inference') {
155
+ throw new Error('runtime optimization: workload.request.inferenceInput requires workload.type="inference".');
156
+ }
157
+ assertJsonValue(assertObject(request.inferenceInput, 'workload.request.inferenceInput'), 'workload.request.inferenceInput');
158
+ }
159
+ if (![undefined, null, 'cold', 'warm'].includes(request.cacheMode)) {
160
+ throw new Error('runtime optimization: workload.request.cacheMode must be "cold", "warm", or null.');
161
+ }
162
+ if (![undefined, null, 'opfs', 'http', 'memory', 'file'].includes(request.loadMode)) {
163
+ throw new Error('runtime optimization: workload.request.loadMode is unsupported.');
164
+ }
165
+ return request;
166
+ }
167
+
168
+ function validateDimension(dimension, index, seenPaths) {
169
+ assertObject(dimension, `mutationPolicy.dimensions[${index}]`);
170
+ assertExactKeys(dimension, ['path', 'values'], `mutationPolicy.dimensions[${index}]`);
171
+ const path = assertSafeMutationPath(
172
+ dimension.path,
173
+ `mutationPolicy.dimensions[${index}].path`
174
+ );
175
+ if (seenPaths.has(path)) {
176
+ throw new Error(`runtime optimization: duplicate mutation dimension "${path}".`);
177
+ }
178
+ seenPaths.add(path);
179
+ if (!Array.isArray(dimension.values) || dimension.values.length === 0) {
180
+ throw new Error(`runtime optimization: mutationPolicy.dimensions[${index}].values must be non-empty.`);
181
+ }
182
+ const valueKeys = new Set();
183
+ for (let valueIndex = 0; valueIndex < dimension.values.length; valueIndex += 1) {
184
+ const value = assertJsonValue(
185
+ dimension.values[valueIndex],
186
+ `mutationPolicy.dimensions[${index}].values[${valueIndex}]`
187
+ );
188
+ const key = canonicalizeJson(value);
189
+ if (valueKeys.has(key)) {
190
+ throw new Error(`runtime optimization: mutation dimension "${path}" contains duplicate values.`);
191
+ }
192
+ valueKeys.add(key);
193
+ }
194
+ }
195
+
196
+ export function validateRuntimeOptimizationContract(input) {
197
+ const contract = cloneJsonValue(assertObject(input, 'contract'));
198
+ assertExactKeys(contract, [
199
+ 'schema', 'contractId', 'kind', 'model', 'baseline', 'workload',
200
+ 'mutationPolicy', 'verification', 'measurement',
201
+ ], 'contract');
202
+ if (contract.schema !== RUNTIME_OPTIMIZATION_CONTRACT_SCHEMA) {
203
+ throw new Error(`runtime optimization: contract.schema must be "${RUNTIME_OPTIMIZATION_CONTRACT_SCHEMA}".`);
204
+ }
205
+ assertString(contract.contractId, 'contract.contractId');
206
+ if (contract.kind !== 'runtime_profile') {
207
+ throw new Error('runtime optimization: contract.kind must be "runtime_profile".');
208
+ }
209
+
210
+ assertObject(contract.model, 'contract.model');
211
+ assertExactKeys(contract.model, ['modelId', 'modelUrl', 'expectedExecutionContractHash'], 'contract.model');
212
+ assertString(contract.model.modelId, 'contract.model.modelId');
213
+ if (contract.model.modelUrl !== null) {
214
+ assertString(contract.model.modelUrl, 'contract.model.modelUrl');
215
+ }
216
+ assertDigestOrNull(
217
+ contract.model.expectedExecutionContractHash,
218
+ 'contract.model.expectedExecutionContractHash'
219
+ );
220
+
221
+ assertObject(contract.baseline, 'contract.baseline');
222
+ assertExactKeys(contract.baseline, ['runtimeProfile', 'runtimeConfig'], 'contract.baseline');
223
+ if (contract.baseline.runtimeProfile !== null) {
224
+ throw new Error(
225
+ 'runtime optimization: contract.baseline.runtimeProfile must be null in v1; provide an explicit runtimeConfig overlay.'
226
+ );
227
+ }
228
+ assertJsonValue(
229
+ assertObject(contract.baseline.runtimeConfig, 'contract.baseline.runtimeConfig'),
230
+ 'contract.baseline.runtimeConfig'
231
+ );
232
+
233
+ assertObject(contract.workload, 'contract.workload');
234
+ assertExactKeys(contract.workload, ['type', 'request'], 'contract.workload');
235
+ if (!WORKLOADS.has(contract.workload.type)) {
236
+ throw new Error(`runtime optimization: unsupported workload "${contract.workload.type}".`);
237
+ }
238
+ validateWorkloadRequest(contract.workload.request, contract.workload.type);
239
+
240
+ assertObject(contract.mutationPolicy, 'contract.mutationPolicy');
241
+ assertExactKeys(contract.mutationPolicy, ['dimensions', 'maxCandidates'], 'contract.mutationPolicy');
242
+ if (!Array.isArray(contract.mutationPolicy.dimensions) || contract.mutationPolicy.dimensions.length === 0) {
243
+ throw new Error('runtime optimization: mutationPolicy.dimensions must be non-empty.');
244
+ }
245
+ const seenPaths = new Set();
246
+ contract.mutationPolicy.dimensions.forEach((dimension, index) => (
247
+ validateDimension(dimension, index, seenPaths)
248
+ ));
249
+ const maxCandidates = assertIntegerRange(
250
+ contract.mutationPolicy.maxCandidates,
251
+ 'contract.mutationPolicy.maxCandidates',
252
+ 1,
253
+ 256
254
+ );
255
+ const candidateCount = contract.mutationPolicy.dimensions.reduce(
256
+ (count, dimension) => count * dimension.values.length,
257
+ 1
258
+ );
259
+ if (candidateCount > maxCandidates) {
260
+ throw new Error(
261
+ `runtime optimization: search grid has ${candidateCount} candidates, exceeding maxCandidates=${maxCandidates}.`
262
+ );
263
+ }
264
+
265
+ assertObject(contract.verification, 'contract.verification');
266
+ assertExactKeys(contract.verification, ['comparisons'], 'contract.verification');
267
+ if (!Array.isArray(contract.verification.comparisons) || contract.verification.comparisons.length === 0) {
268
+ throw new Error('runtime optimization: verification.comparisons must be non-empty.');
269
+ }
270
+ const comparisonPaths = new Set();
271
+ contract.verification.comparisons.forEach((comparison, index) => {
272
+ assertObject(comparison, `verification.comparisons[${index}]`);
273
+ assertExactKeys(comparison, ['path', 'mode'], `verification.comparisons[${index}]`);
274
+ if (!SAFE_COMPARISON_PATHS.has(comparison.path)) {
275
+ throw new Error(`runtime optimization: unsupported comparison path "${comparison.path}".`);
276
+ }
277
+ if (comparison.mode !== 'canonical_exact') {
278
+ throw new Error('runtime optimization: comparison mode must be "canonical_exact".');
279
+ }
280
+ if (comparisonPaths.has(comparison.path)) {
281
+ throw new Error(`runtime optimization: duplicate comparison path "${comparison.path}".`);
282
+ }
283
+ comparisonPaths.add(comparison.path);
284
+ });
285
+
286
+ assertObject(contract.measurement, 'contract.measurement');
287
+ assertExactKeys(contract.measurement, [
288
+ 'metricPath', 'direction', 'pairCount', 'minValidPairs',
289
+ 'minImprovementPercent', 'requirePositiveConfidence', 'maxRelativeStdDevPercent',
290
+ ], 'contract.measurement');
291
+ if (!SAFE_METRIC_PATHS.has(contract.measurement.metricPath)) {
292
+ throw new Error(`runtime optimization: unsupported metric path "${contract.measurement.metricPath}".`);
293
+ }
294
+ if (!DIRECTIONS.has(contract.measurement.direction)) {
295
+ throw new Error('runtime optimization: measurement.direction must be "maximize" or "minimize".');
296
+ }
297
+ const pairCount = assertIntegerRange(contract.measurement.pairCount, 'measurement.pairCount', 1, 64);
298
+ const minValidPairs = assertIntegerRange(
299
+ contract.measurement.minValidPairs,
300
+ 'measurement.minValidPairs',
301
+ 1,
302
+ 64
303
+ );
304
+ if (minValidPairs > pairCount) {
305
+ throw new Error('runtime optimization: measurement.minValidPairs must not exceed pairCount.');
306
+ }
307
+ assertFiniteNumber(contract.measurement.minImprovementPercent, 'measurement.minImprovementPercent');
308
+ if (typeof contract.measurement.requirePositiveConfidence !== 'boolean') {
309
+ throw new Error('runtime optimization: measurement.requirePositiveConfidence must be boolean.');
310
+ }
311
+ if (contract.measurement.maxRelativeStdDevPercent !== null) {
312
+ const maxStdDev = assertFiniteNumber(
313
+ contract.measurement.maxRelativeStdDevPercent,
314
+ 'measurement.maxRelativeStdDevPercent'
315
+ );
316
+ if (maxStdDev < 0) {
317
+ throw new Error('runtime optimization: measurement.maxRelativeStdDevPercent must be non-negative or null.');
318
+ }
319
+ }
320
+ return contract;
321
+ }
322
+
323
+ export function hashRuntimeOptimizationContract(input) {
324
+ return computeCanonicalSha256(validateRuntimeOptimizationContract(input));
325
+ }
326
+
327
+ function buildParentHash(contract) {
328
+ return computeCanonicalSha256({
329
+ runtimeProfile: contract.baseline.runtimeProfile,
330
+ runtimeConfig: contract.baseline.runtimeConfig,
331
+ });
332
+ }
333
+
334
+ function buildCandidate(contract, patch) {
335
+ const contractHash = computeCanonicalSha256(contract);
336
+ const parentHash = buildParentHash(contract);
337
+ const identity = computeCanonicalSha256({
338
+ schema: RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA,
339
+ contractHash,
340
+ parentHash,
341
+ patch,
342
+ });
343
+ return {
344
+ schema: RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA,
345
+ candidateId: `candidate-${identity.slice('sha256:'.length, 'sha256:'.length + 12)}`,
346
+ contractHash,
347
+ parentHash,
348
+ patch,
349
+ };
350
+ }
351
+
352
+ export function enumerateRuntimeOptimizationCandidates(input) {
353
+ const contract = validateRuntimeOptimizationContract(input);
354
+ let patches = [[]];
355
+ for (const dimension of contract.mutationPolicy.dimensions) {
356
+ const next = [];
357
+ for (const patch of patches) {
358
+ for (const value of dimension.values) {
359
+ next.push([
360
+ ...patch,
361
+ { op: 'set', path: dimension.path, value: cloneJsonValue(value) },
362
+ ]);
363
+ }
364
+ }
365
+ patches = next;
366
+ }
367
+ return patches.map((patch) => buildCandidate(contract, patch));
368
+ }
369
+
370
+ function findDimension(contract, path) {
371
+ return contract.mutationPolicy.dimensions.find((dimension) => dimension.path === path) ?? null;
372
+ }
373
+
374
+ export function validateRuntimeOptimizationCandidate(candidateInput, contractInput) {
375
+ const contract = validateRuntimeOptimizationContract(contractInput);
376
+ const candidate = cloneJsonValue(assertObject(candidateInput, 'candidate'));
377
+ assertExactKeys(candidate, ['schema', 'candidateId', 'contractHash', 'parentHash', 'patch'], 'candidate');
378
+ if (candidate.schema !== RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA) {
379
+ throw new Error(`runtime optimization: candidate.schema must be "${RUNTIME_OPTIMIZATION_CANDIDATE_SCHEMA}".`);
380
+ }
381
+ assertString(candidate.candidateId, 'candidate.candidateId');
382
+ const expectedContractHash = computeCanonicalSha256(contract);
383
+ if (candidate.contractHash !== expectedContractHash) {
384
+ throw new Error('runtime optimization: candidate.contractHash does not match the frozen contract.');
385
+ }
386
+ const expectedParentHash = buildParentHash(contract);
387
+ if (candidate.parentHash !== expectedParentHash) {
388
+ throw new Error('runtime optimization: candidate.parentHash does not match the baseline runtime inputs.');
389
+ }
390
+ if (!Array.isArray(candidate.patch) || candidate.patch.length !== contract.mutationPolicy.dimensions.length) {
391
+ throw new Error('runtime optimization: candidate.patch must set every frozen mutation dimension exactly once.');
392
+ }
393
+ const paths = new Set();
394
+ candidate.patch.forEach((operation, index) => {
395
+ assertObject(operation, `candidate.patch[${index}]`);
396
+ assertExactKeys(operation, ['op', 'path', 'value'], `candidate.patch[${index}]`);
397
+ if (operation.op !== 'set') {
398
+ throw new Error('runtime optimization: candidate patch operations must use op="set".');
399
+ }
400
+ const path = assertSafeMutationPath(operation.path, `candidate.patch[${index}].path`);
401
+ if (paths.has(path)) {
402
+ throw new Error(`runtime optimization: candidate.patch contains duplicate path "${path}".`);
403
+ }
404
+ paths.add(path);
405
+ const dimension = findDimension(contract, path);
406
+ if (!dimension) {
407
+ throw new Error(`runtime optimization: candidate path "${path}" is not in the frozen grid.`);
408
+ }
409
+ const candidateValue = canonicalizeJson(assertJsonValue(operation.value, `candidate.patch[${index}].value`));
410
+ if (!dimension.values.some((value) => canonicalizeJson(value) === candidateValue)) {
411
+ throw new Error(`runtime optimization: candidate value for "${path}" is outside the frozen domain.`);
412
+ }
413
+ });
414
+ return candidate;
415
+ }
416
+
417
+ function setPointerValue(target, path, value) {
418
+ const segments = decodeJsonPointer(path);
419
+ let cursor = target;
420
+ for (let index = 0; index < segments.length - 1; index += 1) {
421
+ const segment = segments[index];
422
+ const current = cursor[segment];
423
+ if (current === undefined) {
424
+ cursor[segment] = {};
425
+ } else if (!isPlainObject(current)) {
426
+ throw new Error(`runtime optimization: candidate path "${path}" crosses a non-object field.`);
427
+ }
428
+ cursor = cursor[segment];
429
+ }
430
+ cursor[segments.at(-1)] = cloneJsonValue(value);
431
+ }
432
+
433
+ export function materializeRuntimeOptimizationCandidate(contractInput, candidateInput) {
434
+ const contract = validateRuntimeOptimizationContract(contractInput);
435
+ const candidate = validateRuntimeOptimizationCandidate(candidateInput, contract);
436
+ const runtimeConfig = cloneJsonValue(contract.baseline.runtimeConfig);
437
+ for (const operation of candidate.patch) {
438
+ setPointerValue(runtimeConfig, operation.path, operation.value);
439
+ }
440
+ return {
441
+ runtimeProfile: null,
442
+ runtimeConfig,
443
+ };
444
+ }
445
+
446
+ function valueAtPath(value, path) {
447
+ const segments = String(path).split('.');
448
+ let cursor = value;
449
+ for (const segment of segments) {
450
+ if (cursor == null || !Object.prototype.hasOwnProperty.call(cursor, segment)) {
451
+ return undefined;
452
+ }
453
+ cursor = cursor[segment];
454
+ }
455
+ return cursor;
456
+ }
457
+
458
+ function summarizeError(error) {
459
+ return {
460
+ name: typeof error?.name === 'string' ? error.name : 'Error',
461
+ message: typeof error?.message === 'string' ? error.message : String(error),
462
+ code: typeof error?.code === 'string' ? error.code : null,
463
+ retryable: typeof error?.retryable === 'boolean' ? error.retryable : null,
464
+ };
465
+ }
466
+
467
+ function assertRunEnvelope(envelope, label, modelId) {
468
+ if (!isPlainObject(envelope) || envelope.ok !== true || !isPlainObject(envelope.result)) {
469
+ throw new Error(`${label} did not return a Doppler success envelope.`);
470
+ }
471
+ if (envelope.result.modelId !== modelId) {
472
+ throw new Error(`${label} modelId mismatch: expected "${modelId}", got "${envelope.result.modelId}".`);
473
+ }
474
+ if (!Number.isInteger(envelope.result.passed) || envelope.result.passed < 1) {
475
+ throw new Error(`${label} did not report a passing suite result.`);
476
+ }
477
+ if (!Number.isInteger(envelope.result.failed) || envelope.result.failed !== 0) {
478
+ throw new Error(`${label} reported one or more failed suite results.`);
479
+ }
480
+ return envelope;
481
+ }
482
+
483
+ function summarizeRun(envelope, metricPath = null) {
484
+ const executionContract = envelope.result?.metrics?.executionContractArtifact ?? null;
485
+ const metric = metricPath ? valueAtPath(envelope, metricPath) : null;
486
+ return {
487
+ envelopeHash: computeCanonicalSha256(envelope),
488
+ modelId: envelope.result?.modelId ?? null,
489
+ suite: envelope.result?.suite ?? null,
490
+ passed: envelope.result?.passed ?? null,
491
+ failed: envelope.result?.failed ?? null,
492
+ outputHash: computeCanonicalSha256(envelope.result?.output ?? null),
493
+ executionContractHash: executionContract == null
494
+ ? null
495
+ : computeCanonicalSha256(executionContract),
496
+ metric: metric == null ? null : metric,
497
+ deviceInfo: cloneJsonValue(envelope.result?.deviceInfo ?? null),
498
+ };
499
+ }
500
+
501
+ function buildCommandRequest(contract, runtimeInputs, command) {
502
+ const request = contract.workload.request;
503
+ return {
504
+ command,
505
+ workload: contract.workload.type,
506
+ modelId: contract.model.modelId,
507
+ ...(contract.model.modelUrl === null ? {} : { modelUrl: contract.model.modelUrl }),
508
+ ...(request.inferenceInput == null ? {} : { inferenceInput: cloneJsonValue(request.inferenceInput) }),
509
+ ...(request.cacheMode == null ? {} : { cacheMode: request.cacheMode }),
510
+ ...(request.loadMode == null ? {} : { loadMode: request.loadMode }),
511
+ runtimeProfile: null,
512
+ runtimeConfig: { runtime: cloneJsonValue(runtimeInputs.runtimeConfig) },
513
+ captureOutput: true,
514
+ keepPipeline: false,
515
+ };
516
+ }
517
+
518
+ function compareVerificationRuns(contract, baselineEnvelope, candidateEnvelope) {
519
+ const comparisons = contract.verification.comparisons.map((comparison) => {
520
+ const baselineValue = valueAtPath(baselineEnvelope, comparison.path);
521
+ const candidateValue = valueAtPath(candidateEnvelope, comparison.path);
522
+ const baselineHash = computeCanonicalSha256(baselineValue);
523
+ const candidateHash = computeCanonicalSha256(candidateValue);
524
+ return {
525
+ path: comparison.path,
526
+ mode: comparison.mode,
527
+ passed: baselineValue !== undefined
528
+ && candidateValue !== undefined
529
+ && baselineHash === candidateHash,
530
+ baselineHash,
531
+ candidateHash,
532
+ };
533
+ });
534
+ const baselineSummary = summarizeRun(baselineEnvelope);
535
+ const candidateSummary = summarizeRun(candidateEnvelope);
536
+ const artifactMatches = baselineSummary.executionContractHash === candidateSummary.executionContractHash;
537
+ const expectedArtifact = contract.model.expectedExecutionContractHash;
538
+ const expectedArtifactMatches = expectedArtifact === null
539
+ || (
540
+ baselineSummary.executionContractHash === expectedArtifact
541
+ && candidateSummary.executionContractHash === expectedArtifact
542
+ );
543
+ return {
544
+ passed: comparisons.every((comparison) => comparison.passed)
545
+ && artifactMatches
546
+ && expectedArtifactMatches,
547
+ comparisons,
548
+ artifactMatches,
549
+ expectedArtifactMatches,
550
+ baseline: baselineSummary,
551
+ candidate: candidateSummary,
552
+ };
553
+ }
554
+
555
+ function median(values) {
556
+ if (values.length === 0) return null;
557
+ const sorted = [...values].sort((left, right) => left - right);
558
+ const midpoint = Math.floor(sorted.length / 2);
559
+ return sorted.length % 2 === 0
560
+ ? (sorted[midpoint - 1] + sorted[midpoint]) / 2
561
+ : sorted[midpoint];
562
+ }
563
+
564
+ function sampleStats(values) {
565
+ if (values.length === 0) {
566
+ return {
567
+ count: 0, min: null, max: null, mean: null, median: null,
568
+ stdDev: null, relativeStdDevPercent: null, confidence95: null,
569
+ };
570
+ }
571
+ const count = values.length;
572
+ const mean = values.reduce((sum, value) => sum + value, 0) / count;
573
+ const variance = count > 1
574
+ ? values.reduce((sum, value) => sum + ((value - mean) ** 2), 0) / (count - 1)
575
+ : 0;
576
+ const stdDev = Math.sqrt(variance);
577
+ const relativeStdDevPercent = mean === 0 ? null : Math.abs((stdDev / mean) * 100);
578
+ let confidence95 = null;
579
+ if (count > 1) {
580
+ const degrees = count - 1;
581
+ const critical = STUDENT_T_95[degrees] ?? 1.96;
582
+ const halfWidth = critical * stdDev / Math.sqrt(count);
583
+ confidence95 = { low: mean - halfWidth, high: mean + halfWidth };
584
+ }
585
+ return {
586
+ count,
587
+ min: Math.min(...values),
588
+ max: Math.max(...values),
589
+ mean,
590
+ median: median(values),
591
+ stdDev,
592
+ relativeStdDevPercent,
593
+ confidence95,
594
+ };
595
+ }
596
+
597
+ function computeImprovementPercent(baseline, candidate, direction) {
598
+ if (!Number.isFinite(baseline) || !Number.isFinite(candidate) || baseline <= 0 || candidate <= 0) {
599
+ return null;
600
+ }
601
+ return direction === 'maximize'
602
+ ? ((candidate - baseline) / baseline) * 100
603
+ : ((baseline - candidate) / baseline) * 100;
604
+ }
605
+
606
+ function assertNotAborted(signal) {
607
+ if (signal?.aborted) {
608
+ const error = new Error('runtime optimization: evaluation aborted.');
609
+ error.name = 'AbortError';
610
+ throw error;
611
+ }
612
+ }
613
+
614
+ async function runCommandSafely(runCommand, request, options, label) {
615
+ assertNotAborted(options.signal);
616
+ options.onEvent?.({ type: 'command:start', label, request: cloneJsonValue(request) });
617
+ try {
618
+ const envelope = await runCommand(request, {
619
+ ...(options.commandOptions ?? {}),
620
+ runtimeLoadOptions: {
621
+ ...(options.commandOptions?.runtimeLoadOptions ?? {}),
622
+ ...(options.signal ? { signal: options.signal } : {}),
623
+ },
624
+ });
625
+ options.onEvent?.({ type: 'command:complete', label, ok: true });
626
+ return { ok: true, envelope };
627
+ } catch (error) {
628
+ const summarized = summarizeError(error);
629
+ options.onEvent?.({ type: 'command:complete', label, ok: false, error: summarized });
630
+ return { ok: false, error: summarized };
631
+ }
632
+ }
633
+
634
+ function baseReceipt(contract, candidate, runtimeInputs) {
635
+ return {
636
+ schema: RUNTIME_OPTIMIZATION_RECEIPT_SCHEMA,
637
+ contractId: contract.contractId,
638
+ contractHash: candidate.contractHash,
639
+ candidateId: candidate.candidateId,
640
+ candidateHash: computeCanonicalSha256(candidate),
641
+ parentHash: candidate.parentHash,
642
+ model: cloneJsonValue(contract.model),
643
+ runtimeInputs: {
644
+ baseline: cloneJsonValue(contract.baseline),
645
+ candidate: cloneJsonValue(runtimeInputs),
646
+ candidateRuntimeConfigHash: computeCanonicalSha256(runtimeInputs.runtimeConfig),
647
+ },
648
+ };
649
+ }
650
+
651
+ function finalizeReceipt(receipt) {
652
+ return {
653
+ ...receipt,
654
+ receiptHash: computeCanonicalSha256(receipt),
655
+ };
656
+ }
657
+
658
+ function rejectedReceipt(base, verification, measurement, reasons, status = 'rejected') {
659
+ return finalizeReceipt({
660
+ ...base,
661
+ verification,
662
+ measurement,
663
+ decision: {
664
+ accepted: false,
665
+ status,
666
+ reasons,
667
+ },
668
+ });
669
+ }
670
+
671
+ export async function evaluateBrowserRuntimeOptimizationCandidate(
672
+ contractInput,
673
+ candidateInput,
674
+ options = {}
675
+ ) {
676
+ const contract = validateRuntimeOptimizationContract(contractInput);
677
+ const candidate = validateRuntimeOptimizationCandidate(candidateInput, contract);
678
+ const runtimeInputs = materializeRuntimeOptimizationCandidate(contract, candidate);
679
+ const baselineInputs = cloneJsonValue(contract.baseline);
680
+ const runCommand = options.runCommand ?? runBrowserCommand;
681
+ if (typeof runCommand !== 'function') {
682
+ throw new Error('runtime optimization: options.runCommand must be a function.');
683
+ }
684
+ const base = baseReceipt(contract, candidate, runtimeInputs);
685
+ options.onEvent?.({
686
+ type: 'candidate:start',
687
+ contractHash: candidate.contractHash,
688
+ candidateId: candidate.candidateId,
689
+ candidateHash: base.candidateHash,
690
+ });
691
+
692
+ const baselineVerifyRequest = buildCommandRequest(contract, baselineInputs, 'verify');
693
+ const candidateVerifyRequest = buildCommandRequest(contract, runtimeInputs, 'verify');
694
+ const baselineVerifyRun = await runCommandSafely(
695
+ runCommand,
696
+ baselineVerifyRequest,
697
+ options,
698
+ 'verification:baseline'
699
+ );
700
+ if (!baselineVerifyRun.ok) {
701
+ return rejectedReceipt(base, {
702
+ passed: false,
703
+ baselineError: baselineVerifyRun.error,
704
+ candidateError: null,
705
+ }, { completedPairs: 0, pairs: [] }, ['baseline_verification_failed'], 'invalid');
706
+ }
707
+ try {
708
+ assertRunEnvelope(baselineVerifyRun.envelope, 'baseline verification', contract.model.modelId);
709
+ } catch (error) {
710
+ return rejectedReceipt(base, {
711
+ passed: false,
712
+ baselineError: summarizeError(error),
713
+ candidateError: null,
714
+ }, { completedPairs: 0, pairs: [] }, ['baseline_verification_failed'], 'invalid');
715
+ }
716
+
717
+ const candidateVerifyRun = await runCommandSafely(
718
+ runCommand,
719
+ candidateVerifyRequest,
720
+ options,
721
+ 'verification:candidate'
722
+ );
723
+ if (!candidateVerifyRun.ok) {
724
+ return rejectedReceipt(base, {
725
+ passed: false,
726
+ baseline: summarizeRun(baselineVerifyRun.envelope),
727
+ candidateError: candidateVerifyRun.error,
728
+ }, { completedPairs: 0, pairs: [] }, ['candidate_verification_failed']);
729
+ }
730
+ try {
731
+ assertRunEnvelope(candidateVerifyRun.envelope, 'candidate verification', contract.model.modelId);
732
+ } catch (error) {
733
+ return rejectedReceipt(base, {
734
+ passed: false,
735
+ baseline: summarizeRun(baselineVerifyRun.envelope),
736
+ candidateError: summarizeError(error),
737
+ }, { completedPairs: 0, pairs: [] }, ['candidate_verification_failed']);
738
+ }
739
+
740
+ const verification = compareVerificationRuns(
741
+ contract,
742
+ baselineVerifyRun.envelope,
743
+ candidateVerifyRun.envelope
744
+ );
745
+ if (!verification.passed) {
746
+ return rejectedReceipt(
747
+ base,
748
+ verification,
749
+ { completedPairs: 0, pairs: [] },
750
+ ['candidate_parity_failed']
751
+ );
752
+ }
753
+
754
+ const baselineBenchRequest = buildCommandRequest(contract, baselineInputs, 'bench');
755
+ const candidateBenchRequest = buildCommandRequest(contract, runtimeInputs, 'bench');
756
+ const pairs = [];
757
+ const baselineValues = [];
758
+ const candidateValues = [];
759
+ const improvements = [];
760
+ for (let pairIndex = 0; pairIndex < contract.measurement.pairCount; pairIndex += 1) {
761
+ const order = pairIndex % 2 === 0
762
+ ? ['baseline', 'candidate']
763
+ : ['candidate', 'baseline'];
764
+ const pairRuns = {};
765
+ for (const role of order) {
766
+ const request = role === 'baseline' ? baselineBenchRequest : candidateBenchRequest;
767
+ pairRuns[role] = await runCommandSafely(
768
+ runCommand,
769
+ request,
770
+ options,
771
+ `measurement:${pairIndex}:${role}`
772
+ );
773
+ }
774
+ const pair = { index: pairIndex, order, valid: false, baseline: null, candidate: null, improvementPercent: null };
775
+ try {
776
+ if (!pairRuns.baseline.ok || !pairRuns.candidate.ok) {
777
+ throw new Error('one or more paired commands failed');
778
+ }
779
+ assertRunEnvelope(pairRuns.baseline.envelope, 'baseline benchmark', contract.model.modelId);
780
+ assertRunEnvelope(pairRuns.candidate.envelope, 'candidate benchmark', contract.model.modelId);
781
+ const baselineValue = valueAtPath(pairRuns.baseline.envelope, contract.measurement.metricPath);
782
+ const candidateValue = valueAtPath(pairRuns.candidate.envelope, contract.measurement.metricPath);
783
+ const improvementPercent = computeImprovementPercent(
784
+ baselineValue,
785
+ candidateValue,
786
+ contract.measurement.direction
787
+ );
788
+ if (improvementPercent === null) {
789
+ throw new Error('paired metric values must be finite and greater than zero');
790
+ }
791
+ pair.valid = true;
792
+ pair.baseline = summarizeRun(pairRuns.baseline.envelope, contract.measurement.metricPath);
793
+ pair.candidate = summarizeRun(pairRuns.candidate.envelope, contract.measurement.metricPath);
794
+ pair.improvementPercent = improvementPercent;
795
+ baselineValues.push(baselineValue);
796
+ candidateValues.push(candidateValue);
797
+ improvements.push(improvementPercent);
798
+ } catch (error) {
799
+ pair.error = summarizeError(error);
800
+ if (pairRuns.baseline.ok) {
801
+ pair.baseline = summarizeRun(pairRuns.baseline.envelope, contract.measurement.metricPath);
802
+ } else {
803
+ pair.baselineError = pairRuns.baseline.error;
804
+ }
805
+ if (pairRuns.candidate.ok) {
806
+ pair.candidate = summarizeRun(pairRuns.candidate.envelope, contract.measurement.metricPath);
807
+ } else {
808
+ pair.candidateError = pairRuns.candidate.error;
809
+ }
810
+ }
811
+ pairs.push(pair);
812
+ options.onEvent?.({ type: 'measurement:pair', candidateId: candidate.candidateId, pair });
813
+ }
814
+
815
+ const baselineStats = sampleStats(baselineValues);
816
+ const candidateStats = sampleStats(candidateValues);
817
+ const improvementStats = sampleStats(improvements);
818
+ const measurement = {
819
+ metricPath: contract.measurement.metricPath,
820
+ direction: contract.measurement.direction,
821
+ requestedPairs: contract.measurement.pairCount,
822
+ completedPairs: improvements.length,
823
+ pairs,
824
+ baseline: baselineStats,
825
+ candidate: candidateStats,
826
+ improvementPercent: improvementStats,
827
+ };
828
+ const reasons = [];
829
+ if (improvements.length < contract.measurement.minValidPairs) {
830
+ reasons.push('insufficient_valid_pairs');
831
+ }
832
+ if (
833
+ improvementStats.median === null
834
+ || improvementStats.median < contract.measurement.minImprovementPercent
835
+ ) {
836
+ reasons.push('improvement_below_threshold');
837
+ }
838
+ if (contract.measurement.requirePositiveConfidence) {
839
+ if (
840
+ improvementStats.confidence95 === null
841
+ || improvementStats.confidence95.low < contract.measurement.minImprovementPercent
842
+ ) {
843
+ reasons.push('confidence_interval_below_threshold');
844
+ }
845
+ }
846
+ if (
847
+ contract.measurement.maxRelativeStdDevPercent !== null
848
+ && (
849
+ candidateStats.relativeStdDevPercent === null
850
+ || candidateStats.relativeStdDevPercent > contract.measurement.maxRelativeStdDevPercent
851
+ )
852
+ ) {
853
+ reasons.push('candidate_variance_above_threshold');
854
+ }
855
+ const receipt = finalizeReceipt({
856
+ ...base,
857
+ verification,
858
+ measurement,
859
+ decision: {
860
+ accepted: reasons.length === 0,
861
+ status: reasons.length === 0 ? 'accepted' : 'rejected',
862
+ reasons,
863
+ },
864
+ });
865
+ options.onEvent?.({
866
+ type: 'candidate:complete',
867
+ candidateId: candidate.candidateId,
868
+ candidateHash: receipt.candidateHash,
869
+ decision: cloneJsonValue(receipt.decision),
870
+ });
871
+ return receipt;
872
+ }