@seanhogg/builderforce-webcontainers 2026.10.12 → 2026.10.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/adaptive/controller.d.ts +39 -0
- package/dist/adaptive/controller.d.ts.map +1 -0
- package/dist/adaptive/controller.js +108 -0
- package/dist/adaptive/controller.js.map +1 -0
- package/dist/adaptive/costModel.d.ts +37 -0
- package/dist/adaptive/costModel.d.ts.map +1 -0
- package/dist/adaptive/costModel.js +102 -0
- package/dist/adaptive/costModel.js.map +1 -0
- package/dist/adaptive/planner.d.ts +22 -0
- package/dist/adaptive/planner.d.ts.map +1 -0
- package/dist/adaptive/planner.js +114 -0
- package/dist/adaptive/planner.js.map +1 -0
- package/dist/adaptive/types.d.ts +138 -0
- package/dist/adaptive/types.d.ts.map +1 -0
- package/dist/adaptive/types.js +12 -0
- package/dist/adaptive/types.js.map +1 -0
- package/dist/assets.js +2 -2
- package/dist/check/cachedFetch.js.map +1 -1
- package/dist/check/index.d.ts +6 -1
- package/dist/check/index.d.ts.map +1 -1
- package/dist/check/index.js +6 -1
- package/dist/check/index.js.map +1 -1
- package/dist/componentCompilers.d.ts.map +1 -1
- package/dist/componentCompilers.js.map +1 -1
- package/dist/esbuildWasm.js.map +1 -1
- package/dist/host.d.ts +11 -5
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +19 -8
- package/dist/host.js.map +1 -1
- package/dist/inference/backends/mock.d.ts +24 -0
- package/dist/inference/backends/mock.d.ts.map +1 -0
- package/dist/inference/backends/mock.js +55 -0
- package/dist/inference/backends/mock.js.map +1 -0
- package/dist/inference/backends/onnx.d.ts +33 -0
- package/dist/inference/backends/onnx.d.ts.map +1 -0
- package/dist/inference/backends/onnx.js +60 -0
- package/dist/inference/backends/onnx.js.map +1 -0
- package/dist/inference/backends/webllm.d.ts +27 -0
- package/dist/inference/backends/webllm.d.ts.map +1 -0
- package/dist/inference/backends/webllm.js +52 -0
- package/dist/inference/backends/webllm.js.map +1 -0
- package/dist/inference/backends/wllama.d.ts +29 -0
- package/dist/inference/backends/wllama.d.ts.map +1 -0
- package/dist/inference/backends/wllama.js +51 -0
- package/dist/inference/backends/wllama.js.map +1 -0
- package/dist/inference/capabilities.d.ts +20 -0
- package/dist/inference/capabilities.d.ts.map +1 -0
- package/dist/inference/capabilities.js +33 -0
- package/dist/inference/capabilities.js.map +1 -0
- package/dist/inference/client.d.ts +45 -0
- package/dist/inference/client.d.ts.map +1 -0
- package/dist/inference/client.js +201 -0
- package/dist/inference/client.js.map +1 -0
- package/dist/inference/modelCache.d.ts +71 -0
- package/dist/inference/modelCache.d.ts.map +1 -0
- package/dist/inference/modelCache.js +91 -0
- package/dist/inference/modelCache.js.map +1 -0
- package/dist/inference/protocol.d.ts +94 -0
- package/dist/inference/protocol.d.ts.map +1 -0
- package/dist/inference/protocol.js +19 -0
- package/dist/inference/protocol.js.map +1 -0
- package/dist/inference/tokenStream.d.ts +26 -0
- package/dist/inference/tokenStream.d.ts.map +1 -0
- package/dist/inference/tokenStream.js +79 -0
- package/dist/inference/tokenStream.js.map +1 -0
- package/dist/inference/types.d.ts +108 -0
- package/dist/inference/types.d.ts.map +1 -0
- package/dist/inference/types.js +8 -0
- package/dist/inference/types.js.map +1 -0
- package/dist/inference/worker.bundle.js +5 -0
- package/dist/inference/worker.d.ts +35 -0
- package/dist/inference/worker.d.ts.map +1 -0
- package/dist/inference/worker.js +157 -0
- package/dist/inference/worker.js.map +1 -0
- package/dist/inference/workerEntry.d.ts +15 -0
- package/dist/inference/workerEntry.d.ts.map +1 -0
- package/dist/inference/workerEntry.js +17 -0
- package/dist/inference/workerEntry.js.map +1 -0
- package/dist/kvcache/advanced.d.ts +39 -0
- package/dist/kvcache/advanced.d.ts.map +1 -0
- package/dist/kvcache/advanced.js +59 -0
- package/dist/kvcache/advanced.js.map +1 -0
- package/dist/kvcache/compatibility.d.ts +14 -0
- package/dist/kvcache/compatibility.d.ts.map +1 -0
- package/dist/kvcache/compatibility.js +44 -0
- package/dist/kvcache/compatibility.js.map +1 -0
- package/dist/kvcache/lru.d.ts +45 -0
- package/dist/kvcache/lru.d.ts.map +1 -0
- package/dist/kvcache/lru.js +91 -0
- package/dist/kvcache/lru.js.map +1 -0
- package/dist/kvcache/paged.d.ts +86 -0
- package/dist/kvcache/paged.d.ts.map +1 -0
- package/dist/kvcache/paged.js +238 -0
- package/dist/kvcache/paged.js.map +1 -0
- package/dist/kvcache/quant.d.ts +30 -0
- package/dist/kvcache/quant.d.ts.map +1 -0
- package/dist/kvcache/quant.js +73 -0
- package/dist/kvcache/quant.js.map +1 -0
- package/dist/kvcache/radix.d.ts +28 -0
- package/dist/kvcache/radix.d.ts.map +1 -0
- package/dist/kvcache/radix.js +58 -0
- package/dist/kvcache/radix.js.map +1 -0
- package/dist/kvcache/runtime.d.ts +11 -0
- package/dist/kvcache/runtime.d.ts.map +1 -0
- package/dist/kvcache/runtime.js +204 -0
- package/dist/kvcache/runtime.js.map +1 -0
- package/dist/kvcache/types.d.ts +151 -0
- package/dist/kvcache/types.d.ts.map +1 -0
- package/dist/kvcache/types.js +10 -0
- package/dist/kvcache/types.js.map +1 -0
- package/dist/learning/fingerprint.d.ts +17 -0
- package/dist/learning/fingerprint.d.ts.map +1 -0
- package/dist/learning/fingerprint.js +46 -0
- package/dist/learning/fingerprint.js.map +1 -0
- package/dist/learning/index.d.ts +5 -0
- package/dist/learning/index.d.ts.map +1 -0
- package/dist/learning/index.js +3 -0
- package/dist/learning/index.js.map +1 -0
- package/dist/learning/learner.d.ts +21 -0
- package/dist/learning/learner.d.ts.map +1 -0
- package/dist/learning/learner.js +272 -0
- package/dist/learning/learner.js.map +1 -0
- package/dist/learning/ledger.d.ts +26 -0
- package/dist/learning/ledger.d.ts.map +1 -0
- package/dist/learning/ledger.js +40 -0
- package/dist/learning/ledger.js.map +1 -0
- package/dist/learning/selection.d.ts +44 -0
- package/dist/learning/selection.d.ts.map +1 -0
- package/dist/learning/selection.js +111 -0
- package/dist/learning/selection.js.map +1 -0
- package/dist/learning/stats.d.ts +19 -0
- package/dist/learning/stats.d.ts.map +1 -0
- package/dist/learning/stats.js +31 -0
- package/dist/learning/stats.js.map +1 -0
- package/dist/learning/types.d.ts +133 -0
- package/dist/learning/types.d.ts.map +1 -0
- package/dist/learning/types.js +8 -0
- package/dist/learning/types.js.map +1 -0
- package/dist/memory/envProbe.d.ts +16 -0
- package/dist/memory/envProbe.d.ts.map +1 -0
- package/dist/memory/envProbe.js +28 -0
- package/dist/memory/envProbe.js.map +1 -0
- package/dist/memory/estimates.d.ts +46 -0
- package/dist/memory/estimates.d.ts.map +1 -0
- package/dist/memory/estimates.js +72 -0
- package/dist/memory/estimates.js.map +1 -0
- package/dist/memory/eviction.d.ts +42 -0
- package/dist/memory/eviction.d.ts.map +1 -0
- package/dist/memory/eviction.js +143 -0
- package/dist/memory/eviction.js.map +1 -0
- package/dist/memory/manager.d.ts +34 -0
- package/dist/memory/manager.d.ts.map +1 -0
- package/dist/memory/manager.js +346 -0
- package/dist/memory/manager.js.map +1 -0
- package/dist/memory/managerTypes.d.ts +49 -0
- package/dist/memory/managerTypes.d.ts.map +1 -0
- package/dist/memory/managerTypes.js +7 -0
- package/dist/memory/managerTypes.js.map +1 -0
- package/dist/memory/pressure.d.ts +56 -0
- package/dist/memory/pressure.d.ts.map +1 -0
- package/dist/memory/pressure.js +80 -0
- package/dist/memory/pressure.js.map +1 -0
- package/dist/memory/spillTier.d.ts +47 -0
- package/dist/memory/spillTier.d.ts.map +1 -0
- package/dist/memory/spillTier.js +128 -0
- package/dist/memory/spillTier.js.map +1 -0
- package/dist/memory/storageSink.d.ts +69 -0
- package/dist/memory/storageSink.d.ts.map +1 -0
- package/dist/memory/storageSink.js +221 -0
- package/dist/memory/storageSink.js.map +1 -0
- package/dist/memory/types.d.ts +202 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +15 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/memory/victims.d.ts +32 -0
- package/dist/memory/victims.d.ts.map +1 -0
- package/dist/memory/victims.js +74 -0
- package/dist/memory/victims.js.map +1 -0
- package/dist/memory/workspace.d.ts +34 -0
- package/dist/memory/workspace.d.ts.map +1 -0
- package/dist/memory/workspace.js +55 -0
- package/dist/memory/workspace.js.map +1 -0
- package/dist/memoryIntegration/index.d.ts +8 -0
- package/dist/memoryIntegration/index.d.ts.map +1 -0
- package/dist/memoryIntegration/index.js +4 -0
- package/dist/memoryIntegration/index.js.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts +43 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.js +285 -0
- package/dist/memoryIntegration/knowledgeMemory.js.map +1 -0
- package/dist/memoryIntegration/persistence.d.ts +15 -0
- package/dist/memoryIntegration/persistence.d.ts.map +1 -0
- package/dist/memoryIntegration/persistence.js +43 -0
- package/dist/memoryIntegration/persistence.js.map +1 -0
- package/dist/memoryIntegration/text.d.ts +57 -0
- package/dist/memoryIntegration/text.d.ts.map +1 -0
- package/dist/memoryIntegration/text.js +179 -0
- package/dist/memoryIntegration/text.js.map +1 -0
- package/dist/memoryIntegration/types.d.ts +131 -0
- package/dist/memoryIntegration/types.d.ts.map +1 -0
- package/dist/memoryIntegration/types.js +11 -0
- package/dist/memoryIntegration/types.js.map +1 -0
- package/dist/ml.d.ts +52 -0
- package/dist/ml.d.ts.map +1 -0
- package/dist/ml.js +33 -0
- package/dist/ml.js.map +1 -0
- package/dist/node/childRelay.d.ts +10 -0
- package/dist/node/childRelay.d.ts.map +1 -1
- package/dist/node/childRelay.js +12 -2
- package/dist/node/childRelay.js.map +1 -1
- package/dist/node/index.js.map +1 -1
- package/dist/node/portResponder.d.ts +2 -0
- package/dist/node/portResponder.d.ts.map +1 -1
- package/dist/node/portResponder.js +3 -3
- package/dist/node/portResponder.js.map +1 -1
- package/dist/node/processHost.d.ts +4 -1
- package/dist/node/processHost.d.ts.map +1 -1
- package/dist/node/processHost.js +19 -16
- package/dist/node/processHost.js.map +1 -1
- package/dist/node/protocol.d.ts +1 -0
- package/dist/node/protocol.d.ts.map +1 -1
- package/dist/node/responseWire.d.ts.map +1 -1
- package/dist/node/worker.js +127 -120
- package/dist/node/workerProcess.js +1 -1
- package/dist/node/workerProcess.js.map +1 -1
- package/dist/prefetch/admission.d.ts +41 -0
- package/dist/prefetch/admission.d.ts.map +1 -0
- package/dist/prefetch/admission.js +49 -0
- package/dist/prefetch/admission.js.map +1 -0
- package/dist/prefetch/loadRegistry.d.ts +56 -0
- package/dist/prefetch/loadRegistry.d.ts.map +1 -0
- package/dist/prefetch/loadRegistry.js +108 -0
- package/dist/prefetch/loadRegistry.js.map +1 -0
- package/dist/prefetch/predictor.d.ts +40 -0
- package/dist/prefetch/predictor.d.ts.map +1 -0
- package/dist/prefetch/predictor.js +178 -0
- package/dist/prefetch/predictor.js.map +1 -0
- package/dist/prefetch/prefetcher.d.ts +37 -0
- package/dist/prefetch/prefetcher.d.ts.map +1 -0
- package/dist/prefetch/prefetcher.js +339 -0
- package/dist/prefetch/prefetcher.js.map +1 -0
- package/dist/prefetch/residentStore.d.ts +49 -0
- package/dist/prefetch/residentStore.d.ts.map +1 -0
- package/dist/prefetch/residentStore.js +87 -0
- package/dist/prefetch/residentStore.js.map +1 -0
- package/dist/prefetch/selfTuner.d.ts +40 -0
- package/dist/prefetch/selfTuner.d.ts.map +1 -0
- package/dist/prefetch/selfTuner.js +78 -0
- package/dist/prefetch/selfTuner.js.map +1 -0
- package/dist/prefetch/stats.d.ts +52 -0
- package/dist/prefetch/stats.d.ts.map +1 -0
- package/dist/prefetch/stats.js +72 -0
- package/dist/prefetch/stats.js.map +1 -0
- package/dist/prefetch/streaming.d.ts +132 -0
- package/dist/prefetch/streaming.d.ts.map +1 -0
- package/dist/prefetch/streaming.js +141 -0
- package/dist/prefetch/streaming.js.map +1 -0
- package/dist/prefetch/targetIndex.d.ts +18 -0
- package/dist/prefetch/targetIndex.d.ts.map +1 -0
- package/dist/prefetch/targetIndex.js +32 -0
- package/dist/prefetch/targetIndex.js.map +1 -0
- package/dist/prefetch/types.d.ts +283 -0
- package/dist/prefetch/types.d.ts.map +1 -0
- package/dist/prefetch/types.js +19 -0
- package/dist/prefetch/types.js.map +1 -0
- package/dist/previewBridge.d.ts +6 -0
- package/dist/previewBridge.d.ts.map +1 -1
- package/dist/previewBridge.js +25 -5
- package/dist/previewBridge.js.map +1 -1
- package/dist/previewMessages.d.ts +4 -3
- package/dist/previewMessages.d.ts.map +1 -1
- package/dist/previewMessages.js +12 -6
- package/dist/previewMessages.js.map +1 -1
- package/dist/process-worker.js +127 -120
- package/dist/relay.html +1 -1
- package/dist/shared/clock.d.ts +7 -0
- package/dist/shared/clock.d.ts.map +1 -0
- package/dist/shared/clock.js +10 -0
- package/dist/shared/clock.js.map +1 -0
- package/dist/shared/hardwareProbe.d.ts +26 -0
- package/dist/shared/hardwareProbe.d.ts.map +1 -0
- package/dist/shared/hardwareProbe.js +62 -0
- package/dist/shared/hardwareProbe.js.map +1 -0
- package/dist/shared/idb.d.ts +41 -0
- package/dist/shared/idb.d.ts.map +1 -0
- package/dist/shared/idb.js +184 -0
- package/dist/shared/idb.js.map +1 -0
- package/dist/shared/tokens.d.ts +7 -0
- package/dist/shared/tokens.d.ts.map +1 -0
- package/dist/shared/tokens.js +9 -0
- package/dist/shared/tokens.js.map +1 -0
- package/dist/snapshotStore.d.ts +2 -1
- package/dist/snapshotStore.d.ts.map +1 -1
- package/dist/snapshotStore.js +27 -12
- package/dist/snapshotStore.js.map +1 -1
- package/dist/styleCompilers.d.ts.map +1 -1
- package/dist/styleCompilers.js.map +1 -1
- package/dist/transport.js.map +1 -1
- package/package.json +11 -5
- package/src/adaptive/controller.ts +151 -0
- package/src/adaptive/costModel.ts +131 -0
- package/src/adaptive/planner.ts +127 -0
- package/src/adaptive/types.ts +137 -0
- package/src/check/index.ts +6 -1
- package/src/host.ts +33 -11
- package/src/inference/backends/mock.ts +77 -0
- package/src/inference/backends/onnx.ts +91 -0
- package/src/inference/backends/webllm.ts +79 -0
- package/src/inference/backends/wllama.ts +79 -0
- package/src/inference/capabilities.ts +46 -0
- package/src/inference/client.ts +276 -0
- package/src/inference/modelCache.ts +145 -0
- package/src/inference/protocol.ts +119 -0
- package/src/inference/tokenStream.ts +92 -0
- package/src/inference/types.ts +115 -0
- package/src/inference/worker.ts +191 -0
- package/src/inference/workerEntry.ts +18 -0
- package/src/kvcache/advanced.ts +86 -0
- package/src/kvcache/compatibility.ts +51 -0
- package/src/kvcache/lru.ts +100 -0
- package/src/kvcache/paged.ts +270 -0
- package/src/kvcache/quant.ts +88 -0
- package/src/kvcache/radix.ts +68 -0
- package/src/kvcache/runtime.ts +239 -0
- package/src/kvcache/types.ts +167 -0
- package/src/learning/fingerprint.ts +50 -0
- package/src/learning/index.ts +15 -0
- package/src/learning/learner.ts +304 -0
- package/src/learning/ledger.ts +54 -0
- package/src/learning/selection.ts +137 -0
- package/src/learning/stats.ts +39 -0
- package/src/learning/types.ts +142 -0
- package/src/memory/envProbe.ts +30 -0
- package/src/memory/estimates.ts +98 -0
- package/src/memory/eviction.ts +145 -0
- package/src/memory/manager.ts +380 -0
- package/src/memory/managerTypes.ts +61 -0
- package/src/memory/pressure.ts +120 -0
- package/src/memory/spillTier.ts +159 -0
- package/src/memory/storageSink.ts +289 -0
- package/src/memory/types.ts +217 -0
- package/src/memory/victims.ts +80 -0
- package/src/memory/workspace.ts +74 -0
- package/src/memoryIntegration/index.ts +7 -0
- package/src/memoryIntegration/knowledgeMemory.ts +339 -0
- package/src/memoryIntegration/persistence.ts +54 -0
- package/src/memoryIntegration/text.ts +185 -0
- package/src/memoryIntegration/types.ts +145 -0
- package/src/ml.ts +180 -0
- package/src/node/childRelay.ts +16 -3
- package/src/node/portResponder.ts +5 -3
- package/src/node/processHost.ts +16 -15
- package/src/node/protocol.ts +1 -1
- package/src/node/workerProcess.ts +1 -1
- package/src/prefetch/admission.ts +74 -0
- package/src/prefetch/loadRegistry.ts +156 -0
- package/src/prefetch/predictor.ts +199 -0
- package/src/prefetch/prefetcher.ts +369 -0
- package/src/prefetch/residentStore.ts +121 -0
- package/src/prefetch/selfTuner.ts +110 -0
- package/src/prefetch/stats.ts +120 -0
- package/src/prefetch/streaming.ts +231 -0
- package/src/prefetch/targetIndex.ts +40 -0
- package/src/prefetch/types.ts +307 -0
- package/src/previewBridge.ts +25 -5
- package/src/previewMessages.ts +11 -6
- package/src/shared/clock.ts +9 -0
- package/src/shared/hardwareProbe.ts +68 -0
- package/src/shared/idb.ts +203 -0
- package/src/shared/tokens.ts +8 -0
- package/src/snapshotStore.ts +25 -10
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — quantization-aware execution.
|
|
3
|
+
*
|
|
4
|
+
* Picks the runtime format from `ModelRef.quantization` (`q4`|`q5`|`q8`|`fp16`)
|
|
5
|
+
* and avoids converting unless `preconvertQuant` is opted in. Conversion is
|
|
6
|
+
* never implicit: the native format is used unless a flag + target justify it.
|
|
7
|
+
*/
|
|
8
|
+
import { bytesPerParam } from '../memory/estimates.js';
|
|
9
|
+
/** Relative quality vs fp16, used by the default scorer. */
|
|
10
|
+
const QUALITY = {
|
|
11
|
+
fp16: 1,
|
|
12
|
+
q8: 0.99,
|
|
13
|
+
q5: 0.97,
|
|
14
|
+
q4: 0.94,
|
|
15
|
+
};
|
|
16
|
+
export function nativeQuality(q) {
|
|
17
|
+
return QUALITY[q];
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Decide the runtime quantization. Conversion is skipped unless
|
|
21
|
+
* `preconvert` is true AND the target is a strictly smaller format
|
|
22
|
+
* (justified by a bandwidth saving).
|
|
23
|
+
*/
|
|
24
|
+
export function planQuantization(input) {
|
|
25
|
+
const native = input.native;
|
|
26
|
+
const target = input.preconvertTo ?? 'q4';
|
|
27
|
+
const justified = input.preconvert && bytesPerParam(target) < bytesPerParam(native) && backendSupports(input.backend, target);
|
|
28
|
+
if (!justified) {
|
|
29
|
+
return {
|
|
30
|
+
native,
|
|
31
|
+
runtime: native,
|
|
32
|
+
converted: false,
|
|
33
|
+
conversionCost: 0,
|
|
34
|
+
bytesPerParam: bytesPerParam(native),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
native,
|
|
39
|
+
runtime: target,
|
|
40
|
+
converted: true,
|
|
41
|
+
conversionCost: bytesPerParam(native) - bytesPerParam(target),
|
|
42
|
+
bytesPerParam: bytesPerParam(target),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export function backendSupports(backend, q) {
|
|
46
|
+
if (backend === 'webllm')
|
|
47
|
+
return q === 'fp16' || q === 'q4';
|
|
48
|
+
if (backend === 'wllama')
|
|
49
|
+
return q !== 'fp16';
|
|
50
|
+
return true;
|
|
51
|
+
}
|
|
52
|
+
export function bandwidthBytes(params, plan, tokens, kvDim) {
|
|
53
|
+
const weightBytes = params * plan.bytesPerParam;
|
|
54
|
+
const kvBytes = tokens * kvDim * plan.bytesPerParam;
|
|
55
|
+
return weightBytes + kvBytes;
|
|
56
|
+
}
|
|
57
|
+
export const DEFAULT_QUALITY_TOLERANCE = 0.08;
|
|
58
|
+
export function defaultQualityScorer(input) {
|
|
59
|
+
const q = nativeQuality(input.quantization) / nativeQuality(input.baselineQuantization);
|
|
60
|
+
if (!input.baselineOutput)
|
|
61
|
+
return q;
|
|
62
|
+
const a = input.output;
|
|
63
|
+
const b = input.baselineOutput;
|
|
64
|
+
const n = Math.max(a.length, b.length, 1);
|
|
65
|
+
let same = 0;
|
|
66
|
+
for (let i = 0; i < Math.min(a.length, b.length); i++) {
|
|
67
|
+
if (a.charCodeAt(i) === b.charCodeAt(i))
|
|
68
|
+
same++;
|
|
69
|
+
}
|
|
70
|
+
const overlap = same / n;
|
|
71
|
+
return Math.min(1, q * 0.5 + overlap * 0.5);
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=quant.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quant.js","sourceRoot":"","sources":["../../src/kvcache/quant.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAGvD,4DAA4D;AAC5D,MAAM,OAAO,GAAmC;IAC9C,IAAI,EAAE,CAAC;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;CACT,CAAC;AAEF,MAAM,UAAU,aAAa,CAAC,CAAiB;IAC7C,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAKhC;IACC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC;IAC1C,MAAM,SAAS,GACb,KAAK,CAAC,UAAU,IAAI,aAAa,CAAC,MAAM,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC9G,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO;YACL,MAAM;YACN,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,KAAK;YAChB,cAAc,EAAE,CAAC;YACjB,aAAa,EAAE,aAAa,CAAC,MAAM,CAAC;SACrC,CAAC;IACJ,CAAC;IACD,OAAO;QACL,MAAM;QACN,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI;QACf,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC;QAC7D,aAAa,EAAE,aAAa,CAAC,MAAM,CAAC;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAsB,EAAE,CAAiB;IACvE,IAAI,OAAO,KAAK,QAAQ;QAAE,OAAO,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,IAAI,CAAC;IAC5D,IAAI,OAAO,KAAK,QAAQ;QAAE,OAAO,CAAC,KAAK,MAAM,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,IAAe,EAAE,MAAc,EAAE,KAAa;IAC3F,MAAM,WAAW,GAAG,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;IACpD,OAAO,WAAW,GAAG,OAAO,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC;AAE9C,MAAM,UAAU,oBAAoB,CAAC,KAKpC;IACC,MAAM,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxF,IAAI,CAAC,KAAK,CAAC,cAAc;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,CAAC,GAAG,KAAK,CAAC,cAAc,CAAC;IAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,IAAI,EAAE,CAAC;IAClD,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — radix-tree prefix index over token sequences.
|
|
3
|
+
*
|
|
4
|
+
* Each node is keyed by a single token id. A match walks the tree while
|
|
5
|
+
* tokens equal the query; the first mismatch stops. This is EXACT prefix
|
|
6
|
+
* reuse only — never semantic / approximate.
|
|
7
|
+
*
|
|
8
|
+
* Public primitive (exported from the kvcache barrel). The paged KV cache no
|
|
9
|
+
* longer stores per-sequence trees: its hash-chained page ids already index
|
|
10
|
+
* every shared prefix, inside the page LRU's byte budget.
|
|
11
|
+
*/
|
|
12
|
+
export interface RadixMatch<V> {
|
|
13
|
+
length: number;
|
|
14
|
+
value: V;
|
|
15
|
+
}
|
|
16
|
+
export declare class TokenRadix<V> {
|
|
17
|
+
private readonly root;
|
|
18
|
+
private n;
|
|
19
|
+
get size(): number;
|
|
20
|
+
insert(tokens: readonly number[], value: V): void;
|
|
21
|
+
/**
|
|
22
|
+
* Longest stored prefix that is an exact prefix of `tokens`.
|
|
23
|
+
* Returns undefined when no stored sequence shares the first token.
|
|
24
|
+
*/
|
|
25
|
+
longestPrefix(tokens: readonly number[]): RadixMatch<V> | undefined;
|
|
26
|
+
clear(): void;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=radix.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"radix.d.ts","sourceRoot":"","sources":["../../src/kvcache/radix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,CAAC,CAAC;CACV;AAOD,qBAAa,UAAU,CAAC,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAyC;IAC9D,OAAO,CAAC,CAAC,CAAK;IAEd,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAYhD;IAED;;;OAGG;IACH,aAAa,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,SAAS,CAYlE;IAED,KAAK,IAAI,IAAI,CAIZ;CACF"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — radix-tree prefix index over token sequences.
|
|
3
|
+
*
|
|
4
|
+
* Each node is keyed by a single token id. A match walks the tree while
|
|
5
|
+
* tokens equal the query; the first mismatch stops. This is EXACT prefix
|
|
6
|
+
* reuse only — never semantic / approximate.
|
|
7
|
+
*
|
|
8
|
+
* Public primitive (exported from the kvcache barrel). The paged KV cache no
|
|
9
|
+
* longer stores per-sequence trees: its hash-chained page ids already index
|
|
10
|
+
* every shared prefix, inside the page LRU's byte budget.
|
|
11
|
+
*/
|
|
12
|
+
export class TokenRadix {
|
|
13
|
+
root = { children: new Map() };
|
|
14
|
+
n = 0;
|
|
15
|
+
get size() {
|
|
16
|
+
return this.n;
|
|
17
|
+
}
|
|
18
|
+
insert(tokens, value) {
|
|
19
|
+
let node = this.root;
|
|
20
|
+
for (const t of tokens) {
|
|
21
|
+
let child = node.children.get(t);
|
|
22
|
+
if (!child) {
|
|
23
|
+
child = { children: new Map() };
|
|
24
|
+
node.children.set(t, child);
|
|
25
|
+
}
|
|
26
|
+
node = child;
|
|
27
|
+
}
|
|
28
|
+
if (node.value === undefined)
|
|
29
|
+
this.n++;
|
|
30
|
+
node.value = value;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Longest stored prefix that is an exact prefix of `tokens`.
|
|
34
|
+
* Returns undefined when no stored sequence shares the first token.
|
|
35
|
+
*/
|
|
36
|
+
longestPrefix(tokens) {
|
|
37
|
+
let node = this.root;
|
|
38
|
+
let last;
|
|
39
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
40
|
+
const t = tokens[i];
|
|
41
|
+
if (t === undefined)
|
|
42
|
+
break;
|
|
43
|
+
const child = node.children.get(t);
|
|
44
|
+
if (!child)
|
|
45
|
+
break;
|
|
46
|
+
node = child;
|
|
47
|
+
if (node.value !== undefined)
|
|
48
|
+
last = { length: i + 1, value: node.value };
|
|
49
|
+
}
|
|
50
|
+
return last;
|
|
51
|
+
}
|
|
52
|
+
clear() {
|
|
53
|
+
this.root.children.clear();
|
|
54
|
+
this.root.value = undefined;
|
|
55
|
+
this.n = 0;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=radix.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"radix.js","sourceRoot":"","sources":["../../src/kvcache/radix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAYH,MAAM,OAAO,UAAU;IACJ,IAAI,GAAiB,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IACtD,CAAC,GAAG,CAAC,CAAC;IAEd,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,MAAyB,EAAE,KAAQ;QACxC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACrB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACjC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,KAAK,GAAG,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;gBAChC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC9B,CAAC;YACD,IAAI,GAAG,KAAK,CAAC;QACf,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;YAAE,IAAI,CAAC,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAyB;QACrC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACrB,IAAI,IAA+B,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,CAAC,KAAK,SAAS;gBAAE,MAAM;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,KAAK;gBAAE,MAAM;YAClB,IAAI,GAAG,KAAK,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;gBAAE,IAAI,GAAG,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC5E,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACb,CAAC;CACF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — KV-cache runtime.
|
|
3
|
+
*
|
|
4
|
+
* Orchestrates paged exact-prefix reuse, quantization-aware execution, and
|
|
5
|
+
* gated advanced decode. The execution engine is mock/injected: generate()
|
|
6
|
+
* simulates prefill+decode cost so tests can assert reuse is cheaper than
|
|
7
|
+
* recompute without real GGUF/ONNX weights.
|
|
8
|
+
*/
|
|
9
|
+
import type { KvCacheRuntime, KvCacheRuntimeOptions } from './types.js';
|
|
10
|
+
export declare function createKvCacheRuntime(options?: KvCacheRuntimeOptions): KvCacheRuntime;
|
|
11
|
+
//# sourceMappingURL=runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/kvcache/runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAWH,OAAO,KAAK,EAGV,cAAc,EACd,qBAAqB,EAOtB,MAAM,YAAY,CAAC;AAmCpB,wBAAgB,oBAAoB,CAAC,OAAO,GAAE,qBAA0B,GAAG,cAAc,CA8KxF"}
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — KV-cache runtime.
|
|
3
|
+
*
|
|
4
|
+
* Orchestrates paged exact-prefix reuse, quantization-aware execution, and
|
|
5
|
+
* gated advanced decode. The execution engine is mock/injected: generate()
|
|
6
|
+
* simulates prefill+decode cost so tests can assert reuse is cheaper than
|
|
7
|
+
* recompute without real GGUF/ONNX weights.
|
|
8
|
+
*/
|
|
9
|
+
import { canActivationSparsity, canContinuousBatch, canSpeculative, decodeCost, isEnabled } from './advanced.js';
|
|
10
|
+
import { encodeCompatibilityKey } from './compatibility.js';
|
|
11
|
+
import { PagedKvCache } from './paged.js';
|
|
12
|
+
import { bandwidthBytes, DEFAULT_QUALITY_TOLERANCE, defaultQualityScorer, planQuantization, } from './quant.js';
|
|
13
|
+
const DEFAULT_COSTS = {
|
|
14
|
+
cacheHitPerToken: 0.05,
|
|
15
|
+
computePerToken: 1,
|
|
16
|
+
conversionPerToken: 0.2,
|
|
17
|
+
draftPerToken: 0.15,
|
|
18
|
+
verifyPerToken: 0.25,
|
|
19
|
+
};
|
|
20
|
+
const DEFAULT_MAX_SESSIONS = 1024;
|
|
21
|
+
const DEFAULT_CAPS = {
|
|
22
|
+
speculativeDecoding: false,
|
|
23
|
+
continuousBatching: false,
|
|
24
|
+
activationSparsity: false,
|
|
25
|
+
hasDraftModel: false,
|
|
26
|
+
sparsityFraction: 0,
|
|
27
|
+
architecture: 'decoder-transformer',
|
|
28
|
+
};
|
|
29
|
+
function partsOf(req) {
|
|
30
|
+
return {
|
|
31
|
+
modelId: req.model.id,
|
|
32
|
+
weightsHash: req.model.weightsHash ?? req.model.id,
|
|
33
|
+
tokenizerId: req.tokenizerId,
|
|
34
|
+
positionalConfig: req.positionalConfig,
|
|
35
|
+
attentionConfig: req.attentionConfig,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
function mockOutput(tokens) {
|
|
39
|
+
return tokens.map((t) => String.fromCharCode(97 + (Math.abs(t) % 26))).join('');
|
|
40
|
+
}
|
|
41
|
+
export function createKvCacheRuntime(options = {}) {
|
|
42
|
+
const pageTokens = options.pageTokens ?? 16;
|
|
43
|
+
const layers = options.layers ?? 12;
|
|
44
|
+
const kvDim = options.kvDim ?? 64;
|
|
45
|
+
const bytesPerToken = layers * kvDim * 2; // K+V, fp-ish units
|
|
46
|
+
const maxBytes = options.maxBytes ?? 1 * 1024 * 1024;
|
|
47
|
+
const cache = new PagedKvCache(maxBytes, pageTokens, bytesPerToken);
|
|
48
|
+
const flags = {
|
|
49
|
+
prefixCache: options.flags?.prefixCache !== false,
|
|
50
|
+
preconvertQuant: options.flags?.preconvertQuant === true,
|
|
51
|
+
speculativeDecoding: options.flags?.speculativeDecoding === true,
|
|
52
|
+
continuousBatching: options.flags?.continuousBatching === true,
|
|
53
|
+
activationSparsity: options.flags?.activationSparsity === true,
|
|
54
|
+
};
|
|
55
|
+
const caps = { ...DEFAULT_CAPS, ...options.capabilities };
|
|
56
|
+
const costs = { ...DEFAULT_COSTS, ...options.costs };
|
|
57
|
+
const scorer = options.qualityScorer ?? defaultQualityScorer;
|
|
58
|
+
const qualityBaseline = options.qualityBaseline ?? 'fp16';
|
|
59
|
+
const qualityTolerance = options.qualityTolerance ?? DEFAULT_QUALITY_TOLERANCE;
|
|
60
|
+
const draftLen = options.draftLen ?? 4;
|
|
61
|
+
const draftAccuracy = options.draftAccuracy ?? 0.8;
|
|
62
|
+
const preconvertTo = options.preconvertTo ?? 'q4';
|
|
63
|
+
const maxSessions = Math.max(1, options.maxSessions ?? DEFAULT_MAX_SESSIONS);
|
|
64
|
+
// Recently seen session ids, least recently seen first (Map insertion order).
|
|
65
|
+
const sessions = new Map();
|
|
66
|
+
function noteSession(sessionId) {
|
|
67
|
+
sessions.delete(sessionId);
|
|
68
|
+
sessions.set(sessionId, true);
|
|
69
|
+
for (const oldest of sessions.keys()) {
|
|
70
|
+
if (sessions.size <= maxSessions)
|
|
71
|
+
break;
|
|
72
|
+
sessions.delete(oldest);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
function generateOne(request, batchShare = 1) {
|
|
76
|
+
const compat = partsOf(request);
|
|
77
|
+
const encoded = encodeCompatibilityKey(compat);
|
|
78
|
+
if (request.sessionId)
|
|
79
|
+
noteSession(request.sessionId);
|
|
80
|
+
const quantPlan = planQuantization({
|
|
81
|
+
native: request.model.quantization,
|
|
82
|
+
backend: 'mock',
|
|
83
|
+
preconvert: flags.preconvertQuant,
|
|
84
|
+
preconvertTo,
|
|
85
|
+
});
|
|
86
|
+
const applied = [];
|
|
87
|
+
let cachedTokens = 0;
|
|
88
|
+
let servedFromCache = false;
|
|
89
|
+
let evictedPages = 0;
|
|
90
|
+
if (flags.prefixCache) {
|
|
91
|
+
applied.push('prefixCache');
|
|
92
|
+
const hit = cache.lookup(compat, request.tokens);
|
|
93
|
+
if (hit.compatKey === encoded.key && hit.matched > 0) {
|
|
94
|
+
cachedTokens = hit.matched;
|
|
95
|
+
servedFromCache = true;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
const computeTokens = Math.max(0, request.tokens.length - cachedTokens) + (request.maxNewTokens ?? 0);
|
|
99
|
+
const speculative = flags.speculativeDecoding && canSpeculative(caps);
|
|
100
|
+
const sparsity = flags.activationSparsity && canActivationSparsity(caps);
|
|
101
|
+
const decode = decodeCost({
|
|
102
|
+
tokens: computeTokens,
|
|
103
|
+
computePerToken: costs.computePerToken,
|
|
104
|
+
speculative,
|
|
105
|
+
sparsity,
|
|
106
|
+
draftLen,
|
|
107
|
+
draftAccuracy,
|
|
108
|
+
draftPerToken: costs.draftPerToken,
|
|
109
|
+
verifyPerToken: costs.verifyPerToken,
|
|
110
|
+
sparsityFraction: caps.sparsityFraction,
|
|
111
|
+
});
|
|
112
|
+
for (const f of decode.applied)
|
|
113
|
+
if (!applied.includes(f))
|
|
114
|
+
applied.push(f);
|
|
115
|
+
const hitCost = cachedTokens * costs.cacheHitPerToken;
|
|
116
|
+
const convertCost = quantPlan.converted ? request.tokens.length * costs.conversionPerToken : 0;
|
|
117
|
+
if (quantPlan.converted)
|
|
118
|
+
applied.push('preconvertQuant');
|
|
119
|
+
const simulatedMs = (hitCost + decode.cost + convertCost) * batchShare;
|
|
120
|
+
const params = (request.model.paramsB ?? 1.1) * 1e9;
|
|
121
|
+
const bw = bandwidthBytes(params, quantPlan, request.tokens.length, kvDim);
|
|
122
|
+
const output = request.output ?? mockOutput(request.tokens);
|
|
123
|
+
const baselineOutput = request.baselineOutput ?? output;
|
|
124
|
+
const qualityScore = scorer({
|
|
125
|
+
output,
|
|
126
|
+
baselineOutput,
|
|
127
|
+
quantization: quantPlan.runtime,
|
|
128
|
+
baselineQuantization: qualityBaseline,
|
|
129
|
+
});
|
|
130
|
+
const withinTolerance = qualityScore >= 1 - qualityTolerance;
|
|
131
|
+
if (flags.prefixCache) {
|
|
132
|
+
const put = cache.put(compat, request.tokens);
|
|
133
|
+
evictedPages = put.evicted;
|
|
134
|
+
}
|
|
135
|
+
return {
|
|
136
|
+
output,
|
|
137
|
+
compatibilityKey: encoded.key,
|
|
138
|
+
cachedTokens,
|
|
139
|
+
computeTokens,
|
|
140
|
+
simulatedMs,
|
|
141
|
+
bandwidthBytes: bw,
|
|
142
|
+
qualityScore,
|
|
143
|
+
withinTolerance,
|
|
144
|
+
servedFromCache,
|
|
145
|
+
optimizationsApplied: applied,
|
|
146
|
+
evictedPages,
|
|
147
|
+
quantPlan,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
async generate(request) {
|
|
152
|
+
return generateOne(request, 1);
|
|
153
|
+
},
|
|
154
|
+
async generateBatch(requests) {
|
|
155
|
+
if (requests.length === 0)
|
|
156
|
+
return [];
|
|
157
|
+
const continuous = flags.continuousBatching && canContinuousBatch(caps) && requests.length > 1;
|
|
158
|
+
if (!continuous) {
|
|
159
|
+
const out = [];
|
|
160
|
+
for (const r of requests)
|
|
161
|
+
out.push(generateOne(r, 1));
|
|
162
|
+
return out;
|
|
163
|
+
}
|
|
164
|
+
// Shared decode step: each request pays max(seq) / n of the batch cost
|
|
165
|
+
// so the sum is demonstrably cheaper than sequential generate().
|
|
166
|
+
const maxTokens = requests.reduce((m, r) => Math.max(m, r.tokens.length + (r.maxNewTokens ?? 0)), 0);
|
|
167
|
+
const sequential = requests.reduce((s, r) => s + (r.tokens.length + (r.maxNewTokens ?? 0)), 0);
|
|
168
|
+
const share = sequential === 0 ? 1 : maxTokens / sequential;
|
|
169
|
+
return requests.map((r) => {
|
|
170
|
+
const result = generateOne(r, share);
|
|
171
|
+
if (!result.optimizationsApplied.includes('continuousBatching')) {
|
|
172
|
+
result.optimizationsApplied.push('continuousBatching');
|
|
173
|
+
}
|
|
174
|
+
return result;
|
|
175
|
+
});
|
|
176
|
+
},
|
|
177
|
+
lookup(request) {
|
|
178
|
+
const compat = partsOf(request);
|
|
179
|
+
const hit = cache.lookup(compat, request.tokens);
|
|
180
|
+
return { matched: hit.matched, key: hit.compatKey };
|
|
181
|
+
},
|
|
182
|
+
setEnabled(flag, enabled) {
|
|
183
|
+
flags[flag] = enabled;
|
|
184
|
+
},
|
|
185
|
+
isEnabled(flag) {
|
|
186
|
+
return isEnabled(flags, flag);
|
|
187
|
+
},
|
|
188
|
+
stats() {
|
|
189
|
+
return {
|
|
190
|
+
hits: cache.hits,
|
|
191
|
+
misses: cache.misses,
|
|
192
|
+
pages: cache.pages,
|
|
193
|
+
bytes: cache.bytes,
|
|
194
|
+
evictions: cache.evictions,
|
|
195
|
+
sessions: sessions.size,
|
|
196
|
+
};
|
|
197
|
+
},
|
|
198
|
+
clear() {
|
|
199
|
+
cache.clear();
|
|
200
|
+
sessions.clear();
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
//# sourceMappingURL=runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../src/kvcache/runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACjH,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EACL,cAAc,EACd,yBAAyB,EACzB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,YAAY,CAAC;AAcpB,MAAM,aAAa,GAAgB;IACjC,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,CAAC;IAClB,kBAAkB,EAAE,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;CACrB,CAAC;AAEF,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,MAAM,YAAY,GAAyB;IACzC,mBAAmB,EAAE,KAAK;IAC1B,kBAAkB,EAAE,KAAK;IACzB,kBAAkB,EAAE,KAAK;IACzB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,CAAC;IACnB,YAAY,EAAE,qBAAqB;CACpC,CAAC;AAEF,SAAS,OAAO,CAAC,GAAsB;IACrC,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE;QACrB,WAAW,EAAE,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE;QAClD,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,eAAe,EAAE,GAAG,CAAC,eAAe;KACrC,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,MAAyB;IAC3C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAO,GAA0B,EAAE;IACtE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,aAAa,GAAG,MAAM,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,oBAAoB;IAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IACrD,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,QAAQ,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;IACpE,MAAM,KAAK,GAAsC;QAC/C,WAAW,EAAE,OAAO,CAAC,KAAK,EAAE,WAAW,KAAK,KAAK;QACjD,eAAe,EAAE,OAAO,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI;QACxD,mBAAmB,EAAE,OAAO,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI;QAChE,kBAAkB,EAAE,OAAO,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI;QAC9D,kBAAkB,EAAE,OAAO,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI;KAC/D,CAAC;IACF,MAAM,IAAI,GAAyB,EAAE,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAChF,MAAM,KAAK,GAAgB,EAAE,GAAG,aAAa,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAClE,MAAM,MAAM,GAAkB,OAAO,CAAC,aAAa,IAAI,oBAAoB,CAAC;IAC5E,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC;IAC1D,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,yBAAyB,CAAC;IAC/E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;IACvC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC;IACnD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IAClD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,oBAAoB,CAAC,CAAC;IAC7E,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;IAEzC,SAAS,WAAW,CAAC,SAAiB;QACpC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC9B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,QAAQ,CAAC,IAAI,IAAI,WAAW;gBAAE,MAAM;YACxC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,SAAS,WAAW,CAAC,OAA0B,EAAE,UAAU,GAAG,CAAC;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,OAAO,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,SAAS;YAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,gBAAgB,CAAC;YACjC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,YAAY;YAClC,OAAO,EAAE,MAAM;YACf,UAAU,EAAE,KAAK,CAAC,eAAe;YACjC,YAAY;SACb,CAAC,CAAC;QAEH,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,eAAe,GAAG,KAAK,CAAC;QAC5B,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,GAAG,CAAC,SAAS,KAAK,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACrD,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC3B,eAAe,GAAG,IAAI,CAAC;YACzB,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;QACtG,MAAM,WAAW,GAAG,KAAK,CAAC,mBAAmB,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,kBAAkB,IAAI,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,MAAM,EAAE,aAAa;YACrB,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,aAAa;YACb,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACxC,CAAC,CAAC;QACH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO;YAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE1E,MAAM,OAAO,GAAG,YAAY,GAAG,KAAK,CAAC,gBAAgB,CAAC;QACtD,MAAM,WAAW,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/F,IAAI,SAAS,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAEzD,MAAM,WAAW,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,IAAI,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;QACvE,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC;QACpD,MAAM,EAAE,GAAG,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAE3E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC;QACxD,MAAM,YAAY,GAAG,MAAM,CAAC;YAC1B,MAAM;YACN,cAAc;YACd,YAAY,EAAE,SAAS,CAAC,OAAO;YAC/B,oBAAoB,EAAE,eAAe;SACtC,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,YAAY,IAAI,CAAC,GAAG,gBAAgB,CAAC;QAE7D,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAC9C,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC;QAC7B,CAAC;QAED,OAAO;YACL,MAAM;YACN,gBAAgB,EAAE,OAAO,CAAC,GAAG;YAC7B,YAAY;YACZ,aAAa;YACb,WAAW;YACX,cAAc,EAAE,EAAE;YAClB,YAAY;YACZ,eAAe;YACf,eAAe;YACf,oBAAoB,EAAE,OAAO;YAC7B,YAAY;YACZ,SAAS;SACV,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK,CAAC,QAAQ,CAAC,OAAO;YACpB,OAAO,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjC,CAAC;QAED,KAAK,CAAC,aAAa,CAAC,QAAQ;YAC1B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,KAAK,CAAC,kBAAkB,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/F,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAuB,EAAE,CAAC;gBACnC,KAAK,MAAM,CAAC,IAAI,QAAQ;oBAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACtD,OAAO,GAAG,CAAC;YACb,CAAC;YACD,uEAAuE;YACvE,iEAAiE;YACjE,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACrG,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/F,MAAM,KAAK,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC;YAC5D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxB,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;oBAChE,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;gBACzD,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,OAAO;YACZ,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACjD,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC;QACtD,CAAC;QAED,UAAU,CAAC,IAAI,EAAE,OAAO;YACtB,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;QACxB,CAAC;QAED,SAAS,CAAC,IAAI;YACZ,OAAO,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAChC,CAAC;QAED,KAAK;YACH,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,QAAQ,EAAE,QAAQ,CAAC,IAAI;aACxB,CAAC;QACJ,CAAC;QAED,KAAK;YACH,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — Advanced Model & KV Cache Optimization: shared types.
|
|
3
|
+
*
|
|
4
|
+
* Browser-compatible only: pure TypeScript, no native code, no dependencies.
|
|
5
|
+
* Decoupled from `src/inference/` (which only knows HOW to run) and from
|
|
6
|
+
* `src/adaptive/` (which only decides WHERE/HOW). This module owns KV reuse,
|
|
7
|
+
* quantization-aware execution, and gated advanced decode paths.
|
|
8
|
+
*/
|
|
9
|
+
/** Quantization formats Phase 5 routes on. `fp16` is local to this module. */
|
|
10
|
+
export type KvQuantization = 'q4' | 'q5' | 'q8' | 'fp16';
|
|
11
|
+
/** Backend kinds a quant plan may target. Matches Phase 1 `BackendKind`. */
|
|
12
|
+
export type KvBackendKind = 'onnx' | 'wllama' | 'webllm' | 'mock';
|
|
13
|
+
/**
|
|
14
|
+
* Everything that MUST match before any KV state is reused.
|
|
15
|
+
* Semantic / approximate retrieval is intentionally NOT a match criterion.
|
|
16
|
+
*/
|
|
17
|
+
export interface CompatibilityParts {
|
|
18
|
+
/** Model identity (id). */
|
|
19
|
+
modelId: string;
|
|
20
|
+
/** Weight-file / checksum identity. Different weights ⇒ no reuse. */
|
|
21
|
+
weightsHash: string;
|
|
22
|
+
/** Tokenizer identity (vocab + special-token layout). */
|
|
23
|
+
tokenizerId: string;
|
|
24
|
+
/** Positional encoding config, e.g. `rope:theta=10000:scale=1`. */
|
|
25
|
+
positionalConfig: string;
|
|
26
|
+
/** Attention config, e.g. `mha:heads=32:kvHeads=8:headDim=64`. */
|
|
27
|
+
attentionConfig: string;
|
|
28
|
+
}
|
|
29
|
+
/** Compatibility key with a canonical string form used as a map key. */
|
|
30
|
+
export interface CompatibilityKey extends CompatibilityParts {
|
|
31
|
+
/** Canonical encoding of the five parts. */
|
|
32
|
+
key: string;
|
|
33
|
+
}
|
|
34
|
+
export type OptimizationFlag = 'prefixCache' | 'preconvertQuant' | 'speculativeDecoding' | 'continuousBatching' | 'activationSparsity';
|
|
35
|
+
export interface AdvancedCapabilities {
|
|
36
|
+
speculativeDecoding: boolean;
|
|
37
|
+
continuousBatching: boolean;
|
|
38
|
+
activationSparsity: boolean;
|
|
39
|
+
/** Draft model present — required for speculative decoding. */
|
|
40
|
+
hasDraftModel: boolean;
|
|
41
|
+
/** Fraction of activations that are zero/skippable (0 = dense). */
|
|
42
|
+
sparsityFraction: number;
|
|
43
|
+
/**
|
|
44
|
+
* Model family. Architecture-specific opts are refused on incompatible
|
|
45
|
+
* families (e.g. speculative decoding is decoder-transformer only).
|
|
46
|
+
*/
|
|
47
|
+
architecture: string;
|
|
48
|
+
}
|
|
49
|
+
export interface QuantPlan {
|
|
50
|
+
native: KvQuantization;
|
|
51
|
+
runtime: KvQuantization;
|
|
52
|
+
converted: boolean;
|
|
53
|
+
conversionCost: number;
|
|
54
|
+
bytesPerParam: number;
|
|
55
|
+
}
|
|
56
|
+
export interface KvModelRef {
|
|
57
|
+
id: string;
|
|
58
|
+
quantization: KvQuantization;
|
|
59
|
+
paramsB?: number;
|
|
60
|
+
weightsHash?: string;
|
|
61
|
+
}
|
|
62
|
+
export interface KvGenerateRequest {
|
|
63
|
+
tokens: number[];
|
|
64
|
+
/** Decode steps after prefill. Default 0. */
|
|
65
|
+
maxNewTokens?: number;
|
|
66
|
+
model: KvModelRef;
|
|
67
|
+
tokenizerId: string;
|
|
68
|
+
positionalConfig: string;
|
|
69
|
+
attentionConfig: string;
|
|
70
|
+
/**
|
|
71
|
+
* Session identifier. Compatible sessions SHARE the cache — this is
|
|
72
|
+
* deliberately NOT part of the compatibility key.
|
|
73
|
+
*/
|
|
74
|
+
sessionId?: string;
|
|
75
|
+
/** Optional text used by an injectable quality scorer. */
|
|
76
|
+
output?: string;
|
|
77
|
+
baselineOutput?: string;
|
|
78
|
+
}
|
|
79
|
+
export interface KvGenerateResult {
|
|
80
|
+
output: string;
|
|
81
|
+
compatibilityKey: string;
|
|
82
|
+
cachedTokens: number;
|
|
83
|
+
computeTokens: number;
|
|
84
|
+
simulatedMs: number;
|
|
85
|
+
bandwidthBytes: number;
|
|
86
|
+
qualityScore: number;
|
|
87
|
+
withinTolerance: boolean;
|
|
88
|
+
servedFromCache: boolean;
|
|
89
|
+
optimizationsApplied: OptimizationFlag[];
|
|
90
|
+
evictedPages: number;
|
|
91
|
+
quantPlan: QuantPlan;
|
|
92
|
+
}
|
|
93
|
+
export type QualityScorer = (input: {
|
|
94
|
+
output: string;
|
|
95
|
+
baselineOutput: string;
|
|
96
|
+
quantization: KvQuantization;
|
|
97
|
+
baselineQuantization: KvQuantization;
|
|
98
|
+
}) => number;
|
|
99
|
+
export interface KvCostModel {
|
|
100
|
+
cacheHitPerToken: number;
|
|
101
|
+
computePerToken: number;
|
|
102
|
+
conversionPerToken: number;
|
|
103
|
+
draftPerToken: number;
|
|
104
|
+
verifyPerToken: number;
|
|
105
|
+
}
|
|
106
|
+
export interface KvCacheStats {
|
|
107
|
+
hits: number;
|
|
108
|
+
misses: number;
|
|
109
|
+
pages: number;
|
|
110
|
+
bytes: number;
|
|
111
|
+
evictions: number;
|
|
112
|
+
/** Distinct recently seen session ids (bounded by `maxSessions`). */
|
|
113
|
+
sessions: number;
|
|
114
|
+
}
|
|
115
|
+
export interface KvCacheRuntimeOptions {
|
|
116
|
+
maxBytes?: number;
|
|
117
|
+
pageTokens?: number;
|
|
118
|
+
flags?: Partial<Record<OptimizationFlag, boolean>>;
|
|
119
|
+
capabilities?: Partial<AdvancedCapabilities>;
|
|
120
|
+
qualityScorer?: QualityScorer;
|
|
121
|
+
qualityBaseline?: KvQuantization;
|
|
122
|
+
qualityTolerance?: number;
|
|
123
|
+
now?: () => number;
|
|
124
|
+
costs?: Partial<KvCostModel>;
|
|
125
|
+
layers?: number;
|
|
126
|
+
kvDim?: number;
|
|
127
|
+
/** Draft-token block size when speculative decoding is active. */
|
|
128
|
+
draftLen?: number;
|
|
129
|
+
/** 0..1 fraction of draft tokens accepted (injected; tests set 1). */
|
|
130
|
+
draftAccuracy?: number;
|
|
131
|
+
/** Target format when `preconvertQuant` is on. Default `q4`. */
|
|
132
|
+
preconvertTo?: KvQuantization;
|
|
133
|
+
/**
|
|
134
|
+
* Distinct session ids remembered for `stats().sessions`, least recently
|
|
135
|
+
* seen forgotten first. Default 1024.
|
|
136
|
+
*/
|
|
137
|
+
maxSessions?: number;
|
|
138
|
+
}
|
|
139
|
+
export interface KvCacheRuntime {
|
|
140
|
+
generate(request: KvGenerateRequest): Promise<KvGenerateResult>;
|
|
141
|
+
generateBatch(requests: KvGenerateRequest[]): Promise<KvGenerateResult[]>;
|
|
142
|
+
lookup(request: KvGenerateRequest): {
|
|
143
|
+
matched: number;
|
|
144
|
+
key: string;
|
|
145
|
+
};
|
|
146
|
+
setEnabled(flag: OptimizationFlag, enabled: boolean): void;
|
|
147
|
+
isEnabled(flag: OptimizationFlag): boolean;
|
|
148
|
+
stats(): KvCacheStats;
|
|
149
|
+
clear(): void;
|
|
150
|
+
}
|
|
151
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/kvcache/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,8EAA8E;AAC9E,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC;AAEzD,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;AAElE;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,2BAA2B;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,yDAAyD;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,gBAAgB,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,wEAAwE;AACxE,MAAM,WAAW,gBAAiB,SAAQ,kBAAkB;IAC1D,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,iBAAiB,GACjB,qBAAqB,GACrB,oBAAoB,GACpB,oBAAoB,CAAC;AAEzB,MAAM,WAAW,oBAAoB;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+DAA+D;IAC/D,aAAa,EAAE,OAAO,CAAC;IACvB,mEAAmE;IACnE,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,cAAc,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,cAAc,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,6CAA6C;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,UAAU,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,oBAAoB,EAAE,gBAAgB,EAAE,CAAC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;CACtB;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,cAAc,CAAC;IAC7B,oBAAoB,EAAE,cAAc,CAAC;CACtC,KAAK,MAAM,CAAC;AAEb,MAAM,WAAW,WAAW;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC;IACnD,YAAY,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC7C,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gEAAgE;IAChE,YAAY,CAAC,EAAE,cAAc,CAAC;IAC9B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAChE,aAAa,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACrE,UAAU,CAAC,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3D,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC;IAC3C,KAAK,IAAI,YAAY,CAAC;IACtB,KAAK,IAAI,IAAI,CAAC;CACf"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — Advanced Model & KV Cache Optimization: shared types.
|
|
3
|
+
*
|
|
4
|
+
* Browser-compatible only: pure TypeScript, no native code, no dependencies.
|
|
5
|
+
* Decoupled from `src/inference/` (which only knows HOW to run) and from
|
|
6
|
+
* `src/adaptive/` (which only decides WHERE/HOW). This module owns KV reuse,
|
|
7
|
+
* quantization-aware execution, and gated advanced decode paths.
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/kvcache/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 7 — workload fingerprinting.
|
|
3
|
+
*
|
|
4
|
+
* Buckets a workload into a stable signature (the learner's table key) and
|
|
5
|
+
* maps a Phase 2 `ExecutionOutcome` into an observation. Type-only coupling to
|
|
6
|
+
* the adaptive controller: nothing here imports or calls it.
|
|
7
|
+
*/
|
|
8
|
+
import type { ExecutionOutcome } from '../adaptive/types.js';
|
|
9
|
+
import type { OutcomeObservation, WorkloadFingerprintInput } from './types.js';
|
|
10
|
+
/** Stable signature for a workload. Same buckets always produce the same key. */
|
|
11
|
+
export declare function fingerprintWorkload(input: WorkloadFingerprintInput): string;
|
|
12
|
+
/**
|
|
13
|
+
* Map a Phase 2 `ExecutionOutcome` into an observation. Type-only coupling:
|
|
14
|
+
* this does not import the controller, and it does not apply anything.
|
|
15
|
+
*/
|
|
16
|
+
export declare function observationFromExecution(outcome: ExecutionOutcome, correct?: boolean): OutcomeObservation;
|
|
17
|
+
//# sourceMappingURL=fingerprint.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../../src/learning/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAY/E,iFAAiF;AACjF,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAM3E;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,gBAAgB,EAAE,OAAO,UAAO,GAAG,kBAAkB,CAetG"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 7 — workload fingerprinting.
|
|
3
|
+
*
|
|
4
|
+
* Buckets a workload into a stable signature (the learner's table key) and
|
|
5
|
+
* maps a Phase 2 `ExecutionOutcome` into an observation. Type-only coupling to
|
|
6
|
+
* the adaptive controller: nothing here imports or calls it.
|
|
7
|
+
*/
|
|
8
|
+
function bucket(value, edges) {
|
|
9
|
+
let lower = 0;
|
|
10
|
+
for (const edge of edges) {
|
|
11
|
+
if (value < edge)
|
|
12
|
+
return `${lower}-${edge}`;
|
|
13
|
+
lower = edge;
|
|
14
|
+
}
|
|
15
|
+
const last = edges[edges.length - 1] ?? 0;
|
|
16
|
+
return `${last}+`;
|
|
17
|
+
}
|
|
18
|
+
/** Stable signature for a workload. Same buckets always produce the same key. */
|
|
19
|
+
export function fingerprintWorkload(input) {
|
|
20
|
+
const params = bucket(input.paramsB ?? 0, [1, 3, 8]);
|
|
21
|
+
const prompt = bucket(input.promptTokens ?? 0, [128, 512, 2048]);
|
|
22
|
+
const decode = bucket(input.decodeTokens ?? 0, [64, 256, 1024]);
|
|
23
|
+
const quant = input.quantization && input.quantization.length > 0 ? input.quantization : 'unknown';
|
|
24
|
+
return `p${params}|q:${quant}|prompt:${prompt}|decode:${decode}`;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Map a Phase 2 `ExecutionOutcome` into an observation. Type-only coupling:
|
|
28
|
+
* this does not import the controller, and it does not apply anything.
|
|
29
|
+
*/
|
|
30
|
+
export function observationFromExecution(outcome, correct = true) {
|
|
31
|
+
const workload = outcome.decision.workload;
|
|
32
|
+
return {
|
|
33
|
+
signature: fingerprintWorkload({
|
|
34
|
+
paramsB: workload.paramsB,
|
|
35
|
+
quantization: workload.quantization,
|
|
36
|
+
promptTokens: workload.promptTokens,
|
|
37
|
+
decodeTokens: workload.decodeTokens,
|
|
38
|
+
}),
|
|
39
|
+
strategy: outcome.decision.plan.strategy,
|
|
40
|
+
costMs: outcome.measuredCostMs,
|
|
41
|
+
overheadMs: outcome.overheadMs,
|
|
42
|
+
benefitMs: outcome.benefitMs,
|
|
43
|
+
correct,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=fingerprint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../../src/learning/fingerprint.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,SAAS,MAAM,CAAC,KAAa,EAAE,KAAwB;IACrD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,KAAK,GAAG,IAAI;YAAE,OAAO,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;QAC5C,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1C,OAAO,GAAG,IAAI,GAAG,CAAC;AACpB,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC;IACnG,OAAO,IAAI,MAAM,MAAM,KAAK,WAAW,MAAM,WAAW,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAAyB,EAAE,OAAO,GAAG,IAAI;IAChF,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAC3C,OAAO;QACL,SAAS,EAAE,mBAAmB,CAAC;YAC7B,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,YAAY,EAAE,QAAQ,CAAC,YAAY;SACpC,CAAC;QACF,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ;QACxC,MAAM,EAAE,OAAO,CAAC,cAAc;QAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,OAAO;KACR,CAAC;AACJ,CAAC"}
|