@seanhogg/builderforce-webcontainers 2026.10.11 → 2026.10.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/adaptive/controller.d.ts +39 -0
- package/dist/adaptive/controller.d.ts.map +1 -0
- package/dist/adaptive/controller.js +108 -0
- package/dist/adaptive/controller.js.map +1 -0
- package/dist/adaptive/costModel.d.ts +37 -0
- package/dist/adaptive/costModel.d.ts.map +1 -0
- package/dist/adaptive/costModel.js +102 -0
- package/dist/adaptive/costModel.js.map +1 -0
- package/dist/adaptive/planner.d.ts +22 -0
- package/dist/adaptive/planner.d.ts.map +1 -0
- package/dist/adaptive/planner.js +114 -0
- package/dist/adaptive/planner.js.map +1 -0
- package/dist/adaptive/types.d.ts +138 -0
- package/dist/adaptive/types.d.ts.map +1 -0
- package/dist/adaptive/types.js +12 -0
- package/dist/adaptive/types.js.map +1 -0
- package/dist/assets.js +2 -2
- package/dist/check/cachedFetch.js.map +1 -1
- package/dist/check/index.d.ts +6 -1
- package/dist/check/index.d.ts.map +1 -1
- package/dist/check/index.js +6 -1
- package/dist/check/index.js.map +1 -1
- package/dist/componentCompilers.d.ts.map +1 -1
- package/dist/componentCompilers.js.map +1 -1
- package/dist/esbuildWasm.js.map +1 -1
- package/dist/host.d.ts +11 -5
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +19 -8
- package/dist/host.js.map +1 -1
- package/dist/inference/backends/mock.d.ts +24 -0
- package/dist/inference/backends/mock.d.ts.map +1 -0
- package/dist/inference/backends/mock.js +55 -0
- package/dist/inference/backends/mock.js.map +1 -0
- package/dist/inference/backends/onnx.d.ts +33 -0
- package/dist/inference/backends/onnx.d.ts.map +1 -0
- package/dist/inference/backends/onnx.js +60 -0
- package/dist/inference/backends/onnx.js.map +1 -0
- package/dist/inference/backends/webllm.d.ts +27 -0
- package/dist/inference/backends/webllm.d.ts.map +1 -0
- package/dist/inference/backends/webllm.js +52 -0
- package/dist/inference/backends/webllm.js.map +1 -0
- package/dist/inference/backends/wllama.d.ts +29 -0
- package/dist/inference/backends/wllama.d.ts.map +1 -0
- package/dist/inference/backends/wllama.js +51 -0
- package/dist/inference/backends/wllama.js.map +1 -0
- package/dist/inference/capabilities.d.ts +20 -0
- package/dist/inference/capabilities.d.ts.map +1 -0
- package/dist/inference/capabilities.js +33 -0
- package/dist/inference/capabilities.js.map +1 -0
- package/dist/inference/client.d.ts +45 -0
- package/dist/inference/client.d.ts.map +1 -0
- package/dist/inference/client.js +201 -0
- package/dist/inference/client.js.map +1 -0
- package/dist/inference/modelCache.d.ts +71 -0
- package/dist/inference/modelCache.d.ts.map +1 -0
- package/dist/inference/modelCache.js +91 -0
- package/dist/inference/modelCache.js.map +1 -0
- package/dist/inference/protocol.d.ts +94 -0
- package/dist/inference/protocol.d.ts.map +1 -0
- package/dist/inference/protocol.js +19 -0
- package/dist/inference/protocol.js.map +1 -0
- package/dist/inference/tokenStream.d.ts +26 -0
- package/dist/inference/tokenStream.d.ts.map +1 -0
- package/dist/inference/tokenStream.js +79 -0
- package/dist/inference/tokenStream.js.map +1 -0
- package/dist/inference/types.d.ts +108 -0
- package/dist/inference/types.d.ts.map +1 -0
- package/dist/inference/types.js +8 -0
- package/dist/inference/types.js.map +1 -0
- package/dist/inference/worker.bundle.js +5 -0
- package/dist/inference/worker.d.ts +35 -0
- package/dist/inference/worker.d.ts.map +1 -0
- package/dist/inference/worker.js +157 -0
- package/dist/inference/worker.js.map +1 -0
- package/dist/inference/workerEntry.d.ts +15 -0
- package/dist/inference/workerEntry.d.ts.map +1 -0
- package/dist/inference/workerEntry.js +17 -0
- package/dist/inference/workerEntry.js.map +1 -0
- package/dist/kvcache/advanced.d.ts +39 -0
- package/dist/kvcache/advanced.d.ts.map +1 -0
- package/dist/kvcache/advanced.js +59 -0
- package/dist/kvcache/advanced.js.map +1 -0
- package/dist/kvcache/compatibility.d.ts +14 -0
- package/dist/kvcache/compatibility.d.ts.map +1 -0
- package/dist/kvcache/compatibility.js +44 -0
- package/dist/kvcache/compatibility.js.map +1 -0
- package/dist/kvcache/lru.d.ts +45 -0
- package/dist/kvcache/lru.d.ts.map +1 -0
- package/dist/kvcache/lru.js +91 -0
- package/dist/kvcache/lru.js.map +1 -0
- package/dist/kvcache/paged.d.ts +86 -0
- package/dist/kvcache/paged.d.ts.map +1 -0
- package/dist/kvcache/paged.js +238 -0
- package/dist/kvcache/paged.js.map +1 -0
- package/dist/kvcache/quant.d.ts +30 -0
- package/dist/kvcache/quant.d.ts.map +1 -0
- package/dist/kvcache/quant.js +73 -0
- package/dist/kvcache/quant.js.map +1 -0
- package/dist/kvcache/radix.d.ts +28 -0
- package/dist/kvcache/radix.d.ts.map +1 -0
- package/dist/kvcache/radix.js +58 -0
- package/dist/kvcache/radix.js.map +1 -0
- package/dist/kvcache/runtime.d.ts +11 -0
- package/dist/kvcache/runtime.d.ts.map +1 -0
- package/dist/kvcache/runtime.js +204 -0
- package/dist/kvcache/runtime.js.map +1 -0
- package/dist/kvcache/types.d.ts +151 -0
- package/dist/kvcache/types.d.ts.map +1 -0
- package/dist/kvcache/types.js +10 -0
- package/dist/kvcache/types.js.map +1 -0
- package/dist/learning/fingerprint.d.ts +17 -0
- package/dist/learning/fingerprint.d.ts.map +1 -0
- package/dist/learning/fingerprint.js +46 -0
- package/dist/learning/fingerprint.js.map +1 -0
- package/dist/learning/index.d.ts +5 -0
- package/dist/learning/index.d.ts.map +1 -0
- package/dist/learning/index.js +3 -0
- package/dist/learning/index.js.map +1 -0
- package/dist/learning/learner.d.ts +21 -0
- package/dist/learning/learner.d.ts.map +1 -0
- package/dist/learning/learner.js +272 -0
- package/dist/learning/learner.js.map +1 -0
- package/dist/learning/ledger.d.ts +26 -0
- package/dist/learning/ledger.d.ts.map +1 -0
- package/dist/learning/ledger.js +40 -0
- package/dist/learning/ledger.js.map +1 -0
- package/dist/learning/selection.d.ts +44 -0
- package/dist/learning/selection.d.ts.map +1 -0
- package/dist/learning/selection.js +111 -0
- package/dist/learning/selection.js.map +1 -0
- package/dist/learning/stats.d.ts +19 -0
- package/dist/learning/stats.d.ts.map +1 -0
- package/dist/learning/stats.js +31 -0
- package/dist/learning/stats.js.map +1 -0
- package/dist/learning/types.d.ts +133 -0
- package/dist/learning/types.d.ts.map +1 -0
- package/dist/learning/types.js +8 -0
- package/dist/learning/types.js.map +1 -0
- package/dist/memory/envProbe.d.ts +16 -0
- package/dist/memory/envProbe.d.ts.map +1 -0
- package/dist/memory/envProbe.js +28 -0
- package/dist/memory/envProbe.js.map +1 -0
- package/dist/memory/estimates.d.ts +46 -0
- package/dist/memory/estimates.d.ts.map +1 -0
- package/dist/memory/estimates.js +72 -0
- package/dist/memory/estimates.js.map +1 -0
- package/dist/memory/eviction.d.ts +42 -0
- package/dist/memory/eviction.d.ts.map +1 -0
- package/dist/memory/eviction.js +143 -0
- package/dist/memory/eviction.js.map +1 -0
- package/dist/memory/manager.d.ts +34 -0
- package/dist/memory/manager.d.ts.map +1 -0
- package/dist/memory/manager.js +346 -0
- package/dist/memory/manager.js.map +1 -0
- package/dist/memory/managerTypes.d.ts +49 -0
- package/dist/memory/managerTypes.d.ts.map +1 -0
- package/dist/memory/managerTypes.js +7 -0
- package/dist/memory/managerTypes.js.map +1 -0
- package/dist/memory/pressure.d.ts +56 -0
- package/dist/memory/pressure.d.ts.map +1 -0
- package/dist/memory/pressure.js +80 -0
- package/dist/memory/pressure.js.map +1 -0
- package/dist/memory/spillTier.d.ts +47 -0
- package/dist/memory/spillTier.d.ts.map +1 -0
- package/dist/memory/spillTier.js +128 -0
- package/dist/memory/spillTier.js.map +1 -0
- package/dist/memory/storageSink.d.ts +69 -0
- package/dist/memory/storageSink.d.ts.map +1 -0
- package/dist/memory/storageSink.js +221 -0
- package/dist/memory/storageSink.js.map +1 -0
- package/dist/memory/types.d.ts +202 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +15 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/memory/victims.d.ts +32 -0
- package/dist/memory/victims.d.ts.map +1 -0
- package/dist/memory/victims.js +74 -0
- package/dist/memory/victims.js.map +1 -0
- package/dist/memory/workspace.d.ts +34 -0
- package/dist/memory/workspace.d.ts.map +1 -0
- package/dist/memory/workspace.js +55 -0
- package/dist/memory/workspace.js.map +1 -0
- package/dist/memoryIntegration/index.d.ts +8 -0
- package/dist/memoryIntegration/index.d.ts.map +1 -0
- package/dist/memoryIntegration/index.js +4 -0
- package/dist/memoryIntegration/index.js.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts +43 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.js +285 -0
- package/dist/memoryIntegration/knowledgeMemory.js.map +1 -0
- package/dist/memoryIntegration/persistence.d.ts +15 -0
- package/dist/memoryIntegration/persistence.d.ts.map +1 -0
- package/dist/memoryIntegration/persistence.js +43 -0
- package/dist/memoryIntegration/persistence.js.map +1 -0
- package/dist/memoryIntegration/text.d.ts +57 -0
- package/dist/memoryIntegration/text.d.ts.map +1 -0
- package/dist/memoryIntegration/text.js +179 -0
- package/dist/memoryIntegration/text.js.map +1 -0
- package/dist/memoryIntegration/types.d.ts +131 -0
- package/dist/memoryIntegration/types.d.ts.map +1 -0
- package/dist/memoryIntegration/types.js +11 -0
- package/dist/memoryIntegration/types.js.map +1 -0
- package/dist/ml.d.ts +52 -0
- package/dist/ml.d.ts.map +1 -0
- package/dist/ml.js +33 -0
- package/dist/ml.js.map +1 -0
- package/dist/node/childRelay.d.ts +10 -0
- package/dist/node/childRelay.d.ts.map +1 -1
- package/dist/node/childRelay.js +12 -2
- package/dist/node/childRelay.js.map +1 -1
- package/dist/node/index.js.map +1 -1
- package/dist/node/portResponder.d.ts +2 -0
- package/dist/node/portResponder.d.ts.map +1 -1
- package/dist/node/portResponder.js +3 -3
- package/dist/node/portResponder.js.map +1 -1
- package/dist/node/processHost.d.ts +4 -1
- package/dist/node/processHost.d.ts.map +1 -1
- package/dist/node/processHost.js +19 -16
- package/dist/node/processHost.js.map +1 -1
- package/dist/node/protocol.d.ts +1 -0
- package/dist/node/protocol.d.ts.map +1 -1
- package/dist/node/responseWire.d.ts.map +1 -1
- package/dist/node/worker.js +116 -109
- package/dist/node/workerProcess.js +1 -1
- package/dist/node/workerProcess.js.map +1 -1
- package/dist/prefetch/admission.d.ts +41 -0
- package/dist/prefetch/admission.d.ts.map +1 -0
- package/dist/prefetch/admission.js +49 -0
- package/dist/prefetch/admission.js.map +1 -0
- package/dist/prefetch/loadRegistry.d.ts +56 -0
- package/dist/prefetch/loadRegistry.d.ts.map +1 -0
- package/dist/prefetch/loadRegistry.js +108 -0
- package/dist/prefetch/loadRegistry.js.map +1 -0
- package/dist/prefetch/predictor.d.ts +40 -0
- package/dist/prefetch/predictor.d.ts.map +1 -0
- package/dist/prefetch/predictor.js +178 -0
- package/dist/prefetch/predictor.js.map +1 -0
- package/dist/prefetch/prefetcher.d.ts +37 -0
- package/dist/prefetch/prefetcher.d.ts.map +1 -0
- package/dist/prefetch/prefetcher.js +339 -0
- package/dist/prefetch/prefetcher.js.map +1 -0
- package/dist/prefetch/residentStore.d.ts +49 -0
- package/dist/prefetch/residentStore.d.ts.map +1 -0
- package/dist/prefetch/residentStore.js +87 -0
- package/dist/prefetch/residentStore.js.map +1 -0
- package/dist/prefetch/selfTuner.d.ts +40 -0
- package/dist/prefetch/selfTuner.d.ts.map +1 -0
- package/dist/prefetch/selfTuner.js +78 -0
- package/dist/prefetch/selfTuner.js.map +1 -0
- package/dist/prefetch/stats.d.ts +52 -0
- package/dist/prefetch/stats.d.ts.map +1 -0
- package/dist/prefetch/stats.js +72 -0
- package/dist/prefetch/stats.js.map +1 -0
- package/dist/prefetch/streaming.d.ts +132 -0
- package/dist/prefetch/streaming.d.ts.map +1 -0
- package/dist/prefetch/streaming.js +141 -0
- package/dist/prefetch/streaming.js.map +1 -0
- package/dist/prefetch/targetIndex.d.ts +18 -0
- package/dist/prefetch/targetIndex.d.ts.map +1 -0
- package/dist/prefetch/targetIndex.js +32 -0
- package/dist/prefetch/targetIndex.js.map +1 -0
- package/dist/prefetch/types.d.ts +283 -0
- package/dist/prefetch/types.d.ts.map +1 -0
- package/dist/prefetch/types.js +19 -0
- package/dist/prefetch/types.js.map +1 -0
- package/dist/previewBridge.d.ts +6 -0
- package/dist/previewBridge.d.ts.map +1 -1
- package/dist/previewBridge.js +25 -5
- package/dist/previewBridge.js.map +1 -1
- package/dist/previewMessages.d.ts +4 -3
- package/dist/previewMessages.d.ts.map +1 -1
- package/dist/previewMessages.js +12 -6
- package/dist/previewMessages.js.map +1 -1
- package/dist/process-worker.js +116 -109
- package/dist/relay.html +1 -1
- package/dist/shared/clock.d.ts +7 -0
- package/dist/shared/clock.d.ts.map +1 -0
- package/dist/shared/clock.js +10 -0
- package/dist/shared/clock.js.map +1 -0
- package/dist/shared/hardwareProbe.d.ts +26 -0
- package/dist/shared/hardwareProbe.d.ts.map +1 -0
- package/dist/shared/hardwareProbe.js +62 -0
- package/dist/shared/hardwareProbe.js.map +1 -0
- package/dist/shared/idb.d.ts +41 -0
- package/dist/shared/idb.d.ts.map +1 -0
- package/dist/shared/idb.js +184 -0
- package/dist/shared/idb.js.map +1 -0
- package/dist/shared/tokens.d.ts +7 -0
- package/dist/shared/tokens.d.ts.map +1 -0
- package/dist/shared/tokens.js +9 -0
- package/dist/shared/tokens.js.map +1 -0
- package/dist/snapshotStore.d.ts +2 -1
- package/dist/snapshotStore.d.ts.map +1 -1
- package/dist/snapshotStore.js +27 -12
- package/dist/snapshotStore.js.map +1 -1
- package/dist/styleCompilers.d.ts.map +1 -1
- package/dist/styleCompilers.js.map +1 -1
- package/dist/transport.js.map +1 -1
- package/package.json +11 -5
- package/src/adaptive/controller.ts +151 -0
- package/src/adaptive/costModel.ts +131 -0
- package/src/adaptive/planner.ts +127 -0
- package/src/adaptive/types.ts +137 -0
- package/src/check/index.ts +6 -1
- package/src/host.ts +33 -11
- package/src/inference/backends/mock.ts +77 -0
- package/src/inference/backends/onnx.ts +91 -0
- package/src/inference/backends/webllm.ts +79 -0
- package/src/inference/backends/wllama.ts +79 -0
- package/src/inference/capabilities.ts +46 -0
- package/src/inference/client.ts +276 -0
- package/src/inference/modelCache.ts +145 -0
- package/src/inference/protocol.ts +119 -0
- package/src/inference/tokenStream.ts +92 -0
- package/src/inference/types.ts +115 -0
- package/src/inference/worker.ts +191 -0
- package/src/inference/workerEntry.ts +18 -0
- package/src/kvcache/advanced.ts +86 -0
- package/src/kvcache/compatibility.ts +51 -0
- package/src/kvcache/lru.ts +100 -0
- package/src/kvcache/paged.ts +270 -0
- package/src/kvcache/quant.ts +88 -0
- package/src/kvcache/radix.ts +68 -0
- package/src/kvcache/runtime.ts +239 -0
- package/src/kvcache/types.ts +167 -0
- package/src/learning/fingerprint.ts +50 -0
- package/src/learning/index.ts +15 -0
- package/src/learning/learner.ts +304 -0
- package/src/learning/ledger.ts +54 -0
- package/src/learning/selection.ts +137 -0
- package/src/learning/stats.ts +39 -0
- package/src/learning/types.ts +142 -0
- package/src/memory/envProbe.ts +30 -0
- package/src/memory/estimates.ts +98 -0
- package/src/memory/eviction.ts +145 -0
- package/src/memory/manager.ts +380 -0
- package/src/memory/managerTypes.ts +61 -0
- package/src/memory/pressure.ts +120 -0
- package/src/memory/spillTier.ts +159 -0
- package/src/memory/storageSink.ts +289 -0
- package/src/memory/types.ts +217 -0
- package/src/memory/victims.ts +80 -0
- package/src/memory/workspace.ts +74 -0
- package/src/memoryIntegration/index.ts +7 -0
- package/src/memoryIntegration/knowledgeMemory.ts +339 -0
- package/src/memoryIntegration/persistence.ts +54 -0
- package/src/memoryIntegration/text.ts +185 -0
- package/src/memoryIntegration/types.ts +145 -0
- package/src/ml.ts +180 -0
- package/src/node/childRelay.ts +16 -3
- package/src/node/portResponder.ts +5 -3
- package/src/node/processHost.ts +16 -15
- package/src/node/protocol.ts +1 -1
- package/src/node/workerProcess.ts +1 -1
- package/src/prefetch/admission.ts +74 -0
- package/src/prefetch/loadRegistry.ts +156 -0
- package/src/prefetch/predictor.ts +199 -0
- package/src/prefetch/prefetcher.ts +369 -0
- package/src/prefetch/residentStore.ts +121 -0
- package/src/prefetch/selfTuner.ts +110 -0
- package/src/prefetch/stats.ts +120 -0
- package/src/prefetch/streaming.ts +231 -0
- package/src/prefetch/targetIndex.ts +40 -0
- package/src/prefetch/types.ts +307 -0
- package/src/previewBridge.ts +25 -5
- package/src/previewMessages.ts +11 -6
- package/src/shared/clock.ts +9 -0
- package/src/shared/hardwareProbe.ts +68 -0
- package/src/shared/idb.ts +203 -0
- package/src/shared/tokens.ts +8 -0
- package/src/snapshotStore.ts +25 -10
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — paged KV cache.
|
|
3
|
+
*
|
|
4
|
+
* Token sequences are stored as fixed-size pages keyed by a compatibility
|
|
5
|
+
* key. Pages form a chain: a page's id is a rolling hash of (parent page id,
|
|
6
|
+
* the page's own tokens), rooted at a hash of the compatibility key. So a
|
|
7
|
+
* page id names its whole token prefix without embedding it — ids stay a few
|
|
8
|
+
* dozen bytes however long the sequence is.
|
|
9
|
+
*
|
|
10
|
+
* Lookup walks the query page by page and stops at the first page that is not
|
|
11
|
+
* resident, which finds the longest shared PAGE-ALIGNED prefix with ANY stored
|
|
12
|
+
* sequence — sequences that branch after a shared prefix all reuse it. A
|
|
13
|
+
* trailing partial page is matched too, via the per-parent tail index.
|
|
14
|
+
*
|
|
15
|
+
* Exactness: every hop re-verifies the page's compatibility key, parent id,
|
|
16
|
+
* parent serial (the exact parent page object it was stored under) and
|
|
17
|
+
* tokens, so a hash collision can only cause a miss, never reuse of the wrong
|
|
18
|
+
* KV state. Semantic / approximate matching is never done.
|
|
19
|
+
*
|
|
20
|
+
* Bounds: the only state is the page LRU (charged against `maxBytes`) and the
|
|
21
|
+
* tail index, whose entries are removed when their page is evicted. Sessions
|
|
22
|
+
* that share a compatibility key reuse pages — the session id is NOT part of
|
|
23
|
+
* the key.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { encodeCompatibilityKey } from './compatibility.js';
|
|
27
|
+
import { LruCache } from './lru.js';
|
|
28
|
+
import type { CompatibilityParts } from './types.js';
|
|
29
|
+
|
|
30
|
+
export interface KvPage {
|
|
31
|
+
id: string;
|
|
32
|
+
compatKey: string;
|
|
33
|
+
/** Id of the page this one continues (the chain root for the first page). */
|
|
34
|
+
parentId: string;
|
|
35
|
+
/** Unique per inserted page. A replaced page gets a new serial, orphaning its old children. */
|
|
36
|
+
serial: number;
|
|
37
|
+
/** Serial of the exact parent page this one was stored under (0 = chain root). */
|
|
38
|
+
parentSerial: number;
|
|
39
|
+
/** Token ids this page covers (length ≤ pageTokens). */
|
|
40
|
+
tokens: number[];
|
|
41
|
+
/** Simulated KV bytes for this page. */
|
|
42
|
+
bytes: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface PrefixHit {
|
|
46
|
+
matched: number;
|
|
47
|
+
pages: KvPage[];
|
|
48
|
+
compatKey: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const FNV_OFFSET = 0x811c9dc5;
|
|
52
|
+
const FNV_PRIME = 0x01000193;
|
|
53
|
+
const MIX_SEED = 0x9747b28c;
|
|
54
|
+
const MIX_PRIME = 0x5bd1e995;
|
|
55
|
+
|
|
56
|
+
/** Two independent 32-bit lanes over a string and a token run → a compact id. */
|
|
57
|
+
function chainHash(parent: string, tokens: readonly number[]): string {
|
|
58
|
+
let a = FNV_OFFSET;
|
|
59
|
+
let b = MIX_SEED ^ tokens.length;
|
|
60
|
+
for (let i = 0; i < parent.length; i++) {
|
|
61
|
+
const c = parent.charCodeAt(i);
|
|
62
|
+
a = Math.imul(a ^ c, FNV_PRIME);
|
|
63
|
+
b = Math.imul(b ^ c, MIX_PRIME);
|
|
64
|
+
b ^= b >>> 13;
|
|
65
|
+
}
|
|
66
|
+
for (const token of tokens) {
|
|
67
|
+
const t = token | 0;
|
|
68
|
+
for (let shift = 0; shift < 32; shift += 8) {
|
|
69
|
+
const byte = (t >>> shift) & 0xff;
|
|
70
|
+
a = Math.imul(a ^ byte, FNV_PRIME);
|
|
71
|
+
b = Math.imul(b ^ byte, MIX_PRIME);
|
|
72
|
+
b ^= b >>> 15;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return `${(a >>> 0).toString(36)}.${(b >>> 0).toString(36)}.${tokens.length}`;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function sameTokens(a: readonly number[], b: readonly number[]): boolean {
|
|
79
|
+
if (a.length !== b.length) return false;
|
|
80
|
+
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function isPrefixOf(prefix: readonly number[], tokens: readonly number[], offset: number): boolean {
|
|
85
|
+
if (offset + prefix.length > tokens.length) return false;
|
|
86
|
+
for (let i = 0; i < prefix.length; i++) if (prefix[i] !== tokens[offset + i]) return false;
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export class PagedKvCache {
|
|
91
|
+
private readonly lru: LruCache<KvPage>;
|
|
92
|
+
/** Partial (shorter than pageTokens) pages, by parent id. Entries leave with their page. */
|
|
93
|
+
private readonly tails = new Map<string, Set<string>>();
|
|
94
|
+
private readonly tailParent = new Map<string, string>();
|
|
95
|
+
private nextSerial = 1;
|
|
96
|
+
evictions = 0;
|
|
97
|
+
hits = 0;
|
|
98
|
+
misses = 0;
|
|
99
|
+
|
|
100
|
+
constructor(
|
|
101
|
+
readonly maxBytes: number,
|
|
102
|
+
readonly pageTokens: number,
|
|
103
|
+
readonly bytesPerToken: number,
|
|
104
|
+
) {
|
|
105
|
+
this.lru = new LruCache<KvPage>(maxBytes);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
get pages(): number {
|
|
109
|
+
return this.lru.size;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
get bytes(): number {
|
|
113
|
+
return this.lru.bytes;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
lookup(compat: CompatibilityParts, tokens: readonly number[]): PrefixHit {
|
|
117
|
+
const compatKey = encodeCompatibilityKey(compat).key;
|
|
118
|
+
const pages: KvPage[] = [];
|
|
119
|
+
let parent = chainHash(compatKey, []);
|
|
120
|
+
let parentSerial = 0;
|
|
121
|
+
let matched = 0;
|
|
122
|
+
while (matched < tokens.length) {
|
|
123
|
+
const slice = tokens.slice(matched, matched + this.pageTokens);
|
|
124
|
+
const page = this.residentPage(chainHash(parent, slice), compatKey, parent, parentSerial, slice);
|
|
125
|
+
if (page) {
|
|
126
|
+
pages.push(page);
|
|
127
|
+
matched += page.tokens.length;
|
|
128
|
+
parent = page.id;
|
|
129
|
+
parentSerial = page.serial;
|
|
130
|
+
if (page.tokens.length < this.pageTokens) break;
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
const tail = this.longestTail(parent, parentSerial, compatKey, tokens, matched);
|
|
134
|
+
if (tail) {
|
|
135
|
+
pages.push(tail);
|
|
136
|
+
matched += tail.tokens.length;
|
|
137
|
+
}
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
if (matched === 0) {
|
|
141
|
+
this.misses++;
|
|
142
|
+
return { matched: 0, pages: [], compatKey };
|
|
143
|
+
}
|
|
144
|
+
this.hits++;
|
|
145
|
+
return { matched, pages, compatKey };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Store (or extend) a sequence under a compatibility key. Pages covering
|
|
150
|
+
* tokens already resident are reused; new pages are inserted and may
|
|
151
|
+
* trigger LRU eviction of the least-recently-used pages.
|
|
152
|
+
*/
|
|
153
|
+
put(compat: CompatibilityParts, tokens: readonly number[]): { evicted: number } {
|
|
154
|
+
const compatKey = encodeCompatibilityKey(compat).key;
|
|
155
|
+
let parent = chainHash(compatKey, []);
|
|
156
|
+
let parentSerial = 0;
|
|
157
|
+
let evictedThis = 0;
|
|
158
|
+
for (let i = 0; i < tokens.length; i += this.pageTokens) {
|
|
159
|
+
const slice = tokens.slice(i, i + this.pageTokens);
|
|
160
|
+
const id = chainHash(parent, slice);
|
|
161
|
+
let page = this.residentPage(id, compatKey, parent, parentSerial, slice);
|
|
162
|
+
if (!page) {
|
|
163
|
+
page = {
|
|
164
|
+
id,
|
|
165
|
+
compatKey,
|
|
166
|
+
parentId: parent,
|
|
167
|
+
serial: this.nextSerial++,
|
|
168
|
+
parentSerial,
|
|
169
|
+
tokens: slice,
|
|
170
|
+
bytes: slice.length * this.bytesPerToken,
|
|
171
|
+
};
|
|
172
|
+
// A stale or colliding page under this id (verification failed above) is replaced.
|
|
173
|
+
this.forgetTail(id);
|
|
174
|
+
const evicted = this.lru.set(id, page, page.bytes);
|
|
175
|
+
this.evictions += evicted.length;
|
|
176
|
+
evictedThis += evicted.length;
|
|
177
|
+
this.dropEvicted(evicted);
|
|
178
|
+
if (!this.lru.has(id)) break; // A page that cannot fit ends the chain.
|
|
179
|
+
if (slice.length < this.pageTokens) this.rememberTail(parent, id);
|
|
180
|
+
}
|
|
181
|
+
parent = id;
|
|
182
|
+
parentSerial = page.serial;
|
|
183
|
+
}
|
|
184
|
+
return { evicted: evictedThis };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
clear(): void {
|
|
188
|
+
this.lru.clear();
|
|
189
|
+
this.tails.clear();
|
|
190
|
+
this.tailParent.clear();
|
|
191
|
+
this.evictions = 0;
|
|
192
|
+
this.hits = 0;
|
|
193
|
+
this.misses = 0;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** A resident page whose identity is verified, touched as most recently used. */
|
|
197
|
+
private residentPage(
|
|
198
|
+
id: string,
|
|
199
|
+
compatKey: string,
|
|
200
|
+
parent: string,
|
|
201
|
+
parentSerial: number,
|
|
202
|
+
slice: readonly number[],
|
|
203
|
+
): KvPage | undefined {
|
|
204
|
+
const page = this.lru.peek(id);
|
|
205
|
+
if (!page || !this.childOf(page, compatKey, parent, parentSerial) || !sameTokens(page.tokens, slice)) {
|
|
206
|
+
return undefined;
|
|
207
|
+
}
|
|
208
|
+
this.lru.get(id);
|
|
209
|
+
return page;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
private childOf(page: KvPage, compatKey: string, parent: string, parentSerial: number): boolean {
|
|
213
|
+
return page.compatKey === compatKey && page.parentId === parent && page.parentSerial === parentSerial;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Longest resident partial page under `parent` whose tokens continue the query at `offset`. */
|
|
217
|
+
private longestTail(
|
|
218
|
+
parent: string,
|
|
219
|
+
parentSerial: number,
|
|
220
|
+
compatKey: string,
|
|
221
|
+
tokens: readonly number[],
|
|
222
|
+
offset: number,
|
|
223
|
+
): KvPage | undefined {
|
|
224
|
+
const ids = this.tails.get(parent);
|
|
225
|
+
if (!ids) return undefined;
|
|
226
|
+
let best: KvPage | undefined;
|
|
227
|
+
for (const id of ids) {
|
|
228
|
+
const page = this.lru.peek(id);
|
|
229
|
+
if (!page || !this.childOf(page, compatKey, parent, parentSerial)) continue;
|
|
230
|
+
if (!isPrefixOf(page.tokens, tokens, offset)) continue;
|
|
231
|
+
if (!best || page.tokens.length > best.tokens.length) best = page;
|
|
232
|
+
}
|
|
233
|
+
if (best) this.lru.get(best.id);
|
|
234
|
+
return best;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
private rememberTail(parent: string, id: string): void {
|
|
238
|
+
let ids = this.tails.get(parent);
|
|
239
|
+
if (!ids) {
|
|
240
|
+
ids = new Set();
|
|
241
|
+
this.tails.set(parent, ids);
|
|
242
|
+
}
|
|
243
|
+
ids.add(id);
|
|
244
|
+
this.tailParent.set(id, parent);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
private forgetTail(id: string): void {
|
|
248
|
+
const parent = this.tailParent.get(id);
|
|
249
|
+
if (parent === undefined) return;
|
|
250
|
+
this.tailParent.delete(id);
|
|
251
|
+
const ids = this.tails.get(parent);
|
|
252
|
+
if (!ids) return;
|
|
253
|
+
ids.delete(id);
|
|
254
|
+
if (ids.size === 0) this.tails.delete(parent);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Evicted pages leave every index with them. A descendant of an evicted
|
|
259
|
+
* page stays resident until the LRU reaches it, but is unreachable: lookup
|
|
260
|
+
* stops at the first missing page, so it is a miss and recompute.
|
|
261
|
+
*/
|
|
262
|
+
private dropEvicted(ids: readonly string[]): void {
|
|
263
|
+
for (const id of ids) this.forgetTail(id);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** Number of tail-index entries (diagnostics / bound checks). */
|
|
267
|
+
get indexedTails(): number {
|
|
268
|
+
return this.tailParent.size;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — quantization-aware execution.
|
|
3
|
+
*
|
|
4
|
+
* Picks the runtime format from `ModelRef.quantization` (`q4`|`q5`|`q8`|`fp16`)
|
|
5
|
+
* and avoids converting unless `preconvertQuant` is opted in. Conversion is
|
|
6
|
+
* never implicit: the native format is used unless a flag + target justify it.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { bytesPerParam } from '../memory/estimates.js';
|
|
10
|
+
import type { KvBackendKind, KvQuantization, QuantPlan } from './types.js';
|
|
11
|
+
|
|
12
|
+
/** Relative quality vs fp16, used by the default scorer. */
|
|
13
|
+
const QUALITY: Record<KvQuantization, number> = {
|
|
14
|
+
fp16: 1,
|
|
15
|
+
q8: 0.99,
|
|
16
|
+
q5: 0.97,
|
|
17
|
+
q4: 0.94,
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export function nativeQuality(q: KvQuantization): number {
|
|
21
|
+
return QUALITY[q];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Decide the runtime quantization. Conversion is skipped unless
|
|
26
|
+
* `preconvert` is true AND the target is a strictly smaller format
|
|
27
|
+
* (justified by a bandwidth saving).
|
|
28
|
+
*/
|
|
29
|
+
export function planQuantization(input: {
|
|
30
|
+
native: KvQuantization;
|
|
31
|
+
backend: KvBackendKind;
|
|
32
|
+
preconvert: boolean;
|
|
33
|
+
preconvertTo?: KvQuantization;
|
|
34
|
+
}): QuantPlan {
|
|
35
|
+
const native = input.native;
|
|
36
|
+
const target = input.preconvertTo ?? 'q4';
|
|
37
|
+
const justified =
|
|
38
|
+
input.preconvert && bytesPerParam(target) < bytesPerParam(native) && backendSupports(input.backend, target);
|
|
39
|
+
if (!justified) {
|
|
40
|
+
return {
|
|
41
|
+
native,
|
|
42
|
+
runtime: native,
|
|
43
|
+
converted: false,
|
|
44
|
+
conversionCost: 0,
|
|
45
|
+
bytesPerParam: bytesPerParam(native),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
return {
|
|
49
|
+
native,
|
|
50
|
+
runtime: target,
|
|
51
|
+
converted: true,
|
|
52
|
+
conversionCost: bytesPerParam(native) - bytesPerParam(target),
|
|
53
|
+
bytesPerParam: bytesPerParam(target),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function backendSupports(backend: KvBackendKind, q: KvQuantization): boolean {
|
|
58
|
+
if (backend === 'webllm') return q === 'fp16' || q === 'q4';
|
|
59
|
+
if (backend === 'wllama') return q !== 'fp16';
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function bandwidthBytes(params: number, plan: QuantPlan, tokens: number, kvDim: number): number {
|
|
64
|
+
const weightBytes = params * plan.bytesPerParam;
|
|
65
|
+
const kvBytes = tokens * kvDim * plan.bytesPerParam;
|
|
66
|
+
return weightBytes + kvBytes;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export const DEFAULT_QUALITY_TOLERANCE = 0.08;
|
|
70
|
+
|
|
71
|
+
export function defaultQualityScorer(input: {
|
|
72
|
+
output: string;
|
|
73
|
+
baselineOutput: string;
|
|
74
|
+
quantization: KvQuantization;
|
|
75
|
+
baselineQuantization: KvQuantization;
|
|
76
|
+
}): number {
|
|
77
|
+
const q = nativeQuality(input.quantization) / nativeQuality(input.baselineQuantization);
|
|
78
|
+
if (!input.baselineOutput) return q;
|
|
79
|
+
const a = input.output;
|
|
80
|
+
const b = input.baselineOutput;
|
|
81
|
+
const n = Math.max(a.length, b.length, 1);
|
|
82
|
+
let same = 0;
|
|
83
|
+
for (let i = 0; i < Math.min(a.length, b.length); i++) {
|
|
84
|
+
if (a.charCodeAt(i) === b.charCodeAt(i)) same++;
|
|
85
|
+
}
|
|
86
|
+
const overlap = same / n;
|
|
87
|
+
return Math.min(1, q * 0.5 + overlap * 0.5);
|
|
88
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — radix-tree prefix index over token sequences.
|
|
3
|
+
*
|
|
4
|
+
* Each node is keyed by a single token id. A match walks the tree while
|
|
5
|
+
* tokens equal the query; the first mismatch stops. This is EXACT prefix
|
|
6
|
+
* reuse only — never semantic / approximate.
|
|
7
|
+
*
|
|
8
|
+
* Public primitive (exported from the kvcache barrel). The paged KV cache no
|
|
9
|
+
* longer stores per-sequence trees: its hash-chained page ids already index
|
|
10
|
+
* every shared prefix, inside the page LRU's byte budget.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
export interface RadixMatch<V> {
|
|
14
|
+
length: number;
|
|
15
|
+
value: V;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
interface RadixNode<V> {
|
|
19
|
+
children: Map<number, RadixNode<V>>;
|
|
20
|
+
value?: V;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class TokenRadix<V> {
|
|
24
|
+
private readonly root: RadixNode<V> = { children: new Map() };
|
|
25
|
+
private n = 0;
|
|
26
|
+
|
|
27
|
+
get size(): number {
|
|
28
|
+
return this.n;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
insert(tokens: readonly number[], value: V): void {
|
|
32
|
+
let node = this.root;
|
|
33
|
+
for (const t of tokens) {
|
|
34
|
+
let child = node.children.get(t);
|
|
35
|
+
if (!child) {
|
|
36
|
+
child = { children: new Map() };
|
|
37
|
+
node.children.set(t, child);
|
|
38
|
+
}
|
|
39
|
+
node = child;
|
|
40
|
+
}
|
|
41
|
+
if (node.value === undefined) this.n++;
|
|
42
|
+
node.value = value;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Longest stored prefix that is an exact prefix of `tokens`.
|
|
47
|
+
* Returns undefined when no stored sequence shares the first token.
|
|
48
|
+
*/
|
|
49
|
+
longestPrefix(tokens: readonly number[]): RadixMatch<V> | undefined {
|
|
50
|
+
let node = this.root;
|
|
51
|
+
let last: RadixMatch<V> | undefined;
|
|
52
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
53
|
+
const t = tokens[i];
|
|
54
|
+
if (t === undefined) break;
|
|
55
|
+
const child = node.children.get(t);
|
|
56
|
+
if (!child) break;
|
|
57
|
+
node = child;
|
|
58
|
+
if (node.value !== undefined) last = { length: i + 1, value: node.value };
|
|
59
|
+
}
|
|
60
|
+
return last;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
clear(): void {
|
|
64
|
+
this.root.children.clear();
|
|
65
|
+
this.root.value = undefined;
|
|
66
|
+
this.n = 0;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — KV-cache runtime.
|
|
3
|
+
*
|
|
4
|
+
* Orchestrates paged exact-prefix reuse, quantization-aware execution, and
|
|
5
|
+
* gated advanced decode. The execution engine is mock/injected: generate()
|
|
6
|
+
* simulates prefill+decode cost so tests can assert reuse is cheaper than
|
|
7
|
+
* recompute without real GGUF/ONNX weights.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { canActivationSparsity, canContinuousBatch, canSpeculative, decodeCost, isEnabled } from './advanced.js';
|
|
11
|
+
import { encodeCompatibilityKey } from './compatibility.js';
|
|
12
|
+
import { PagedKvCache } from './paged.js';
|
|
13
|
+
import {
|
|
14
|
+
bandwidthBytes,
|
|
15
|
+
DEFAULT_QUALITY_TOLERANCE,
|
|
16
|
+
defaultQualityScorer,
|
|
17
|
+
planQuantization,
|
|
18
|
+
} from './quant.js';
|
|
19
|
+
import type {
|
|
20
|
+
AdvancedCapabilities,
|
|
21
|
+
CompatibilityParts,
|
|
22
|
+
KvCacheRuntime,
|
|
23
|
+
KvCacheRuntimeOptions,
|
|
24
|
+
KvCacheStats,
|
|
25
|
+
KvCostModel,
|
|
26
|
+
KvGenerateRequest,
|
|
27
|
+
KvGenerateResult,
|
|
28
|
+
OptimizationFlag,
|
|
29
|
+
QualityScorer,
|
|
30
|
+
} from './types.js';
|
|
31
|
+
|
|
32
|
+
const DEFAULT_COSTS: KvCostModel = {
|
|
33
|
+
cacheHitPerToken: 0.05,
|
|
34
|
+
computePerToken: 1,
|
|
35
|
+
conversionPerToken: 0.2,
|
|
36
|
+
draftPerToken: 0.15,
|
|
37
|
+
verifyPerToken: 0.25,
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const DEFAULT_MAX_SESSIONS = 1024;
|
|
41
|
+
|
|
42
|
+
const DEFAULT_CAPS: AdvancedCapabilities = {
|
|
43
|
+
speculativeDecoding: false,
|
|
44
|
+
continuousBatching: false,
|
|
45
|
+
activationSparsity: false,
|
|
46
|
+
hasDraftModel: false,
|
|
47
|
+
sparsityFraction: 0,
|
|
48
|
+
architecture: 'decoder-transformer',
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
function partsOf(req: KvGenerateRequest): CompatibilityParts {
|
|
52
|
+
return {
|
|
53
|
+
modelId: req.model.id,
|
|
54
|
+
weightsHash: req.model.weightsHash ?? req.model.id,
|
|
55
|
+
tokenizerId: req.tokenizerId,
|
|
56
|
+
positionalConfig: req.positionalConfig,
|
|
57
|
+
attentionConfig: req.attentionConfig,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function mockOutput(tokens: readonly number[]): string {
|
|
62
|
+
return tokens.map((t) => String.fromCharCode(97 + (Math.abs(t) % 26))).join('');
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function createKvCacheRuntime(options: KvCacheRuntimeOptions = {}): KvCacheRuntime {
|
|
66
|
+
const pageTokens = options.pageTokens ?? 16;
|
|
67
|
+
const layers = options.layers ?? 12;
|
|
68
|
+
const kvDim = options.kvDim ?? 64;
|
|
69
|
+
const bytesPerToken = layers * kvDim * 2; // K+V, fp-ish units
|
|
70
|
+
const maxBytes = options.maxBytes ?? 1 * 1024 * 1024;
|
|
71
|
+
const cache = new PagedKvCache(maxBytes, pageTokens, bytesPerToken);
|
|
72
|
+
const flags: Record<OptimizationFlag, boolean> = {
|
|
73
|
+
prefixCache: options.flags?.prefixCache !== false,
|
|
74
|
+
preconvertQuant: options.flags?.preconvertQuant === true,
|
|
75
|
+
speculativeDecoding: options.flags?.speculativeDecoding === true,
|
|
76
|
+
continuousBatching: options.flags?.continuousBatching === true,
|
|
77
|
+
activationSparsity: options.flags?.activationSparsity === true,
|
|
78
|
+
};
|
|
79
|
+
const caps: AdvancedCapabilities = { ...DEFAULT_CAPS, ...options.capabilities };
|
|
80
|
+
const costs: KvCostModel = { ...DEFAULT_COSTS, ...options.costs };
|
|
81
|
+
const scorer: QualityScorer = options.qualityScorer ?? defaultQualityScorer;
|
|
82
|
+
const qualityBaseline = options.qualityBaseline ?? 'fp16';
|
|
83
|
+
const qualityTolerance = options.qualityTolerance ?? DEFAULT_QUALITY_TOLERANCE;
|
|
84
|
+
const draftLen = options.draftLen ?? 4;
|
|
85
|
+
const draftAccuracy = options.draftAccuracy ?? 0.8;
|
|
86
|
+
const preconvertTo = options.preconvertTo ?? 'q4';
|
|
87
|
+
const maxSessions = Math.max(1, options.maxSessions ?? DEFAULT_MAX_SESSIONS);
|
|
88
|
+
// Recently seen session ids, least recently seen first (Map insertion order).
|
|
89
|
+
const sessions = new Map<string, true>();
|
|
90
|
+
|
|
91
|
+
function noteSession(sessionId: string): void {
|
|
92
|
+
sessions.delete(sessionId);
|
|
93
|
+
sessions.set(sessionId, true);
|
|
94
|
+
for (const oldest of sessions.keys()) {
|
|
95
|
+
if (sessions.size <= maxSessions) break;
|
|
96
|
+
sessions.delete(oldest);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function generateOne(request: KvGenerateRequest, batchShare = 1): KvGenerateResult {
|
|
101
|
+
const compat = partsOf(request);
|
|
102
|
+
const encoded = encodeCompatibilityKey(compat);
|
|
103
|
+
if (request.sessionId) noteSession(request.sessionId);
|
|
104
|
+
|
|
105
|
+
const quantPlan = planQuantization({
|
|
106
|
+
native: request.model.quantization,
|
|
107
|
+
backend: 'mock',
|
|
108
|
+
preconvert: flags.preconvertQuant,
|
|
109
|
+
preconvertTo,
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
const applied: OptimizationFlag[] = [];
|
|
113
|
+
let cachedTokens = 0;
|
|
114
|
+
let servedFromCache = false;
|
|
115
|
+
let evictedPages = 0;
|
|
116
|
+
|
|
117
|
+
if (flags.prefixCache) {
|
|
118
|
+
applied.push('prefixCache');
|
|
119
|
+
const hit = cache.lookup(compat, request.tokens);
|
|
120
|
+
if (hit.compatKey === encoded.key && hit.matched > 0) {
|
|
121
|
+
cachedTokens = hit.matched;
|
|
122
|
+
servedFromCache = true;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const computeTokens = Math.max(0, request.tokens.length - cachedTokens) + (request.maxNewTokens ?? 0);
|
|
127
|
+
const speculative = flags.speculativeDecoding && canSpeculative(caps);
|
|
128
|
+
const sparsity = flags.activationSparsity && canActivationSparsity(caps);
|
|
129
|
+
|
|
130
|
+
const decode = decodeCost({
|
|
131
|
+
tokens: computeTokens,
|
|
132
|
+
computePerToken: costs.computePerToken,
|
|
133
|
+
speculative,
|
|
134
|
+
sparsity,
|
|
135
|
+
draftLen,
|
|
136
|
+
draftAccuracy,
|
|
137
|
+
draftPerToken: costs.draftPerToken,
|
|
138
|
+
verifyPerToken: costs.verifyPerToken,
|
|
139
|
+
sparsityFraction: caps.sparsityFraction,
|
|
140
|
+
});
|
|
141
|
+
for (const f of decode.applied) if (!applied.includes(f)) applied.push(f);
|
|
142
|
+
|
|
143
|
+
const hitCost = cachedTokens * costs.cacheHitPerToken;
|
|
144
|
+
const convertCost = quantPlan.converted ? request.tokens.length * costs.conversionPerToken : 0;
|
|
145
|
+
if (quantPlan.converted) applied.push('preconvertQuant');
|
|
146
|
+
|
|
147
|
+
const simulatedMs = (hitCost + decode.cost + convertCost) * batchShare;
|
|
148
|
+
const params = (request.model.paramsB ?? 1.1) * 1e9;
|
|
149
|
+
const bw = bandwidthBytes(params, quantPlan, request.tokens.length, kvDim);
|
|
150
|
+
|
|
151
|
+
const output = request.output ?? mockOutput(request.tokens);
|
|
152
|
+
const baselineOutput = request.baselineOutput ?? output;
|
|
153
|
+
const qualityScore = scorer({
|
|
154
|
+
output,
|
|
155
|
+
baselineOutput,
|
|
156
|
+
quantization: quantPlan.runtime,
|
|
157
|
+
baselineQuantization: qualityBaseline,
|
|
158
|
+
});
|
|
159
|
+
const withinTolerance = qualityScore >= 1 - qualityTolerance;
|
|
160
|
+
|
|
161
|
+
if (flags.prefixCache) {
|
|
162
|
+
const put = cache.put(compat, request.tokens);
|
|
163
|
+
evictedPages = put.evicted;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return {
|
|
167
|
+
output,
|
|
168
|
+
compatibilityKey: encoded.key,
|
|
169
|
+
cachedTokens,
|
|
170
|
+
computeTokens,
|
|
171
|
+
simulatedMs,
|
|
172
|
+
bandwidthBytes: bw,
|
|
173
|
+
qualityScore,
|
|
174
|
+
withinTolerance,
|
|
175
|
+
servedFromCache,
|
|
176
|
+
optimizationsApplied: applied,
|
|
177
|
+
evictedPages,
|
|
178
|
+
quantPlan,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return {
|
|
183
|
+
async generate(request) {
|
|
184
|
+
return generateOne(request, 1);
|
|
185
|
+
},
|
|
186
|
+
|
|
187
|
+
async generateBatch(requests) {
|
|
188
|
+
if (requests.length === 0) return [];
|
|
189
|
+
const continuous = flags.continuousBatching && canContinuousBatch(caps) && requests.length > 1;
|
|
190
|
+
if (!continuous) {
|
|
191
|
+
const out: KvGenerateResult[] = [];
|
|
192
|
+
for (const r of requests) out.push(generateOne(r, 1));
|
|
193
|
+
return out;
|
|
194
|
+
}
|
|
195
|
+
// Shared decode step: each request pays max(seq) / n of the batch cost
|
|
196
|
+
// so the sum is demonstrably cheaper than sequential generate().
|
|
197
|
+
const maxTokens = requests.reduce((m, r) => Math.max(m, r.tokens.length + (r.maxNewTokens ?? 0)), 0);
|
|
198
|
+
const sequential = requests.reduce((s, r) => s + (r.tokens.length + (r.maxNewTokens ?? 0)), 0);
|
|
199
|
+
const share = sequential === 0 ? 1 : maxTokens / sequential;
|
|
200
|
+
return requests.map((r) => {
|
|
201
|
+
const result = generateOne(r, share);
|
|
202
|
+
if (!result.optimizationsApplied.includes('continuousBatching')) {
|
|
203
|
+
result.optimizationsApplied.push('continuousBatching');
|
|
204
|
+
}
|
|
205
|
+
return result;
|
|
206
|
+
});
|
|
207
|
+
},
|
|
208
|
+
|
|
209
|
+
lookup(request) {
|
|
210
|
+
const compat = partsOf(request);
|
|
211
|
+
const hit = cache.lookup(compat, request.tokens);
|
|
212
|
+
return { matched: hit.matched, key: hit.compatKey };
|
|
213
|
+
},
|
|
214
|
+
|
|
215
|
+
setEnabled(flag, enabled) {
|
|
216
|
+
flags[flag] = enabled;
|
|
217
|
+
},
|
|
218
|
+
|
|
219
|
+
isEnabled(flag) {
|
|
220
|
+
return isEnabled(flags, flag);
|
|
221
|
+
},
|
|
222
|
+
|
|
223
|
+
stats(): KvCacheStats {
|
|
224
|
+
return {
|
|
225
|
+
hits: cache.hits,
|
|
226
|
+
misses: cache.misses,
|
|
227
|
+
pages: cache.pages,
|
|
228
|
+
bytes: cache.bytes,
|
|
229
|
+
evictions: cache.evictions,
|
|
230
|
+
sessions: sessions.size,
|
|
231
|
+
};
|
|
232
|
+
},
|
|
233
|
+
|
|
234
|
+
clear() {
|
|
235
|
+
cache.clear();
|
|
236
|
+
sessions.clear();
|
|
237
|
+
},
|
|
238
|
+
};
|
|
239
|
+
}
|