@seanhogg/builderforce-webcontainers 2026.10.12 → 2026.10.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/adaptive/controller.d.ts +39 -0
- package/dist/adaptive/controller.d.ts.map +1 -0
- package/dist/adaptive/controller.js +108 -0
- package/dist/adaptive/controller.js.map +1 -0
- package/dist/adaptive/costModel.d.ts +37 -0
- package/dist/adaptive/costModel.d.ts.map +1 -0
- package/dist/adaptive/costModel.js +102 -0
- package/dist/adaptive/costModel.js.map +1 -0
- package/dist/adaptive/planner.d.ts +22 -0
- package/dist/adaptive/planner.d.ts.map +1 -0
- package/dist/adaptive/planner.js +114 -0
- package/dist/adaptive/planner.js.map +1 -0
- package/dist/adaptive/types.d.ts +138 -0
- package/dist/adaptive/types.d.ts.map +1 -0
- package/dist/adaptive/types.js +12 -0
- package/dist/adaptive/types.js.map +1 -0
- package/dist/assets.js +2 -2
- package/dist/check/cachedFetch.js.map +1 -1
- package/dist/check/index.d.ts +6 -1
- package/dist/check/index.d.ts.map +1 -1
- package/dist/check/index.js +6 -1
- package/dist/check/index.js.map +1 -1
- package/dist/componentCompilers.d.ts.map +1 -1
- package/dist/componentCompilers.js.map +1 -1
- package/dist/esbuildWasm.js.map +1 -1
- package/dist/host.d.ts +11 -5
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +19 -8
- package/dist/host.js.map +1 -1
- package/dist/inference/backends/mock.d.ts +24 -0
- package/dist/inference/backends/mock.d.ts.map +1 -0
- package/dist/inference/backends/mock.js +55 -0
- package/dist/inference/backends/mock.js.map +1 -0
- package/dist/inference/backends/onnx.d.ts +33 -0
- package/dist/inference/backends/onnx.d.ts.map +1 -0
- package/dist/inference/backends/onnx.js +60 -0
- package/dist/inference/backends/onnx.js.map +1 -0
- package/dist/inference/backends/webllm.d.ts +27 -0
- package/dist/inference/backends/webllm.d.ts.map +1 -0
- package/dist/inference/backends/webllm.js +52 -0
- package/dist/inference/backends/webllm.js.map +1 -0
- package/dist/inference/backends/wllama.d.ts +29 -0
- package/dist/inference/backends/wllama.d.ts.map +1 -0
- package/dist/inference/backends/wllama.js +51 -0
- package/dist/inference/backends/wllama.js.map +1 -0
- package/dist/inference/capabilities.d.ts +20 -0
- package/dist/inference/capabilities.d.ts.map +1 -0
- package/dist/inference/capabilities.js +33 -0
- package/dist/inference/capabilities.js.map +1 -0
- package/dist/inference/client.d.ts +45 -0
- package/dist/inference/client.d.ts.map +1 -0
- package/dist/inference/client.js +201 -0
- package/dist/inference/client.js.map +1 -0
- package/dist/inference/modelCache.d.ts +71 -0
- package/dist/inference/modelCache.d.ts.map +1 -0
- package/dist/inference/modelCache.js +91 -0
- package/dist/inference/modelCache.js.map +1 -0
- package/dist/inference/protocol.d.ts +94 -0
- package/dist/inference/protocol.d.ts.map +1 -0
- package/dist/inference/protocol.js +19 -0
- package/dist/inference/protocol.js.map +1 -0
- package/dist/inference/tokenStream.d.ts +26 -0
- package/dist/inference/tokenStream.d.ts.map +1 -0
- package/dist/inference/tokenStream.js +79 -0
- package/dist/inference/tokenStream.js.map +1 -0
- package/dist/inference/types.d.ts +108 -0
- package/dist/inference/types.d.ts.map +1 -0
- package/dist/inference/types.js +8 -0
- package/dist/inference/types.js.map +1 -0
- package/dist/inference/worker.bundle.js +5 -0
- package/dist/inference/worker.d.ts +35 -0
- package/dist/inference/worker.d.ts.map +1 -0
- package/dist/inference/worker.js +157 -0
- package/dist/inference/worker.js.map +1 -0
- package/dist/inference/workerEntry.d.ts +15 -0
- package/dist/inference/workerEntry.d.ts.map +1 -0
- package/dist/inference/workerEntry.js +17 -0
- package/dist/inference/workerEntry.js.map +1 -0
- package/dist/kvcache/advanced.d.ts +39 -0
- package/dist/kvcache/advanced.d.ts.map +1 -0
- package/dist/kvcache/advanced.js +59 -0
- package/dist/kvcache/advanced.js.map +1 -0
- package/dist/kvcache/compatibility.d.ts +14 -0
- package/dist/kvcache/compatibility.d.ts.map +1 -0
- package/dist/kvcache/compatibility.js +44 -0
- package/dist/kvcache/compatibility.js.map +1 -0
- package/dist/kvcache/lru.d.ts +45 -0
- package/dist/kvcache/lru.d.ts.map +1 -0
- package/dist/kvcache/lru.js +91 -0
- package/dist/kvcache/lru.js.map +1 -0
- package/dist/kvcache/paged.d.ts +86 -0
- package/dist/kvcache/paged.d.ts.map +1 -0
- package/dist/kvcache/paged.js +238 -0
- package/dist/kvcache/paged.js.map +1 -0
- package/dist/kvcache/quant.d.ts +30 -0
- package/dist/kvcache/quant.d.ts.map +1 -0
- package/dist/kvcache/quant.js +73 -0
- package/dist/kvcache/quant.js.map +1 -0
- package/dist/kvcache/radix.d.ts +28 -0
- package/dist/kvcache/radix.d.ts.map +1 -0
- package/dist/kvcache/radix.js +58 -0
- package/dist/kvcache/radix.js.map +1 -0
- package/dist/kvcache/runtime.d.ts +11 -0
- package/dist/kvcache/runtime.d.ts.map +1 -0
- package/dist/kvcache/runtime.js +204 -0
- package/dist/kvcache/runtime.js.map +1 -0
- package/dist/kvcache/types.d.ts +151 -0
- package/dist/kvcache/types.d.ts.map +1 -0
- package/dist/kvcache/types.js +10 -0
- package/dist/kvcache/types.js.map +1 -0
- package/dist/learning/fingerprint.d.ts +17 -0
- package/dist/learning/fingerprint.d.ts.map +1 -0
- package/dist/learning/fingerprint.js +46 -0
- package/dist/learning/fingerprint.js.map +1 -0
- package/dist/learning/index.d.ts +5 -0
- package/dist/learning/index.d.ts.map +1 -0
- package/dist/learning/index.js +3 -0
- package/dist/learning/index.js.map +1 -0
- package/dist/learning/learner.d.ts +21 -0
- package/dist/learning/learner.d.ts.map +1 -0
- package/dist/learning/learner.js +272 -0
- package/dist/learning/learner.js.map +1 -0
- package/dist/learning/ledger.d.ts +26 -0
- package/dist/learning/ledger.d.ts.map +1 -0
- package/dist/learning/ledger.js +40 -0
- package/dist/learning/ledger.js.map +1 -0
- package/dist/learning/selection.d.ts +44 -0
- package/dist/learning/selection.d.ts.map +1 -0
- package/dist/learning/selection.js +111 -0
- package/dist/learning/selection.js.map +1 -0
- package/dist/learning/stats.d.ts +19 -0
- package/dist/learning/stats.d.ts.map +1 -0
- package/dist/learning/stats.js +31 -0
- package/dist/learning/stats.js.map +1 -0
- package/dist/learning/types.d.ts +133 -0
- package/dist/learning/types.d.ts.map +1 -0
- package/dist/learning/types.js +8 -0
- package/dist/learning/types.js.map +1 -0
- package/dist/memory/envProbe.d.ts +16 -0
- package/dist/memory/envProbe.d.ts.map +1 -0
- package/dist/memory/envProbe.js +28 -0
- package/dist/memory/envProbe.js.map +1 -0
- package/dist/memory/estimates.d.ts +46 -0
- package/dist/memory/estimates.d.ts.map +1 -0
- package/dist/memory/estimates.js +72 -0
- package/dist/memory/estimates.js.map +1 -0
- package/dist/memory/eviction.d.ts +42 -0
- package/dist/memory/eviction.d.ts.map +1 -0
- package/dist/memory/eviction.js +143 -0
- package/dist/memory/eviction.js.map +1 -0
- package/dist/memory/manager.d.ts +34 -0
- package/dist/memory/manager.d.ts.map +1 -0
- package/dist/memory/manager.js +346 -0
- package/dist/memory/manager.js.map +1 -0
- package/dist/memory/managerTypes.d.ts +49 -0
- package/dist/memory/managerTypes.d.ts.map +1 -0
- package/dist/memory/managerTypes.js +7 -0
- package/dist/memory/managerTypes.js.map +1 -0
- package/dist/memory/pressure.d.ts +56 -0
- package/dist/memory/pressure.d.ts.map +1 -0
- package/dist/memory/pressure.js +80 -0
- package/dist/memory/pressure.js.map +1 -0
- package/dist/memory/spillTier.d.ts +47 -0
- package/dist/memory/spillTier.d.ts.map +1 -0
- package/dist/memory/spillTier.js +128 -0
- package/dist/memory/spillTier.js.map +1 -0
- package/dist/memory/storageSink.d.ts +69 -0
- package/dist/memory/storageSink.d.ts.map +1 -0
- package/dist/memory/storageSink.js +221 -0
- package/dist/memory/storageSink.js.map +1 -0
- package/dist/memory/types.d.ts +202 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +15 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/memory/victims.d.ts +32 -0
- package/dist/memory/victims.d.ts.map +1 -0
- package/dist/memory/victims.js +74 -0
- package/dist/memory/victims.js.map +1 -0
- package/dist/memory/workspace.d.ts +34 -0
- package/dist/memory/workspace.d.ts.map +1 -0
- package/dist/memory/workspace.js +55 -0
- package/dist/memory/workspace.js.map +1 -0
- package/dist/memoryIntegration/index.d.ts +8 -0
- package/dist/memoryIntegration/index.d.ts.map +1 -0
- package/dist/memoryIntegration/index.js +4 -0
- package/dist/memoryIntegration/index.js.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts +43 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.js +285 -0
- package/dist/memoryIntegration/knowledgeMemory.js.map +1 -0
- package/dist/memoryIntegration/persistence.d.ts +15 -0
- package/dist/memoryIntegration/persistence.d.ts.map +1 -0
- package/dist/memoryIntegration/persistence.js +43 -0
- package/dist/memoryIntegration/persistence.js.map +1 -0
- package/dist/memoryIntegration/text.d.ts +57 -0
- package/dist/memoryIntegration/text.d.ts.map +1 -0
- package/dist/memoryIntegration/text.js +179 -0
- package/dist/memoryIntegration/text.js.map +1 -0
- package/dist/memoryIntegration/types.d.ts +131 -0
- package/dist/memoryIntegration/types.d.ts.map +1 -0
- package/dist/memoryIntegration/types.js +11 -0
- package/dist/memoryIntegration/types.js.map +1 -0
- package/dist/ml.d.ts +52 -0
- package/dist/ml.d.ts.map +1 -0
- package/dist/ml.js +33 -0
- package/dist/ml.js.map +1 -0
- package/dist/node/childRelay.d.ts +10 -0
- package/dist/node/childRelay.d.ts.map +1 -1
- package/dist/node/childRelay.js +12 -2
- package/dist/node/childRelay.js.map +1 -1
- package/dist/node/index.js.map +1 -1
- package/dist/node/portResponder.d.ts +2 -0
- package/dist/node/portResponder.d.ts.map +1 -1
- package/dist/node/portResponder.js +3 -3
- package/dist/node/portResponder.js.map +1 -1
- package/dist/node/processHost.d.ts +4 -1
- package/dist/node/processHost.d.ts.map +1 -1
- package/dist/node/processHost.js +19 -16
- package/dist/node/processHost.js.map +1 -1
- package/dist/node/protocol.d.ts +1 -0
- package/dist/node/protocol.d.ts.map +1 -1
- package/dist/node/responseWire.d.ts.map +1 -1
- package/dist/node/worker.js +127 -120
- package/dist/node/workerProcess.js +1 -1
- package/dist/node/workerProcess.js.map +1 -1
- package/dist/prefetch/admission.d.ts +41 -0
- package/dist/prefetch/admission.d.ts.map +1 -0
- package/dist/prefetch/admission.js +49 -0
- package/dist/prefetch/admission.js.map +1 -0
- package/dist/prefetch/loadRegistry.d.ts +56 -0
- package/dist/prefetch/loadRegistry.d.ts.map +1 -0
- package/dist/prefetch/loadRegistry.js +108 -0
- package/dist/prefetch/loadRegistry.js.map +1 -0
- package/dist/prefetch/predictor.d.ts +40 -0
- package/dist/prefetch/predictor.d.ts.map +1 -0
- package/dist/prefetch/predictor.js +178 -0
- package/dist/prefetch/predictor.js.map +1 -0
- package/dist/prefetch/prefetcher.d.ts +37 -0
- package/dist/prefetch/prefetcher.d.ts.map +1 -0
- package/dist/prefetch/prefetcher.js +339 -0
- package/dist/prefetch/prefetcher.js.map +1 -0
- package/dist/prefetch/residentStore.d.ts +49 -0
- package/dist/prefetch/residentStore.d.ts.map +1 -0
- package/dist/prefetch/residentStore.js +87 -0
- package/dist/prefetch/residentStore.js.map +1 -0
- package/dist/prefetch/selfTuner.d.ts +40 -0
- package/dist/prefetch/selfTuner.d.ts.map +1 -0
- package/dist/prefetch/selfTuner.js +78 -0
- package/dist/prefetch/selfTuner.js.map +1 -0
- package/dist/prefetch/stats.d.ts +52 -0
- package/dist/prefetch/stats.d.ts.map +1 -0
- package/dist/prefetch/stats.js +72 -0
- package/dist/prefetch/stats.js.map +1 -0
- package/dist/prefetch/streaming.d.ts +132 -0
- package/dist/prefetch/streaming.d.ts.map +1 -0
- package/dist/prefetch/streaming.js +141 -0
- package/dist/prefetch/streaming.js.map +1 -0
- package/dist/prefetch/targetIndex.d.ts +18 -0
- package/dist/prefetch/targetIndex.d.ts.map +1 -0
- package/dist/prefetch/targetIndex.js +32 -0
- package/dist/prefetch/targetIndex.js.map +1 -0
- package/dist/prefetch/types.d.ts +283 -0
- package/dist/prefetch/types.d.ts.map +1 -0
- package/dist/prefetch/types.js +19 -0
- package/dist/prefetch/types.js.map +1 -0
- package/dist/previewBridge.d.ts +6 -0
- package/dist/previewBridge.d.ts.map +1 -1
- package/dist/previewBridge.js +25 -5
- package/dist/previewBridge.js.map +1 -1
- package/dist/previewMessages.d.ts +4 -3
- package/dist/previewMessages.d.ts.map +1 -1
- package/dist/previewMessages.js +12 -6
- package/dist/previewMessages.js.map +1 -1
- package/dist/process-worker.js +127 -120
- package/dist/relay.html +1 -1
- package/dist/shared/clock.d.ts +7 -0
- package/dist/shared/clock.d.ts.map +1 -0
- package/dist/shared/clock.js +10 -0
- package/dist/shared/clock.js.map +1 -0
- package/dist/shared/hardwareProbe.d.ts +26 -0
- package/dist/shared/hardwareProbe.d.ts.map +1 -0
- package/dist/shared/hardwareProbe.js +62 -0
- package/dist/shared/hardwareProbe.js.map +1 -0
- package/dist/shared/idb.d.ts +41 -0
- package/dist/shared/idb.d.ts.map +1 -0
- package/dist/shared/idb.js +184 -0
- package/dist/shared/idb.js.map +1 -0
- package/dist/shared/tokens.d.ts +7 -0
- package/dist/shared/tokens.d.ts.map +1 -0
- package/dist/shared/tokens.js +9 -0
- package/dist/shared/tokens.js.map +1 -0
- package/dist/snapshotStore.d.ts +2 -1
- package/dist/snapshotStore.d.ts.map +1 -1
- package/dist/snapshotStore.js +27 -12
- package/dist/snapshotStore.js.map +1 -1
- package/dist/styleCompilers.d.ts.map +1 -1
- package/dist/styleCompilers.js.map +1 -1
- package/dist/transport.js.map +1 -1
- package/package.json +11 -5
- package/src/adaptive/controller.ts +151 -0
- package/src/adaptive/costModel.ts +131 -0
- package/src/adaptive/planner.ts +127 -0
- package/src/adaptive/types.ts +137 -0
- package/src/check/index.ts +6 -1
- package/src/host.ts +33 -11
- package/src/inference/backends/mock.ts +77 -0
- package/src/inference/backends/onnx.ts +91 -0
- package/src/inference/backends/webllm.ts +79 -0
- package/src/inference/backends/wllama.ts +79 -0
- package/src/inference/capabilities.ts +46 -0
- package/src/inference/client.ts +276 -0
- package/src/inference/modelCache.ts +145 -0
- package/src/inference/protocol.ts +119 -0
- package/src/inference/tokenStream.ts +92 -0
- package/src/inference/types.ts +115 -0
- package/src/inference/worker.ts +191 -0
- package/src/inference/workerEntry.ts +18 -0
- package/src/kvcache/advanced.ts +86 -0
- package/src/kvcache/compatibility.ts +51 -0
- package/src/kvcache/lru.ts +100 -0
- package/src/kvcache/paged.ts +270 -0
- package/src/kvcache/quant.ts +88 -0
- package/src/kvcache/radix.ts +68 -0
- package/src/kvcache/runtime.ts +239 -0
- package/src/kvcache/types.ts +167 -0
- package/src/learning/fingerprint.ts +50 -0
- package/src/learning/index.ts +15 -0
- package/src/learning/learner.ts +304 -0
- package/src/learning/ledger.ts +54 -0
- package/src/learning/selection.ts +137 -0
- package/src/learning/stats.ts +39 -0
- package/src/learning/types.ts +142 -0
- package/src/memory/envProbe.ts +30 -0
- package/src/memory/estimates.ts +98 -0
- package/src/memory/eviction.ts +145 -0
- package/src/memory/manager.ts +380 -0
- package/src/memory/managerTypes.ts +61 -0
- package/src/memory/pressure.ts +120 -0
- package/src/memory/spillTier.ts +159 -0
- package/src/memory/storageSink.ts +289 -0
- package/src/memory/types.ts +217 -0
- package/src/memory/victims.ts +80 -0
- package/src/memory/workspace.ts +74 -0
- package/src/memoryIntegration/index.ts +7 -0
- package/src/memoryIntegration/knowledgeMemory.ts +339 -0
- package/src/memoryIntegration/persistence.ts +54 -0
- package/src/memoryIntegration/text.ts +185 -0
- package/src/memoryIntegration/types.ts +145 -0
- package/src/ml.ts +180 -0
- package/src/node/childRelay.ts +16 -3
- package/src/node/portResponder.ts +5 -3
- package/src/node/processHost.ts +16 -15
- package/src/node/protocol.ts +1 -1
- package/src/node/workerProcess.ts +1 -1
- package/src/prefetch/admission.ts +74 -0
- package/src/prefetch/loadRegistry.ts +156 -0
- package/src/prefetch/predictor.ts +199 -0
- package/src/prefetch/prefetcher.ts +369 -0
- package/src/prefetch/residentStore.ts +121 -0
- package/src/prefetch/selfTuner.ts +110 -0
- package/src/prefetch/stats.ts +120 -0
- package/src/prefetch/streaming.ts +231 -0
- package/src/prefetch/targetIndex.ts +40 -0
- package/src/prefetch/types.ts +307 -0
- package/src/previewBridge.ts +25 -5
- package/src/previewMessages.ts +11 -6
- package/src/shared/clock.ts +9 -0
- package/src/shared/hardwareProbe.ts +68 -0
- package/src/shared/idb.ts +203 -0
- package/src/shared/tokens.ts +8 -0
- package/src/snapshotStore.ts +25 -10
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: model and capability types.
|
|
3
|
+
*
|
|
4
|
+
* The runtime abstraction lives in `types.ts`; this file holds the shared
|
|
5
|
+
* shapes that cross the worker boundary and the capability probe results.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/** Quantization formats a backend may support. */
|
|
9
|
+
export type Quantization = 'q4' | 'q5' | 'q8';
|
|
10
|
+
|
|
11
|
+
/** A reference to a local model asset. */
|
|
12
|
+
export interface ModelRef {
|
|
13
|
+
/** Unique model identifier, e.g. `builderforce/tiny-instruct-q4`. */
|
|
14
|
+
id: string;
|
|
15
|
+
/** Parameter count (billions), used for tier recommendations. */
|
|
16
|
+
paramsB: number;
|
|
17
|
+
quantization: Quantization;
|
|
18
|
+
/** Number of weights in the model (informational). */
|
|
19
|
+
numWeights?: number;
|
|
20
|
+
/** Where the model bytes live. `url` is fetched once at provisioning time. */
|
|
21
|
+
source?: { kind: 'url'; url: string; bytes?: number };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Runtime/backend kinds the abstraction can host. */
|
|
25
|
+
export type BackendKind = 'onnx' | 'wllama' | 'webllm' | 'mock';
|
|
26
|
+
|
|
27
|
+
/** Backend + capability metadata returned by `probeCapabilities()`. */
|
|
28
|
+
export interface BackendInfo {
|
|
29
|
+
kind: BackendKind;
|
|
30
|
+
available: boolean;
|
|
31
|
+
/** Human-readable reason when unavailable (e.g. "WebGPU not present"). */
|
|
32
|
+
reason?: string;
|
|
33
|
+
/** WASM SIMD usable by this backend (best-effort probe). */
|
|
34
|
+
wasmSimd: boolean;
|
|
35
|
+
/** WebGPU usable by this backend. */
|
|
36
|
+
webgpu: boolean;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Everything the runtime could detect about the environment. */
|
|
40
|
+
export interface RuntimeCapabilities {
|
|
41
|
+
wasm: boolean;
|
|
42
|
+
wasmSimd: boolean;
|
|
43
|
+
webgpu: boolean;
|
|
44
|
+
crossOriginIsolated: boolean;
|
|
45
|
+
sharedArrayBuffer: boolean;
|
|
46
|
+
backends: BackendInfo[];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** A generated token, streamed back to the caller. */
|
|
50
|
+
export interface InferenceToken {
|
|
51
|
+
token: string;
|
|
52
|
+
/** Cumulative tokens generated so far in this request. */
|
|
53
|
+
index: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Final stats for one `generate()` call. */
|
|
57
|
+
export interface GenerationStats {
|
|
58
|
+
/** Prompt length in tokens: the backend's tokenizer count, or the ~4 chars/token estimate (see `promptTokensEstimated`). */
|
|
59
|
+
promptTokens: number;
|
|
60
|
+
/** True when `promptTokens` is the heuristic estimate because the backend reports no count. */
|
|
61
|
+
promptTokensEstimated: boolean;
|
|
62
|
+
generatedTokens: number;
|
|
63
|
+
/** Wall-clock ms from generate() call to first token. */
|
|
64
|
+
ttftMs: number;
|
|
65
|
+
/** Wall-clock ms from first token to stream end. */
|
|
66
|
+
decodeMs: number;
|
|
67
|
+
tokensPerSecond: number;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** One message in a chat-style prompt. */
|
|
71
|
+
export interface ChatMessage {
|
|
72
|
+
role: 'system' | 'user' | 'assistant';
|
|
73
|
+
content: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Options accepted by `generate()`. */
|
|
77
|
+
export interface GenerateOptions {
|
|
78
|
+
maxTokens?: number;
|
|
79
|
+
temperature?: number;
|
|
80
|
+
signal?: AbortSignal;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Public API of a loaded, ready-to-generate model instance. */
|
|
84
|
+
export interface InferenceModel {
|
|
85
|
+
readonly ref: ModelRef;
|
|
86
|
+
generate(messages: ChatMessage[], options?: GenerateOptions): AsyncIterable<InferenceToken>;
|
|
87
|
+
/** Release backend resources; the instance is unusable after. */
|
|
88
|
+
dispose(): Promise<void>;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* A backend plugin: it can report availability, load a model from cached
|
|
93
|
+
* bytes, and generate tokens. The worker loop in `inference/worker.ts`
|
|
94
|
+
* drives a single backend at a time; the host-side `InferenceClient`
|
|
95
|
+
* (`inference/client.ts`) is what callers use.
|
|
96
|
+
*/
|
|
97
|
+
export interface InferenceBackend {
|
|
98
|
+
readonly kind: BackendKind;
|
|
99
|
+
probe(): Promise<BackendInfo>;
|
|
100
|
+
load(model: ModelRef, weights: ArrayBuffer, context: BackendContext): Promise<void>;
|
|
101
|
+
generate(messages: ChatMessage[], options?: GenerateOptions): AsyncIterable<InferenceToken>;
|
|
102
|
+
/** Exact prompt token count from the loaded model's tokenizer, when the backend has one. */
|
|
103
|
+
countPromptTokens?(messages: ChatMessage[]): number;
|
|
104
|
+
dispose(): Promise<void>;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** What a backend needs from its host environment. */
|
|
108
|
+
export interface BackendContext {
|
|
109
|
+
/** Log lines go to the host as worker log messages (debugging only). */
|
|
110
|
+
log(message: string): void;
|
|
111
|
+
/** True when the host supports the SharedArrayBuffer fast path. */
|
|
112
|
+
sharedArrayBuffer: boolean;
|
|
113
|
+
wasmSimd: boolean;
|
|
114
|
+
webgpu: boolean;
|
|
115
|
+
}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: worker host (runs inside the Worker).
|
|
3
|
+
*
|
|
4
|
+
* Owns the backend registry, drives load/generate/cancel, and streams tokens
|
|
5
|
+
* back to the `InferenceClient` over postMessage. Generation runs here, on
|
|
6
|
+
* the worker thread, so the UI/main thread never blocks on decode. The
|
|
7
|
+
* worker global is typed locally (as in `sw.ts` / `check/worker.ts`) because
|
|
8
|
+
* `lib.webworker` cannot share a compilation with the DOM lib.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
INFERENCE_CANCEL,
|
|
13
|
+
INFERENCE_DISPOSE_MODEL,
|
|
14
|
+
INFERENCE_DONE,
|
|
15
|
+
INFERENCE_ERROR,
|
|
16
|
+
INFERENCE_GENERATE,
|
|
17
|
+
INFERENCE_LOAD,
|
|
18
|
+
INFERENCE_LOG,
|
|
19
|
+
INFERENCE_PROBE,
|
|
20
|
+
INFERENCE_READY,
|
|
21
|
+
INFERENCE_TOKEN,
|
|
22
|
+
type DoneReply,
|
|
23
|
+
type GenerateRequest,
|
|
24
|
+
type InferenceReply,
|
|
25
|
+
type InferenceRequest,
|
|
26
|
+
} from './protocol.js';
|
|
27
|
+
import { probeRuntimeCapabilities } from './capabilities.js';
|
|
28
|
+
import { detectSharedArrayBuffer, detectWasmSimd, detectWebGpu } from '../shared/hardwareProbe.js';
|
|
29
|
+
import { estimateTokens } from '../shared/tokens.js';
|
|
30
|
+
import type { BackendContext, ChatMessage, InferenceBackend } from './types.js';
|
|
31
|
+
|
|
32
|
+
interface WorkerGlobalLike {
|
|
33
|
+
postMessage(message: InferenceReply): void;
|
|
34
|
+
addEventListener(type: 'message', listener: (event: { data: unknown }) => void): void;
|
|
35
|
+
removeEventListener?(type: 'message', listener: (event: { data: unknown }) => void): void;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface InferenceWorkerHostOptions {
|
|
39
|
+
/** Backends in preference order; the first available one is used. */
|
|
40
|
+
backends: InferenceBackend[];
|
|
41
|
+
now?: () => number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Prompt size: the backend's tokenizer count when it has one, else the shared estimate. */
|
|
45
|
+
function countPrompt(backend: InferenceBackend, messages: ChatMessage[]): { tokens: number; estimated: boolean } {
|
|
46
|
+
const exact = backend.countPromptTokens?.(messages);
|
|
47
|
+
if (typeof exact === 'number' && Number.isFinite(exact) && exact >= 0) return { tokens: exact, estimated: false };
|
|
48
|
+
return { tokens: messages.reduce((n, m) => n + estimateTokens(m.content), 0), estimated: true };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Wire the host into a worker global. Returns a dispose function that
|
|
53
|
+
* cancels any in-flight generation. Exported for tests, which drive it with
|
|
54
|
+
* a fake worker global.
|
|
55
|
+
*/
|
|
56
|
+
export function startInferenceWorkerHost(
|
|
57
|
+
worker: WorkerGlobalLike,
|
|
58
|
+
options: InferenceWorkerHostOptions,
|
|
59
|
+
): () => void {
|
|
60
|
+
const { backends } = options;
|
|
61
|
+
const now = options.now ?? (() => Date.now());
|
|
62
|
+
const context: BackendContext = {
|
|
63
|
+
log: (message) => worker.postMessage({ type: INFERENCE_LOG, id: -1, message }),
|
|
64
|
+
get sharedArrayBuffer() {
|
|
65
|
+
return detectSharedArrayBuffer();
|
|
66
|
+
},
|
|
67
|
+
get wasmSimd() {
|
|
68
|
+
return detectWasmSimd();
|
|
69
|
+
},
|
|
70
|
+
get webgpu() {
|
|
71
|
+
return detectWebGpu();
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
let active: InferenceBackend | undefined;
|
|
76
|
+
let activeAbort: AbortController | undefined;
|
|
77
|
+
let activeRequestId: number | undefined;
|
|
78
|
+
|
|
79
|
+
const reply = (message: InferenceReply) => worker.postMessage(message);
|
|
80
|
+
const fail = (id: number, error: unknown) =>
|
|
81
|
+
reply({ type: INFERENCE_ERROR, id, message: error instanceof Error ? error.message : String(error) });
|
|
82
|
+
|
|
83
|
+
async function selectBackend(): Promise<InferenceBackend> {
|
|
84
|
+
for (const backend of backends) {
|
|
85
|
+
const info = await backend.probe();
|
|
86
|
+
if (info.available) return backend;
|
|
87
|
+
}
|
|
88
|
+
throw new Error('no inference backend available in this environment');
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Abort any running generation and release the loaded model. */
|
|
92
|
+
async function releaseActive(): Promise<void> {
|
|
93
|
+
activeAbort?.abort();
|
|
94
|
+
const previous = active;
|
|
95
|
+
active = undefined;
|
|
96
|
+
await previous?.dispose();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async function handleGenerate(id: number, request: GenerateRequest) {
|
|
100
|
+
const backend = active;
|
|
101
|
+
if (!backend) throw new Error('no model loaded');
|
|
102
|
+
activeAbort?.abort();
|
|
103
|
+
const abort = new AbortController();
|
|
104
|
+
activeAbort = abort;
|
|
105
|
+
activeRequestId = id;
|
|
106
|
+
const started = now();
|
|
107
|
+
let firstTokenAt = -1;
|
|
108
|
+
let generated = 0;
|
|
109
|
+
try {
|
|
110
|
+
// The wire options never carry a signal (not cloneable); the worker-side
|
|
111
|
+
// controller is the ONE cancellation source (cancel request / next generate).
|
|
112
|
+
for await (const { token, index } of backend.generate(request.messages, {
|
|
113
|
+
...request.options,
|
|
114
|
+
signal: abort.signal,
|
|
115
|
+
})) {
|
|
116
|
+
if (firstTokenAt < 0) firstTokenAt = now();
|
|
117
|
+
generated++;
|
|
118
|
+
worker.postMessage({ type: INFERENCE_TOKEN, id, requestId: id, token, index });
|
|
119
|
+
}
|
|
120
|
+
const finished = now();
|
|
121
|
+
const ttftMs = firstTokenAt >= 0 ? firstTokenAt - started : finished - started;
|
|
122
|
+
const decodeMs = firstTokenAt >= 0 ? finished - firstTokenAt : 0;
|
|
123
|
+
const prompt = countPrompt(backend, request.messages);
|
|
124
|
+
const done: DoneReply = {
|
|
125
|
+
type: INFERENCE_DONE,
|
|
126
|
+
id,
|
|
127
|
+
requestId: id,
|
|
128
|
+
promptTokens: prompt.tokens,
|
|
129
|
+
promptTokensEstimated: prompt.estimated,
|
|
130
|
+
generatedTokens: generated,
|
|
131
|
+
ttftMs,
|
|
132
|
+
decodeMs,
|
|
133
|
+
tokensPerSecond: decodeMs > 0 ? (generated / decodeMs) * 1000 : 0,
|
|
134
|
+
};
|
|
135
|
+
reply(done);
|
|
136
|
+
} finally {
|
|
137
|
+
if (activeAbort === abort) {
|
|
138
|
+
activeAbort = undefined;
|
|
139
|
+
activeRequestId = undefined;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const listener = (event: { data: unknown }) => {
|
|
145
|
+
const request = event.data as InferenceRequest | null;
|
|
146
|
+
if (!request || typeof request !== 'object') return;
|
|
147
|
+
const id = request.id;
|
|
148
|
+
void (async () => {
|
|
149
|
+
try {
|
|
150
|
+
switch (request.type) {
|
|
151
|
+
case INFERENCE_PROBE: {
|
|
152
|
+
const capabilities = probeRuntimeCapabilities(backends);
|
|
153
|
+
reply({ type: INFERENCE_READY, id, capabilities });
|
|
154
|
+
break;
|
|
155
|
+
}
|
|
156
|
+
case INFERENCE_LOAD: {
|
|
157
|
+
// Abort + dispose the previous model BEFORE loading the next, so two
|
|
158
|
+
// models are never resident and no stale generation keeps decoding.
|
|
159
|
+
await releaseActive();
|
|
160
|
+
const next = await selectBackend();
|
|
161
|
+
await next.load(request.ref, request.weights, context);
|
|
162
|
+
active = next;
|
|
163
|
+
reply({ type: INFERENCE_READY, id, capabilities: probeRuntimeCapabilities(backends) });
|
|
164
|
+
break;
|
|
165
|
+
}
|
|
166
|
+
case INFERENCE_GENERATE:
|
|
167
|
+
await handleGenerate(id, request);
|
|
168
|
+
break;
|
|
169
|
+
case INFERENCE_CANCEL:
|
|
170
|
+
if (request.requestId === undefined || request.requestId === activeRequestId) activeAbort?.abort();
|
|
171
|
+
break;
|
|
172
|
+
case INFERENCE_DISPOSE_MODEL: {
|
|
173
|
+
await releaseActive();
|
|
174
|
+
reply({ type: INFERENCE_READY, id, capabilities: probeRuntimeCapabilities(backends) });
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
} catch (error) {
|
|
179
|
+
fail(id, error);
|
|
180
|
+
}
|
|
181
|
+
})();
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
worker.addEventListener('message', listener);
|
|
185
|
+
return () => worker.removeEventListener?.('message', listener as never);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Entry point used by the real worker bundle. */
|
|
189
|
+
export function runInferenceWorker(backends: InferenceBackend[]): void {
|
|
190
|
+
startInferenceWorkerHost(globalThis as unknown as WorkerGlobalLike, { backends });
|
|
191
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: worker entry.
|
|
3
|
+
*
|
|
4
|
+
* Bundled to `dist/inference/worker.bundle.js` (see `scripts/build-sw.mjs`;
|
|
5
|
+
* exported as `@seanhogg/builderforce-webcontainers/ml/worker.js`). Backends
|
|
6
|
+
* are listed in preference order; `mock` is last so a real backend always
|
|
7
|
+
* wins when its module is present. The ONNX/Wllama/WebLLM adapters take an
|
|
8
|
+
* injected `loadModule()` so this package stays dependency-free — a host app
|
|
9
|
+
* that installs `onnxruntime-web` (or wllama / `@mlc-ai/web-llm`) and wants
|
|
10
|
+
* the worker to use it wires that in here. For now the worker runs the
|
|
11
|
+
* deterministic mock backend, which is what the tests and the
|
|
12
|
+
* offline/caching paths exercise.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { runInferenceWorker } from './worker.js';
|
|
16
|
+
import { createMockBackend } from './backends/mock.js';
|
|
17
|
+
|
|
18
|
+
runInferenceWorker([createMockBackend()]);
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — optional advanced inference, gated by explicit opt-in flags
|
|
3
|
+
* AND capability checks. Architecture-specific optimizations are never
|
|
4
|
+
* applied to incompatible models; the plain path is the default.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { AdvancedCapabilities, OptimizationFlag } from './types.js';
|
|
8
|
+
|
|
9
|
+
const DECODER_FAMILIES = new Set(['decoder-transformer', 'llama', 'gpt', 'mistral', 'qwen']);
|
|
10
|
+
|
|
11
|
+
export function isEnabled(
|
|
12
|
+
flags: Partial<Record<OptimizationFlag, boolean>> | undefined,
|
|
13
|
+
flag: OptimizationFlag,
|
|
14
|
+
): boolean {
|
|
15
|
+
return flags?.[flag] === true;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function canSpeculative(caps: AdvancedCapabilities): boolean {
|
|
19
|
+
return (
|
|
20
|
+
caps.speculativeDecoding &&
|
|
21
|
+
caps.hasDraftModel &&
|
|
22
|
+
DECODER_FAMILIES.has(caps.architecture)
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function canContinuousBatch(caps: AdvancedCapabilities): boolean {
|
|
27
|
+
return caps.continuousBatching && DECODER_FAMILIES.has(caps.architecture);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function canActivationSparsity(caps: AdvancedCapabilities): boolean {
|
|
31
|
+
return caps.activationSparsity && caps.sparsityFraction > 0 && DECODER_FAMILIES.has(caps.architecture);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Simulated decode cost in abstract time units.
|
|
36
|
+
* Speculative: draft + verify of a block; accepted tokens skip full compute.
|
|
37
|
+
* Sparsity: compute scaled by (1 - sparsityFraction).
|
|
38
|
+
* Continuous batching is applied at the batch level (see runtime).
|
|
39
|
+
*/
|
|
40
|
+
export function decodeCost(input: {
|
|
41
|
+
tokens: number;
|
|
42
|
+
computePerToken: number;
|
|
43
|
+
speculative: boolean;
|
|
44
|
+
sparsity: boolean;
|
|
45
|
+
draftLen: number;
|
|
46
|
+
draftAccuracy: number;
|
|
47
|
+
draftPerToken: number;
|
|
48
|
+
verifyPerToken: number;
|
|
49
|
+
sparsityFraction: number;
|
|
50
|
+
}): { cost: number; applied: OptimizationFlag[] } {
|
|
51
|
+
const applied: OptimizationFlag[] = [];
|
|
52
|
+
let per = input.computePerToken;
|
|
53
|
+
if (input.sparsity) {
|
|
54
|
+
per *= Math.max(0, 1 - input.sparsityFraction);
|
|
55
|
+
applied.push('activationSparsity');
|
|
56
|
+
}
|
|
57
|
+
if (!input.speculative || input.tokens <= 0) {
|
|
58
|
+
return { cost: input.tokens * per, applied };
|
|
59
|
+
}
|
|
60
|
+
applied.push('speculativeDecoding');
|
|
61
|
+
const block = Math.max(1, input.draftLen);
|
|
62
|
+
const blocks = Math.ceil(input.tokens / block);
|
|
63
|
+
const accepted = Math.min(1, Math.max(0, input.draftAccuracy));
|
|
64
|
+
// Each block: draft `block` tokens cheaply, then verify. Accepted tokens
|
|
65
|
+
// pay only draft+verify; rejected tokens fall back to full compute.
|
|
66
|
+
const draftCost = blocks * block * input.draftPerToken;
|
|
67
|
+
const verifyCost = blocks * input.verifyPerToken;
|
|
68
|
+
const rejectCost = (1 - accepted) * input.tokens * per;
|
|
69
|
+
const acceptCost = accepted * input.tokens * input.verifyPerToken;
|
|
70
|
+
return { cost: draftCost + verifyCost + rejectCost + acceptCost, applied };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function batchCost(input: {
|
|
74
|
+
tokenCounts: number[];
|
|
75
|
+
computePerToken: number;
|
|
76
|
+
continuous: boolean;
|
|
77
|
+
}): { cost: number; applied: OptimizationFlag[] } {
|
|
78
|
+
const sum = input.tokenCounts.reduce((a, b) => a + b, 0);
|
|
79
|
+
if (!input.continuous || input.tokenCounts.length <= 1) {
|
|
80
|
+
return { cost: sum * input.computePerToken, applied: [] };
|
|
81
|
+
}
|
|
82
|
+
// Continuous batching: shared prefill/decode step, cost ≈ max sequence
|
|
83
|
+
// rather than the sum. Demonstrably cheaper than sequential.
|
|
84
|
+
const max = input.tokenCounts.reduce((a, b) => Math.max(a, b), 0);
|
|
85
|
+
return { cost: max * input.computePerToken, applied: ['continuousBatching'] };
|
|
86
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — compatibility key for EXACT-prefix KV reuse.
|
|
3
|
+
*
|
|
4
|
+
* A cache hit is allowed only when model weights, tokenizer, positional
|
|
5
|
+
* config, attention config, AND the exact preceding token prefix all match.
|
|
6
|
+
* Semantic retrieval is never treated as KV reuse.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { CompatibilityKey, CompatibilityParts } from './types.js';
|
|
10
|
+
|
|
11
|
+
const SEP = '\x1f';
|
|
12
|
+
|
|
13
|
+
export function encodeCompatibilityKey(parts: CompatibilityParts): CompatibilityKey {
|
|
14
|
+
const key = [
|
|
15
|
+
parts.modelId,
|
|
16
|
+
parts.weightsHash,
|
|
17
|
+
parts.tokenizerId,
|
|
18
|
+
parts.positionalConfig,
|
|
19
|
+
parts.attentionConfig,
|
|
20
|
+
].join(SEP);
|
|
21
|
+
return { ...parts, key };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function parseCompatibilityKey(key: string): CompatibilityParts {
|
|
25
|
+
const [modelId, weightsHash, tokenizerId, positionalConfig, attentionConfig] = key.split(SEP);
|
|
26
|
+
return {
|
|
27
|
+
modelId: modelId ?? '',
|
|
28
|
+
weightsHash: weightsHash ?? '',
|
|
29
|
+
tokenizerId: tokenizerId ?? '',
|
|
30
|
+
positionalConfig: positionalConfig ?? '',
|
|
31
|
+
attentionConfig: attentionConfig ?? '',
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function compatibilityEquals(a: CompatibilityParts, b: CompatibilityParts): boolean {
|
|
36
|
+
return (
|
|
37
|
+
a.modelId === b.modelId &&
|
|
38
|
+
a.weightsHash === b.weightsHash &&
|
|
39
|
+
a.tokenizerId === b.tokenizerId &&
|
|
40
|
+
a.positionalConfig === b.positionalConfig &&
|
|
41
|
+
a.attentionConfig === b.attentionConfig
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Longest exact token-prefix match length. 0 when the first token differs. */
|
|
46
|
+
export function exactPrefixLength(cached: readonly number[], query: readonly number[]): number {
|
|
47
|
+
const n = Math.min(cached.length, query.length);
|
|
48
|
+
let i = 0;
|
|
49
|
+
while (i < n && cached[i] === query[i]) i++;
|
|
50
|
+
return i;
|
|
51
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 5 — byte-bounded LRU container for KV pages. Pure TypeScript.
|
|
3
|
+
*
|
|
4
|
+
* Why this is not `src/memory/eviction.ts` `lruStrategy()`: that strategy is a
|
|
5
|
+
* ranking POLICY — it sorts a snapshot of `TrackedEntry` records (group,
|
|
6
|
+
* reload cost, reuse probability, admission sequence) that the residency
|
|
7
|
+
* manager owns, O(n log n) per eviction. The paged KV cache needs a keyed
|
|
8
|
+
* CONTAINER that owns its values and evicts in O(1) on every page insert, for
|
|
9
|
+
* values that carry none of the residency metadata. Routing KV pages through
|
|
10
|
+
* the residency manager would charge them against the model-weight budget and
|
|
11
|
+
* make every page insert an async admission; the shapes genuinely differ.
|
|
12
|
+
*
|
|
13
|
+
* Invariant: `bytes <= maxBytes` after every call. An entry larger than
|
|
14
|
+
* `maxBytes` is rejected (and any previous value under its key is dropped).
|
|
15
|
+
* Recency is the Map's insertion order: a touch deletes and re-inserts.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
export interface LruNode<V> {
|
|
19
|
+
key: string;
|
|
20
|
+
value: V;
|
|
21
|
+
bytes: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export class LruCache<V> {
|
|
25
|
+
private readonly map = new Map<string, LruNode<V>>();
|
|
26
|
+
private used = 0;
|
|
27
|
+
|
|
28
|
+
constructor(readonly maxBytes: number) {}
|
|
29
|
+
|
|
30
|
+
get size(): number {
|
|
31
|
+
return this.map.size;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
get bytes(): number {
|
|
35
|
+
return this.used;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
has(key: string): boolean {
|
|
39
|
+
return this.map.has(key);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
get(key: string): V | undefined {
|
|
43
|
+
const node = this.map.get(key);
|
|
44
|
+
if (!node) return undefined;
|
|
45
|
+
this.map.delete(key);
|
|
46
|
+
this.map.set(key, node);
|
|
47
|
+
return node.value;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
peek(key: string): V | undefined {
|
|
51
|
+
return this.map.get(key)?.value;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Insert or replace `key`, evicting least-recently-used entries until the
|
|
56
|
+
* cache is back inside `maxBytes`. Returns every key that left the cache —
|
|
57
|
+
* including `key` itself when the new value is larger than `maxBytes` and
|
|
58
|
+
* replaced a resident value (a rejected NEW key is simply not inserted).
|
|
59
|
+
*/
|
|
60
|
+
set(key: string, value: V, bytes: number): string[] {
|
|
61
|
+
const evicted: string[] = [];
|
|
62
|
+
const size = Number.isFinite(bytes) && bytes > 0 ? bytes : 0;
|
|
63
|
+
const existed = this.remove(key);
|
|
64
|
+
if (size > this.maxBytes) {
|
|
65
|
+
if (existed) evicted.push(key);
|
|
66
|
+
return evicted;
|
|
67
|
+
}
|
|
68
|
+
this.map.set(key, { key, value, bytes: size });
|
|
69
|
+
this.used += size;
|
|
70
|
+
for (const victim of this.map.keys()) {
|
|
71
|
+
if (this.used <= this.maxBytes) break;
|
|
72
|
+
if (victim === key) continue;
|
|
73
|
+
this.remove(victim);
|
|
74
|
+
evicted.push(victim);
|
|
75
|
+
}
|
|
76
|
+
return evicted;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
delete(key: string): boolean {
|
|
80
|
+
return this.remove(key);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
clear(): void {
|
|
84
|
+
this.map.clear();
|
|
85
|
+
this.used = 0;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Keys, least recently used first. */
|
|
89
|
+
keys(): string[] {
|
|
90
|
+
return [...this.map.keys()];
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private remove(key: string): boolean {
|
|
94
|
+
const node = this.map.get(key);
|
|
95
|
+
if (!node) return false;
|
|
96
|
+
this.map.delete(key);
|
|
97
|
+
this.used -= node.bytes;
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
}
|