@seanhogg/builderforce-webcontainers 2026.10.11 → 2026.10.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/adaptive/controller.d.ts +39 -0
- package/dist/adaptive/controller.d.ts.map +1 -0
- package/dist/adaptive/controller.js +108 -0
- package/dist/adaptive/controller.js.map +1 -0
- package/dist/adaptive/costModel.d.ts +37 -0
- package/dist/adaptive/costModel.d.ts.map +1 -0
- package/dist/adaptive/costModel.js +102 -0
- package/dist/adaptive/costModel.js.map +1 -0
- package/dist/adaptive/planner.d.ts +22 -0
- package/dist/adaptive/planner.d.ts.map +1 -0
- package/dist/adaptive/planner.js +114 -0
- package/dist/adaptive/planner.js.map +1 -0
- package/dist/adaptive/types.d.ts +138 -0
- package/dist/adaptive/types.d.ts.map +1 -0
- package/dist/adaptive/types.js +12 -0
- package/dist/adaptive/types.js.map +1 -0
- package/dist/assets.js +2 -2
- package/dist/check/cachedFetch.js.map +1 -1
- package/dist/check/index.d.ts +6 -1
- package/dist/check/index.d.ts.map +1 -1
- package/dist/check/index.js +6 -1
- package/dist/check/index.js.map +1 -1
- package/dist/componentCompilers.d.ts.map +1 -1
- package/dist/componentCompilers.js.map +1 -1
- package/dist/esbuildWasm.js.map +1 -1
- package/dist/host.d.ts +11 -5
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +19 -8
- package/dist/host.js.map +1 -1
- package/dist/inference/backends/mock.d.ts +24 -0
- package/dist/inference/backends/mock.d.ts.map +1 -0
- package/dist/inference/backends/mock.js +55 -0
- package/dist/inference/backends/mock.js.map +1 -0
- package/dist/inference/backends/onnx.d.ts +33 -0
- package/dist/inference/backends/onnx.d.ts.map +1 -0
- package/dist/inference/backends/onnx.js +60 -0
- package/dist/inference/backends/onnx.js.map +1 -0
- package/dist/inference/backends/webllm.d.ts +27 -0
- package/dist/inference/backends/webllm.d.ts.map +1 -0
- package/dist/inference/backends/webllm.js +52 -0
- package/dist/inference/backends/webllm.js.map +1 -0
- package/dist/inference/backends/wllama.d.ts +29 -0
- package/dist/inference/backends/wllama.d.ts.map +1 -0
- package/dist/inference/backends/wllama.js +51 -0
- package/dist/inference/backends/wllama.js.map +1 -0
- package/dist/inference/capabilities.d.ts +20 -0
- package/dist/inference/capabilities.d.ts.map +1 -0
- package/dist/inference/capabilities.js +33 -0
- package/dist/inference/capabilities.js.map +1 -0
- package/dist/inference/client.d.ts +45 -0
- package/dist/inference/client.d.ts.map +1 -0
- package/dist/inference/client.js +201 -0
- package/dist/inference/client.js.map +1 -0
- package/dist/inference/modelCache.d.ts +71 -0
- package/dist/inference/modelCache.d.ts.map +1 -0
- package/dist/inference/modelCache.js +91 -0
- package/dist/inference/modelCache.js.map +1 -0
- package/dist/inference/protocol.d.ts +94 -0
- package/dist/inference/protocol.d.ts.map +1 -0
- package/dist/inference/protocol.js +19 -0
- package/dist/inference/protocol.js.map +1 -0
- package/dist/inference/tokenStream.d.ts +26 -0
- package/dist/inference/tokenStream.d.ts.map +1 -0
- package/dist/inference/tokenStream.js +79 -0
- package/dist/inference/tokenStream.js.map +1 -0
- package/dist/inference/types.d.ts +108 -0
- package/dist/inference/types.d.ts.map +1 -0
- package/dist/inference/types.js +8 -0
- package/dist/inference/types.js.map +1 -0
- package/dist/inference/worker.bundle.js +5 -0
- package/dist/inference/worker.d.ts +35 -0
- package/dist/inference/worker.d.ts.map +1 -0
- package/dist/inference/worker.js +157 -0
- package/dist/inference/worker.js.map +1 -0
- package/dist/inference/workerEntry.d.ts +15 -0
- package/dist/inference/workerEntry.d.ts.map +1 -0
- package/dist/inference/workerEntry.js +17 -0
- package/dist/inference/workerEntry.js.map +1 -0
- package/dist/kvcache/advanced.d.ts +39 -0
- package/dist/kvcache/advanced.d.ts.map +1 -0
- package/dist/kvcache/advanced.js +59 -0
- package/dist/kvcache/advanced.js.map +1 -0
- package/dist/kvcache/compatibility.d.ts +14 -0
- package/dist/kvcache/compatibility.d.ts.map +1 -0
- package/dist/kvcache/compatibility.js +44 -0
- package/dist/kvcache/compatibility.js.map +1 -0
- package/dist/kvcache/lru.d.ts +45 -0
- package/dist/kvcache/lru.d.ts.map +1 -0
- package/dist/kvcache/lru.js +91 -0
- package/dist/kvcache/lru.js.map +1 -0
- package/dist/kvcache/paged.d.ts +86 -0
- package/dist/kvcache/paged.d.ts.map +1 -0
- package/dist/kvcache/paged.js +238 -0
- package/dist/kvcache/paged.js.map +1 -0
- package/dist/kvcache/quant.d.ts +30 -0
- package/dist/kvcache/quant.d.ts.map +1 -0
- package/dist/kvcache/quant.js +73 -0
- package/dist/kvcache/quant.js.map +1 -0
- package/dist/kvcache/radix.d.ts +28 -0
- package/dist/kvcache/radix.d.ts.map +1 -0
- package/dist/kvcache/radix.js +58 -0
- package/dist/kvcache/radix.js.map +1 -0
- package/dist/kvcache/runtime.d.ts +11 -0
- package/dist/kvcache/runtime.d.ts.map +1 -0
- package/dist/kvcache/runtime.js +204 -0
- package/dist/kvcache/runtime.js.map +1 -0
- package/dist/kvcache/types.d.ts +151 -0
- package/dist/kvcache/types.d.ts.map +1 -0
- package/dist/kvcache/types.js +10 -0
- package/dist/kvcache/types.js.map +1 -0
- package/dist/learning/fingerprint.d.ts +17 -0
- package/dist/learning/fingerprint.d.ts.map +1 -0
- package/dist/learning/fingerprint.js +46 -0
- package/dist/learning/fingerprint.js.map +1 -0
- package/dist/learning/index.d.ts +5 -0
- package/dist/learning/index.d.ts.map +1 -0
- package/dist/learning/index.js +3 -0
- package/dist/learning/index.js.map +1 -0
- package/dist/learning/learner.d.ts +21 -0
- package/dist/learning/learner.d.ts.map +1 -0
- package/dist/learning/learner.js +272 -0
- package/dist/learning/learner.js.map +1 -0
- package/dist/learning/ledger.d.ts +26 -0
- package/dist/learning/ledger.d.ts.map +1 -0
- package/dist/learning/ledger.js +40 -0
- package/dist/learning/ledger.js.map +1 -0
- package/dist/learning/selection.d.ts +44 -0
- package/dist/learning/selection.d.ts.map +1 -0
- package/dist/learning/selection.js +111 -0
- package/dist/learning/selection.js.map +1 -0
- package/dist/learning/stats.d.ts +19 -0
- package/dist/learning/stats.d.ts.map +1 -0
- package/dist/learning/stats.js +31 -0
- package/dist/learning/stats.js.map +1 -0
- package/dist/learning/types.d.ts +133 -0
- package/dist/learning/types.d.ts.map +1 -0
- package/dist/learning/types.js +8 -0
- package/dist/learning/types.js.map +1 -0
- package/dist/memory/envProbe.d.ts +16 -0
- package/dist/memory/envProbe.d.ts.map +1 -0
- package/dist/memory/envProbe.js +28 -0
- package/dist/memory/envProbe.js.map +1 -0
- package/dist/memory/estimates.d.ts +46 -0
- package/dist/memory/estimates.d.ts.map +1 -0
- package/dist/memory/estimates.js +72 -0
- package/dist/memory/estimates.js.map +1 -0
- package/dist/memory/eviction.d.ts +42 -0
- package/dist/memory/eviction.d.ts.map +1 -0
- package/dist/memory/eviction.js +143 -0
- package/dist/memory/eviction.js.map +1 -0
- package/dist/memory/manager.d.ts +34 -0
- package/dist/memory/manager.d.ts.map +1 -0
- package/dist/memory/manager.js +346 -0
- package/dist/memory/manager.js.map +1 -0
- package/dist/memory/managerTypes.d.ts +49 -0
- package/dist/memory/managerTypes.d.ts.map +1 -0
- package/dist/memory/managerTypes.js +7 -0
- package/dist/memory/managerTypes.js.map +1 -0
- package/dist/memory/pressure.d.ts +56 -0
- package/dist/memory/pressure.d.ts.map +1 -0
- package/dist/memory/pressure.js +80 -0
- package/dist/memory/pressure.js.map +1 -0
- package/dist/memory/spillTier.d.ts +47 -0
- package/dist/memory/spillTier.d.ts.map +1 -0
- package/dist/memory/spillTier.js +128 -0
- package/dist/memory/spillTier.js.map +1 -0
- package/dist/memory/storageSink.d.ts +69 -0
- package/dist/memory/storageSink.d.ts.map +1 -0
- package/dist/memory/storageSink.js +221 -0
- package/dist/memory/storageSink.js.map +1 -0
- package/dist/memory/types.d.ts +202 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +15 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/memory/victims.d.ts +32 -0
- package/dist/memory/victims.d.ts.map +1 -0
- package/dist/memory/victims.js +74 -0
- package/dist/memory/victims.js.map +1 -0
- package/dist/memory/workspace.d.ts +34 -0
- package/dist/memory/workspace.d.ts.map +1 -0
- package/dist/memory/workspace.js +55 -0
- package/dist/memory/workspace.js.map +1 -0
- package/dist/memoryIntegration/index.d.ts +8 -0
- package/dist/memoryIntegration/index.d.ts.map +1 -0
- package/dist/memoryIntegration/index.js +4 -0
- package/dist/memoryIntegration/index.js.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts +43 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.js +285 -0
- package/dist/memoryIntegration/knowledgeMemory.js.map +1 -0
- package/dist/memoryIntegration/persistence.d.ts +15 -0
- package/dist/memoryIntegration/persistence.d.ts.map +1 -0
- package/dist/memoryIntegration/persistence.js +43 -0
- package/dist/memoryIntegration/persistence.js.map +1 -0
- package/dist/memoryIntegration/text.d.ts +57 -0
- package/dist/memoryIntegration/text.d.ts.map +1 -0
- package/dist/memoryIntegration/text.js +179 -0
- package/dist/memoryIntegration/text.js.map +1 -0
- package/dist/memoryIntegration/types.d.ts +131 -0
- package/dist/memoryIntegration/types.d.ts.map +1 -0
- package/dist/memoryIntegration/types.js +11 -0
- package/dist/memoryIntegration/types.js.map +1 -0
- package/dist/ml.d.ts +52 -0
- package/dist/ml.d.ts.map +1 -0
- package/dist/ml.js +33 -0
- package/dist/ml.js.map +1 -0
- package/dist/node/childRelay.d.ts +10 -0
- package/dist/node/childRelay.d.ts.map +1 -1
- package/dist/node/childRelay.js +12 -2
- package/dist/node/childRelay.js.map +1 -1
- package/dist/node/index.js.map +1 -1
- package/dist/node/portResponder.d.ts +2 -0
- package/dist/node/portResponder.d.ts.map +1 -1
- package/dist/node/portResponder.js +3 -3
- package/dist/node/portResponder.js.map +1 -1
- package/dist/node/processHost.d.ts +4 -1
- package/dist/node/processHost.d.ts.map +1 -1
- package/dist/node/processHost.js +19 -16
- package/dist/node/processHost.js.map +1 -1
- package/dist/node/protocol.d.ts +1 -0
- package/dist/node/protocol.d.ts.map +1 -1
- package/dist/node/responseWire.d.ts.map +1 -1
- package/dist/node/worker.js +116 -109
- package/dist/node/workerProcess.js +1 -1
- package/dist/node/workerProcess.js.map +1 -1
- package/dist/prefetch/admission.d.ts +41 -0
- package/dist/prefetch/admission.d.ts.map +1 -0
- package/dist/prefetch/admission.js +49 -0
- package/dist/prefetch/admission.js.map +1 -0
- package/dist/prefetch/loadRegistry.d.ts +56 -0
- package/dist/prefetch/loadRegistry.d.ts.map +1 -0
- package/dist/prefetch/loadRegistry.js +108 -0
- package/dist/prefetch/loadRegistry.js.map +1 -0
- package/dist/prefetch/predictor.d.ts +40 -0
- package/dist/prefetch/predictor.d.ts.map +1 -0
- package/dist/prefetch/predictor.js +178 -0
- package/dist/prefetch/predictor.js.map +1 -0
- package/dist/prefetch/prefetcher.d.ts +37 -0
- package/dist/prefetch/prefetcher.d.ts.map +1 -0
- package/dist/prefetch/prefetcher.js +339 -0
- package/dist/prefetch/prefetcher.js.map +1 -0
- package/dist/prefetch/residentStore.d.ts +49 -0
- package/dist/prefetch/residentStore.d.ts.map +1 -0
- package/dist/prefetch/residentStore.js +87 -0
- package/dist/prefetch/residentStore.js.map +1 -0
- package/dist/prefetch/selfTuner.d.ts +40 -0
- package/dist/prefetch/selfTuner.d.ts.map +1 -0
- package/dist/prefetch/selfTuner.js +78 -0
- package/dist/prefetch/selfTuner.js.map +1 -0
- package/dist/prefetch/stats.d.ts +52 -0
- package/dist/prefetch/stats.d.ts.map +1 -0
- package/dist/prefetch/stats.js +72 -0
- package/dist/prefetch/stats.js.map +1 -0
- package/dist/prefetch/streaming.d.ts +132 -0
- package/dist/prefetch/streaming.d.ts.map +1 -0
- package/dist/prefetch/streaming.js +141 -0
- package/dist/prefetch/streaming.js.map +1 -0
- package/dist/prefetch/targetIndex.d.ts +18 -0
- package/dist/prefetch/targetIndex.d.ts.map +1 -0
- package/dist/prefetch/targetIndex.js +32 -0
- package/dist/prefetch/targetIndex.js.map +1 -0
- package/dist/prefetch/types.d.ts +283 -0
- package/dist/prefetch/types.d.ts.map +1 -0
- package/dist/prefetch/types.js +19 -0
- package/dist/prefetch/types.js.map +1 -0
- package/dist/previewBridge.d.ts +6 -0
- package/dist/previewBridge.d.ts.map +1 -1
- package/dist/previewBridge.js +25 -5
- package/dist/previewBridge.js.map +1 -1
- package/dist/previewMessages.d.ts +4 -3
- package/dist/previewMessages.d.ts.map +1 -1
- package/dist/previewMessages.js +12 -6
- package/dist/previewMessages.js.map +1 -1
- package/dist/process-worker.js +116 -109
- package/dist/relay.html +1 -1
- package/dist/shared/clock.d.ts +7 -0
- package/dist/shared/clock.d.ts.map +1 -0
- package/dist/shared/clock.js +10 -0
- package/dist/shared/clock.js.map +1 -0
- package/dist/shared/hardwareProbe.d.ts +26 -0
- package/dist/shared/hardwareProbe.d.ts.map +1 -0
- package/dist/shared/hardwareProbe.js +62 -0
- package/dist/shared/hardwareProbe.js.map +1 -0
- package/dist/shared/idb.d.ts +41 -0
- package/dist/shared/idb.d.ts.map +1 -0
- package/dist/shared/idb.js +184 -0
- package/dist/shared/idb.js.map +1 -0
- package/dist/shared/tokens.d.ts +7 -0
- package/dist/shared/tokens.d.ts.map +1 -0
- package/dist/shared/tokens.js +9 -0
- package/dist/shared/tokens.js.map +1 -0
- package/dist/snapshotStore.d.ts +2 -1
- package/dist/snapshotStore.d.ts.map +1 -1
- package/dist/snapshotStore.js +27 -12
- package/dist/snapshotStore.js.map +1 -1
- package/dist/styleCompilers.d.ts.map +1 -1
- package/dist/styleCompilers.js.map +1 -1
- package/dist/transport.js.map +1 -1
- package/package.json +11 -5
- package/src/adaptive/controller.ts +151 -0
- package/src/adaptive/costModel.ts +131 -0
- package/src/adaptive/planner.ts +127 -0
- package/src/adaptive/types.ts +137 -0
- package/src/check/index.ts +6 -1
- package/src/host.ts +33 -11
- package/src/inference/backends/mock.ts +77 -0
- package/src/inference/backends/onnx.ts +91 -0
- package/src/inference/backends/webllm.ts +79 -0
- package/src/inference/backends/wllama.ts +79 -0
- package/src/inference/capabilities.ts +46 -0
- package/src/inference/client.ts +276 -0
- package/src/inference/modelCache.ts +145 -0
- package/src/inference/protocol.ts +119 -0
- package/src/inference/tokenStream.ts +92 -0
- package/src/inference/types.ts +115 -0
- package/src/inference/worker.ts +191 -0
- package/src/inference/workerEntry.ts +18 -0
- package/src/kvcache/advanced.ts +86 -0
- package/src/kvcache/compatibility.ts +51 -0
- package/src/kvcache/lru.ts +100 -0
- package/src/kvcache/paged.ts +270 -0
- package/src/kvcache/quant.ts +88 -0
- package/src/kvcache/radix.ts +68 -0
- package/src/kvcache/runtime.ts +239 -0
- package/src/kvcache/types.ts +167 -0
- package/src/learning/fingerprint.ts +50 -0
- package/src/learning/index.ts +15 -0
- package/src/learning/learner.ts +304 -0
- package/src/learning/ledger.ts +54 -0
- package/src/learning/selection.ts +137 -0
- package/src/learning/stats.ts +39 -0
- package/src/learning/types.ts +142 -0
- package/src/memory/envProbe.ts +30 -0
- package/src/memory/estimates.ts +98 -0
- package/src/memory/eviction.ts +145 -0
- package/src/memory/manager.ts +380 -0
- package/src/memory/managerTypes.ts +61 -0
- package/src/memory/pressure.ts +120 -0
- package/src/memory/spillTier.ts +159 -0
- package/src/memory/storageSink.ts +289 -0
- package/src/memory/types.ts +217 -0
- package/src/memory/victims.ts +80 -0
- package/src/memory/workspace.ts +74 -0
- package/src/memoryIntegration/index.ts +7 -0
- package/src/memoryIntegration/knowledgeMemory.ts +339 -0
- package/src/memoryIntegration/persistence.ts +54 -0
- package/src/memoryIntegration/text.ts +185 -0
- package/src/memoryIntegration/types.ts +145 -0
- package/src/ml.ts +180 -0
- package/src/node/childRelay.ts +16 -3
- package/src/node/portResponder.ts +5 -3
- package/src/node/processHost.ts +16 -15
- package/src/node/protocol.ts +1 -1
- package/src/node/workerProcess.ts +1 -1
- package/src/prefetch/admission.ts +74 -0
- package/src/prefetch/loadRegistry.ts +156 -0
- package/src/prefetch/predictor.ts +199 -0
- package/src/prefetch/prefetcher.ts +369 -0
- package/src/prefetch/residentStore.ts +121 -0
- package/src/prefetch/selfTuner.ts +110 -0
- package/src/prefetch/stats.ts +120 -0
- package/src/prefetch/streaming.ts +231 -0
- package/src/prefetch/targetIndex.ts +40 -0
- package/src/prefetch/types.ts +307 -0
- package/src/previewBridge.ts +25 -5
- package/src/previewMessages.ts +11 -6
- package/src/shared/clock.ts +9 -0
- package/src/shared/hardwareProbe.ts +68 -0
- package/src/shared/idb.ts +203 -0
- package/src/shared/tokens.ts +8 -0
- package/src/snapshotStore.ts +25 -10
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: Wllama (llama.cpp WASM) backend.
|
|
3
|
+
*
|
|
4
|
+
* Registered as an alternative CPU backend. Like the ONNX adapter, it wraps
|
|
5
|
+
* an injected `WllamaModule` so this package carries no llama.cpp/wllama
|
|
6
|
+
* dependency; a host that prefers GGUF models provides the module. Kept as
|
|
7
|
+
* the secondary path pending the Phase 1 decision recorded in
|
|
8
|
+
* `docs/phase-1/report.md`.
|
|
9
|
+
*/
|
|
10
|
+
import { renderChatPrompt } from './onnx.js';
|
|
11
|
+
import { detectBackendEnv } from '../capabilities.js';
|
|
12
|
+
export function createWllamaBackend(options) {
|
|
13
|
+
let module;
|
|
14
|
+
let model;
|
|
15
|
+
return {
|
|
16
|
+
kind: 'wllama',
|
|
17
|
+
async probe() {
|
|
18
|
+
const env = detectBackendEnv();
|
|
19
|
+
return {
|
|
20
|
+
kind: 'wllama',
|
|
21
|
+
available: env.wasm,
|
|
22
|
+
reason: env.wasm ? undefined : 'WebAssembly not available',
|
|
23
|
+
wasmSimd: env.wasmSimd,
|
|
24
|
+
webgpu: false,
|
|
25
|
+
};
|
|
26
|
+
},
|
|
27
|
+
async load(modelRef, weights, context) {
|
|
28
|
+
module ??= await options.loadModule();
|
|
29
|
+
await module.init({ simd: context.wasmSimd });
|
|
30
|
+
model = await module.loadModel(weights, modelRef.id);
|
|
31
|
+
context.log(`wllama: loaded ${modelRef.id}`);
|
|
32
|
+
},
|
|
33
|
+
async *generate(messages, options) {
|
|
34
|
+
if (!model)
|
|
35
|
+
throw new Error('wllama: generate called before load');
|
|
36
|
+
let index = 0;
|
|
37
|
+
for await (const token of model.generate(renderChatPrompt(messages), {
|
|
38
|
+
maxTokens: options?.maxTokens ?? 256,
|
|
39
|
+
temperature: options?.temperature ?? 0.7,
|
|
40
|
+
signal: options?.signal,
|
|
41
|
+
})) {
|
|
42
|
+
yield { token, index: index++ };
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
async dispose() {
|
|
46
|
+
await model?.release();
|
|
47
|
+
model = undefined;
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=wllama.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wllama.js","sourceRoot":"","sources":["../../../src/inference/backends/wllama.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAWH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAgBtD,MAAM,UAAU,mBAAmB,CAAC,OAA6B;IAC/D,IAAI,MAAgC,CAAC;IACrC,IAAI,KAA8B,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,QAAQ;QAEd,KAAK,CAAC,KAAK;YACT,MAAM,GAAG,GAAG,gBAAgB,EAAE,CAAC;YAC/B,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,SAAS,EAAE,GAAG,CAAC,IAAI;gBACnB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,2BAA2B;gBAC1D,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,KAAK;aACd,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,QAAkB,EAAE,OAAoB,EAAE,OAAuB;YAC1E,MAAM,KAAK,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9C,KAAK,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,CAAC;QAED,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAuB,EAAE,OAAyB;YAChE,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACnE,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE;gBACnE,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,GAAG;gBACpC,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,GAAG;gBACxC,MAAM,EAAE,OAAO,EAAE,MAAM;aACxB,CAAC,EAAE,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;YAClC,CAAC;QACH,CAAC;QAED,KAAK,CAAC,OAAO;YACX,MAAM,KAAK,EAAE,OAAO,EAAE,CAAC;YACvB,KAAK,GAAG,SAAS,CAAC;QACpB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: environment capability probe.
|
|
3
|
+
*
|
|
4
|
+
* Best-effort detection of what the current browser can run. SharedArrayBuffer
|
|
5
|
+
* and cross-origin isolation are OPTIONAL — the runtime must work without
|
|
6
|
+
* them — so these are reported, never assumed. The individual probes live in
|
|
7
|
+
* `shared/hardwareProbe.ts` (the one home shared with the adaptive controller
|
|
8
|
+
* and the memory manager); this module only assembles the report.
|
|
9
|
+
*/
|
|
10
|
+
import type { RuntimeCapabilities } from './types.js';
|
|
11
|
+
/** Shared wasm / wasm-SIMD detection for backend probes (onnx, wllama). */
|
|
12
|
+
export declare function detectBackendEnv(): {
|
|
13
|
+
wasm: boolean;
|
|
14
|
+
wasmSimd: boolean;
|
|
15
|
+
};
|
|
16
|
+
export declare function probeRuntimeCapabilities(backends?: InferenceBackendLike[]): RuntimeCapabilities;
|
|
17
|
+
export interface InferenceBackendLike {
|
|
18
|
+
kind: string;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=capabilities.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"capabilities.d.ts","sourceRoot":"","sources":["../../src/inference/capabilities.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH,OAAO,KAAK,EAAe,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEnE,2EAA2E;AAC3E,wBAAgB,gBAAgB,IAAI;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CAGvE;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,GAAE,oBAAoB,EAAO,GAAG,mBAAmB,CAgBnG;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;CACd"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: environment capability probe.
|
|
3
|
+
*
|
|
4
|
+
* Best-effort detection of what the current browser can run. SharedArrayBuffer
|
|
5
|
+
* and cross-origin isolation are OPTIONAL — the runtime must work without
|
|
6
|
+
* them — so these are reported, never assumed. The individual probes live in
|
|
7
|
+
* `shared/hardwareProbe.ts` (the one home shared with the adaptive controller
|
|
8
|
+
* and the memory manager); this module only assembles the report.
|
|
9
|
+
*/
|
|
10
|
+
import { detectCrossOriginIsolated, detectSharedArrayBuffer, detectWasm, detectWasmSimd, detectWebGpu, } from '../shared/hardwareProbe.js';
|
|
11
|
+
/** Shared wasm / wasm-SIMD detection for backend probes (onnx, wllama). */
|
|
12
|
+
export function detectBackendEnv() {
|
|
13
|
+
const wasm = detectWasm();
|
|
14
|
+
return { wasm, wasmSimd: wasm && detectWasmSimd() };
|
|
15
|
+
}
|
|
16
|
+
export function probeRuntimeCapabilities(backends = []) {
|
|
17
|
+
const { wasm, wasmSimd } = detectBackendEnv();
|
|
18
|
+
const webgpu = detectWebGpu();
|
|
19
|
+
return {
|
|
20
|
+
wasm,
|
|
21
|
+
wasmSimd,
|
|
22
|
+
webgpu,
|
|
23
|
+
crossOriginIsolated: detectCrossOriginIsolated(),
|
|
24
|
+
sharedArrayBuffer: detectSharedArrayBuffer(),
|
|
25
|
+
backends: backends.map((b) => ({
|
|
26
|
+
kind: b.kind,
|
|
27
|
+
available: wasm,
|
|
28
|
+
wasmSimd,
|
|
29
|
+
webgpu,
|
|
30
|
+
})),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=capabilities.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"capabilities.js","sourceRoot":"","sources":["../../src/inference/capabilities.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,yBAAyB,EACzB,uBAAuB,EACvB,UAAU,EACV,cAAc,EACd,YAAY,GACb,MAAM,4BAA4B,CAAC;AAGpC,2EAA2E;AAC3E,MAAM,UAAU,gBAAgB;IAC9B,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;IAC1B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,cAAc,EAAE,EAAE,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAAQ,GAA2B,EAAE;IAC5E,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,OAAO;QACL,IAAI;QACJ,QAAQ;QACR,MAAM;QACN,mBAAmB,EAAE,yBAAyB,EAAE;QAChD,iBAAiB,EAAE,uBAAuB,EAAE;QAC5C,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7B,IAAI,EAAE,CAAC,CAAC,IAAmB;YAC3B,SAAS,EAAE,IAAI;YACf,QAAQ;YACR,MAAM;SACP,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { type ModelCache } from './modelCache.js';
|
|
2
|
+
import type { ChatMessage, GenerateOptions, GenerationStats, InferenceToken, ModelRef, RuntimeCapabilities } from './types.js';
|
|
3
|
+
/** Minimal Worker surface the client needs (real Worker, tests, or a port). */
|
|
4
|
+
export interface InferenceWorkerLike {
|
|
5
|
+
postMessage(message: unknown, transfer?: Transferable[]): void;
|
|
6
|
+
terminate(): void;
|
|
7
|
+
onmessage: ((event: {
|
|
8
|
+
data: unknown;
|
|
9
|
+
}) => void) | null;
|
|
10
|
+
onerror: ((event: {
|
|
11
|
+
message?: string;
|
|
12
|
+
}) => void) | null;
|
|
13
|
+
}
|
|
14
|
+
export interface InferenceClientOptions {
|
|
15
|
+
/**
|
|
16
|
+
* Spawn the inference worker. Default: a classic Worker on the bundled
|
|
17
|
+
* `worker.bundle.js` that sits beside this module in `dist/inference/`
|
|
18
|
+
* (also exported as `@seanhogg/builderforce-webcontainers/ml/worker.js`).
|
|
19
|
+
*/
|
|
20
|
+
createWorker?: () => InferenceWorkerLike;
|
|
21
|
+
/** Model cache; default IndexedDB via `createModelCache()`. */
|
|
22
|
+
cache?: ModelCache;
|
|
23
|
+
/**
|
|
24
|
+
* Fetch a model's bytes from its source URL (only called on a cache miss).
|
|
25
|
+
* The returned buffer is TRANSFERRED to the worker, so do not keep using it.
|
|
26
|
+
*/
|
|
27
|
+
fetchModel?: (ref: ModelRef) => Promise<ArrayBuffer>;
|
|
28
|
+
logger?: (message: string) => void;
|
|
29
|
+
}
|
|
30
|
+
export interface InferenceClient {
|
|
31
|
+
probe(): Promise<RuntimeCapabilities>;
|
|
32
|
+
/**
|
|
33
|
+
* Ensure the model is cached and loaded in the worker. After this resolves,
|
|
34
|
+
* `generate()` makes no network access — the bytes came from the cache.
|
|
35
|
+
*/
|
|
36
|
+
load(ref: ModelRef): Promise<void>;
|
|
37
|
+
generate(messages: ChatMessage[], options?: GenerateOptions): AsyncIterable<InferenceToken> & {
|
|
38
|
+
stats: Promise<GenerationStats>;
|
|
39
|
+
};
|
|
40
|
+
cancel(): void;
|
|
41
|
+
dispose(): Promise<void>;
|
|
42
|
+
readonly cache: ModelCache;
|
|
43
|
+
}
|
|
44
|
+
export declare function createInferenceClient(options?: InferenceClientOptions): InferenceClient;
|
|
45
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/inference/client.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAoB,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEpE,OAAO,KAAK,EACV,WAAW,EACX,eAAe,EACf,eAAe,EACf,cAAc,EACd,QAAQ,EACR,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAEpB,+EAA+E;AAC/E,MAAM,WAAW,mBAAmB;IAClC,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAC/D,SAAS,IAAI,IAAI,CAAC;IAClB,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IACvD,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CACzD;AAED,MAAM,WAAW,sBAAsB;IACrC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,mBAAmB,CAAC;IACzC,+DAA+D;IAC/D,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;IACrD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACpC;AAOD,MAAM,WAAW,eAAe;IAC9B,KAAK,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACtC;;;OAGG;IACH,IAAI,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,aAAa,CAAC,cAAc,CAAC,GAAG;QAC5F,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;KACjC,CAAC;IACF,MAAM,IAAI,IAAI,CAAC;IACf,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAmBD,wBAAgB,qBAAqB,CAAC,OAAO,GAAE,sBAA2B,GAAG,eAAe,CAwL3F"}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { INFERENCE_CANCEL, INFERENCE_DISPOSE_MODEL, INFERENCE_DONE, INFERENCE_ERROR, INFERENCE_GENERATE, INFERENCE_LOAD, INFERENCE_LOG, INFERENCE_PROBE, INFERENCE_TOKEN, } from './protocol.js';
|
|
2
|
+
import { createModelCache } from './modelCache.js';
|
|
3
|
+
import { createTokenStream } from './tokenStream.js';
|
|
4
|
+
const toError = (error) => (error instanceof Error ? error : new Error(String(error)));
|
|
5
|
+
function defaultFetchModel(ref) {
|
|
6
|
+
if (!ref.source || ref.source.kind !== 'url')
|
|
7
|
+
return Promise.reject(new Error(`model ${ref.id} has no url source`));
|
|
8
|
+
const url = ref.source.url;
|
|
9
|
+
return fetch(url).then((res) => {
|
|
10
|
+
if (!res.ok)
|
|
11
|
+
throw new Error(`failed to fetch ${url}: ${res.status}`);
|
|
12
|
+
return res.arrayBuffer();
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
function defaultCreateWorker() {
|
|
16
|
+
// The IIFE bundle emitted by scripts/build-sw.mjs beside dist/inference/client.js.
|
|
17
|
+
// ESM only (import.meta.url): a CommonJS consumer of ./ml must pass `createWorker`.
|
|
18
|
+
return new Worker(new URL('./worker.bundle.js', import.meta.url));
|
|
19
|
+
}
|
|
20
|
+
export function createInferenceClient(options = {}) {
|
|
21
|
+
const cache = options.cache ?? createModelCache();
|
|
22
|
+
const fetchModel = options.fetchModel ?? defaultFetchModel;
|
|
23
|
+
const spawn = options.createWorker ?? defaultCreateWorker;
|
|
24
|
+
let activeWorker;
|
|
25
|
+
let nextId = 1;
|
|
26
|
+
const pending = new Map();
|
|
27
|
+
const streams = new Map();
|
|
28
|
+
let workerFailed;
|
|
29
|
+
const handleReply = (reply) => {
|
|
30
|
+
switch (reply.type) {
|
|
31
|
+
case INFERENCE_LOG:
|
|
32
|
+
options.logger?.(reply.message);
|
|
33
|
+
return;
|
|
34
|
+
case INFERENCE_TOKEN: {
|
|
35
|
+
const token = reply;
|
|
36
|
+
streams.get(token.requestId)?.push({ token: token.token, index: token.index });
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
case INFERENCE_DONE: {
|
|
40
|
+
const done = reply;
|
|
41
|
+
streams.get(done.requestId)?.done({
|
|
42
|
+
promptTokens: done.promptTokens,
|
|
43
|
+
promptTokensEstimated: done.promptTokensEstimated,
|
|
44
|
+
generatedTokens: done.generatedTokens,
|
|
45
|
+
ttftMs: done.ttftMs,
|
|
46
|
+
decodeMs: done.decodeMs,
|
|
47
|
+
tokensPerSecond: done.tokensPerSecond,
|
|
48
|
+
});
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
case INFERENCE_ERROR: {
|
|
52
|
+
const error = reply;
|
|
53
|
+
const waiting = pending.get(error.id);
|
|
54
|
+
if (waiting) {
|
|
55
|
+
pending.delete(error.id);
|
|
56
|
+
waiting.reject(new Error(error.message));
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
streams.get(error.id)?.fail(new Error(error.message));
|
|
60
|
+
}
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const id = reply.id;
|
|
65
|
+
const waiting = pending.get(id);
|
|
66
|
+
if (waiting) {
|
|
67
|
+
pending.delete(id);
|
|
68
|
+
waiting.resolve(reply);
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
/** Reject every in-flight request and stream; later calls reject immediately. */
|
|
72
|
+
const failAll = (error) => {
|
|
73
|
+
workerFailed = error;
|
|
74
|
+
const waiting = [...pending.values()];
|
|
75
|
+
const open = [...streams.values()];
|
|
76
|
+
pending.clear();
|
|
77
|
+
streams.clear();
|
|
78
|
+
for (const p of waiting)
|
|
79
|
+
p.reject(error);
|
|
80
|
+
for (const s of open)
|
|
81
|
+
s.fail(error);
|
|
82
|
+
};
|
|
83
|
+
const getWorker = () => {
|
|
84
|
+
if (activeWorker)
|
|
85
|
+
return activeWorker;
|
|
86
|
+
const w = spawn();
|
|
87
|
+
w.onmessage = (event) => handleReply(event.data);
|
|
88
|
+
w.onerror = (event) => failAll(new Error(event.message ?? 'inference worker failed'));
|
|
89
|
+
activeWorker = w;
|
|
90
|
+
return w;
|
|
91
|
+
};
|
|
92
|
+
const request = (message, transfer) => new Promise((resolve, reject) => {
|
|
93
|
+
if (workerFailed)
|
|
94
|
+
return reject(workerFailed);
|
|
95
|
+
const id = nextId++;
|
|
96
|
+
pending.set(id, { resolve: resolve, reject });
|
|
97
|
+
try {
|
|
98
|
+
getWorker().postMessage({ ...message, id }, transfer);
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
pending.delete(id);
|
|
102
|
+
reject(toError(error));
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
/** Ask the worker to stop one generate request (fire-and-forget). */
|
|
106
|
+
const sendCancel = (requestId) => {
|
|
107
|
+
try {
|
|
108
|
+
activeWorker?.postMessage({ type: INFERENCE_CANCEL, id: nextId++, requestId });
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
/* worker gone: nothing to cancel */
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
const client = {
|
|
115
|
+
cache,
|
|
116
|
+
async probe() {
|
|
117
|
+
const reply = await request({ type: INFERENCE_PROBE });
|
|
118
|
+
return reply.capabilities;
|
|
119
|
+
},
|
|
120
|
+
async load(ref) {
|
|
121
|
+
// Per the ModelCache ownership contract, `get` hands back a buffer we own
|
|
122
|
+
// and `put` never retains ours — so the bytes are transferred to the
|
|
123
|
+
// worker with no copy made here (IndexedDB's own clone is the only one).
|
|
124
|
+
const cached = await cache.get(ref.id);
|
|
125
|
+
let weights;
|
|
126
|
+
if (cached) {
|
|
127
|
+
weights = cached.bytes;
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
weights = await fetchModel(ref);
|
|
131
|
+
// Awaited BEFORE the transfer below: `put` clones the bytes (IndexedDB's
|
|
132
|
+
// structured clone runs once its store has opened), and a transfer first
|
|
133
|
+
// would hand it a detached buffer. The write is best-effort — a quota abort
|
|
134
|
+
// (or any cache failure) must not stop the model loading; it just isn't cached.
|
|
135
|
+
try {
|
|
136
|
+
await cache.put(ref.id, {
|
|
137
|
+
quantization: ref.quantization,
|
|
138
|
+
paramsB: ref.paramsB,
|
|
139
|
+
bytes: weights,
|
|
140
|
+
size: weights.byteLength,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
options.logger?.(`model cache write failed for ${ref.id}: ${toError(error).message}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
await request({ type: INFERENCE_LOAD, ref, weights }, [weights]);
|
|
148
|
+
},
|
|
149
|
+
generate(messages, genOptions = {}) {
|
|
150
|
+
const id = nextId++;
|
|
151
|
+
// An AbortSignal is not structured-cloneable: it stays host-side and is
|
|
152
|
+
// translated into an INFERENCE_CANCEL for this request.
|
|
153
|
+
const { signal, ...wireOptions } = genOptions;
|
|
154
|
+
const onAbort = () => sendCancel(id);
|
|
155
|
+
const stream = createTokenStream(() => {
|
|
156
|
+
streams.delete(id);
|
|
157
|
+
signal?.removeEventListener('abort', onAbort);
|
|
158
|
+
});
|
|
159
|
+
const result = (s) => Object.assign(s.iterate(() => sendCancel(id)), { stats: s.stats });
|
|
160
|
+
if (workerFailed) {
|
|
161
|
+
stream.fail(workerFailed);
|
|
162
|
+
return result(stream);
|
|
163
|
+
}
|
|
164
|
+
try {
|
|
165
|
+
const w = getWorker();
|
|
166
|
+
streams.set(id, stream);
|
|
167
|
+
w.postMessage({ type: INFERENCE_GENERATE, id, messages, options: wireOptions });
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
stream.fail(toError(error)); // also removes the map entry
|
|
171
|
+
return result(stream);
|
|
172
|
+
}
|
|
173
|
+
if (signal) {
|
|
174
|
+
if (signal.aborted)
|
|
175
|
+
onAbort();
|
|
176
|
+
else
|
|
177
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
178
|
+
}
|
|
179
|
+
return result(stream);
|
|
180
|
+
},
|
|
181
|
+
cancel() {
|
|
182
|
+
try {
|
|
183
|
+
activeWorker?.postMessage({ type: INFERENCE_CANCEL, id: nextId++ });
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
/* worker gone */
|
|
187
|
+
}
|
|
188
|
+
},
|
|
189
|
+
async dispose() {
|
|
190
|
+
try {
|
|
191
|
+
await request({ type: INFERENCE_DISPOSE_MODEL });
|
|
192
|
+
}
|
|
193
|
+
finally {
|
|
194
|
+
activeWorker?.terminate();
|
|
195
|
+
activeWorker = undefined;
|
|
196
|
+
}
|
|
197
|
+
},
|
|
198
|
+
};
|
|
199
|
+
return client;
|
|
200
|
+
}
|
|
201
|
+
//# sourceMappingURL=client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/inference/client.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,uBAAuB,EACvB,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,cAAc,EACd,aAAa,EACb,eAAe,EACf,eAAe,GAQhB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAmB,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAoB,MAAM,kBAAkB,CAAC;AAuDvE,MAAM,OAAO,GAAG,CAAC,KAAc,EAAS,EAAE,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEvG,SAAS,iBAAiB,CAAC,GAAa;IACtC,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,oBAAoB,CAAC,CAAC,CAAC;IACpH,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;IAC3B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACtE,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB;IAC1B,mFAAmF;IACnF,oFAAoF;IACpF,OAAO,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,oBAAoB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAmC,CAAC;AACtG,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAO,GAA2B,EAAE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,gBAAgB,EAAE,CAAC;IAClD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,iBAAiB,CAAC;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,IAAI,mBAAmB,CAAC;IAE1D,IAAI,YAA6C,CAAC;IAClD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,IAAI,YAA+B,CAAC;IAEpC,MAAM,WAAW,GAAG,CAAC,KAAqB,EAAE,EAAE;QAC5C,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,aAAa;gBAChB,OAAO,CAAC,MAAM,EAAE,CAAE,KAAkB,CAAC,OAAO,CAAC,CAAC;gBAC9C,OAAO;YACT,KAAK,eAAe,EAAE,CAAC;gBACrB,MAAM,KAAK,GAAG,KAAmB,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC/E,OAAO;YACT,CAAC;YACD,KAAK,cAAc,EAAE,CAAC;gBACpB,MAAM,IAAI,GAAG,KAAkB,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC;oBAChC,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;oBACjD,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,eAAe,EAAE,IAAI,CAAC,eAAe;iBACtC,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YACD,KAAK,eAAe,EAAE,CAAC;gBACrB,MAAM,KAAK,GAAG,KAAmB,CAAC;gBAClC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACtC,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBACzB,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3C,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACxD,CAAC;gBACD,OAAO;YACT,CAAC;QACH,CAAC;QACD,MAAM,EAAE,GAAI,KAAwB,CAAC,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,CAAC,OAAkC,CAAC,KAAc,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC,CAAC;IAEF,iFAAiF;IACjF,MAAM,OAAO,GAAG,CAAC,KAAY,EAAE,EAAE;QAC/B,YAAY,GAAG,KAAK,CAAC;QACrB,MAAM,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,GAAwB,EAAE;QAC1C,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QACtC,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC;QAClB,CAAC,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,IAAsB,CAAC,CAAC;QACnE,CAAC,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,yBAAyB,CAAC,CAAC,CAAC;QACtF,YAAY,GAAG,CAAC,CAAC;QACjB,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAI,OAAgC,EAAE,QAAyB,EAAc,EAAE,CAC7F,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACjC,IAAI,YAAY;YAAE,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,OAAiC,EAAE,MAAM,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC;YACH,SAAS,EAAE,CAAC,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACzB,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,qEAAqE;IACrE,MAAM,UAAU,GAAG,CAAC,SAAiB,EAAE,EAAE;QACvC,IAAI,CAAC;YACH,YAAY,EAAE,WAAW,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACjF,CAAC;QAAC,MAAM,CAAC;YACP,oCAAoC;QACtC,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,MAAM,GAAoB;QAC9B,KAAK;QAEL,KAAK,CAAC,KAAK;YACT,MAAM,KAAK,GAAG,MAAM,OAAO,CAAa,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;YACnE,OAAO,KAAK,CAAC,YAAY,CAAC;QAC5B,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,GAAG;YACZ,0EAA0E;YAC1E,qEAAqE;YACrE,yEAAyE;YACzE,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvC,IAAI,OAAoB,CAAC;YACzB,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;gBAChC,yEAAyE;gBACzE,yEAAyE;gBACzE,4EAA4E;gBAC5E,gFAAgF;gBAChF,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE;wBACtB,YAAY,EAAE,GAAG,CAAC,YAAY;wBAC9B,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,KAAK,EAAE,OAAO;wBACd,IAAI,EAAE,OAAO,CAAC,UAAU;qBACzB,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO,CAAC,MAAM,EAAE,CAAC,gCAAgC,GAAG,CAAC,EAAE,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxF,CAAC;YACH,CAAC;YACD,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACnE,CAAC;QAED,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,EAAE;YAChC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;YACpB,wEAAwE;YACxE,wDAAwD;YACxD,MAAM,EAAE,MAAM,EAAE,GAAG,WAAW,EAAE,GAAG,UAAU,CAAC;YAC9C,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;gBACpC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACnB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,CAAC,CAAc,EAAE,EAAE,CAChC,MAAM,CAAC,MAAM,CACX,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,EAC/B,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CACnB,CAAC;YAEJ,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC;YACxB,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gBACxB,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAyC,EAAE,CAAC,CAAC;YAChH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,6BAA6B;gBAC1D,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC;YACxB,CAAC;YACD,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,MAAM,CAAC,OAAO;oBAAE,OAAO,EAAE,CAAC;;oBACzB,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;QAED,MAAM;YACJ,IAAI,CAAC;gBACH,YAAY,EAAE,WAAW,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YACtE,CAAC;YAAC,MAAM,CAAC;gBACP,iBAAiB;YACnB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,OAAO;YACX,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACnD,CAAC;oBAAS,CAAC;gBACT,YAAY,EAAE,SAAS,EAAE,CAAC;gBAC1B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;QACH,CAAC;KACF,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: model asset cache.
|
|
3
|
+
*
|
|
4
|
+
* Stores quantized model weights in IndexedDB so a model provisioned once is
|
|
5
|
+
* available offline for every later session. Keys are the model id; entries
|
|
6
|
+
* carry the bytes plus a small metadata record (or, with `valueFormat: 'raw'`,
|
|
7
|
+
* just the bytes — see `ModelCacheOptions.valueFormat`). Uses the same defensive
|
|
8
|
+
* posture as the rest of the runtime: if IndexedDB is unavailable the cache
|
|
9
|
+
* is a no-op and provisioning simply re-downloads.
|
|
10
|
+
*
|
|
11
|
+
* Buffer ownership contract (every `ModelCache` implementation): `put` must not
|
|
12
|
+
* retain the caller's `bytes` buffer (IndexedDB structured-clones it; the
|
|
13
|
+
* in-memory cache copies it), and `get` returns a buffer the caller owns and
|
|
14
|
+
* may transfer. That is what lets `InferenceClient.load()` transfer the bytes
|
|
15
|
+
* to the worker without making its own copy.
|
|
16
|
+
*/
|
|
17
|
+
export interface ModelCacheEntry {
|
|
18
|
+
/** Model id, e.g. `builderforce/tiny-instruct-q4`. */
|
|
19
|
+
id: string;
|
|
20
|
+
/** Absent on legacy entries stored as raw bytes. */
|
|
21
|
+
quantization?: string;
|
|
22
|
+
/** Absent on legacy entries stored as raw bytes. */
|
|
23
|
+
paramsB?: number;
|
|
24
|
+
bytes: ArrayBuffer;
|
|
25
|
+
/** Byte length of `bytes`, recorded at put time for bookkeeping. */
|
|
26
|
+
size: number;
|
|
27
|
+
/** When the entry was written, epoch ms (0 for legacy raw-bytes entries). */
|
|
28
|
+
storedAt: number;
|
|
29
|
+
}
|
|
30
|
+
export interface ModelCache {
|
|
31
|
+
get(id: string): Promise<ModelCacheEntry | undefined>;
|
|
32
|
+
put(id: string, entry: Omit<ModelCacheEntry, 'id' | 'storedAt'>): Promise<void>;
|
|
33
|
+
has(id: string): Promise<boolean>;
|
|
34
|
+
delete(id: string): Promise<void>;
|
|
35
|
+
/** Ids currently held, for capacity management and diagnostics. */
|
|
36
|
+
keys(): Promise<string[]>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* IndexedDB-backed model cache. Pass an explicit `indexedDB` in tests (the
|
|
40
|
+
* `fake-indexeddb` package is already a dev dependency of this package);
|
|
41
|
+
* in the browser it defaults to the global.
|
|
42
|
+
*
|
|
43
|
+
* `dbName` / `storeName` / `dbVersion` let a host adopt an existing weights
|
|
44
|
+
* database (e.g. Builderforce Studio's `builderforce-studio-weights` / `weights`
|
|
45
|
+
* / v1, whose values are raw ArrayBuffers keyed by path) without migrating it.
|
|
46
|
+
*/
|
|
47
|
+
export interface ModelCacheOptions {
|
|
48
|
+
indexedDB?: IDBFactory;
|
|
49
|
+
/** Default `bfwc-inference-models`. */
|
|
50
|
+
dbName?: string;
|
|
51
|
+
/** Default `models`. */
|
|
52
|
+
storeName?: string;
|
|
53
|
+
/** Default 1. The upgrade only creates the store when it is missing. */
|
|
54
|
+
dbVersion?: number;
|
|
55
|
+
/**
|
|
56
|
+
* How `put` stores a value. Default `'entry'`.
|
|
57
|
+
* - `'entry'`: a `ModelCacheEntry` object (bytes + metadata).
|
|
58
|
+
* - `'raw'`: the `ArrayBuffer` itself, with no wrapper — byte-identical to
|
|
59
|
+
* Builderforce Studio's legacy weights format, so older builds on the same
|
|
60
|
+
* origin keep reading what this cache writes. `quantization` / `paramsB`
|
|
61
|
+
* are NOT persisted in raw mode; `get` returns `{ id, bytes, size, storedAt: 0 }`.
|
|
62
|
+
*
|
|
63
|
+
* `get` reads both shapes in either mode.
|
|
64
|
+
*/
|
|
65
|
+
valueFormat?: ModelCacheValueFormat;
|
|
66
|
+
}
|
|
67
|
+
export type ModelCacheValueFormat = 'entry' | 'raw';
|
|
68
|
+
export declare function createModelCache(options?: ModelCacheOptions): ModelCache;
|
|
69
|
+
/** In-memory cache for tests and non-persistent fallbacks. Copies on put and get (see the ownership contract). */
|
|
70
|
+
export declare function createMemoryModelCache(): ModelCache;
|
|
71
|
+
//# sourceMappingURL=modelCache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"modelCache.d.ts","sourceRoot":"","sources":["../../src/inference/modelCache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,MAAM,WAAW,eAAe;IAC9B,sDAAsD;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,WAAW,CAAC;IACnB,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IACtD,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,GAAG,UAAU,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChF,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,mEAAmE;IACnE,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAC3B;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,uCAAuC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,qBAAqB,CAAC;CACrC;AAED,MAAM,MAAM,qBAAqB,GAAG,OAAO,GAAG,KAAK,CAAC;AAapD,wBAAgB,gBAAgB,CAAC,OAAO,GAAE,iBAAsB,GAAG,UAAU,CAoC5E;AAED,kHAAkH;AAClH,wBAAgB,sBAAsB,IAAI,UAAU,CAoBnD"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: model asset cache.
|
|
3
|
+
*
|
|
4
|
+
* Stores quantized model weights in IndexedDB so a model provisioned once is
|
|
5
|
+
* available offline for every later session. Keys are the model id; entries
|
|
6
|
+
* carry the bytes plus a small metadata record (or, with `valueFormat: 'raw'`,
|
|
7
|
+
* just the bytes — see `ModelCacheOptions.valueFormat`). Uses the same defensive
|
|
8
|
+
* posture as the rest of the runtime: if IndexedDB is unavailable the cache
|
|
9
|
+
* is a no-op and provisioning simply re-downloads.
|
|
10
|
+
*
|
|
11
|
+
* Buffer ownership contract (every `ModelCache` implementation): `put` must not
|
|
12
|
+
* retain the caller's `bytes` buffer (IndexedDB structured-clones it; the
|
|
13
|
+
* in-memory cache copies it), and `get` returns a buffer the caller owns and
|
|
14
|
+
* may transfer. That is what lets `InferenceClient.load()` transfer the bytes
|
|
15
|
+
* to the worker without making its own copy.
|
|
16
|
+
*/
|
|
17
|
+
import { lazyIdbStore } from '../shared/idb.js';
|
|
18
|
+
function isArrayBuffer(value) {
|
|
19
|
+
return value instanceof ArrayBuffer || Object.prototype.toString.call(value) === '[object ArrayBuffer]';
|
|
20
|
+
}
|
|
21
|
+
/** Normalise a stored value: a legacy raw ArrayBuffer is wrapped as an entry. */
|
|
22
|
+
function toEntry(id, value) {
|
|
23
|
+
if (value === undefined || value === null)
|
|
24
|
+
return undefined;
|
|
25
|
+
if (isArrayBuffer(value))
|
|
26
|
+
return { id, bytes: value, size: value.byteLength, storedAt: 0 };
|
|
27
|
+
return value;
|
|
28
|
+
}
|
|
29
|
+
export function createModelCache(options = {}) {
|
|
30
|
+
const factory = options.indexedDB ?? globalThis.indexedDB;
|
|
31
|
+
const config = {
|
|
32
|
+
dbName: options.dbName ?? 'bfwc-inference-models',
|
|
33
|
+
storeName: options.storeName ?? 'models',
|
|
34
|
+
version: options.dbVersion ?? 1,
|
|
35
|
+
};
|
|
36
|
+
const raw = options.valueFormat === 'raw';
|
|
37
|
+
const open = lazyIdbStore(factory, config);
|
|
38
|
+
return {
|
|
39
|
+
async get(id) {
|
|
40
|
+
const store = await open();
|
|
41
|
+
if (!store)
|
|
42
|
+
return undefined;
|
|
43
|
+
return toEntry(id, await store.run('readonly', (s) => s.get(id)));
|
|
44
|
+
},
|
|
45
|
+
async put(id, entry) {
|
|
46
|
+
const store = await open();
|
|
47
|
+
if (!store)
|
|
48
|
+
return;
|
|
49
|
+
const value = raw ? entry.bytes : { ...entry, id, storedAt: Date.now() };
|
|
50
|
+
await store.run('readwrite', (s) => s.put(value, id));
|
|
51
|
+
},
|
|
52
|
+
async has(id) {
|
|
53
|
+
const store = await open();
|
|
54
|
+
if (!store)
|
|
55
|
+
return false;
|
|
56
|
+
return (await store.run('readonly', (s) => s.count(id))) > 0;
|
|
57
|
+
},
|
|
58
|
+
async delete(id) {
|
|
59
|
+
await (await open())?.run('readwrite', (s) => s.delete(id));
|
|
60
|
+
},
|
|
61
|
+
async keys() {
|
|
62
|
+
const store = await open();
|
|
63
|
+
if (!store)
|
|
64
|
+
return [];
|
|
65
|
+
return (await store.run('readonly', (s) => s.getAllKeys())).map(String);
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/** In-memory cache for tests and non-persistent fallbacks. Copies on put and get (see the ownership contract). */
|
|
70
|
+
export function createMemoryModelCache() {
|
|
71
|
+
const map = new Map();
|
|
72
|
+
return {
|
|
73
|
+
async get(id) {
|
|
74
|
+
const entry = map.get(id);
|
|
75
|
+
return entry ? { ...entry, bytes: entry.bytes.slice(0) } : undefined;
|
|
76
|
+
},
|
|
77
|
+
async put(id, entry) {
|
|
78
|
+
map.set(id, { ...entry, bytes: entry.bytes.slice(0), id, storedAt: Date.now() });
|
|
79
|
+
},
|
|
80
|
+
async has(id) {
|
|
81
|
+
return map.has(id);
|
|
82
|
+
},
|
|
83
|
+
async delete(id) {
|
|
84
|
+
map.delete(id);
|
|
85
|
+
},
|
|
86
|
+
async keys() {
|
|
87
|
+
return [...map.keys()];
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=modelCache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"modelCache.js","sourceRoot":"","sources":["../../src/inference/modelCache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAyDhD,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,KAAK,YAAY,WAAW,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,sBAAsB,CAAC;AAC1G,CAAC;AAED,iFAAiF;AACjF,SAAS,OAAO,CAAC,EAAU,EAAE,KAAc;IACzC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5D,IAAI,aAAa,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC3F,OAAO,KAAwB,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAO,GAAsB,EAAE;IAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,IAAK,UAAyC,CAAC,SAAS,CAAC;IAC1F,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,uBAAuB;QACjD,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,QAAQ;QACxC,OAAO,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC;KAChC,CAAC;IACF,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAE3C,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,EAAE;YACV,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK;gBAAE,OAAO,SAAS,CAAC;YAC7B,OAAO,OAAO,CAAC,EAAE,EAAE,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAwB,CAAC,CAAC,CAAC;QAC3F,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK;YACjB,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,MAAM,KAAK,GAAkC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACxG,MAAM,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,EAAE;YACV,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK;gBAAE,OAAO,KAAK,CAAC;YACzB,OAAO,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,EAAE;YACb,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,CAAC,IAAI;YACR,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK;gBAAE,OAAO,EAAE,CAAC;YACtB,OAAO,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1E,CAAC;KACF,CAAC;AACJ,CAAC;AAED,kHAAkH;AAClH,MAAM,UAAU,sBAAsB;IACpC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC/C,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,EAAE;YACV,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC1B,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACvE,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK;YACjB,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,EAAE;YACV,OAAO,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,EAAE;YACb,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;QACD,KAAK,CAAC,IAAI;YACR,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACzB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1 — Local CPU Inference Engine: worker protocol.
|
|
3
|
+
*
|
|
4
|
+
* Messages between the host-side `InferenceClient` and the inference worker.
|
|
5
|
+
* Structured so a future Phase 4/5 can add streaming KV-cache and prefill
|
|
6
|
+
* messages without reworking the envelope.
|
|
7
|
+
*/
|
|
8
|
+
import type { ChatMessage, GenerateOptions, ModelRef, RuntimeCapabilities } from './types.js';
|
|
9
|
+
/** Generate options as they cross the wire: an AbortSignal is not structured-cloneable, so it never does. */
|
|
10
|
+
export type WireGenerateOptions = Omit<GenerateOptions, 'signal'>;
|
|
11
|
+
export declare const INFERENCE_PROBE = "inference:probe";
|
|
12
|
+
export declare const INFERENCE_LOAD = "inference:load";
|
|
13
|
+
export declare const INFERENCE_GENERATE = "inference:generate";
|
|
14
|
+
export declare const INFERENCE_CANCEL = "inference:cancel";
|
|
15
|
+
export declare const INFERENCE_DISPOSE_MODEL = "inference:dispose-model";
|
|
16
|
+
export declare const INFERENCE_CAPABILITIES = "inference:capabilities";
|
|
17
|
+
export declare const INFERENCE_READY = "inference:ready";
|
|
18
|
+
export declare const INFERENCE_ERROR = "inference:error";
|
|
19
|
+
export declare const INFERENCE_LOG = "inference:log";
|
|
20
|
+
export declare const INFERENCE_TOKEN = "inference:token";
|
|
21
|
+
export declare const INFERENCE_DONE = "inference:done";
|
|
22
|
+
export interface ProbeRequest {
|
|
23
|
+
type: typeof INFERENCE_PROBE;
|
|
24
|
+
id: number;
|
|
25
|
+
}
|
|
26
|
+
export interface LoadRequest {
|
|
27
|
+
type: typeof INFERENCE_LOAD;
|
|
28
|
+
id: number;
|
|
29
|
+
ref: ModelRef;
|
|
30
|
+
/** Model bytes, transferred by the host (ArrayBuffer transfer). */
|
|
31
|
+
weights: ArrayBuffer;
|
|
32
|
+
}
|
|
33
|
+
export interface GenerateRequest {
|
|
34
|
+
type: typeof INFERENCE_GENERATE;
|
|
35
|
+
id: number;
|
|
36
|
+
messages: ChatMessage[];
|
|
37
|
+
options?: WireGenerateOptions;
|
|
38
|
+
}
|
|
39
|
+
export interface CancelRequest {
|
|
40
|
+
type: typeof INFERENCE_CANCEL;
|
|
41
|
+
id: number;
|
|
42
|
+
/** The generate request to cancel; absent = whatever generation is running. */
|
|
43
|
+
requestId?: number;
|
|
44
|
+
}
|
|
45
|
+
export interface DisposeModelRequest {
|
|
46
|
+
type: typeof INFERENCE_DISPOSE_MODEL;
|
|
47
|
+
id: number;
|
|
48
|
+
}
|
|
49
|
+
export type InferenceRequest = ProbeRequest | LoadRequest | GenerateRequest | CancelRequest | DisposeModelRequest;
|
|
50
|
+
export interface ReadyReply {
|
|
51
|
+
type: typeof INFERENCE_READY;
|
|
52
|
+
id: number;
|
|
53
|
+
capabilities: RuntimeCapabilities;
|
|
54
|
+
}
|
|
55
|
+
export interface ErrorReply {
|
|
56
|
+
type: typeof INFERENCE_ERROR;
|
|
57
|
+
id: number;
|
|
58
|
+
message: string;
|
|
59
|
+
}
|
|
60
|
+
export interface LogReply {
|
|
61
|
+
type: typeof INFERENCE_LOG;
|
|
62
|
+
id: number;
|
|
63
|
+
message: string;
|
|
64
|
+
}
|
|
65
|
+
export interface TokenReply {
|
|
66
|
+
type: typeof INFERENCE_TOKEN;
|
|
67
|
+
id: number;
|
|
68
|
+
requestId: number;
|
|
69
|
+
token: string;
|
|
70
|
+
index: number;
|
|
71
|
+
}
|
|
72
|
+
export interface DoneReply {
|
|
73
|
+
type: typeof INFERENCE_DONE;
|
|
74
|
+
id: number;
|
|
75
|
+
requestId: number;
|
|
76
|
+
promptTokens: number;
|
|
77
|
+
/** True when `promptTokens` is the ~4 chars/token heuristic, not the backend's tokenizer count. */
|
|
78
|
+
promptTokensEstimated: boolean;
|
|
79
|
+
generatedTokens: number;
|
|
80
|
+
ttftMs: number;
|
|
81
|
+
decodeMs: number;
|
|
82
|
+
tokensPerSecond: number;
|
|
83
|
+
}
|
|
84
|
+
export type InferenceReply = ReadyReply | ErrorReply | LogReply | TokenReply | DoneReply;
|
|
85
|
+
/** Stats part of a DoneReply, shared with the public API. */
|
|
86
|
+
export interface GenerationStatsPayload {
|
|
87
|
+
promptTokens: number;
|
|
88
|
+
promptTokensEstimated: boolean;
|
|
89
|
+
generatedTokens: number;
|
|
90
|
+
ttftMs: number;
|
|
91
|
+
decodeMs: number;
|
|
92
|
+
tokensPerSecond: number;
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=protocol.d.ts.map
|