@seanhogg/builderforce-webcontainers 2026.10.12 → 2026.10.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/adaptive/controller.d.ts +39 -0
- package/dist/adaptive/controller.d.ts.map +1 -0
- package/dist/adaptive/controller.js +108 -0
- package/dist/adaptive/controller.js.map +1 -0
- package/dist/adaptive/costModel.d.ts +37 -0
- package/dist/adaptive/costModel.d.ts.map +1 -0
- package/dist/adaptive/costModel.js +102 -0
- package/dist/adaptive/costModel.js.map +1 -0
- package/dist/adaptive/planner.d.ts +22 -0
- package/dist/adaptive/planner.d.ts.map +1 -0
- package/dist/adaptive/planner.js +114 -0
- package/dist/adaptive/planner.js.map +1 -0
- package/dist/adaptive/types.d.ts +138 -0
- package/dist/adaptive/types.d.ts.map +1 -0
- package/dist/adaptive/types.js +12 -0
- package/dist/adaptive/types.js.map +1 -0
- package/dist/assets.js +2 -2
- package/dist/check/cachedFetch.js.map +1 -1
- package/dist/check/index.d.ts +6 -1
- package/dist/check/index.d.ts.map +1 -1
- package/dist/check/index.js +6 -1
- package/dist/check/index.js.map +1 -1
- package/dist/componentCompilers.d.ts.map +1 -1
- package/dist/componentCompilers.js.map +1 -1
- package/dist/esbuildWasm.js.map +1 -1
- package/dist/host.d.ts +11 -5
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +19 -8
- package/dist/host.js.map +1 -1
- package/dist/inference/backends/mock.d.ts +24 -0
- package/dist/inference/backends/mock.d.ts.map +1 -0
- package/dist/inference/backends/mock.js +55 -0
- package/dist/inference/backends/mock.js.map +1 -0
- package/dist/inference/backends/onnx.d.ts +33 -0
- package/dist/inference/backends/onnx.d.ts.map +1 -0
- package/dist/inference/backends/onnx.js +60 -0
- package/dist/inference/backends/onnx.js.map +1 -0
- package/dist/inference/backends/webllm.d.ts +27 -0
- package/dist/inference/backends/webllm.d.ts.map +1 -0
- package/dist/inference/backends/webllm.js +52 -0
- package/dist/inference/backends/webllm.js.map +1 -0
- package/dist/inference/backends/wllama.d.ts +29 -0
- package/dist/inference/backends/wllama.d.ts.map +1 -0
- package/dist/inference/backends/wllama.js +51 -0
- package/dist/inference/backends/wllama.js.map +1 -0
- package/dist/inference/capabilities.d.ts +20 -0
- package/dist/inference/capabilities.d.ts.map +1 -0
- package/dist/inference/capabilities.js +33 -0
- package/dist/inference/capabilities.js.map +1 -0
- package/dist/inference/client.d.ts +45 -0
- package/dist/inference/client.d.ts.map +1 -0
- package/dist/inference/client.js +201 -0
- package/dist/inference/client.js.map +1 -0
- package/dist/inference/modelCache.d.ts +71 -0
- package/dist/inference/modelCache.d.ts.map +1 -0
- package/dist/inference/modelCache.js +91 -0
- package/dist/inference/modelCache.js.map +1 -0
- package/dist/inference/protocol.d.ts +94 -0
- package/dist/inference/protocol.d.ts.map +1 -0
- package/dist/inference/protocol.js +19 -0
- package/dist/inference/protocol.js.map +1 -0
- package/dist/inference/tokenStream.d.ts +26 -0
- package/dist/inference/tokenStream.d.ts.map +1 -0
- package/dist/inference/tokenStream.js +79 -0
- package/dist/inference/tokenStream.js.map +1 -0
- package/dist/inference/types.d.ts +108 -0
- package/dist/inference/types.d.ts.map +1 -0
- package/dist/inference/types.js +8 -0
- package/dist/inference/types.js.map +1 -0
- package/dist/inference/worker.bundle.js +5 -0
- package/dist/inference/worker.d.ts +35 -0
- package/dist/inference/worker.d.ts.map +1 -0
- package/dist/inference/worker.js +157 -0
- package/dist/inference/worker.js.map +1 -0
- package/dist/inference/workerEntry.d.ts +15 -0
- package/dist/inference/workerEntry.d.ts.map +1 -0
- package/dist/inference/workerEntry.js +17 -0
- package/dist/inference/workerEntry.js.map +1 -0
- package/dist/kvcache/advanced.d.ts +39 -0
- package/dist/kvcache/advanced.d.ts.map +1 -0
- package/dist/kvcache/advanced.js +59 -0
- package/dist/kvcache/advanced.js.map +1 -0
- package/dist/kvcache/compatibility.d.ts +14 -0
- package/dist/kvcache/compatibility.d.ts.map +1 -0
- package/dist/kvcache/compatibility.js +44 -0
- package/dist/kvcache/compatibility.js.map +1 -0
- package/dist/kvcache/lru.d.ts +45 -0
- package/dist/kvcache/lru.d.ts.map +1 -0
- package/dist/kvcache/lru.js +91 -0
- package/dist/kvcache/lru.js.map +1 -0
- package/dist/kvcache/paged.d.ts +86 -0
- package/dist/kvcache/paged.d.ts.map +1 -0
- package/dist/kvcache/paged.js +238 -0
- package/dist/kvcache/paged.js.map +1 -0
- package/dist/kvcache/quant.d.ts +30 -0
- package/dist/kvcache/quant.d.ts.map +1 -0
- package/dist/kvcache/quant.js +73 -0
- package/dist/kvcache/quant.js.map +1 -0
- package/dist/kvcache/radix.d.ts +28 -0
- package/dist/kvcache/radix.d.ts.map +1 -0
- package/dist/kvcache/radix.js +58 -0
- package/dist/kvcache/radix.js.map +1 -0
- package/dist/kvcache/runtime.d.ts +11 -0
- package/dist/kvcache/runtime.d.ts.map +1 -0
- package/dist/kvcache/runtime.js +204 -0
- package/dist/kvcache/runtime.js.map +1 -0
- package/dist/kvcache/types.d.ts +151 -0
- package/dist/kvcache/types.d.ts.map +1 -0
- package/dist/kvcache/types.js +10 -0
- package/dist/kvcache/types.js.map +1 -0
- package/dist/learning/fingerprint.d.ts +17 -0
- package/dist/learning/fingerprint.d.ts.map +1 -0
- package/dist/learning/fingerprint.js +46 -0
- package/dist/learning/fingerprint.js.map +1 -0
- package/dist/learning/index.d.ts +5 -0
- package/dist/learning/index.d.ts.map +1 -0
- package/dist/learning/index.js +3 -0
- package/dist/learning/index.js.map +1 -0
- package/dist/learning/learner.d.ts +21 -0
- package/dist/learning/learner.d.ts.map +1 -0
- package/dist/learning/learner.js +272 -0
- package/dist/learning/learner.js.map +1 -0
- package/dist/learning/ledger.d.ts +26 -0
- package/dist/learning/ledger.d.ts.map +1 -0
- package/dist/learning/ledger.js +40 -0
- package/dist/learning/ledger.js.map +1 -0
- package/dist/learning/selection.d.ts +44 -0
- package/dist/learning/selection.d.ts.map +1 -0
- package/dist/learning/selection.js +111 -0
- package/dist/learning/selection.js.map +1 -0
- package/dist/learning/stats.d.ts +19 -0
- package/dist/learning/stats.d.ts.map +1 -0
- package/dist/learning/stats.js +31 -0
- package/dist/learning/stats.js.map +1 -0
- package/dist/learning/types.d.ts +133 -0
- package/dist/learning/types.d.ts.map +1 -0
- package/dist/learning/types.js +8 -0
- package/dist/learning/types.js.map +1 -0
- package/dist/memory/envProbe.d.ts +16 -0
- package/dist/memory/envProbe.d.ts.map +1 -0
- package/dist/memory/envProbe.js +28 -0
- package/dist/memory/envProbe.js.map +1 -0
- package/dist/memory/estimates.d.ts +46 -0
- package/dist/memory/estimates.d.ts.map +1 -0
- package/dist/memory/estimates.js +72 -0
- package/dist/memory/estimates.js.map +1 -0
- package/dist/memory/eviction.d.ts +42 -0
- package/dist/memory/eviction.d.ts.map +1 -0
- package/dist/memory/eviction.js +143 -0
- package/dist/memory/eviction.js.map +1 -0
- package/dist/memory/manager.d.ts +34 -0
- package/dist/memory/manager.d.ts.map +1 -0
- package/dist/memory/manager.js +346 -0
- package/dist/memory/manager.js.map +1 -0
- package/dist/memory/managerTypes.d.ts +49 -0
- package/dist/memory/managerTypes.d.ts.map +1 -0
- package/dist/memory/managerTypes.js +7 -0
- package/dist/memory/managerTypes.js.map +1 -0
- package/dist/memory/pressure.d.ts +56 -0
- package/dist/memory/pressure.d.ts.map +1 -0
- package/dist/memory/pressure.js +80 -0
- package/dist/memory/pressure.js.map +1 -0
- package/dist/memory/spillTier.d.ts +47 -0
- package/dist/memory/spillTier.d.ts.map +1 -0
- package/dist/memory/spillTier.js +128 -0
- package/dist/memory/spillTier.js.map +1 -0
- package/dist/memory/storageSink.d.ts +69 -0
- package/dist/memory/storageSink.d.ts.map +1 -0
- package/dist/memory/storageSink.js +221 -0
- package/dist/memory/storageSink.js.map +1 -0
- package/dist/memory/types.d.ts +202 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +15 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/memory/victims.d.ts +32 -0
- package/dist/memory/victims.d.ts.map +1 -0
- package/dist/memory/victims.js +74 -0
- package/dist/memory/victims.js.map +1 -0
- package/dist/memory/workspace.d.ts +34 -0
- package/dist/memory/workspace.d.ts.map +1 -0
- package/dist/memory/workspace.js +55 -0
- package/dist/memory/workspace.js.map +1 -0
- package/dist/memoryIntegration/index.d.ts +8 -0
- package/dist/memoryIntegration/index.d.ts.map +1 -0
- package/dist/memoryIntegration/index.js +4 -0
- package/dist/memoryIntegration/index.js.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts +43 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.js +285 -0
- package/dist/memoryIntegration/knowledgeMemory.js.map +1 -0
- package/dist/memoryIntegration/persistence.d.ts +15 -0
- package/dist/memoryIntegration/persistence.d.ts.map +1 -0
- package/dist/memoryIntegration/persistence.js +43 -0
- package/dist/memoryIntegration/persistence.js.map +1 -0
- package/dist/memoryIntegration/text.d.ts +57 -0
- package/dist/memoryIntegration/text.d.ts.map +1 -0
- package/dist/memoryIntegration/text.js +179 -0
- package/dist/memoryIntegration/text.js.map +1 -0
- package/dist/memoryIntegration/types.d.ts +131 -0
- package/dist/memoryIntegration/types.d.ts.map +1 -0
- package/dist/memoryIntegration/types.js +11 -0
- package/dist/memoryIntegration/types.js.map +1 -0
- package/dist/ml.d.ts +52 -0
- package/dist/ml.d.ts.map +1 -0
- package/dist/ml.js +33 -0
- package/dist/ml.js.map +1 -0
- package/dist/node/childRelay.d.ts +10 -0
- package/dist/node/childRelay.d.ts.map +1 -1
- package/dist/node/childRelay.js +12 -2
- package/dist/node/childRelay.js.map +1 -1
- package/dist/node/index.js.map +1 -1
- package/dist/node/portResponder.d.ts +2 -0
- package/dist/node/portResponder.d.ts.map +1 -1
- package/dist/node/portResponder.js +3 -3
- package/dist/node/portResponder.js.map +1 -1
- package/dist/node/processHost.d.ts +4 -1
- package/dist/node/processHost.d.ts.map +1 -1
- package/dist/node/processHost.js +19 -16
- package/dist/node/processHost.js.map +1 -1
- package/dist/node/protocol.d.ts +1 -0
- package/dist/node/protocol.d.ts.map +1 -1
- package/dist/node/responseWire.d.ts.map +1 -1
- package/dist/node/worker.js +127 -120
- package/dist/node/workerProcess.js +1 -1
- package/dist/node/workerProcess.js.map +1 -1
- package/dist/prefetch/admission.d.ts +41 -0
- package/dist/prefetch/admission.d.ts.map +1 -0
- package/dist/prefetch/admission.js +49 -0
- package/dist/prefetch/admission.js.map +1 -0
- package/dist/prefetch/loadRegistry.d.ts +56 -0
- package/dist/prefetch/loadRegistry.d.ts.map +1 -0
- package/dist/prefetch/loadRegistry.js +108 -0
- package/dist/prefetch/loadRegistry.js.map +1 -0
- package/dist/prefetch/predictor.d.ts +40 -0
- package/dist/prefetch/predictor.d.ts.map +1 -0
- package/dist/prefetch/predictor.js +178 -0
- package/dist/prefetch/predictor.js.map +1 -0
- package/dist/prefetch/prefetcher.d.ts +37 -0
- package/dist/prefetch/prefetcher.d.ts.map +1 -0
- package/dist/prefetch/prefetcher.js +339 -0
- package/dist/prefetch/prefetcher.js.map +1 -0
- package/dist/prefetch/residentStore.d.ts +49 -0
- package/dist/prefetch/residentStore.d.ts.map +1 -0
- package/dist/prefetch/residentStore.js +87 -0
- package/dist/prefetch/residentStore.js.map +1 -0
- package/dist/prefetch/selfTuner.d.ts +40 -0
- package/dist/prefetch/selfTuner.d.ts.map +1 -0
- package/dist/prefetch/selfTuner.js +78 -0
- package/dist/prefetch/selfTuner.js.map +1 -0
- package/dist/prefetch/stats.d.ts +52 -0
- package/dist/prefetch/stats.d.ts.map +1 -0
- package/dist/prefetch/stats.js +72 -0
- package/dist/prefetch/stats.js.map +1 -0
- package/dist/prefetch/streaming.d.ts +132 -0
- package/dist/prefetch/streaming.d.ts.map +1 -0
- package/dist/prefetch/streaming.js +141 -0
- package/dist/prefetch/streaming.js.map +1 -0
- package/dist/prefetch/targetIndex.d.ts +18 -0
- package/dist/prefetch/targetIndex.d.ts.map +1 -0
- package/dist/prefetch/targetIndex.js +32 -0
- package/dist/prefetch/targetIndex.js.map +1 -0
- package/dist/prefetch/types.d.ts +283 -0
- package/dist/prefetch/types.d.ts.map +1 -0
- package/dist/prefetch/types.js +19 -0
- package/dist/prefetch/types.js.map +1 -0
- package/dist/previewBridge.d.ts +6 -0
- package/dist/previewBridge.d.ts.map +1 -1
- package/dist/previewBridge.js +25 -5
- package/dist/previewBridge.js.map +1 -1
- package/dist/previewMessages.d.ts +4 -3
- package/dist/previewMessages.d.ts.map +1 -1
- package/dist/previewMessages.js +12 -6
- package/dist/previewMessages.js.map +1 -1
- package/dist/process-worker.js +127 -120
- package/dist/relay.html +1 -1
- package/dist/shared/clock.d.ts +7 -0
- package/dist/shared/clock.d.ts.map +1 -0
- package/dist/shared/clock.js +10 -0
- package/dist/shared/clock.js.map +1 -0
- package/dist/shared/hardwareProbe.d.ts +26 -0
- package/dist/shared/hardwareProbe.d.ts.map +1 -0
- package/dist/shared/hardwareProbe.js +62 -0
- package/dist/shared/hardwareProbe.js.map +1 -0
- package/dist/shared/idb.d.ts +41 -0
- package/dist/shared/idb.d.ts.map +1 -0
- package/dist/shared/idb.js +184 -0
- package/dist/shared/idb.js.map +1 -0
- package/dist/shared/tokens.d.ts +7 -0
- package/dist/shared/tokens.d.ts.map +1 -0
- package/dist/shared/tokens.js +9 -0
- package/dist/shared/tokens.js.map +1 -0
- package/dist/snapshotStore.d.ts +2 -1
- package/dist/snapshotStore.d.ts.map +1 -1
- package/dist/snapshotStore.js +27 -12
- package/dist/snapshotStore.js.map +1 -1
- package/dist/styleCompilers.d.ts.map +1 -1
- package/dist/styleCompilers.js.map +1 -1
- package/dist/transport.js.map +1 -1
- package/package.json +11 -5
- package/src/adaptive/controller.ts +151 -0
- package/src/adaptive/costModel.ts +131 -0
- package/src/adaptive/planner.ts +127 -0
- package/src/adaptive/types.ts +137 -0
- package/src/check/index.ts +6 -1
- package/src/host.ts +33 -11
- package/src/inference/backends/mock.ts +77 -0
- package/src/inference/backends/onnx.ts +91 -0
- package/src/inference/backends/webllm.ts +79 -0
- package/src/inference/backends/wllama.ts +79 -0
- package/src/inference/capabilities.ts +46 -0
- package/src/inference/client.ts +276 -0
- package/src/inference/modelCache.ts +145 -0
- package/src/inference/protocol.ts +119 -0
- package/src/inference/tokenStream.ts +92 -0
- package/src/inference/types.ts +115 -0
- package/src/inference/worker.ts +191 -0
- package/src/inference/workerEntry.ts +18 -0
- package/src/kvcache/advanced.ts +86 -0
- package/src/kvcache/compatibility.ts +51 -0
- package/src/kvcache/lru.ts +100 -0
- package/src/kvcache/paged.ts +270 -0
- package/src/kvcache/quant.ts +88 -0
- package/src/kvcache/radix.ts +68 -0
- package/src/kvcache/runtime.ts +239 -0
- package/src/kvcache/types.ts +167 -0
- package/src/learning/fingerprint.ts +50 -0
- package/src/learning/index.ts +15 -0
- package/src/learning/learner.ts +304 -0
- package/src/learning/ledger.ts +54 -0
- package/src/learning/selection.ts +137 -0
- package/src/learning/stats.ts +39 -0
- package/src/learning/types.ts +142 -0
- package/src/memory/envProbe.ts +30 -0
- package/src/memory/estimates.ts +98 -0
- package/src/memory/eviction.ts +145 -0
- package/src/memory/manager.ts +380 -0
- package/src/memory/managerTypes.ts +61 -0
- package/src/memory/pressure.ts +120 -0
- package/src/memory/spillTier.ts +159 -0
- package/src/memory/storageSink.ts +289 -0
- package/src/memory/types.ts +217 -0
- package/src/memory/victims.ts +80 -0
- package/src/memory/workspace.ts +74 -0
- package/src/memoryIntegration/index.ts +7 -0
- package/src/memoryIntegration/knowledgeMemory.ts +339 -0
- package/src/memoryIntegration/persistence.ts +54 -0
- package/src/memoryIntegration/text.ts +185 -0
- package/src/memoryIntegration/types.ts +145 -0
- package/src/ml.ts +180 -0
- package/src/node/childRelay.ts +16 -3
- package/src/node/portResponder.ts +5 -3
- package/src/node/processHost.ts +16 -15
- package/src/node/protocol.ts +1 -1
- package/src/node/workerProcess.ts +1 -1
- package/src/prefetch/admission.ts +74 -0
- package/src/prefetch/loadRegistry.ts +156 -0
- package/src/prefetch/predictor.ts +199 -0
- package/src/prefetch/prefetcher.ts +369 -0
- package/src/prefetch/residentStore.ts +121 -0
- package/src/prefetch/selfTuner.ts +110 -0
- package/src/prefetch/stats.ts +120 -0
- package/src/prefetch/streaming.ts +231 -0
- package/src/prefetch/targetIndex.ts +40 -0
- package/src/prefetch/types.ts +307 -0
- package/src/previewBridge.ts +25 -5
- package/src/previewMessages.ts +11 -6
- package/src/shared/clock.ts +9 -0
- package/src/shared/hardwareProbe.ts +68 -0
- package/src/shared/idb.ts +203 -0
- package/src/shared/tokens.ts +8 -0
- package/src/snapshotStore.ts +25 -10
package/README.md
CHANGED
|
@@ -33,3 +33,25 @@ the project in a Web Worker. `.vue` and `.svelte` files need no setup. See the
|
|
|
33
33
|
|
|
34
34
|
The package re-exports everything from
|
|
35
35
|
[`@seanhogg/builderforce-webcontainers-core`](../core).
|
|
36
|
+
|
|
37
|
+
## The `./ml` subpath
|
|
38
|
+
|
|
39
|
+
The in-browser ML runtime lives on its own subpath so the root stays the dev
|
|
40
|
+
runtime only: local inference (`createInferenceClient`, backends, the IndexedDB
|
|
41
|
+
`createModelCache`), the adaptive execution controller, memory residency, the
|
|
42
|
+
knowledge/retrieval layer, prefetch, KV-cache optimisation and online learning.
|
|
43
|
+
|
|
44
|
+
```ts
|
|
45
|
+
import { createInferenceClient, createModelCache } from '@seanhogg/builderforce-webcontainers/ml';
|
|
46
|
+
|
|
47
|
+
const client = createInferenceClient({
|
|
48
|
+
// Optional: adopt an existing weights database instead of `bfwc-inference-models`.
|
|
49
|
+
cache: createModelCache({ dbName: 'my-weights', storeName: 'weights', dbVersion: 1 }),
|
|
50
|
+
});
|
|
51
|
+
await client.load({ id: 'tiny-q4', paramsB: 1.1, quantization: 'q4', source: { kind: 'url', url: modelUrl } });
|
|
52
|
+
for await (const { token } of client.generate([{ role: 'user', content: 'hi' }], { signal })) render(token);
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The client starts the self-contained worker bundle that ships beside it. If your
|
|
56
|
+
bundler does not copy it, serve `@seanhogg/builderforce-webcontainers/ml/worker.js`
|
|
57
|
+
yourself and pass `createWorker: () => new Worker(url)`.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AHEC controller — the Adaptive Heterogeneous Execution Controller.
|
|
3
|
+
*
|
|
4
|
+
* Owns the decide -> execute -> measure loop. For each request it:
|
|
5
|
+
* 1. probes hardware facts,
|
|
6
|
+
* 2. builds an explainable execution plan (cost-based, CPU-first),
|
|
7
|
+
* 3. routes local vs remote (local-only unless configured otherwise),
|
|
8
|
+
* 4. drives an injected executor,
|
|
9
|
+
* 5. measures its own overhead against the benefit the plan provided.
|
|
10
|
+
*
|
|
11
|
+
* CPU-only is a first-class configuration; heterogeneous execution is enabled
|
|
12
|
+
* only when the plan estimates it beats CPU-only by the configured margin.
|
|
13
|
+
*/
|
|
14
|
+
import type { ControllerConfig, ExecutionDecision, ExecutionOutcome, ExecutorLike, HardwareProfile } from './types.js';
|
|
15
|
+
/** Probe hardware facts from the environment (never throws; facts not gates). */
|
|
16
|
+
export declare function probeHardware(): HardwareProfile;
|
|
17
|
+
export interface AdaptiveController {
|
|
18
|
+
/** Decide (and optionally execute) one request. */
|
|
19
|
+
execute(model: {
|
|
20
|
+
id: string;
|
|
21
|
+
paramsB: number;
|
|
22
|
+
quantization: 'q4' | 'q5' | 'q8' | 'fp16';
|
|
23
|
+
}, request: {
|
|
24
|
+
prompt: string;
|
|
25
|
+
maxTokens?: number;
|
|
26
|
+
signal?: AbortSignal;
|
|
27
|
+
}): Promise<ExecutionOutcome>;
|
|
28
|
+
/** Aggregate controller statistics (overhead vs benefit accounting). */
|
|
29
|
+
stats(): {
|
|
30
|
+
runs: number;
|
|
31
|
+
totalOverheadMs: number;
|
|
32
|
+
totalBenefitMs: number;
|
|
33
|
+
lastDecision?: ExecutionDecision;
|
|
34
|
+
};
|
|
35
|
+
/** The most recent explainable decision, for audit/UI. */
|
|
36
|
+
lastDecision(): ExecutionDecision | undefined;
|
|
37
|
+
}
|
|
38
|
+
export declare function createAdaptiveController(executor: ExecutorLike, config?: ControllerConfig): AdaptiveController;
|
|
39
|
+
//# sourceMappingURL=controller.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"controller.d.ts","sourceRoot":"","sources":["../../src/adaptive/controller.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,YAAY,EACZ,eAAe,EAEhB,MAAM,YAAY,CAAC;AAwCpB,iFAAiF;AACjF,wBAAgB,aAAa,IAAI,eAAe,CAS/C;AAED,MAAM,WAAW,kBAAkB;IACjC,mDAAmD;IACnD,OAAO,CACL,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAA;KAAE,EACjF,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GACpE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC7B,wEAAwE;IACxE,KAAK,IAAI;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,iBAAiB,CAAA;KAAE,CAAC;IAC7G,0DAA0D;IAC1D,YAAY,IAAI,iBAAiB,GAAG,SAAS,CAAC;CAC/C;AAED,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,YAAY,EACtB,MAAM,GAAE,gBAAqB,GAC5B,kBAAkB,CAyDpB"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AHEC controller — the Adaptive Heterogeneous Execution Controller.
|
|
3
|
+
*
|
|
4
|
+
* Owns the decide -> execute -> measure loop. For each request it:
|
|
5
|
+
* 1. probes hardware facts,
|
|
6
|
+
* 2. builds an explainable execution plan (cost-based, CPU-first),
|
|
7
|
+
* 3. routes local vs remote (local-only unless configured otherwise),
|
|
8
|
+
* 4. drives an injected executor,
|
|
9
|
+
* 5. measures its own overhead against the benefit the plan provided.
|
|
10
|
+
*
|
|
11
|
+
* CPU-only is a first-class configuration; heterogeneous execution is enabled
|
|
12
|
+
* only when the plan estimates it beats CPU-only by the configured margin.
|
|
13
|
+
*/
|
|
14
|
+
import { planExecution } from './planner.js';
|
|
15
|
+
import { estimateWeightBytes } from '../memory/estimates.js';
|
|
16
|
+
import { now as nowMs } from '../shared/clock.js';
|
|
17
|
+
import { estimateTokens } from '../shared/tokens.js';
|
|
18
|
+
import { detectCrossOriginIsolated, detectSharedArrayBuffer, detectWasmSimd, detectWebGpu, probeDeviceMemoryBytes, probeHardwareConcurrency, } from '../shared/hardwareProbe.js';
|
|
19
|
+
/** A linear layer graph derived from the workload, for the planner to partition. */
|
|
20
|
+
function defaultGraph(workload) {
|
|
21
|
+
// A coarse 12-layer transformer-style graph sized to the model. The planner
|
|
22
|
+
// only needs relative per-layer costs to pick a split point.
|
|
23
|
+
const layers = 12;
|
|
24
|
+
const attentionFraction = workload.attentionFraction ?? 0.25;
|
|
25
|
+
const perLayerCpu = workload.residentBytes > 0 ? 0 : 0; // placeholder, costs set below
|
|
26
|
+
void perLayerCpu;
|
|
27
|
+
const base = (workload.promptTokens + workload.decodeTokens) / layers;
|
|
28
|
+
return {
|
|
29
|
+
layers: Array.from({ length: layers }, (_, i) => {
|
|
30
|
+
const isAttention = (i % Math.max(1, Math.round(1 / Math.max(attentionFraction, 0.01)))) === 0;
|
|
31
|
+
const cpuCostMs = base * (isAttention ? 1.2 : 1.0);
|
|
32
|
+
return {
|
|
33
|
+
id: `L${i}`,
|
|
34
|
+
cpuCostMs,
|
|
35
|
+
gpuCostMs: cpuCostMs * 0.3,
|
|
36
|
+
inputBytes: 1024 * workload.promptTokens * 2,
|
|
37
|
+
};
|
|
38
|
+
}),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/** RAM assumed when `navigator.deviceMemory` is unreported (4 GiB). */
|
|
42
|
+
const DEFAULT_RAM_BYTES = 4 * 1024 * 1024 * 1024;
|
|
43
|
+
/** Probe hardware facts from the environment (never throws; facts not gates). */
|
|
44
|
+
export function probeHardware() {
|
|
45
|
+
return {
|
|
46
|
+
cores: probeHardwareConcurrency() ?? 4,
|
|
47
|
+
wasmSimd: detectWasmSimd(),
|
|
48
|
+
webgpu: detectWebGpu(),
|
|
49
|
+
crossOriginIsolated: detectCrossOriginIsolated(),
|
|
50
|
+
sharedArrayBuffer: detectSharedArrayBuffer(),
|
|
51
|
+
ramBytes: probeDeviceMemoryBytes() ?? DEFAULT_RAM_BYTES,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
export function createAdaptiveController(executor, config = {}) {
|
|
55
|
+
// Probe once; the controller treats hardware as stable within a session.
|
|
56
|
+
const hardware = config.hardwareOverride ?? probeHardware();
|
|
57
|
+
let last;
|
|
58
|
+
let runs = 0;
|
|
59
|
+
let totalOverheadMs = 0;
|
|
60
|
+
let totalBenefitMs = 0;
|
|
61
|
+
return {
|
|
62
|
+
async execute(model, request) {
|
|
63
|
+
const workload = {
|
|
64
|
+
paramsB: model.paramsB,
|
|
65
|
+
quantization: model.quantization,
|
|
66
|
+
promptTokens: Math.max(1, estimateTokens(request.prompt)),
|
|
67
|
+
decodeTokens: request.maxTokens ?? 128,
|
|
68
|
+
residentBytes: estimateResidentBytes(model),
|
|
69
|
+
};
|
|
70
|
+
const planStart = nowMs();
|
|
71
|
+
const plan = planExecution({ workload, hardware, graph: defaultGraph(workload), config });
|
|
72
|
+
const overheadMs = nowMs() - planStart;
|
|
73
|
+
const route = config.allowRemote === true && hardware.webgpu === false && workload.paramsB > 13
|
|
74
|
+
? 'remote'
|
|
75
|
+
: 'local';
|
|
76
|
+
const decision = { plan, route, hardware, workload };
|
|
77
|
+
if (route === 'remote') {
|
|
78
|
+
decision.plan.reasoning.push('remote route: model too large for local memory and remote allowed');
|
|
79
|
+
}
|
|
80
|
+
last = decision;
|
|
81
|
+
config.onDecision?.(decision);
|
|
82
|
+
const execStart = nowMs();
|
|
83
|
+
const stream = executor.generate({ id: model.id, paramsB: model.paramsB, quantization: model.quantization }, { prompt: request.prompt, maxTokens: request.maxTokens ?? 128, signal: request.signal });
|
|
84
|
+
let tokens = 0;
|
|
85
|
+
for await (const event of stream) {
|
|
86
|
+
if (event.token)
|
|
87
|
+
tokens += 1;
|
|
88
|
+
}
|
|
89
|
+
const measuredCostMs = nowMs() - execStart;
|
|
90
|
+
const benefitMs = decision.plan.estimatedCostMs - measuredCostMs;
|
|
91
|
+
runs += 1;
|
|
92
|
+
totalOverheadMs += overheadMs;
|
|
93
|
+
totalBenefitMs += benefitMs;
|
|
94
|
+
return { decision, measuredCostMs, overheadMs, benefitMs, tokens };
|
|
95
|
+
},
|
|
96
|
+
stats() {
|
|
97
|
+
return { runs, totalOverheadMs, totalBenefitMs, lastDecision: last };
|
|
98
|
+
},
|
|
99
|
+
lastDecision() {
|
|
100
|
+
return last;
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** Weights + ~20% KV/activation headroom, from the canonical memory estimate. */
|
|
105
|
+
function estimateResidentBytes(model) {
|
|
106
|
+
return estimateWeightBytes(model.paramsB, model.quantization, 1.2);
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=controller.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"controller.js","sourceRoot":"","sources":["../../src/adaptive/controller.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAUH,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,GAAG,IAAI,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EACL,yBAAyB,EACzB,uBAAuB,EACvB,cAAc,EACd,YAAY,EACZ,sBAAsB,EACtB,wBAAwB,GACzB,MAAM,4BAA4B,CAAC;AAEpC,oFAAoF;AACpF,SAAS,YAAY,CAAC,QAAyB;IAC7C,4EAA4E;IAC5E,6DAA6D;IAC7D,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,MAAM,iBAAiB,GAAG,QAAQ,CAAC,iBAAiB,IAAI,IAAI,CAAC;IAC7D,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,+BAA+B;IACvF,KAAK,WAAW,CAAC;IACjB,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;IACtE,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC9C,MAAM,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC/F,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACnD,OAAO;gBACL,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,SAAS;gBACT,SAAS,EAAE,SAAS,GAAG,GAAG;gBAC1B,UAAU,EAAE,IAAI,GAAG,QAAQ,CAAC,YAAY,GAAG,CAAC;aAC7C,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAEjD,iFAAiF;AACjF,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,KAAK,EAAE,wBAAwB,EAAE,IAAI,CAAC;QACtC,QAAQ,EAAE,cAAc,EAAE;QAC1B,MAAM,EAAE,YAAY,EAAE;QACtB,mBAAmB,EAAE,yBAAyB,EAAE;QAChD,iBAAiB,EAAE,uBAAuB,EAAE;QAC5C,QAAQ,EAAE,sBAAsB,EAAE,IAAI,iBAAiB;KACxD,CAAC;AACJ,CAAC;AAcD,MAAM,UAAU,wBAAwB,CACtC,QAAsB,EACtB,MAAM,GAAqB,EAAE;IAE7B,yEAAyE;IACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,gBAAgB,IAAI,aAAa,EAAE,CAAC;IAC5D,IAAI,IAAmC,CAAC;IACxC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,OAAO;QACL,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO;YAC1B,MAAM,QAAQ,GAAoB;gBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBACzD,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;gBACtC,aAAa,EAAE,qBAAqB,CAAC,KAAK,CAAC;aAC5C,CAAC;YAEF,MAAM,SAAS,GAAG,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,aAAa,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1F,MAAM,UAAU,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC;YAEvC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,KAAK,IAAI,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,IAAI,QAAQ,CAAC,OAAO,GAAG,EAAE;gBAC7F,CAAC,CAAE,QAAkB;gBACrB,CAAC,CAAE,OAAiB,CAAC;YACvB,MAAM,QAAQ,GAAsB,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;YACxE,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACvB,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;YACpG,CAAC;YACD,IAAI,GAAG,QAAQ,CAAC;YAChB,MAAM,CAAC,UAAU,EAAE,CAAC,QAAQ,CAAC,CAAC;YAE9B,MAAM,SAAS,GAAG,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAC9B,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,EAC1E,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CACxF,CAAC;YACF,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,IAAI,KAAK,CAAC,KAAK;oBAAE,MAAM,IAAI,CAAC,CAAC;YAC/B,CAAC;YACD,MAAM,cAAc,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC;YAE3C,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;YACjE,IAAI,IAAI,CAAC,CAAC;YACV,eAAe,IAAI,UAAU,CAAC;YAC9B,cAAc,IAAI,SAAS,CAAC;YAE5B,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;QACrE,CAAC;QACD,KAAK;YACH,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,cAAc,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;QACvE,CAAC;QACD,YAAY;YACV,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC;AAED,iFAAiF;AACjF,SAAS,qBAAqB,CAAC,KAAgD;IAC7E,OAAO,mBAAmB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;AACrE,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AHEC cost model — estimates execution cost from workload + hardware facts.
|
|
3
|
+
*
|
|
4
|
+
* Pure and deterministic: the same inputs always produce the same estimate, so
|
|
5
|
+
* planning decisions are reproducible and explainable. Estimates are coarse by
|
|
6
|
+
* design (they rank strategies, they do not predict wall-clock to the ms).
|
|
7
|
+
*/
|
|
8
|
+
import type { HardwareProfile, WorkloadProfile } from './types.js';
|
|
9
|
+
/** Cost estimate breakdown for one candidate execution. */
|
|
10
|
+
export interface CostEstimate {
|
|
11
|
+
/** Compute time in ms. */
|
|
12
|
+
computeMs: number;
|
|
13
|
+
/** Data-transfer time in ms (activations/weights crossing the device boundary). */
|
|
14
|
+
transferMs: number;
|
|
15
|
+
/** Fixed per-sync-point overhead in ms. */
|
|
16
|
+
syncMs: number;
|
|
17
|
+
/** Memory-allocation penalty in ms (cold buffers). */
|
|
18
|
+
allocMs: number;
|
|
19
|
+
/** Cache-miss penalty in ms (weights/activations not resident). */
|
|
20
|
+
cacheMissMs: number;
|
|
21
|
+
/** Total wall-clock estimate in ms. */
|
|
22
|
+
totalMs: number;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Estimate the cost of running `workload` entirely on CPU.
|
|
26
|
+
*/
|
|
27
|
+
export declare function estimateCpuCost(workload: WorkloadProfile, hw: HardwareProfile): CostEstimate;
|
|
28
|
+
/**
|
|
29
|
+
* Estimate the cost of running `workload` entirely on GPU (when available).
|
|
30
|
+
*/
|
|
31
|
+
export declare function estimateGpuCost(workload: WorkloadProfile, hw: HardwareProfile): CostEstimate | null;
|
|
32
|
+
/**
|
|
33
|
+
* Estimate a split (heterogeneous) plan: `gpuFraction` of the compute runs on
|
|
34
|
+
* GPU, the rest on CPU, with one transfer + sync at the device boundary.
|
|
35
|
+
*/
|
|
36
|
+
export declare function estimateSplitCost(workload: WorkloadProfile, hw: HardwareProfile, gpuFraction: number, transferSyncPenaltyMs: number): CostEstimate | null;
|
|
37
|
+
//# sourceMappingURL=costModel.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"costModel.d.ts","sourceRoot":"","sources":["../../src/adaptive/costModel.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,2DAA2D;AAC3D,MAAM,WAAW,YAAY;IAC3B,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,mFAAmF;IACnF,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;CACjB;AA6BD;;GAEG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,EAAE,eAAe,GAAG,YAAY,CAqB5F;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,EAAE,eAAe,GAAG,YAAY,GAAG,IAAI,CAgBnG;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,eAAe,EACzB,EAAE,EAAE,eAAe,EACnB,WAAW,EAAE,MAAM,EACnB,qBAAqB,EAAE,MAAM,GAC5B,YAAY,GAAG,IAAI,CAqBrB"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AHEC cost model — estimates execution cost from workload + hardware facts.
|
|
3
|
+
*
|
|
4
|
+
* Pure and deterministic: the same inputs always produce the same estimate, so
|
|
5
|
+
* planning decisions are reproducible and explainable. Estimates are coarse by
|
|
6
|
+
* design (they rank strategies, they do not predict wall-clock to the ms).
|
|
7
|
+
*/
|
|
8
|
+
/** Bytes per activation element (fp16 KV / activations in the 1B–4B target range). */
|
|
9
|
+
const BYTES_PER_ELEM = 2;
|
|
10
|
+
/** Rough per-param per-token FLOPs multiplier for decode (2 for forward pass). */
|
|
11
|
+
const FLOPS_PER_PARAM_PER_TOKEN = 2;
|
|
12
|
+
/** Assume ~1 GFLOP/ms effective on a single scalar wasm core; SIMD ~4x; multi-core adds a sub-linear gain. */
|
|
13
|
+
function cpuGflopsPerMs(hw) {
|
|
14
|
+
let g = 1.0;
|
|
15
|
+
if (hw.wasmSimd)
|
|
16
|
+
g *= 4;
|
|
17
|
+
// Sub-linear multi-core gain: cores help prefill (parallel) more than decode.
|
|
18
|
+
g *= 1 + Math.min(hw.cores, 8) * 0.25;
|
|
19
|
+
return g;
|
|
20
|
+
}
|
|
21
|
+
/** Rough effective GPU throughput multiplier over a single scalar CPU core. */
|
|
22
|
+
function gpuGflopsPerMs(_hw) {
|
|
23
|
+
// Deliberately conservative: integrated/mobile GPUs are far weaker than discrete.
|
|
24
|
+
return 40;
|
|
25
|
+
}
|
|
26
|
+
/** Working estimate of effective active parameters from the profile. */
|
|
27
|
+
function effectiveParams(workload) {
|
|
28
|
+
const dense = workload.paramsB * 1e9;
|
|
29
|
+
return dense * (workload.activeFraction ?? 1);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Estimate the cost of running `workload` entirely on CPU.
|
|
33
|
+
*/
|
|
34
|
+
export function estimateCpuCost(workload, hw) {
|
|
35
|
+
const params = effectiveParams(workload);
|
|
36
|
+
const flops = params * FLOPS_PER_PARAM_PER_TOKEN * (workload.promptTokens + workload.decodeTokens);
|
|
37
|
+
const computeMs = flops / (cpuGflopsPerMs(hw) * 1e9) * 1000;
|
|
38
|
+
// KV cache + weight residency: if the model does not fit comfortably in RAM,
|
|
39
|
+
// add a cache-miss penalty proportional to the shortfall.
|
|
40
|
+
const ramBudget = hw.ramBytes * 0.5;
|
|
41
|
+
const cacheMissMs = workload.residentBytes > ramBudget
|
|
42
|
+
? ((workload.residentBytes - ramBudget) / 1e6) * 2
|
|
43
|
+
: 0;
|
|
44
|
+
const allocMs = Math.min(workload.residentBytes / 1e7, 200); // cold buffer setup, capped
|
|
45
|
+
return {
|
|
46
|
+
computeMs,
|
|
47
|
+
transferMs: 0,
|
|
48
|
+
syncMs: 0,
|
|
49
|
+
allocMs,
|
|
50
|
+
cacheMissMs,
|
|
51
|
+
totalMs: computeMs + allocMs + cacheMissMs,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Estimate the cost of running `workload` entirely on GPU (when available).
|
|
56
|
+
*/
|
|
57
|
+
export function estimateGpuCost(workload, hw) {
|
|
58
|
+
if (!hw.webgpu)
|
|
59
|
+
return null;
|
|
60
|
+
const params = effectiveParams(workload);
|
|
61
|
+
const flops = params * FLOPS_PER_PARAM_PER_TOKEN * (workload.promptTokens + workload.decodeTokens);
|
|
62
|
+
const computeMs = flops / (gpuGflopsPerMs(hw) * 1e9) * 1000;
|
|
63
|
+
// Upload weights once (residentBytes across the bus at ~5 GB/s effective).
|
|
64
|
+
const transferMs = workload.residentBytes / 5e9 * 1000;
|
|
65
|
+
const allocMs = Math.min(workload.residentBytes / 1e7, 200);
|
|
66
|
+
return {
|
|
67
|
+
computeMs,
|
|
68
|
+
transferMs,
|
|
69
|
+
syncMs: 0,
|
|
70
|
+
allocMs,
|
|
71
|
+
cacheMissMs: 0,
|
|
72
|
+
totalMs: computeMs + transferMs + allocMs,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Estimate a split (heterogeneous) plan: `gpuFraction` of the compute runs on
|
|
77
|
+
* GPU, the rest on CPU, with one transfer + sync at the device boundary.
|
|
78
|
+
*/
|
|
79
|
+
export function estimateSplitCost(workload, hw, gpuFraction, transferSyncPenaltyMs) {
|
|
80
|
+
const gpu = estimateGpuCost(workload, hw);
|
|
81
|
+
if (!gpu)
|
|
82
|
+
return null;
|
|
83
|
+
const cpu = estimateCpuCost(workload, hw);
|
|
84
|
+
const gpuPart = gpu.computeMs * gpuFraction;
|
|
85
|
+
const cpuPart = cpu.computeMs * (1 - gpuFraction);
|
|
86
|
+
// Layers run in dependency order; without device overlap the compute is serial.
|
|
87
|
+
const computeMs = gpuPart + cpuPart;
|
|
88
|
+
// Only the boundary activations cross the bus, not the full weights.
|
|
89
|
+
const boundaryBytes = workload.residentBytes * 0.02 + BYTES_PER_ELEM * workload.promptTokens * 1024;
|
|
90
|
+
const transferMs = boundaryBytes / 5e9 * 1000;
|
|
91
|
+
const syncMs = transferSyncPenaltyMs;
|
|
92
|
+
const allocMs = Math.max(cpu.allocMs, gpu.allocMs);
|
|
93
|
+
return {
|
|
94
|
+
computeMs,
|
|
95
|
+
transferMs,
|
|
96
|
+
syncMs,
|
|
97
|
+
allocMs,
|
|
98
|
+
cacheMissMs: cpu.cacheMissMs * (1 - gpuFraction),
|
|
99
|
+
totalMs: computeMs + transferMs + syncMs + allocMs,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=costModel.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"costModel.js","sourceRoot":"","sources":["../../src/adaptive/costModel.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAoBH,sFAAsF;AACtF,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,kFAAkF;AAClF,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAEpC,8GAA8G;AAC9G,SAAS,cAAc,CAAC,EAAmB;IACzC,IAAI,CAAC,GAAG,GAAG,CAAC;IACZ,IAAI,EAAE,CAAC,QAAQ;QAAE,CAAC,IAAI,CAAC,CAAC;IACxB,8EAA8E;IAC9E,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;IACtC,OAAO,CAAC,CAAC;AACX,CAAC;AAED,+EAA+E;AAC/E,SAAS,cAAc,CAAC,GAAoB;IAC1C,kFAAkF;IAClF,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,wEAAwE;AACxE,SAAS,eAAe,CAAC,QAAyB;IAChD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,GAAG,GAAG,CAAC;IACrC,OAAO,KAAK,GAAG,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,QAAyB,EAAE,EAAmB;IAC5E,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,GAAG,yBAAyB,GAAG,CAAC,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IACnG,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC;IAE5D,6EAA6E;IAC7E,0DAA0D;IAC1D,MAAM,SAAS,GAAG,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC;IACpC,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,GAAG,SAAS;QACpD,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;QAClD,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,4BAA4B;IACzF,OAAO;QACL,SAAS;QACT,UAAU,EAAE,CAAC;QACb,MAAM,EAAE,CAAC;QACT,OAAO;QACP,WAAW;QACX,OAAO,EAAE,SAAS,GAAG,OAAO,GAAG,WAAW;KAC3C,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,QAAyB,EAAE,EAAmB;IAC5E,IAAI,CAAC,EAAE,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,GAAG,yBAAyB,GAAG,CAAC,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IACnG,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC;IAC5D,2EAA2E;IAC3E,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,GAAG,GAAG,GAAG,IAAI,CAAC;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;IAC5D,OAAO;QACL,SAAS;QACT,UAAU;QACV,MAAM,EAAE,CAAC;QACT,OAAO;QACP,WAAW,EAAE,CAAC;QACd,OAAO,EAAE,SAAS,GAAG,UAAU,GAAG,OAAO;KAC1C,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,QAAyB,EACzB,EAAmB,EACnB,WAAmB,EACnB,qBAA6B;IAE7B,MAAM,GAAG,GAAG,eAAe,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,GAAG,GAAG,eAAe,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,GAAG,WAAW,CAAC;IAC5C,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;IAClD,gFAAgF;IAChF,MAAM,SAAS,GAAG,OAAO,GAAG,OAAO,CAAC;IACpC,qEAAqE;IACrE,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,GAAG,IAAI,GAAG,cAAc,GAAG,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;IACpG,MAAM,UAAU,GAAG,aAAa,GAAG,GAAG,GAAG,IAAI,CAAC;IAC9C,MAAM,MAAM,GAAG,qBAAqB,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACnD,OAAO;QACL,SAAS;QACT,UAAU;QACV,MAAM;QACN,OAAO;QACP,WAAW,EAAE,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC;QAChD,OAAO,EAAE,SAAS,GAAG,UAAU,GAAG,MAAM,GAAG,OAAO;KACnD,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AHEC planner — cost-based execution planning over the layer graph.
|
|
3
|
+
*
|
|
4
|
+
* HEFT-lite: for the linear transformer-style dependency chain (L0 -> L1 -> ...)
|
|
5
|
+
* the partition problem reduces to choosing a split point, which dynamic
|
|
6
|
+
* programming solves exactly in O(n). We also evaluate the all-CPU and all-GPU
|
|
7
|
+
* extremes. A heterogeneous plan is only returned when it beats the best
|
|
8
|
+
* homogeneous plan by the configured margin — heterogeneity is never assumed.
|
|
9
|
+
*/
|
|
10
|
+
import type { ControllerConfig, ExecutionGraph, ExecutionPlan, HardwareProfile, WorkloadProfile } from './types.js';
|
|
11
|
+
export interface PlanRequest {
|
|
12
|
+
workload: WorkloadProfile;
|
|
13
|
+
hardware: HardwareProfile;
|
|
14
|
+
graph: ExecutionGraph;
|
|
15
|
+
config: ControllerConfig;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Plan execution for one request. Pure given its inputs — the same request
|
|
19
|
+
* always yields the same plan, which keeps decisions explainable and testable.
|
|
20
|
+
*/
|
|
21
|
+
export declare function planExecution(req: PlanRequest): ExecutionPlan;
|
|
22
|
+
//# sourceMappingURL=planner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../../src/adaptive/planner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAGpH,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,eAAe,CAAC;IAC1B,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,WAAW,GAAG,aAAa,CA8F7D"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AHEC planner — cost-based execution planning over the layer graph.
|
|
3
|
+
*
|
|
4
|
+
* HEFT-lite: for the linear transformer-style dependency chain (L0 -> L1 -> ...)
|
|
5
|
+
* the partition problem reduces to choosing a split point, which dynamic
|
|
6
|
+
* programming solves exactly in O(n). We also evaluate the all-CPU and all-GPU
|
|
7
|
+
* extremes. A heterogeneous plan is only returned when it beats the best
|
|
8
|
+
* homogeneous plan by the configured margin — heterogeneity is never assumed.
|
|
9
|
+
*/
|
|
10
|
+
import { estimateCpuCost, estimateGpuCost, estimateSplitCost } from './costModel.js';
|
|
11
|
+
/**
|
|
12
|
+
* Plan execution for one request. Pure given its inputs — the same request
|
|
13
|
+
* always yields the same plan, which keeps decisions explainable and testable.
|
|
14
|
+
*/
|
|
15
|
+
export function planExecution(req) {
|
|
16
|
+
const { workload, hardware, graph, config } = req;
|
|
17
|
+
const reasoning = [];
|
|
18
|
+
const margin = config.heterogeneityMargin ?? 0.05;
|
|
19
|
+
const syncPenalty = config.transferSyncPenaltyMs ?? 4;
|
|
20
|
+
const cpuOnlyForced = config.cpuOnly === true;
|
|
21
|
+
if (cpuOnlyForced)
|
|
22
|
+
reasoning.push('cpu-only forced by config');
|
|
23
|
+
const cpu = estimateCpuCost(workload, hardware);
|
|
24
|
+
reasoning.push(`cpu estimate ${cpu.totalMs.toFixed(1)}ms`);
|
|
25
|
+
// CPU-only candidate.
|
|
26
|
+
let best = {
|
|
27
|
+
strategy: 'cpu-only',
|
|
28
|
+
backend: pickBackend(hardware, 'cpu-only'),
|
|
29
|
+
reasoning: [...reasoning, 'cpu-only selected'],
|
|
30
|
+
estimatedCostMs: cpu.totalMs,
|
|
31
|
+
gpuLayerIds: [],
|
|
32
|
+
cpuLayerIds: graph.layers.map((l) => l.id),
|
|
33
|
+
};
|
|
34
|
+
let bestCost = cpu.totalMs;
|
|
35
|
+
const gpuUsable = !cpuOnlyForced && hardware.webgpu;
|
|
36
|
+
if (!hardware.webgpu)
|
|
37
|
+
reasoning.push('webgpu unavailable');
|
|
38
|
+
if (gpuUsable) {
|
|
39
|
+
const gpu = estimateGpuCost(workload, hardware);
|
|
40
|
+
if (gpu) {
|
|
41
|
+
reasoning.push(`gpu estimate ${gpu.totalMs.toFixed(1)}ms`);
|
|
42
|
+
// GPU-only candidate.
|
|
43
|
+
if (gpu.totalMs < bestCost * (1 - margin)) {
|
|
44
|
+
best = {
|
|
45
|
+
strategy: 'gpu-only',
|
|
46
|
+
backend: pickBackend(hardware, 'gpu-only'),
|
|
47
|
+
reasoning: [...reasoning, `gpu-only beats cpu by ${(bestCost - gpu.totalMs).toFixed(1)}ms`],
|
|
48
|
+
estimatedCostMs: gpu.totalMs,
|
|
49
|
+
gpuLayerIds: graph.layers.map((l) => l.id),
|
|
50
|
+
cpuLayerIds: [],
|
|
51
|
+
};
|
|
52
|
+
bestCost = gpu.totalMs;
|
|
53
|
+
}
|
|
54
|
+
// Split candidates: prefix-on-CPU/suffix-on-GPU and the reverse, plus a
|
|
55
|
+
// mid split. DP over all split points for exactness on the linear chain.
|
|
56
|
+
const n = graph.layers.length;
|
|
57
|
+
if (n > 0) {
|
|
58
|
+
// Cumulative per-layer cost lets us score any split point in O(1).
|
|
59
|
+
let cpuPrefix = 0;
|
|
60
|
+
const cpuCumulative = [];
|
|
61
|
+
let gpuPrefix = 0;
|
|
62
|
+
const gpuCumulative = [];
|
|
63
|
+
for (const layer of graph.layers) {
|
|
64
|
+
cpuPrefix += layer.cpuCostMs;
|
|
65
|
+
gpuPrefix += Number.isFinite(layer.gpuCostMs) ? layer.gpuCostMs : layer.cpuCostMs * 4;
|
|
66
|
+
cpuCumulative.push(cpuPrefix);
|
|
67
|
+
gpuCumulative.push(gpuPrefix);
|
|
68
|
+
}
|
|
69
|
+
const totalCpu = cpuPrefix;
|
|
70
|
+
const totalGpu = gpuPrefix;
|
|
71
|
+
for (let split = 0; split <= n; split++) {
|
|
72
|
+
const cpuPart = cpuCumulative[split - 1] ?? 0;
|
|
73
|
+
const gpuPart = totalGpu - (gpuCumulative[split - 1] ?? 0);
|
|
74
|
+
const gpuFraction = totalGpu > 0 ? gpuPart / totalGpu : 0;
|
|
75
|
+
const est = estimateSplitCost(workload, hardware, gpuFraction, syncPenalty);
|
|
76
|
+
if (!est)
|
|
77
|
+
continue;
|
|
78
|
+
// Scale the analytic split estimate to the per-layer graph costs so the
|
|
79
|
+
// plan honors the actual dependency structure, not just the analytic model.
|
|
80
|
+
const graphCompute = cpuPart + gpuPart;
|
|
81
|
+
const analyticCompute = est.computeMs || 1;
|
|
82
|
+
const scaled = (est.totalMs - est.computeMs) + graphCompute * (est.computeMs / analyticCompute);
|
|
83
|
+
if (scaled < bestCost * (1 - margin)) {
|
|
84
|
+
const gpuIds = graph.layers.slice(split).map((l) => l.id);
|
|
85
|
+
const cpuIds = graph.layers.slice(0, split).map((l) => l.id);
|
|
86
|
+
best = {
|
|
87
|
+
strategy: split === 0 ? 'gpu-only' : split === n ? 'cpu-only' : 'layer-split',
|
|
88
|
+
backend: pickBackend(hardware, 'layer-split'),
|
|
89
|
+
reasoning: [
|
|
90
|
+
...reasoning,
|
|
91
|
+
`layer-split at ${split}/${n} (gpu layers ${gpuIds.length}) saves est. ${(bestCost - scaled).toFixed(1)}ms after ${(est.transferMs + est.syncMs).toFixed(1)}ms transfer+sync`,
|
|
92
|
+
],
|
|
93
|
+
estimatedCostMs: scaled,
|
|
94
|
+
gpuLayerIds: gpuIds,
|
|
95
|
+
cpuLayerIds: cpuIds,
|
|
96
|
+
};
|
|
97
|
+
bestCost = scaled;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return best;
|
|
104
|
+
}
|
|
105
|
+
function pickBackend(hw, strategy) {
|
|
106
|
+
// WebGPU-present strategies prefer the WebLLM adapter; CPU paths prefer ONNX
|
|
107
|
+
// (Wllama when cross-origin isolation enables threaded decode).
|
|
108
|
+
if (strategy === 'gpu-only')
|
|
109
|
+
return 'webllm';
|
|
110
|
+
if (hw.crossOriginIsolated && hw.sharedArrayBuffer)
|
|
111
|
+
return 'wllama';
|
|
112
|
+
return 'onnx';
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=planner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"planner.js","sourceRoot":"","sources":["../../src/adaptive/planner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AASrF;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,GAAgB;IAC5C,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC;IAClD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC;IAClD,MAAM,WAAW,GAAG,MAAM,CAAC,qBAAqB,IAAI,CAAC,CAAC;IAEtD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC;IAC9C,IAAI,aAAa;QAAE,SAAS,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAE/D,MAAM,GAAG,GAAG,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAChD,SAAS,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE3D,sBAAsB;IACtB,IAAI,IAAI,GAAkB;QACxB,QAAQ,EAAE,UAAU;QACpB,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC1C,SAAS,EAAE,CAAC,GAAG,SAAS,EAAE,mBAAmB,CAAC;QAC9C,eAAe,EAAE,GAAG,CAAC,OAAO;QAC5B,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3C,CAAC;IACF,IAAI,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC;IAE3B,MAAM,SAAS,GAAG,CAAC,aAAa,IAAI,QAAQ,CAAC,MAAM,CAAC;IACpD,IAAI,CAAC,QAAQ,CAAC,MAAM;QAAE,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAE3D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,GAAG,GAAG,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAChD,IAAI,GAAG,EAAE,CAAC;YACR,SAAS,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3D,sBAAsB;YACtB,IAAI,GAAG,CAAC,OAAO,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC;gBAC1C,IAAI,GAAG;oBACL,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC;oBAC1C,SAAS,EAAE,CAAC,GAAG,SAAS,EAAE,yBAAyB,CAAC,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC3F,eAAe,EAAE,GAAG,CAAC,OAAO;oBAC5B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC1C,WAAW,EAAE,EAAE;iBAChB,CAAC;gBACF,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC;YACzB,CAAC;YAED,wEAAwE;YACxE,yEAAyE;YACzE,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACV,mEAAmE;gBACnE,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,MAAM,aAAa,GAAa,EAAE,CAAC;gBACnC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,MAAM,aAAa,GAAa,EAAE,CAAC;gBACnC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;oBACjC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;oBAC7B,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;oBACtF,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC9B,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAChC,CAAC;gBACD,MAAM,QAAQ,GAAG,SAAS,CAAC;gBAC3B,MAAM,QAAQ,GAAG,SAAS,CAAC;gBAE3B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;oBAC9C,MAAM,OAAO,GAAG,QAAQ,GAAG,CAAC,aAAa,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC3D,MAAM,WAAW,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC1D,MAAM,GAAG,GAAG,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;oBAC5E,IAAI,CAAC,GAAG;wBAAE,SAAS;oBACnB,wEAAwE;oBACxE,4EAA4E;oBAC5E,MAAM,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC;oBACvC,MAAM,eAAe,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC;oBAC3C,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,YAAY,GAAG,CAAC,GAAG,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;oBAChG,IAAI,MAAM,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC;wBACrC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC7D,IAAI,GAAG;4BACL,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa;4BAC7E,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,aAAa,CAAC;4BAC7C,SAAS,EAAE;gCACT,GAAG,SAAS;gCACZ,kBAAkB,KAAK,IAAI,CAAC,gBAAgB,MAAM,CAAC,MAAM,gBAAgB,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB;6BAC9K;4BACD,eAAe,EAAE,MAAM;4BACvB,WAAW,EAAE,MAAM;4BACnB,WAAW,EAAE,MAAM;yBACpB,CAAC;wBACF,QAAQ,GAAG,MAAM,CAAC;oBACpB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,EAAmB,EAAE,QAAmC;IAC3E,6EAA6E;IAC7E,gEAAgE;IAChE,IAAI,QAAQ,KAAK,UAAU;QAAE,OAAO,QAAQ,CAAC;IAC7C,IAAI,EAAE,CAAC,mBAAmB,IAAI,EAAE,CAAC,iBAAiB;QAAE,OAAO,QAAQ,CAAC;IACpE,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adaptive Heterogeneous Execution Controller (AHEC) — types.
|
|
3
|
+
*
|
|
4
|
+
* The controller decides WHERE/HOW to execute an inference workload and WHY.
|
|
5
|
+
* It is deliberately decoupled from the execution engine in `src/inference/`
|
|
6
|
+
* (which only knows HOW to run): the controller routes to an injected executor
|
|
7
|
+
* and picks a backend + strategy from measured hardware capabilities.
|
|
8
|
+
*
|
|
9
|
+
* Browser-compatible only: pure TypeScript, no native code, no dependencies.
|
|
10
|
+
*/
|
|
11
|
+
/** Hardware facts probed from the environment. Facts, not gates. */
|
|
12
|
+
export interface HardwareProfile {
|
|
13
|
+
/** Logical CPU cores available to workers. */
|
|
14
|
+
cores: number;
|
|
15
|
+
/** WebAssembly SIMD is available (v128 probe). */
|
|
16
|
+
wasmSimd: boolean;
|
|
17
|
+
/** WebGPU adapter is present and usable. */
|
|
18
|
+
webgpu: boolean;
|
|
19
|
+
/** Page is cross-origin isolated. */
|
|
20
|
+
crossOriginIsolated: boolean;
|
|
21
|
+
/** SharedArrayBuffer is available. */
|
|
22
|
+
sharedArrayBuffer: boolean;
|
|
23
|
+
/** Usable RAM hint in bytes (navigator.deviceMemory * ~1GiB when present). */
|
|
24
|
+
ramBytes: number;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Characterization of one inference request, used for planning.
|
|
28
|
+
* All fields are hints the planner reasons over; none are required to be exact.
|
|
29
|
+
*/
|
|
30
|
+
export interface WorkloadProfile {
|
|
31
|
+
/** Model size in billions of parameters. */
|
|
32
|
+
paramsB: number;
|
|
33
|
+
/** Quantization bucket, matching `ModelRef.quantization`. */
|
|
34
|
+
quantization: 'q4' | 'q5' | 'q8' | 'fp16';
|
|
35
|
+
/** Expected prompt (prefill) token count. */
|
|
36
|
+
promptTokens: number;
|
|
37
|
+
/** Expected decode (generation) token count. */
|
|
38
|
+
decodeTokens: number;
|
|
39
|
+
/** Approximate resident bytes the model needs (weights + KV headroom). */
|
|
40
|
+
residentBytes: number;
|
|
41
|
+
/** Fraction of layers that are attention (rest treated as FFN). */
|
|
42
|
+
attentionFraction?: number;
|
|
43
|
+
/** Sparsity / MoE: fraction of parameters active per token (1 = dense). */
|
|
44
|
+
activeFraction?: number;
|
|
45
|
+
}
|
|
46
|
+
/** One node in the layer-dependency graph the planner partitions. */
|
|
47
|
+
export interface LayerNode {
|
|
48
|
+
id: string;
|
|
49
|
+
/** Estimated cost to run this layer on CPU, in ms. */
|
|
50
|
+
cpuCostMs: number;
|
|
51
|
+
/** Estimated cost on GPU, in ms (Infinity if unsupported). */
|
|
52
|
+
gpuCostMs: number;
|
|
53
|
+
/** Bytes of activations this layer consumes from its predecessor. */
|
|
54
|
+
inputBytes: number;
|
|
55
|
+
}
|
|
56
|
+
/** A linear-dependency layer graph (transformer-style: L0 -> L1 -> ... -> Ln). */
|
|
57
|
+
export interface ExecutionGraph {
|
|
58
|
+
layers: LayerNode[];
|
|
59
|
+
}
|
|
60
|
+
/** Where a plan runs. */
|
|
61
|
+
export type ExecutionStrategy = 'cpu-only' | 'gpu-only' | 'cpu-prefill-gpu-decode' | 'gpu-prefill-cpu-decode' | 'layer-split';
|
|
62
|
+
/** A chosen execution strategy with its justification. */
|
|
63
|
+
export interface ExecutionPlan {
|
|
64
|
+
strategy: ExecutionStrategy;
|
|
65
|
+
/** Which backend kind the executor should use. */
|
|
66
|
+
backend: 'onnx' | 'wllama' | 'webllm' | 'mock';
|
|
67
|
+
/** Human-readable, auditable reasoning trail for this decision. */
|
|
68
|
+
reasoning: string[];
|
|
69
|
+
/** Estimated wall-clock cost in ms (including transfer + sync). */
|
|
70
|
+
estimatedCostMs: number;
|
|
71
|
+
/** Layers assigned to GPU (empty for cpu-only). */
|
|
72
|
+
gpuLayerIds: string[];
|
|
73
|
+
/** Layers assigned to CPU. */
|
|
74
|
+
cpuLayerIds: string[];
|
|
75
|
+
}
|
|
76
|
+
/** The decision the controller makes for one request. */
|
|
77
|
+
export interface ExecutionDecision {
|
|
78
|
+
plan: ExecutionPlan;
|
|
79
|
+
/** 'local' | 'remote' — remote is only chosen when explicitly configured. */
|
|
80
|
+
route: 'local' | 'remote';
|
|
81
|
+
/** Probed hardware snapshot the decision was made against. */
|
|
82
|
+
hardware: HardwareProfile;
|
|
83
|
+
/** Workload snapshot the decision was made against. */
|
|
84
|
+
workload: WorkloadProfile;
|
|
85
|
+
}
|
|
86
|
+
/** Result of running one request through the controller. */
|
|
87
|
+
export interface ExecutionOutcome {
|
|
88
|
+
decision: ExecutionDecision;
|
|
89
|
+
/** Wall-clock ms actually observed for the executor call. */
|
|
90
|
+
measuredCostMs: number;
|
|
91
|
+
/** ms the controller spent planning (its own overhead). */
|
|
92
|
+
overheadMs: number;
|
|
93
|
+
/** planned - measured; positive means the plan beat the estimate. */
|
|
94
|
+
benefitMs: number;
|
|
95
|
+
/** Tokens produced (when the executor reports them). */
|
|
96
|
+
tokens?: number;
|
|
97
|
+
}
|
|
98
|
+
/** Controller configuration. */
|
|
99
|
+
export interface ControllerConfig {
|
|
100
|
+
/** Force CPU-only planning regardless of probe. Default false. */
|
|
101
|
+
cpuOnly?: boolean;
|
|
102
|
+
/** Allow routing to a remote executor. Default false (local-only). */
|
|
103
|
+
allowRemote?: boolean;
|
|
104
|
+
/** Heterogeneous plans must beat the CPU-only estimate by this margin. Default 0.05 (5%). */
|
|
105
|
+
heterogeneityMargin?: number;
|
|
106
|
+
/** Per-transfer-point fixed sync penalty in ms. Default 4. */
|
|
107
|
+
transferSyncPenaltyMs?: number;
|
|
108
|
+
/** Optional decision logger (explainability sink). */
|
|
109
|
+
onDecision?: (decision: ExecutionDecision) => void;
|
|
110
|
+
/** Test hook: override the probed hardware profile. */
|
|
111
|
+
hardwareOverride?: HardwareProfile;
|
|
112
|
+
}
|
|
113
|
+
/** Minimal executor surface the controller drives (matches InferenceClient). */
|
|
114
|
+
export interface ExecutorLike {
|
|
115
|
+
load(model: {
|
|
116
|
+
id: string;
|
|
117
|
+
paramsB: number;
|
|
118
|
+
quantization: string;
|
|
119
|
+
}): Promise<unknown>;
|
|
120
|
+
generate(model: {
|
|
121
|
+
id: string;
|
|
122
|
+
paramsB: number;
|
|
123
|
+
quantization: string;
|
|
124
|
+
}, options: {
|
|
125
|
+
prompt: string;
|
|
126
|
+
maxTokens: number;
|
|
127
|
+
signal?: AbortSignal;
|
|
128
|
+
}): AsyncIterable<{
|
|
129
|
+
token: string;
|
|
130
|
+
}> & {
|
|
131
|
+
stats?: () => Promise<{
|
|
132
|
+
ttftMs: number;
|
|
133
|
+
tokensPerSecond: number;
|
|
134
|
+
}>;
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
export type { Quantization } from '../inference/types.js';
|
|
138
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/adaptive/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,oEAAoE;AACpE,MAAM,WAAW,eAAe;IAC9B,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,EAAE,OAAO,CAAC;IAClB,4CAA4C;IAC5C,MAAM,EAAE,OAAO,CAAC;IAChB,qCAAqC;IACrC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,sCAAsC;IACtC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,YAAY,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC;IAC1C,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,gDAAgD;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,0EAA0E;IAC1E,aAAa,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qEAAqE;AACrE,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,SAAS,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kFAAkF;AAClF,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,SAAS,EAAE,CAAC;CACrB;AAED,yBAAyB;AACzB,MAAM,MAAM,iBAAiB,GACzB,UAAU,GACV,UAAU,GACV,wBAAwB,GACxB,wBAAwB,GACxB,aAAa,CAAC;AAElB,0DAA0D;AAC1D,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,kDAAkD;IAClD,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC/C,mEAAmE;IACnE,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,mEAAmE;IACnE,eAAe,EAAE,MAAM,CAAC;IACxB,mDAAmD;IACnD,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,yDAAyD;AACzD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,aAAa,CAAC;IACpB,6EAA6E;IAC7E,KAAK,EAAE,OAAO,GAAG,QAAQ,CAAC;IAC1B,8DAA8D;IAC9D,QAAQ,EAAE,eAAe,CAAC;IAC1B,uDAAuD;IACvD,QAAQ,EAAE,eAAe,CAAC;CAC3B;AAED,4DAA4D;AAC5D,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,6DAA6D;IAC7D,cAAc,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,gCAAgC;AAChC,MAAM,WAAW,gBAAgB;IAC/B,kEAAkE;IAClE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,sEAAsE;IACtE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,6FAA6F;IAC7F,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,8DAA8D;IAC9D,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sDAAsD;IACtD,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACnD,uDAAuD;IACvD,gBAAgB,CAAC,EAAE,eAAe,CAAC;CACpC;AAED,gFAAgF;AAChF,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACrF,QAAQ,CACN,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,EAC5D,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GACnE,aAAa,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG;QAAE,KAAK,CAAC,EAAE,MAAM,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,eAAe,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC9G;AAED,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adaptive Heterogeneous Execution Controller (AHEC) — types.
|
|
3
|
+
*
|
|
4
|
+
* The controller decides WHERE/HOW to execute an inference workload and WHY.
|
|
5
|
+
* It is deliberately decoupled from the execution engine in `src/inference/`
|
|
6
|
+
* (which only knows HOW to run): the controller routes to an injected executor
|
|
7
|
+
* and picks a backend + strategy from measured hardware capabilities.
|
|
8
|
+
*
|
|
9
|
+
* Browser-compatible only: pure TypeScript, no native code, no dependencies.
|
|
10
|
+
*/
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=types.js.map
|