@seanhogg/builderforce-webcontainers 2026.10.11 → 2026.10.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/adaptive/controller.d.ts +39 -0
- package/dist/adaptive/controller.d.ts.map +1 -0
- package/dist/adaptive/controller.js +108 -0
- package/dist/adaptive/controller.js.map +1 -0
- package/dist/adaptive/costModel.d.ts +37 -0
- package/dist/adaptive/costModel.d.ts.map +1 -0
- package/dist/adaptive/costModel.js +102 -0
- package/dist/adaptive/costModel.js.map +1 -0
- package/dist/adaptive/planner.d.ts +22 -0
- package/dist/adaptive/planner.d.ts.map +1 -0
- package/dist/adaptive/planner.js +114 -0
- package/dist/adaptive/planner.js.map +1 -0
- package/dist/adaptive/types.d.ts +138 -0
- package/dist/adaptive/types.d.ts.map +1 -0
- package/dist/adaptive/types.js +12 -0
- package/dist/adaptive/types.js.map +1 -0
- package/dist/assets.js +2 -2
- package/dist/check/cachedFetch.js.map +1 -1
- package/dist/check/index.d.ts +6 -1
- package/dist/check/index.d.ts.map +1 -1
- package/dist/check/index.js +6 -1
- package/dist/check/index.js.map +1 -1
- package/dist/componentCompilers.d.ts.map +1 -1
- package/dist/componentCompilers.js.map +1 -1
- package/dist/esbuildWasm.js.map +1 -1
- package/dist/host.d.ts +11 -5
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +19 -8
- package/dist/host.js.map +1 -1
- package/dist/inference/backends/mock.d.ts +24 -0
- package/dist/inference/backends/mock.d.ts.map +1 -0
- package/dist/inference/backends/mock.js +55 -0
- package/dist/inference/backends/mock.js.map +1 -0
- package/dist/inference/backends/onnx.d.ts +33 -0
- package/dist/inference/backends/onnx.d.ts.map +1 -0
- package/dist/inference/backends/onnx.js +60 -0
- package/dist/inference/backends/onnx.js.map +1 -0
- package/dist/inference/backends/webllm.d.ts +27 -0
- package/dist/inference/backends/webllm.d.ts.map +1 -0
- package/dist/inference/backends/webllm.js +52 -0
- package/dist/inference/backends/webllm.js.map +1 -0
- package/dist/inference/backends/wllama.d.ts +29 -0
- package/dist/inference/backends/wllama.d.ts.map +1 -0
- package/dist/inference/backends/wllama.js +51 -0
- package/dist/inference/backends/wllama.js.map +1 -0
- package/dist/inference/capabilities.d.ts +20 -0
- package/dist/inference/capabilities.d.ts.map +1 -0
- package/dist/inference/capabilities.js +33 -0
- package/dist/inference/capabilities.js.map +1 -0
- package/dist/inference/client.d.ts +45 -0
- package/dist/inference/client.d.ts.map +1 -0
- package/dist/inference/client.js +201 -0
- package/dist/inference/client.js.map +1 -0
- package/dist/inference/modelCache.d.ts +71 -0
- package/dist/inference/modelCache.d.ts.map +1 -0
- package/dist/inference/modelCache.js +91 -0
- package/dist/inference/modelCache.js.map +1 -0
- package/dist/inference/protocol.d.ts +94 -0
- package/dist/inference/protocol.d.ts.map +1 -0
- package/dist/inference/protocol.js +19 -0
- package/dist/inference/protocol.js.map +1 -0
- package/dist/inference/tokenStream.d.ts +26 -0
- package/dist/inference/tokenStream.d.ts.map +1 -0
- package/dist/inference/tokenStream.js +79 -0
- package/dist/inference/tokenStream.js.map +1 -0
- package/dist/inference/types.d.ts +108 -0
- package/dist/inference/types.d.ts.map +1 -0
- package/dist/inference/types.js +8 -0
- package/dist/inference/types.js.map +1 -0
- package/dist/inference/worker.bundle.js +5 -0
- package/dist/inference/worker.d.ts +35 -0
- package/dist/inference/worker.d.ts.map +1 -0
- package/dist/inference/worker.js +157 -0
- package/dist/inference/worker.js.map +1 -0
- package/dist/inference/workerEntry.d.ts +15 -0
- package/dist/inference/workerEntry.d.ts.map +1 -0
- package/dist/inference/workerEntry.js +17 -0
- package/dist/inference/workerEntry.js.map +1 -0
- package/dist/kvcache/advanced.d.ts +39 -0
- package/dist/kvcache/advanced.d.ts.map +1 -0
- package/dist/kvcache/advanced.js +59 -0
- package/dist/kvcache/advanced.js.map +1 -0
- package/dist/kvcache/compatibility.d.ts +14 -0
- package/dist/kvcache/compatibility.d.ts.map +1 -0
- package/dist/kvcache/compatibility.js +44 -0
- package/dist/kvcache/compatibility.js.map +1 -0
- package/dist/kvcache/lru.d.ts +45 -0
- package/dist/kvcache/lru.d.ts.map +1 -0
- package/dist/kvcache/lru.js +91 -0
- package/dist/kvcache/lru.js.map +1 -0
- package/dist/kvcache/paged.d.ts +86 -0
- package/dist/kvcache/paged.d.ts.map +1 -0
- package/dist/kvcache/paged.js +238 -0
- package/dist/kvcache/paged.js.map +1 -0
- package/dist/kvcache/quant.d.ts +30 -0
- package/dist/kvcache/quant.d.ts.map +1 -0
- package/dist/kvcache/quant.js +73 -0
- package/dist/kvcache/quant.js.map +1 -0
- package/dist/kvcache/radix.d.ts +28 -0
- package/dist/kvcache/radix.d.ts.map +1 -0
- package/dist/kvcache/radix.js +58 -0
- package/dist/kvcache/radix.js.map +1 -0
- package/dist/kvcache/runtime.d.ts +11 -0
- package/dist/kvcache/runtime.d.ts.map +1 -0
- package/dist/kvcache/runtime.js +204 -0
- package/dist/kvcache/runtime.js.map +1 -0
- package/dist/kvcache/types.d.ts +151 -0
- package/dist/kvcache/types.d.ts.map +1 -0
- package/dist/kvcache/types.js +10 -0
- package/dist/kvcache/types.js.map +1 -0
- package/dist/learning/fingerprint.d.ts +17 -0
- package/dist/learning/fingerprint.d.ts.map +1 -0
- package/dist/learning/fingerprint.js +46 -0
- package/dist/learning/fingerprint.js.map +1 -0
- package/dist/learning/index.d.ts +5 -0
- package/dist/learning/index.d.ts.map +1 -0
- package/dist/learning/index.js +3 -0
- package/dist/learning/index.js.map +1 -0
- package/dist/learning/learner.d.ts +21 -0
- package/dist/learning/learner.d.ts.map +1 -0
- package/dist/learning/learner.js +272 -0
- package/dist/learning/learner.js.map +1 -0
- package/dist/learning/ledger.d.ts +26 -0
- package/dist/learning/ledger.d.ts.map +1 -0
- package/dist/learning/ledger.js +40 -0
- package/dist/learning/ledger.js.map +1 -0
- package/dist/learning/selection.d.ts +44 -0
- package/dist/learning/selection.d.ts.map +1 -0
- package/dist/learning/selection.js +111 -0
- package/dist/learning/selection.js.map +1 -0
- package/dist/learning/stats.d.ts +19 -0
- package/dist/learning/stats.d.ts.map +1 -0
- package/dist/learning/stats.js +31 -0
- package/dist/learning/stats.js.map +1 -0
- package/dist/learning/types.d.ts +133 -0
- package/dist/learning/types.d.ts.map +1 -0
- package/dist/learning/types.js +8 -0
- package/dist/learning/types.js.map +1 -0
- package/dist/memory/envProbe.d.ts +16 -0
- package/dist/memory/envProbe.d.ts.map +1 -0
- package/dist/memory/envProbe.js +28 -0
- package/dist/memory/envProbe.js.map +1 -0
- package/dist/memory/estimates.d.ts +46 -0
- package/dist/memory/estimates.d.ts.map +1 -0
- package/dist/memory/estimates.js +72 -0
- package/dist/memory/estimates.js.map +1 -0
- package/dist/memory/eviction.d.ts +42 -0
- package/dist/memory/eviction.d.ts.map +1 -0
- package/dist/memory/eviction.js +143 -0
- package/dist/memory/eviction.js.map +1 -0
- package/dist/memory/manager.d.ts +34 -0
- package/dist/memory/manager.d.ts.map +1 -0
- package/dist/memory/manager.js +346 -0
- package/dist/memory/manager.js.map +1 -0
- package/dist/memory/managerTypes.d.ts +49 -0
- package/dist/memory/managerTypes.d.ts.map +1 -0
- package/dist/memory/managerTypes.js +7 -0
- package/dist/memory/managerTypes.js.map +1 -0
- package/dist/memory/pressure.d.ts +56 -0
- package/dist/memory/pressure.d.ts.map +1 -0
- package/dist/memory/pressure.js +80 -0
- package/dist/memory/pressure.js.map +1 -0
- package/dist/memory/spillTier.d.ts +47 -0
- package/dist/memory/spillTier.d.ts.map +1 -0
- package/dist/memory/spillTier.js +128 -0
- package/dist/memory/spillTier.js.map +1 -0
- package/dist/memory/storageSink.d.ts +69 -0
- package/dist/memory/storageSink.d.ts.map +1 -0
- package/dist/memory/storageSink.js +221 -0
- package/dist/memory/storageSink.js.map +1 -0
- package/dist/memory/types.d.ts +202 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +15 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/memory/victims.d.ts +32 -0
- package/dist/memory/victims.d.ts.map +1 -0
- package/dist/memory/victims.js +74 -0
- package/dist/memory/victims.js.map +1 -0
- package/dist/memory/workspace.d.ts +34 -0
- package/dist/memory/workspace.d.ts.map +1 -0
- package/dist/memory/workspace.js +55 -0
- package/dist/memory/workspace.js.map +1 -0
- package/dist/memoryIntegration/index.d.ts +8 -0
- package/dist/memoryIntegration/index.d.ts.map +1 -0
- package/dist/memoryIntegration/index.js +4 -0
- package/dist/memoryIntegration/index.js.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts +43 -0
- package/dist/memoryIntegration/knowledgeMemory.d.ts.map +1 -0
- package/dist/memoryIntegration/knowledgeMemory.js +285 -0
- package/dist/memoryIntegration/knowledgeMemory.js.map +1 -0
- package/dist/memoryIntegration/persistence.d.ts +15 -0
- package/dist/memoryIntegration/persistence.d.ts.map +1 -0
- package/dist/memoryIntegration/persistence.js +43 -0
- package/dist/memoryIntegration/persistence.js.map +1 -0
- package/dist/memoryIntegration/text.d.ts +57 -0
- package/dist/memoryIntegration/text.d.ts.map +1 -0
- package/dist/memoryIntegration/text.js +179 -0
- package/dist/memoryIntegration/text.js.map +1 -0
- package/dist/memoryIntegration/types.d.ts +131 -0
- package/dist/memoryIntegration/types.d.ts.map +1 -0
- package/dist/memoryIntegration/types.js +11 -0
- package/dist/memoryIntegration/types.js.map +1 -0
- package/dist/ml.d.ts +52 -0
- package/dist/ml.d.ts.map +1 -0
- package/dist/ml.js +33 -0
- package/dist/ml.js.map +1 -0
- package/dist/node/childRelay.d.ts +10 -0
- package/dist/node/childRelay.d.ts.map +1 -1
- package/dist/node/childRelay.js +12 -2
- package/dist/node/childRelay.js.map +1 -1
- package/dist/node/index.js.map +1 -1
- package/dist/node/portResponder.d.ts +2 -0
- package/dist/node/portResponder.d.ts.map +1 -1
- package/dist/node/portResponder.js +3 -3
- package/dist/node/portResponder.js.map +1 -1
- package/dist/node/processHost.d.ts +4 -1
- package/dist/node/processHost.d.ts.map +1 -1
- package/dist/node/processHost.js +19 -16
- package/dist/node/processHost.js.map +1 -1
- package/dist/node/protocol.d.ts +1 -0
- package/dist/node/protocol.d.ts.map +1 -1
- package/dist/node/responseWire.d.ts.map +1 -1
- package/dist/node/worker.js +116 -109
- package/dist/node/workerProcess.js +1 -1
- package/dist/node/workerProcess.js.map +1 -1
- package/dist/prefetch/admission.d.ts +41 -0
- package/dist/prefetch/admission.d.ts.map +1 -0
- package/dist/prefetch/admission.js +49 -0
- package/dist/prefetch/admission.js.map +1 -0
- package/dist/prefetch/loadRegistry.d.ts +56 -0
- package/dist/prefetch/loadRegistry.d.ts.map +1 -0
- package/dist/prefetch/loadRegistry.js +108 -0
- package/dist/prefetch/loadRegistry.js.map +1 -0
- package/dist/prefetch/predictor.d.ts +40 -0
- package/dist/prefetch/predictor.d.ts.map +1 -0
- package/dist/prefetch/predictor.js +178 -0
- package/dist/prefetch/predictor.js.map +1 -0
- package/dist/prefetch/prefetcher.d.ts +37 -0
- package/dist/prefetch/prefetcher.d.ts.map +1 -0
- package/dist/prefetch/prefetcher.js +339 -0
- package/dist/prefetch/prefetcher.js.map +1 -0
- package/dist/prefetch/residentStore.d.ts +49 -0
- package/dist/prefetch/residentStore.d.ts.map +1 -0
- package/dist/prefetch/residentStore.js +87 -0
- package/dist/prefetch/residentStore.js.map +1 -0
- package/dist/prefetch/selfTuner.d.ts +40 -0
- package/dist/prefetch/selfTuner.d.ts.map +1 -0
- package/dist/prefetch/selfTuner.js +78 -0
- package/dist/prefetch/selfTuner.js.map +1 -0
- package/dist/prefetch/stats.d.ts +52 -0
- package/dist/prefetch/stats.d.ts.map +1 -0
- package/dist/prefetch/stats.js +72 -0
- package/dist/prefetch/stats.js.map +1 -0
- package/dist/prefetch/streaming.d.ts +132 -0
- package/dist/prefetch/streaming.d.ts.map +1 -0
- package/dist/prefetch/streaming.js +141 -0
- package/dist/prefetch/streaming.js.map +1 -0
- package/dist/prefetch/targetIndex.d.ts +18 -0
- package/dist/prefetch/targetIndex.d.ts.map +1 -0
- package/dist/prefetch/targetIndex.js +32 -0
- package/dist/prefetch/targetIndex.js.map +1 -0
- package/dist/prefetch/types.d.ts +283 -0
- package/dist/prefetch/types.d.ts.map +1 -0
- package/dist/prefetch/types.js +19 -0
- package/dist/prefetch/types.js.map +1 -0
- package/dist/previewBridge.d.ts +6 -0
- package/dist/previewBridge.d.ts.map +1 -1
- package/dist/previewBridge.js +25 -5
- package/dist/previewBridge.js.map +1 -1
- package/dist/previewMessages.d.ts +4 -3
- package/dist/previewMessages.d.ts.map +1 -1
- package/dist/previewMessages.js +12 -6
- package/dist/previewMessages.js.map +1 -1
- package/dist/process-worker.js +116 -109
- package/dist/relay.html +1 -1
- package/dist/shared/clock.d.ts +7 -0
- package/dist/shared/clock.d.ts.map +1 -0
- package/dist/shared/clock.js +10 -0
- package/dist/shared/clock.js.map +1 -0
- package/dist/shared/hardwareProbe.d.ts +26 -0
- package/dist/shared/hardwareProbe.d.ts.map +1 -0
- package/dist/shared/hardwareProbe.js +62 -0
- package/dist/shared/hardwareProbe.js.map +1 -0
- package/dist/shared/idb.d.ts +41 -0
- package/dist/shared/idb.d.ts.map +1 -0
- package/dist/shared/idb.js +184 -0
- package/dist/shared/idb.js.map +1 -0
- package/dist/shared/tokens.d.ts +7 -0
- package/dist/shared/tokens.d.ts.map +1 -0
- package/dist/shared/tokens.js +9 -0
- package/dist/shared/tokens.js.map +1 -0
- package/dist/snapshotStore.d.ts +2 -1
- package/dist/snapshotStore.d.ts.map +1 -1
- package/dist/snapshotStore.js +27 -12
- package/dist/snapshotStore.js.map +1 -1
- package/dist/styleCompilers.d.ts.map +1 -1
- package/dist/styleCompilers.js.map +1 -1
- package/dist/transport.js.map +1 -1
- package/package.json +11 -5
- package/src/adaptive/controller.ts +151 -0
- package/src/adaptive/costModel.ts +131 -0
- package/src/adaptive/planner.ts +127 -0
- package/src/adaptive/types.ts +137 -0
- package/src/check/index.ts +6 -1
- package/src/host.ts +33 -11
- package/src/inference/backends/mock.ts +77 -0
- package/src/inference/backends/onnx.ts +91 -0
- package/src/inference/backends/webllm.ts +79 -0
- package/src/inference/backends/wllama.ts +79 -0
- package/src/inference/capabilities.ts +46 -0
- package/src/inference/client.ts +276 -0
- package/src/inference/modelCache.ts +145 -0
- package/src/inference/protocol.ts +119 -0
- package/src/inference/tokenStream.ts +92 -0
- package/src/inference/types.ts +115 -0
- package/src/inference/worker.ts +191 -0
- package/src/inference/workerEntry.ts +18 -0
- package/src/kvcache/advanced.ts +86 -0
- package/src/kvcache/compatibility.ts +51 -0
- package/src/kvcache/lru.ts +100 -0
- package/src/kvcache/paged.ts +270 -0
- package/src/kvcache/quant.ts +88 -0
- package/src/kvcache/radix.ts +68 -0
- package/src/kvcache/runtime.ts +239 -0
- package/src/kvcache/types.ts +167 -0
- package/src/learning/fingerprint.ts +50 -0
- package/src/learning/index.ts +15 -0
- package/src/learning/learner.ts +304 -0
- package/src/learning/ledger.ts +54 -0
- package/src/learning/selection.ts +137 -0
- package/src/learning/stats.ts +39 -0
- package/src/learning/types.ts +142 -0
- package/src/memory/envProbe.ts +30 -0
- package/src/memory/estimates.ts +98 -0
- package/src/memory/eviction.ts +145 -0
- package/src/memory/manager.ts +380 -0
- package/src/memory/managerTypes.ts +61 -0
- package/src/memory/pressure.ts +120 -0
- package/src/memory/spillTier.ts +159 -0
- package/src/memory/storageSink.ts +289 -0
- package/src/memory/types.ts +217 -0
- package/src/memory/victims.ts +80 -0
- package/src/memory/workspace.ts +74 -0
- package/src/memoryIntegration/index.ts +7 -0
- package/src/memoryIntegration/knowledgeMemory.ts +339 -0
- package/src/memoryIntegration/persistence.ts +54 -0
- package/src/memoryIntegration/text.ts +185 -0
- package/src/memoryIntegration/types.ts +145 -0
- package/src/ml.ts +180 -0
- package/src/node/childRelay.ts +16 -3
- package/src/node/portResponder.ts +5 -3
- package/src/node/processHost.ts +16 -15
- package/src/node/protocol.ts +1 -1
- package/src/node/workerProcess.ts +1 -1
- package/src/prefetch/admission.ts +74 -0
- package/src/prefetch/loadRegistry.ts +156 -0
- package/src/prefetch/predictor.ts +199 -0
- package/src/prefetch/prefetcher.ts +369 -0
- package/src/prefetch/residentStore.ts +121 -0
- package/src/prefetch/selfTuner.ts +110 -0
- package/src/prefetch/stats.ts +120 -0
- package/src/prefetch/streaming.ts +231 -0
- package/src/prefetch/targetIndex.ts +40 -0
- package/src/prefetch/types.ts +307 -0
- package/src/previewBridge.ts +25 -5
- package/src/previewMessages.ts +11 -6
- package/src/shared/clock.ts +9 -0
- package/src/shared/hardwareProbe.ts +68 -0
- package/src/shared/idb.ts +203 -0
- package/src/shared/tokens.ts +8 -0
- package/src/snapshotStore.ts +25 -10
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 3 — victim selection and quota accounting for the residency manager.
|
|
3
|
+
*
|
|
4
|
+
* Pure over the snapshot it is given: it never mutates the residents map, so
|
|
5
|
+
* eviction stays transactional (the manager drops victims only once the
|
|
6
|
+
* admission is known to fit).
|
|
7
|
+
*
|
|
8
|
+
* Cost: usage is computed ONCE per selection — the global total plus one
|
|
9
|
+
* running sum per quota that applies to the incoming entry — and each chosen
|
|
10
|
+
* candidate is subtracted incrementally. A selection is O(n · q) plus the
|
|
11
|
+
* strategy's rank, not O(n² · q).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { EvictionStrategy, Quota, ResidentEntry, TrackedEntry } from './types.js';
|
|
15
|
+
|
|
16
|
+
/** True when `quota` charges `entry` (same group, and same component when the quota names one). */
|
|
17
|
+
export function quotaApplies(quota: Quota, entry: Pick<ResidentEntry, 'group' | 'component'>): boolean {
|
|
18
|
+
if (quota.group !== entry.group) return false;
|
|
19
|
+
if (quota.component !== undefined && quota.component !== entry.component) return false;
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface VictimQuery {
|
|
24
|
+
incoming: ResidentEntry;
|
|
25
|
+
residents: Iterable<TrackedEntry>;
|
|
26
|
+
/** Ids treated as already gone (the same-id resident being replaced). */
|
|
27
|
+
except: ReadonlySet<string>;
|
|
28
|
+
strategy: EvictionStrategy;
|
|
29
|
+
quotas: readonly Quota[];
|
|
30
|
+
byteBudget: number;
|
|
31
|
+
workspaceBytes: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Choose victims without mutating. Same-group residents are offered first,
|
|
36
|
+
* each group in strategy order. Returns `[]` when the admission already fits,
|
|
37
|
+
* and undefined when no subset of unpinned residents makes it fit.
|
|
38
|
+
*/
|
|
39
|
+
export function selectVictims(query: VictimQuery): TrackedEntry[] | undefined {
|
|
40
|
+
const { incoming, except } = query;
|
|
41
|
+
const applicable = query.quotas.filter((quota) => quotaApplies(quota, incoming));
|
|
42
|
+
const quotaUsed = applicable.map(() => 0);
|
|
43
|
+
let total = 0;
|
|
44
|
+
const evictable: TrackedEntry[] = [];
|
|
45
|
+
|
|
46
|
+
for (const entry of query.residents) {
|
|
47
|
+
if (except.has(entry.id)) continue;
|
|
48
|
+
total += entry.bytes;
|
|
49
|
+
for (let i = 0; i < applicable.length; i++) {
|
|
50
|
+
if (quotaApplies(applicable[i]!, entry)) quotaUsed[i] = (quotaUsed[i] ?? 0) + entry.bytes;
|
|
51
|
+
}
|
|
52
|
+
if (!entry.pinned) evictable.push(entry);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const fits = (): boolean => {
|
|
56
|
+
if (total + query.workspaceBytes + incoming.bytes > query.byteBudget) return false;
|
|
57
|
+
for (let i = 0; i < applicable.length; i++) {
|
|
58
|
+
if ((quotaUsed[i] ?? 0) + incoming.bytes > applicable[i]!.byteCap) return false;
|
|
59
|
+
}
|
|
60
|
+
return true;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
if (fits()) return [];
|
|
64
|
+
const ranked = query.strategy.rank(evictable);
|
|
65
|
+
const sameGroup = ranked.filter((entry) => entry.group === incoming.group);
|
|
66
|
+
const rest = ranked.filter((entry) => entry.group !== incoming.group);
|
|
67
|
+
const chosen: TrackedEntry[] = [];
|
|
68
|
+
const seen = new Set<string>();
|
|
69
|
+
for (const candidate of [...sameGroup, ...rest]) {
|
|
70
|
+
if (seen.has(candidate.id)) continue;
|
|
71
|
+
seen.add(candidate.id);
|
|
72
|
+
total -= candidate.bytes;
|
|
73
|
+
for (let i = 0; i < applicable.length; i++) {
|
|
74
|
+
if (quotaApplies(applicable[i]!, candidate)) quotaUsed[i] = (quotaUsed[i] ?? 0) - candidate.bytes;
|
|
75
|
+
}
|
|
76
|
+
chosen.push(candidate);
|
|
77
|
+
if (fits()) return chosen.map((entry) => ({ ...entry }));
|
|
78
|
+
}
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 3 — the residency manager's dynamic workspace.
|
|
3
|
+
*
|
|
4
|
+
* The workspace is charged against the hard byte budget. It grows on hits
|
|
5
|
+
* while headroom remains and shrinks under pressure (the plan comes from
|
|
6
|
+
* `planWorkspace`), always inside [min, max] and never so far that residents
|
|
7
|
+
* plus workspace would cross the budget.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { planWorkspace, type WorkspaceBounds } from './pressure.js';
|
|
11
|
+
import type { MemoryPressure, ResidencyEvent } from './types.js';
|
|
12
|
+
|
|
13
|
+
export interface WorkspaceControllerOptions {
|
|
14
|
+
bounds: WorkspaceBounds;
|
|
15
|
+
byteBudget: number;
|
|
16
|
+
initialBytes?: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** A workspace change, ready for the manager's event log. */
|
|
20
|
+
export type WorkspaceChange = Omit<ResidencyEvent, 'at' | 'id'> & { kind: 'workspace-grow' | 'workspace-shrink' };
|
|
21
|
+
|
|
22
|
+
export interface WorkspaceController {
|
|
23
|
+
/** Bytes currently allocated to the workspace. */
|
|
24
|
+
bytes(): number;
|
|
25
|
+
/**
|
|
26
|
+
* Re-plan after a hit (`onHit`) or a pressure reading. `residentBytes` is
|
|
27
|
+
* what the residents alone hold. Returns the change applied, if any.
|
|
28
|
+
*/
|
|
29
|
+
apply(input: { onHit: boolean; pressure: MemoryPressure; residentBytes: number }): WorkspaceChange | undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function createWorkspaceController(options: WorkspaceControllerOptions): WorkspaceController {
|
|
33
|
+
const { bounds, byteBudget } = options;
|
|
34
|
+
const active = bounds.maxBytes > 0 || bounds.minBytes > 0;
|
|
35
|
+
|
|
36
|
+
function clamp(bytes: number): number {
|
|
37
|
+
if (!active) return 0;
|
|
38
|
+
// Charged against the hard budget, so it can never start (or grow) past
|
|
39
|
+
// it — even if the caller asked for a larger floor.
|
|
40
|
+
const cap = Math.min(bounds.maxBytes, byteBudget);
|
|
41
|
+
const floor = Math.min(bounds.minBytes, cap);
|
|
42
|
+
const value = Number.isFinite(bytes) ? bytes : floor;
|
|
43
|
+
return Math.min(cap, Math.max(floor, value));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
let current = clamp(options.initialBytes ?? bounds.minBytes);
|
|
47
|
+
|
|
48
|
+
return {
|
|
49
|
+
bytes: () => current,
|
|
50
|
+
apply({ onHit, pressure, residentBytes }) {
|
|
51
|
+
if (!active) return undefined;
|
|
52
|
+
const decision = planWorkspace({
|
|
53
|
+
currentBytes: current,
|
|
54
|
+
bounds,
|
|
55
|
+
pressure,
|
|
56
|
+
byteBudget,
|
|
57
|
+
usedBytes: residentBytes + current,
|
|
58
|
+
onHit,
|
|
59
|
+
});
|
|
60
|
+
if (decision.deltaBytes === 0) return undefined;
|
|
61
|
+
const next = clamp(decision.nextBytes);
|
|
62
|
+
if (next === current) return undefined;
|
|
63
|
+
// Never let a workspace change push the total over the hard budget.
|
|
64
|
+
if (residentBytes + next > byteBudget) return undefined;
|
|
65
|
+
const delta = next - current;
|
|
66
|
+
current = next;
|
|
67
|
+
return {
|
|
68
|
+
kind: delta > 0 ? 'workspace-grow' : 'workspace-shrink',
|
|
69
|
+
bytes: Math.abs(delta),
|
|
70
|
+
reason: decision.reason,
|
|
71
|
+
};
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { createKnowledgeMemory } from './knowledgeMemory.js';
|
|
2
|
+
export type { KnowledgeMemory, KnowledgeMemoryOptions, KnowledgeMemoryStats } from './knowledgeMemory.js';
|
|
3
|
+
export { createIndexedDbKnowledgePersistence, createMemoryKnowledgePersistence } from './persistence.js';
|
|
4
|
+
export type { IndexedDbKnowledgePersistenceOptions } from './persistence.js';
|
|
5
|
+
export { chunkText, normalizeText, contentHash, estimateTokens, tokenize, LexicalIndex } from './text.js';
|
|
6
|
+
export type { TextChunk, ChunkOptions } from './text.js';
|
|
7
|
+
export type * from './types.js';
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 6 — the persistent knowledge/retrieval layer.
|
|
3
|
+
*
|
|
4
|
+
* Lifecycle: capture → normalise → chunk → metadata → embed (injected embedder)
|
|
5
|
+
* → index (BM25 + dense) → retrieve (hybrid RRF + filters + recency + rerank hook)
|
|
6
|
+
* → assemble context (dedup, compress, token budget, citations) → update/delete →
|
|
7
|
+
* compact (retention). Incremental: a document whose normalised content hash is
|
|
8
|
+
* unchanged is never re-embedded; a changed document re-embeds only chunks whose
|
|
9
|
+
* hash is new. Vectors are stamped with the embedder fingerprint; on restore a
|
|
10
|
+
* document made under another fingerprint is reported stale, never compared.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
LexicalIndex,
|
|
15
|
+
chunkText,
|
|
16
|
+
compareHits,
|
|
17
|
+
contentHash,
|
|
18
|
+
normalizeText,
|
|
19
|
+
toUnit,
|
|
20
|
+
type ChunkOptions,
|
|
21
|
+
} from './text.js';
|
|
22
|
+
import { estimateTokens } from '../shared/tokens.js';
|
|
23
|
+
import type {
|
|
24
|
+
AssembledContext,
|
|
25
|
+
ContextOptions,
|
|
26
|
+
IngestResult,
|
|
27
|
+
KnowledgeDocument,
|
|
28
|
+
KnowledgeHit,
|
|
29
|
+
KnowledgePersistence,
|
|
30
|
+
RetentionPolicy,
|
|
31
|
+
RetrieveOptions,
|
|
32
|
+
SourceReference,
|
|
33
|
+
StoredChunk,
|
|
34
|
+
StoredKnowledgeDoc,
|
|
35
|
+
TextEmbedder,
|
|
36
|
+
} from './types.js';
|
|
37
|
+
|
|
38
|
+
export interface KnowledgeMemoryOptions {
|
|
39
|
+
embedder: TextEmbedder;
|
|
40
|
+
persistence?: KnowledgePersistence;
|
|
41
|
+
chunk?: ChunkOptions;
|
|
42
|
+
/** Clock (epoch ms). Default `Date.now`. */
|
|
43
|
+
now?: () => number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface KnowledgeMemoryStats {
|
|
47
|
+
documents: number;
|
|
48
|
+
chunks: number;
|
|
49
|
+
totalChunkTokens: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface KnowledgeMemory {
|
|
53
|
+
/** Restore the persisted index. Returns ids of documents made under another embedder fingerprint (need re-ingest). */
|
|
54
|
+
load(): Promise<{ restored: number; stale: string[] }>;
|
|
55
|
+
ingest(doc: KnowledgeDocument): Promise<IngestResult>;
|
|
56
|
+
ingestMany(docs: KnowledgeDocument[]): Promise<IngestResult[]>;
|
|
57
|
+
remove(id: string): Promise<boolean>;
|
|
58
|
+
get(id: string): StoredKnowledgeDoc | undefined;
|
|
59
|
+
retrieve(query: string, opts?: RetrieveOptions): Promise<KnowledgeHit[]>;
|
|
60
|
+
assembleContext(query: string, opts: ContextOptions): Promise<AssembledContext>;
|
|
61
|
+
/** Retention/maintenance; returns removed ids. */
|
|
62
|
+
compact(policy: RetentionPolicy): Promise<string[]>;
|
|
63
|
+
stats(): KnowledgeMemoryStats;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const DAY_MS = 86_400_000;
|
|
67
|
+
|
|
68
|
+
export function createKnowledgeMemory(options: KnowledgeMemoryOptions): KnowledgeMemory {
|
|
69
|
+
const { embedder, persistence } = options;
|
|
70
|
+
const clock = options.now ?? Date.now;
|
|
71
|
+
const docs = new Map<string, StoredKnowledgeDoc>();
|
|
72
|
+
const chunks = new Map<string, { doc: StoredKnowledgeDoc; chunk: StoredChunk }>();
|
|
73
|
+
const lexical = new LexicalIndex();
|
|
74
|
+
|
|
75
|
+
const indexDoc = (doc: StoredKnowledgeDoc) => {
|
|
76
|
+
docs.set(doc.id, doc);
|
|
77
|
+
for (const c of doc.chunks) {
|
|
78
|
+
chunks.set(c.id, { doc, chunk: c });
|
|
79
|
+
lexical.set(c.id, doc.title ? `${doc.title}\n${c.text}` : c.text);
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
const unindexDoc = (id: string) => {
|
|
83
|
+
const doc = docs.get(id);
|
|
84
|
+
if (!doc) return;
|
|
85
|
+
for (const c of doc.chunks) {
|
|
86
|
+
chunks.delete(c.id);
|
|
87
|
+
lexical.delete(c.id);
|
|
88
|
+
}
|
|
89
|
+
docs.delete(id);
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const refOf = (doc: StoredKnowledgeDoc, c: StoredChunk): SourceReference => ({
|
|
93
|
+
sourceId: doc.id,
|
|
94
|
+
version: doc.version,
|
|
95
|
+
contentHash: doc.contentHash,
|
|
96
|
+
chunkId: c.id,
|
|
97
|
+
chunkIndex: c.index,
|
|
98
|
+
start: c.start,
|
|
99
|
+
kind: doc.kind,
|
|
100
|
+
...(doc.title !== undefined ? { title: doc.title } : {}),
|
|
101
|
+
...(doc.uri !== undefined ? { uri: doc.uri } : {}),
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
const passes = (doc: StoredKnowledgeDoc, opts: RetrieveOptions): boolean => {
|
|
105
|
+
if (opts.kinds && !opts.kinds.includes(doc.kind)) return false;
|
|
106
|
+
const f = opts.filter;
|
|
107
|
+
if (!f) return true;
|
|
108
|
+
if (typeof f === 'function') return f(doc);
|
|
109
|
+
for (const [k, v] of Object.entries(f)) if (doc.metadata[k] !== v) return false;
|
|
110
|
+
return true;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
// Per-id mutation chain: ingest/remove of the same id run strictly in CALL
|
|
114
|
+
// order, so the last call wins in the index AND in persistence (a slow embed
|
|
115
|
+
// of an older version can never overwrite a newer one). Different ids run
|
|
116
|
+
// concurrently.
|
|
117
|
+
const tails = new Map<string, Promise<unknown>>();
|
|
118
|
+
function serialize<T>(id: string, task: () => Promise<T>): Promise<T> {
|
|
119
|
+
const run = (tails.get(id) ?? Promise.resolve()).then(task, task);
|
|
120
|
+
const tail = run.then(
|
|
121
|
+
() => undefined,
|
|
122
|
+
() => undefined,
|
|
123
|
+
);
|
|
124
|
+
tails.set(id, tail);
|
|
125
|
+
void tail.then(() => {
|
|
126
|
+
if (tails.get(id) === tail) tails.delete(id);
|
|
127
|
+
});
|
|
128
|
+
return run;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const ingest = (input: KnowledgeDocument): Promise<IngestResult> => serialize(input.id, () => ingestNow(input));
|
|
132
|
+
|
|
133
|
+
async function ingestNow(input: KnowledgeDocument): Promise<IngestResult> {
|
|
134
|
+
const text = normalizeText(input.text);
|
|
135
|
+
const hash = contentHash(text);
|
|
136
|
+
const prior = docs.get(input.id);
|
|
137
|
+
const kind = input.kind ?? 'business-doc';
|
|
138
|
+
const metaSame =
|
|
139
|
+
prior !== undefined &&
|
|
140
|
+
prior.kind === kind &&
|
|
141
|
+
prior.title === input.title &&
|
|
142
|
+
prior.uri === input.uri &&
|
|
143
|
+
JSON.stringify(prior.metadata) === JSON.stringify(input.metadata ?? {});
|
|
144
|
+
if (prior && prior.contentHash === hash && prior.fingerprint === embedder.fingerprint && metaSame) {
|
|
145
|
+
return { id: input.id, status: 'unchanged', version: prior.version, chunks: prior.chunks.length, embedded: 0 };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Reuse vectors of chunks whose text is unchanged (same fingerprint only).
|
|
149
|
+
const reusable = new Map<string, Float32Array>();
|
|
150
|
+
if (prior && prior.fingerprint === embedder.fingerprint) for (const c of prior.chunks) reusable.set(c.hash, c.vector);
|
|
151
|
+
|
|
152
|
+
const seen = new Set<string>();
|
|
153
|
+
const stored: StoredChunk[] = [];
|
|
154
|
+
let embedded = 0;
|
|
155
|
+
for (const piece of chunkText(text, options.chunk)) {
|
|
156
|
+
const h = contentHash(piece.text);
|
|
157
|
+
if (seen.has(h)) continue; // duplicate chunk within the source
|
|
158
|
+
seen.add(h);
|
|
159
|
+
let vector = reusable.get(h);
|
|
160
|
+
if (!vector) {
|
|
161
|
+
vector = toUnit(await embedder.embed(piece.text));
|
|
162
|
+
embedded++;
|
|
163
|
+
}
|
|
164
|
+
stored.push({
|
|
165
|
+
id: `${input.id}#${stored.length}`,
|
|
166
|
+
index: stored.length,
|
|
167
|
+
start: piece.start,
|
|
168
|
+
text: piece.text,
|
|
169
|
+
hash: h,
|
|
170
|
+
tokens: estimateTokens(piece.text),
|
|
171
|
+
vector,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const doc: StoredKnowledgeDoc = {
|
|
176
|
+
id: input.id,
|
|
177
|
+
kind,
|
|
178
|
+
...(input.title !== undefined ? { title: input.title } : {}),
|
|
179
|
+
...(input.uri !== undefined ? { uri: input.uri } : {}),
|
|
180
|
+
metadata: { ...(input.metadata ?? {}) },
|
|
181
|
+
updatedAt: input.updatedAt ?? clock(),
|
|
182
|
+
version: prior && prior.contentHash === hash ? prior.version : (prior?.version ?? 0) + 1,
|
|
183
|
+
contentHash: hash,
|
|
184
|
+
fingerprint: embedder.fingerprint,
|
|
185
|
+
chunks: stored,
|
|
186
|
+
};
|
|
187
|
+
unindexDoc(input.id);
|
|
188
|
+
indexDoc(doc);
|
|
189
|
+
await persistence?.put(doc);
|
|
190
|
+
return { id: doc.id, status: prior ? 'updated' : 'added', version: doc.version, chunks: stored.length, embedded };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async function retrieve(query: string, opts: RetrieveOptions = {}): Promise<KnowledgeHit[]> {
|
|
194
|
+
const topK = opts.topK ?? 5;
|
|
195
|
+
const pool = opts.candidatePool ?? 100;
|
|
196
|
+
const rrfK = opts.rrfK ?? 60;
|
|
197
|
+
if (chunks.size === 0 || topK <= 0) return [];
|
|
198
|
+
|
|
199
|
+
const sparse: string[] = [];
|
|
200
|
+
for (const h of lexical.search(query)) {
|
|
201
|
+
if (sparse.length >= pool) break;
|
|
202
|
+
if (passes(chunks.get(h.id)!.doc, opts)) sparse.push(h.id);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const qv = toUnit(await embedder.embed(query));
|
|
206
|
+
const dense: Array<{ id: string; score: number }> = [];
|
|
207
|
+
for (const [id, { doc, chunk }] of chunks) {
|
|
208
|
+
if (!passes(doc, opts)) continue;
|
|
209
|
+
const v = chunk.vector;
|
|
210
|
+
const n = Math.min(v.length, qv.length);
|
|
211
|
+
let dot = 0;
|
|
212
|
+
for (let i = 0; i < n; i++) dot += v[i]! * qv[i]!;
|
|
213
|
+
dense.push({ id, score: dot });
|
|
214
|
+
}
|
|
215
|
+
dense.sort(compareHits);
|
|
216
|
+
|
|
217
|
+
const fused = new Map<string, number>();
|
|
218
|
+
const add = (ids: string[], w: number) =>
|
|
219
|
+
ids.forEach((id, rank) => fused.set(id, (fused.get(id) ?? 0) + w / (rrfK + rank + 1)));
|
|
220
|
+
add(dense.slice(0, pool).map((d) => d.id), opts.denseWeight ?? 1);
|
|
221
|
+
add(sparse, opts.sparseWeight ?? 1);
|
|
222
|
+
|
|
223
|
+
const now = opts.now ?? clock();
|
|
224
|
+
const halfLife = opts.recencyHalfLifeDays ?? 0;
|
|
225
|
+
const ranked = [...fused].map(([id, score]) => {
|
|
226
|
+
if (halfLife > 0) {
|
|
227
|
+
const age = Math.max(0, (now - chunks.get(id)!.doc.updatedAt) / DAY_MS);
|
|
228
|
+
score *= Math.exp((-Math.LN2 / halfLife) * age);
|
|
229
|
+
}
|
|
230
|
+
return { id, score };
|
|
231
|
+
});
|
|
232
|
+
ranked.sort(compareHits);
|
|
233
|
+
|
|
234
|
+
let hits: KnowledgeHit[] = ranked.slice(0, opts.rerank ? Math.max(topK * 4, topK) : topK).map(({ id, score }) => {
|
|
235
|
+
const { doc, chunk } = chunks.get(id)!;
|
|
236
|
+
return { chunkId: id, text: chunk.text, score, source: refOf(doc, chunk), metadata: { ...doc.metadata } };
|
|
237
|
+
});
|
|
238
|
+
if (opts.rerank) hits = (await opts.rerank(query, hits)).slice(0, topK);
|
|
239
|
+
return hits;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async function assembleContext(query: string, opts: ContextOptions): Promise<AssembledContext> {
|
|
243
|
+
const hits = await retrieve(query, { ...opts, topK: opts.topK ?? 8 });
|
|
244
|
+
const kept: KnowledgeHit[] = [];
|
|
245
|
+
const parts: string[] = [];
|
|
246
|
+
const seenHashes = new Set<string>();
|
|
247
|
+
let tokens = 0;
|
|
248
|
+
let duplicatesRemoved = 0;
|
|
249
|
+
let droppedForBudget = 0;
|
|
250
|
+
for (const hit of hits) {
|
|
251
|
+
const compressed = hit.text.replace(/\s+/g, ' ').trim();
|
|
252
|
+
const h = contentHash(compressed);
|
|
253
|
+
if (seenHashes.has(h) || parts.some((p) => p.includes(compressed))) {
|
|
254
|
+
duplicatesRemoved++;
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
const s = hit.source;
|
|
258
|
+
const block = `[${kept.length + 1}] ${s.sourceId} v${s.version} §${s.chunkIndex}\n${compressed}`;
|
|
259
|
+
const t = estimateTokens(block) + (parts.length > 0 ? 1 : 0);
|
|
260
|
+
if (tokens + t > opts.tokenBudget) {
|
|
261
|
+
droppedForBudget++;
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
seenHashes.add(h);
|
|
265
|
+
parts.push(block);
|
|
266
|
+
kept.push(hit);
|
|
267
|
+
tokens += t;
|
|
268
|
+
}
|
|
269
|
+
return {
|
|
270
|
+
text: parts.join('\n\n'),
|
|
271
|
+
tokens,
|
|
272
|
+
citations: kept.map((k) => k.source),
|
|
273
|
+
hits: kept,
|
|
274
|
+
duplicatesRemoved,
|
|
275
|
+
droppedForBudget,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function remove(id: string): Promise<boolean> {
|
|
280
|
+
return serialize(id, async () => {
|
|
281
|
+
const had = docs.has(id);
|
|
282
|
+
unindexDoc(id);
|
|
283
|
+
await persistence?.delete(id);
|
|
284
|
+
return had;
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
return {
|
|
289
|
+
async load() {
|
|
290
|
+
const stale: string[] = [];
|
|
291
|
+
let restored = 0;
|
|
292
|
+
for (const d of persistence ? await persistence.load() : []) {
|
|
293
|
+
if (d.fingerprint !== embedder.fingerprint) {
|
|
294
|
+
stale.push(d.id);
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
unindexDoc(d.id);
|
|
298
|
+
indexDoc({ ...d, chunks: d.chunks.map((c) => ({ ...c, vector: Float32Array.from(c.vector) })) });
|
|
299
|
+
restored++;
|
|
300
|
+
}
|
|
301
|
+
return { restored, stale };
|
|
302
|
+
},
|
|
303
|
+
ingest,
|
|
304
|
+
async ingestMany(list) {
|
|
305
|
+
const out: IngestResult[] = [];
|
|
306
|
+
for (const d of list) out.push(await ingest(d));
|
|
307
|
+
return out;
|
|
308
|
+
},
|
|
309
|
+
remove,
|
|
310
|
+
get: (id) => docs.get(id),
|
|
311
|
+
retrieve,
|
|
312
|
+
assembleContext,
|
|
313
|
+
async compact(policy) {
|
|
314
|
+
const now = policy.now ?? clock();
|
|
315
|
+
const keep = new Set(policy.keepKinds ?? ['approved']);
|
|
316
|
+
const removed: string[] = [];
|
|
317
|
+
const candidates = [...docs.values()].filter((d) => !keep.has(d.kind));
|
|
318
|
+
if (policy.maxAgeMs !== undefined) {
|
|
319
|
+
for (const d of candidates) if (now - d.updatedAt > policy.maxAgeMs) removed.push(d.id);
|
|
320
|
+
}
|
|
321
|
+
if (policy.maxDocuments !== undefined) {
|
|
322
|
+
const over = docs.size - removed.length - policy.maxDocuments;
|
|
323
|
+
if (over > 0) {
|
|
324
|
+
const rest = candidates
|
|
325
|
+
.filter((d) => !removed.includes(d.id))
|
|
326
|
+
.sort((a, b) => a.updatedAt - b.updatedAt || (a.id < b.id ? -1 : 1));
|
|
327
|
+
removed.push(...rest.slice(0, over).map((d) => d.id));
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
for (const id of removed) await remove(id);
|
|
331
|
+
return removed;
|
|
332
|
+
},
|
|
333
|
+
stats() {
|
|
334
|
+
let total = 0;
|
|
335
|
+
for (const { chunk } of chunks.values()) total += chunk.tokens;
|
|
336
|
+
return { documents: docs.size, chunks: chunks.size, totalChunkTokens: total };
|
|
337
|
+
},
|
|
338
|
+
};
|
|
339
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 6 — persistence for the knowledge layer. IndexedDB in the browser via the
|
|
3
|
+
* shared `lazyIdbStore` helper (no IndexedDB ⇒ a no-op store and the index is
|
|
4
|
+
* simply rebuilt), plus an in-memory implementation for tests.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { lazyIdbStore } from '../shared/idb.js';
|
|
8
|
+
import type { KnowledgePersistence, StoredKnowledgeDoc } from './types.js';
|
|
9
|
+
|
|
10
|
+
const DB_VERSION = 1;
|
|
11
|
+
const STORE = 'docs';
|
|
12
|
+
|
|
13
|
+
export interface IndexedDbKnowledgePersistenceOptions {
|
|
14
|
+
indexedDB?: IDBFactory;
|
|
15
|
+
/** Database name; one per knowledge collection. Default `bfwc-memory-knowledge`. */
|
|
16
|
+
dbName?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function createIndexedDbKnowledgePersistence(
|
|
20
|
+
options: IndexedDbKnowledgePersistenceOptions = {},
|
|
21
|
+
): KnowledgePersistence {
|
|
22
|
+
const factory = options.indexedDB ?? (globalThis as { indexedDB?: IDBFactory }).indexedDB;
|
|
23
|
+
const config = { dbName: options.dbName ?? 'bfwc-memory-knowledge', storeName: STORE, version: DB_VERSION };
|
|
24
|
+
const open = lazyIdbStore(factory, config);
|
|
25
|
+
return {
|
|
26
|
+
async load() {
|
|
27
|
+
const store = await open();
|
|
28
|
+
if (!store) return [];
|
|
29
|
+
return store.run('readonly', (s) => s.getAll() as IDBRequest<StoredKnowledgeDoc[]>);
|
|
30
|
+
},
|
|
31
|
+
async put(doc) {
|
|
32
|
+
await (await open())?.run('readwrite', (s) => s.put(doc, doc.id));
|
|
33
|
+
},
|
|
34
|
+
async delete(id) {
|
|
35
|
+
await (await open())?.run('readwrite', (s) => s.delete(id));
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Non-persistent store (tests, or a session-only knowledge layer). */
|
|
41
|
+
export function createMemoryKnowledgePersistence(): KnowledgePersistence {
|
|
42
|
+
const map = new Map<string, StoredKnowledgeDoc>();
|
|
43
|
+
return {
|
|
44
|
+
async load() {
|
|
45
|
+
return [...map.values()];
|
|
46
|
+
},
|
|
47
|
+
async put(doc) {
|
|
48
|
+
map.set(doc.id, doc);
|
|
49
|
+
},
|
|
50
|
+
async delete(id) {
|
|
51
|
+
map.delete(id);
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
}
|