@soulcraft/cor 3.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +328 -0
- package/dist/aggregation/NativeAggregationEngine.d.ts +118 -0
- package/dist/aggregation/NativeAggregationEngine.js +186 -0
- package/dist/cli.d.ts +12 -0
- package/dist/cli.js +236 -0
- package/dist/graph/GraphAccelerationAdapter.d.ts +153 -0
- package/dist/graph/GraphAccelerationAdapter.js +326 -0
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +286 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +944 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.d.ts +187 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.js +313 -0
- package/dist/hnsw/NativeDiskAnnWrapper.d.ts +489 -0
- package/dist/hnsw/NativeDiskAnnWrapper.js +1456 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +29 -0
- package/dist/legacyLayoutGuard.d.ts +93 -0
- package/dist/legacyLayoutGuard.js +237 -0
- package/dist/license/constants.d.ts +30 -0
- package/dist/license/constants.js +30 -0
- package/dist/license/onlineValidator.d.ts +43 -0
- package/dist/license/onlineValidator.js +225 -0
- package/dist/license/types.d.ts +39 -0
- package/dist/license/types.js +5 -0
- package/dist/license.d.ts +64 -0
- package/dist/license.js +165 -0
- package/dist/migration/MigrationCoordinator.d.ts +171 -0
- package/dist/migration/MigrationCoordinator.js +249 -0
- package/dist/migration/indexEpochGuard.d.ts +163 -0
- package/dist/migration/indexEpochGuard.js +202 -0
- package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
- package/dist/native/NativeEmbeddingEngine.js +318 -0
- package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
- package/dist/native/NativeRoaringBitmap32.js +221 -0
- package/dist/native/ffi.d.ts +20 -0
- package/dist/native/ffi.js +48 -0
- package/dist/native/idMapperTestSupport.d.ts +67 -0
- package/dist/native/idMapperTestSupport.js +112 -0
- package/dist/native/index.d.ts +49 -0
- package/dist/native/index.js +87 -0
- package/dist/native/napi.d.ts +21 -0
- package/dist/native/napi.js +88 -0
- package/dist/native/types.d.ts +1298 -0
- package/dist/native/types.js +16 -0
- package/dist/plugin.d.ts +50 -0
- package/dist/plugin.js +388 -0
- package/dist/providerContracts.d.ts +277 -0
- package/dist/providerContracts.js +38 -0
- package/dist/resource/OsMemoryProbe.d.ts +175 -0
- package/dist/resource/OsMemoryProbe.js +206 -0
- package/dist/resource/ResourceManager.d.ts +491 -0
- package/dist/resource/ResourceManager.js +960 -0
- package/dist/utils/ColumnManifest.d.ts +97 -0
- package/dist/utils/ColumnManifest.js +129 -0
- package/dist/utils/NativeColumnStore.d.ts +284 -0
- package/dist/utils/NativeColumnStore.js +685 -0
- package/dist/utils/NativeMetadataIndex.d.ts +882 -0
- package/dist/utils/NativeMetadataIndex.js +2631 -0
- package/dist/utils/NativeUnifiedCache.d.ts +87 -0
- package/dist/utils/NativeUnifiedCache.js +273 -0
- package/dist/utils/binaryIdMapperFactory.d.ts +59 -0
- package/dist/utils/binaryIdMapperFactory.js +94 -0
- package/dist/utils/collation.d.ts +30 -0
- package/dist/utils/collation.js +40 -0
- package/dist/utils/columnStoreTypes.d.ts +161 -0
- package/dist/utils/columnStoreTypes.js +29 -0
- package/dist/utils/nativeBinaryEntityIdMapper.d.ts +211 -0
- package/dist/utils/nativeBinaryEntityIdMapper.js +381 -0
- package/dist/utils/nativeEntityIdMapper.d.ts +111 -0
- package/dist/utils/nativeEntityIdMapper.js +170 -0
- package/docs/ADR-002-diskann-100-percent-rust.md +294 -0
- package/docs/ADR-003-semantic-time-travel.md +100 -0
- package/docs/aggregation.md +251 -0
- package/docs/comparison.md +162 -0
- package/docs/deployment-limits.md +87 -0
- package/docs/diskann.md +184 -0
- package/docs/migration-3.0.md +396 -0
- package/docs/performance-budget.md +274 -0
- package/docs/performance.md +117 -0
- package/docs/scaling.md +439 -0
- package/docs/snapshot-safety.md +246 -0
- package/docs/u64-id-space.md +214 -0
- package/docs/verification-report.md +670 -0
- package/native/brainy-native.node +0 -0
- package/native/index.d.ts +3916 -0
- package/package.json +92 -0
- package/scripts/migrate-cortex-2x-to-3x.mjs +970 -0
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module migration/MigrationCoordinator
|
|
3
|
+
* @description The cor-side driver of the #18 whole-brain 7.x → 8.0 upgrade — a
|
|
4
|
+
* **coordinated migration lock**. When cor opens a LARGE brain whose derived-index
|
|
5
|
+
* epoch is stale (drifted or absent; see {@link module:migration/indexEpochGuard}),
|
|
6
|
+
* the correct-and-clean upgrade is a BLOCKING one: rebuild every derived index
|
|
7
|
+
* (metadata LSM, DiskANN vector, graph adjacency) from the canonical records to a
|
|
8
|
+
* known-good state BEFORE serving — never serve a half-built or stale index. That
|
|
9
|
+
* decision (David, 2026-07-01) replaced the earlier "no-freeze background swap":
|
|
10
|
+
* no-freeze's availability is illusory (it serves degraded/half-built results),
|
|
11
|
+
* and *unknown/halfway states are more dangerous than a clean block*.
|
|
12
|
+
*
|
|
13
|
+
* ## The lock, coordinated with Brainy
|
|
14
|
+
*
|
|
15
|
+
* cor sets a lock the instant it detects a large stale brain; Brainy holds every
|
|
16
|
+
* public read AND write behind that lock (its `isMigrating()` gate, rc.9) and maps
|
|
17
|
+
* the lock to an HTTP-503-style readiness signal, so nothing observes a partial
|
|
18
|
+
* result and no write reaches a half-built index. The rebuild runs asynchronously
|
|
19
|
+
* — the LOCK, not a synchronous init hang, is what serializes access — and when
|
|
20
|
+
* all three indexes are rebuilt-from-canonical + the version marker is stamped,
|
|
21
|
+
* cor clears the lock and Brainy releases the queued operations against a fully
|
|
22
|
+
* correct brain. One invariant: *the brain is not readable/writable until all
|
|
23
|
+
* three indexes are rebuilt + stamped.*
|
|
24
|
+
*
|
|
25
|
+
* Brainy feature-detects three optional provider methods; every cor derived-index
|
|
26
|
+
* provider delegates them to the single coordinator so the answer is consistent
|
|
27
|
+
* across all three: {@link MigrationCoordinator#isMigrating} (the lock signal),
|
|
28
|
+
* {@link MigrationCoordinator#whenMigrationComplete} (the no-poll wait Brainy
|
|
29
|
+
* `Promise.race`s against its bounded timeout), and
|
|
30
|
+
* {@link MigrationCoordinator#migrationStatus} (the progress cor owns, which Brainy
|
|
31
|
+
* relays verbatim into `getIndexStatus().migration`).
|
|
32
|
+
*
|
|
33
|
+
* ## Failure is a locked-with-error state, never a half-served one
|
|
34
|
+
*
|
|
35
|
+
* If a rebuild throws, the coordinator KEEPS the brain locked (so Brainy never
|
|
36
|
+
* releases reads onto a half-built index) and rejects {@link
|
|
37
|
+
* MigrationCoordinator#whenMigrationComplete} with a loud, actionable error — the
|
|
38
|
+
* same "blocking beats halfway" principle applied to the failure path. A restart
|
|
39
|
+
* re-detects the drift and re-runs the idempotent in-place rebuild, so a transient
|
|
40
|
+
* hiccup self-heals; genuinely-corrupt canonical data stays safely blocked with a
|
|
41
|
+
* pointer to the offline `scripts/migrate-cortex-2x-to-3x.mjs` escape hatch.
|
|
42
|
+
*
|
|
43
|
+
* The rebuild is IN-PLACE (no shadow directories, no serve-from-canonical, no
|
|
44
|
+
* concurrent-write reconciliation) precisely because everything is locked: no live
|
|
45
|
+
* index is read while it is being rebuilt.
|
|
46
|
+
*/
|
|
47
|
+
import { prodLog } from '@soulcraft/brainy/internals';
|
|
48
|
+
import { writeBrainFormat } from '@soulcraft/brainy/brain-format';
|
|
49
|
+
/**
|
|
50
|
+
* @description Drives the #18 coordinated blocking migration and answers the three
|
|
51
|
+
* Brainy-feature-detected lock methods. One instance per brain, created in the
|
|
52
|
+
* plugin's `activate` closure and shared by all three derived-index providers.
|
|
53
|
+
*
|
|
54
|
+
* @example
|
|
55
|
+
* const coordinator = new MigrationCoordinator(storage, {
|
|
56
|
+
* metadata: () => metadataIndexInstance,
|
|
57
|
+
* vector: () => vectorInstance,
|
|
58
|
+
* graph: () => graphEngine,
|
|
59
|
+
* })
|
|
60
|
+
* // ...in metadata init(), on a large stale brain:
|
|
61
|
+
* coordinator.start() // flips isMigrating() true synchronously, rebuilds async
|
|
62
|
+
*/
|
|
63
|
+
export class MigrationCoordinator {
|
|
64
|
+
storage;
|
|
65
|
+
providers;
|
|
66
|
+
/**
|
|
67
|
+
* `idle` before/without a migration; `migrating` while rebuilding; `done` once
|
|
68
|
+
* stamped; `failed` when a rebuild threw. `failed` reads as LOCKED (see
|
|
69
|
+
* {@link isMigrating}) so Brainy never releases reads onto a half-built index.
|
|
70
|
+
*/
|
|
71
|
+
state = 'idle';
|
|
72
|
+
status = null;
|
|
73
|
+
completePromise = null;
|
|
74
|
+
resolveComplete = null;
|
|
75
|
+
rejectComplete = null;
|
|
76
|
+
failure = null;
|
|
77
|
+
/**
|
|
78
|
+
* @param storage - The brain's storage adapter — passed to `writeBrainFormat`
|
|
79
|
+
* as the final stamp step (Brainy's own `dataFormat`/`indexEpoch` constants;
|
|
80
|
+
* cor never authors the format string).
|
|
81
|
+
* @param providers - Lazy resolvers for the three derived-index providers.
|
|
82
|
+
*/
|
|
83
|
+
constructor(storage, providers) {
|
|
84
|
+
this.storage = storage;
|
|
85
|
+
this.providers = providers;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* @description The lock signal Brainy gates every public read AND write on.
|
|
89
|
+
* `true` while migrating OR after a failure (a failed migration stays LOCKED so
|
|
90
|
+
* Brainy never serves the half-built index — a restart re-runs the idempotent
|
|
91
|
+
* rebuild).
|
|
92
|
+
* @returns `true` when the brain must not be read from or written to.
|
|
93
|
+
*/
|
|
94
|
+
isMigrating() {
|
|
95
|
+
return this.state === 'migrating' || this.state === 'failed';
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* @description Resolves when the migration completes (lock clears), or REJECTS
|
|
99
|
+
* with the loud failure error if it failed. Used internally by {@link start} (the
|
|
100
|
+
* already-migrating idempotency path) and by tests as the completion signal.
|
|
101
|
+
*
|
|
102
|
+
* Brainy does NOT call this: rc.9 dropped the provider-level `whenMigrationComplete`
|
|
103
|
+
* hook and polls `isMigrating()` at 250 ms instead (the event-driven variant
|
|
104
|
+
* busy-spun with mixed providers). `getIndexStatus().migrating` is the readiness
|
|
105
|
+
* signal, and a blocked op waits out the lock up to `migrationWaitTimeoutMs` before
|
|
106
|
+
* a retryable `MigrationInProgressError`.
|
|
107
|
+
* @returns A promise that settles when the lock is released or the migration fails.
|
|
108
|
+
*/
|
|
109
|
+
whenMigrationComplete() {
|
|
110
|
+
if (this.state === 'done')
|
|
111
|
+
return Promise.resolve();
|
|
112
|
+
if (this.state === 'failed') {
|
|
113
|
+
return Promise.reject(this.failure ?? new Error('migration failed'));
|
|
114
|
+
}
|
|
115
|
+
if (!this.completePromise) {
|
|
116
|
+
// idle and not yet started — a wait registered before start() still settles.
|
|
117
|
+
this.completePromise = new Promise((resolve, reject) => {
|
|
118
|
+
this.resolveComplete = resolve;
|
|
119
|
+
this.rejectComplete = reject;
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
return this.completePromise;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* @description The structured progress Brainy relays into
|
|
126
|
+
* `getIndexStatus().migration`. `null` unless a migration is in flight.
|
|
127
|
+
* @returns The current {@link MigrationStatus}, or `null`.
|
|
128
|
+
*/
|
|
129
|
+
migrationStatus() {
|
|
130
|
+
// Report during 'migrating' (rebuild progress) AND 'failed' — so a brain
|
|
131
|
+
// hard-locked by a failed migration surfaces WHY (phase: 'failed' + which index)
|
|
132
|
+
// via getIndexStatus().migration, not just `migrating: true` with no reason.
|
|
133
|
+
// Null when idle / done.
|
|
134
|
+
return this.state === 'migrating' || this.state === 'failed'
|
|
135
|
+
? this.status
|
|
136
|
+
: null;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* @description Kick the coordinated blocking rebuild. Flips {@link isMigrating}
|
|
140
|
+
* to `true` **synchronously** (so Brainy sees the lock the instant the triggering
|
|
141
|
+
* `init()` returns), then runs the rebuild asynchronously — the LOCK, not this
|
|
142
|
+
* promise, is what serializes reads/writes, so callers do NOT await this.
|
|
143
|
+
* Idempotent: calling it again while already migrating returns the in-flight
|
|
144
|
+
* completion promise without starting a second rebuild.
|
|
145
|
+
* @returns The completion promise (settles when the lock clears or the migration fails).
|
|
146
|
+
*/
|
|
147
|
+
start() {
|
|
148
|
+
// Already migrating → return the in-flight completion (no second rebuild).
|
|
149
|
+
if (this.state === 'migrating')
|
|
150
|
+
return this.whenMigrationComplete();
|
|
151
|
+
// Already migrated + stamped → no-op. (A re-open won't reach here — the epoch
|
|
152
|
+
// is current — but guard defensively against a double start() re-running the
|
|
153
|
+
// whole rebuild and re-stamping.)
|
|
154
|
+
if (this.state === 'done')
|
|
155
|
+
return Promise.resolve();
|
|
156
|
+
// Fresh run — 'idle', or the RETRY path after a prior 'failed' (the epoch was
|
|
157
|
+
// never stamped, so a re-open re-detects drift). run() NULLS the completion
|
|
158
|
+
// triplet when it settles, so `this.completePromise` here is either null (→ a
|
|
159
|
+
// fresh promise below) or a still-pending pre-start wait we reuse.
|
|
160
|
+
this.state = 'migrating';
|
|
161
|
+
this.failure = null;
|
|
162
|
+
this.status = { phase: 'starting' };
|
|
163
|
+
// Reuse a wait registered before start(); otherwise create the promise now.
|
|
164
|
+
if (!this.completePromise) {
|
|
165
|
+
this.completePromise = new Promise((resolve, reject) => {
|
|
166
|
+
this.resolveComplete = resolve;
|
|
167
|
+
this.rejectComplete = reject;
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
// The lock is already set above (synchronous); run the rebuild in the background.
|
|
171
|
+
void this.run();
|
|
172
|
+
return this.completePromise;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* The async body: rebuild metadata → vector → graph from canonical, verify by
|
|
176
|
+
* stamping (the marker certifies all three; Brainy withholds its own stamp while
|
|
177
|
+
* we are migrating), then clear the lock. Any throw leaves the brain LOCKED and
|
|
178
|
+
* rejects the completion promise.
|
|
179
|
+
*/
|
|
180
|
+
async run() {
|
|
181
|
+
const startedAt = Date.now();
|
|
182
|
+
// Track the index being rebuilt so a failure can report WHICH one failed.
|
|
183
|
+
let currentIndex;
|
|
184
|
+
try {
|
|
185
|
+
const steps = [
|
|
186
|
+
['metadata', this.providers.metadata()],
|
|
187
|
+
['vector', this.providers.vector()],
|
|
188
|
+
['graph', this.providers.graph()],
|
|
189
|
+
];
|
|
190
|
+
for (const [index, provider] of steps) {
|
|
191
|
+
// A resolver may return undefined (e.g. a provider a cloud adapter never
|
|
192
|
+
// engages) — skip it rather than fail the whole upgrade.
|
|
193
|
+
if (!provider)
|
|
194
|
+
continue;
|
|
195
|
+
currentIndex = index;
|
|
196
|
+
this.status = { phase: 'rebuilding', index };
|
|
197
|
+
prodLog.info(`[Cor] migration: rebuilding ${index} index from canonical records…`);
|
|
198
|
+
await provider.rebuild();
|
|
199
|
+
prodLog.info(`[Cor] migration: ${index} index rebuilt`);
|
|
200
|
+
}
|
|
201
|
+
// Stamp LAST — Brainy's own dataFormat + indexEpoch constants, written via
|
|
202
|
+
// the exported writer (no brain handle, no hardcoded '8.0'). The marker
|
|
203
|
+
// advancing is what certifies all three indexes are current; a crash before
|
|
204
|
+
// this leaves it stale → the next open re-detects drift → idempotent re-run.
|
|
205
|
+
// `writeRawObject` is on the concrete BaseStorage (used by the generational
|
|
206
|
+
// record layer for `_system/` artifacts), not the public `StorageAdapter`
|
|
207
|
+
// type — the same structural cast readOnDiskBrainFormat uses for the read side.
|
|
208
|
+
await writeBrainFormat(this.storage);
|
|
209
|
+
const elapsedMs = Date.now() - startedAt;
|
|
210
|
+
this.state = 'done';
|
|
211
|
+
this.status = null;
|
|
212
|
+
prodLog.info(`[Cor] migration complete (${elapsedMs} ms) — brain upgraded to 8.0 across all 3 derived indexes`);
|
|
213
|
+
// Clear the completion triplet THEN settle it: a subsequent run (retry path)
|
|
214
|
+
// must get a fresh promise + resolvers, never this already-settled one. The
|
|
215
|
+
// callers that hold the returned promise still settle (they reference the
|
|
216
|
+
// object, not the field); post-settle whenMigrationComplete() answers from
|
|
217
|
+
// `state`, so nulling the field is safe.
|
|
218
|
+
const resolve = this.resolveComplete;
|
|
219
|
+
this.completePromise = null;
|
|
220
|
+
this.resolveComplete = null;
|
|
221
|
+
this.rejectComplete = null;
|
|
222
|
+
resolve?.();
|
|
223
|
+
}
|
|
224
|
+
catch (error) {
|
|
225
|
+
// Keep the brain LOCKED on failure: releasing reads onto a half-built index
|
|
226
|
+
// would surface stale/partial results with no signal — worse than a clear
|
|
227
|
+
// block. Surface loudly; a restart re-detects the drift and re-runs the
|
|
228
|
+
// idempotent in-place rebuild (a transient hiccup self-heals; genuinely
|
|
229
|
+
// corrupt canonical data stays safely blocked).
|
|
230
|
+
this.failure = error instanceof Error ? error : new Error(String(error));
|
|
231
|
+
this.state = 'failed';
|
|
232
|
+
this.status = { phase: 'failed', index: currentIndex };
|
|
233
|
+
prodLog.error(`[Cor] migration FAILED — the brain stays locked (unmigrated) rather than ` +
|
|
234
|
+
`serve a half-built index. Restart to retry the automatic upgrade, or run ` +
|
|
235
|
+
`scripts/migrate-cortex-2x-to-3x.mjs for an offline migration. Cause: ` +
|
|
236
|
+
`${this.failure.message}`);
|
|
237
|
+
// Clear the completion triplet THEN reject it (see the success path) so the
|
|
238
|
+
// RETRY after this failure gets a fresh promise. Until that retry's start()
|
|
239
|
+
// resets `state`, whenMigrationComplete() rejects from the 'failed' state +
|
|
240
|
+
// the retained `this.failure`, not from this (now-detached) promise.
|
|
241
|
+
const reject = this.rejectComplete;
|
|
242
|
+
this.completePromise = null;
|
|
243
|
+
this.resolveComplete = null;
|
|
244
|
+
this.rejectComplete = null;
|
|
245
|
+
reject?.(this.failure);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
//# sourceMappingURL=MigrationCoordinator.js.map
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module migration/indexEpochGuard
|
|
3
|
+
* @description Read-side of the 7.x → 8.0 whole-brain auto-migration (#18) —
|
|
4
|
+
* the open-time epoch-drift detection cor performs on its own derived indexes
|
|
5
|
+
* (Adaptive DiskANN vectors, the LSM metadata postings, the graph adjacency
|
|
6
|
+
* trees).
|
|
7
|
+
*
|
|
8
|
+
* ## The handshake cor binds against
|
|
9
|
+
*
|
|
10
|
+
* Brainy 8.0 persists a tiny version-handshake marker at
|
|
11
|
+
* `_system/brain-format.json` (`{ dataFormat, indexEpoch }`). `indexEpoch` is a
|
|
12
|
+
* SHARED, lockstep-bumped integer that Brainy and cor advance TOGETHER on any
|
|
13
|
+
* coordinated release whose on-disk derived-index format changes. On open,
|
|
14
|
+
* Brainy compares the on-disk `indexEpoch` against its compiled
|
|
15
|
+
* `EXPECTED_INDEX_EPOCH`; a mismatch — or an absent marker (a pre-handshake 7.x
|
|
16
|
+
* brain, or a brand-new store) — means every derived index on disk predates
|
|
17
|
+
* this build and must be rebuilt from the canonical records.
|
|
18
|
+
*
|
|
19
|
+
* cor cannot reach Brainy's value: `brain.formatInfo()` is a brain method, and
|
|
20
|
+
* cor's providers receive only the storage adapter. cor also cannot import
|
|
21
|
+
* Brainy's `brainFormat` module (it is not in Brainy's public exports). So cor
|
|
22
|
+
* reads the same marker off storage directly and compiles its OWN
|
|
23
|
+
* {@link COR_EXPECTED_INDEX_EPOCH}.
|
|
24
|
+
*
|
|
25
|
+
* ## Lockstep contract — DO NOT bump unilaterally
|
|
26
|
+
*
|
|
27
|
+
* {@link COR_EXPECTED_INDEX_EPOCH} MUST equal Brainy's `EXPECTED_INDEX_EPOCH`
|
|
28
|
+
* (the constant declared at `@soulcraft/brainy/dist/storage/brainFormat.d.ts:69`
|
|
29
|
+
* — value `1`, the 8.0 GA baseline). The two are a duplicated constant on
|
|
30
|
+
* purpose: cor has no compile-time dependency on Brainy's marker module, so the
|
|
31
|
+
* value lives here too. NEVER advance one side alone. It advances only on a
|
|
32
|
+
* coordinated brainy+cor release whose on-disk derived-index format actually
|
|
33
|
+
* changed, and both projects ship the SAME new value in the same release — so a
|
|
34
|
+
* brain written by either side is recognised as current by the other, and a
|
|
35
|
+
* brain written by an older build of either side is recognised as stale and
|
|
36
|
+
* rebuilt.
|
|
37
|
+
*
|
|
38
|
+
* ## What this module does NOT do
|
|
39
|
+
*
|
|
40
|
+
* It performs read-side detection only — reading the marker, classifying drift,
|
|
41
|
+
* sizing the brain, and resolving the auto-migrate policy. The actual rebuild
|
|
42
|
+
* of a SMALL brain is driven by Brainy's `rebuildIndexesIfNeeded` (which calls
|
|
43
|
+
* each cor provider's `rebuild()` from canonical and re-stamps the marker). The
|
|
44
|
+
* non-blocking background build-new → verify → swap for a LARGE brain is the
|
|
45
|
+
* rc.8 increment (it needs Brainy's deference hook + a stamp method that are not
|
|
46
|
+
* yet shipped) and lives nowhere in this module.
|
|
47
|
+
*/
|
|
48
|
+
import type { StorageAdapter } from '@soulcraft/brainy';
|
|
49
|
+
/**
|
|
50
|
+
* @description The derived-index format epoch this cor build expects on disk.
|
|
51
|
+
* SHARED and lockstep-bumped with Brainy's `EXPECTED_INDEX_EPOCH`
|
|
52
|
+
* (`@soulcraft/brainy` → `storage/brainFormat`, value `1` — the 8.0 GA
|
|
53
|
+
* baseline). Advance it (in BOTH projects, to the same value, in one
|
|
54
|
+
* coordinated release) ONLY on a change to the on-disk format of any derived
|
|
55
|
+
* index — never on one side alone. An on-disk `indexEpoch` that differs from
|
|
56
|
+
* this — or an absent marker — means the derived indexes predate this build and
|
|
57
|
+
* are stale.
|
|
58
|
+
*/
|
|
59
|
+
export declare const COR_EXPECTED_INDEX_EPOCH = 1;
|
|
60
|
+
/**
|
|
61
|
+
* @description Storage-root-relative path of the version-handshake marker.
|
|
62
|
+
* Mirrors Brainy's `BRAIN_FORMAT_PATH` (the constant is not exported from
|
|
63
|
+
* Brainy, so it is re-declared here — a stable on-disk path).
|
|
64
|
+
*/
|
|
65
|
+
export declare const BRAIN_FORMAT_PATH = "_system/brain-format.json";
|
|
66
|
+
/**
|
|
67
|
+
* @description Default brain-size boundary (canonical noun count) above which a
|
|
68
|
+
* 7.x → 8.0 migration is treated as a LARGE brain. Mirrors Brainy's
|
|
69
|
+
* `AUTO_REBUILD_THRESHOLD` (10000): below it Brainy auto-rebuilds all derived
|
|
70
|
+
* indexes inline on open; at or above it Brainy defers to lazy first-query
|
|
71
|
+
* rebuilds. cor uses the SAME boundary so the two agree on which brains take the
|
|
72
|
+
* small (Brainy-driven) versus large (cor-owned, rc.8 background-swap) path.
|
|
73
|
+
* Overridable via the `COR_LARGE_BRAIN_THRESHOLD` env var.
|
|
74
|
+
*/
|
|
75
|
+
export declare const COR_LARGE_BRAIN_THRESHOLD_DEFAULT = 10000;
|
|
76
|
+
/**
|
|
77
|
+
* @description The parsed shape of `_system/brain-format.json` — the same two
|
|
78
|
+
* fields Brainy's `BrainFormat` carries.
|
|
79
|
+
*/
|
|
80
|
+
export interface OnDiskBrainFormat {
|
|
81
|
+
/** Brainy-owned data-layer version string (e.g. `'8.0'`). */
|
|
82
|
+
dataFormat: string;
|
|
83
|
+
/** Shared, lockstep-bumped derived-index format epoch (e.g. `1`). */
|
|
84
|
+
indexEpoch: number;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* @description Read the on-disk version-handshake marker, reimplementing
|
|
88
|
+
* Brainy's `readBrainFormat` validation so cor classifies drift identically
|
|
89
|
+
* without importing Brainy's (non-exported) marker module. Returns `null` when
|
|
90
|
+
* the marker is absent (a pre-handshake 7.x brain, or a brand-new store) OR
|
|
91
|
+
* malformed (not an object, missing either field, or a non-finite epoch) — a
|
|
92
|
+
* corrupt marker forces a safe rebuild rather than trusting a bad epoch.
|
|
93
|
+
*
|
|
94
|
+
* Storage that does not implement `readRawObject` (a non-`BaseStorage` adapter)
|
|
95
|
+
* is treated as having no marker — the cloud/JS path cor never accelerates.
|
|
96
|
+
*
|
|
97
|
+
* @param storage - The brain's storage adapter (raw-object surface accessed via
|
|
98
|
+
* a structural cast — `readRawObject` is on `BaseStorage`, not the
|
|
99
|
+
* `StorageAdapter` type).
|
|
100
|
+
* @returns The parsed {@link OnDiskBrainFormat}, or `null`.
|
|
101
|
+
* @example
|
|
102
|
+
* const onDisk = await readOnDiskBrainFormat(storage)
|
|
103
|
+
* const stale = isEpochStale(onDisk)
|
|
104
|
+
*/
|
|
105
|
+
export declare function readOnDiskBrainFormat(storage: StorageAdapter): Promise<OnDiskBrainFormat | null>;
|
|
106
|
+
/**
|
|
107
|
+
* @description Classify the on-disk marker as stale (derived indexes predate
|
|
108
|
+
* this build) or current. Stale when the marker is absent ({@link
|
|
109
|
+
* readOnDiskBrainFormat} returned `null`) OR its `indexEpoch` differs from
|
|
110
|
+
* {@link COR_EXPECTED_INDEX_EPOCH}.
|
|
111
|
+
*
|
|
112
|
+
* @param onDisk - The result of {@link readOnDiskBrainFormat}.
|
|
113
|
+
* @returns `true` when a derived-index rebuild is required.
|
|
114
|
+
*/
|
|
115
|
+
export declare function isEpochStale(onDisk: OnDiskBrainFormat | null): boolean;
|
|
116
|
+
/**
|
|
117
|
+
* @description Resolve the large-brain boundary, honouring the
|
|
118
|
+
* `COR_LARGE_BRAIN_THRESHOLD` env override (a non-negative finite integer);
|
|
119
|
+
* falls back to {@link COR_LARGE_BRAIN_THRESHOLD_DEFAULT} when unset or
|
|
120
|
+
* unparseable.
|
|
121
|
+
*
|
|
122
|
+
* @returns The canonical noun count at or above which a migration is LARGE.
|
|
123
|
+
*/
|
|
124
|
+
export declare function resolveLargeBrainThreshold(): number;
|
|
125
|
+
/**
|
|
126
|
+
* @description Size the brain by its canonical noun count, via
|
|
127
|
+
* `getNouns({ pagination: { limit: 1 } }).totalCount` — the same one-row probe
|
|
128
|
+
* Brainy's `rebuildIndexesIfNeeded` uses to decide small-versus-large. A
|
|
129
|
+
* storage error (or a missing `totalCount`) reports `0`, which classifies the
|
|
130
|
+
* brain as small and lets Brainy's path drive — the conservative default that
|
|
131
|
+
* never throws the loud large-brain migration error on an unreadable probe.
|
|
132
|
+
*
|
|
133
|
+
* @param storage - The brain's storage adapter.
|
|
134
|
+
* @returns The canonical noun count (`0` when unknown).
|
|
135
|
+
*/
|
|
136
|
+
export declare function detectBrainSize(storage: StorageAdapter): Promise<number>;
|
|
137
|
+
/**
|
|
138
|
+
* @description Resolve the auto-migrate policy from the environment.
|
|
139
|
+
* `config.autoMigrate` is unreachable from a cor provider (the provider receives
|
|
140
|
+
* only storage, not the brain config), so the policy is an env var:
|
|
141
|
+
* `COR_AUTO_MIGRATE`. Defaults to `true` (migrate automatically). Only the
|
|
142
|
+
* explicit opt-outs `'false'` / `'0'` (case-insensitive, trimmed) disable it —
|
|
143
|
+
* any other value (or unset) is `true`.
|
|
144
|
+
*
|
|
145
|
+
* @returns `true` when cor should migrate automatically; `false` when the
|
|
146
|
+
* operator has opted out and wants the loud {@link IndexMigrationRequiredError}
|
|
147
|
+
* on a large stale brain.
|
|
148
|
+
*/
|
|
149
|
+
export declare function resolveAutoMigrate(): boolean;
|
|
150
|
+
/**
|
|
151
|
+
* @description Thrown from `MetadataIndexManager.init()` when a LARGE brain
|
|
152
|
+
* carries stale (drifted or absent) derived-index epoch state on disk AND
|
|
153
|
+
* auto-migrate is disabled ({@link resolveAutoMigrate} is `false`). Modeled on
|
|
154
|
+
* `MetadataIndexLegacyJsonStateDetected`: it is loud and pre-serve — a service
|
|
155
|
+
* that refuses to start is recoverable, whereas serving on indexes that predate
|
|
156
|
+
* this build would surface stale/empty results with no signal. Catchable for
|
|
157
|
+
* callers that prefer a one-shot warning, but the default (uncaught) is the
|
|
158
|
+
* right behaviour for production.
|
|
159
|
+
*/
|
|
160
|
+
export declare class IndexMigrationRequiredError extends Error {
|
|
161
|
+
readonly name = "IndexMigrationRequiredError";
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=indexEpochGuard.d.ts.map
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module migration/indexEpochGuard
|
|
3
|
+
* @description Read-side of the 7.x → 8.0 whole-brain auto-migration (#18) —
|
|
4
|
+
* the open-time epoch-drift detection cor performs on its own derived indexes
|
|
5
|
+
* (Adaptive DiskANN vectors, the LSM metadata postings, the graph adjacency
|
|
6
|
+
* trees).
|
|
7
|
+
*
|
|
8
|
+
* ## The handshake cor binds against
|
|
9
|
+
*
|
|
10
|
+
* Brainy 8.0 persists a tiny version-handshake marker at
|
|
11
|
+
* `_system/brain-format.json` (`{ dataFormat, indexEpoch }`). `indexEpoch` is a
|
|
12
|
+
* SHARED, lockstep-bumped integer that Brainy and cor advance TOGETHER on any
|
|
13
|
+
* coordinated release whose on-disk derived-index format changes. On open,
|
|
14
|
+
* Brainy compares the on-disk `indexEpoch` against its compiled
|
|
15
|
+
* `EXPECTED_INDEX_EPOCH`; a mismatch — or an absent marker (a pre-handshake 7.x
|
|
16
|
+
* brain, or a brand-new store) — means every derived index on disk predates
|
|
17
|
+
* this build and must be rebuilt from the canonical records.
|
|
18
|
+
*
|
|
19
|
+
* cor cannot reach Brainy's value: `brain.formatInfo()` is a brain method, and
|
|
20
|
+
* cor's providers receive only the storage adapter. cor also cannot import
|
|
21
|
+
* Brainy's `brainFormat` module (it is not in Brainy's public exports). So cor
|
|
22
|
+
* reads the same marker off storage directly and compiles its OWN
|
|
23
|
+
* {@link COR_EXPECTED_INDEX_EPOCH}.
|
|
24
|
+
*
|
|
25
|
+
* ## Lockstep contract — DO NOT bump unilaterally
|
|
26
|
+
*
|
|
27
|
+
* {@link COR_EXPECTED_INDEX_EPOCH} MUST equal Brainy's `EXPECTED_INDEX_EPOCH`
|
|
28
|
+
* (the constant declared at `@soulcraft/brainy/dist/storage/brainFormat.d.ts:69`
|
|
29
|
+
* — value `1`, the 8.0 GA baseline). The two are a duplicated constant on
|
|
30
|
+
* purpose: cor has no compile-time dependency on Brainy's marker module, so the
|
|
31
|
+
* value lives here too. NEVER advance one side alone. It advances only on a
|
|
32
|
+
* coordinated brainy+cor release whose on-disk derived-index format actually
|
|
33
|
+
* changed, and both projects ship the SAME new value in the same release — so a
|
|
34
|
+
* brain written by either side is recognised as current by the other, and a
|
|
35
|
+
* brain written by an older build of either side is recognised as stale and
|
|
36
|
+
* rebuilt.
|
|
37
|
+
*
|
|
38
|
+
* ## What this module does NOT do
|
|
39
|
+
*
|
|
40
|
+
* It performs read-side detection only — reading the marker, classifying drift,
|
|
41
|
+
* sizing the brain, and resolving the auto-migrate policy. The actual rebuild
|
|
42
|
+
* of a SMALL brain is driven by Brainy's `rebuildIndexesIfNeeded` (which calls
|
|
43
|
+
* each cor provider's `rebuild()` from canonical and re-stamps the marker). The
|
|
44
|
+
* non-blocking background build-new → verify → swap for a LARGE brain is the
|
|
45
|
+
* rc.8 increment (it needs Brainy's deference hook + a stamp method that are not
|
|
46
|
+
* yet shipped) and lives nowhere in this module.
|
|
47
|
+
*/
|
|
48
|
+
// brainy rc.8 exports the shared epoch constant. Import it as the SINGLE SOURCE
|
|
49
|
+
// OF TRUTH rather than duplicating the literal here — the two can no longer drift.
|
|
50
|
+
import { EXPECTED_INDEX_EPOCH } from '@soulcraft/brainy/brain-format';
|
|
51
|
+
/**
|
|
52
|
+
* @description The derived-index format epoch this cor build expects on disk.
|
|
53
|
+
* SHARED and lockstep-bumped with Brainy's `EXPECTED_INDEX_EPOCH`
|
|
54
|
+
* (`@soulcraft/brainy` → `storage/brainFormat`, value `1` — the 8.0 GA
|
|
55
|
+
* baseline). Advance it (in BOTH projects, to the same value, in one
|
|
56
|
+
* coordinated release) ONLY on a change to the on-disk format of any derived
|
|
57
|
+
* index — never on one side alone. An on-disk `indexEpoch` that differs from
|
|
58
|
+
* this — or an absent marker — means the derived indexes predate this build and
|
|
59
|
+
* are stale.
|
|
60
|
+
*/
|
|
61
|
+
export const COR_EXPECTED_INDEX_EPOCH = EXPECTED_INDEX_EPOCH;
|
|
62
|
+
/**
|
|
63
|
+
* @description Storage-root-relative path of the version-handshake marker.
|
|
64
|
+
* Mirrors Brainy's `BRAIN_FORMAT_PATH` (the constant is not exported from
|
|
65
|
+
* Brainy, so it is re-declared here — a stable on-disk path).
|
|
66
|
+
*/
|
|
67
|
+
export const BRAIN_FORMAT_PATH = '_system/brain-format.json';
|
|
68
|
+
/**
|
|
69
|
+
* @description Default brain-size boundary (canonical noun count) above which a
|
|
70
|
+
* 7.x → 8.0 migration is treated as a LARGE brain. Mirrors Brainy's
|
|
71
|
+
* `AUTO_REBUILD_THRESHOLD` (10000): below it Brainy auto-rebuilds all derived
|
|
72
|
+
* indexes inline on open; at or above it Brainy defers to lazy first-query
|
|
73
|
+
* rebuilds. cor uses the SAME boundary so the two agree on which brains take the
|
|
74
|
+
* small (Brainy-driven) versus large (cor-owned, rc.8 background-swap) path.
|
|
75
|
+
* Overridable via the `COR_LARGE_BRAIN_THRESHOLD` env var.
|
|
76
|
+
*/
|
|
77
|
+
export const COR_LARGE_BRAIN_THRESHOLD_DEFAULT = 10000;
|
|
78
|
+
/**
|
|
79
|
+
* @description Read the on-disk version-handshake marker, reimplementing
|
|
80
|
+
* Brainy's `readBrainFormat` validation so cor classifies drift identically
|
|
81
|
+
* without importing Brainy's (non-exported) marker module. Returns `null` when
|
|
82
|
+
* the marker is absent (a pre-handshake 7.x brain, or a brand-new store) OR
|
|
83
|
+
* malformed (not an object, missing either field, or a non-finite epoch) — a
|
|
84
|
+
* corrupt marker forces a safe rebuild rather than trusting a bad epoch.
|
|
85
|
+
*
|
|
86
|
+
* Storage that does not implement `readRawObject` (a non-`BaseStorage` adapter)
|
|
87
|
+
* is treated as having no marker — the cloud/JS path cor never accelerates.
|
|
88
|
+
*
|
|
89
|
+
* @param storage - The brain's storage adapter (raw-object surface accessed via
|
|
90
|
+
* a structural cast — `readRawObject` is on `BaseStorage`, not the
|
|
91
|
+
* `StorageAdapter` type).
|
|
92
|
+
* @returns The parsed {@link OnDiskBrainFormat}, or `null`.
|
|
93
|
+
* @example
|
|
94
|
+
* const onDisk = await readOnDiskBrainFormat(storage)
|
|
95
|
+
* const stale = isEpochStale(onDisk)
|
|
96
|
+
*/
|
|
97
|
+
export async function readOnDiskBrainFormat(storage) {
|
|
98
|
+
const reader = storage;
|
|
99
|
+
if (typeof reader.readRawObject !== 'function')
|
|
100
|
+
return null;
|
|
101
|
+
let raw;
|
|
102
|
+
try {
|
|
103
|
+
raw = await reader.readRawObject(BRAIN_FORMAT_PATH);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
// A read error during the probe is treated as "no marker": we cannot read
|
|
107
|
+
// it, so the safe default (drift → rebuild) applies rather than erroring on
|
|
108
|
+
// a transient storage hiccup at every open.
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
if (raw === null || raw === undefined || typeof raw !== 'object')
|
|
112
|
+
return null;
|
|
113
|
+
const obj = raw;
|
|
114
|
+
if (typeof obj.dataFormat !== 'string' ||
|
|
115
|
+
typeof obj.indexEpoch !== 'number' ||
|
|
116
|
+
!Number.isFinite(obj.indexEpoch)) {
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
return { dataFormat: obj.dataFormat, indexEpoch: obj.indexEpoch };
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* @description Classify the on-disk marker as stale (derived indexes predate
|
|
123
|
+
* this build) or current. Stale when the marker is absent ({@link
|
|
124
|
+
* readOnDiskBrainFormat} returned `null`) OR its `indexEpoch` differs from
|
|
125
|
+
* {@link COR_EXPECTED_INDEX_EPOCH}.
|
|
126
|
+
*
|
|
127
|
+
* @param onDisk - The result of {@link readOnDiskBrainFormat}.
|
|
128
|
+
* @returns `true` when a derived-index rebuild is required.
|
|
129
|
+
*/
|
|
130
|
+
export function isEpochStale(onDisk) {
|
|
131
|
+
return onDisk === null || onDisk.indexEpoch !== COR_EXPECTED_INDEX_EPOCH;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* @description Resolve the large-brain boundary, honouring the
|
|
135
|
+
* `COR_LARGE_BRAIN_THRESHOLD` env override (a non-negative finite integer);
|
|
136
|
+
* falls back to {@link COR_LARGE_BRAIN_THRESHOLD_DEFAULT} when unset or
|
|
137
|
+
* unparseable.
|
|
138
|
+
*
|
|
139
|
+
* @returns The canonical noun count at or above which a migration is LARGE.
|
|
140
|
+
*/
|
|
141
|
+
export function resolveLargeBrainThreshold() {
|
|
142
|
+
const raw = process.env.COR_LARGE_BRAIN_THRESHOLD;
|
|
143
|
+
if (raw === undefined)
|
|
144
|
+
return COR_LARGE_BRAIN_THRESHOLD_DEFAULT;
|
|
145
|
+
const n = Number(raw.trim());
|
|
146
|
+
return Number.isFinite(n) && n >= 0 ? n : COR_LARGE_BRAIN_THRESHOLD_DEFAULT;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* @description Size the brain by its canonical noun count, via
|
|
150
|
+
* `getNouns({ pagination: { limit: 1 } }).totalCount` — the same one-row probe
|
|
151
|
+
* Brainy's `rebuildIndexesIfNeeded` uses to decide small-versus-large. A
|
|
152
|
+
* storage error (or a missing `totalCount`) reports `0`, which classifies the
|
|
153
|
+
* brain as small and lets Brainy's path drive — the conservative default that
|
|
154
|
+
* never throws the loud large-brain migration error on an unreadable probe.
|
|
155
|
+
*
|
|
156
|
+
* @param storage - The brain's storage adapter.
|
|
157
|
+
* @returns The canonical noun count (`0` when unknown).
|
|
158
|
+
*/
|
|
159
|
+
export async function detectBrainSize(storage) {
|
|
160
|
+
try {
|
|
161
|
+
const probe = await storage.getNouns({
|
|
162
|
+
pagination: { limit: 1 },
|
|
163
|
+
});
|
|
164
|
+
return probe?.totalCount ?? 0;
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
return 0;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* @description Resolve the auto-migrate policy from the environment.
|
|
172
|
+
* `config.autoMigrate` is unreachable from a cor provider (the provider receives
|
|
173
|
+
* only storage, not the brain config), so the policy is an env var:
|
|
174
|
+
* `COR_AUTO_MIGRATE`. Defaults to `true` (migrate automatically). Only the
|
|
175
|
+
* explicit opt-outs `'false'` / `'0'` (case-insensitive, trimmed) disable it —
|
|
176
|
+
* any other value (or unset) is `true`.
|
|
177
|
+
*
|
|
178
|
+
* @returns `true` when cor should migrate automatically; `false` when the
|
|
179
|
+
* operator has opted out and wants the loud {@link IndexMigrationRequiredError}
|
|
180
|
+
* on a large stale brain.
|
|
181
|
+
*/
|
|
182
|
+
export function resolveAutoMigrate() {
|
|
183
|
+
const raw = process.env.COR_AUTO_MIGRATE;
|
|
184
|
+
if (raw === undefined)
|
|
185
|
+
return true;
|
|
186
|
+
const normalized = raw.trim().toLowerCase();
|
|
187
|
+
return !(normalized === 'false' || normalized === '0');
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* @description Thrown from `MetadataIndexManager.init()` when a LARGE brain
|
|
191
|
+
* carries stale (drifted or absent) derived-index epoch state on disk AND
|
|
192
|
+
* auto-migrate is disabled ({@link resolveAutoMigrate} is `false`). Modeled on
|
|
193
|
+
* `MetadataIndexLegacyJsonStateDetected`: it is loud and pre-serve — a service
|
|
194
|
+
* that refuses to start is recoverable, whereas serving on indexes that predate
|
|
195
|
+
* this build would surface stale/empty results with no signal. Catchable for
|
|
196
|
+
* callers that prefer a one-shot warning, but the default (uncaught) is the
|
|
197
|
+
* right behaviour for production.
|
|
198
|
+
*/
|
|
199
|
+
export class IndexMigrationRequiredError extends Error {
|
|
200
|
+
name = 'IndexMigrationRequiredError';
|
|
201
|
+
}
|
|
202
|
+
//# sourceMappingURL=indexEpochGuard.js.map
|