@soulcraft/cor 3.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/LICENSE +16 -0
  2. package/README.md +328 -0
  3. package/dist/aggregation/NativeAggregationEngine.d.ts +118 -0
  4. package/dist/aggregation/NativeAggregationEngine.js +186 -0
  5. package/dist/cli.d.ts +12 -0
  6. package/dist/cli.js +236 -0
  7. package/dist/graph/GraphAccelerationAdapter.d.ts +153 -0
  8. package/dist/graph/GraphAccelerationAdapter.js +326 -0
  9. package/dist/graph/NativeGraphAdjacencyIndex.d.ts +286 -0
  10. package/dist/graph/NativeGraphAdjacencyIndex.js +944 -0
  11. package/dist/hnsw/AdaptiveDiskAnnModeSelector.d.ts +187 -0
  12. package/dist/hnsw/AdaptiveDiskAnnModeSelector.js +313 -0
  13. package/dist/hnsw/NativeDiskAnnWrapper.d.ts +489 -0
  14. package/dist/hnsw/NativeDiskAnnWrapper.js +1456 -0
  15. package/dist/index.d.ts +28 -0
  16. package/dist/index.js +29 -0
  17. package/dist/legacyLayoutGuard.d.ts +93 -0
  18. package/dist/legacyLayoutGuard.js +237 -0
  19. package/dist/license/constants.d.ts +30 -0
  20. package/dist/license/constants.js +30 -0
  21. package/dist/license/onlineValidator.d.ts +43 -0
  22. package/dist/license/onlineValidator.js +225 -0
  23. package/dist/license/types.d.ts +39 -0
  24. package/dist/license/types.js +5 -0
  25. package/dist/license.d.ts +64 -0
  26. package/dist/license.js +165 -0
  27. package/dist/migration/MigrationCoordinator.d.ts +171 -0
  28. package/dist/migration/MigrationCoordinator.js +249 -0
  29. package/dist/migration/indexEpochGuard.d.ts +163 -0
  30. package/dist/migration/indexEpochGuard.js +202 -0
  31. package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
  32. package/dist/native/NativeEmbeddingEngine.js +318 -0
  33. package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
  34. package/dist/native/NativeRoaringBitmap32.js +221 -0
  35. package/dist/native/ffi.d.ts +20 -0
  36. package/dist/native/ffi.js +48 -0
  37. package/dist/native/idMapperTestSupport.d.ts +67 -0
  38. package/dist/native/idMapperTestSupport.js +112 -0
  39. package/dist/native/index.d.ts +49 -0
  40. package/dist/native/index.js +87 -0
  41. package/dist/native/napi.d.ts +21 -0
  42. package/dist/native/napi.js +88 -0
  43. package/dist/native/types.d.ts +1298 -0
  44. package/dist/native/types.js +16 -0
  45. package/dist/plugin.d.ts +50 -0
  46. package/dist/plugin.js +388 -0
  47. package/dist/providerContracts.d.ts +277 -0
  48. package/dist/providerContracts.js +38 -0
  49. package/dist/resource/OsMemoryProbe.d.ts +175 -0
  50. package/dist/resource/OsMemoryProbe.js +206 -0
  51. package/dist/resource/ResourceManager.d.ts +491 -0
  52. package/dist/resource/ResourceManager.js +960 -0
  53. package/dist/utils/ColumnManifest.d.ts +97 -0
  54. package/dist/utils/ColumnManifest.js +129 -0
  55. package/dist/utils/NativeColumnStore.d.ts +284 -0
  56. package/dist/utils/NativeColumnStore.js +685 -0
  57. package/dist/utils/NativeMetadataIndex.d.ts +882 -0
  58. package/dist/utils/NativeMetadataIndex.js +2631 -0
  59. package/dist/utils/NativeUnifiedCache.d.ts +87 -0
  60. package/dist/utils/NativeUnifiedCache.js +273 -0
  61. package/dist/utils/binaryIdMapperFactory.d.ts +59 -0
  62. package/dist/utils/binaryIdMapperFactory.js +94 -0
  63. package/dist/utils/collation.d.ts +30 -0
  64. package/dist/utils/collation.js +40 -0
  65. package/dist/utils/columnStoreTypes.d.ts +161 -0
  66. package/dist/utils/columnStoreTypes.js +29 -0
  67. package/dist/utils/nativeBinaryEntityIdMapper.d.ts +211 -0
  68. package/dist/utils/nativeBinaryEntityIdMapper.js +381 -0
  69. package/dist/utils/nativeEntityIdMapper.d.ts +111 -0
  70. package/dist/utils/nativeEntityIdMapper.js +170 -0
  71. package/docs/ADR-002-diskann-100-percent-rust.md +294 -0
  72. package/docs/ADR-003-semantic-time-travel.md +100 -0
  73. package/docs/aggregation.md +251 -0
  74. package/docs/comparison.md +162 -0
  75. package/docs/deployment-limits.md +87 -0
  76. package/docs/diskann.md +184 -0
  77. package/docs/migration-3.0.md +396 -0
  78. package/docs/performance-budget.md +274 -0
  79. package/docs/performance.md +117 -0
  80. package/docs/scaling.md +439 -0
  81. package/docs/snapshot-safety.md +246 -0
  82. package/docs/u64-id-space.md +214 -0
  83. package/docs/verification-report.md +670 -0
  84. package/native/brainy-native.node +0 -0
  85. package/native/index.d.ts +3916 -0
  86. package/package.json +92 -0
  87. package/scripts/migrate-cortex-2x-to-3x.mjs +970 -0
@@ -0,0 +1,249 @@
1
+ /**
2
+ * @module migration/MigrationCoordinator
3
+ * @description The cor-side driver of the #18 whole-brain 7.x → 8.0 upgrade — a
4
+ * **coordinated migration lock**. When cor opens a LARGE brain whose derived-index
5
+ * epoch is stale (drifted or absent; see {@link module:migration/indexEpochGuard}),
6
+ * the correct-and-clean upgrade is a BLOCKING one: rebuild every derived index
7
+ * (metadata LSM, DiskANN vector, graph adjacency) from the canonical records to a
8
+ * known-good state BEFORE serving — never serve a half-built or stale index. That
9
+ * decision (David, 2026-07-01) replaced the earlier "no-freeze background swap":
10
+ * no-freeze's availability is illusory (it serves degraded/half-built results),
11
+ * and *unknown/halfway states are more dangerous than a clean block*.
12
+ *
13
+ * ## The lock, coordinated with Brainy
14
+ *
15
+ * cor sets a lock the instant it detects a large stale brain; Brainy holds every
16
+ * public read AND write behind that lock (its `isMigrating()` gate, rc.9) and maps
17
+ * the lock to an HTTP-503-style readiness signal, so nothing observes a partial
18
+ * result and no write reaches a half-built index. The rebuild runs asynchronously
19
+ * — the LOCK, not a synchronous init hang, is what serializes access — and when
20
+ * all three indexes are rebuilt-from-canonical + the version marker is stamped,
21
+ * cor clears the lock and Brainy releases the queued operations against a fully
22
+ * correct brain. One invariant: *the brain is not readable/writable until all
23
+ * three indexes are rebuilt + stamped.*
24
+ *
25
+ * Brainy feature-detects three optional provider methods; every cor derived-index
26
+ * provider delegates them to the single coordinator so the answer is consistent
27
+ * across all three: {@link MigrationCoordinator#isMigrating} (the lock signal),
28
+ * {@link MigrationCoordinator#whenMigrationComplete} (the no-poll wait Brainy
29
+ * `Promise.race`s against its bounded timeout), and
30
+ * {@link MigrationCoordinator#migrationStatus} (the progress cor owns, which Brainy
31
+ * relays verbatim into `getIndexStatus().migration`).
32
+ *
33
+ * ## Failure is a locked-with-error state, never a half-served one
34
+ *
35
+ * If a rebuild throws, the coordinator KEEPS the brain locked (so Brainy never
36
+ * releases reads onto a half-built index) and rejects {@link
37
+ * MigrationCoordinator#whenMigrationComplete} with a loud, actionable error — the
38
+ * same "blocking beats halfway" principle applied to the failure path. A restart
39
+ * re-detects the drift and re-runs the idempotent in-place rebuild, so a transient
40
+ * hiccup self-heals; genuinely-corrupt canonical data stays safely blocked with a
41
+ * pointer to the offline `scripts/migrate-cortex-2x-to-3x.mjs` escape hatch.
42
+ *
43
+ * The rebuild is IN-PLACE (no shadow directories, no serve-from-canonical, no
44
+ * concurrent-write reconciliation) precisely because everything is locked: no live
45
+ * index is read while it is being rebuilt.
46
+ */
47
+ import { prodLog } from '@soulcraft/brainy/internals';
48
+ import { writeBrainFormat } from '@soulcraft/brainy/brain-format';
49
+ /**
50
+ * @description Drives the #18 coordinated blocking migration and answers the three
51
+ * Brainy-feature-detected lock methods. One instance per brain, created in the
52
+ * plugin's `activate` closure and shared by all three derived-index providers.
53
+ *
54
+ * @example
55
+ * const coordinator = new MigrationCoordinator(storage, {
56
+ * metadata: () => metadataIndexInstance,
57
+ * vector: () => vectorInstance,
58
+ * graph: () => graphEngine,
59
+ * })
60
+ * // ...in metadata init(), on a large stale brain:
61
+ * coordinator.start() // flips isMigrating() true synchronously, rebuilds async
62
+ */
63
+ export class MigrationCoordinator {
64
+ storage;
65
+ providers;
66
+ /**
67
+ * `idle` before/without a migration; `migrating` while rebuilding; `done` once
68
+ * stamped; `failed` when a rebuild threw. `failed` reads as LOCKED (see
69
+ * {@link isMigrating}) so Brainy never releases reads onto a half-built index.
70
+ */
71
+ state = 'idle';
72
+ status = null;
73
+ completePromise = null;
74
+ resolveComplete = null;
75
+ rejectComplete = null;
76
+ failure = null;
77
+ /**
78
+ * @param storage - The brain's storage adapter — passed to `writeBrainFormat`
79
+ * as the final stamp step (Brainy's own `dataFormat`/`indexEpoch` constants;
80
+ * cor never authors the format string).
81
+ * @param providers - Lazy resolvers for the three derived-index providers.
82
+ */
83
+ constructor(storage, providers) {
84
+ this.storage = storage;
85
+ this.providers = providers;
86
+ }
87
+ /**
88
+ * @description The lock signal Brainy gates every public read AND write on.
89
+ * `true` while migrating OR after a failure (a failed migration stays LOCKED so
90
+ * Brainy never serves the half-built index — a restart re-runs the idempotent
91
+ * rebuild).
92
+ * @returns `true` when the brain must not be read from or written to.
93
+ */
94
+ isMigrating() {
95
+ return this.state === 'migrating' || this.state === 'failed';
96
+ }
97
+ /**
98
+ * @description Resolves when the migration completes (lock clears), or REJECTS
99
+ * with the loud failure error if it failed. Used internally by {@link start} (the
100
+ * already-migrating idempotency path) and by tests as the completion signal.
101
+ *
102
+ * Brainy does NOT call this: rc.9 dropped the provider-level `whenMigrationComplete`
103
+ * hook and polls `isMigrating()` at 250 ms instead (the event-driven variant
104
+ * busy-spun with mixed providers). `getIndexStatus().migrating` is the readiness
105
+ * signal, and a blocked op waits out the lock up to `migrationWaitTimeoutMs` before
106
+ * a retryable `MigrationInProgressError`.
107
+ * @returns A promise that settles when the lock is released or the migration fails.
108
+ */
109
+ whenMigrationComplete() {
110
+ if (this.state === 'done')
111
+ return Promise.resolve();
112
+ if (this.state === 'failed') {
113
+ return Promise.reject(this.failure ?? new Error('migration failed'));
114
+ }
115
+ if (!this.completePromise) {
116
+ // idle and not yet started — a wait registered before start() still settles.
117
+ this.completePromise = new Promise((resolve, reject) => {
118
+ this.resolveComplete = resolve;
119
+ this.rejectComplete = reject;
120
+ });
121
+ }
122
+ return this.completePromise;
123
+ }
124
+ /**
125
+ * @description The structured progress Brainy relays into
126
+ * `getIndexStatus().migration`. `null` unless a migration is in flight.
127
+ * @returns The current {@link MigrationStatus}, or `null`.
128
+ */
129
+ migrationStatus() {
130
+ // Report during 'migrating' (rebuild progress) AND 'failed' — so a brain
131
+ // hard-locked by a failed migration surfaces WHY (phase: 'failed' + which index)
132
+ // via getIndexStatus().migration, not just `migrating: true` with no reason.
133
+ // Null when idle / done.
134
+ return this.state === 'migrating' || this.state === 'failed'
135
+ ? this.status
136
+ : null;
137
+ }
138
+ /**
139
+ * @description Kick the coordinated blocking rebuild. Flips {@link isMigrating}
140
+ * to `true` **synchronously** (so Brainy sees the lock the instant the triggering
141
+ * `init()` returns), then runs the rebuild asynchronously — the LOCK, not this
142
+ * promise, is what serializes reads/writes, so callers do NOT await this.
143
+ * Idempotent: calling it again while already migrating returns the in-flight
144
+ * completion promise without starting a second rebuild.
145
+ * @returns The completion promise (settles when the lock clears or the migration fails).
146
+ */
147
+ start() {
148
+ // Already migrating → return the in-flight completion (no second rebuild).
149
+ if (this.state === 'migrating')
150
+ return this.whenMigrationComplete();
151
+ // Already migrated + stamped → no-op. (A re-open won't reach here — the epoch
152
+ // is current — but guard defensively against a double start() re-running the
153
+ // whole rebuild and re-stamping.)
154
+ if (this.state === 'done')
155
+ return Promise.resolve();
156
+ // Fresh run — 'idle', or the RETRY path after a prior 'failed' (the epoch was
157
+ // never stamped, so a re-open re-detects drift). run() NULLS the completion
158
+ // triplet when it settles, so `this.completePromise` here is either null (→ a
159
+ // fresh promise below) or a still-pending pre-start wait we reuse.
160
+ this.state = 'migrating';
161
+ this.failure = null;
162
+ this.status = { phase: 'starting' };
163
+ // Reuse a wait registered before start(); otherwise create the promise now.
164
+ if (!this.completePromise) {
165
+ this.completePromise = new Promise((resolve, reject) => {
166
+ this.resolveComplete = resolve;
167
+ this.rejectComplete = reject;
168
+ });
169
+ }
170
+ // The lock is already set above (synchronous); run the rebuild in the background.
171
+ void this.run();
172
+ return this.completePromise;
173
+ }
174
+ /**
175
+ * The async body: rebuild metadata → vector → graph from canonical, verify by
176
+ * stamping (the marker certifies all three; Brainy withholds its own stamp while
177
+ * we are migrating), then clear the lock. Any throw leaves the brain LOCKED and
178
+ * rejects the completion promise.
179
+ */
180
+ async run() {
181
+ const startedAt = Date.now();
182
+ // Track the index being rebuilt so a failure can report WHICH one failed.
183
+ let currentIndex;
184
+ try {
185
+ const steps = [
186
+ ['metadata', this.providers.metadata()],
187
+ ['vector', this.providers.vector()],
188
+ ['graph', this.providers.graph()],
189
+ ];
190
+ for (const [index, provider] of steps) {
191
+ // A resolver may return undefined (e.g. a provider a cloud adapter never
192
+ // engages) — skip it rather than fail the whole upgrade.
193
+ if (!provider)
194
+ continue;
195
+ currentIndex = index;
196
+ this.status = { phase: 'rebuilding', index };
197
+ prodLog.info(`[Cor] migration: rebuilding ${index} index from canonical records…`);
198
+ await provider.rebuild();
199
+ prodLog.info(`[Cor] migration: ${index} index rebuilt`);
200
+ }
201
+ // Stamp LAST — Brainy's own dataFormat + indexEpoch constants, written via
202
+ // the exported writer (no brain handle, no hardcoded '8.0'). The marker
203
+ // advancing is what certifies all three indexes are current; a crash before
204
+ // this leaves it stale → the next open re-detects drift → idempotent re-run.
205
+ // `writeRawObject` is on the concrete BaseStorage (used by the generational
206
+ // record layer for `_system/` artifacts), not the public `StorageAdapter`
207
+ // type — the same structural cast readOnDiskBrainFormat uses for the read side.
208
+ await writeBrainFormat(this.storage);
209
+ const elapsedMs = Date.now() - startedAt;
210
+ this.state = 'done';
211
+ this.status = null;
212
+ prodLog.info(`[Cor] migration complete (${elapsedMs} ms) — brain upgraded to 8.0 across all 3 derived indexes`);
213
+ // Clear the completion triplet THEN settle it: a subsequent run (retry path)
214
+ // must get a fresh promise + resolvers, never this already-settled one. The
215
+ // callers that hold the returned promise still settle (they reference the
216
+ // object, not the field); post-settle whenMigrationComplete() answers from
217
+ // `state`, so nulling the field is safe.
218
+ const resolve = this.resolveComplete;
219
+ this.completePromise = null;
220
+ this.resolveComplete = null;
221
+ this.rejectComplete = null;
222
+ resolve?.();
223
+ }
224
+ catch (error) {
225
+ // Keep the brain LOCKED on failure: releasing reads onto a half-built index
226
+ // would surface stale/partial results with no signal — worse than a clear
227
+ // block. Surface loudly; a restart re-detects the drift and re-runs the
228
+ // idempotent in-place rebuild (a transient hiccup self-heals; genuinely
229
+ // corrupt canonical data stays safely blocked).
230
+ this.failure = error instanceof Error ? error : new Error(String(error));
231
+ this.state = 'failed';
232
+ this.status = { phase: 'failed', index: currentIndex };
233
+ prodLog.error(`[Cor] migration FAILED — the brain stays locked (unmigrated) rather than ` +
234
+ `serve a half-built index. Restart to retry the automatic upgrade, or run ` +
235
+ `scripts/migrate-cortex-2x-to-3x.mjs for an offline migration. Cause: ` +
236
+ `${this.failure.message}`);
237
+ // Clear the completion triplet THEN reject it (see the success path) so the
238
+ // RETRY after this failure gets a fresh promise. Until that retry's start()
239
+ // resets `state`, whenMigrationComplete() rejects from the 'failed' state +
240
+ // the retained `this.failure`, not from this (now-detached) promise.
241
+ const reject = this.rejectComplete;
242
+ this.completePromise = null;
243
+ this.resolveComplete = null;
244
+ this.rejectComplete = null;
245
+ reject?.(this.failure);
246
+ }
247
+ }
248
+ }
249
+ //# sourceMappingURL=MigrationCoordinator.js.map
@@ -0,0 +1,163 @@
1
+ /**
2
+ * @module migration/indexEpochGuard
3
+ * @description Read-side of the 7.x → 8.0 whole-brain auto-migration (#18) —
4
+ * the open-time epoch-drift detection cor performs on its own derived indexes
5
+ * (Adaptive DiskANN vectors, the LSM metadata postings, the graph adjacency
6
+ * trees).
7
+ *
8
+ * ## The handshake cor binds against
9
+ *
10
+ * Brainy 8.0 persists a tiny version-handshake marker at
11
+ * `_system/brain-format.json` (`{ dataFormat, indexEpoch }`). `indexEpoch` is a
12
+ * SHARED, lockstep-bumped integer that Brainy and cor advance TOGETHER on any
13
+ * coordinated release whose on-disk derived-index format changes. On open,
14
+ * Brainy compares the on-disk `indexEpoch` against its compiled
15
+ * `EXPECTED_INDEX_EPOCH`; a mismatch — or an absent marker (a pre-handshake 7.x
16
+ * brain, or a brand-new store) — means every derived index on disk predates
17
+ * this build and must be rebuilt from the canonical records.
18
+ *
19
+ * cor cannot reach Brainy's value: `brain.formatInfo()` is a brain method, and
20
+ * cor's providers receive only the storage adapter. cor also cannot import
21
+ * Brainy's `brainFormat` module (it is not in Brainy's public exports). So cor
22
+ * reads the same marker off storage directly and compiles its OWN
23
+ * {@link COR_EXPECTED_INDEX_EPOCH}.
24
+ *
25
+ * ## Lockstep contract — DO NOT bump unilaterally
26
+ *
27
+ * {@link COR_EXPECTED_INDEX_EPOCH} MUST equal Brainy's `EXPECTED_INDEX_EPOCH`
28
+ * (the constant declared at `@soulcraft/brainy/dist/storage/brainFormat.d.ts:69`
29
+ * — value `1`, the 8.0 GA baseline). The two are a duplicated constant on
30
+ * purpose: cor has no compile-time dependency on Brainy's marker module, so the
31
+ * value lives here too. NEVER advance one side alone. It advances only on a
32
+ * coordinated brainy+cor release whose on-disk derived-index format actually
33
+ * changed, and both projects ship the SAME new value in the same release — so a
34
+ * brain written by either side is recognised as current by the other, and a
35
+ * brain written by an older build of either side is recognised as stale and
36
+ * rebuilt.
37
+ *
38
+ * ## What this module does NOT do
39
+ *
40
+ * It performs read-side detection only — reading the marker, classifying drift,
41
+ * sizing the brain, and resolving the auto-migrate policy. The actual rebuild
42
+ * of a SMALL brain is driven by Brainy's `rebuildIndexesIfNeeded` (which calls
43
+ * each cor provider's `rebuild()` from canonical and re-stamps the marker). The
44
+ * non-blocking background build-new → verify → swap for a LARGE brain is the
45
+ * rc.8 increment (it needs Brainy's deference hook + a stamp method that are not
46
+ * yet shipped) and lives nowhere in this module.
47
+ */
48
+ import type { StorageAdapter } from '@soulcraft/brainy';
49
+ /**
50
+ * @description The derived-index format epoch this cor build expects on disk.
51
+ * SHARED and lockstep-bumped with Brainy's `EXPECTED_INDEX_EPOCH`
52
+ * (`@soulcraft/brainy` → `storage/brainFormat`, value `1` — the 8.0 GA
53
+ * baseline). Advance it (in BOTH projects, to the same value, in one
54
+ * coordinated release) ONLY on a change to the on-disk format of any derived
55
+ * index — never on one side alone. An on-disk `indexEpoch` that differs from
56
+ * this — or an absent marker — means the derived indexes predate this build and
57
+ * are stale.
58
+ */
59
+ export declare const COR_EXPECTED_INDEX_EPOCH = 1;
60
+ /**
61
+ * @description Storage-root-relative path of the version-handshake marker.
62
+ * Mirrors Brainy's `BRAIN_FORMAT_PATH` (the constant is not exported from
63
+ * Brainy, so it is re-declared here — a stable on-disk path).
64
+ */
65
+ export declare const BRAIN_FORMAT_PATH = "_system/brain-format.json";
66
+ /**
67
+ * @description Default brain-size boundary (canonical noun count) above which a
68
+ * 7.x → 8.0 migration is treated as a LARGE brain. Mirrors Brainy's
69
+ * `AUTO_REBUILD_THRESHOLD` (10000): below it Brainy auto-rebuilds all derived
70
+ * indexes inline on open; at or above it Brainy defers to lazy first-query
71
+ * rebuilds. cor uses the SAME boundary so the two agree on which brains take the
72
+ * small (Brainy-driven) versus large (cor-owned, rc.8 background-swap) path.
73
+ * Overridable via the `COR_LARGE_BRAIN_THRESHOLD` env var.
74
+ */
75
+ export declare const COR_LARGE_BRAIN_THRESHOLD_DEFAULT = 10000;
76
+ /**
77
+ * @description The parsed shape of `_system/brain-format.json` — the same two
78
+ * fields Brainy's `BrainFormat` carries.
79
+ */
80
+ export interface OnDiskBrainFormat {
81
+ /** Brainy-owned data-layer version string (e.g. `'8.0'`). */
82
+ dataFormat: string;
83
+ /** Shared, lockstep-bumped derived-index format epoch (e.g. `1`). */
84
+ indexEpoch: number;
85
+ }
86
+ /**
87
+ * @description Read the on-disk version-handshake marker, reimplementing
88
+ * Brainy's `readBrainFormat` validation so cor classifies drift identically
89
+ * without importing Brainy's (non-exported) marker module. Returns `null` when
90
+ * the marker is absent (a pre-handshake 7.x brain, or a brand-new store) OR
91
+ * malformed (not an object, missing either field, or a non-finite epoch) — a
92
+ * corrupt marker forces a safe rebuild rather than trusting a bad epoch.
93
+ *
94
+ * Storage that does not implement `readRawObject` (a non-`BaseStorage` adapter)
95
+ * is treated as having no marker — the cloud/JS path cor never accelerates.
96
+ *
97
+ * @param storage - The brain's storage adapter (raw-object surface accessed via
98
+ * a structural cast — `readRawObject` is on `BaseStorage`, not the
99
+ * `StorageAdapter` type).
100
+ * @returns The parsed {@link OnDiskBrainFormat}, or `null`.
101
+ * @example
102
+ * const onDisk = await readOnDiskBrainFormat(storage)
103
+ * const stale = isEpochStale(onDisk)
104
+ */
105
+ export declare function readOnDiskBrainFormat(storage: StorageAdapter): Promise<OnDiskBrainFormat | null>;
106
+ /**
107
+ * @description Classify the on-disk marker as stale (derived indexes predate
108
+ * this build) or current. Stale when the marker is absent ({@link
109
+ * readOnDiskBrainFormat} returned `null`) OR its `indexEpoch` differs from
110
+ * {@link COR_EXPECTED_INDEX_EPOCH}.
111
+ *
112
+ * @param onDisk - The result of {@link readOnDiskBrainFormat}.
113
+ * @returns `true` when a derived-index rebuild is required.
114
+ */
115
+ export declare function isEpochStale(onDisk: OnDiskBrainFormat | null): boolean;
116
+ /**
117
+ * @description Resolve the large-brain boundary, honouring the
118
+ * `COR_LARGE_BRAIN_THRESHOLD` env override (a non-negative finite integer);
119
+ * falls back to {@link COR_LARGE_BRAIN_THRESHOLD_DEFAULT} when unset or
120
+ * unparseable.
121
+ *
122
+ * @returns The canonical noun count at or above which a migration is LARGE.
123
+ */
124
+ export declare function resolveLargeBrainThreshold(): number;
125
+ /**
126
+ * @description Size the brain by its canonical noun count, via
127
+ * `getNouns({ pagination: { limit: 1 } }).totalCount` — the same one-row probe
128
+ * Brainy's `rebuildIndexesIfNeeded` uses to decide small-versus-large. A
129
+ * storage error (or a missing `totalCount`) reports `0`, which classifies the
130
+ * brain as small and lets Brainy's path drive — the conservative default that
131
+ * never throws the loud large-brain migration error on an unreadable probe.
132
+ *
133
+ * @param storage - The brain's storage adapter.
134
+ * @returns The canonical noun count (`0` when unknown).
135
+ */
136
+ export declare function detectBrainSize(storage: StorageAdapter): Promise<number>;
137
+ /**
138
+ * @description Resolve the auto-migrate policy from the environment.
139
+ * `config.autoMigrate` is unreachable from a cor provider (the provider receives
140
+ * only storage, not the brain config), so the policy is an env var:
141
+ * `COR_AUTO_MIGRATE`. Defaults to `true` (migrate automatically). Only the
142
+ * explicit opt-outs `'false'` / `'0'` (case-insensitive, trimmed) disable it —
143
+ * any other value (or unset) is `true`.
144
+ *
145
+ * @returns `true` when cor should migrate automatically; `false` when the
146
+ * operator has opted out and wants the loud {@link IndexMigrationRequiredError}
147
+ * on a large stale brain.
148
+ */
149
+ export declare function resolveAutoMigrate(): boolean;
150
+ /**
151
+ * @description Thrown from `MetadataIndexManager.init()` when a LARGE brain
152
+ * carries stale (drifted or absent) derived-index epoch state on disk AND
153
+ * auto-migrate is disabled ({@link resolveAutoMigrate} is `false`). Modeled on
154
+ * `MetadataIndexLegacyJsonStateDetected`: it is loud and pre-serve — a service
155
+ * that refuses to start is recoverable, whereas serving on indexes that predate
156
+ * this build would surface stale/empty results with no signal. Catchable for
157
+ * callers that prefer a one-shot warning, but the default (uncaught) is the
158
+ * right behaviour for production.
159
+ */
160
+ export declare class IndexMigrationRequiredError extends Error {
161
+ readonly name = "IndexMigrationRequiredError";
162
+ }
163
+ //# sourceMappingURL=indexEpochGuard.d.ts.map
@@ -0,0 +1,202 @@
1
+ /**
2
+ * @module migration/indexEpochGuard
3
+ * @description Read-side of the 7.x → 8.0 whole-brain auto-migration (#18) —
4
+ * the open-time epoch-drift detection cor performs on its own derived indexes
5
+ * (Adaptive DiskANN vectors, the LSM metadata postings, the graph adjacency
6
+ * trees).
7
+ *
8
+ * ## The handshake cor binds against
9
+ *
10
+ * Brainy 8.0 persists a tiny version-handshake marker at
11
+ * `_system/brain-format.json` (`{ dataFormat, indexEpoch }`). `indexEpoch` is a
12
+ * SHARED, lockstep-bumped integer that Brainy and cor advance TOGETHER on any
13
+ * coordinated release whose on-disk derived-index format changes. On open,
14
+ * Brainy compares the on-disk `indexEpoch` against its compiled
15
+ * `EXPECTED_INDEX_EPOCH`; a mismatch — or an absent marker (a pre-handshake 7.x
16
+ * brain, or a brand-new store) — means every derived index on disk predates
17
+ * this build and must be rebuilt from the canonical records.
18
+ *
19
+ * cor cannot reach Brainy's value: `brain.formatInfo()` is a brain method, and
20
+ * cor's providers receive only the storage adapter. cor also cannot import
21
+ * Brainy's `brainFormat` module (it is not in Brainy's public exports). So cor
22
+ * reads the same marker off storage directly and compiles its OWN
23
+ * {@link COR_EXPECTED_INDEX_EPOCH}.
24
+ *
25
+ * ## Lockstep contract — DO NOT bump unilaterally
26
+ *
27
+ * {@link COR_EXPECTED_INDEX_EPOCH} MUST equal Brainy's `EXPECTED_INDEX_EPOCH`
28
+ * (the constant declared at `@soulcraft/brainy/dist/storage/brainFormat.d.ts:69`
29
+ * — value `1`, the 8.0 GA baseline). The two are a duplicated constant on
30
+ * purpose: cor has no compile-time dependency on Brainy's marker module, so the
31
+ * value lives here too. NEVER advance one side alone. It advances only on a
32
+ * coordinated brainy+cor release whose on-disk derived-index format actually
33
+ * changed, and both projects ship the SAME new value in the same release — so a
34
+ * brain written by either side is recognised as current by the other, and a
35
+ * brain written by an older build of either side is recognised as stale and
36
+ * rebuilt.
37
+ *
38
+ * ## What this module does NOT do
39
+ *
40
+ * It performs read-side detection only — reading the marker, classifying drift,
41
+ * sizing the brain, and resolving the auto-migrate policy. The actual rebuild
42
+ * of a SMALL brain is driven by Brainy's `rebuildIndexesIfNeeded` (which calls
43
+ * each cor provider's `rebuild()` from canonical and re-stamps the marker). The
44
+ * non-blocking background build-new → verify → swap for a LARGE brain is the
45
+ * rc.8 increment (it needs Brainy's deference hook + a stamp method that are not
46
+ * yet shipped) and lives nowhere in this module.
47
+ */
48
+ // brainy rc.8 exports the shared epoch constant. Import it as the SINGLE SOURCE
49
+ // OF TRUTH rather than duplicating the literal here — the two can no longer drift.
50
+ import { EXPECTED_INDEX_EPOCH } from '@soulcraft/brainy/brain-format';
51
+ /**
52
+ * @description The derived-index format epoch this cor build expects on disk.
53
+ * SHARED and lockstep-bumped with Brainy's `EXPECTED_INDEX_EPOCH`
54
+ * (`@soulcraft/brainy` → `storage/brainFormat`, value `1` — the 8.0 GA
55
+ * baseline). Advance it (in BOTH projects, to the same value, in one
56
+ * coordinated release) ONLY on a change to the on-disk format of any derived
57
+ * index — never on one side alone. An on-disk `indexEpoch` that differs from
58
+ * this — or an absent marker — means the derived indexes predate this build and
59
+ * are stale.
60
+ */
61
+ export const COR_EXPECTED_INDEX_EPOCH = EXPECTED_INDEX_EPOCH;
62
+ /**
63
+ * @description Storage-root-relative path of the version-handshake marker.
64
+ * Mirrors Brainy's `BRAIN_FORMAT_PATH` (the constant is not exported from
65
+ * Brainy, so it is re-declared here — a stable on-disk path).
66
+ */
67
+ export const BRAIN_FORMAT_PATH = '_system/brain-format.json';
68
+ /**
69
+ * @description Default brain-size boundary (canonical noun count) above which a
70
+ * 7.x → 8.0 migration is treated as a LARGE brain. Mirrors Brainy's
71
+ * `AUTO_REBUILD_THRESHOLD` (10000): below it Brainy auto-rebuilds all derived
72
+ * indexes inline on open; at or above it Brainy defers to lazy first-query
73
+ * rebuilds. cor uses the SAME boundary so the two agree on which brains take the
74
+ * small (Brainy-driven) versus large (cor-owned, rc.8 background-swap) path.
75
+ * Overridable via the `COR_LARGE_BRAIN_THRESHOLD` env var.
76
+ */
77
+ export const COR_LARGE_BRAIN_THRESHOLD_DEFAULT = 10000;
78
+ /**
79
+ * @description Read the on-disk version-handshake marker, reimplementing
80
+ * Brainy's `readBrainFormat` validation so cor classifies drift identically
81
+ * without importing Brainy's (non-exported) marker module. Returns `null` when
82
+ * the marker is absent (a pre-handshake 7.x brain, or a brand-new store) OR
83
+ * malformed (not an object, missing either field, or a non-finite epoch) — a
84
+ * corrupt marker forces a safe rebuild rather than trusting a bad epoch.
85
+ *
86
+ * Storage that does not implement `readRawObject` (a non-`BaseStorage` adapter)
87
+ * is treated as having no marker — the cloud/JS path cor never accelerates.
88
+ *
89
+ * @param storage - The brain's storage adapter (raw-object surface accessed via
90
+ * a structural cast — `readRawObject` is on `BaseStorage`, not the
91
+ * `StorageAdapter` type).
92
+ * @returns The parsed {@link OnDiskBrainFormat}, or `null`.
93
+ * @example
94
+ * const onDisk = await readOnDiskBrainFormat(storage)
95
+ * const stale = isEpochStale(onDisk)
96
+ */
97
+ export async function readOnDiskBrainFormat(storage) {
98
+ const reader = storage;
99
+ if (typeof reader.readRawObject !== 'function')
100
+ return null;
101
+ let raw;
102
+ try {
103
+ raw = await reader.readRawObject(BRAIN_FORMAT_PATH);
104
+ }
105
+ catch {
106
+ // A read error during the probe is treated as "no marker": we cannot read
107
+ // it, so the safe default (drift → rebuild) applies rather than erroring on
108
+ // a transient storage hiccup at every open.
109
+ return null;
110
+ }
111
+ if (raw === null || raw === undefined || typeof raw !== 'object')
112
+ return null;
113
+ const obj = raw;
114
+ if (typeof obj.dataFormat !== 'string' ||
115
+ typeof obj.indexEpoch !== 'number' ||
116
+ !Number.isFinite(obj.indexEpoch)) {
117
+ return null;
118
+ }
119
+ return { dataFormat: obj.dataFormat, indexEpoch: obj.indexEpoch };
120
+ }
121
+ /**
122
+ * @description Classify the on-disk marker as stale (derived indexes predate
123
+ * this build) or current. Stale when the marker is absent ({@link
124
+ * readOnDiskBrainFormat} returned `null`) OR its `indexEpoch` differs from
125
+ * {@link COR_EXPECTED_INDEX_EPOCH}.
126
+ *
127
+ * @param onDisk - The result of {@link readOnDiskBrainFormat}.
128
+ * @returns `true` when a derived-index rebuild is required.
129
+ */
130
+ export function isEpochStale(onDisk) {
131
+ return onDisk === null || onDisk.indexEpoch !== COR_EXPECTED_INDEX_EPOCH;
132
+ }
133
+ /**
134
+ * @description Resolve the large-brain boundary, honouring the
135
+ * `COR_LARGE_BRAIN_THRESHOLD` env override (a non-negative finite integer);
136
+ * falls back to {@link COR_LARGE_BRAIN_THRESHOLD_DEFAULT} when unset or
137
+ * unparseable.
138
+ *
139
+ * @returns The canonical noun count at or above which a migration is LARGE.
140
+ */
141
+ export function resolveLargeBrainThreshold() {
142
+ const raw = process.env.COR_LARGE_BRAIN_THRESHOLD;
143
+ if (raw === undefined)
144
+ return COR_LARGE_BRAIN_THRESHOLD_DEFAULT;
145
+ const n = Number(raw.trim());
146
+ return Number.isFinite(n) && n >= 0 ? n : COR_LARGE_BRAIN_THRESHOLD_DEFAULT;
147
+ }
148
+ /**
149
+ * @description Size the brain by its canonical noun count, via
150
+ * `getNouns({ pagination: { limit: 1 } }).totalCount` — the same one-row probe
151
+ * Brainy's `rebuildIndexesIfNeeded` uses to decide small-versus-large. A
152
+ * storage error (or a missing `totalCount`) reports `0`, which classifies the
153
+ * brain as small and lets Brainy's path drive — the conservative default that
154
+ * never throws the loud large-brain migration error on an unreadable probe.
155
+ *
156
+ * @param storage - The brain's storage adapter.
157
+ * @returns The canonical noun count (`0` when unknown).
158
+ */
159
+ export async function detectBrainSize(storage) {
160
+ try {
161
+ const probe = await storage.getNouns({
162
+ pagination: { limit: 1 },
163
+ });
164
+ return probe?.totalCount ?? 0;
165
+ }
166
+ catch {
167
+ return 0;
168
+ }
169
+ }
170
+ /**
171
+ * @description Resolve the auto-migrate policy from the environment.
172
+ * `config.autoMigrate` is unreachable from a cor provider (the provider receives
173
+ * only storage, not the brain config), so the policy is an env var:
174
+ * `COR_AUTO_MIGRATE`. Defaults to `true` (migrate automatically). Only the
175
+ * explicit opt-outs `'false'` / `'0'` (case-insensitive, trimmed) disable it —
176
+ * any other value (or unset) is `true`.
177
+ *
178
+ * @returns `true` when cor should migrate automatically; `false` when the
179
+ * operator has opted out and wants the loud {@link IndexMigrationRequiredError}
180
+ * on a large stale brain.
181
+ */
182
+ export function resolveAutoMigrate() {
183
+ const raw = process.env.COR_AUTO_MIGRATE;
184
+ if (raw === undefined)
185
+ return true;
186
+ const normalized = raw.trim().toLowerCase();
187
+ return !(normalized === 'false' || normalized === '0');
188
+ }
189
+ /**
190
+ * @description Thrown from `MetadataIndexManager.init()` when a LARGE brain
191
+ * carries stale (drifted or absent) derived-index epoch state on disk AND
192
+ * auto-migrate is disabled ({@link resolveAutoMigrate} is `false`). Modeled on
193
+ * `MetadataIndexLegacyJsonStateDetected`: it is loud and pre-serve — a service
194
+ * that refuses to start is recoverable, whereas serving on indexes that predate
195
+ * this build would surface stale/empty results with no signal. Catchable for
196
+ * callers that prefer a one-shot warning, but the default (uncaught) is the
197
+ * right behaviour for production.
198
+ */
199
+ export class IndexMigrationRequiredError extends Error {
200
+ name = 'IndexMigrationRequiredError';
201
+ }
202
+ //# sourceMappingURL=indexEpochGuard.js.map