@git-stunts/git-warp 11.5.1 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -10
- package/bin/cli/commands/registry.js +4 -0
- package/bin/cli/commands/reindex.js +41 -0
- package/bin/cli/commands/verify-index.js +59 -0
- package/bin/cli/infrastructure.js +7 -2
- package/bin/cli/schemas.js +19 -0
- package/bin/cli/types.js +2 -0
- package/index.d.ts +49 -12
- package/package.json +2 -2
- package/src/domain/WarpGraph.js +40 -0
- package/src/domain/errors/ShardIdOverflowError.js +28 -0
- package/src/domain/errors/index.js +1 -0
- package/src/domain/services/AdjacencyNeighborProvider.js +140 -0
- package/src/domain/services/BitmapNeighborProvider.js +178 -0
- package/src/domain/services/CheckpointMessageCodec.js +3 -3
- package/src/domain/services/CheckpointService.js +77 -12
- package/src/domain/services/GraphTraversal.js +1239 -0
- package/src/domain/services/IncrementalIndexUpdater.js +765 -0
- package/src/domain/services/JoinReducer.js +233 -5
- package/src/domain/services/LogicalBitmapIndexBuilder.js +323 -0
- package/src/domain/services/LogicalIndexBuildService.js +108 -0
- package/src/domain/services/LogicalIndexReader.js +315 -0
- package/src/domain/services/LogicalTraversal.js +321 -202
- package/src/domain/services/MaterializedViewService.js +379 -0
- package/src/domain/services/ObserverView.js +138 -47
- package/src/domain/services/PatchBuilderV2.js +3 -3
- package/src/domain/services/PropertyIndexBuilder.js +64 -0
- package/src/domain/services/PropertyIndexReader.js +111 -0
- package/src/domain/services/TemporalQuery.js +128 -14
- package/src/domain/types/PatchDiff.js +90 -0
- package/src/domain/types/WarpTypesV2.js +4 -4
- package/src/domain/utils/MinHeap.js +45 -17
- package/src/domain/utils/canonicalCbor.js +36 -0
- package/src/domain/utils/fnv1a.js +20 -0
- package/src/domain/utils/roaring.js +14 -3
- package/src/domain/utils/shardKey.js +40 -0
- package/src/domain/utils/toBytes.js +17 -0
- package/src/domain/warp/_wiredMethods.d.ts +7 -1
- package/src/domain/warp/checkpoint.methods.js +21 -5
- package/src/domain/warp/materialize.methods.js +17 -5
- package/src/domain/warp/materializeAdvanced.methods.js +142 -3
- package/src/domain/warp/query.methods.js +78 -12
- package/src/infrastructure/adapters/CasSeekCacheAdapter.js +26 -5
- package/src/ports/BlobPort.js +1 -1
- package/src/ports/NeighborProviderPort.js +59 -0
- package/src/ports/SeekCachePort.js +4 -3
|
@@ -50,6 +50,7 @@ const NOT_CHECKED = Symbol('NOT_CHECKED');
|
|
|
50
50
|
* @typedef {Object} RoaringBitmapSubset
|
|
51
51
|
* @property {number} size
|
|
52
52
|
* @property {function(number): void} add
|
|
53
|
+
* @property {function(number): void} remove
|
|
53
54
|
* @property {function(number): boolean} has
|
|
54
55
|
* @property {function(Iterable<number>): void} orInPlace
|
|
55
56
|
* @property {function(boolean): Uint8Array} serialize
|
|
@@ -63,6 +64,13 @@ const NOT_CHECKED = Symbol('NOT_CHECKED');
|
|
|
63
64
|
*/
|
|
64
65
|
let roaringModule = null;
|
|
65
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Captures module initialization failure so callers can see the root cause.
|
|
69
|
+
* @type {unknown}
|
|
70
|
+
* @private
|
|
71
|
+
*/
|
|
72
|
+
let initError = null;
|
|
73
|
+
|
|
66
74
|
/**
|
|
67
75
|
* Cached result of native availability check.
|
|
68
76
|
* `NOT_CHECKED` means not yet checked, `null` means indeterminate.
|
|
@@ -83,7 +91,8 @@ let nativeAvailability = NOT_CHECKED;
|
|
|
83
91
|
*/
|
|
84
92
|
function loadRoaring() {
|
|
85
93
|
if (!roaringModule) {
|
|
86
|
-
|
|
94
|
+
const cause = initError instanceof Error ? ` Caused by: ${initError.message}` : '';
|
|
95
|
+
throw new Error(`Roaring module not loaded. Call initRoaring() first or ensure top-level await import completed.${cause}`);
|
|
87
96
|
}
|
|
88
97
|
return roaringModule;
|
|
89
98
|
}
|
|
@@ -99,6 +108,7 @@ function loadRoaring() {
|
|
|
99
108
|
export async function initRoaring(mod) {
|
|
100
109
|
if (mod) {
|
|
101
110
|
roaringModule = mod;
|
|
111
|
+
initError = null;
|
|
102
112
|
return;
|
|
103
113
|
}
|
|
104
114
|
if (!roaringModule) {
|
|
@@ -113,8 +123,9 @@ export async function initRoaring(mod) {
|
|
|
113
123
|
// Auto-initialize on module load (top-level await)
|
|
114
124
|
try {
|
|
115
125
|
await initRoaring();
|
|
116
|
-
} catch {
|
|
117
|
-
// Roaring may not be installed;
|
|
126
|
+
} catch (err) {
|
|
127
|
+
// Roaring may not be installed; keep root cause for downstream diagnostics.
|
|
128
|
+
initError = err;
|
|
118
129
|
}
|
|
119
130
|
|
|
120
131
|
/**
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const HEX_RE = /^[0-9a-fA-F]{40}$|^[0-9a-fA-F]{64}$/;
|
|
2
|
+
|
|
3
|
+
const encoder = new TextEncoder();
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* FNV-1a 32-bit over raw bytes (Uint8Array).
|
|
7
|
+
*
|
|
8
|
+
* @param {Uint8Array} bytes
|
|
9
|
+
* @returns {number} Unsigned 32-bit hash
|
|
10
|
+
*/
|
|
11
|
+
function fnv1aBytes(bytes) {
|
|
12
|
+
let hash = 0x811c9dc5;
|
|
13
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
14
|
+
hash ^= bytes[i];
|
|
15
|
+
hash = Math.imul(hash, 0x01000193);
|
|
16
|
+
}
|
|
17
|
+
return hash >>> 0;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Computes a 2-character hex shard key for a given ID.
|
|
22
|
+
*
|
|
23
|
+
* For hex SHAs (exactly 40 or 64 hex chars), uses the first two characters (lowercased).
|
|
24
|
+
* For all other strings, computes FNV-1a hash over UTF-8 bytes and takes the low byte.
|
|
25
|
+
*
|
|
26
|
+
* Returns '00' for null, undefined, or non-string inputs (graceful fallback).
|
|
27
|
+
*
|
|
28
|
+
* @param {string} id - Node ID or SHA
|
|
29
|
+
* @returns {string} 2-character lowercase hex shard key (e.g. 'ab', '0f')
|
|
30
|
+
*/
|
|
31
|
+
export default function computeShardKey(id) {
|
|
32
|
+
if (id === null || id === undefined || typeof id !== 'string') {
|
|
33
|
+
return '00';
|
|
34
|
+
}
|
|
35
|
+
if (HEX_RE.test(id)) {
|
|
36
|
+
return id.substring(0, 2).toLowerCase();
|
|
37
|
+
}
|
|
38
|
+
const hash = fnv1aBytes(encoder.encode(id));
|
|
39
|
+
return (hash & 0xff).toString(16).padStart(2, '0');
|
|
40
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Normalizes a decoded byte payload to a Uint8Array.
|
|
3
|
+
*
|
|
4
|
+
* CBOR decoders may yield Buffer, Uint8Array, or plain number[] depending
|
|
5
|
+
* on runtime and codec implementation (e.g. cbor-x on Node vs Deno).
|
|
6
|
+
* This helper ensures Roaring bitmap deserialization and other binary APIs
|
|
7
|
+
* always receive a Uint8Array.
|
|
8
|
+
*
|
|
9
|
+
* @param {Uint8Array|ArrayLike<number>} value
|
|
10
|
+
* @returns {Uint8Array}
|
|
11
|
+
*/
|
|
12
|
+
export default function toBytes(value) {
|
|
13
|
+
if (value instanceof Uint8Array) {
|
|
14
|
+
return value;
|
|
15
|
+
}
|
|
16
|
+
return Uint8Array.from(value);
|
|
17
|
+
}
|
|
@@ -151,6 +151,7 @@ interface CheckpointData {
|
|
|
151
151
|
stateHash: string;
|
|
152
152
|
schema: number;
|
|
153
153
|
provenanceIndex?: unknown;
|
|
154
|
+
indexShardOids?: Record<string, string>;
|
|
154
155
|
}
|
|
155
156
|
|
|
156
157
|
export {};
|
|
@@ -247,8 +248,13 @@ declare module '../WarpGraph.js' {
|
|
|
247
248
|
// ── materializeAdvanced.methods.js ────────────────────────────────────
|
|
248
249
|
_resolveCeiling(options?: { ceiling?: number | null }): number | null;
|
|
249
250
|
_buildAdjacency(state: WarpStateV5): { outgoing: Map<string, Array<{ neighborId: string; label: string }>>; incoming: Map<string, Array<{ neighborId: string; label: string }>> };
|
|
250
|
-
|
|
251
|
+
_buildView(state: WarpStateV5, stateHash: string, diff?: import('../types/PatchDiff.js').PatchDiff): void;
|
|
252
|
+
_setMaterializedState(state: WarpStateV5, diff?: import('../types/PatchDiff.js').PatchDiff): Promise<{ state: WarpStateV5; stateHash: string; adjacency: unknown }>;
|
|
251
253
|
_materializeWithCeiling(ceiling: number, collectReceipts: boolean, t0: number): Promise<WarpStateV5 | { state: WarpStateV5; receipts: TickReceipt[] }>;
|
|
254
|
+
_persistSeekCacheEntry(cacheKey: string, buf: Buffer, state: WarpStateV5): Promise<void>;
|
|
255
|
+
_restoreIndexFromCache(indexTreeOid: string): Promise<void>;
|
|
252
256
|
materializeAt(checkpointSha: string): Promise<WarpStateV5>;
|
|
257
|
+
verifyIndex(options?: { seed?: number; sampleRate?: number }): { passed: number; failed: number; errors: Array<{ nodeId: string; direction: string; expected: string[]; actual: string[] }> };
|
|
258
|
+
invalidateIndex(): void;
|
|
253
259
|
}
|
|
254
260
|
}
|
|
@@ -60,7 +60,22 @@ export async function createCheckpoint() {
|
|
|
60
60
|
this._checkpointing = prevCheckpointing;
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
// 4.
|
|
63
|
+
// 4. Reuse cached index tree or rebuild from view service
|
|
64
|
+
let indexTree = this._cachedIndexTree;
|
|
65
|
+
if (!indexTree && this._viewService) {
|
|
66
|
+
try {
|
|
67
|
+
const { tree } = this._viewService.build(state);
|
|
68
|
+
indexTree = tree;
|
|
69
|
+
} catch (err) {
|
|
70
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
71
|
+
this._logger?.warn('[warp] checkpoint index build failed; saving checkpoint without index', {
|
|
72
|
+
error: message,
|
|
73
|
+
});
|
|
74
|
+
indexTree = null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// 5. Create checkpoint commit with provenance index + index tree
|
|
64
79
|
/** @type {CorePersistence} */
|
|
65
80
|
const persistence = this._persistence;
|
|
66
81
|
const checkpointSha = await createCheckpointCommit({
|
|
@@ -72,15 +87,16 @@ export async function createCheckpoint() {
|
|
|
72
87
|
provenanceIndex: this._provenanceIndex || undefined,
|
|
73
88
|
crypto: this._crypto,
|
|
74
89
|
codec: this._codec,
|
|
90
|
+
indexTree: indexTree || undefined,
|
|
75
91
|
});
|
|
76
92
|
|
|
77
|
-
//
|
|
93
|
+
// 6. Update checkpoint ref
|
|
78
94
|
const checkpointRef = buildCheckpointRef(this._graphName);
|
|
79
95
|
await this._persistence.updateRef(checkpointRef, checkpointSha);
|
|
80
96
|
|
|
81
97
|
this._logTiming('createCheckpoint', t0);
|
|
82
98
|
|
|
83
|
-
//
|
|
99
|
+
// 7. Return checkpoint SHA
|
|
84
100
|
return checkpointSha;
|
|
85
101
|
} catch (err) {
|
|
86
102
|
this._logTiming('createCheckpoint', t0, { error: /** @type {Error} */ (err) });
|
|
@@ -137,7 +153,7 @@ export async function syncCoverage() {
|
|
|
137
153
|
* Loads the latest checkpoint for this graph.
|
|
138
154
|
*
|
|
139
155
|
* @this {import('../WarpGraph.js').default}
|
|
140
|
-
* @returns {Promise<{state: import('../services/JoinReducer.js').WarpStateV5, frontier: Map<string, string>, stateHash: string, schema: number, provenanceIndex?: import('../services/ProvenanceIndex.js').ProvenanceIndex}|null>} The checkpoint or null
|
|
156
|
+
* @returns {Promise<{state: import('../services/JoinReducer.js').WarpStateV5, frontier: Map<string, string>, stateHash: string, schema: number, provenanceIndex?: import('../services/ProvenanceIndex.js').ProvenanceIndex, indexShardOids?: Record<string, string>|null}|null>} The checkpoint or null
|
|
141
157
|
* @private
|
|
142
158
|
*/
|
|
143
159
|
export async function _loadLatestCheckpoint() {
|
|
@@ -197,7 +213,7 @@ export async function _loadPatchesSince(checkpoint) {
|
|
|
197
213
|
*/
|
|
198
214
|
export async function _validateMigrationBoundary() {
|
|
199
215
|
const checkpoint = await this._loadLatestCheckpoint();
|
|
200
|
-
if (checkpoint?.schema === 2 || checkpoint?.schema === 3) {
|
|
216
|
+
if (checkpoint?.schema === 2 || checkpoint?.schema === 3 || checkpoint?.schema === 4) {
|
|
201
217
|
return; // Already migrated
|
|
202
218
|
}
|
|
203
219
|
|
|
@@ -51,6 +51,7 @@ function scanPatchesForMaxLamport(graph, patches) {
|
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
+
|
|
54
55
|
/**
|
|
55
56
|
* Materializes the current graph state.
|
|
56
57
|
*
|
|
@@ -99,10 +100,13 @@ export async function materialize(options) {
|
|
|
99
100
|
let state;
|
|
100
101
|
/** @type {import('../types/TickReceipt.js').TickReceipt[]|undefined} */
|
|
101
102
|
let receipts;
|
|
103
|
+
/** @type {import('../types/PatchDiff.js').PatchDiff|undefined} */
|
|
104
|
+
let diff;
|
|
102
105
|
let patchCount = 0;
|
|
106
|
+
const wantDiff = !collectReceipts && !!this._cachedIndexTree;
|
|
103
107
|
|
|
104
108
|
// If checkpoint exists, use incremental materialization
|
|
105
|
-
if (checkpoint?.schema === 2 || checkpoint?.schema === 3) {
|
|
109
|
+
if (checkpoint?.schema === 2 || checkpoint?.schema === 3 || checkpoint?.schema === 4) {
|
|
106
110
|
const patches = await this._loadPatchesSince(checkpoint);
|
|
107
111
|
// Update max observed Lamport so _nextLamport() issues globally-monotonic ticks.
|
|
108
112
|
// Read the checkpoint frontier's tip commit messages to capture the pre-checkpoint max,
|
|
@@ -115,6 +119,10 @@ export async function materialize(options) {
|
|
|
115
119
|
const result = /** @type {{state: import('../services/JoinReducer.js').WarpStateV5, receipts: import('../types/TickReceipt.js').TickReceipt[]}} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (patches), checkpoint.state, { receipts: true }));
|
|
116
120
|
state = result.state;
|
|
117
121
|
receipts = result.receipts;
|
|
122
|
+
} else if (wantDiff) {
|
|
123
|
+
const result = /** @type {{state: import('../services/JoinReducer.js').WarpStateV5, diff: import('../types/PatchDiff.js').PatchDiff}} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (patches), checkpoint.state, { trackDiff: true }));
|
|
124
|
+
state = result.state;
|
|
125
|
+
diff = result.diff;
|
|
118
126
|
} else {
|
|
119
127
|
state = /** @type {import('../services/JoinReducer.js').WarpStateV5} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (patches), checkpoint.state));
|
|
120
128
|
}
|
|
@@ -164,6 +172,10 @@ export async function materialize(options) {
|
|
|
164
172
|
const result = /** @type {{state: import('../services/JoinReducer.js').WarpStateV5, receipts: import('../types/TickReceipt.js').TickReceipt[]}} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (allPatches), undefined, { receipts: true }));
|
|
165
173
|
state = result.state;
|
|
166
174
|
receipts = result.receipts;
|
|
175
|
+
} else if (wantDiff) {
|
|
176
|
+
const result = /** @type {{state: import('../services/JoinReducer.js').WarpStateV5, diff: import('../types/PatchDiff.js').PatchDiff}} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (allPatches), undefined, { trackDiff: true }));
|
|
177
|
+
state = result.state;
|
|
178
|
+
diff = result.diff;
|
|
167
179
|
} else {
|
|
168
180
|
state = /** @type {import('../services/JoinReducer.js').WarpStateV5} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (allPatches)));
|
|
169
181
|
}
|
|
@@ -178,7 +190,7 @@ export async function materialize(options) {
|
|
|
178
190
|
}
|
|
179
191
|
}
|
|
180
192
|
|
|
181
|
-
await this._setMaterializedState(state);
|
|
193
|
+
await this._setMaterializedState(state, diff);
|
|
182
194
|
this._provenanceDegraded = false;
|
|
183
195
|
this._cachedCeiling = null;
|
|
184
196
|
this._cachedFrontier = null;
|
|
@@ -202,9 +214,9 @@ export async function materialize(options) {
|
|
|
202
214
|
// Also handles deferred replay for subscribers added with replay: true before cached state
|
|
203
215
|
if (this._subscribers.length > 0) {
|
|
204
216
|
const hasPendingReplay = this._subscribers.some(s => s.pendingReplay);
|
|
205
|
-
const
|
|
206
|
-
if (!isEmptyDiff(
|
|
207
|
-
this._notifySubscribers(
|
|
217
|
+
const stateDelta = diffStates(this._lastNotifiedState, state);
|
|
218
|
+
if (!isEmptyDiff(stateDelta) || hasPendingReplay) {
|
|
219
|
+
this._notifySubscribers(stateDelta, state);
|
|
208
220
|
}
|
|
209
221
|
}
|
|
210
222
|
// Clone state to prevent eager path mutations from affecting the baseline
|
|
@@ -18,6 +18,7 @@ import { serializeFullStateV5, deserializeFullStateV5 } from '../services/Checkp
|
|
|
18
18
|
import { buildSeekCacheKey } from '../utils/seekCacheKey.js';
|
|
19
19
|
import { materializeIncremental } from '../services/CheckpointService.js';
|
|
20
20
|
import { createFrontier, updateFrontier } from '../services/Frontier.js';
|
|
21
|
+
import BitmapNeighborProvider from '../services/BitmapNeighborProvider.js';
|
|
21
22
|
|
|
22
23
|
/** @typedef {import('../types/WarpPersistence.js').CorePersistence} CorePersistence */
|
|
23
24
|
/** @typedef {import('../services/JoinReducer.js').WarpStateV5} WarpStateV5 */
|
|
@@ -106,10 +107,11 @@ export function _buildAdjacency(state) {
|
|
|
106
107
|
*
|
|
107
108
|
* @this {import('../WarpGraph.js').default}
|
|
108
109
|
* @param {import('../services/JoinReducer.js').WarpStateV5} state
|
|
110
|
+
* @param {import('../types/PatchDiff.js').PatchDiff} [diff] - Optional diff for incremental index
|
|
109
111
|
* @returns {Promise<MaterializedResult>}
|
|
110
112
|
* @private
|
|
111
113
|
*/
|
|
112
|
-
export async function _setMaterializedState(state) {
|
|
114
|
+
export async function _setMaterializedState(state, diff) {
|
|
113
115
|
this._cachedState = state;
|
|
114
116
|
this._stateDirty = false;
|
|
115
117
|
this._versionVector = vvClone(state.observedFrontier);
|
|
@@ -128,9 +130,58 @@ export async function _setMaterializedState(state) {
|
|
|
128
130
|
}
|
|
129
131
|
|
|
130
132
|
this._materializedGraph = { state, stateHash, adjacency };
|
|
133
|
+
this._buildView(state, stateHash, diff);
|
|
131
134
|
return this._materializedGraph;
|
|
132
135
|
}
|
|
133
136
|
|
|
137
|
+
/**
|
|
138
|
+
* Builds the MaterializedView (logicalIndex + propertyReader) and attaches
|
|
139
|
+
* a BitmapNeighborProvider to the materialized graph. Skips rebuild when
|
|
140
|
+
* the stateHash matches the previous build. Uses incremental update when
|
|
141
|
+
* a diff and cached index tree are available.
|
|
142
|
+
*
|
|
143
|
+
* @this {import('../WarpGraph.js').default}
|
|
144
|
+
* @param {import('../services/JoinReducer.js').WarpStateV5} state
|
|
145
|
+
* @param {string} stateHash
|
|
146
|
+
* @param {import('../types/PatchDiff.js').PatchDiff} [diff] - Optional diff for incremental update
|
|
147
|
+
* @private
|
|
148
|
+
*/
|
|
149
|
+
export function _buildView(state, stateHash, diff) {
|
|
150
|
+
if (this._cachedViewHash === stateHash) {
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
try {
|
|
154
|
+
/** @type {import('../services/MaterializedViewService.js').BuildResult} */
|
|
155
|
+
let result;
|
|
156
|
+
if (diff && this._cachedIndexTree) {
|
|
157
|
+
result = this._viewService.applyDiff({
|
|
158
|
+
existingTree: this._cachedIndexTree,
|
|
159
|
+
diff,
|
|
160
|
+
state,
|
|
161
|
+
});
|
|
162
|
+
} else {
|
|
163
|
+
result = this._viewService.build(state);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
this._logicalIndex = result.logicalIndex;
|
|
167
|
+
this._propertyReader = result.propertyReader;
|
|
168
|
+
this._cachedViewHash = stateHash;
|
|
169
|
+
this._cachedIndexTree = result.tree;
|
|
170
|
+
|
|
171
|
+
const provider = new BitmapNeighborProvider({ logicalIndex: result.logicalIndex });
|
|
172
|
+
if (this._materializedGraph) {
|
|
173
|
+
this._materializedGraph.provider = provider;
|
|
174
|
+
}
|
|
175
|
+
} catch (err) {
|
|
176
|
+
this._logger?.warn('[warp] index build failed, falling back to linear scan', {
|
|
177
|
+
error: /** @type {Error} */ (err).message,
|
|
178
|
+
});
|
|
179
|
+
this._logicalIndex = null;
|
|
180
|
+
this._propertyReader = null;
|
|
181
|
+
this._cachedIndexTree = null;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
134
185
|
/**
|
|
135
186
|
* Materializes the graph with a Lamport ceiling (time-travel).
|
|
136
187
|
*
|
|
@@ -196,12 +247,15 @@ export async function _materializeWithCeiling(ceiling, collectReceipts, t0) {
|
|
|
196
247
|
const cached = await this._seekCache.get(cacheKey);
|
|
197
248
|
if (cached) {
|
|
198
249
|
try {
|
|
199
|
-
const state = deserializeFullStateV5(cached, { codec: this._codec });
|
|
250
|
+
const state = deserializeFullStateV5(cached.buffer, { codec: this._codec });
|
|
200
251
|
this._provenanceIndex = new ProvenanceIndex();
|
|
201
252
|
this._provenanceDegraded = true;
|
|
202
253
|
await this._setMaterializedState(state);
|
|
203
254
|
this._cachedCeiling = ceiling;
|
|
204
255
|
this._cachedFrontier = frontier;
|
|
256
|
+
if (cached.indexTreeOid) {
|
|
257
|
+
await this._restoreIndexFromCache(cached.indexTreeOid);
|
|
258
|
+
}
|
|
205
259
|
this._logTiming('materialize', t0, { metrics: `cache hit (ceiling=${ceiling})` });
|
|
206
260
|
return state;
|
|
207
261
|
} catch {
|
|
@@ -258,7 +312,8 @@ export async function _materializeWithCeiling(ceiling, collectReceipts, t0) {
|
|
|
258
312
|
cacheKey = buildSeekCacheKey(ceiling, frontier);
|
|
259
313
|
}
|
|
260
314
|
const buf = serializeFullStateV5(state, { codec: this._codec });
|
|
261
|
-
this.
|
|
315
|
+
this._persistSeekCacheEntry(cacheKey, /** @type {Buffer} */ (buf), state)
|
|
316
|
+
.catch(() => {});
|
|
262
317
|
}
|
|
263
318
|
|
|
264
319
|
// Skip auto-checkpoint and GC — this is an exploratory read
|
|
@@ -270,6 +325,62 @@ export async function _materializeWithCeiling(ceiling, collectReceipts, t0) {
|
|
|
270
325
|
return state;
|
|
271
326
|
}
|
|
272
327
|
|
|
328
|
+
/**
|
|
329
|
+
* Persists a seek cache entry with an optional index tree snapshot.
|
|
330
|
+
*
|
|
331
|
+
* Builds the bitmap index tree from the materialized state, writes it
|
|
332
|
+
* to Git storage, and includes the resulting tree OID in the cache
|
|
333
|
+
* entry metadata. Index persistence failure is non-fatal — the state
|
|
334
|
+
* buffer is still cached without the index.
|
|
335
|
+
*
|
|
336
|
+
* @this {import('../WarpGraph.js').default}
|
|
337
|
+
* @param {string} cacheKey - Seek cache key
|
|
338
|
+
* @param {Buffer} buf - Serialized WarpStateV5 buffer
|
|
339
|
+
* @param {import('../services/JoinReducer.js').WarpStateV5} state
|
|
340
|
+
* @returns {Promise<void>}
|
|
341
|
+
* @private
|
|
342
|
+
*/
|
|
343
|
+
export async function _persistSeekCacheEntry(cacheKey, buf, state) {
|
|
344
|
+
/** @type {{ indexTreeOid?: string }} */
|
|
345
|
+
const opts = {};
|
|
346
|
+
try {
|
|
347
|
+
const { tree } = this._viewService.build(state);
|
|
348
|
+
opts.indexTreeOid = await this._viewService.persistIndexTree(
|
|
349
|
+
tree,
|
|
350
|
+
this._persistence,
|
|
351
|
+
);
|
|
352
|
+
} catch {
|
|
353
|
+
// Non-fatal — cache the state without the index
|
|
354
|
+
}
|
|
355
|
+
if (this._seekCache) {
|
|
356
|
+
await this._seekCache.set(cacheKey, buf, opts);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Restores a LogicalIndex and PropertyReader from a cached index tree OID.
|
|
362
|
+
*
|
|
363
|
+
* Reads the tree entries from Git storage and delegates hydration to
|
|
364
|
+
* the MaterializedViewService. Failure is non-fatal — the in-memory
|
|
365
|
+
* index built by `_buildView` remains as fallback.
|
|
366
|
+
*
|
|
367
|
+
* @this {import('../WarpGraph.js').default}
|
|
368
|
+
* @param {string} indexTreeOid - Git tree OID of the bitmap index snapshot
|
|
369
|
+
* @returns {Promise<void>}
|
|
370
|
+
* @private
|
|
371
|
+
*/
|
|
372
|
+
export async function _restoreIndexFromCache(indexTreeOid) {
|
|
373
|
+
try {
|
|
374
|
+
const shardOids = await this._persistence.readTreeOids(indexTreeOid);
|
|
375
|
+
const { logicalIndex, propertyReader } =
|
|
376
|
+
await this._viewService.loadFromOids(shardOids, this._persistence);
|
|
377
|
+
this._logicalIndex = logicalIndex;
|
|
378
|
+
this._propertyReader = propertyReader;
|
|
379
|
+
} catch {
|
|
380
|
+
// Non-fatal — fall back to in-memory index from _buildView
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
273
384
|
/**
|
|
274
385
|
* Materializes the graph state at a specific checkpoint.
|
|
275
386
|
*
|
|
@@ -348,3 +459,31 @@ export async function materializeAt(checkpointSha) {
|
|
|
348
459
|
await this._setMaterializedState(state);
|
|
349
460
|
return state;
|
|
350
461
|
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Verifies the bitmap index against adjacency ground truth.
|
|
465
|
+
*
|
|
466
|
+
* @this {import('../WarpGraph.js').default}
|
|
467
|
+
* @param {{ seed?: number, sampleRate?: number }} [options]
|
|
468
|
+
* @returns {{ passed: number, failed: number, errors: Array<{nodeId: string, direction: string, expected: string[], actual: string[]}> }}
|
|
469
|
+
*/
|
|
470
|
+
export function verifyIndex(options) {
|
|
471
|
+
if (!this._logicalIndex || !this._cachedState || !this._viewService) {
|
|
472
|
+
throw new Error('Cannot verify index: graph not materialized or index not built');
|
|
473
|
+
}
|
|
474
|
+
return this._viewService.verifyIndex({
|
|
475
|
+
state: this._cachedState,
|
|
476
|
+
logicalIndex: this._logicalIndex,
|
|
477
|
+
options,
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* Clears the cached bitmap index, forcing a full rebuild on next materialize.
|
|
483
|
+
*
|
|
484
|
+
* @this {import('../WarpGraph.js').default}
|
|
485
|
+
*/
|
|
486
|
+
export function invalidateIndex() {
|
|
487
|
+
this._cachedIndexTree = null;
|
|
488
|
+
this._cachedViewHash = null;
|
|
489
|
+
}
|
|
@@ -42,6 +42,18 @@ export async function hasNode(nodeId) {
|
|
|
42
42
|
*/
|
|
43
43
|
export async function getNodeProps(nodeId) {
|
|
44
44
|
await this._ensureFreshState();
|
|
45
|
+
|
|
46
|
+
// ── Indexed fast path (positive results only; stale index falls through) ──
|
|
47
|
+
if (this._propertyReader && this._logicalIndex?.isAlive(nodeId)) {
|
|
48
|
+
try {
|
|
49
|
+
const record = await this._propertyReader.getNodeProps(nodeId);
|
|
50
|
+
return record ? new Map(Object.entries(record)) : new Map();
|
|
51
|
+
} catch {
|
|
52
|
+
// Fall through to linear scan on index read failures.
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// ── Linear scan fallback ─────────────────────────────────────────────
|
|
45
57
|
const s = /** @type {import('../services/JoinReducer.js').WarpStateV5} */ (this._cachedState);
|
|
46
58
|
|
|
47
59
|
if (!orsetContains(s.nodeAlive, nodeId)) {
|
|
@@ -103,6 +115,17 @@ export async function getEdgeProps(from, to, label) {
|
|
|
103
115
|
return props;
|
|
104
116
|
}
|
|
105
117
|
|
|
118
|
+
/**
|
|
119
|
+
* Converts NeighborEdge[] to the query-method shape with a direction tag.
|
|
120
|
+
*
|
|
121
|
+
* @param {Array<{neighborId: string, label: string}>} edges
|
|
122
|
+
* @param {'outgoing' | 'incoming'} dir
|
|
123
|
+
* @returns {Array<{nodeId: string, label: string, direction: 'outgoing' | 'incoming'}>}
|
|
124
|
+
*/
|
|
125
|
+
function tagDirection(edges, dir) {
|
|
126
|
+
return edges.map((e) => ({ nodeId: e.neighborId, label: e.label, direction: dir }));
|
|
127
|
+
}
|
|
128
|
+
|
|
106
129
|
/**
|
|
107
130
|
* Gets neighbors of a node from the materialized state.
|
|
108
131
|
*
|
|
@@ -115,28 +138,71 @@ export async function getEdgeProps(from, to, label) {
|
|
|
115
138
|
*/
|
|
116
139
|
export async function neighbors(nodeId, direction = 'both', edgeLabel = undefined) {
|
|
117
140
|
await this._ensureFreshState();
|
|
118
|
-
const s = /** @type {import('../services/JoinReducer.js').WarpStateV5} */ (this._cachedState);
|
|
119
141
|
|
|
142
|
+
// ── Indexed fast path (only when node is in index; stale falls through) ──
|
|
143
|
+
const provider = this._materializedGraph?.provider;
|
|
144
|
+
if (provider && this._logicalIndex?.isAlive(nodeId)) {
|
|
145
|
+
try {
|
|
146
|
+
const opts = edgeLabel ? { labels: new Set([edgeLabel]) } : undefined;
|
|
147
|
+
return await _indexedNeighbors(provider, nodeId, direction, opts);
|
|
148
|
+
} catch {
|
|
149
|
+
// Fall through to linear scan on index/provider failures.
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// ── Linear scan fallback ─────────────────────────────────────────────
|
|
154
|
+
return _linearNeighbors(/** @type {import('../services/JoinReducer.js').WarpStateV5} */ (this._cachedState), nodeId, direction, edgeLabel);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Indexed neighbor lookup using BitmapNeighborProvider.
|
|
159
|
+
*
|
|
160
|
+
* @param {import('../../ports/NeighborProviderPort.js').default} provider
|
|
161
|
+
* @param {string} nodeId
|
|
162
|
+
* @param {'outgoing' | 'incoming' | 'both'} direction
|
|
163
|
+
* @param {import('../../ports/NeighborProviderPort.js').NeighborOptions} [opts]
|
|
164
|
+
* @returns {Promise<Array<{nodeId: string, label: string, direction: 'outgoing' | 'incoming'}>>}
|
|
165
|
+
*/
|
|
166
|
+
async function _indexedNeighbors(provider, nodeId, direction, opts) {
|
|
167
|
+
if (direction === 'both') {
|
|
168
|
+
const [outEdges, inEdges] = await Promise.all([
|
|
169
|
+
provider.getNeighbors(nodeId, 'out', opts),
|
|
170
|
+
provider.getNeighbors(nodeId, 'in', opts),
|
|
171
|
+
]);
|
|
172
|
+
return [...tagDirection(outEdges, 'outgoing'), ...tagDirection(inEdges, 'incoming')];
|
|
173
|
+
}
|
|
174
|
+
const dir = direction === 'outgoing' ? 'out' : 'in';
|
|
175
|
+
const edges = await provider.getNeighbors(nodeId, dir, opts);
|
|
176
|
+
const tag = direction === 'outgoing' ? /** @type {const} */ ('outgoing') : /** @type {const} */ ('incoming');
|
|
177
|
+
return tagDirection(edges, tag);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Linear-scan neighbor lookup from raw CRDT state.
|
|
182
|
+
*
|
|
183
|
+
* @param {import('../services/JoinReducer.js').WarpStateV5} cachedState
|
|
184
|
+
* @param {string} nodeId
|
|
185
|
+
* @param {'outgoing' | 'incoming' | 'both'} direction
|
|
186
|
+
* @param {string} [edgeLabel]
|
|
187
|
+
* @returns {Array<{nodeId: string, label: string, direction: 'outgoing' | 'incoming'}>}
|
|
188
|
+
*/
|
|
189
|
+
function _linearNeighbors(cachedState, nodeId, direction, edgeLabel) {
|
|
190
|
+
const s = /** @type {import('../services/JoinReducer.js').WarpStateV5} */ (cachedState);
|
|
120
191
|
/** @type {Array<{nodeId: string, label: string, direction: 'outgoing' | 'incoming'}>} */
|
|
121
192
|
const result = [];
|
|
193
|
+
const checkOut = direction === 'outgoing' || direction === 'both';
|
|
194
|
+
const checkIn = direction === 'incoming' || direction === 'both';
|
|
122
195
|
|
|
123
196
|
for (const edgeKey of orsetElements(s.edgeAlive)) {
|
|
124
197
|
const { from, to, label } = decodeEdgeKey(edgeKey);
|
|
125
|
-
|
|
126
198
|
if (edgeLabel !== undefined && label !== edgeLabel) {
|
|
127
199
|
continue;
|
|
128
200
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
if (orsetContains(s.nodeAlive, to)) {
|
|
132
|
-
result.push({ nodeId: to, label, direction: /** @type {const} */ ('outgoing') });
|
|
133
|
-
}
|
|
201
|
+
if (checkOut && from === nodeId && orsetContains(s.nodeAlive, to)) {
|
|
202
|
+
result.push({ nodeId: to, label, direction: /** @type {const} */ ('outgoing') });
|
|
134
203
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
if (orsetContains(s.nodeAlive, from)) {
|
|
138
|
-
result.push({ nodeId: from, label, direction: /** @type {const} */ ('incoming') });
|
|
139
|
-
}
|
|
204
|
+
if (checkIn && to === nodeId && orsetContains(s.nodeAlive, from)) {
|
|
205
|
+
result.push({ nodeId: from, label, direction: /** @type {const} */ ('incoming') });
|
|
140
206
|
}
|
|
141
207
|
}
|
|
142
208
|
|
|
@@ -39,6 +39,7 @@ const MAX_CAS_RETRIES = 3;
|
|
|
39
39
|
* @property {string} codec - Codec identifier (e.g. 'cbor-v1')
|
|
40
40
|
* @property {number} schemaVersion - Index entry schema version
|
|
41
41
|
* @property {string} [lastAccessedAt] - ISO 8601 timestamp of last read (for LRU eviction)
|
|
42
|
+
* @property {string} [indexTreeOid] - Git tree OID of the bitmap index snapshot
|
|
42
43
|
*/
|
|
43
44
|
|
|
44
45
|
/**
|
|
@@ -203,9 +204,18 @@ export default class CasSeekCacheAdapter extends SeekCachePort {
|
|
|
203
204
|
// ---------------------------------------------------------------------------
|
|
204
205
|
|
|
205
206
|
/**
|
|
207
|
+
* Retrieves a cached state buffer by key.
|
|
208
|
+
*
|
|
209
|
+
* Note: This method reads the index twice — once here for the entry lookup,
|
|
210
|
+
* and again inside `_mutateIndex` for the `lastAccessedAt` update. The
|
|
211
|
+
* double-read is a known trade-off: `_mutateIndex` re-reads to provide
|
|
212
|
+
* CAS-safe retry semantics, and deduplicating the reads would complicate
|
|
213
|
+
* the retry logic without meaningful performance impact (the index is a
|
|
214
|
+
* single small JSON blob).
|
|
215
|
+
*
|
|
206
216
|
* @override
|
|
207
217
|
* @param {string} key
|
|
208
|
-
* @returns {Promise<Buffer|null>}
|
|
218
|
+
* @returns {Promise<{ buffer: Buffer|Uint8Array, indexTreeOid?: string } | null>}
|
|
209
219
|
*/
|
|
210
220
|
async get(key) {
|
|
211
221
|
const cas = await this._getCas();
|
|
@@ -225,7 +235,12 @@ export default class CasSeekCacheAdapter extends SeekCachePort {
|
|
|
225
235
|
}
|
|
226
236
|
return idx;
|
|
227
237
|
});
|
|
228
|
-
|
|
238
|
+
/** @type {{ buffer: Buffer|Uint8Array, indexTreeOid?: string }} */
|
|
239
|
+
const result = { buffer };
|
|
240
|
+
if (entry.indexTreeOid) {
|
|
241
|
+
result.indexTreeOid = entry.indexTreeOid;
|
|
242
|
+
}
|
|
243
|
+
return result;
|
|
229
244
|
} catch {
|
|
230
245
|
// Blob GC'd or corrupted — self-heal by removing dead entry
|
|
231
246
|
await this._mutateIndex((idx) => {
|
|
@@ -239,10 +254,11 @@ export default class CasSeekCacheAdapter extends SeekCachePort {
|
|
|
239
254
|
/**
|
|
240
255
|
* @override
|
|
241
256
|
* @param {string} key
|
|
242
|
-
* @param {Buffer} buffer
|
|
257
|
+
* @param {Buffer|Uint8Array} buffer
|
|
258
|
+
* @param {{ indexTreeOid?: string }} [options]
|
|
243
259
|
* @returns {Promise<void>}
|
|
244
260
|
*/
|
|
245
|
-
async set(key, buffer) {
|
|
261
|
+
async set(key, buffer, options) {
|
|
246
262
|
const cas = await this._getCas();
|
|
247
263
|
const { ceiling, frontierHash } = this._parseKey(key);
|
|
248
264
|
|
|
@@ -257,7 +273,8 @@ export default class CasSeekCacheAdapter extends SeekCachePort {
|
|
|
257
273
|
|
|
258
274
|
// Update index with rich metadata
|
|
259
275
|
await this._mutateIndex((index) => {
|
|
260
|
-
|
|
276
|
+
/** @type {IndexEntry} */
|
|
277
|
+
const entry = {
|
|
261
278
|
treeOid,
|
|
262
279
|
createdAt: new Date().toISOString(),
|
|
263
280
|
ceiling,
|
|
@@ -266,6 +283,10 @@ export default class CasSeekCacheAdapter extends SeekCachePort {
|
|
|
266
283
|
codec: 'cbor-v1',
|
|
267
284
|
schemaVersion: INDEX_SCHEMA_VERSION,
|
|
268
285
|
};
|
|
286
|
+
if (options?.indexTreeOid) {
|
|
287
|
+
entry.indexTreeOid = options.indexTreeOid;
|
|
288
|
+
}
|
|
289
|
+
index.entries[key] = entry;
|
|
269
290
|
return this._enforceMaxEntries(index);
|
|
270
291
|
});
|
|
271
292
|
}
|
package/src/ports/BlobPort.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
export default class BlobPort {
|
|
11
11
|
/**
|
|
12
12
|
* Writes content as a Git blob and returns its OID.
|
|
13
|
-
* @param {Buffer|string} _content - The blob content to write
|
|
13
|
+
* @param {Uint8Array|Buffer|string} _content - The blob content to write
|
|
14
14
|
* @returns {Promise<string>} The Git OID of the created blob
|
|
15
15
|
* @throws {Error} If not implemented by a concrete adapter
|
|
16
16
|
*/
|