@git-stunts/git-warp 12.1.0 → 12.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/bin/cli/commands/trust.js +37 -1
- package/bin/cli/infrastructure.js +14 -1
- package/bin/cli/schemas.js +4 -4
- package/bin/warp-graph.js +9 -2
- package/index.d.ts +18 -2
- package/package.json +1 -1
- package/src/domain/WarpGraph.js +4 -1
- package/src/domain/crdt/Dot.js +5 -0
- package/src/domain/crdt/LWW.js +3 -1
- package/src/domain/crdt/ORSet.js +63 -27
- package/src/domain/crdt/VersionVector.js +12 -0
- package/src/domain/errors/PatchError.js +27 -0
- package/src/domain/errors/StorageError.js +8 -0
- package/src/domain/errors/SyncError.js +1 -0
- package/src/domain/errors/TrustError.js +2 -0
- package/src/domain/errors/WriterError.js +5 -0
- package/src/domain/errors/index.js +1 -0
- package/src/domain/services/AuditVerifierService.js +32 -2
- package/src/domain/services/BitmapIndexBuilder.js +14 -9
- package/src/domain/services/CheckpointService.js +12 -8
- package/src/domain/services/Frontier.js +18 -0
- package/src/domain/services/GCPolicy.js +25 -4
- package/src/domain/services/GraphTraversal.js +11 -50
- package/src/domain/services/HttpSyncServer.js +18 -29
- package/src/domain/services/IncrementalIndexUpdater.js +179 -36
- package/src/domain/services/JoinReducer.js +164 -31
- package/src/domain/services/MaterializedViewService.js +13 -2
- package/src/domain/services/PatchBuilderV2.js +210 -145
- package/src/domain/services/QueryBuilder.js +67 -30
- package/src/domain/services/SyncController.js +62 -18
- package/src/domain/services/SyncPayloadSchema.js +236 -0
- package/src/domain/services/SyncProtocol.js +102 -40
- package/src/domain/services/SyncTrustGate.js +146 -0
- package/src/domain/services/TranslationCost.js +2 -2
- package/src/domain/trust/TrustRecordService.js +161 -34
- package/src/domain/utils/CachedValue.js +34 -5
- package/src/domain/utils/EventId.js +4 -1
- package/src/domain/utils/LRUCache.js +3 -1
- package/src/domain/utils/RefLayout.js +4 -0
- package/src/domain/utils/canonicalStringify.js +48 -18
- package/src/domain/utils/matchGlob.js +7 -0
- package/src/domain/warp/PatchSession.js +30 -24
- package/src/domain/warp/Writer.js +12 -5
- package/src/domain/warp/_wiredMethods.d.ts +1 -1
- package/src/domain/warp/checkpoint.methods.js +102 -16
- package/src/domain/warp/materialize.methods.js +47 -5
- package/src/domain/warp/materializeAdvanced.methods.js +52 -10
- package/src/domain/warp/patch.methods.js +24 -8
- package/src/domain/warp/query.methods.js +4 -4
- package/src/domain/warp/subscribe.methods.js +11 -19
- package/src/infrastructure/adapters/GitGraphAdapter.js +57 -54
- package/src/infrastructure/codecs/CborCodec.js +2 -0
- package/src/domain/utils/fnv1a.js +0 -20
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* @see WARP Spec Section 10
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
import {
|
|
14
|
+
import { computeStateHashV5 } from './StateSerializerV5.js';
|
|
15
15
|
import {
|
|
16
16
|
serializeFullStateV5,
|
|
17
17
|
deserializeFullStateV5,
|
|
@@ -86,7 +86,6 @@ function partitionTreeOids(rawOids) {
|
|
|
86
86
|
* ```
|
|
87
87
|
* <checkpoint_commit_tree>/
|
|
88
88
|
* ├── state.cbor # AUTHORITATIVE: Full V5 state (ORSets + props)
|
|
89
|
-
* ├── visible.cbor # CACHE ONLY: Visible projection for fast queries
|
|
90
89
|
* ├── frontier.cbor # Writer frontiers
|
|
91
90
|
* ├── appliedVV.cbor # Version vector of dots in state
|
|
92
91
|
* └── provenanceIndex.cbor # Optional: node-to-patchSha index (HG/IO/2)
|
|
@@ -116,7 +115,6 @@ export async function create({ persistence, graphName, state, frontier, parents
|
|
|
116
115
|
* ```
|
|
117
116
|
* <checkpoint_tree>/
|
|
118
117
|
* ├── state.cbor # AUTHORITATIVE: Full V5 state (ORSets + props)
|
|
119
|
-
* ├── visible.cbor # CACHE ONLY: Visible projection for fast queries
|
|
120
118
|
* ├── frontier.cbor # Writer frontiers
|
|
121
119
|
* ├── appliedVV.cbor # Version vector of dots in state
|
|
122
120
|
* └── provenanceIndex.cbor # Optional: node-to-patchSha index (HG/IO/2)
|
|
@@ -150,7 +148,9 @@ export async function createV5({
|
|
|
150
148
|
// 1. Compute appliedVV from actual state dots
|
|
151
149
|
const appliedVV = computeAppliedVV(state);
|
|
152
150
|
|
|
153
|
-
// 2. Optionally compact (only tombstoned dots <= appliedVV)
|
|
151
|
+
// 2. Optionally compact (only tombstoned dots <= appliedVV).
|
|
152
|
+
// When compact=false, checkpointState aliases the caller's state but the
|
|
153
|
+
// remaining path is read-only (serialize + hash), so no clone is needed.
|
|
154
154
|
let checkpointState = state;
|
|
155
155
|
if (compact) {
|
|
156
156
|
checkpointState = cloneStateV5(state);
|
|
@@ -161,8 +161,7 @@ export async function createV5({
|
|
|
161
161
|
// 3. Serialize full state (AUTHORITATIVE)
|
|
162
162
|
const stateBuffer = serializeFullStateV5(checkpointState, { codec });
|
|
163
163
|
|
|
164
|
-
// 4.
|
|
165
|
-
const visibleBuffer = serializeStateV5(checkpointState, { codec });
|
|
164
|
+
// 4. Compute state hash
|
|
166
165
|
const stateHash = await computeStateHashV5(checkpointState, { codec, crypto: /** @type {import('../../ports/CryptoPort.js').default} */ (crypto) });
|
|
167
166
|
|
|
168
167
|
// 5. Serialize frontier and appliedVV
|
|
@@ -171,7 +170,6 @@ export async function createV5({
|
|
|
171
170
|
|
|
172
171
|
// 6. Write blobs to git
|
|
173
172
|
const stateBlobOid = await persistence.writeBlob(/** @type {Buffer} */ (stateBuffer));
|
|
174
|
-
const visibleBlobOid = await persistence.writeBlob(/** @type {Buffer} */ (visibleBuffer));
|
|
175
173
|
const frontierBlobOid = await persistence.writeBlob(/** @type {Buffer} */ (frontierBuffer));
|
|
176
174
|
const appliedVVBlobOid = await persistence.writeBlob(/** @type {Buffer} */ (appliedVVBuffer));
|
|
177
175
|
|
|
@@ -192,6 +190,11 @@ export async function createV5({
|
|
|
192
190
|
// If patch commits are ever pruned, content blobs remain reachable via
|
|
193
191
|
// the checkpoint tree. Without this, git gc would nuke content blobs
|
|
194
192
|
// whose only anchor was the (now-pruned) patch commit tree.
|
|
193
|
+
//
|
|
194
|
+
// O(P) scan over all properties — acceptable because checkpoint creation
|
|
195
|
+
// is infrequent. The property key format is deterministic (encodePropKey /
|
|
196
|
+
// encodeEdgePropKey), but content keys are interleaved with regular keys
|
|
197
|
+
// so no prefix filter can skip non-content entries without decoding.
|
|
195
198
|
const contentOids = new Set();
|
|
196
199
|
for (const [propKey, register] of checkpointState.prop) {
|
|
197
200
|
const { propKey: decodedKey } = isEdgePropKey(propKey)
|
|
@@ -207,7 +210,6 @@ export async function createV5({
|
|
|
207
210
|
`100644 blob ${appliedVVBlobOid}\tappliedVV.cbor`,
|
|
208
211
|
`100644 blob ${frontierBlobOid}\tfrontier.cbor`,
|
|
209
212
|
`100644 blob ${stateBlobOid}\tstate.cbor`,
|
|
210
|
-
`100644 blob ${visibleBlobOid}\tvisible.cbor`,
|
|
211
213
|
];
|
|
212
214
|
|
|
213
215
|
// Add provenance index if present
|
|
@@ -240,6 +242,8 @@ export async function createV5({
|
|
|
240
242
|
stateHash,
|
|
241
243
|
frontierOid: frontierBlobOid,
|
|
242
244
|
indexOid: treeOid,
|
|
245
|
+
// Schema 3 was used for edge-property-aware patches but is never emitted
|
|
246
|
+
// by checkpoint creation. Schema 4 indicates an index tree is present.
|
|
243
247
|
schema: indexTree ? 4 : 2,
|
|
244
248
|
});
|
|
245
249
|
|
|
@@ -91,6 +91,24 @@ export function cloneFrontier(frontier) {
|
|
|
91
91
|
return new Map(frontier);
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
/**
|
|
95
|
+
* Produces a stable, deterministic fingerprint of a frontier.
|
|
96
|
+
*
|
|
97
|
+
* Sorts entries by writer ID and JSON-stringifies the sorted pairs.
|
|
98
|
+
* Two frontiers produce the same fingerprint iff they have identical
|
|
99
|
+
* writer→SHA mappings. Used for snapshot isolation checks (B63)
|
|
100
|
+
* and diagnostic logging.
|
|
101
|
+
*
|
|
102
|
+
* @param {Frontier} frontier
|
|
103
|
+
* @returns {string} Deterministic JSON string of sorted entries
|
|
104
|
+
*/
|
|
105
|
+
export function frontierFingerprint(frontier) {
|
|
106
|
+
const sorted = [...frontier.entries()].sort(
|
|
107
|
+
([a], [b]) => (a < b ? -1 : a > b ? 1 : 0),
|
|
108
|
+
);
|
|
109
|
+
return JSON.stringify(sorted);
|
|
110
|
+
}
|
|
111
|
+
|
|
94
112
|
/**
|
|
95
113
|
* Merges two frontiers, taking the "later" entry for each writer.
|
|
96
114
|
* Note: This is a simple merge that takes entries from both.
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { orsetCompact } from '../crdt/ORSet.js';
|
|
6
6
|
import { collectGCMetrics } from './GCMetrics.js';
|
|
7
|
+
import WarpError from '../errors/WarpError.js';
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* @typedef {Object} GCPolicy
|
|
@@ -92,21 +93,41 @@ export function shouldRunGC(metrics, policy) {
|
|
|
92
93
|
|
|
93
94
|
/**
|
|
94
95
|
* Executes GC on state. Only compacts tombstoned dots <= appliedVV.
|
|
95
|
-
* Mutates state in place
|
|
96
|
+
* Mutates state **in place** — callers must clone-then-swap to preserve
|
|
97
|
+
* a rollback copy (see CheckpointService for the canonical pattern).
|
|
96
98
|
*
|
|
97
99
|
* @param {import('./JoinReducer.js').WarpStateV5} state - State to compact (mutated!)
|
|
98
100
|
* @param {import('../crdt/VersionVector.js').VersionVector} appliedVV - Version vector cutoff
|
|
99
101
|
* @returns {GCExecuteResult}
|
|
102
|
+
* @throws {WarpError} E_GC_INVALID_VV if appliedVV is not a Map
|
|
103
|
+
* @throws {WarpError} E_GC_COMPACT_FAILED if orsetCompact throws
|
|
100
104
|
*/
|
|
101
105
|
export function executeGC(state, appliedVV) {
|
|
106
|
+
if (!(appliedVV instanceof Map)) {
|
|
107
|
+
throw new WarpError(
|
|
108
|
+
'executeGC requires appliedVV to be a Map (VersionVector)',
|
|
109
|
+
'E_GC_INVALID_VV',
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
|
|
102
113
|
const startTime = performance.now();
|
|
103
114
|
|
|
104
115
|
// Collect metrics before compaction
|
|
105
116
|
const beforeMetrics = collectGCMetrics(state);
|
|
106
117
|
|
|
107
|
-
// Compact both ORSets
|
|
108
|
-
|
|
109
|
-
|
|
118
|
+
// Compact both ORSets — wrap each phase so partial failure is diagnosable
|
|
119
|
+
let nodesDone = false;
|
|
120
|
+
try {
|
|
121
|
+
orsetCompact(state.nodeAlive, appliedVV);
|
|
122
|
+
nodesDone = true;
|
|
123
|
+
orsetCompact(state.edgeAlive, appliedVV);
|
|
124
|
+
} catch {
|
|
125
|
+
throw new WarpError(
|
|
126
|
+
`GC compaction failed during ${nodesDone ? 'edgeAlive' : 'nodeAlive'} phase`,
|
|
127
|
+
'E_GC_COMPACT_FAILED',
|
|
128
|
+
{ context: { phase: nodesDone ? 'edgeAlive' : 'nodeAlive', partialCompaction: nodesDone } },
|
|
129
|
+
);
|
|
130
|
+
}
|
|
110
131
|
|
|
111
132
|
// Collect metrics after compaction
|
|
112
133
|
const afterMetrics = collectGCMetrics(state);
|
|
@@ -165,7 +165,9 @@ export default class GraphTraversal {
|
|
|
165
165
|
return await this._provider.getNeighbors(nodeId, direction, options);
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
-
const labelsKey = options?.labels
|
|
168
|
+
const labelsKey = options?.labels
|
|
169
|
+
? [...options.labels].sort().join('\0')
|
|
170
|
+
: '*';
|
|
169
171
|
const key = `${nodeId}\0${direction}\0${labelsKey}`;
|
|
170
172
|
const cached = cache.get(key);
|
|
171
173
|
if (cached !== undefined) {
|
|
@@ -830,52 +832,38 @@ export default class GraphTraversal {
|
|
|
830
832
|
}
|
|
831
833
|
}
|
|
832
834
|
|
|
833
|
-
// Phase 2: Kahn's —
|
|
834
|
-
|
|
835
|
-
const ready = [];
|
|
835
|
+
// Phase 2: Kahn's — MinHeap for O(N log N) zero-indegree processing
|
|
836
|
+
const ready = new MinHeap({ tieBreaker: lexTieBreaker });
|
|
836
837
|
for (const nodeId of discovered) {
|
|
837
838
|
if ((inDegree.get(nodeId) || 0) === 0) {
|
|
838
|
-
ready.
|
|
839
|
+
ready.insert(nodeId, 0);
|
|
839
840
|
}
|
|
840
841
|
}
|
|
841
|
-
ready.sort(lexTieBreaker);
|
|
842
842
|
|
|
843
|
+
/** @type {string[]} */
|
|
843
844
|
const sorted = [];
|
|
844
|
-
|
|
845
|
-
while (rHead < ready.length && sorted.length < maxNodes) {
|
|
845
|
+
while (!ready.isEmpty() && sorted.length < maxNodes) {
|
|
846
846
|
if (sorted.length % 1000 === 0) {
|
|
847
847
|
checkAborted(signal, 'topologicalSort');
|
|
848
848
|
}
|
|
849
|
-
const nodeId = /** @type {string} */ (ready
|
|
849
|
+
const nodeId = /** @type {string} */ (ready.extractMin());
|
|
850
850
|
sorted.push(nodeId);
|
|
851
851
|
|
|
852
852
|
const neighbors = adjList.get(nodeId) || [];
|
|
853
|
-
/** @type {string[]} */
|
|
854
|
-
const newlyReady = [];
|
|
855
853
|
for (const neighborId of neighbors) {
|
|
856
854
|
const deg = /** @type {number} */ (inDegree.get(neighborId)) - 1;
|
|
857
855
|
inDegree.set(neighborId, deg);
|
|
858
856
|
if (deg === 0) {
|
|
859
|
-
|
|
857
|
+
ready.insert(neighborId, 0);
|
|
860
858
|
}
|
|
861
859
|
}
|
|
862
|
-
// Insert newly ready nodes in sorted position
|
|
863
|
-
if (newlyReady.length > 0) {
|
|
864
|
-
newlyReady.sort(lexTieBreaker);
|
|
865
|
-
// Compact consumed prefix before merge to keep rHead at 0
|
|
866
|
-
if (rHead > 0) {
|
|
867
|
-
ready.splice(0, rHead);
|
|
868
|
-
rHead = 0;
|
|
869
|
-
}
|
|
870
|
-
this._insertSorted(ready, newlyReady);
|
|
871
|
-
}
|
|
872
860
|
}
|
|
873
861
|
|
|
874
862
|
const hasCycle = computeTopoHasCycle({
|
|
875
863
|
sortedLength: sorted.length,
|
|
876
864
|
discoveredSize: discovered.size,
|
|
877
865
|
maxNodes,
|
|
878
|
-
readyRemaining:
|
|
866
|
+
readyRemaining: !ready.isEmpty(),
|
|
879
867
|
});
|
|
880
868
|
if (hasCycle && throwOnCycle) {
|
|
881
869
|
// Find a back-edge as witness
|
|
@@ -1209,31 +1197,4 @@ export default class GraphTraversal {
|
|
|
1209
1197
|
return candidatePred < current;
|
|
1210
1198
|
}
|
|
1211
1199
|
|
|
1212
|
-
/**
|
|
1213
|
-
* Inserts sorted items into a sorted array maintaining order.
|
|
1214
|
-
* Both input arrays must be sorted by lexTieBreaker.
|
|
1215
|
-
*
|
|
1216
|
-
* @param {string[]} target - Sorted array to insert into (mutated in place)
|
|
1217
|
-
* @param {string[]} items - Sorted items to insert
|
|
1218
|
-
* @private
|
|
1219
|
-
*/
|
|
1220
|
-
_insertSorted(target, items) {
|
|
1221
|
-
// O(n+k) merge: build merged array from two sorted inputs
|
|
1222
|
-
const merged = [];
|
|
1223
|
-
let ti = 0;
|
|
1224
|
-
let ii = 0;
|
|
1225
|
-
while (ti < target.length && ii < items.length) {
|
|
1226
|
-
if (target[ti] <= items[ii]) {
|
|
1227
|
-
merged.push(target[ti++]);
|
|
1228
|
-
} else {
|
|
1229
|
-
merged.push(items[ii++]);
|
|
1230
|
-
}
|
|
1231
|
-
}
|
|
1232
|
-
while (ti < target.length) { merged.push(target[ti++]); }
|
|
1233
|
-
while (ii < items.length) { merged.push(items[ii++]); }
|
|
1234
|
-
target.length = 0;
|
|
1235
|
-
for (let i = 0; i < merged.length; i++) {
|
|
1236
|
-
target.push(merged[i]);
|
|
1237
|
-
}
|
|
1238
|
-
}
|
|
1239
1200
|
}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
import { z } from 'zod';
|
|
12
12
|
import SyncAuthService from './SyncAuthService.js';
|
|
13
|
+
import { validateSyncRequest } from './SyncPayloadSchema.js';
|
|
13
14
|
|
|
14
15
|
const DEFAULT_MAX_REQUEST_BYTES = 4 * 1024 * 1024;
|
|
15
16
|
const MAX_REQUEST_BYTES_CEILING = 128 * 1024 * 1024; // 134217728
|
|
@@ -117,26 +118,7 @@ function jsonResponse(data) {
|
|
|
117
118
|
};
|
|
118
119
|
}
|
|
119
120
|
|
|
120
|
-
|
|
121
|
-
* Validates that a sync request object has the expected shape.
|
|
122
|
-
*
|
|
123
|
-
* @param {unknown} parsed - Parsed JSON body
|
|
124
|
-
* @returns {boolean} True if valid
|
|
125
|
-
* @private
|
|
126
|
-
*/
|
|
127
|
-
function isValidSyncRequest(parsed) {
|
|
128
|
-
if (!parsed || typeof parsed !== 'object') {
|
|
129
|
-
return false;
|
|
130
|
-
}
|
|
131
|
-
const rec = /** @type {Record<string, unknown>} */ (parsed);
|
|
132
|
-
if (rec.type !== 'sync-request') {
|
|
133
|
-
return false;
|
|
134
|
-
}
|
|
135
|
-
if (!rec.frontier || typeof rec.frontier !== 'object' || Array.isArray(rec.frontier)) {
|
|
136
|
-
return false;
|
|
137
|
-
}
|
|
138
|
-
return true;
|
|
139
|
-
}
|
|
121
|
+
// isValidSyncRequest replaced by SyncPayloadSchema.validateSyncRequest (B64)
|
|
140
122
|
|
|
141
123
|
/**
|
|
142
124
|
* Checks the content-type header. Returns an error response if the
|
|
@@ -200,6 +182,7 @@ function checkBodySize(body, maxBytes) {
|
|
|
200
182
|
|
|
201
183
|
/**
|
|
202
184
|
* Parses and validates the request body as a sync request.
|
|
185
|
+
* Uses Zod-based SyncPayloadSchema for shape + resource limit validation.
|
|
203
186
|
*
|
|
204
187
|
* @param {Buffer|undefined} body
|
|
205
188
|
* @returns {{ error: { status: number, headers: Object, body: string }, parsed: null } | { error: null, parsed: import('./SyncProtocol.js').SyncRequest }}
|
|
@@ -215,11 +198,12 @@ function parseBody(body) {
|
|
|
215
198
|
return { error: errorResponse(400, 'Invalid JSON'), parsed: null };
|
|
216
199
|
}
|
|
217
200
|
|
|
218
|
-
|
|
219
|
-
|
|
201
|
+
const validation = validateSyncRequest(parsed);
|
|
202
|
+
if (!validation.ok) {
|
|
203
|
+
return { error: errorResponse(400, `Invalid sync request: ${validation.error}`), parsed: null };
|
|
220
204
|
}
|
|
221
205
|
|
|
222
|
-
return { error: null, parsed };
|
|
206
|
+
return { error: null, parsed: /** @type {import('./SyncProtocol.js').SyncRequest} */ (validation.value) };
|
|
223
207
|
}
|
|
224
208
|
|
|
225
209
|
/**
|
|
@@ -298,12 +282,17 @@ export default class HttpSyncServer {
|
|
|
298
282
|
this._auth.recordLogOnlyPassthrough();
|
|
299
283
|
}
|
|
300
284
|
|
|
301
|
-
// Writer whitelist
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
285
|
+
// Writer whitelist: for sync-requests, extract writer IDs from frontier
|
|
286
|
+
// keys (the writers the peer claims to have). Sync-requests don't carry
|
|
287
|
+
// patches — the server generates the response. For sync-responses with
|
|
288
|
+
// patches, trust-gate should be on patch authors (handled client-side).
|
|
289
|
+
if (parsed.frontier && typeof parsed.frontier === 'object') {
|
|
290
|
+
const writerIds = Object.keys(/** @type {Record<string, string>} */ (parsed.frontier));
|
|
291
|
+
if (writerIds.length > 0) {
|
|
292
|
+
const writerResult = this._auth.enforceWriters(writerIds);
|
|
293
|
+
if (!writerResult.ok) {
|
|
294
|
+
return errorResponse(writerResult.status, writerResult.reason);
|
|
295
|
+
}
|
|
307
296
|
}
|
|
308
297
|
}
|
|
309
298
|
|
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Stateful service that computes dirty shard buffers from a PatchDiff.
|
|
3
3
|
*
|
|
4
4
|
* Given a diff of alive-ness transitions + a shard loader for the existing
|
|
5
5
|
* index tree, produces only the shard buffers that changed. The caller
|
|
6
6
|
* merges them back into the tree via `{ ...existingTree, ...dirtyShards }`.
|
|
7
7
|
*
|
|
8
|
+
* Instance state:
|
|
9
|
+
* - `_edgeAdjacencyCache` stores a WeakMap keyed by `state.edgeAlive` ORSet
|
|
10
|
+
* identity, mapping nodeId -> incident alive edge keys.
|
|
11
|
+
* - Cache lifetime is tied to the updater instance and is reconciled per diff
|
|
12
|
+
* once initialized. Reuse one instance for a single linear state stream;
|
|
13
|
+
* create a new instance to reset cache state across independent streams.
|
|
14
|
+
*
|
|
8
15
|
* @module domain/services/IncrementalIndexUpdater
|
|
9
16
|
*/
|
|
10
17
|
|
|
@@ -40,6 +47,15 @@ export default class IncrementalIndexUpdater {
|
|
|
40
47
|
*/
|
|
41
48
|
constructor({ codec } = {}) {
|
|
42
49
|
this._codec = codec || defaultCodec;
|
|
50
|
+
/** @type {WeakMap<import('../crdt/ORSet.js').ORSet, Map<string, Set<string>>>} */
|
|
51
|
+
this._edgeAdjacencyCache = new WeakMap();
|
|
52
|
+
/**
|
|
53
|
+
* Cached next label ID — avoids O(L) max-scan per new label.
|
|
54
|
+
* Initialized lazily from existing labels on first _ensureLabel call.
|
|
55
|
+
* @type {number|null}
|
|
56
|
+
* @private
|
|
57
|
+
*/
|
|
58
|
+
this._nextLabelId = null;
|
|
43
59
|
}
|
|
44
60
|
|
|
45
61
|
/**
|
|
@@ -63,8 +79,22 @@ export default class IncrementalIndexUpdater {
|
|
|
63
79
|
const out = {};
|
|
64
80
|
|
|
65
81
|
const labels = this._loadLabels(loadShard);
|
|
82
|
+
// Reset cached next label ID so _ensureLabel re-scans the fresh labels
|
|
83
|
+
// object loaded above. Without this, a stale _nextLabelId from a prior
|
|
84
|
+
// applyDiff call could collide with IDs already present in the new labels.
|
|
85
|
+
this._nextLabelId = null;
|
|
66
86
|
let labelsDirty = false;
|
|
67
87
|
|
|
88
|
+
// Determine which added nodes are true re-adds (already have global IDs).
|
|
89
|
+
// Brand-new nodes cannot have pre-existing indexed edges to restore.
|
|
90
|
+
const readdedNodes = new Set();
|
|
91
|
+
for (const nodeId of diff.nodesAdded) {
|
|
92
|
+
const meta = this._getOrLoadMeta(computeShardKey(nodeId), metaCache, loadShard);
|
|
93
|
+
if (this._findGlobalId(meta, nodeId) !== undefined) {
|
|
94
|
+
readdedNodes.add(nodeId);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
68
98
|
for (const nodeId of diff.nodesAdded) {
|
|
69
99
|
this._handleNodeAdd(nodeId, metaCache, loadShard);
|
|
70
100
|
}
|
|
@@ -96,25 +126,22 @@ export default class IncrementalIndexUpdater {
|
|
|
96
126
|
this._handleEdgeRemove(edge, labels, metaCache, fwdCache, revCache, loadShard);
|
|
97
127
|
}
|
|
98
128
|
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
// edges
|
|
107
|
-
//
|
|
108
|
-
|
|
109
|
-
|
|
129
|
+
// Keep adjacency cache in sync for every diff once initialized, so later
|
|
130
|
+
// re-add restores never consult stale edge membership.
|
|
131
|
+
let readdAdjacency = null;
|
|
132
|
+
if (readdedNodes.size > 0 || this._edgeAdjacencyCache.has(state.edgeAlive)) {
|
|
133
|
+
readdAdjacency = this._getOrBuildAliveEdgeAdjacency(state, diff);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Restore edges for re-added nodes only. When a node transitions
|
|
137
|
+
// not-alive -> alive, alive OR-Set edges touching it become visible again.
|
|
138
|
+
// Brand-new nodes are skipped because they have no prior global ID.
|
|
139
|
+
if (readdedNodes.size > 0 && readdAdjacency) {
|
|
110
140
|
const diffEdgeSet = new Set(
|
|
111
141
|
diff.edgesAdded.map((e) => `${e.from}\0${e.to}\0${e.label}`),
|
|
112
142
|
);
|
|
113
|
-
for (const edgeKey of
|
|
143
|
+
for (const edgeKey of this._collectReaddedEdgeKeys(readdAdjacency, readdedNodes)) {
|
|
114
144
|
const { from, to, label } = decodeEdgeKey(edgeKey);
|
|
115
|
-
if (!addedSet.has(from) && !addedSet.has(to)) {
|
|
116
|
-
continue;
|
|
117
|
-
}
|
|
118
145
|
if (!orsetContains(state.nodeAlive, from) || !orsetContains(state.nodeAlive, to)) {
|
|
119
146
|
continue;
|
|
120
147
|
}
|
|
@@ -255,15 +282,16 @@ export default class IncrementalIndexUpdater {
|
|
|
255
282
|
for (const bucket of Object.keys(fwdData)) {
|
|
256
283
|
const gidStr = String(deadGid);
|
|
257
284
|
if (fwdData[bucket] && fwdData[bucket][gidStr]) {
|
|
258
|
-
//
|
|
259
|
-
const
|
|
285
|
+
// Deserialize once, collect targets, then clear+serialize in place.
|
|
286
|
+
const bm = RoaringBitmap32.deserialize(
|
|
260
287
|
toBytes(fwdData[bucket][gidStr]),
|
|
261
288
|
true,
|
|
262
|
-
)
|
|
289
|
+
);
|
|
290
|
+
const targets = bm.toArray();
|
|
263
291
|
|
|
264
292
|
// Clear this node's outgoing bitmap
|
|
265
|
-
|
|
266
|
-
fwdData[bucket][gidStr] =
|
|
293
|
+
bm.clear();
|
|
294
|
+
fwdData[bucket][gidStr] = bm.serialize(true);
|
|
267
295
|
|
|
268
296
|
// Remove deadGid from each target's reverse bitmap
|
|
269
297
|
for (const targetGid of targets) {
|
|
@@ -273,12 +301,12 @@ export default class IncrementalIndexUpdater {
|
|
|
273
301
|
const revData = this._getOrLoadEdgeShard(revCache, 'rev', targetShard, loadShard);
|
|
274
302
|
const targetGidStr = String(targetGid);
|
|
275
303
|
if (revData[bucket] && revData[bucket][targetGidStr]) {
|
|
276
|
-
const
|
|
304
|
+
const targetBm = RoaringBitmap32.deserialize(
|
|
277
305
|
toBytes(revData[bucket][targetGidStr]),
|
|
278
306
|
true,
|
|
279
307
|
);
|
|
280
|
-
|
|
281
|
-
revData[bucket][targetGidStr] =
|
|
308
|
+
targetBm.remove(deadGid);
|
|
309
|
+
revData[bucket][targetGidStr] = targetBm.serialize(true);
|
|
282
310
|
}
|
|
283
311
|
}
|
|
284
312
|
}
|
|
@@ -290,13 +318,14 @@ export default class IncrementalIndexUpdater {
|
|
|
290
318
|
for (const bucket of Object.keys(revData)) {
|
|
291
319
|
const gidStr = String(deadGid);
|
|
292
320
|
if (revData[bucket] && revData[bucket][gidStr]) {
|
|
293
|
-
const
|
|
321
|
+
const bm = RoaringBitmap32.deserialize(
|
|
294
322
|
toBytes(revData[bucket][gidStr]),
|
|
295
323
|
true,
|
|
296
|
-
)
|
|
324
|
+
);
|
|
325
|
+
const sources = bm.toArray();
|
|
297
326
|
|
|
298
|
-
|
|
299
|
-
revData[bucket][gidStr] =
|
|
327
|
+
bm.clear();
|
|
328
|
+
revData[bucket][gidStr] = bm.serialize(true);
|
|
300
329
|
|
|
301
330
|
// Remove deadGid from each source's forward bitmap
|
|
302
331
|
for (const sourceGid of sources) {
|
|
@@ -306,12 +335,12 @@ export default class IncrementalIndexUpdater {
|
|
|
306
335
|
const fwdDataPeer = this._getOrLoadEdgeShard(fwdCache, 'fwd', sourceShard, loadShard);
|
|
307
336
|
const sourceGidStr = String(sourceGid);
|
|
308
337
|
if (fwdDataPeer[bucket] && fwdDataPeer[bucket][sourceGidStr]) {
|
|
309
|
-
const
|
|
338
|
+
const sourceBm = RoaringBitmap32.deserialize(
|
|
310
339
|
toBytes(fwdDataPeer[bucket][sourceGidStr]),
|
|
311
340
|
true,
|
|
312
341
|
);
|
|
313
|
-
|
|
314
|
-
fwdDataPeer[bucket][sourceGidStr] =
|
|
342
|
+
sourceBm.remove(deadGid);
|
|
343
|
+
fwdDataPeer[bucket][sourceGidStr] = sourceBm.serialize(true);
|
|
315
344
|
}
|
|
316
345
|
}
|
|
317
346
|
}
|
|
@@ -350,13 +379,19 @@ export default class IncrementalIndexUpdater {
|
|
|
350
379
|
if (Object.prototype.hasOwnProperty.call(labels, label)) {
|
|
351
380
|
return false;
|
|
352
381
|
}
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
382
|
+
// Lazily initialize _nextLabelId from existing labels (O(L) once),
|
|
383
|
+
// then O(1) per subsequent new label.
|
|
384
|
+
if (this._nextLabelId === null) {
|
|
385
|
+
let maxId = -1;
|
|
386
|
+
for (const id of Object.values(labels)) {
|
|
387
|
+
if (id > maxId) {
|
|
388
|
+
maxId = id;
|
|
389
|
+
}
|
|
357
390
|
}
|
|
391
|
+
this._nextLabelId = maxId + 1;
|
|
358
392
|
}
|
|
359
|
-
labels[label] =
|
|
393
|
+
labels[label] = this._nextLabelId;
|
|
394
|
+
this._nextLabelId++;
|
|
360
395
|
return true;
|
|
361
396
|
}
|
|
362
397
|
|
|
@@ -743,6 +778,111 @@ export default class IncrementalIndexUpdater {
|
|
|
743
778
|
return meta.nodeToGlobalMap.get(nodeId);
|
|
744
779
|
}
|
|
745
780
|
|
|
781
|
+
/**
|
|
782
|
+
* Collects alive edge keys incident to re-added nodes.
|
|
783
|
+
*
|
|
784
|
+
* Uses an ORSet-keyed adjacency cache so repeated updates can enumerate
|
|
785
|
+
* candidates by degree rather than scanning all alive edges each time.
|
|
786
|
+
*
|
|
787
|
+
* @param {Map<string, Set<string>>} adjacency
|
|
788
|
+
* @param {Set<string>} readdedNodes
|
|
789
|
+
* @returns {Set<string>}
|
|
790
|
+
* @private
|
|
791
|
+
*/
|
|
792
|
+
_collectReaddedEdgeKeys(adjacency, readdedNodes) {
|
|
793
|
+
const keys = new Set();
|
|
794
|
+
for (const nodeId of readdedNodes) {
|
|
795
|
+
const incident = adjacency.get(nodeId);
|
|
796
|
+
if (!incident) {
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
799
|
+
for (const edgeKey of incident) {
|
|
800
|
+
keys.add(edgeKey);
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
return keys;
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
/**
|
|
807
|
+
* Gets or builds a node -> alive edgeKey adjacency map for state.edgeAlive.
|
|
808
|
+
*
|
|
809
|
+
* For cached maps, applies diff edge transitions to keep membership current.
|
|
810
|
+
*
|
|
811
|
+
* @param {import('./JoinReducer.js').WarpStateV5} state
|
|
812
|
+
* @param {import('../types/PatchDiff.js').PatchDiff} diff
|
|
813
|
+
* @returns {Map<string, Set<string>>}
|
|
814
|
+
* @private
|
|
815
|
+
*/
|
|
816
|
+
_getOrBuildAliveEdgeAdjacency(state, diff) {
|
|
817
|
+
const { edgeAlive } = state;
|
|
818
|
+
let adjacency = this._edgeAdjacencyCache.get(edgeAlive);
|
|
819
|
+
if (!adjacency) {
|
|
820
|
+
adjacency = new Map();
|
|
821
|
+
for (const edgeKey of orsetElements(edgeAlive)) {
|
|
822
|
+
const { from, to } = decodeEdgeKey(edgeKey);
|
|
823
|
+
this._addEdgeKeyToAdjacency(adjacency, from, edgeKey);
|
|
824
|
+
this._addEdgeKeyToAdjacency(adjacency, to, edgeKey);
|
|
825
|
+
}
|
|
826
|
+
this._edgeAdjacencyCache.set(edgeAlive, adjacency);
|
|
827
|
+
return adjacency;
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
for (const edge of diff.edgesAdded) {
|
|
831
|
+
const edgeKey = `${edge.from}\0${edge.to}\0${edge.label}`;
|
|
832
|
+
if (!orsetContains(edgeAlive, edgeKey)) {
|
|
833
|
+
continue;
|
|
834
|
+
}
|
|
835
|
+
this._addEdgeKeyToAdjacency(adjacency, edge.from, edgeKey);
|
|
836
|
+
this._addEdgeKeyToAdjacency(adjacency, edge.to, edgeKey);
|
|
837
|
+
}
|
|
838
|
+
for (const edge of diff.edgesRemoved) {
|
|
839
|
+
const edgeKey = `${edge.from}\0${edge.to}\0${edge.label}`;
|
|
840
|
+
if (orsetContains(edgeAlive, edgeKey)) {
|
|
841
|
+
continue;
|
|
842
|
+
}
|
|
843
|
+
this._removeEdgeKeyFromAdjacency(adjacency, edge.from, edgeKey);
|
|
844
|
+
this._removeEdgeKeyFromAdjacency(adjacency, edge.to, edgeKey);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
return adjacency;
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Adds an edge key to one endpoint's adjacency set.
|
|
852
|
+
*
|
|
853
|
+
* @param {Map<string, Set<string>>} adjacency
|
|
854
|
+
* @param {string} nodeId
|
|
855
|
+
* @param {string} edgeKey
|
|
856
|
+
* @private
|
|
857
|
+
*/
|
|
858
|
+
_addEdgeKeyToAdjacency(adjacency, nodeId, edgeKey) {
|
|
859
|
+
let set = adjacency.get(nodeId);
|
|
860
|
+
if (!set) {
|
|
861
|
+
set = new Set();
|
|
862
|
+
adjacency.set(nodeId, set);
|
|
863
|
+
}
|
|
864
|
+
set.add(edgeKey);
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
/**
|
|
868
|
+
* Removes an edge key from one endpoint's adjacency set.
|
|
869
|
+
*
|
|
870
|
+
* @param {Map<string, Set<string>>} adjacency
|
|
871
|
+
* @param {string} nodeId
|
|
872
|
+
* @param {string} edgeKey
|
|
873
|
+
* @private
|
|
874
|
+
*/
|
|
875
|
+
_removeEdgeKeyFromAdjacency(adjacency, nodeId, edgeKey) {
|
|
876
|
+
const set = adjacency.get(nodeId);
|
|
877
|
+
if (!set) {
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
set.delete(edgeKey);
|
|
881
|
+
if (set.size === 0) {
|
|
882
|
+
adjacency.delete(nodeId);
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
|
|
746
886
|
/**
|
|
747
887
|
* Deserializes a bitmap from edge shard data, or creates a new one.
|
|
748
888
|
*
|
|
@@ -753,6 +893,9 @@ export default class IncrementalIndexUpdater {
|
|
|
753
893
|
* @private
|
|
754
894
|
*/
|
|
755
895
|
_deserializeBitmap(data, bucket, ownerStr) {
|
|
896
|
+
// getRoaringBitmap32() is internally memoized (returns cached constructor
|
|
897
|
+
// after first resolution). The repeated calls are cheap but the pattern
|
|
898
|
+
// is noisy. A future cleanup could cache the constructor at instance level.
|
|
756
899
|
const RoaringBitmap32 = getRoaringBitmap32();
|
|
757
900
|
if (data[bucket] && data[bucket][ownerStr]) {
|
|
758
901
|
return RoaringBitmap32.deserialize(
|