@git-stunts/git-warp 11.5.1 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -10
- package/bin/cli/commands/registry.js +4 -0
- package/bin/cli/commands/reindex.js +41 -0
- package/bin/cli/commands/verify-index.js +59 -0
- package/bin/cli/infrastructure.js +7 -2
- package/bin/cli/schemas.js +19 -0
- package/bin/cli/types.js +2 -0
- package/index.d.ts +49 -12
- package/package.json +2 -2
- package/src/domain/WarpGraph.js +40 -0
- package/src/domain/errors/ShardIdOverflowError.js +28 -0
- package/src/domain/errors/index.js +1 -0
- package/src/domain/services/AdjacencyNeighborProvider.js +140 -0
- package/src/domain/services/BitmapNeighborProvider.js +178 -0
- package/src/domain/services/CheckpointMessageCodec.js +3 -3
- package/src/domain/services/CheckpointService.js +77 -12
- package/src/domain/services/GraphTraversal.js +1239 -0
- package/src/domain/services/IncrementalIndexUpdater.js +765 -0
- package/src/domain/services/JoinReducer.js +233 -5
- package/src/domain/services/LogicalBitmapIndexBuilder.js +323 -0
- package/src/domain/services/LogicalIndexBuildService.js +108 -0
- package/src/domain/services/LogicalIndexReader.js +315 -0
- package/src/domain/services/LogicalTraversal.js +321 -202
- package/src/domain/services/MaterializedViewService.js +379 -0
- package/src/domain/services/ObserverView.js +138 -47
- package/src/domain/services/PatchBuilderV2.js +3 -3
- package/src/domain/services/PropertyIndexBuilder.js +64 -0
- package/src/domain/services/PropertyIndexReader.js +111 -0
- package/src/domain/services/TemporalQuery.js +128 -14
- package/src/domain/types/PatchDiff.js +90 -0
- package/src/domain/types/WarpTypesV2.js +4 -4
- package/src/domain/utils/MinHeap.js +45 -17
- package/src/domain/utils/canonicalCbor.js +36 -0
- package/src/domain/utils/fnv1a.js +20 -0
- package/src/domain/utils/roaring.js +14 -3
- package/src/domain/utils/shardKey.js +40 -0
- package/src/domain/utils/toBytes.js +17 -0
- package/src/domain/warp/_wiredMethods.d.ts +7 -1
- package/src/domain/warp/checkpoint.methods.js +21 -5
- package/src/domain/warp/materialize.methods.js +17 -5
- package/src/domain/warp/materializeAdvanced.methods.js +142 -3
- package/src/domain/warp/query.methods.js +78 -12
- package/src/infrastructure/adapters/CasSeekCacheAdapter.js +26 -5
- package/src/ports/BlobPort.js +1 -1
- package/src/ports/NeighborProviderPort.js +59 -0
- package/src/ports/SeekCachePort.js +4 -3
|
@@ -9,13 +9,14 @@
|
|
|
9
9
|
* }
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
import { createORSet, orsetAdd, orsetRemove, orsetJoin } from '../crdt/ORSet.js';
|
|
12
|
+
import { createORSet, orsetAdd, orsetRemove, orsetJoin, orsetContains } from '../crdt/ORSet.js';
|
|
13
13
|
import { createVersionVector, vvMerge, vvClone, vvDeserialize } from '../crdt/VersionVector.js';
|
|
14
14
|
import { lwwSet, lwwMax } from '../crdt/LWW.js';
|
|
15
15
|
import { createEventId, compareEventIds } from '../utils/EventId.js';
|
|
16
16
|
import { createTickReceipt, OP_TYPES } from '../types/TickReceipt.js';
|
|
17
17
|
import { encodeDot } from '../crdt/Dot.js';
|
|
18
|
-
import { encodeEdgeKey, encodePropKey } from './KeyCodec.js';
|
|
18
|
+
import { encodeEdgeKey, decodeEdgeKey, encodePropKey } from './KeyCodec.js';
|
|
19
|
+
import { createEmptyDiff, mergeDiffs } from '../types/PatchDiff.js';
|
|
19
20
|
|
|
20
21
|
// Re-export key codec functions for backward compatibility
|
|
21
22
|
export {
|
|
@@ -378,6 +379,218 @@ export function applyFast(state, patch, patchSha) {
|
|
|
378
379
|
return state;
|
|
379
380
|
}
|
|
380
381
|
|
|
382
|
+
/**
|
|
383
|
+
* Builds a reverse map from dot string → element ID for an OR-Set.
|
|
384
|
+
*
|
|
385
|
+
* Only includes mappings for dots that appear in the given targetDots set,
|
|
386
|
+
* allowing early termination once all target dots are accounted for.
|
|
387
|
+
*
|
|
388
|
+
* @param {import('../crdt/ORSet.js').ORSet} orset
|
|
389
|
+
* @param {Set<string>} targetDots - The dots we care about
|
|
390
|
+
* @returns {Map<string, string>} dot → elementId
|
|
391
|
+
*/
|
|
392
|
+
function buildDotToElement(orset, targetDots) {
|
|
393
|
+
const dotToElement = new Map();
|
|
394
|
+
let remaining = targetDots.size;
|
|
395
|
+
for (const [element, dots] of orset.entries) {
|
|
396
|
+
if (remaining === 0) { break; }
|
|
397
|
+
for (const d of dots) {
|
|
398
|
+
if (targetDots.has(d)) {
|
|
399
|
+
dotToElement.set(d, element);
|
|
400
|
+
remaining--;
|
|
401
|
+
if (remaining === 0) { break; }
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
return dotToElement;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Collects the set of alive elements that own at least one of the target dots.
|
|
410
|
+
*
|
|
411
|
+
* Uses a reverse-index from dot → element to avoid scanning every entry in the
|
|
412
|
+
* OR-Set. Complexity: O(total_dots_in_orset) for index build (with early exit)
|
|
413
|
+
* + O(|targetDots|) for lookups, vs the previous O(N * |targetDots|) full scan.
|
|
414
|
+
*
|
|
415
|
+
* @param {import('../crdt/ORSet.js').ORSet} orset
|
|
416
|
+
* @param {Set<string>} observedDots
|
|
417
|
+
* @returns {Set<string>} element IDs that were alive and own at least one observed dot
|
|
418
|
+
*/
|
|
419
|
+
function aliveElementsForDots(orset, observedDots) {
|
|
420
|
+
const result = new Set();
|
|
421
|
+
const dotToElement = buildDotToElement(orset, observedDots);
|
|
422
|
+
for (const d of observedDots) {
|
|
423
|
+
const element = dotToElement.get(d);
|
|
424
|
+
if (element !== undefined && !result.has(element) && orsetContains(orset, element)) {
|
|
425
|
+
result.add(element);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
return result;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* @typedef {Object} SnapshotBeforeOp
|
|
433
|
+
* @property {boolean} [nodeWasAlive]
|
|
434
|
+
* @property {boolean} [edgeWasAlive]
|
|
435
|
+
* @property {string} [edgeKey]
|
|
436
|
+
* @property {unknown} [prevPropValue]
|
|
437
|
+
* @property {string} [propKey]
|
|
438
|
+
* @property {Set<string>} [aliveBeforeNodes]
|
|
439
|
+
* @property {Set<string>} [aliveBeforeEdges]
|
|
440
|
+
*/
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* Snapshots alive-ness of a node or edge before an op is applied.
|
|
444
|
+
*
|
|
445
|
+
* @param {WarpStateV5} state
|
|
446
|
+
* @param {import('../types/WarpTypesV2.js').OpV2} op
|
|
447
|
+
* @returns {SnapshotBeforeOp}
|
|
448
|
+
*/
|
|
449
|
+
function snapshotBeforeOp(state, op) {
|
|
450
|
+
switch (op.type) {
|
|
451
|
+
case 'NodeAdd':
|
|
452
|
+
return { nodeWasAlive: orsetContains(state.nodeAlive, op.node) };
|
|
453
|
+
case 'NodeRemove': {
|
|
454
|
+
const rawDots = /** @type {Iterable<string>} */ (op.observedDots);
|
|
455
|
+
/** @type {Set<string>} */
|
|
456
|
+
const nodeDots = rawDots instanceof Set ? rawDots : new Set(rawDots);
|
|
457
|
+
const aliveBeforeNodes = aliveElementsForDots(state.nodeAlive, nodeDots);
|
|
458
|
+
return { aliveBeforeNodes };
|
|
459
|
+
}
|
|
460
|
+
case 'EdgeAdd': {
|
|
461
|
+
const ek = encodeEdgeKey(op.from, op.to, op.label);
|
|
462
|
+
return { edgeWasAlive: orsetContains(state.edgeAlive, ek), edgeKey: ek };
|
|
463
|
+
}
|
|
464
|
+
case 'EdgeRemove': {
|
|
465
|
+
const rawEdgeDots = /** @type {Iterable<string>} */ (op.observedDots);
|
|
466
|
+
/** @type {Set<string>} */
|
|
467
|
+
const edgeDots = rawEdgeDots instanceof Set ? rawEdgeDots : new Set(rawEdgeDots);
|
|
468
|
+
const aliveBeforeEdges = aliveElementsForDots(state.edgeAlive, edgeDots);
|
|
469
|
+
return { aliveBeforeEdges };
|
|
470
|
+
}
|
|
471
|
+
case 'PropSet': {
|
|
472
|
+
const pk = encodePropKey(op.node, op.key);
|
|
473
|
+
const reg = state.prop.get(pk);
|
|
474
|
+
return { prevPropValue: reg ? reg.value : undefined, propKey: pk };
|
|
475
|
+
}
|
|
476
|
+
default:
|
|
477
|
+
return {};
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* Computes diff entries by comparing pre/post alive-ness after an op.
|
|
483
|
+
*
|
|
484
|
+
* @param {import('../types/PatchDiff.js').PatchDiff} diff
|
|
485
|
+
* @param {WarpStateV5} state
|
|
486
|
+
* @param {import('../types/WarpTypesV2.js').OpV2} op
|
|
487
|
+
* @param {SnapshotBeforeOp} before
|
|
488
|
+
*/
|
|
489
|
+
function accumulateOpDiff(diff, state, op, before) {
|
|
490
|
+
switch (op.type) {
|
|
491
|
+
case 'NodeAdd': {
|
|
492
|
+
if (!before.nodeWasAlive && orsetContains(state.nodeAlive, op.node)) {
|
|
493
|
+
diff.nodesAdded.push(op.node);
|
|
494
|
+
}
|
|
495
|
+
break;
|
|
496
|
+
}
|
|
497
|
+
case 'NodeRemove': {
|
|
498
|
+
collectNodeRemovals(diff, state, before);
|
|
499
|
+
break;
|
|
500
|
+
}
|
|
501
|
+
case 'EdgeAdd': {
|
|
502
|
+
if (!before.edgeWasAlive && orsetContains(state.edgeAlive, /** @type {string} */ (before.edgeKey))) {
|
|
503
|
+
const { from, to, label } = op;
|
|
504
|
+
diff.edgesAdded.push({ from, to, label });
|
|
505
|
+
}
|
|
506
|
+
break;
|
|
507
|
+
}
|
|
508
|
+
case 'EdgeRemove': {
|
|
509
|
+
collectEdgeRemovals(diff, state, before);
|
|
510
|
+
break;
|
|
511
|
+
}
|
|
512
|
+
case 'PropSet': {
|
|
513
|
+
const reg = state.prop.get(/** @type {string} */ (before.propKey));
|
|
514
|
+
const newVal = reg ? reg.value : undefined;
|
|
515
|
+
if (newVal !== before.prevPropValue) {
|
|
516
|
+
diff.propsChanged.push({
|
|
517
|
+
nodeId: op.node,
|
|
518
|
+
key: op.key,
|
|
519
|
+
value: newVal,
|
|
520
|
+
prevValue: before.prevPropValue,
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
break;
|
|
524
|
+
}
|
|
525
|
+
default:
|
|
526
|
+
break;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* Records removal only for elements that were alive before AND dead after.
|
|
532
|
+
*
|
|
533
|
+
* @param {import('../types/PatchDiff.js').PatchDiff} diff
|
|
534
|
+
* @param {WarpStateV5} state
|
|
535
|
+
* @param {SnapshotBeforeOp} before
|
|
536
|
+
*/
|
|
537
|
+
function collectNodeRemovals(diff, state, before) {
|
|
538
|
+
if (!before.aliveBeforeNodes) { return; }
|
|
539
|
+
for (const element of before.aliveBeforeNodes) {
|
|
540
|
+
if (!orsetContains(state.nodeAlive, element)) {
|
|
541
|
+
diff.nodesRemoved.push(element);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/**
|
|
547
|
+
* Records removal only for edges that were alive before AND dead after.
|
|
548
|
+
*
|
|
549
|
+
* @param {import('../types/PatchDiff.js').PatchDiff} diff
|
|
550
|
+
* @param {WarpStateV5} state
|
|
551
|
+
* @param {SnapshotBeforeOp} before
|
|
552
|
+
*/
|
|
553
|
+
function collectEdgeRemovals(diff, state, before) {
|
|
554
|
+
if (!before.aliveBeforeEdges) { return; }
|
|
555
|
+
for (const edgeKey of before.aliveBeforeEdges) {
|
|
556
|
+
if (!orsetContains(state.edgeAlive, edgeKey)) {
|
|
557
|
+
diff.edgesRemoved.push(decodeEdgeKey(edgeKey));
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Applies a patch to state with diff tracking for incremental index updates.
|
|
564
|
+
*
|
|
565
|
+
* Captures alive-ness transitions: only records a diff entry when the
|
|
566
|
+
* alive-ness of a node/edge actually changes, or when an LWW property
|
|
567
|
+
* winner changes. Redundant ops produce no diff entries.
|
|
568
|
+
*
|
|
569
|
+
* @param {WarpStateV5} state - The state to mutate in place
|
|
570
|
+
* @param {Object} patch - The patch to apply
|
|
571
|
+
* @param {string} patch.writer
|
|
572
|
+
* @param {number} patch.lamport
|
|
573
|
+
* @param {Array<Object>} patch.ops
|
|
574
|
+
* @param {Map<string, number>|{[x: string]: number}} patch.context
|
|
575
|
+
* @param {string} patchSha - Git SHA of the patch commit
|
|
576
|
+
* @returns {{state: WarpStateV5, diff: import('../types/PatchDiff.js').PatchDiff}}
|
|
577
|
+
*/
|
|
578
|
+
export function applyWithDiff(state, patch, patchSha) {
|
|
579
|
+
const diff = createEmptyDiff();
|
|
580
|
+
|
|
581
|
+
for (let i = 0; i < patch.ops.length; i++) {
|
|
582
|
+
const op = patch.ops[i];
|
|
583
|
+
const eventId = createEventId(patch.lamport, patch.writer, patchSha, i);
|
|
584
|
+
const typedOp = /** @type {import('../types/WarpTypesV2.js').OpV2} */ (op);
|
|
585
|
+
const before = snapshotBeforeOp(state, typedOp);
|
|
586
|
+
applyOpV2(state, typedOp, eventId);
|
|
587
|
+
accumulateOpDiff(diff, state, typedOp, before);
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
updateFrontierFromPatch(state, patch);
|
|
591
|
+
return { state, diff };
|
|
592
|
+
}
|
|
593
|
+
|
|
381
594
|
/**
|
|
382
595
|
* Applies a patch to state with receipt collection for provenance tracking.
|
|
383
596
|
*
|
|
@@ -585,9 +798,15 @@ function mergeEdgeBirthEvent(a, b) {
|
|
|
585
798
|
* @param {WarpStateV5} [initialState] - Optional starting state (for incremental materialization from checkpoint)
|
|
586
799
|
* @param {Object} [options] - Optional configuration
|
|
587
800
|
* @param {boolean} [options.receipts=false] - When true, collect and return TickReceipts
|
|
588
|
-
* @
|
|
589
|
-
*
|
|
590
|
-
*
|
|
801
|
+
* @param {boolean} [options.trackDiff=false] - When true, collect and return PatchDiff
|
|
802
|
+
* @returns {WarpStateV5|{state: WarpStateV5, receipts: import('../types/TickReceipt.js').TickReceipt[]}|{state: WarpStateV5, diff: import('../types/PatchDiff.js').PatchDiff}}
|
|
803
|
+
* Returns state directly when no options;
|
|
804
|
+
* returns {state, receipts} when receipts is true;
|
|
805
|
+
* returns {state, diff} when trackDiff is true
|
|
806
|
+
*
|
|
807
|
+
* @note When initialState is provided, the returned diff records transitions
|
|
808
|
+
* relative to that state. The caller must ensure any index tree passed to
|
|
809
|
+
* IncrementalIndexUpdater was built from the same initialState.
|
|
591
810
|
*/
|
|
592
811
|
export function reduceV5(patches, initialState, options) {
|
|
593
812
|
const state = initialState ? cloneStateV5(initialState) : createEmptyStateV5();
|
|
@@ -602,6 +821,15 @@ export function reduceV5(patches, initialState, options) {
|
|
|
602
821
|
return { state, receipts };
|
|
603
822
|
}
|
|
604
823
|
|
|
824
|
+
if (options && options.trackDiff) {
|
|
825
|
+
let merged = createEmptyDiff();
|
|
826
|
+
for (const { patch, sha } of patches) {
|
|
827
|
+
const { diff } = applyWithDiff(state, patch, sha);
|
|
828
|
+
merged = mergeDiffs(merged, diff);
|
|
829
|
+
}
|
|
830
|
+
return { state, diff: merged };
|
|
831
|
+
}
|
|
832
|
+
|
|
605
833
|
for (const { patch, sha } of patches) {
|
|
606
834
|
applyFast(state, patch, sha);
|
|
607
835
|
}
|
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Builder for constructing CBOR-based bitmap indexes over the logical graph.
|
|
3
|
+
*
|
|
4
|
+
* Produces sharded index with stable numeric IDs, append-only label registry,
|
|
5
|
+
* per-label forward/reverse bitmaps, and alive bitmaps per shard.
|
|
6
|
+
*
|
|
7
|
+
* Shard output:
|
|
8
|
+
* meta_XX.cbor — nodeId → globalId mappings, nextLocalId, alive bitmap
|
|
9
|
+
* labels.cbor — label registry (append-only)
|
|
10
|
+
* fwd_XX.cbor — forward edge bitmaps (all + byLabel)
|
|
11
|
+
* rev_XX.cbor — reverse edge bitmaps (all + byLabel)
|
|
12
|
+
* receipt.cbor — build metadata
|
|
13
|
+
*
|
|
14
|
+
* @module domain/services/LogicalBitmapIndexBuilder
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import defaultCodec from '../utils/defaultCodec.js';
|
|
18
|
+
import computeShardKey from '../utils/shardKey.js';
|
|
19
|
+
import { getRoaringBitmap32 } from '../utils/roaring.js';
|
|
20
|
+
import { ShardIdOverflowError } from '../errors/index.js';
|
|
21
|
+
|
|
22
|
+
/** Maximum local IDs per shard (2^24). */
|
|
23
|
+
const MAX_LOCAL_ID = 1 << 24;
|
|
24
|
+
|
|
25
|
+
export default class LogicalBitmapIndexBuilder {
|
|
26
|
+
/**
|
|
27
|
+
* @param {Object} [options]
|
|
28
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec]
|
|
29
|
+
*/
|
|
30
|
+
constructor({ codec } = {}) {
|
|
31
|
+
this._codec = codec || defaultCodec;
|
|
32
|
+
|
|
33
|
+
/** @type {Map<string, number>} nodeId → globalId */
|
|
34
|
+
this._nodeToGlobal = new Map();
|
|
35
|
+
|
|
36
|
+
/** @type {Map<string, string>} globalId(string) → nodeId */
|
|
37
|
+
this._globalToNode = new Map();
|
|
38
|
+
|
|
39
|
+
/** Per-shard next local ID counters. @type {Map<string, number>} */
|
|
40
|
+
this._shardNextLocal = new Map();
|
|
41
|
+
|
|
42
|
+
/** Alive bitmap per shard. @type {Map<string, import('../utils/roaring.js').RoaringBitmapSubset>} */
|
|
43
|
+
this._aliveBitmaps = new Map();
|
|
44
|
+
|
|
45
|
+
/** Label → labelId (append-only). @type {Map<string, number>} */
|
|
46
|
+
this._labelToId = new Map();
|
|
47
|
+
|
|
48
|
+
/** @type {number} */
|
|
49
|
+
this._nextLabelId = 0;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Forward edge bitmaps.
|
|
53
|
+
* Key: `${shardKey}:all:${globalId}` or `${shardKey}:${labelId}:${globalId}`
|
|
54
|
+
* @type {Map<string, import('../utils/roaring.js').RoaringBitmapSubset>}
|
|
55
|
+
*/
|
|
56
|
+
this._fwdBitmaps = new Map();
|
|
57
|
+
|
|
58
|
+
/** Reverse edge bitmaps. Same key scheme as _fwdBitmaps. @type {Map<string, import('../utils/roaring.js').RoaringBitmapSubset>} */
|
|
59
|
+
this._revBitmaps = new Map();
|
|
60
|
+
|
|
61
|
+
/** Per-shard node list for O(shard) serialize. @type {Map<string, Array<[string, number]>>} */
|
|
62
|
+
this._shardNodes = new Map();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Registers a node and returns its stable global ID.
|
|
67
|
+
* GlobalId = (shardByte << 24) | localId.
|
|
68
|
+
*
|
|
69
|
+
* @param {string} nodeId
|
|
70
|
+
* @returns {number} globalId
|
|
71
|
+
* @throws {ShardIdOverflowError} If the shard is full
|
|
72
|
+
*/
|
|
73
|
+
registerNode(nodeId) {
|
|
74
|
+
const existing = this._nodeToGlobal.get(nodeId);
|
|
75
|
+
if (existing !== undefined) {
|
|
76
|
+
return existing;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const shardKey = computeShardKey(nodeId);
|
|
80
|
+
const shardByte = parseInt(shardKey, 16);
|
|
81
|
+
const nextLocal = this._shardNextLocal.get(shardKey) ?? 0;
|
|
82
|
+
|
|
83
|
+
if (nextLocal >= MAX_LOCAL_ID) {
|
|
84
|
+
throw new ShardIdOverflowError(
|
|
85
|
+
`Shard '${shardKey}' exceeded max local ID (${MAX_LOCAL_ID})`,
|
|
86
|
+
{ shardKey, nextLocalId: nextLocal },
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const globalId = ((shardByte << 24) | nextLocal) >>> 0;
|
|
91
|
+
this._nodeToGlobal.set(nodeId, globalId);
|
|
92
|
+
this._globalToNode.set(String(globalId), nodeId);
|
|
93
|
+
this._shardNextLocal.set(shardKey, nextLocal + 1);
|
|
94
|
+
|
|
95
|
+
let shardList = this._shardNodes.get(shardKey);
|
|
96
|
+
if (!shardList) {
|
|
97
|
+
shardList = [];
|
|
98
|
+
this._shardNodes.set(shardKey, shardList);
|
|
99
|
+
}
|
|
100
|
+
shardList.push([nodeId, globalId]);
|
|
101
|
+
|
|
102
|
+
return globalId;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Marks a node as alive in its shard's alive bitmap.
|
|
107
|
+
*
|
|
108
|
+
* @param {string} nodeId
|
|
109
|
+
*/
|
|
110
|
+
markAlive(nodeId) {
|
|
111
|
+
const globalId = this._nodeToGlobal.get(nodeId);
|
|
112
|
+
if (globalId === undefined) {
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
const shardKey = computeShardKey(nodeId);
|
|
116
|
+
let bitmap = this._aliveBitmaps.get(shardKey);
|
|
117
|
+
if (!bitmap) {
|
|
118
|
+
const RoaringBitmap32 = getRoaringBitmap32();
|
|
119
|
+
bitmap = new RoaringBitmap32();
|
|
120
|
+
this._aliveBitmaps.set(shardKey, bitmap);
|
|
121
|
+
}
|
|
122
|
+
bitmap.add(globalId);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Registers a label and returns its append-only labelId.
|
|
127
|
+
*
|
|
128
|
+
* @param {string} label
|
|
129
|
+
* @returns {number}
|
|
130
|
+
*/
|
|
131
|
+
registerLabel(label) {
|
|
132
|
+
const existing = this._labelToId.get(label);
|
|
133
|
+
if (existing !== undefined) {
|
|
134
|
+
return existing;
|
|
135
|
+
}
|
|
136
|
+
const id = this._nextLabelId++;
|
|
137
|
+
this._labelToId.set(label, id);
|
|
138
|
+
return id;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Adds a directed edge, populating forward/reverse bitmaps
|
|
143
|
+
* for both the 'all' bucket and the per-label bucket.
|
|
144
|
+
*
|
|
145
|
+
* @param {string} fromId - Source node ID (must be registered)
|
|
146
|
+
* @param {string} toId - Target node ID (must be registered)
|
|
147
|
+
* @param {string} label - Edge label (must be registered)
|
|
148
|
+
*/
|
|
149
|
+
addEdge(fromId, toId, label) {
|
|
150
|
+
const fromGlobal = this._nodeToGlobal.get(fromId);
|
|
151
|
+
const toGlobal = this._nodeToGlobal.get(toId);
|
|
152
|
+
if (fromGlobal === undefined || toGlobal === undefined) {
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const labelId = this._labelToId.get(label);
|
|
157
|
+
if (labelId === undefined) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const fromShard = computeShardKey(fromId);
|
|
162
|
+
const toShard = computeShardKey(toId);
|
|
163
|
+
|
|
164
|
+
// Forward: from's shard, keyed by fromGlobal, value contains toGlobal
|
|
165
|
+
this._addToBitmap(this._fwdBitmaps, { shardKey: fromShard, bucket: 'all', owner: fromGlobal, target: toGlobal });
|
|
166
|
+
this._addToBitmap(this._fwdBitmaps, { shardKey: fromShard, bucket: String(labelId), owner: fromGlobal, target: toGlobal });
|
|
167
|
+
|
|
168
|
+
// Reverse: to's shard, keyed by toGlobal, value contains fromGlobal
|
|
169
|
+
this._addToBitmap(this._revBitmaps, { shardKey: toShard, bucket: 'all', owner: toGlobal, target: fromGlobal });
|
|
170
|
+
this._addToBitmap(this._revBitmaps, { shardKey: toShard, bucket: String(labelId), owner: toGlobal, target: fromGlobal });
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Seeds ID mappings from a previously built meta shard for ID stability.
|
|
175
|
+
*
|
|
176
|
+
* @param {string} shardKey
|
|
177
|
+
* @param {{ nodeToGlobal: Array<[string, number]>|Record<string, number>, nextLocalId: number }} metaShard
|
|
178
|
+
*/
|
|
179
|
+
loadExistingMeta(shardKey, metaShard) {
|
|
180
|
+
const entries = Array.isArray(metaShard.nodeToGlobal)
|
|
181
|
+
? metaShard.nodeToGlobal
|
|
182
|
+
: Object.entries(metaShard.nodeToGlobal);
|
|
183
|
+
let shardList = this._shardNodes.get(shardKey);
|
|
184
|
+
if (!shardList) {
|
|
185
|
+
shardList = [];
|
|
186
|
+
this._shardNodes.set(shardKey, shardList);
|
|
187
|
+
}
|
|
188
|
+
for (const [nodeId, globalId] of entries) {
|
|
189
|
+
this._nodeToGlobal.set(nodeId, /** @type {number} */ (globalId));
|
|
190
|
+
this._globalToNode.set(String(globalId), /** @type {string} */ (nodeId));
|
|
191
|
+
shardList.push([/** @type {string} */ (nodeId), /** @type {number} */ (globalId)]);
|
|
192
|
+
}
|
|
193
|
+
const current = this._shardNextLocal.get(shardKey) ?? 0;
|
|
194
|
+
if (metaShard.nextLocalId > current) {
|
|
195
|
+
this._shardNextLocal.set(shardKey, metaShard.nextLocalId);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Seeds the label registry from a previous build for append-only stability.
|
|
201
|
+
*
|
|
202
|
+
* @param {Record<string, number>|Array<[string, number]>} registry - label → labelId
|
|
203
|
+
*/
|
|
204
|
+
loadExistingLabels(registry) {
|
|
205
|
+
const entries = Array.isArray(registry) ? registry : Object.entries(registry);
|
|
206
|
+
let maxId = this._nextLabelId;
|
|
207
|
+
for (const [label, id] of entries) {
|
|
208
|
+
this._labelToId.set(label, id);
|
|
209
|
+
if (id >= maxId) {
|
|
210
|
+
maxId = id + 1;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
this._nextLabelId = maxId;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Serializes the full index to a Record<string, Uint8Array>.
|
|
218
|
+
*
|
|
219
|
+
* @returns {Record<string, Uint8Array>}
|
|
220
|
+
*/
|
|
221
|
+
serialize() {
|
|
222
|
+
/** @type {Record<string, Uint8Array>} */
|
|
223
|
+
const tree = {};
|
|
224
|
+
|
|
225
|
+
// Collect all shard keys that have any data
|
|
226
|
+
const allShardKeys = new Set([
|
|
227
|
+
...this._shardNextLocal.keys(),
|
|
228
|
+
]);
|
|
229
|
+
|
|
230
|
+
// Meta shards
|
|
231
|
+
for (const shardKey of allShardKeys) {
|
|
232
|
+
// Use array of [nodeId, globalId] pairs to avoid __proto__ key issues
|
|
233
|
+
// Sort by nodeId for deterministic output
|
|
234
|
+
const nodeToGlobal = (this._shardNodes.get(shardKey) ?? [])
|
|
235
|
+
.slice()
|
|
236
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
|
237
|
+
|
|
238
|
+
const aliveBitmap = this._aliveBitmaps.get(shardKey);
|
|
239
|
+
const aliveBytes = aliveBitmap ? aliveBitmap.serialize(true) : new Uint8Array(0);
|
|
240
|
+
|
|
241
|
+
const shard = {
|
|
242
|
+
nodeToGlobal,
|
|
243
|
+
nextLocalId: this._shardNextLocal.get(shardKey) ?? 0,
|
|
244
|
+
alive: aliveBytes,
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
tree[`meta_${shardKey}.cbor`] = this._codec.encode(shard).slice();
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Labels registry
|
|
251
|
+
/** @type {Array<[string, number]>} */
|
|
252
|
+
const labelRegistry = [];
|
|
253
|
+
for (const [label, id] of this._labelToId) {
|
|
254
|
+
labelRegistry.push([label, id]);
|
|
255
|
+
}
|
|
256
|
+
tree['labels.cbor'] = this._codec.encode(labelRegistry).slice();
|
|
257
|
+
|
|
258
|
+
// Forward/reverse edge shards
|
|
259
|
+
this._serializeEdgeShards(tree, 'fwd', this._fwdBitmaps);
|
|
260
|
+
this._serializeEdgeShards(tree, 'rev', this._revBitmaps);
|
|
261
|
+
|
|
262
|
+
// Receipt
|
|
263
|
+
const receipt = {
|
|
264
|
+
version: 1,
|
|
265
|
+
nodeCount: this._nodeToGlobal.size,
|
|
266
|
+
labelCount: this._labelToId.size,
|
|
267
|
+
shardCount: allShardKeys.size,
|
|
268
|
+
};
|
|
269
|
+
tree['receipt.cbor'] = this._codec.encode(receipt).slice();
|
|
270
|
+
|
|
271
|
+
return tree;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* @param {Record<string, Uint8Array>} tree
|
|
276
|
+
* @param {string} direction - 'fwd' or 'rev'
|
|
277
|
+
* @param {Map<string, import('../utils/roaring.js').RoaringBitmapSubset>} bitmaps
|
|
278
|
+
* @private
|
|
279
|
+
*/
|
|
280
|
+
_serializeEdgeShards(tree, direction, bitmaps) {
|
|
281
|
+
// Group by shardKey
|
|
282
|
+
/** @type {Map<string, Record<string, Record<string, Uint8Array>>>} */
|
|
283
|
+
const byShardKey = new Map();
|
|
284
|
+
|
|
285
|
+
for (const [key, bitmap] of bitmaps) {
|
|
286
|
+
// key: `${shardKey}:${bucketName}:${globalId}`
|
|
287
|
+
const firstColon = key.indexOf(':');
|
|
288
|
+
const secondColon = key.indexOf(':', firstColon + 1);
|
|
289
|
+
const shardKey = key.substring(0, firstColon);
|
|
290
|
+
const bucketName = key.substring(firstColon + 1, secondColon);
|
|
291
|
+
const globalIdStr = key.substring(secondColon + 1);
|
|
292
|
+
|
|
293
|
+
if (!byShardKey.has(shardKey)) {
|
|
294
|
+
byShardKey.set(shardKey, {});
|
|
295
|
+
}
|
|
296
|
+
const shardData = /** @type {Record<string, Record<string, Uint8Array>>} */ (byShardKey.get(shardKey));
|
|
297
|
+
if (!shardData[bucketName]) {
|
|
298
|
+
shardData[bucketName] = {};
|
|
299
|
+
}
|
|
300
|
+
shardData[bucketName][globalIdStr] = bitmap.serialize(true);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
for (const [shardKey, shardData] of byShardKey) {
|
|
304
|
+
tree[`${direction}_${shardKey}.cbor`] = this._codec.encode(shardData).slice();
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* @param {Map<string, import('../utils/roaring.js').RoaringBitmapSubset>} store
|
|
310
|
+
* @param {{ shardKey: string, bucket: string, owner: number, target: number }} opts
|
|
311
|
+
* @private
|
|
312
|
+
*/
|
|
313
|
+
_addToBitmap(store, { shardKey, bucket, owner, target }) {
|
|
314
|
+
const key = `${shardKey}:${bucket}:${owner}`;
|
|
315
|
+
let bitmap = store.get(key);
|
|
316
|
+
if (!bitmap) {
|
|
317
|
+
const RoaringBitmap32 = getRoaringBitmap32();
|
|
318
|
+
bitmap = new RoaringBitmap32();
|
|
319
|
+
store.set(key, bitmap);
|
|
320
|
+
}
|
|
321
|
+
bitmap.add(target);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orchestrates a full logical bitmap index build from WarpStateV5.
|
|
3
|
+
*
|
|
4
|
+
* Extracts the visible projection (nodes, edges, properties) from materialized
|
|
5
|
+
* state and delegates to LogicalBitmapIndexBuilder + PropertyIndexBuilder.
|
|
6
|
+
*
|
|
7
|
+
* @module domain/services/LogicalIndexBuildService
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import defaultCodec from '../utils/defaultCodec.js';
|
|
11
|
+
import nullLogger from '../utils/nullLogger.js';
|
|
12
|
+
import LogicalBitmapIndexBuilder from './LogicalBitmapIndexBuilder.js';
|
|
13
|
+
import PropertyIndexBuilder from './PropertyIndexBuilder.js';
|
|
14
|
+
import { orsetElements } from '../crdt/ORSet.js';
|
|
15
|
+
import { decodeEdgeKey, decodePropKey, isEdgePropKey } from './KeyCodec.js';
|
|
16
|
+
import { nodeVisibleV5, edgeVisibleV5 } from './StateSerializerV5.js';
|
|
17
|
+
|
|
18
|
+
export default class LogicalIndexBuildService {
|
|
19
|
+
/**
|
|
20
|
+
* @param {Object} [options]
|
|
21
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec]
|
|
22
|
+
* @param {import('../../ports/LoggerPort.js').default} [options.logger]
|
|
23
|
+
*/
|
|
24
|
+
constructor({ codec, logger } = {}) {
|
|
25
|
+
this._codec = codec || defaultCodec;
|
|
26
|
+
this._logger = logger || nullLogger;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Builds a complete logical index from materialized state.
|
|
31
|
+
*
|
|
32
|
+
* @param {import('./JoinReducer.js').WarpStateV5} state
|
|
33
|
+
* @param {Object} [options]
|
|
34
|
+
* @param {Record<string, { nodeToGlobal: Record<string, number>, nextLocalId: number }>} [options.existingMeta] - Prior meta shards for ID stability
|
|
35
|
+
* @param {Record<string, number>|Array<[string, number]>} [options.existingLabels] - Prior label registry for append-only stability
|
|
36
|
+
* @returns {{ tree: Record<string, Uint8Array>, receipt: Record<string, unknown> }}
|
|
37
|
+
*/
|
|
38
|
+
build(state, options = {}) {
|
|
39
|
+
const indexBuilder = new LogicalBitmapIndexBuilder({ codec: this._codec });
|
|
40
|
+
const propBuilder = new PropertyIndexBuilder({ codec: this._codec });
|
|
41
|
+
|
|
42
|
+
// Seed existing data for stability
|
|
43
|
+
if (options.existingMeta) {
|
|
44
|
+
for (const [shardKey, meta] of Object.entries(options.existingMeta)) {
|
|
45
|
+
indexBuilder.loadExistingMeta(shardKey, meta);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (options.existingLabels) {
|
|
49
|
+
indexBuilder.loadExistingLabels(options.existingLabels);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// 1. Register and mark alive all visible nodes (sorted for deterministic ID assignment)
|
|
53
|
+
const aliveNodes = [...orsetElements(state.nodeAlive)].sort();
|
|
54
|
+
for (const nodeId of aliveNodes) {
|
|
55
|
+
indexBuilder.registerNode(nodeId);
|
|
56
|
+
indexBuilder.markAlive(nodeId);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// 2. Collect visible edges and register labels (sorted for deterministic ID assignment)
|
|
60
|
+
const visibleEdges = [];
|
|
61
|
+
for (const edgeKey of orsetElements(state.edgeAlive)) {
|
|
62
|
+
if (edgeVisibleV5(state, edgeKey)) {
|
|
63
|
+
visibleEdges.push(decodeEdgeKey(edgeKey));
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
visibleEdges.sort((a, b) => {
|
|
67
|
+
if (a.from !== b.from) {
|
|
68
|
+
return a.from < b.from ? -1 : 1;
|
|
69
|
+
}
|
|
70
|
+
if (a.to !== b.to) {
|
|
71
|
+
return a.to < b.to ? -1 : 1;
|
|
72
|
+
}
|
|
73
|
+
if (a.label !== b.label) {
|
|
74
|
+
return a.label < b.label ? -1 : 1;
|
|
75
|
+
}
|
|
76
|
+
return 0;
|
|
77
|
+
});
|
|
78
|
+
const uniqueLabels = [...new Set(visibleEdges.map(e => e.label))].sort();
|
|
79
|
+
for (const label of uniqueLabels) {
|
|
80
|
+
indexBuilder.registerLabel(label);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// 3. Add edges
|
|
84
|
+
for (const { from, to, label } of visibleEdges) {
|
|
85
|
+
indexBuilder.addEdge(from, to, label);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// 4. Build property index from visible props
|
|
89
|
+
for (const [propKey, register] of state.prop) {
|
|
90
|
+
if (isEdgePropKey(propKey)) {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const { nodeId, propKey: key } = decodePropKey(propKey);
|
|
94
|
+
if (nodeVisibleV5(state, nodeId)) {
|
|
95
|
+
propBuilder.addProperty(nodeId, key, register.value);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// 5. Serialize
|
|
100
|
+
const indexTree = indexBuilder.serialize();
|
|
101
|
+
const propTree = propBuilder.serialize();
|
|
102
|
+
const tree = { ...indexTree, ...propTree };
|
|
103
|
+
|
|
104
|
+
const receipt = /** @type {Record<string, unknown>} */ (this._codec.decode(indexTree['receipt.cbor']));
|
|
105
|
+
|
|
106
|
+
return { tree, receipt };
|
|
107
|
+
}
|
|
108
|
+
}
|