@git-stunts/git-warp 11.5.1 → 12.1.0

package/src/domain/utils/MinHeap.js

@@ -8,10 +8,18 @@
 class MinHeap {
   /**
    * Creates an empty MinHeap.
+   *
+   * @param {Object} [options] - Configuration options
+   * @param {((a: T, b: T) => number)} [options.tieBreaker] - Comparator invoked when two
+   *   entries have equal priority. Negative return = a wins (comes out first).
+   *   When omitted, equal-priority extraction order is unspecified (heap-natural).
    */
-  constructor() {
+  constructor(options) {
+    const { tieBreaker } = options || {};
     /** @type {Array<{item: T, priority: number}>} */
-    this.heap = [];
+    this._heap = [];
+    /** @type {((a: T, b: T) => number) | undefined} */
+    this._tieBreaker = tieBreaker;
   }
 
   /**
@@ -22,8 +30,8 @@ class MinHeap {
    * @returns {void}
    */
   insert(item, priority) {
-    this.heap.push({ item, priority });
-    this._bubbleUp(this.heap.length - 1);
+    this._heap.push({ item, priority });
+    this._bubbleUp(this._heap.length - 1);
   }
 
   /**
@@ -32,11 +40,11 @@ class MinHeap {
    * @returns {T | undefined} The item with lowest priority, or undefined if empty
    */
   extractMin() {
-    if (this.heap.length === 0) { return undefined; }
-    if (this.heap.length === 1) { return /** @type {{item: T, priority: number}} */ (this.heap.pop()).item; }
+    if (this._heap.length === 0) { return undefined; }
+    if (this._heap.length === 1) { return /** @type {{item: T, priority: number}} */ (this._heap.pop()).item; }
 
-    const min = this.heap[0];
-    this.heap[0] = /** @type {{item: T, priority: number}} */ (this.heap.pop());
+    const min = this._heap[0];
+    this._heap[0] = /** @type {{item: T, priority: number}} */ (this._heap.pop());
     this._bubbleDown(0);
     return min.item;
   }
@@ -47,7 +55,7 @@ class MinHeap {
    * @returns {boolean} True if empty
    */
   isEmpty() {
-    return this.heap.length === 0;
+    return this._heap.length === 0;
   }
 
   /**
@@ -56,7 +64,7 @@ class MinHeap {
    * @returns {number} Number of items
    */
   size() {
-    return this.heap.length;
+    return this._heap.length;
   }
 
   /**
@@ -65,7 +73,27 @@ class MinHeap {
    * @returns {number} The minimum priority value, or Infinity if empty
    */
   peekPriority() {
-    return this.heap.length > 0 ? this.heap[0].priority : Infinity;
+    return this._heap.length > 0 ? this._heap[0].priority : Infinity;
+  }
+
+  /**
+   * Compares two heap entries. Returns negative if a should come before b.
+   *
+   * @private
+   * @param {number} idxA - Index of first entry
+   * @param {number} idxB - Index of second entry
+   * @returns {number} Negative if a < b, positive if a > b, zero if equal
+   */
+  _compare(idxA, idxB) {
+    const a = this._heap[idxA];
+    const b = this._heap[idxB];
+    if (a.priority !== b.priority) {
+      return a.priority - b.priority;
+    }
+    if (this._tieBreaker) {
+      return this._tieBreaker(a.item, b.item);
+    }
+    return 0;
   }
 
   /**
@@ -78,8 +106,8 @@ class MinHeap {
     let current = pos;
     while (current > 0) {
       const parentIndex = Math.floor((current - 1) / 2);
-      if (this.heap[parentIndex].priority <= this.heap[current].priority) { break; }
-      [this.heap[parentIndex], this.heap[current]] = [this.heap[current], this.heap[parentIndex]];
+      if (this._compare(parentIndex, current) <= 0) { break; }
+      [this._heap[parentIndex], this._heap[current]] = [this._heap[current], this._heap[parentIndex]];
       current = parentIndex;
     }
   }
@@ -91,22 +119,22 @@ class MinHeap {
    * @param {number} pos - Starting index
    */
   _bubbleDown(pos) {
-    const {length} = this.heap;
+    const {length} = this._heap;
     let current = pos;
     while (true) {
       const leftChild = 2 * current + 1;
       const rightChild = 2 * current + 2;
       let smallest = current;
 
-      if (leftChild < length && this.heap[leftChild].priority < this.heap[smallest].priority) {
+      if (leftChild < length && this._compare(leftChild, smallest) < 0) {
         smallest = leftChild;
       }
-      if (rightChild < length && this.heap[rightChild].priority < this.heap[smallest].priority) {
+      if (rightChild < length && this._compare(rightChild, smallest) < 0) {
         smallest = rightChild;
       }
       if (smallest === current) { break; }
 
-      [this.heap[current], this.heap[smallest]] = [this.heap[smallest], this.heap[current]];
+      [this._heap[current], this._heap[smallest]] = [this._heap[smallest], this._heap[current]];
       current = smallest;
     }
   }

package/src/domain/utils/canonicalCbor.js

@@ -0,0 +1,36 @@
+/**
+ * Canonical CBOR encoding/decoding.
+ *
+ * Delegates to defaultCodec which already sorts keys recursively
+ * and handles Maps, null-prototype objects, and arrays.
+ *
+ * Deterministic output relies on cbor-x's key-sorting behaviour,
+ * which approximates RFC 7049 Section 3.9 (Canonical CBOR) by sorting
+ * map keys in length-first lexicographic order. This is sufficient for
+ * content-addressed equality within the WARP system but should not be
+ * assumed to match other canonical CBOR implementations byte-for-byte.
+ *
+ * @module domain/utils/canonicalCbor
+ */
+
+import defaultCodec from './defaultCodec.js';
+
+/**
+ * Encodes a value to canonical CBOR bytes with sorted keys.
+ *
+ * @param {unknown} value - The value to encode
+ * @returns {Uint8Array} CBOR-encoded bytes
+ */
+export function encodeCanonicalCbor(value) {
+  return defaultCodec.encode(value);
+}
+
+/**
+ * Decodes CBOR bytes to a value.
+ *
+ * @param {Buffer|Uint8Array} buffer - CBOR bytes
+ * @returns {unknown} Decoded value
+ */
+export function decodeCanonicalCbor(buffer) {
+  return defaultCodec.decode(buffer);
+}

package/src/domain/utils/fnv1a.js

@@ -0,0 +1,20 @@
+/**
+ * FNV-1a 32-bit hash function.
+ *
+ * Used for shard key computation when the input is not a hex SHA.
+ * Uses Math.imul for correct 32-bit multiplication semantics.
+ *
+ * @note Callers with non-ASCII node IDs should normalize to NFC before
+ * hashing to ensure consistent shard placement.
+ *
+ * @param {string} str - Input string
+ * @returns {number} Unsigned 32-bit FNV-1a hash
+ */
+export default function fnv1a(str) {
+  let hash = 0x811c9dc5; // FNV offset basis
+  for (let i = 0; i < str.length; i++) {
+    hash ^= str.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193); // FNV prime
+  }
+  return hash >>> 0; // Ensure unsigned
+}

package/src/domain/utils/matchGlob.js

@@ -0,0 +1,51 @@
+/** @type {Map<string, RegExp>} Module-level cache for compiled glob regexes. */
+const globRegexCache = new Map();
+
+/**
+ * Escapes special regex characters in a string so it can be used as a literal match.
+ *
+ * @param {string} value - The string to escape
+ * @returns {string} The escaped string safe for use in a RegExp
+ * @private
+ */
+function escapeRegex(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Tests whether a string matches a glob-style pattern or an array of patterns.
+ *
+ * Supports:
+ * - `*` as the default pattern, matching all strings
+ * - Wildcard `*` anywhere in the pattern, matching zero or more characters
+ * - Literal match when pattern contains no wildcards
+ * - Array of patterns: returns true if ANY pattern matches (OR semantics)
+ *
+ * @param {string|string[]} pattern - The glob pattern(s) to match against
+ * @param {string} str - The string to test
+ * @returns {boolean} True if the string matches any of the patterns
+ */
+export function matchGlob(pattern, str) {
+  if (Array.isArray(pattern)) {
+    return pattern.some((p) => matchGlob(p, str));
+  }
+
+  if (pattern === '*') {
+    return true;
+  }
+
+  if (typeof pattern !== 'string') {
+    return false;
+  }
+
+  if (!pattern.includes('*')) {
+    return pattern === str;
+  }
+
+  let regex = globRegexCache.get(pattern);
+  if (!regex) {
+    regex = new RegExp(`^${escapeRegex(pattern).replace(/\\\*/g, '.*')}$`);
+    globRegexCache.set(pattern, regex);
+  }
+  return regex.test(str);
+}

package/src/domain/utils/roaring.js

@@ -50,6 +50,7 @@ const NOT_CHECKED = Symbol('NOT_CHECKED');
  * @typedef {Object} RoaringBitmapSubset
  * @property {number} size
  * @property {function(number): void} add
+ * @property {function(number): void} remove
  * @property {function(number): boolean} has
  * @property {function(Iterable<number>): void} orInPlace
  * @property {function(boolean): Uint8Array} serialize
@@ -63,6 +64,13 @@ const NOT_CHECKED = Symbol('NOT_CHECKED');
  */
 let roaringModule = null;
 
+/**
+ * Captures module initialization failure so callers can see the root cause.
+ * @type {unknown}
+ * @private
+ */
+let initError = null;
+
 /**
  * Cached result of native availability check.
  * `NOT_CHECKED` means not yet checked, `null` means indeterminate.
@@ -83,7 +91,8 @@ let nativeAvailability = NOT_CHECKED;
  */
 function loadRoaring() {
   if (!roaringModule) {
-    throw new Error('Roaring module not loaded. Call initRoaring() first or ensure top-level await import completed.');
+    const cause = initError instanceof Error ? ` Caused by: ${initError.message}` : '';
+    throw new Error(`Roaring module not loaded. Call initRoaring() first or ensure top-level await import completed.${cause}`);
   }
   return roaringModule;
 }
@@ -99,6 +108,7 @@ function loadRoaring() {
 export async function initRoaring(mod) {
   if (mod) {
     roaringModule = mod;
+    initError = null;
     return;
   }
   if (!roaringModule) {
@@ -113,8 +123,9 @@ export async function initRoaring(mod) {
 // Auto-initialize on module load (top-level await)
 try {
   await initRoaring();
-} catch {
-  // Roaring may not be installed; functions will throw on use
+} catch (err) {
+  // Roaring may not be installed; keep root cause for downstream diagnostics.
+  initError = err;
 }
 
 /**

package/src/domain/utils/shardKey.js

@@ -0,0 +1,40 @@
+const HEX_RE = /^[0-9a-fA-F]{40}$|^[0-9a-fA-F]{64}$/;
+
+const encoder = new TextEncoder();
+
+/**
+ * FNV-1a 32-bit over raw bytes (Uint8Array).
+ *
+ * @param {Uint8Array} bytes
+ * @returns {number} Unsigned 32-bit hash
+ */
+function fnv1aBytes(bytes) {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < bytes.length; i++) {
+    hash ^= bytes[i];
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return hash >>> 0;
+}
+
+/**
+ * Computes a 2-character hex shard key for a given ID.
+ *
+ * For hex SHAs (exactly 40 or 64 hex chars), uses the first two characters (lowercased).
+ * For all other strings, computes FNV-1a hash over UTF-8 bytes and takes the low byte.
+ *
+ * Returns '00' for null, undefined, or non-string inputs (graceful fallback).
+ *
+ * @param {string} id - Node ID or SHA
+ * @returns {string} 2-character lowercase hex shard key (e.g. 'ab', '0f')
+ */
+export default function computeShardKey(id) {
+  if (id === null || id === undefined || typeof id !== 'string') {
+    return '00';
+  }
+  if (HEX_RE.test(id)) {
+    return id.substring(0, 2).toLowerCase();
+  }
+  const hash = fnv1aBytes(encoder.encode(id));
+  return (hash & 0xff).toString(16).padStart(2, '0');
+}

package/src/domain/utils/toBytes.js

@@ -0,0 +1,17 @@
+/**
+ * Normalizes a decoded byte payload to a Uint8Array.
+ *
+ * CBOR decoders may yield Buffer, Uint8Array, or plain number[] depending
+ * on runtime and codec implementation (e.g. cbor-x on Node vs Deno).
+ * This helper ensures Roaring bitmap deserialization and other binary APIs
+ * always receive a Uint8Array.
+ *
+ * @param {Uint8Array|ArrayLike<number>} value
+ * @returns {Uint8Array}
+ */
+export default function toBytes(value) {
+  if (value instanceof Uint8Array) {
+    return value;
+  }
+  return Uint8Array.from(value);
+}

package/src/domain/warp/_wiredMethods.d.ts

@@ -151,6 +151,7 @@ interface CheckpointData {
   stateHash: string;
   schema: number;
   provenanceIndex?: unknown;
+  indexShardOids?: Record<string, string>;
 }
 
 export {};
@@ -247,8 +248,13 @@ declare module '../WarpGraph.js' {
     // ── materializeAdvanced.methods.js ────────────────────────────────────
     _resolveCeiling(options?: { ceiling?: number | null }): number | null;
     _buildAdjacency(state: WarpStateV5): { outgoing: Map<string, Array<{ neighborId: string; label: string }>>; incoming: Map<string, Array<{ neighborId: string; label: string }>> };
-    _setMaterializedState(state: WarpStateV5): Promise<{ state: WarpStateV5; stateHash: string; adjacency: unknown }>;
+    _buildView(state: WarpStateV5, stateHash: string, diff?: import('../types/PatchDiff.js').PatchDiff): void;
+    _setMaterializedState(state: WarpStateV5, diff?: import('../types/PatchDiff.js').PatchDiff): Promise<{ state: WarpStateV5; stateHash: string; adjacency: unknown }>;
     _materializeWithCeiling(ceiling: number, collectReceipts: boolean, t0: number): Promise<WarpStateV5 | { state: WarpStateV5; receipts: TickReceipt[] }>;
+    _persistSeekCacheEntry(cacheKey: string, buf: Buffer, state: WarpStateV5): Promise<void>;
+    _restoreIndexFromCache(indexTreeOid: string): Promise<void>;
     materializeAt(checkpointSha: string): Promise<WarpStateV5>;
+    verifyIndex(options?: { seed?: number; sampleRate?: number }): { passed: number; failed: number; errors: Array<{ nodeId: string; direction: string; expected: string[]; actual: string[] }> };
+    invalidateIndex(): void;
   }
 }

package/src/domain/warp/checkpoint.methods.js

@@ -60,7 +60,22 @@ export async function createCheckpoint() {
       this._checkpointing = prevCheckpointing;
     }
 
-    // 4. Call CheckpointService.create() with provenance index if available
+    // 4. Reuse cached index tree or rebuild from view service
+    let indexTree = this._cachedIndexTree;
+    if (!indexTree && this._viewService) {
+      try {
+        const { tree } = this._viewService.build(state);
+        indexTree = tree;
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        this._logger?.warn('[warp] checkpoint index build failed; saving checkpoint without index', {
+          error: message,
+        });
+        indexTree = null;
+      }
+    }
+
+    // 5. Create checkpoint commit with provenance index + index tree
     /** @type {CorePersistence} */
     const persistence = this._persistence;
     const checkpointSha = await createCheckpointCommit({
@@ -72,15 +87,16 @@ export async function createCheckpoint() {
       provenanceIndex: this._provenanceIndex || undefined,
       crypto: this._crypto,
       codec: this._codec,
+      indexTree: indexTree || undefined,
     });
 
-    // 5. Update checkpoint ref
+    // 6. Update checkpoint ref
     const checkpointRef = buildCheckpointRef(this._graphName);
     await this._persistence.updateRef(checkpointRef, checkpointSha);
 
     this._logTiming('createCheckpoint', t0);
 
-    // 6. Return checkpoint SHA
+    // 7. Return checkpoint SHA
     return checkpointSha;
   } catch (err) {
     this._logTiming('createCheckpoint', t0, { error: /** @type {Error} */ (err) });
@@ -137,7 +153,7 @@ export async function syncCoverage() {
  * Loads the latest checkpoint for this graph.
  *
  * @this {import('../WarpGraph.js').default}
- * @returns {Promise<{state: import('../services/JoinReducer.js').WarpStateV5, frontier: Map<string, string>, stateHash: string, schema: number, provenanceIndex?: import('../services/ProvenanceIndex.js').ProvenanceIndex}|null>} The checkpoint or null
+ * @returns {Promise<{state: import('../services/JoinReducer.js').WarpStateV5, frontier: Map<string, string>, stateHash: string, schema: number, provenanceIndex?: import('../services/ProvenanceIndex.js').ProvenanceIndex, indexShardOids?: Record<string, string>|null}|null>} The checkpoint or null
  * @private
  */
 export async function _loadLatestCheckpoint() {
@@ -197,7 +213,7 @@ export async function _loadPatchesSince(checkpoint) {
  */
 export async function _validateMigrationBoundary() {
   const checkpoint = await this._loadLatestCheckpoint();
-  if (checkpoint?.schema === 2 || checkpoint?.schema === 3) {
+  if (checkpoint?.schema === 2 || checkpoint?.schema === 3 || checkpoint?.schema === 4) {
     return;  // Already migrated
   }
 

package/src/domain/warp/materialize.methods.js

@@ -51,6 +51,7 @@ function scanPatchesForMaxLamport(graph, patches) {
   }
 }
 
+
 /**
  * Materializes the current graph state.
  *
@@ -99,10 +100,13 @@ export async function materialize(options) {
     let state;
     /** @type {import('../types/TickReceipt.js').TickReceipt[]|undefined} */
     let receipts;
+    /** @type {import('../types/PatchDiff.js').PatchDiff|undefined} */
+    let diff;
     let patchCount = 0;
+    const wantDiff = !collectReceipts && !!this._cachedIndexTree;
 
     // If checkpoint exists, use incremental materialization
-    if (checkpoint?.schema === 2 || checkpoint?.schema === 3) {
+    if (checkpoint?.schema === 2 || checkpoint?.schema === 3 || checkpoint?.schema === 4) {
       const patches = await this._loadPatchesSince(checkpoint);
       // Update max observed Lamport so _nextLamport() issues globally-monotonic ticks.
       // Read the checkpoint frontier's tip commit messages to capture the pre-checkpoint max,
@@ -115,6 +119,10 @@ export async function materialize(options) {
         const result = /** @type {{state: import('../services/JoinReducer.js').WarpStateV5, receipts: import('../types/TickReceipt.js').TickReceipt[]}} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (patches), checkpoint.state, { receipts: true }));
         state = result.state;
         receipts = result.receipts;
+      } else if (wantDiff) {
+        const result = /** @type {{state: import('../services/JoinReducer.js').WarpStateV5, diff: import('../types/PatchDiff.js').PatchDiff}} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (patches), checkpoint.state, { trackDiff: true }));
+        state = result.state;
+        diff = result.diff;
       } else {
         state = /** @type {import('../services/JoinReducer.js').WarpStateV5} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (patches), checkpoint.state));
       }
@@ -164,6 +172,10 @@ export async function materialize(options) {
             const result = /** @type {{state: import('../services/JoinReducer.js').WarpStateV5, receipts: import('../types/TickReceipt.js').TickReceipt[]}} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (allPatches), undefined, { receipts: true }));
             state = result.state;
             receipts = result.receipts;
+          } else if (wantDiff) {
+            const result = /** @type {{state: import('../services/JoinReducer.js').WarpStateV5, diff: import('../types/PatchDiff.js').PatchDiff}} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (allPatches), undefined, { trackDiff: true }));
+            state = result.state;
+            diff = result.diff;
           } else {
             state = /** @type {import('../services/JoinReducer.js').WarpStateV5} */ (reduceV5(/** @type {Parameters<typeof reduceV5>[0]} */ (allPatches)));
           }
@@ -178,7 +190,7 @@ export async function materialize(options) {
       }
     }
 
-    await this._setMaterializedState(state);
+    await this._setMaterializedState(state, diff);
     this._provenanceDegraded = false;
     this._cachedCeiling = null;
     this._cachedFrontier = null;
@@ -202,9 +214,9 @@ export async function materialize(options) {
     // Also handles deferred replay for subscribers added with replay: true before cached state
     if (this._subscribers.length > 0) {
       const hasPendingReplay = this._subscribers.some(s => s.pendingReplay);
-      const diff = diffStates(this._lastNotifiedState, state);
-      if (!isEmptyDiff(diff) || hasPendingReplay) {
-        this._notifySubscribers(diff, state);
+      const stateDelta = diffStates(this._lastNotifiedState, state);
+      if (!isEmptyDiff(stateDelta) || hasPendingReplay) {
+        this._notifySubscribers(stateDelta, state);
       }
     }
     // Clone state to prevent eager path mutations from affecting the baseline