@rljson/db 0.0.13 → 0.0.15

package/dist/db.js

@@ -1,75 +1,276 @@
-import { timeId, Route, createInsertHistoryTableCfg, Validate, BaseValidator, treeFromObject, isTimeId, getTimeIdTimestamp, createSliceIdsTableCfg, createLayerTableCfg, createCakeTableCfg } from "@rljson/rljson";
+import { timeId, syncEvents, clientId, Route, createInsertHistoryTableCfg, Validate, BaseValidator, treeFromObject, isTimeId, getTimeIdTimestamp, createSliceIdsTableCfg, createLayerTableCfg, createCakeTableCfg } from "@rljson/rljson";
 import { rmhsh, hsh, Hash, hip } from "@rljson/hash";
 import { equals, merge } from "@rljson/json";
 import { IoMem } from "@rljson/io";
 import { traverse } from "object-traversal";
 import { compileExpression } from "filtrex";
 class Connector {
-  constructor(_db, _route, _socket) {
+  constructor(_db, _route, _socket, syncConfig, clientIdentity) {
     this._db = _db;
     this._route = _route;
     this._socket = _socket;
     this._origin = timeId();
+    this._syncConfig = syncConfig;
+    this._events = syncEvents(this._route.flat);
+    if (clientIdentity) {
+      this._clientId = clientIdentity;
+    } else if (syncConfig?.includeClientIdentity) {
+      this._clientId = clientId();
+    }
+    this._maxDedup = syncConfig?.maxDedupSetSize ?? 1e4;
     this._init();
   }
   _origin;
   _callbacks = [];
+  _conflictCallbacks = [];
   _isListening = false;
-  _sentRefs = /* @__PURE__ */ new Set();
-  _receivedRefs = /* @__PURE__ */ new Set();
+  // Two-generation dedup sets — bounded memory
+  _sentRefsCurrent = /* @__PURE__ */ new Set();
+  _sentRefsPrevious = /* @__PURE__ */ new Set();
+  _receivedRefsCurrent = /* @__PURE__ */ new Set();
+  _receivedRefsPrevious = /* @__PURE__ */ new Set();
+  _maxDedup;
+  // Sync protocol state
+  _syncConfig;
+  _clientId;
+  _events;
+  _seq = 0;
+  _lastPredecessors = [];
+  _peerSeqs = /* @__PURE__ */ new Map();
+  // ...........................................................................
+  /**
+   * Sends a ref to the server via the socket.
+   * Enriches the payload based on SyncConfig flags.
+   * @param ref - The ref to send
+   */
   send(ref) {
-    if (this._sentRefs.has(ref) || this._receivedRefs.has(ref)) return;
-    this._sentRefs.add(ref);
-    this.socket.emit(this.route.flat, {
+    if (this._hasSentRef(ref) || this._hasReceivedRef(ref)) return;
+    this._addSentRef(ref);
+    const payload = {
       o: this._origin,
       r: ref
+    };
+    if (this._syncConfig?.includeClientIdentity && this._clientId) {
+      payload.c = this._clientId;
+      payload.t = Date.now();
+    }
+    if (this._syncConfig?.causalOrdering) {
+      payload.seq = ++this._seq;
+      if (this._lastPredecessors.length > 0) {
+        payload.p = [...this._lastPredecessors];
+      }
+    }
+    this.socket.emit(this._events.ref, payload);
+  }
+  // ...........................................................................
+  /**
+   * Sends a ref and waits for server acknowledgment.
+   * Only meaningful when `syncConfig.requireAck` is `true`.
+   * @param ref - The ref to send
+   * @returns A promise that resolves with the AckPayload
+   */
+  async sendWithAck(ref) {
+    const timeoutMs = this._syncConfig?.ackTimeoutMs ?? 1e4;
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this._socket.off(this._events.ack, handler);
+        reject(new Error(`ACK timeout for ref ${ref} after ${timeoutMs}ms`));
+      }, timeoutMs);
+      const handler = (ack) => {
+        if (ack.r === ref) {
+          clearTimeout(timeout);
+          this._socket.off(this._events.ack, handler);
+          resolve(ack);
+        }
+      };
+      this._socket.on(this._events.ack, handler);
+      this.send(ref);
     });
   }
+  // ...........................................................................
+  /**
+   * Sets the causal predecessors for the next send.
+   * @param predecessors - The InsertHistory timeIds of causal predecessors
+   */
+  setPredecessors(predecessors) {
+    this._lastPredecessors = predecessors;
+  }
+  // ...........................................................................
+  /**
+   * Registers a callback for incoming refs on this route.
+   *
+   * Incoming refs are processed through the full sync pipeline:
+   * origin filtering, dedup, gap detection, and ACK.
+   *
+   * @param callback - The callback to invoke with each deduplicated incoming ref
+   */
   listen(callback) {
-    this._socket.on(this._route.flat, async (payload) => {
-      try {
-        await callback(payload.r);
-      } catch (error) {
-        console.error("Error in connector listener callback:", error);
-      }
-    });
+    this._callbacks.push(callback);
+  }
+  // ...........................................................................
+  /**
+   * Registers a callback that fires when a DAG conflict is detected.
+   *
+   * A conflict occurs when the InsertHistory for this route's table
+   * has multiple "tips" (leaf nodes), indicating concurrent writes
+   * from different clients that have not yet been merged.
+   *
+   * Detection-only: the callback receives a `Conflict` object
+   * describing the branches. Resolution is left to upper layers.
+   * @param callback - Invoked with the detected Conflict
+   */
+  onConflict(callback) {
+    this._conflictCallbacks.push(callback);
+  }
+  // ...........................................................................
+  /**
+   * Returns the current sequence number.
+   * Only meaningful when `causalOrdering` is enabled.
+   */
+  get seq() {
+    return this._seq;
+  }
+  // ...........................................................................
+  /**
+   * Returns the stable client identity.
+   * Only available when `includeClientIdentity` is enabled.
+   */
+  get clientIdentity() {
+    return this._clientId;
+  }
+  // ...........................................................................
+  /**
+   * Returns the sync configuration, if any.
+   */
+  get syncConfig() {
+    return this._syncConfig;
+  }
+  // ...........................................................................
+  /**
+   * Returns the typed event names for this connector's route.
+   */
+  get events() {
+    return this._events;
   }
+  // ######################
+  // Private
+  // ######################
   _init() {
     this._registerSocketObserver();
+    this._registerBootstrapHandler();
     this._registerDbObserver();
+    this._registerConflictObserver();
+    if (this._syncConfig?.causalOrdering) {
+      this._registerGapFillHandler();
+    }
     this._isListening = true;
   }
-  teardown() {
-    this._socket.removeAllListeners(this._route.flat);
+  tearDown() {
+    this._socket.removeAllListeners(this._events.ref);
+    this._socket.removeAllListeners(this._events.bootstrap);
+    if (this._syncConfig?.causalOrdering) {
+      this._socket.removeAllListeners(this._events.gapFillRes);
+    }
+    if (this._syncConfig?.requireAck) {
+      this._socket.removeAllListeners(this._events.ack);
+    }
     this._db.unregisterAllObservers(this._route);
+    this._db.unregisterAllConflictObservers(this._route);
     this._isListening = false;
   }
+  // ...........................................................................
+  // Two-generation dedup helpers
+  // ...........................................................................
+  _hasSentRef(ref) {
+    return this._sentRefsCurrent.has(ref) || this._sentRefsPrevious.has(ref);
+  }
+  _addSentRef(ref) {
+    this._sentRefsCurrent.add(ref);
+    if (this._sentRefsCurrent.size >= this._maxDedup) {
+      this._sentRefsPrevious = this._sentRefsCurrent;
+      this._sentRefsCurrent = /* @__PURE__ */ new Set();
+    }
+  }
+  _hasReceivedRef(ref) {
+    return this._receivedRefsCurrent.has(ref) || this._receivedRefsPrevious.has(ref);
+  }
+  _addReceivedRef(ref) {
+    this._receivedRefsCurrent.add(ref);
+    if (this._receivedRefsCurrent.size >= this._maxDedup) {
+      this._receivedRefsPrevious = this._receivedRefsCurrent;
+      this._receivedRefsCurrent = /* @__PURE__ */ new Set();
+    }
+  }
   _notifyCallbacks(ref) {
     Promise.all(this._callbacks.map((cb) => cb(ref))).catch((err) => {
       console.error(`Error notifying connector callbacks for ref ${ref}:`, err);
     });
   }
+  _processIncoming(payload) {
+    const ref = payload.r;
+    if (this._hasReceivedRef(ref)) {
+      return;
+    }
+    if (this._syncConfig?.causalOrdering && payload.seq != null && payload.c) {
+      const lastSeq = this._peerSeqs.get(payload.c) ?? 0;
+      if (payload.seq > lastSeq + 1) {
+        const gapReq = {
+          route: this._route.flat,
+          afterSeq: lastSeq
+        };
+        this._socket.emit(this._events.gapFillReq, gapReq);
+      }
+      this._peerSeqs.set(payload.c, payload.seq);
+    }
+    this._addReceivedRef(ref);
+    this._notifyCallbacks(ref);
+    if (this._syncConfig?.requireAck) {
+      this._socket.emit(this._events.ackClient, { r: ref });
+    }
+  }
   _registerSocketObserver() {
-    this.socket.on(this.route.flat, (p) => {
+    this.socket.on(this._events.ref, (p) => {
       if (p.o === this._origin) {
         return;
       }
-      const ref = p.r;
-      if (this._receivedRefs.has(ref)) {
-        return;
+      this._processIncoming(p);
+    });
+  }
+  _registerGapFillHandler() {
+    this._socket.on(this._events.gapFillRes, (res) => {
+      for (const p of res.refs) {
+        this._processIncoming(p);
+      }
+    });
+  }
+  /**
+   * Listens for bootstrap messages from the server.
+   * The server sends the latest ref on connect and optionally via heartbeat.
+   * _processIncoming handles dedup so already-seen refs are filtered out.
+   */
+  _registerBootstrapHandler() {
+    this._socket.on(this._events.bootstrap, (p) => {
+      this._processIncoming(p);
+    });
+  }
+  _registerConflictObserver() {
+    this._db.registerConflictObserver(this._route, (conflict) => {
+      for (const cb of this._conflictCallbacks) {
+        cb(conflict);
       }
-      this._receivedRefs.add(p.r);
-      this._notifyCallbacks(p.r);
     });
   }
   _registerDbObserver() {
     this._db.registerObserver(this._route, (ins) => {
       return new Promise((resolve) => {
         const ref = ins[this.route.root.tableKey + "Ref"];
-        if (this._sentRefs.has(ref)) {
+        if (this._hasSentRef(ref)) {
           resolve();
           return;
         }
+        if (this._syncConfig?.causalOrdering && ins.previous?.length) {
+          this._lastPredecessors = [...ins.previous];
+        }
         this.send(ref);
         resolve();
       });
@@ -2134,7 +2335,7 @@ class ColumnFilterProcessor {
     }
   }
   // ...........................................................................
-  /* v8 ignore stop */
+  /* v8 ignore stop -- @preserve */
   static operatorsForType(type) {
     switch (type) {
       case "string":
@@ -2916,6 +3117,7 @@ class Db {
    * Notification system to register callbacks on data changes
    */
   notify;
+  _conflictCallbacks = /* @__PURE__ */ new Map();
   _cache = /* @__PURE__ */ new Map();
   // ...........................................................................
   /**
@@ -3585,6 +3787,56 @@ class Db {
     return insertHistoryRow;
   }
   // ...........................................................................
+  /**
+   * Insert pre-decomposed tree nodes into a tree table.
+   *
+   * Unlike `insert()`, which expects a plain nested object and decomposes it
+   * via `treeFromObject()`, this method accepts an array of already-decomposed
+   * `Tree` nodes (e.g. from FsScanner). The **root node must be the last
+   * element** in the array.
+   *
+   * The method goes through the full insert pipeline:
+   * 1. Writes each node via TreeController
+   * 2. Creates an InsertHistoryRow automatically
+   * 3. Calls `notify.notify()` so Connector observers fire
+   *
+   * @param treeKey - The tree table key (must end with "Tree")
+   * @param trees - Pre-decomposed Tree nodes, root LAST
+   * @param options - Optional: skip notification or history
+   * @returns The InsertHistoryRow for the root node
+   */
+  async insertTrees(treeKey, trees, options) {
+    if (!trees || trees.length === 0) {
+      throw new Error(
+        "Db.insertTrees: trees array must contain at least one node."
+      );
+    }
+    const controller = await this.getController(treeKey);
+    const writePromises = trees.map(
+      (tree) => controller.insert("add", tree, "db.insertTrees")
+    );
+    const writeResults = await Promise.all(writePromises);
+    const lastResult = writeResults[writeResults.length - 1];
+    if (!lastResult || lastResult.length === 0) {
+      throw new Error(
+        `Db.insertTrees: TreeController returned no result for root node of table "${treeKey}".`
+      );
+    }
+    const rootResult = lastResult[0];
+    const route = Route.fromFlat(`/${treeKey}`);
+    const result = {
+      ...rootResult,
+      route: route.flat
+    };
+    if (!options?.skipHistory) {
+      await this._writeInsertHistory(treeKey, result);
+    }
+    if (!options?.skipNotification) {
+      this.notify.notify(route, result);
+    }
+    return [result];
+  }
+  // ...........................................................................
   /**
    * Recursively runs controllers based on the route of the Insert
    * @param insert - The Insert to run
@@ -3835,6 +4087,44 @@ class Db {
     this.notify.unregisterAll(route);
   }
   // ...........................................................................
+  /**
+   * Registers a callback to be called when a DAG conflict is detected
+   * on the given route.
+   * @param route - The route to register the conflict callback on
+   * @param callback - The callback to invoke with the Conflict
+   */
+  registerConflictObserver(route, callback) {
+    const key = route.flat;
+    this._conflictCallbacks.set(key, [
+      ...this._conflictCallbacks.get(key) || [],
+      callback
+    ]);
+  }
+  // ...........................................................................
+  /**
+   * Unregisters a specific conflict callback from the given route.
+   * @param route - The route to unregister the callback from
+   * @param callback - The callback to remove
+   */
+  unregisterConflictObserver(route, callback) {
+    const key = route.flat;
+    const callbacks = this._conflictCallbacks.get(key);
+    if (callbacks) {
+      this._conflictCallbacks.set(
+        key,
+        callbacks.filter((cb) => cb !== callback)
+      );
+    }
+  }
+  // ...........................................................................
+  /**
+   * Unregisters all conflict callbacks from the given route.
+   * @param route - The route to clear conflict callbacks for
+   */
+  unregisterAllConflictObservers(route) {
+    this._conflictCallbacks.delete(route.flat);
+  }
+  // ...........................................................................
   /**
    * Get a controller for a specific table
    * @param tableKey - The key of the table to get the controller for
@@ -3867,6 +4157,57 @@ class Db {
     return controllers;
   }
   // ...........................................................................
+  /**
+   * Detects whether the InsertHistory for a table has diverged into
+   * multiple branches (multiple "tips" — leaf nodes in the DAG).
+   *
+   * A tip is a timeId that is NOT referenced as `previous` by any other
+   * InsertHistory row. Two or more tips indicate a DAG fork (conflict).
+   * @param table - The table name (without "InsertHistory" suffix)
+   * @returns A Conflict if a DAG branch is detected, or null otherwise
+   */
+  async detectDagBranch(table) {
+    const insertHistoryTable = table + "InsertHistory";
+    const hasTable = await this.core.hasTable(insertHistoryTable);
+    if (!hasTable) return null;
+    const dump = await this.core.dumpTable(insertHistoryTable);
+    const rows = dump[insertHistoryTable]._data;
+    if (rows.length < 2) return null;
+    const referencedAsParent = /* @__PURE__ */ new Set();
+    for (const row of rows) {
+      if (row.previous) {
+        for (const p of row.previous) {
+          referencedAsParent.add(p);
+        }
+      }
+    }
+    const tips = rows.filter((row) => !referencedAsParent.has(row.timeId));
+    if (tips.length > 1) {
+      return {
+        table,
+        type: "dagBranch",
+        detectedAt: Date.now(),
+        branches: tips.map((t) => t.timeId)
+      };
+    }
+    return null;
+  }
+  // ...........................................................................
+  /**
+   * Fires conflict callbacks registered on the route derived from the table.
+   * @param table - The table name
+   * @param conflict - The detected conflict
+   */
+  _notifyConflict(table, conflict) {
+    const route = Route.fromFlat(`/${table}`);
+    const callbacks = this._conflictCallbacks.get(route.flat);
+    if (callbacks) {
+      for (const cb of callbacks) {
+        cb(conflict);
+      }
+    }
+  }
+  // ...........................................................................
   /**
    * Adds an InsertHistory row to the InsertHistory table of a table
    * @param table - The table the Insert was made on
@@ -3881,6 +4222,10 @@ class Db {
         _type: "insertHistory"
       }
     });
+    const conflict = await this.detectDagBranch(table);
+    if (conflict) {
+      this._notifyConflict(table, conflict);
+    }
   }
   // ...........................................................................
   /**