@rljson/db 0.0.13 → 0.0.14

package/dist/db.js

@@ -1,75 +1,251 @@
-import { timeId, Route, createInsertHistoryTableCfg, Validate, BaseValidator, treeFromObject, isTimeId, getTimeIdTimestamp, createSliceIdsTableCfg, createLayerTableCfg, createCakeTableCfg } from "@rljson/rljson";
+import { timeId, syncEvents, clientId, Route, createInsertHistoryTableCfg, Validate, BaseValidator, treeFromObject, isTimeId, getTimeIdTimestamp, createSliceIdsTableCfg, createLayerTableCfg, createCakeTableCfg } from "@rljson/rljson";
 import { rmhsh, hsh, Hash, hip } from "@rljson/hash";
 import { equals, merge } from "@rljson/json";
 import { IoMem } from "@rljson/io";
 import { traverse } from "object-traversal";
 import { compileExpression } from "filtrex";
 class Connector {
-  constructor(_db, _route, _socket) {
+  constructor(_db, _route, _socket, syncConfig, clientIdentity) {
     this._db = _db;
     this._route = _route;
     this._socket = _socket;
     this._origin = timeId();
+    this._syncConfig = syncConfig;
+    this._events = syncEvents(this._route.flat);
+    if (clientIdentity) {
+      this._clientId = clientIdentity;
+    } else if (syncConfig?.includeClientIdentity) {
+      this._clientId = clientId();
+    }
+    this._maxDedup = syncConfig?.maxDedupSetSize ?? 1e4;
     this._init();
   }
   _origin;
   _callbacks = [];
   _isListening = false;
-  _sentRefs = /* @__PURE__ */ new Set();
-  _receivedRefs = /* @__PURE__ */ new Set();
+  // Two-generation dedup sets — bounded memory
+  _sentRefsCurrent = /* @__PURE__ */ new Set();
+  _sentRefsPrevious = /* @__PURE__ */ new Set();
+  _receivedRefsCurrent = /* @__PURE__ */ new Set();
+  _receivedRefsPrevious = /* @__PURE__ */ new Set();
+  _maxDedup;
+  // Sync protocol state
+  _syncConfig;
+  _clientId;
+  _events;
+  _seq = 0;
+  _lastPredecessors = [];
+  _peerSeqs = /* @__PURE__ */ new Map();
+  // ...........................................................................
+  /**
+   * Sends a ref to the server via the socket.
+   * Enriches the payload based on SyncConfig flags.
+   * @param ref - The ref to send
+   */
   send(ref) {
-    if (this._sentRefs.has(ref) || this._receivedRefs.has(ref)) return;
-    this._sentRefs.add(ref);
-    this.socket.emit(this.route.flat, {
+    if (this._hasSentRef(ref) || this._hasReceivedRef(ref)) return;
+    this._addSentRef(ref);
+    const payload = {
       o: this._origin,
       r: ref
+    };
+    if (this._syncConfig?.includeClientIdentity && this._clientId) {
+      payload.c = this._clientId;
+      payload.t = Date.now();
+    }
+    if (this._syncConfig?.causalOrdering) {
+      payload.seq = ++this._seq;
+      if (this._lastPredecessors.length > 0) {
+        payload.p = [...this._lastPredecessors];
+      }
+    }
+    this.socket.emit(this._events.ref, payload);
+  }
+  // ...........................................................................
+  /**
+   * Sends a ref and waits for server acknowledgment.
+   * Only meaningful when `syncConfig.requireAck` is `true`.
+   * @param ref - The ref to send
+   * @returns A promise that resolves with the AckPayload
+   */
+  async sendWithAck(ref) {
+    const timeoutMs = this._syncConfig?.ackTimeoutMs ?? 1e4;
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this._socket.off(this._events.ack, handler);
+        reject(new Error(`ACK timeout for ref ${ref} after ${timeoutMs}ms`));
+      }, timeoutMs);
+      const handler = (ack) => {
+        if (ack.r === ref) {
+          clearTimeout(timeout);
+          this._socket.off(this._events.ack, handler);
+          resolve(ack);
+        }
+      };
+      this._socket.on(this._events.ack, handler);
+      this.send(ref);
     });
   }
+  // ...........................................................................
+  /**
+   * Sets the causal predecessors for the next send.
+   * @param predecessors - The InsertHistory timeIds of causal predecessors
+   */
+  setPredecessors(predecessors) {
+    this._lastPredecessors = predecessors;
+  }
+  // ...........................................................................
+  /**
+   * Registers a callback for incoming refs on this route.
+   *
+   * Incoming refs are processed through the full sync pipeline:
+   * origin filtering, dedup, gap detection, and ACK.
+   *
+   * @param callback - The callback to invoke with each deduplicated incoming ref
+   */
   listen(callback) {
-    this._socket.on(this._route.flat, async (payload) => {
-      try {
-        await callback(payload.r);
-      } catch (error) {
-        console.error("Error in connector listener callback:", error);
-      }
-    });
+    this._callbacks.push(callback);
+  }
+  // ...........................................................................
+  /**
+   * Returns the current sequence number.
+   * Only meaningful when `causalOrdering` is enabled.
+   */
+  get seq() {
+    return this._seq;
+  }
+  // ...........................................................................
+  /**
+   * Returns the stable client identity.
+   * Only available when `includeClientIdentity` is enabled.
+   */
+  get clientIdentity() {
+    return this._clientId;
+  }
+  // ...........................................................................
+  /**
+   * Returns the sync configuration, if any.
+   */
+  get syncConfig() {
+    return this._syncConfig;
+  }
+  // ...........................................................................
+  /**
+   * Returns the typed event names for this connector's route.
+   */
+  get events() {
+    return this._events;
   }
+  // ######################
+  // Private
+  // ######################
   _init() {
     this._registerSocketObserver();
+    this._registerBootstrapHandler();
     this._registerDbObserver();
+    if (this._syncConfig?.causalOrdering) {
+      this._registerGapFillHandler();
+    }
     this._isListening = true;
   }
-  teardown() {
-    this._socket.removeAllListeners(this._route.flat);
+  tearDown() {
+    this._socket.removeAllListeners(this._events.ref);
+    this._socket.removeAllListeners(this._events.bootstrap);
+    if (this._syncConfig?.causalOrdering) {
+      this._socket.removeAllListeners(this._events.gapFillRes);
+    }
+    if (this._syncConfig?.requireAck) {
+      this._socket.removeAllListeners(this._events.ack);
+    }
     this._db.unregisterAllObservers(this._route);
     this._isListening = false;
   }
+  // ...........................................................................
+  // Two-generation dedup helpers
+  // ...........................................................................
+  _hasSentRef(ref) {
+    return this._sentRefsCurrent.has(ref) || this._sentRefsPrevious.has(ref);
+  }
+  _addSentRef(ref) {
+    this._sentRefsCurrent.add(ref);
+    if (this._sentRefsCurrent.size >= this._maxDedup) {
+      this._sentRefsPrevious = this._sentRefsCurrent;
+      this._sentRefsCurrent = /* @__PURE__ */ new Set();
+    }
+  }
+  _hasReceivedRef(ref) {
+    return this._receivedRefsCurrent.has(ref) || this._receivedRefsPrevious.has(ref);
+  }
+  _addReceivedRef(ref) {
+    this._receivedRefsCurrent.add(ref);
+    if (this._receivedRefsCurrent.size >= this._maxDedup) {
+      this._receivedRefsPrevious = this._receivedRefsCurrent;
+      this._receivedRefsCurrent = /* @__PURE__ */ new Set();
+    }
+  }
   _notifyCallbacks(ref) {
     Promise.all(this._callbacks.map((cb) => cb(ref))).catch((err) => {
       console.error(`Error notifying connector callbacks for ref ${ref}:`, err);
     });
   }
+  _processIncoming(payload) {
+    const ref = payload.r;
+    if (this._hasReceivedRef(ref)) {
+      return;
+    }
+    if (this._syncConfig?.causalOrdering && payload.seq != null && payload.c) {
+      const lastSeq = this._peerSeqs.get(payload.c) ?? 0;
+      if (payload.seq > lastSeq + 1) {
+        const gapReq = {
+          route: this._route.flat,
+          afterSeq: lastSeq
+        };
+        this._socket.emit(this._events.gapFillReq, gapReq);
+      }
+      this._peerSeqs.set(payload.c, payload.seq);
+    }
+    this._addReceivedRef(ref);
+    this._notifyCallbacks(ref);
+    if (this._syncConfig?.requireAck) {
+      this._socket.emit(this._events.ackClient, { r: ref });
+    }
+  }
   _registerSocketObserver() {
-    this.socket.on(this.route.flat, (p) => {
+    this.socket.on(this._events.ref, (p) => {
       if (p.o === this._origin) {
         return;
       }
-      const ref = p.r;
-      if (this._receivedRefs.has(ref)) {
-        return;
+      this._processIncoming(p);
+    });
+  }
+  _registerGapFillHandler() {
+    this._socket.on(this._events.gapFillRes, (res) => {
+      for (const p of res.refs) {
+        this._processIncoming(p);
       }
-      this._receivedRefs.add(p.r);
-      this._notifyCallbacks(p.r);
+    });
+  }
+  /**
+   * Listens for bootstrap messages from the server.
+   * The server sends the latest ref on connect and optionally via heartbeat.
+   * _processIncoming handles dedup so already-seen refs are filtered out.
+   */
+  _registerBootstrapHandler() {
+    this._socket.on(this._events.bootstrap, (p) => {
+      this._processIncoming(p);
     });
   }
   _registerDbObserver() {
     this._db.registerObserver(this._route, (ins) => {
       return new Promise((resolve) => {
         const ref = ins[this.route.root.tableKey + "Ref"];
-        if (this._sentRefs.has(ref)) {
+        if (this._hasSentRef(ref)) {
           resolve();
           return;
         }
+        if (this._syncConfig?.causalOrdering && ins.previous?.length) {
+          this._lastPredecessors = [...ins.previous];
+        }
         this.send(ref);
         resolve();
       });
@@ -2134,7 +2310,7 @@ class ColumnFilterProcessor {
     }
   }
   // ...........................................................................
-  /* v8 ignore stop */
+  /* v8 ignore stop -- @preserve */
   static operatorsForType(type) {
     switch (type) {
       case "string":
@@ -3585,6 +3761,56 @@ class Db {
     return insertHistoryRow;
   }
   // ...........................................................................
+  /**
+   * Insert pre-decomposed tree nodes into a tree table.
+   *
+   * Unlike `insert()`, which expects a plain nested object and decomposes it
+   * via `treeFromObject()`, this method accepts an array of already-decomposed
+   * `Tree` nodes (e.g. from FsScanner). The **root node must be the last
+   * element** in the array.
+   *
+   * The method goes through the full insert pipeline:
+   * 1. Writes each node via TreeController
+   * 2. Creates an InsertHistoryRow automatically
+   * 3. Calls `notify.notify()` so Connector observers fire
+   *
+   * @param treeKey - The tree table key (must end with "Tree")
+   * @param trees - Pre-decomposed Tree nodes, root LAST
+   * @param options - Optional: skip notification or history
+   * @returns The InsertHistoryRow for the root node
+   */
+  async insertTrees(treeKey, trees, options) {
+    if (!trees || trees.length === 0) {
+      throw new Error(
+        "Db.insertTrees: trees array must contain at least one node."
+      );
+    }
+    const controller = await this.getController(treeKey);
+    const writePromises = trees.map(
+      (tree) => controller.insert("add", tree, "db.insertTrees")
+    );
+    const writeResults = await Promise.all(writePromises);
+    const lastResult = writeResults[writeResults.length - 1];
+    if (!lastResult || lastResult.length === 0) {
+      throw new Error(
+        `Db.insertTrees: TreeController returned no result for root node of table "${treeKey}".`
+      );
+    }
+    const rootResult = lastResult[0];
+    const route = Route.fromFlat(`/${treeKey}`);
+    const result = {
+      ...rootResult,
+      route: route.flat
+    };
+    if (!options?.skipHistory) {
+      await this._writeInsertHistory(treeKey, result);
+    }
+    if (!options?.skipNotification) {
+      this.notify.notify(route, result);
+    }
+    return [result];
+  }
+  // ...........................................................................
   /**
    * Recursively runs controllers based on the route of the Insert
    * @param insert - The Insert to run