@rljson/db 0.0.12 → 0.0.14

package/dist/README.trouble.md

@@ -6,9 +6,58 @@
 
 ## Table of contents <!-- omit in toc -->
 
+- [Tree INSERT operations failing with empty results](#tree-insert-operations-failing-with-empty-results)
 - [Vscode Windows: Debugging is not working](#vscode-windows-debugging-is-not-working)
 - [GitHub actions: Cannot find module @rollup/rollup-linux-x64-gnu](#github-actions-cannot-find-module-rolluprollup-linux-x64-gnu)
 
+## Tree INSERT operations failing with empty results
+
+Date: 2026-01-28
+
+### Symptoms
+
+- Tree INSERT operations complete without errors
+- GET queries after INSERT return empty results or only root node
+- Cell length is 1 instead of expected count
+- Navigation stops at root level
+
+### Root Cause
+
+The `treeFromObject` function from `@rljson/rljson` v0.0.74+ creates an explicit root node with `id='root'` at the end of the tree array. When inserting an already-isolated subtree (from `isolate()`), this created a double-root structure:
+
+```
+Auto-root (id='root')
+  └─ User-root (id='root')
+       └─ actual data nodes
+```
+
+When `TreeController` navigates the tree, it stops at the first node with `id='root'` (the auto-root), which has no meaningful data.
+
+### Solution
+
+✅ **Fixed in current version**: The `db.ts` file now calls `treeFromObject` with `skipRootCreation: true` parameter:
+
+```typescript
+const trees = treeFromObject(treeObject, true);
+```
+
+This prevents the automatic root wrapper from being created during INSERT operations.
+
+### Verification
+
+Run the tree INSERT tests:
+
+```bash
+pnpm test --run --grep "insert on tree"
+```
+
+All tree INSERT tests should pass:
+
+- "insert on tree root node"
+- "insert on tree deeper leaf"
+- "insert on tree simple branch"
+- "insert new child on branch"
+
 ## Vscode Windows: Debugging is not working
 
 Date: 2025-03-08

package/dist/connector/connector.d.ts

@@ -1,10 +1,7 @@
 import { Socket } from '@rljson/io';
-import { Route } from '@rljson/rljson';
+import { AckPayload, ClientId, InsertHistoryTimeId, Route, SyncConfig, SyncEventNames } from '@rljson/rljson';
 import { Db } from '../db.ts';
-export type ConnectorPayload = {
-    o: string;
-    r: string;
-};
+export type { ConnectorPayload } from '@rljson/rljson';
 export type ConnectorCallback = (ref: string) => Promise<any>;
 export declare class Connector {
     private readonly _db;
@@ -13,15 +10,79 @@ export declare class Connector {
     private _origin;
     private _callbacks;
     private _isListening;
-    private _sentRefs;
-    private _receivedRefs;
-    constructor(_db: Db, _route: Route, _socket: Socket);
+    private _sentRefsCurrent;
+    private _sentRefsPrevious;
+    private _receivedRefsCurrent;
+    private _receivedRefsPrevious;
+    private readonly _maxDedup;
+    private readonly _syncConfig;
+    private readonly _clientId;
+    private readonly _events;
+    private _seq;
+    private _lastPredecessors;
+    private _peerSeqs;
+    constructor(_db: Db, _route: Route, _socket: Socket, syncConfig?: SyncConfig, clientIdentity?: ClientId);
+    /**
+     * Sends a ref to the server via the socket.
+     * Enriches the payload based on SyncConfig flags.
+     * @param ref - The ref to send
+     */
     send(ref: string): void;
-    listen(callback: (editHistoryRef: string) => Promise<void>): void;
+    /**
+     * Sends a ref and waits for server acknowledgment.
+     * Only meaningful when `syncConfig.requireAck` is `true`.
+     * @param ref - The ref to send
+     * @returns A promise that resolves with the AckPayload
+     */
+    sendWithAck(ref: string): Promise<AckPayload>;
+    /**
+     * Sets the causal predecessors for the next send.
+     * @param predecessors - The InsertHistory timeIds of causal predecessors
+     */
+    setPredecessors(predecessors: InsertHistoryTimeId[]): void;
+    /**
+     * Registers a callback for incoming refs on this route.
+     *
+     * Incoming refs are processed through the full sync pipeline:
+     * origin filtering, dedup, gap detection, and ACK.
+     *
+     * @param callback - The callback to invoke with each deduplicated incoming ref
+     */
+    listen(callback: ConnectorCallback): void;
+    /**
+     * Returns the current sequence number.
+     * Only meaningful when `causalOrdering` is enabled.
+     */
+    get seq(): number;
+    /**
+     * Returns the stable client identity.
+     * Only available when `includeClientIdentity` is enabled.
+     */
+    get clientIdentity(): ClientId | undefined;
+    /**
+     * Returns the sync configuration, if any.
+     */
+    get syncConfig(): SyncConfig | undefined;
+    /**
+     * Returns the typed event names for this connector's route.
+     */
+    get events(): SyncEventNames;
     private _init;
-    teardown(): void;
+    tearDown(): void;
+    private _hasSentRef;
+    private _addSentRef;
+    private _hasReceivedRef;
+    private _addReceivedRef;
     private _notifyCallbacks;
+    private _processIncoming;
     private _registerSocketObserver;
+    private _registerGapFillHandler;
+    /**
+     * Listens for bootstrap messages from the server.
+     * The server sends the latest ref on connect and optionally via heartbeat.
+     * _processIncoming handles dedup so already-seen refs are filtered out.
+     */
+    private _registerBootstrapHandler;
     private _registerDbObserver;
     get socket(): Socket;
     get route(): Route;

package/dist/db.d.ts

@@ -1,6 +1,6 @@
 import { Io } from '@rljson/io';
 import { Json, JsonValue } from '@rljson/json';
-import { Edit, EditHistory, InsertHistoryRow, InsertHistoryTimeId, MultiEdit, Ref, Rljson, Route, SliceId, TableType } from '@rljson/rljson';
+import { Edit, EditHistory, InsertHistoryRow, InsertHistoryTimeId, MultiEdit, Ref, Rljson, Route, SliceId, TableType, Tree } from '@rljson/rljson';
 import { Controller, ControllerChildProperty, ControllerRefs } from './controller/controller.ts';
 import { Core } from './core.ts';
 import { Join } from './join/join.ts';
@@ -90,6 +90,28 @@ export declare class Db {
         skipNotification?: boolean;
         skipHistory?: boolean;
     }): Promise<InsertHistoryRow<any>[]>;
+    /**
+     * Insert pre-decomposed tree nodes into a tree table.
+     *
+     * Unlike `insert()`, which expects a plain nested object and decomposes it
+     * via `treeFromObject()`, this method accepts an array of already-decomposed
+     * `Tree` nodes (e.g. from FsScanner). The **root node must be the last
+     * element** in the array.
+     *
+     * The method goes through the full insert pipeline:
+     * 1. Writes each node via TreeController
+     * 2. Creates an InsertHistoryRow automatically
+     * 3. Calls `notify.notify()` so Connector observers fire
+     *
+     * @param treeKey - The tree table key (must end with "Tree")
+     * @param trees - Pre-decomposed Tree nodes, root LAST
+     * @param options - Optional: skip notification or history
+     * @returns The InsertHistoryRow for the root node
+     */
+    insertTrees(treeKey: string, trees: Tree[], options?: {
+        skipNotification?: boolean;
+        skipHistory?: boolean;
+    }): Promise<InsertHistoryRow<any>[]>;
     /**
      * Recursively runs controllers based on the route of the Insert
      * @param insert - The Insert to run

package/dist/db.js

@@ -1,75 +1,251 @@
-import { timeId, Route, createInsertHistoryTableCfg, Validate, BaseValidator, treeFromObject, isTimeId, getTimeIdTimestamp, createSliceIdsTableCfg, createLayerTableCfg, createCakeTableCfg } from "@rljson/rljson";
+import { timeId, syncEvents, clientId, Route, createInsertHistoryTableCfg, Validate, BaseValidator, treeFromObject, isTimeId, getTimeIdTimestamp, createSliceIdsTableCfg, createLayerTableCfg, createCakeTableCfg } from "@rljson/rljson";
 import { rmhsh, hsh, Hash, hip } from "@rljson/hash";
 import { equals, merge } from "@rljson/json";
 import { IoMem } from "@rljson/io";
 import { traverse } from "object-traversal";
 import { compileExpression } from "filtrex";
 class Connector {
-  constructor(_db, _route, _socket) {
+  constructor(_db, _route, _socket, syncConfig, clientIdentity) {
     this._db = _db;
     this._route = _route;
     this._socket = _socket;
     this._origin = timeId();
+    this._syncConfig = syncConfig;
+    this._events = syncEvents(this._route.flat);
+    if (clientIdentity) {
+      this._clientId = clientIdentity;
+    } else if (syncConfig?.includeClientIdentity) {
+      this._clientId = clientId();
+    }
+    this._maxDedup = syncConfig?.maxDedupSetSize ?? 1e4;
     this._init();
   }
   _origin;
   _callbacks = [];
   _isListening = false;
-  _sentRefs = /* @__PURE__ */ new Set();
-  _receivedRefs = /* @__PURE__ */ new Set();
+  // Two-generation dedup sets — bounded memory
+  _sentRefsCurrent = /* @__PURE__ */ new Set();
+  _sentRefsPrevious = /* @__PURE__ */ new Set();
+  _receivedRefsCurrent = /* @__PURE__ */ new Set();
+  _receivedRefsPrevious = /* @__PURE__ */ new Set();
+  _maxDedup;
+  // Sync protocol state
+  _syncConfig;
+  _clientId;
+  _events;
+  _seq = 0;
+  _lastPredecessors = [];
+  _peerSeqs = /* @__PURE__ */ new Map();
+  // ...........................................................................
+  /**
+   * Sends a ref to the server via the socket.
+   * Enriches the payload based on SyncConfig flags.
+   * @param ref - The ref to send
+   */
   send(ref) {
-    if (this._sentRefs.has(ref) || this._receivedRefs.has(ref)) return;
-    this._sentRefs.add(ref);
-    this.socket.emit(this.route.flat, {
+    if (this._hasSentRef(ref) || this._hasReceivedRef(ref)) return;
+    this._addSentRef(ref);
+    const payload = {
       o: this._origin,
       r: ref
+    };
+    if (this._syncConfig?.includeClientIdentity && this._clientId) {
+      payload.c = this._clientId;
+      payload.t = Date.now();
+    }
+    if (this._syncConfig?.causalOrdering) {
+      payload.seq = ++this._seq;
+      if (this._lastPredecessors.length > 0) {
+        payload.p = [...this._lastPredecessors];
+      }
+    }
+    this.socket.emit(this._events.ref, payload);
+  }
+  // ...........................................................................
+  /**
+   * Sends a ref and waits for server acknowledgment.
+   * Only meaningful when `syncConfig.requireAck` is `true`.
+   * @param ref - The ref to send
+   * @returns A promise that resolves with the AckPayload
+   */
+  async sendWithAck(ref) {
+    const timeoutMs = this._syncConfig?.ackTimeoutMs ?? 1e4;
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this._socket.off(this._events.ack, handler);
+        reject(new Error(`ACK timeout for ref ${ref} after ${timeoutMs}ms`));
+      }, timeoutMs);
+      const handler = (ack) => {
+        if (ack.r === ref) {
+          clearTimeout(timeout);
+          this._socket.off(this._events.ack, handler);
+          resolve(ack);
+        }
+      };
+      this._socket.on(this._events.ack, handler);
+      this.send(ref);
     });
   }
+  // ...........................................................................
+  /**
+   * Sets the causal predecessors for the next send.
+   * @param predecessors - The InsertHistory timeIds of causal predecessors
+   */
+  setPredecessors(predecessors) {
+    this._lastPredecessors = predecessors;
+  }
+  // ...........................................................................
+  /**
+   * Registers a callback for incoming refs on this route.
+   *
+   * Incoming refs are processed through the full sync pipeline:
+   * origin filtering, dedup, gap detection, and ACK.
+   *
+   * @param callback - The callback to invoke with each deduplicated incoming ref
+   */
   listen(callback) {
-    this._socket.on(this._route.flat, async (payload) => {
-      try {
-        await callback(payload.r);
-      } catch (error) {
-        console.error("Error in connector listener callback:", error);
-      }
-    });
+    this._callbacks.push(callback);
+  }
+  // ...........................................................................
+  /**
+   * Returns the current sequence number.
+   * Only meaningful when `causalOrdering` is enabled.
+   */
+  get seq() {
+    return this._seq;
   }
+  // ...........................................................................
+  /**
+   * Returns the stable client identity.
+   * Only available when `includeClientIdentity` is enabled.
+   */
+  get clientIdentity() {
+    return this._clientId;
+  }
+  // ...........................................................................
+  /**
+   * Returns the sync configuration, if any.
+   */
+  get syncConfig() {
+    return this._syncConfig;
+  }
+  // ...........................................................................
+  /**
+   * Returns the typed event names for this connector's route.
+   */
+  get events() {
+    return this._events;
+  }
+  // ######################
+  // Private
+  // ######################
   _init() {
     this._registerSocketObserver();
+    this._registerBootstrapHandler();
     this._registerDbObserver();
+    if (this._syncConfig?.causalOrdering) {
+      this._registerGapFillHandler();
+    }
     this._isListening = true;
   }
-  teardown() {
-    this._socket.removeAllListeners(this._route.flat);
+  tearDown() {
+    this._socket.removeAllListeners(this._events.ref);
+    this._socket.removeAllListeners(this._events.bootstrap);
+    if (this._syncConfig?.causalOrdering) {
+      this._socket.removeAllListeners(this._events.gapFillRes);
+    }
+    if (this._syncConfig?.requireAck) {
+      this._socket.removeAllListeners(this._events.ack);
+    }
     this._db.unregisterAllObservers(this._route);
     this._isListening = false;
   }
+  // ...........................................................................
+  // Two-generation dedup helpers
+  // ...........................................................................
+  _hasSentRef(ref) {
+    return this._sentRefsCurrent.has(ref) || this._sentRefsPrevious.has(ref);
+  }
+  _addSentRef(ref) {
+    this._sentRefsCurrent.add(ref);
+    if (this._sentRefsCurrent.size >= this._maxDedup) {
+      this._sentRefsPrevious = this._sentRefsCurrent;
+      this._sentRefsCurrent = /* @__PURE__ */ new Set();
+    }
+  }
+  _hasReceivedRef(ref) {
+    return this._receivedRefsCurrent.has(ref) || this._receivedRefsPrevious.has(ref);
+  }
+  _addReceivedRef(ref) {
+    this._receivedRefsCurrent.add(ref);
+    if (this._receivedRefsCurrent.size >= this._maxDedup) {
+      this._receivedRefsPrevious = this._receivedRefsCurrent;
+      this._receivedRefsCurrent = /* @__PURE__ */ new Set();
+    }
+  }
   _notifyCallbacks(ref) {
     Promise.all(this._callbacks.map((cb) => cb(ref))).catch((err) => {
       console.error(`Error notifying connector callbacks for ref ${ref}:`, err);
     });
   }
+  _processIncoming(payload) {
+    const ref = payload.r;
+    if (this._hasReceivedRef(ref)) {
+      return;
+    }
+    if (this._syncConfig?.causalOrdering && payload.seq != null && payload.c) {
+      const lastSeq = this._peerSeqs.get(payload.c) ?? 0;
+      if (payload.seq > lastSeq + 1) {
+        const gapReq = {
+          route: this._route.flat,
+          afterSeq: lastSeq
+        };
+        this._socket.emit(this._events.gapFillReq, gapReq);
+      }
+      this._peerSeqs.set(payload.c, payload.seq);
+    }
+    this._addReceivedRef(ref);
+    this._notifyCallbacks(ref);
+    if (this._syncConfig?.requireAck) {
+      this._socket.emit(this._events.ackClient, { r: ref });
+    }
+  }
   _registerSocketObserver() {
-    this.socket.on(this.route.flat, (p) => {
+    this.socket.on(this._events.ref, (p) => {
       if (p.o === this._origin) {
         return;
       }
-      const ref = p.r;
-      if (this._receivedRefs.has(ref)) {
-        return;
+      this._processIncoming(p);
+    });
+  }
+  _registerGapFillHandler() {
+    this._socket.on(this._events.gapFillRes, (res) => {
+      for (const p of res.refs) {
+        this._processIncoming(p);
       }
-      this._receivedRefs.add(p.r);
-      this._notifyCallbacks(p.r);
+    });
+  }
+  /**
+   * Listens for bootstrap messages from the server.
+   * The server sends the latest ref on connect and optionally via heartbeat.
+   * _processIncoming handles dedup so already-seen refs are filtered out.
+   */
+  _registerBootstrapHandler() {
+    this._socket.on(this._events.bootstrap, (p) => {
+      this._processIncoming(p);
     });
   }
   _registerDbObserver() {
     this._db.registerObserver(this._route, (ins) => {
       return new Promise((resolve) => {
         const ref = ins[this.route.root.tableKey + "Ref"];
-        if (this._sentRefs.has(ref)) {
+        if (this._hasSentRef(ref)) {
           resolve();
           return;
         }
+        if (this._syncConfig?.causalOrdering && ins.previous?.length) {
+          this._lastPredecessors = [...ins.previous];
+        }
         this.send(ref);
         resolve();
       });
@@ -984,6 +1160,15 @@ class TreeController extends BaseController {
     if (treeId && treeId !== tree.id) {
       return { [this._tableKey]: { _data: [], _type: "trees" } };
     }
+    const shouldExpandChildren = path !== void 0;
+    if (!shouldExpandChildren) {
+      return {
+        [this._tableKey]: {
+          _data: [tree],
+          _type: "trees"
+        }
+      };
+    }
     const children = [];
     for (const childRef of tree.children ?? []) {
       const child = await this.get(
@@ -1005,21 +1190,47 @@ class TreeController extends BaseController {
     if (trees.length === 0) {
       return {};
     }
+    if (trees.length > 1e5) {
+      throw new Error(
+        `TreeController.buildTreeFromTrees: Tree size exceeds limit (${trees.length} > 100000 nodes). This may indicate a performance issue or data structure problem.`
+      );
+    }
     const treeMap = /* @__PURE__ */ new Map();
     for (const tree of trees) {
       treeMap.set(tree._hash, tree);
     }
-    const buildObject = (tree) => {
+    const memo = /* @__PURE__ */ new Map();
+    let buildObjectCallCount = 0;
+    const MAX_ITERATIONS = 1e6;
+    const buildObject = (tree, depth = 0) => {
+      buildObjectCallCount++;
+      if (buildObjectCallCount > MAX_ITERATIONS) {
+        throw new Error(
+          `TreeController.buildTreeFromTrees: Maximum iterations (${MAX_ITERATIONS}) exceeded. This likely indicates a bug. Processed ${buildObjectCallCount} nodes from ${trees.length} total.`
+        );
+      }
+      if (depth > 1e4) {
+        throw new Error(
+          `TreeController.buildTreeFromTrees: Tree depth exceeds limit (${depth} > 10000). This may indicate a circular reference or extremely deep structure.`
+        );
+      }
+      const hash = tree._hash;
+      if (memo.has(hash)) {
+        return memo.get(hash);
+      }
       if (!tree.isParent || !tree.children || tree.children.length === 0) {
-        return tree;
+        const result3 = tree;
+        memo.set(hash, result3);
+        return result3;
       }
       const result2 = {};
       for (const childHash of tree.children) {
         const childTree = treeMap.get(childHash);
         if (childTree && childTree.id) {
-          result2[childTree.id] = buildObject(childTree);
+          result2[childTree.id] = buildObject(childTree, depth + 1);
         }
       }
+      memo.set(hash, result2);
       return result2;
     };
     const referencedHashes = /* @__PURE__ */ new Set();
@@ -1105,6 +1316,9 @@ class TreeController extends BaseController {
     return cells;
   }
   async getChildRefs(where, filter) {
+    if (typeof where === "string") {
+      return [];
+    }
     const childRefs = [];
     const { [this._tableKey]: table } = await this.get(where, filter);
     const trees = table._data;
@@ -1241,7 +1455,11 @@ class Core {
   // ...........................................................................
   /** Reads a specific row from a database table */
   async readRow(table, rowHash) {
-    return await this._io.readRows({ table, where: { _hash: rowHash } });
+    const result = await this._io.readRows({
+      table,
+      where: { _hash: rowHash }
+    });
+    return result;
   }
   // ...........................................................................
   async readRows(table, where) {
@@ -2092,7 +2310,7 @@ class ColumnFilterProcessor {
     }
   }
   // ...........................................................................
-  /* v8 ignore stop */
+  /* v8 ignore stop -- @preserve */
   static operatorsForType(type) {
     switch (type) {
       case "string":
@@ -2919,6 +3137,12 @@ class Db {
    * @returns - An Rljson object matching the route and filters
    */
   async _get(route, where, controllers, filter, sliceIds, routeAccumulator, options) {
+    const depth = routeAccumulator ? routeAccumulator.flat.split("/").length : 1;
+    if (depth > 100) {
+      throw new Error(
+        `Db._get: Maximum recursion depth (100) exceeded. This likely indicates a bug in route resolution or where clause construction. Current route: ${route.flat}, routeAccumulator: ${routeAccumulator?.flat}`
+      );
+    }
     const opts = options ?? {};
     const routeHasRefs = route.flat != route.flatWithoutRefs;
     const hasFilter = filter !== void 0 && filter.length > 0;
@@ -3537,6 +3761,56 @@ class Db {
     return insertHistoryRow;
   }
   // ...........................................................................
+  /**
+   * Insert pre-decomposed tree nodes into a tree table.
+   *
+   * Unlike `insert()`, which expects a plain nested object and decomposes it
+   * via `treeFromObject()`, this method accepts an array of already-decomposed
+   * `Tree` nodes (e.g. from FsScanner). The **root node must be the last
+   * element** in the array.
+   *
+   * The method goes through the full insert pipeline:
+   * 1. Writes each node via TreeController
+   * 2. Creates an InsertHistoryRow automatically
+   * 3. Calls `notify.notify()` so Connector observers fire
+   *
+   * @param treeKey - The tree table key (must end with "Tree")
+   * @param trees - Pre-decomposed Tree nodes, root LAST
+   * @param options - Optional: skip notification or history
+   * @returns The InsertHistoryRow for the root node
+   */
+  async insertTrees(treeKey, trees, options) {
+    if (!trees || trees.length === 0) {
+      throw new Error(
+        "Db.insertTrees: trees array must contain at least one node."
+      );
+    }
+    const controller = await this.getController(treeKey);
+    const writePromises = trees.map(
+      (tree) => controller.insert("add", tree, "db.insertTrees")
+    );
+    const writeResults = await Promise.all(writePromises);
+    const lastResult = writeResults[writeResults.length - 1];
+    if (!lastResult || lastResult.length === 0) {
+      throw new Error(
+        `Db.insertTrees: TreeController returned no result for root node of table "${treeKey}".`
+      );
+    }
+    const rootResult = lastResult[0];
+    const route = Route.fromFlat(`/${treeKey}`);
+    const result = {
+      ...rootResult,
+      route: route.flat
+    };
+    if (!options?.skipHistory) {
+      await this._writeInsertHistory(treeKey, result);
+    }
+    if (!options?.skipNotification) {
+      this.notify.notify(route, result);
+    }
+    return [result];
+  }
+  // ...........................................................................
   /**
    * Recursively runs controllers based on the route of the Insert
    * @param insert - The Insert to run
@@ -3738,7 +4012,7 @@ class Db {
             `Db._insert: No tree data found for table "${nodeTableKey}" in route "${route.flat}".`
           );
         }
-        const trees = treeFromObject(treeObject);
+        const trees = treeFromObject(treeObject, true);
         const writePromises = trees.map(
           (tree2) => runFn("add", tree2, "db.insert")
         );
@@ -3862,9 +4136,7 @@ class Db {
     const multiEditController = await this.getController(
       cakeKey + "MultiEdits"
     );
-    const { [cakeKey + "MultiEdits"]: result } = await multiEditController.get(
-      where
-    );
+    const { [cakeKey + "MultiEdits"]: result } = await multiEditController.get(where);
     return result._data;
   }
   // ...........................................................................
@@ -6091,6 +6363,10 @@ export {
   exampleEditActionRowSort,
   exampleEditActionSetValue,
   exampleEditSetValueReferenced,
+  inject,
+  isolate,
+  makeUnique,
+  mergeTrees,
   staticExample
 };
 //# sourceMappingURL=db.js.map