@rljson/io 0.0.68 → 0.0.70

package/README.architecture.md

@@ -18,7 +18,7 @@ found in the LICENSE file in the root of this package.
 
 The central abstraction that defines all database operations:
 
-```
+```text
 ┌─────────────────────────────────────────────┐
 │              Io Interface                   │
 ├─────────────────────────────────────────────┤
@@ -42,7 +42,7 @@ The central abstraction that defines all database operations:
 
 In-memory implementation using JavaScript objects.
 
-```
+```text
 ┌─────────────────────┐
 │      IoMem          │
 ├─────────────────────┤
@@ -61,6 +61,21 @@ In-memory implementation using JavaScript objects.
 - Uses `@rljson/hash` for data hashing and identity
 - `IsReady` pattern for initialization tracking
 
+**Performance internals** (no observable API change):
+
+- Rows are indexed per table by content hash — write dedup is O(1) and
+  `readRows` with a string `_hash` in the where clause resolves through
+  the index instead of scanning the table
+- Table data is kept hash-sorted incrementally (binary-searched insert)
+  instead of re-sorting the whole table per write
+- Table and global hashes are updated lazily: writes mark tables dirty,
+  hashes are recomputed before they become observable (`dump`,
+  `dumpTable`, `createOrExtendTable`, re-`init`) — deterministic hashes
+  make the refreshed state identical to eager updates
+- The latest table configuration and column keys are cached per table
+  (updated on create/extend), so validation does not re-scan all
+  configurations per read/write
+
 **Data Structure:**
 
 ```typescript
@@ -76,7 +91,7 @@ _mem = {
 
 Remote database connection over sockets (Socket.IO compatible).
 
-```
+```text
 ┌──────────────┐         Socket          ┌──────────────┐
 │   IoPeer     │◄───────────────────────►│ IoPeerBridge │
 │  (Client)    │     Events/Acks         │   (Server)   │
@@ -107,7 +122,7 @@ Remote database connection over sockets (Socket.IO compatible).
 
 Server-side handler that bridges socket events to Io operations.
 
-```
+```text
 ┌─────────────────────────────────────┐
 │        IoPeerBridge                 │
 ├─────────────────────────────────────┤
@@ -132,7 +147,7 @@ Server-side handler that bridges socket events to Io operations.
 
 Aggregates multiple Io instances with priority-based cascading.
 
-```
+```text
 ┌────────────────────────────────────────────┐
 │             IoMulti                        │
 ├────────────────────────────────────────────┤
@@ -182,7 +197,7 @@ IoMultiIo {
 
 Provides name mapping between different table name formats.
 
-```
+```text
 ┌──────────────────────────────────────┐
 │     IoDbNameMapping                  │
 ├──────────────────────────────────────┤
@@ -202,7 +217,7 @@ Provides name mapping between different table name formats.
 
 Server implementation that combines Socket.IO with Io backends.
 
-```
+```text
 ┌─────────────────────────────────────┐
 │         IoServer                    │
 ├─────────────────────────────────────┤
@@ -230,7 +245,7 @@ Server implementation that combines Socket.IO with Io backends.
 
 ### Write Operation
 
-```
+```text
 Client Code
     │
     ▼
@@ -248,7 +263,7 @@ io.write(data)
 
 ### Read Operation (IoMulti Cascade)
 
-```
+```text
 io.readRows({table: 'users', where: {id: 1}})
     │
     ▼
@@ -293,7 +308,7 @@ return rljson; // Only after loop completes
 
 ### Event-Based Protocol
 
-```
+```text
 Client (IoPeer)                    Server (IoPeerBridge)
       │                                    │
       ├─── emit('readRows', request) ────►│

package/dist/README.architecture.md

@@ -18,7 +18,7 @@ found in the LICENSE file in the root of this package.
 
 The central abstraction that defines all database operations:
 
-```
+```text
 ┌─────────────────────────────────────────────┐
 │              Io Interface                   │
 ├─────────────────────────────────────────────┤
@@ -42,7 +42,7 @@ The central abstraction that defines all database operations:
 
 In-memory implementation using JavaScript objects.
 
-```
+```text
 ┌─────────────────────┐
 │      IoMem          │
 ├─────────────────────┤
@@ -61,6 +61,21 @@ In-memory implementation using JavaScript objects.
 - Uses `@rljson/hash` for data hashing and identity
 - `IsReady` pattern for initialization tracking
 
+**Performance internals** (no observable API change):
+
+- Rows are indexed per table by content hash — write dedup is O(1) and
+  `readRows` with a string `_hash` in the where clause resolves through
+  the index instead of scanning the table
+- Table data is kept hash-sorted incrementally (binary-searched insert)
+  instead of re-sorting the whole table per write
+- Table and global hashes are updated lazily: writes mark tables dirty,
+  hashes are recomputed before they become observable (`dump`,
+  `dumpTable`, `createOrExtendTable`, re-`init`) — deterministic hashes
+  make the refreshed state identical to eager updates
+- The latest table configuration and column keys are cached per table
+  (updated on create/extend), so validation does not re-scan all
+  configurations per read/write
+
 **Data Structure:**
 
 ```typescript
@@ -76,7 +91,7 @@ _mem = {
 
 Remote database connection over sockets (Socket.IO compatible).
 
-```
+```text
 ┌──────────────┐         Socket          ┌──────────────┐
 │   IoPeer     │◄───────────────────────►│ IoPeerBridge │
 │  (Client)    │     Events/Acks         │   (Server)   │
@@ -107,7 +122,7 @@ Remote database connection over sockets (Socket.IO compatible).
 
 Server-side handler that bridges socket events to Io operations.
 
-```
+```text
 ┌─────────────────────────────────────┐
 │        IoPeerBridge                 │
 ├─────────────────────────────────────┤
@@ -132,7 +147,7 @@ Server-side handler that bridges socket events to Io operations.
 
 Aggregates multiple Io instances with priority-based cascading.
 
-```
+```text
 ┌────────────────────────────────────────────┐
 │             IoMulti                        │
 ├────────────────────────────────────────────┤
@@ -182,7 +197,7 @@ IoMultiIo {
 
 Provides name mapping between different table name formats.
 
-```
+```text
 ┌──────────────────────────────────────┐
 │     IoDbNameMapping                  │
 ├──────────────────────────────────────┤
@@ -202,7 +217,7 @@ Provides name mapping between different table name formats.
 
 Server implementation that combines Socket.IO with Io backends.
 
-```
+```text
 ┌─────────────────────────────────────┐
 │         IoServer                    │
 ├─────────────────────────────────────┤
@@ -230,7 +245,7 @@ Server implementation that combines Socket.IO with Io backends.
 
 ### Write Operation
 
-```
+```text
 Client Code
     │
     ▼
@@ -248,7 +263,7 @@ io.write(data)
 
 ### Read Operation (IoMulti Cascade)
 
-```
+```text
 io.readRows({table: 'users', where: {id: 1}})
     │
     ▼
@@ -293,7 +308,7 @@ return rljson; // Only after loop completes
 
 ### Event-Based Protocol
 
-```
+```text
 Client (IoPeer)                    Server (IoPeerBridge)
       │                                    │
       ├─── emit('readRows', request) ────►│

package/dist/io-mem.d.ts

@@ -36,6 +36,79 @@ export declare class IoMem implements Io {
     private _isReady;
     private _isOpen;
     private _mem;
+    /**
+     * Latest table configuration per table (the one with the most
+     * columns). Kept in sync by _createTable/_extendTable so that reads
+     * and writes need no repeated scan over all configurations.
+     */
+    private readonly _latestCfgs;
+    /** Column key sets per table, derived from _latestCfgs */
+    private readonly _columnKeys;
+    /**
+     * Per-table index of rows by content hash. Rows are only ever added,
+     * never removed, so the index cannot go stale.
+     */
+    private readonly _rowIndex;
+    /**
+     * Tables whose table hash (and thus the global hash) is outdated
+     * after writes. Hashes are refreshed lazily before they become
+     * observable (dump, dumpTable, create/extend) instead of after every
+     * single write — row hashes themselves are always up to date.
+     */
+    private readonly _dirtyTableHashes;
+    /**
+     * Recomputes the hashes of all dirty tables and the global hash.
+     * Produces exactly the state an eager per-write update would have
+     * produced (hashes are deterministic over the same data).
+     */
+    private _refreshHashes;
+    /**
+     * Returns the row index of a table, building it lazily from the
+     * table data on first access.
+     * @param table - The table to index
+     */
+    private _rowIndexFor;
+    /**
+     * Inserts a row into hash-sorted table data at its sorted position —
+     * preserves the order sortTableDataAndUpdateHash establishes without
+     * a full re-sort.
+     * @param data - The hash-sorted table data
+     * @param row - The row to insert
+     */
+    private static _insertSortedByHash;
+    /**
+     * The row filter predicate of readRows. Extracted so that the indexed
+     * fast path and the full scan share identical semantics.
+     * @param row - The row to check
+     * @param where - The where clause
+     */
+    private static _rowMatchesWhere;
+    /**
+     * Returns the latest table configuration, filling the cache lazily
+     * from IoTools (which picks the config with the most columns).
+     * @param table - The table to get the configuration for
+     */
+    private _latestCfg;
+    /**
+     * Updates the cached latest configuration of a table
+     * @param cfg - The new latest configuration
+     */
+    private _setLatestCfg;
+    /**
+     * Throws when one of the given columns does not exist in the table.
+     * Mirrors IoTools.throwWhenColumnDoesNotExist but uses the cached
+     * configuration.
+     * @param table - The table to check
+     * @param columns - The columns to check
+     */
+    private _throwWhenColumnDoesNotExist;
+    /**
+     * Throws when the data does not match the table configurations.
+     * Mirrors IoTools.throwWhenTableDataDoesNotMatchCfg but uses the
+     * cached configurations.
+     * @param data - The data to validate
+     */
+    private _throwWhenTableDataDoesNotMatchCfg;
     private _init;
     private _initTableCfgs;
     private _updateGlobalHash;
@@ -48,5 +121,5 @@ export declare class IoMem implements Io {
     private _contentType;
     private _write;
     private _readRows;
-    _removeNullValues(rljson: Rljson): void;
+    _removeNullValues(rljson: Rljson): boolean;
 }

package/dist/io.js

@@ -1,6 +1,6 @@
 import { hip, hsh } from "@rljson/hash";
 import { IsReady } from "@rljson/is-ready";
-import { copy, equals, merge } from "@rljson/json";
+import { equals, copy, merge } from "@rljson/json";
 import { iterateTables, throwOnInvalidTableCfg, validateRljsonAgainstTableCfg, iterateTablesSync } from "@rljson/rljson";
 class IoDbNameMapping {
   // The primary key column is always named '_hash'
@@ -443,8 +443,182 @@ class IoMem {
   _isReady = new IsReady();
   _isOpen = false;
   _mem = hip({});
+  /**
+   * Latest table configuration per table (the one with the most
+   * columns). Kept in sync by _createTable/_extendTable so that reads
+   * and writes need no repeated scan over all configurations.
+   */
+  _latestCfgs = /* @__PURE__ */ new Map();
+  /** Column key sets per table, derived from _latestCfgs */
+  _columnKeys = /* @__PURE__ */ new Map();
+  /**
+   * Per-table index of rows by content hash. Rows are only ever added,
+   * never removed, so the index cannot go stale.
+   */
+  _rowIndex = /* @__PURE__ */ new Map();
+  /**
+   * Tables whose table hash (and thus the global hash) is outdated
+   * after writes. Hashes are refreshed lazily before they become
+   * observable (dump, dumpTable, create/extend) instead of after every
+   * single write — row hashes themselves are always up to date.
+   */
+  _dirtyTableHashes = /* @__PURE__ */ new Set();
+  // ...........................................................................
+  /**
+   * Recomputes the hashes of all dirty tables and the global hash.
+   * Produces exactly the state an eager per-write update would have
+   * produced (hashes are deterministic over the same data).
+   */
+  _refreshHashes() {
+    if (this._dirtyTableHashes.size === 0) {
+      return;
+    }
+    for (const tableKey of this._dirtyTableHashes) {
+      const table = this._mem[tableKey];
+      table._hash = "";
+      hip(table, {
+        updateExistingHashes: false,
+        throwOnWrongHashes: false
+      });
+    }
+    this._dirtyTableHashes.clear();
+    this._updateGlobalHash();
+  }
+  // ...........................................................................
+  /**
+   * Returns the row index of a table, building it lazily from the
+   * table data on first access.
+   * @param table - The table to index
+   */
+  _rowIndexFor(table) {
+    let index = this._rowIndex.get(table);
+    if (!index) {
+      index = /* @__PURE__ */ new Map();
+      const tableData = this._mem[table]._data;
+      for (const row of tableData) {
+        index.set(row._hash, row);
+      }
+      this._rowIndex.set(table, index);
+    }
+    return index;
+  }
+  // ...........................................................................
+  /**
+   * Inserts a row into hash-sorted table data at its sorted position —
+   * preserves the order sortTableDataAndUpdateHash establishes without
+   * a full re-sort.
+   * @param data - The hash-sorted table data
+   * @param row - The row to insert
+   */
+  static _insertSortedByHash(data, row) {
+    const hash = row._hash;
+    let lo = 0;
+    let hi = data.length;
+    while (lo < hi) {
+      const mid = lo + hi >> 1;
+      if (data[mid]._hash < hash) {
+        lo = mid + 1;
+      } else {
+        hi = mid;
+      }
+    }
+    data.splice(lo, 0, row);
+  }
+  // ...........................................................................
+  /**
+   * The row filter predicate of readRows. Extracted so that the indexed
+   * fast path and the full scan share identical semantics.
+   * @param row - The row to check
+   * @param where - The where clause
+   */
+  static _rowMatchesWhere(row, where) {
+    for (const column in where) {
+      const a = row[column];
+      const b = where[column];
+      if (b === null && a === void 0) {
+        return true;
+      }
+      if (!equals(a, b)) {
+        return false;
+      }
+    }
+    return true;
+  }
+  // ...........................................................................
+  /**
+   * Returns the latest table configuration, filling the cache lazily
+   * from IoTools (which picks the config with the most columns).
+   * @param table - The table to get the configuration for
+   */
+  async _latestCfg(table) {
+    let cfg = this._latestCfgs.get(table);
+    if (!cfg) {
+      cfg = await this._ioTools.tableCfg(table);
+      this._latestCfgs.set(table, cfg);
+    }
+    return cfg;
+  }
+  /**
+   * Updates the cached latest configuration of a table
+   * @param cfg - The new latest configuration
+   */
+  _setLatestCfg(cfg) {
+    this._latestCfgs.set(cfg.key, cfg);
+    this._columnKeys.set(cfg.key, new Set(cfg.columns.map((c) => c.key)));
+  }
+  /**
+   * Throws when one of the given columns does not exist in the table.
+   * Mirrors IoTools.throwWhenColumnDoesNotExist but uses the cached
+   * configuration.
+   * @param table - The table to check
+   * @param columns - The columns to check
+   */
+  async _throwWhenColumnDoesNotExist(table, columns) {
+    let columnKeys = this._columnKeys.get(table);
+    if (!columnKeys) {
+      const cfg = await this._latestCfg(table);
+      columnKeys = new Set(cfg.columns.map((c) => c.key));
+      this._columnKeys.set(table, columnKeys);
+    }
+    const missingColumns = columns.filter((column) => !columnKeys.has(column));
+    if (missingColumns.length > 0) {
+      throw new Error(
+        `The following columns do not exist in table "${table}": ${missingColumns.join(
+          ", "
+        )}.`
+      );
+    }
+  }
+  /**
+   * Throws when the data does not match the table configurations.
+   * Mirrors IoTools.throwWhenTableDataDoesNotMatchCfg but uses the
+   * cached configurations.
+   * @param data - The data to validate
+   */
+  async _throwWhenTableDataDoesNotMatchCfg(data) {
+    const errors = [];
+    for (const tableKey of Object.keys(data)) {
+      const table = data[tableKey];
+      if (typeof table !== "object" || !Array.isArray(table?._data)) continue;
+      if (table._type === "tableCfgs") continue;
+      const tableCfg = await this._latestCfg(tableKey);
+      errors.push(...validateRljsonAgainstTableCfg(table._data, tableCfg));
+    }
+    if (errors.length > 0) {
+      throw new Error(
+        `Table data does not match the configuration.
+
+Errors:
+${errors.map((e) => `- ${e}`).join("\n")}`
+      );
+    }
+  }
   // ...........................................................................
   async _init() {
+    this._refreshHashes();
+    this._rowIndex.clear();
+    this._latestCfgs.clear();
+    this._columnKeys.clear();
     this._ioTools = new IoTools(this);
     this._initTableCfgs();
     this._updateGlobalHash();
@@ -476,6 +650,7 @@ class IoMem {
   }
   // ...........................................................................
   async _createOrExtendTable(request) {
+    this._refreshHashes();
     const tableCfg = request.tableCfg;
     await this._ioTools.throwWhenTableIsNotCompatible(tableCfg);
     const { key } = tableCfg;
@@ -492,6 +667,8 @@ class IoMem {
     newConfig = hsh(newConfig);
     this._mem.tableCfgs._data.push(newConfig);
     this._ioTools.sortTableDataAndUpdateHash(this._mem.tableCfgs);
+    this._setLatestCfg(newConfig);
+    this._rowIndex.get("tableCfgs")?.set(newConfig._hash, newConfig);
     const table = {
       _type: newConfig.type,
       _data: [],
@@ -509,6 +686,8 @@ class IoMem {
     newConfig = hsh(newConfig);
     this._mem.tableCfgs._data.push(newConfig);
     this._ioTools.sortTableDataAndUpdateHash(this._mem.tableCfgs);
+    this._setLatestCfg(newConfig);
+    this._rowIndex.get("tableCfgs")?.set(newConfig._hash, newConfig);
     const table = this._mem[newConfig.key];
     table._tableCfg = newConfig._hash;
     this._updateTableHash(newConfig.key);
@@ -516,11 +695,13 @@ class IoMem {
   }
   // ...........................................................................
   async _dump() {
+    this._refreshHashes();
     return copy(this._mem);
   }
   // ...........................................................................
   async _dumpTable(request) {
     await this._ioTools.throwWhenTableDoesNotExist(request.table);
+    this._refreshHashes();
     const table = this._mem[request.table];
     return {
       [request.table]: copy(table)
@@ -534,49 +715,48 @@ class IoMem {
   // ...........................................................................
   async _write(request) {
     const addedData = hsh(request.data);
-    this._removeNullValues(addedData);
+    const removedNullValues = this._removeNullValues(addedData);
     const tables = Object.keys(addedData);
-    hsh(addedData);
+    if (removedNullValues) {
+      hsh(addedData);
+    }
     await this._ioTools.throwWhenTablesDoNotExist(request.data);
-    await this._ioTools.throwWhenTableDataDoesNotMatchCfg(request.data);
+    await this._throwWhenTableDataDoesNotMatchCfg(request.data);
     for (const table of tables) {
       if (table.startsWith("_")) {
         continue;
       }
       const oldTable = this._mem[table];
       const newTable = addedData[table];
+      const rowIndex = this._rowIndexFor(table);
       for (const item of newTable._data) {
         const hash = item._hash;
-        const exists = oldTable._data.find((i) => i._hash === hash);
-        if (!exists) {
-          oldTable._data.push(item);
+        if (!rowIndex.has(hash)) {
+          rowIndex.set(hash, item);
+          IoMem._insertSortedByHash(oldTable._data, item);
         }
       }
-      this._ioTools.sortTableDataAndUpdateHash(oldTable);
+      this._dirtyTableHashes.add(table);
     }
-    this._updateGlobalHash();
   }
   // ...........................................................................
   async _readRows(request) {
     await this._ioTools.throwWhenTableDoesNotExist(request.table);
-    await this._ioTools.throwWhenColumnDoesNotExist(
+    await this._throwWhenColumnDoesNotExist(
       request.table,
       Object.keys(request.where)
     );
     const table = this._mem[request.table];
-    const tableDataFiltered = table._data.filter((row) => {
-      for (const column in request.where) {
-        const a = row[column];
-        const b = request.where[column];
-        if (b === null && a === void 0) {
-          return true;
-        }
-        if (!equals(a, b)) {
-          return false;
-        }
-      }
-      return true;
-    });
+    const whereHash = request.where["_hash"];
+    let tableDataFiltered;
+    if (typeof whereHash === "string") {
+      const row = this._rowIndexFor(request.table).get(whereHash);
+      tableDataFiltered = row && IoMem._rowMatchesWhere(row, request.where) ? [row] : [];
+    } else {
+      tableDataFiltered = table._data.filter(
+        (row) => IoMem._rowMatchesWhere(row, request.where)
+      );
+    }
     const tableFiltered = {
       _type: table._type,
       _data: tableDataFiltered
@@ -588,16 +768,19 @@ class IoMem {
     return result;
   }
   _removeNullValues(rljson) {
+    let removedAny = false;
     iterateTablesSync(rljson, (table) => {
       const data = rljson[table]._data;
       for (const row of data) {
         for (const key in row) {
           if (row[key] === null) {
             delete row[key];
+            removedAny = true;
           }
         }
       }
     });
+    return removedAny;
   }
 }
 class PeerSocketMock {