@rljson/io 0.0.69 → 0.0.71

package/dist/io.js

@@ -1,6 +1,6 @@
 import { hip, hsh } from "@rljson/hash";
 import { IsReady } from "@rljson/is-ready";
-import { copy, equals, merge } from "@rljson/json";
+import { equals, copy, merge } from "@rljson/json";
 import { iterateTables, throwOnInvalidTableCfg, validateRljsonAgainstTableCfg, iterateTablesSync } from "@rljson/rljson";
 class IoDbNameMapping {
   // The primary key column is always named '_hash'
@@ -411,6 +411,27 @@ class IoMem {
   readRows(request) {
     return this._readRows(request);
   }
+  async readRowsByHashes(request) {
+    await this._ioTools.throwWhenTableDoesNotExist(request.table);
+    const table = this._mem[request.table];
+    const rowIndex = this._rowIndexFor(request.table);
+    const seen = /* @__PURE__ */ new Set();
+    const rows = [];
+    for (const hash of request.hashes) {
+      if (seen.has(hash)) continue;
+      seen.add(hash);
+      const row = rowIndex.get(hash);
+      if (row) {
+        rows.push(row);
+      }
+    }
+    const tableFiltered = {
+      _type: table._type,
+      _data: rows
+    };
+    this._ioTools.sortTableDataAndUpdateHash(tableFiltered);
+    return { [request.table]: tableFiltered };
+  }
   async rowCount(table) {
     const tableData = this._mem[table];
     if (!tableData) {
@@ -443,8 +464,182 @@ class IoMem {
   _isReady = new IsReady();
   _isOpen = false;
   _mem = hip({});
+  /**
+   * Latest table configuration per table (the one with the most
+   * columns). Kept in sync by _createTable/_extendTable so that reads
+   * and writes need no repeated scan over all configurations.
+   */
+  _latestCfgs = /* @__PURE__ */ new Map();
+  /** Column key sets per table, derived from _latestCfgs */
+  _columnKeys = /* @__PURE__ */ new Map();
+  /**
+   * Per-table index of rows by content hash. Rows are only ever added,
+   * never removed, so the index cannot go stale.
+   */
+  _rowIndex = /* @__PURE__ */ new Map();
+  /**
+   * Tables whose table hash (and thus the global hash) is outdated
+   * after writes. Hashes are refreshed lazily before they become
+   * observable (dump, dumpTable, create/extend) instead of after every
+   * single write — row hashes themselves are always up to date.
+   */
+  _dirtyTableHashes = /* @__PURE__ */ new Set();
+  // ...........................................................................
+  /**
+   * Recomputes the hashes of all dirty tables and the global hash.
+   * Produces exactly the state an eager per-write update would have
+   * produced (hashes are deterministic over the same data).
+   */
+  _refreshHashes() {
+    if (this._dirtyTableHashes.size === 0) {
+      return;
+    }
+    for (const tableKey of this._dirtyTableHashes) {
+      const table = this._mem[tableKey];
+      table._hash = "";
+      hip(table, {
+        updateExistingHashes: false,
+        throwOnWrongHashes: false
+      });
+    }
+    this._dirtyTableHashes.clear();
+    this._updateGlobalHash();
+  }
+  // ...........................................................................
+  /**
+   * Returns the row index of a table, building it lazily from the
+   * table data on first access.
+   * @param table - The table to index
+   */
+  _rowIndexFor(table) {
+    let index = this._rowIndex.get(table);
+    if (!index) {
+      index = /* @__PURE__ */ new Map();
+      const tableData = this._mem[table]._data;
+      for (const row of tableData) {
+        index.set(row._hash, row);
+      }
+      this._rowIndex.set(table, index);
+    }
+    return index;
+  }
+  // ...........................................................................
+  /**
+   * Inserts a row into hash-sorted table data at its sorted position —
+   * preserves the order sortTableDataAndUpdateHash establishes without
+   * a full re-sort.
+   * @param data - The hash-sorted table data
+   * @param row - The row to insert
+   */
+  static _insertSortedByHash(data, row) {
+    const hash = row._hash;
+    let lo = 0;
+    let hi = data.length;
+    while (lo < hi) {
+      const mid = lo + hi >> 1;
+      if (data[mid]._hash < hash) {
+        lo = mid + 1;
+      } else {
+        hi = mid;
+      }
+    }
+    data.splice(lo, 0, row);
+  }
+  // ...........................................................................
+  /**
+   * The row filter predicate of readRows. Extracted so that the indexed
+   * fast path and the full scan share identical semantics.
+   * @param row - The row to check
+   * @param where - The where clause
+   */
+  static _rowMatchesWhere(row, where) {
+    for (const column in where) {
+      const a = row[column];
+      const b = where[column];
+      if (b === null && a === void 0) {
+        return true;
+      }
+      if (!equals(a, b)) {
+        return false;
+      }
+    }
+    return true;
+  }
+  // ...........................................................................
+  /**
+   * Returns the latest table configuration, filling the cache lazily
+   * from IoTools (which picks the config with the most columns).
+   * @param table - The table to get the configuration for
+   */
+  async _latestCfg(table) {
+    let cfg = this._latestCfgs.get(table);
+    if (!cfg) {
+      cfg = await this._ioTools.tableCfg(table);
+      this._latestCfgs.set(table, cfg);
+    }
+    return cfg;
+  }
+  /**
+   * Updates the cached latest configuration of a table
+   * @param cfg - The new latest configuration
+   */
+  _setLatestCfg(cfg) {
+    this._latestCfgs.set(cfg.key, cfg);
+    this._columnKeys.set(cfg.key, new Set(cfg.columns.map((c) => c.key)));
+  }
+  /**
+   * Throws when one of the given columns does not exist in the table.
+   * Mirrors IoTools.throwWhenColumnDoesNotExist but uses the cached
+   * configuration.
+   * @param table - The table to check
+   * @param columns - The columns to check
+   */
+  async _throwWhenColumnDoesNotExist(table, columns) {
+    let columnKeys = this._columnKeys.get(table);
+    if (!columnKeys) {
+      const cfg = await this._latestCfg(table);
+      columnKeys = new Set(cfg.columns.map((c) => c.key));
+      this._columnKeys.set(table, columnKeys);
+    }
+    const missingColumns = columns.filter((column) => !columnKeys.has(column));
+    if (missingColumns.length > 0) {
+      throw new Error(
+        `The following columns do not exist in table "${table}": ${missingColumns.join(
+          ", "
+        )}.`
+      );
+    }
+  }
+  /**
+   * Throws when the data does not match the table configurations.
+   * Mirrors IoTools.throwWhenTableDataDoesNotMatchCfg but uses the
+   * cached configurations.
+   * @param data - The data to validate
+   */
+  async _throwWhenTableDataDoesNotMatchCfg(data) {
+    const errors = [];
+    for (const tableKey of Object.keys(data)) {
+      const table = data[tableKey];
+      if (typeof table !== "object" || !Array.isArray(table?._data)) continue;
+      if (table._type === "tableCfgs") continue;
+      const tableCfg = await this._latestCfg(tableKey);
+      errors.push(...validateRljsonAgainstTableCfg(table._data, tableCfg));
+    }
+    if (errors.length > 0) {
+      throw new Error(
+        `Table data does not match the configuration.
+
+Errors:
+${errors.map((e) => `- ${e}`).join("\n")}`
+      );
+    }
+  }
   // ...........................................................................
   async _init() {
+    this._refreshHashes();
+    this._rowIndex.clear();
+    this._latestCfgs.clear();
+    this._columnKeys.clear();
     this._ioTools = new IoTools(this);
     this._initTableCfgs();
     this._updateGlobalHash();
@@ -476,6 +671,7 @@ class IoMem {
   }
   // ...........................................................................
   async _createOrExtendTable(request) {
+    this._refreshHashes();
     const tableCfg = request.tableCfg;
     await this._ioTools.throwWhenTableIsNotCompatible(tableCfg);
     const { key } = tableCfg;
@@ -492,6 +688,8 @@ class IoMem {
     newConfig = hsh(newConfig);
     this._mem.tableCfgs._data.push(newConfig);
     this._ioTools.sortTableDataAndUpdateHash(this._mem.tableCfgs);
+    this._setLatestCfg(newConfig);
+    this._rowIndex.get("tableCfgs")?.set(newConfig._hash, newConfig);
     const table = {
       _type: newConfig.type,
       _data: [],
@@ -509,6 +707,8 @@ class IoMem {
     newConfig = hsh(newConfig);
     this._mem.tableCfgs._data.push(newConfig);
     this._ioTools.sortTableDataAndUpdateHash(this._mem.tableCfgs);
+    this._setLatestCfg(newConfig);
+    this._rowIndex.get("tableCfgs")?.set(newConfig._hash, newConfig);
     const table = this._mem[newConfig.key];
     table._tableCfg = newConfig._hash;
     this._updateTableHash(newConfig.key);
@@ -516,11 +716,13 @@ class IoMem {
   }
   // ...........................................................................
   async _dump() {
+    this._refreshHashes();
     return copy(this._mem);
   }
   // ...........................................................................
   async _dumpTable(request) {
     await this._ioTools.throwWhenTableDoesNotExist(request.table);
+    this._refreshHashes();
     const table = this._mem[request.table];
     return {
       [request.table]: copy(table)
@@ -534,49 +736,48 @@ class IoMem {
   // ...........................................................................
   async _write(request) {
     const addedData = hsh(request.data);
-    this._removeNullValues(addedData);
+    const removedNullValues = this._removeNullValues(addedData);
     const tables = Object.keys(addedData);
-    hsh(addedData);
+    if (removedNullValues) {
+      hsh(addedData);
+    }
     await this._ioTools.throwWhenTablesDoNotExist(request.data);
-    await this._ioTools.throwWhenTableDataDoesNotMatchCfg(request.data);
+    await this._throwWhenTableDataDoesNotMatchCfg(request.data);
     for (const table of tables) {
       if (table.startsWith("_")) {
         continue;
       }
       const oldTable = this._mem[table];
       const newTable = addedData[table];
+      const rowIndex = this._rowIndexFor(table);
       for (const item of newTable._data) {
         const hash = item._hash;
-        const exists = oldTable._data.find((i) => i._hash === hash);
-        if (!exists) {
-          oldTable._data.push(item);
+        if (!rowIndex.has(hash)) {
+          rowIndex.set(hash, item);
+          IoMem._insertSortedByHash(oldTable._data, item);
         }
       }
-      this._ioTools.sortTableDataAndUpdateHash(oldTable);
+      this._dirtyTableHashes.add(table);
     }
-    this._updateGlobalHash();
   }
   // ...........................................................................
   async _readRows(request) {
     await this._ioTools.throwWhenTableDoesNotExist(request.table);
-    await this._ioTools.throwWhenColumnDoesNotExist(
+    await this._throwWhenColumnDoesNotExist(
       request.table,
       Object.keys(request.where)
     );
     const table = this._mem[request.table];
-    const tableDataFiltered = table._data.filter((row) => {
-      for (const column in request.where) {
-        const a = row[column];
-        const b = request.where[column];
-        if (b === null && a === void 0) {
-          return true;
-        }
-        if (!equals(a, b)) {
-          return false;
-        }
-      }
-      return true;
-    });
+    const whereHash = request.where["_hash"];
+    let tableDataFiltered;
+    if (typeof whereHash === "string") {
+      const row = this._rowIndexFor(request.table).get(whereHash);
+      tableDataFiltered = row && IoMem._rowMatchesWhere(row, request.where) ? [row] : [];
+    } else {
+      tableDataFiltered = table._data.filter(
+        (row) => IoMem._rowMatchesWhere(row, request.where)
+      );
+    }
     const tableFiltered = {
       _type: table._type,
       _data: tableDataFiltered
@@ -588,16 +789,19 @@ class IoMem {
     return result;
   }
   _removeNullValues(rljson) {
+    let removedAny = false;
     iterateTablesSync(rljson, (table) => {
       const data = rljson[table]._data;
       for (const row of data) {
         for (const key in row) {
           if (row[key] === null) {
             delete row[key];
+            removedAny = true;
           }
         }
       }
     });
+    return removedAny;
   }
 }
 class PeerSocketMock {
@@ -925,6 +1129,65 @@ class IoPeer {
     );
   }
   // ...........................................................................
+  /**
+   * True once the remote side signalled that it does not support batch
+   * reads — all further batch reads then use per-hash readRows.
+   */
+  _batchReadsUnsupported = false;
+  /**
+   * Batch read over the socket. Falls back to per-hash readRows when
+   * the remote side does not support it (older server) and remembers
+   * the capability for subsequent calls.
+   * @param request - The table and the row hashes to read
+   */
+  async readRowsByHashes(request) {
+    if (!this._batchReadsUnsupported) {
+      try {
+        return await this._withTimeout(
+          new Promise((resolve, reject) => {
+            this._socket.emit(
+              "readRowsByHashes",
+              request,
+              (result, error) => {
+                if (error) reject(error);
+                resolve(result);
+              }
+            );
+          }),
+          "readRowsByHashes"
+        );
+      } catch (error) {
+        const message = String(error.message);
+        const unsupported = message.includes("not found on Io instance") || message.includes("not supported") || message.includes("Timeout after");
+        if (!unsupported) {
+          throw error;
+        }
+        this._batchReadsUnsupported = true;
+      }
+    }
+    const hashes = Array.from(new Set(request.hashes));
+    const results = await Promise.all(
+      hashes.map(
+        (hash) => this.readRows({ table: request.table, where: { _hash: hash } })
+      )
+    );
+    let type = void 0;
+    const rows = [];
+    for (const result of results) {
+      const tableData = result[request.table];
+      type ??= tableData._type;
+      rows.push(...tableData._data);
+    }
+    if (type === void 0) {
+      const empty = await this.readRows({
+        table: request.table,
+        where: { _hash: "__NONE__" }
+      });
+      type = empty[request.table]._type;
+    }
+    return { [request.table]: { _data: rows, _type: type } };
+  }
+  // ...........................................................................
   /**
    * Retrieves the number of rows in a specific table.
    * @param table The name of the table to count rows in.
@@ -1265,6 +1528,103 @@ class IoMulti {
     return Promise.resolve(tableData._data.length);
   }
   // ...........................................................................
+  /**
+   * Batch read with PER-HASH cascade: every hash not found in a
+   * higher-priority readable is looked up in the next one. Readables
+   * without readRowsByHashes are queried per hash via readRows.
+   * @param request - The table and the row hashes to read
+   */
+  async readRowsByHashes(request) {
+    if (this.readables.length === 0) {
+      throw new Error("No readable Io available");
+    }
+    let tableExistsAny = false;
+    const rows = /* @__PURE__ */ new Map();
+    let type = void 0;
+    let readFrom = "";
+    const errors = [];
+    let remaining = Array.from(new Set(request.hashes));
+    for (const readable of this.readables) {
+      if (remaining.length === 0) break;
+      try {
+        let result;
+        if (readable.io.readRowsByHashes) {
+          result = await readable.io.readRowsByHashes({
+            table: request.table,
+            hashes: remaining
+          });
+        } else {
+          result = await IoMulti._readHashesViaReadRows(
+            readable.io,
+            request.table,
+            remaining
+          );
+        }
+        const tableData = result[request.table];
+        tableExistsAny = true;
+        type = tableData._type;
+        if (tableData._data.length > 0) {
+          readFrom = readable.id ?? "";
+          for (const tableRow of tableData._data) {
+            rows.set(tableRow._hash, tableRow);
+          }
+          remaining = remaining.filter((hash) => !rows.has(hash));
+        }
+      } catch (e) {
+        errors.push(e);
+      }
+    }
+    if (!tableExistsAny) {
+      if (errors.length === 0) {
+        throw new Error(`Table "${request.table}" not found`);
+      } else {
+        const preciseErrors = errors.filter(
+          (err) => !err.message.includes(`Table "${request.table}" not found`)
+        );
+        if (preciseErrors.length > 0) {
+          throw preciseErrors[0];
+        } else {
+          throw errors[0];
+        }
+      }
+    }
+    const rljson = {
+      [request.table]: hip({ _data: Array.from(rows.values()), _type: type })
+    };
+    if (this.writables.length > 0 && rows.size > 0) {
+      for (const writeable of this.writables) {
+        if (writeable.id === readFrom) {
+          continue;
+        }
+        try {
+          await writeable.io.write({ data: rljson });
+        } catch {
+          continue;
+        }
+      }
+    }
+    return rljson;
+  }
+  /**
+   * Per-hash fallback for readables without readRowsByHashes.
+   * @param io - The readable io
+   * @param table - The table to read from
+   * @param hashes - The row hashes to read
+   */
+  static async _readHashesViaReadRows(io, table, hashes) {
+    const results = await Promise.all(
+      hashes.map((hash) => io.readRows({ table, where: { _hash: hash } }))
+    );
+    let type = void 0;
+    const rows = [];
+    for (const result of results) {
+      const tableData = result[table];
+      type ??= tableData._type;
+      rows.push(...tableData._data);
+    }
+    return { [table]: { _data: rows, _type: type } };
+  }
+  // ...........................................................................
   /**
    * Gets the list of underlying readable Io instances, sorted by priority.
    */
@@ -1364,6 +1724,7 @@ class IoPeerBridge {
       "createOrExtendTable",
       "write",
       "readRows",
+      "readRowsByHashes",
       "rowCount",
       "dumpTable",
       "dump",
@@ -1551,6 +1912,11 @@ class IoServer {
     rawTableCfgs: () => this._io.rawTableCfgs(),
     write: (request) => this._io.write(request),
     readRows: (request) => this._io.readRows(request),
+    readRowsByHashes: (request) => this._io.readRowsByHashes ? this._io.readRowsByHashes(request) : Promise.reject(
+      new Error(
+        'Method "readRowsByHashes" not found on Io instance'
+      )
+    ),
     rowCount: (table) => this._io.rowCount(table)
   });
 }