@workglow/storage 0.2.25 → 0.2.26

package/dist/common.d.ts

@@ -5,6 +5,9 @@
  */
 export * from "./tabular/BaseTabularStorage";
 export * from "./tabular/CachedTabularStorage";
+export * from "./tabular/CoveringIndexMissingError";
+export { pickCoveringIndex } from "./tabular/coveringIndexPicker";
+export type { RegisteredIndex, PickCoveringIndexInput, PickedIndex, } from "./tabular/coveringIndexPicker";
 export * from "./tabular/HuggingFaceTabularStorage";
 export * from "./tabular/InMemoryTabularStorage";
 export * from "./tabular/ITabularStorage";

package/dist/common.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,8BAA8B,CAAC;AAC7C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,wBAAwB,CAAC;AACvC,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAElD,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AAExC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAE9C,cAAc,4CAA4C,CAAC;AAC3D,cAAc,qCAAqC,CAAC;AAEpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAClD,cAAc,eAAe,CAAC;AAE9B,cAAc,gCAAgC,CAAC;AAC/C,cAAc,yBAAyB,CAAC;AACxC,cAAc,iCAAiC,CAAC;AAEhD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC"}
+{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,8BAA8B,CAAC;AAC7C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,qCAAqC,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,YAAY,EACV,eAAe,EACf,sBAAsB,EACtB,WAAW,GACZ,MAAM,+BAA+B,CAAC;AACvC,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,wBAAwB,CAAC;AACvC,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAElD,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AAExC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAE9C,cAAc,4CAA4C,CAAC;AAC3D,cAAc,qCAAqC,CAAC;AAEpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAClD,cAAc,eAAe,CAAC;AAE9B,cAAc,gCAAgC,CAAC;AAC/C,cAAc,yBAAyB,CAAC;AACxC,cAAc,iCAAiC,CAAC;AAEhD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC"}

package/dist/node.js

@@ -48,6 +48,21 @@ class StorageUnsupportedError extends StorageError {
   }
 }
 
+// src/tabular/CoveringIndexMissingError.ts
+class CoveringIndexMissingError extends StorageError {
+  static type = "CoveringIndexMissingError";
+  table;
+  requiredColumns;
+  registeredIndexes;
+  constructor(table, requiredColumns, registeredIndexes) {
+    const indexList = registeredIndexes.map((cols) => `[${cols.join(", ")}]`).join(", ");
+    super(`No covering index for table "${table}". ` + `Required columns: [${requiredColumns.join(", ")}]. ` + `Registered indexes: ${indexList || "(none)"}.`);
+    this.table = table;
+    this.requiredColumns = requiredColumns;
+    this.registeredIndexes = registeredIndexes;
+  }
+}
+
 // src/tabular/BaseTabularStorage.ts
 var TABULAR_REPOSITORY = createServiceToken("storage.tabularRepository");
 
@@ -171,6 +186,15 @@ class BaseTabularStorage {
     const entities = await this.query(criteria);
     return entities?.length ?? 0;
   }
+  queryIndex(criteria, options) {
+    this.validateSelect(options);
+    const requiredColumns = [
+      ...Object.keys(criteria),
+      ...(options.orderBy ?? []).map((o) => String(o.column)),
+      ...options.select.map(String)
+    ];
+    throw new CoveringIndexMissingError(this.constructor.name, requiredColumns, []);
+  }
   async* records(pageSize = 100) {
     if (pageSize <= 0) {
       throw new RangeError(`pageSize must be greater than 0, got ${pageSize}`);
@@ -266,6 +290,18 @@ class BaseTabularStorage {
       }
     }
   }
+  validateSelect(options) {
+    if (!options.select || options.select.length === 0) {
+      throw new StorageValidationError("queryIndex requires a non-empty select array");
+    }
+    const schemaProps = Object.keys(this.schema.properties);
+    for (const col of options.select) {
+      const colStr = String(col);
+      if (!schemaProps.includes(colStr)) {
+        throw new StorageValidationError(`queryIndex select column "${colStr}" is not in schema`);
+      }
+    }
+  }
   primaryKeyColumns() {
     const columns = [];
     for (const key of Object.keys(this.primaryKeySchema.properties)) {
@@ -386,6 +422,61 @@ import { createServiceToken as createServiceToken3, getLogger } from "@workglow/
 
 // src/tabular/InMemoryTabularStorage.ts
 import { createServiceToken as createServiceToken2, makeFingerprint as makeFingerprint2, uuid4 } from "@workglow/util";
+
+// src/tabular/coveringIndexPicker.ts
+function pickCoveringIndex(input) {
+  const { table, indexes, criteriaColumns, orderByColumns, selectColumns, primaryKeyColumns } = input;
+  const required = uniqueColumns([
+    ...criteriaColumns,
+    ...orderByColumns.map((o) => o.column),
+    ...selectColumns
+  ]);
+  const pkSet = new Set(primaryKeyColumns);
+  for (const idx of indexes) {
+    const fit = tryFitIndex(idx, criteriaColumns, orderByColumns, selectColumns, pkSet);
+    if (fit !== undefined) {
+      return { name: idx.name, keyPath: idx.keyPath, reverseDirection: fit.reverseDirection };
+    }
+  }
+  throw new CoveringIndexMissingError(table, required, indexes.map((i) => i.keyPath));
+}
+function tryFitIndex(index, criteria, orderBy, select, pkSet) {
+  const keyPath = index.keyPath;
+  const criteriaSet = new Set(criteria);
+  if (criteria.length > keyPath.length)
+    return;
+  for (let i = 0;i < criteria.length; i++) {
+    if (!criteriaSet.has(keyPath[i]))
+      return;
+  }
+  let reverseDirection = false;
+  if (orderBy.length > 0) {
+    const start = criteria.length;
+    if (start + orderBy.length > keyPath.length)
+      return;
+    for (let i = 0;i < orderBy.length; i++) {
+      if (keyPath[start + i] !== orderBy[i].column)
+        return;
+    }
+    const allDesc = orderBy.every((o) => o.direction === "DESC");
+    const allAsc = orderBy.every((o) => o.direction === "ASC");
+    if (allDesc)
+      reverseDirection = true;
+    else if (!allAsc)
+      return;
+  }
+  const keyPathSet = new Set(keyPath);
+  for (const col of select) {
+    if (!keyPathSet.has(col) && !pkSet.has(col))
+      return;
+  }
+  return { reverseDirection };
+}
+function uniqueColumns(cols) {
+  return Array.from(new Set(cols));
+}
+
+// src/tabular/InMemoryTabularStorage.ts
 var MEMORY_TABULAR_REPOSITORY = createServiceToken2("storage.tabularRepository.inMemory");
 
 class InMemoryTabularStorage extends BaseTabularStorage {
@@ -628,6 +719,38 @@ class InMemoryTabularStorage extends BaseTabularStorage {
     this.events.emit("query", criteria, result);
     return result;
   }
+  async queryIndex(criteria, options) {
+    this.validateSelect(options);
+    this.validateQueryParams(criteria, options);
+    const registered = this.indexes.map((cols, i) => ({
+      name: `idx_${i}`,
+      keyPath: cols
+    }));
+    pickCoveringIndex({
+      table: "InMemoryTabularStorage",
+      indexes: registered,
+      criteriaColumns: Object.keys(criteria),
+      orderByColumns: (options.orderBy ?? []).map((o) => ({
+        column: String(o.column),
+        direction: o.direction
+      })),
+      selectColumns: options.select.map(String),
+      primaryKeyColumns: this.primaryKeyNames.map(String)
+    });
+    const full = await this.query(criteria, {
+      orderBy: options.orderBy,
+      limit: options.limit,
+      offset: options.offset
+    });
+    if (!full)
+      return [];
+    return full.map((row) => {
+      const out = {};
+      for (const k of options.select)
+        out[k] = row[k];
+      return out;
+    });
+  }
   subscribeToChanges(callback, options) {
     const handlePut = (entity) => {
       callback({ type: this._lastPutWasInsert ? "INSERT" : "UPDATE", new: entity });
@@ -770,6 +893,10 @@ class CachedTabularStorage extends BaseTabularStorage {
     await this.initializeCache();
     return await this.cache.query(criteria, options);
   }
+  async queryIndex(criteria, options) {
+    await this.initializeCache();
+    return await this.cache.queryIndex(criteria, options);
+  }
   async deleteSearch(criteria) {
     await this.initializeCache();
     await this.durable.deleteSearch(criteria);
@@ -1210,6 +1337,9 @@ class TelemetryTabularStorage {
   query(criteria, options) {
     return traced("workglow.storage.tabular.query", this.storageName, () => this.inner.query(criteria, options));
   }
+  queryIndex(criteria, options) {
+    return traced("workglow.storage.tabular.queryIndex", this.storageName, () => this.inner.queryIndex(criteria, options));
+  }
   records(pageSize) {
     return this.inner.records(pageSize);
   }
@@ -2495,6 +2625,9 @@ class FsFolderTabularStorage extends BaseTabularStorage {
   async query(_criteria, _options) {
     throw new StorageUnsupportedError("query", "FsFolderTabularStorage");
   }
+  async queryIndex(_criteria, _options) {
+    throw new StorageUnsupportedError("queryIndex", "FsFolderTabularStorage");
+  }
   async deleteSearch(_criteria) {
     throw new Error("deleteSearch is not supported for FsFolderTabularStorage");
   }
@@ -3238,6 +3371,49 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
     this.events.emit("query", criteria, undefined);
     return;
   }
+  async queryIndex(criteria, options) {
+    this.validateSelect(options);
+    this.validateQueryParams(criteria, options);
+    const registered = this.indexes.map((cols2, i) => {
+      const cs = Array.isArray(cols2) ? cols2 : [cols2];
+      return { name: `idx_${i}`, keyPath: cs };
+    });
+    pickCoveringIndex({
+      table: this.table,
+      indexes: registered,
+      criteriaColumns: Object.keys(criteria),
+      orderByColumns: (options.orderBy ?? []).map((o) => ({
+        column: String(o.column),
+        direction: o.direction
+      })),
+      selectColumns: options.select.map(String),
+      primaryKeyColumns: this.primaryKeyNames.map(String)
+    });
+    const cols = options.select.map((c) => `"${String(c)}"`).join(", ");
+    let sql = `SELECT ${cols} FROM "${this.table}"`;
+    const { whereClause, params } = this.buildDeleteSearchWhere(criteria);
+    sql += ` WHERE ${whereClause}`;
+    if (options.orderBy && options.orderBy.length > 0) {
+      sql += ` ORDER BY ` + options.orderBy.map((o) => `"${String(o.column)}" ${o.direction}`).join(", ");
+    }
+    if (options.limit !== undefined) {
+      sql += ` LIMIT $${params.length + 1}`;
+      params.push(options.limit);
+    }
+    if (options.offset !== undefined) {
+      sql += ` OFFSET $${params.length + 1}`;
+      params.push(options.offset);
+    }
+    const db = this.db;
+    const result = await db.query(sql, params);
+    for (const row of result.rows) {
+      const record = row;
+      for (const k of Object.keys(record)) {
+        record[k] = this.sqlToJsValue(k, record[k]);
+      }
+    }
+    return result.rows;
+  }
   subscribeToChanges(callback, options) {
     throw new Error("subscribeToChanges is not supported for PostgresTabularStorage");
   }
@@ -3722,6 +3898,50 @@ class SqliteTabularStorage extends BaseSqlTabularStorage {
     this.events.emit("query", criteria, undefined);
     return;
   }
+  async queryIndex(criteria, options) {
+    this.validateSelect(options);
+    this.validateQueryParams(criteria, options);
+    const registered = this.indexes.map((cols2, i) => {
+      const cs = Array.isArray(cols2) ? cols2 : [cols2];
+      return { name: `idx_${i}`, keyPath: cs };
+    });
+    pickCoveringIndex({
+      table: this.table,
+      indexes: registered,
+      criteriaColumns: Object.keys(criteria),
+      orderByColumns: (options.orderBy ?? []).map((o) => ({
+        column: String(o.column),
+        direction: o.direction
+      })),
+      selectColumns: options.select.map(String),
+      primaryKeyColumns: this.primaryKeyNames.map(String)
+    });
+    const cols = options.select.map((c) => `\`${String(c)}\``).join(", ");
+    let sql = `SELECT ${cols} FROM \`${this.table}\``;
+    const { whereClause, params } = this.buildDeleteSearchWhere(criteria);
+    sql += ` WHERE ${whereClause}`;
+    if (options.orderBy && options.orderBy.length > 0) {
+      sql += ` ORDER BY ` + options.orderBy.map((o) => `\`${String(o.column)}\` ${o.direction}`).join(", ");
+    }
+    if (options.limit !== undefined) {
+      sql += ` LIMIT ?`;
+      params.push(options.limit);
+    }
+    if (options.offset !== undefined) {
+      if (options.limit === undefined)
+        sql += ` LIMIT -1`;
+      sql += ` OFFSET ?`;
+      params.push(options.offset);
+    }
+    const stmt = this.db.prepare(sql);
+    const rows = stmt.all(...params);
+    for (const row of rows) {
+      for (const k of Object.keys(row)) {
+        row[k] = this.sqlToJsValue(k, row[k]);
+      }
+    }
+    return rows;
+  }
   subscribeToChanges(callback, options) {
     throw new Error("subscribeToChanges is not supported for SqliteTabularStorage");
   }
@@ -4196,6 +4416,56 @@ class SupabaseTabularStorage extends BaseSqlTabularStorage {
     this.events.emit("query", criteria, undefined);
     return;
   }
+  async queryIndex(criteria, options) {
+    this.validateSelect(options);
+    this.validateQueryParams(criteria, options);
+    const registered = this.indexes.map((cols, i) => {
+      const cs = Array.isArray(cols) ? cols : [cols];
+      return { name: `idx_${i}`, keyPath: cs };
+    });
+    pickCoveringIndex({
+      table: this.table,
+      indexes: registered,
+      criteriaColumns: Object.keys(criteria),
+      orderByColumns: (options.orderBy ?? []).map((o) => ({
+        column: String(o.column),
+        direction: o.direction
+      })),
+      selectColumns: options.select.map(String),
+      primaryKeyColumns: this.primaryKeyNames.map(String)
+    });
+    const colList = options.select.map(String).join(",");
+    let q = this.applyCriteriaToFilter(this.client.from(this.table).select(colList), criteria);
+    if (options.orderBy) {
+      for (const { column, direction } of options.orderBy) {
+        q = q.order(String(column), { ascending: direction === "ASC" });
+      }
+    }
+    if (options.offset !== undefined && options.limit === undefined) {
+      throw new StorageValidationError("queryIndex with offset requires limit (no implicit cap)");
+    }
+    if (options.offset !== undefined || options.limit !== undefined) {
+      const start = options.offset ?? 0;
+      if (options.limit !== undefined) {
+        q = q.range(start, start + options.limit - 1);
+      }
+    }
+    const { data, error } = await q;
+    if (error)
+      throw error;
+    if (!data)
+      return [];
+    const rows = data;
+    const sel = new Set(options.select.map(String));
+    for (const row of rows) {
+      for (const key of Object.keys(row)) {
+        if (sel.has(key)) {
+          row[key] = this.sqlToJsValue(key, row[key]);
+        }
+      }
+    }
+    return rows;
+  }
   convertRealtimeRow(row) {
     const entity = { ...row };
     const record = entity;
@@ -7621,6 +7891,105 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
       request.onerror = () => reject(request.error);
     });
   }
+  async queryIndex(criteria, options) {
+    this.validateSelect(options);
+    this.validateQueryParams(criteria, options);
+    const registered = this.indexes.map((cols) => {
+      const cs = Array.isArray(cols) ? cols : [cols];
+      return { name: cs.join("_"), keyPath: cs };
+    });
+    const picked = pickCoveringIndex({
+      table: this.table,
+      indexes: registered,
+      criteriaColumns: Object.keys(criteria),
+      orderByColumns: (options.orderBy ?? []).map((o) => ({
+        column: String(o.column),
+        direction: o.direction
+      })),
+      selectColumns: options.select.map(String),
+      primaryKeyColumns: this.primaryKeyColumns().map(String)
+    });
+    const db = await this.getDb();
+    return new Promise((resolve, reject) => {
+      const tx = db.transaction(this.table, "readonly");
+      const store = tx.objectStore(this.table);
+      const idx = store.index(picked.name);
+      const prefix = [];
+      for (const col of picked.keyPath) {
+        const c = criteria[col];
+        if (c === undefined && !(col in criteria))
+          break;
+        if (isSearchCondition(c)) {
+          if (c.operator !== "=")
+            break;
+          prefix.push(c.value);
+        } else {
+          prefix.push(c);
+        }
+      }
+      const range = prefix.length === 0 ? undefined : prefix.length === picked.keyPath.length ? IDBKeyRange.only(prefix.length === 1 ? prefix[0] : prefix) : IDBKeyRange.bound(prefix, [...prefix, []]);
+      const direction = picked.reverseDirection ? "prev" : "next";
+      const request = idx.openKeyCursor(range, direction);
+      const out = [];
+      let toSkip = options.offset ?? 0;
+      const keyPathPositions = new Map;
+      picked.keyPath.forEach((col, i) => keyPathPositions.set(col, i));
+      const pkCols = this.primaryKeyColumns().map(String);
+      const pkPositions = new Map;
+      pkCols.forEach((col, i) => pkPositions.set(col, i));
+      request.onsuccess = () => {
+        const cursor = request.result;
+        if (!cursor) {
+          resolve(out);
+          return;
+        }
+        const key = cursor.key;
+        const row = {};
+        for (const col of options.select) {
+          const colStr = String(col);
+          const pos = keyPathPositions.get(colStr);
+          if (pos !== undefined) {
+            row[colStr] = Array.isArray(key) ? key[pos] : key;
+          } else {
+            if (pkCols.length === 1 && colStr === pkCols[0]) {
+              row[colStr] = cursor.primaryKey;
+            } else {
+              const pkPos = pkPositions.get(colStr);
+              if (pkPos !== undefined) {
+                row[colStr] = Array.isArray(cursor.primaryKey) ? cursor.primaryKey[pkPos] : cursor.primaryKey;
+              }
+            }
+          }
+        }
+        let matches = true;
+        for (const [col, crit] of Object.entries(criteria)) {
+          if (!isSearchCondition(crit))
+            continue;
+          const pos = keyPathPositions.get(col);
+          if (pos === undefined)
+            continue;
+          const valFromKey = Array.isArray(key) ? key[pos] : key;
+          if (!compareWithOperator(valFromKey, crit.operator, crit.value)) {
+            matches = false;
+            break;
+          }
+        }
+        if (matches) {
+          if (toSkip > 0) {
+            toSkip -= 1;
+          } else {
+            out.push(row);
+            if (options.limit !== undefined && out.length >= options.limit) {
+              resolve(out);
+              return;
+            }
+          }
+        }
+        cursor.continue();
+      };
+      request.onerror = () => reject(request.error);
+    });
+  }
   getHybridManager() {
     if (!this.hybridManager) {
       const channelName = `indexeddb-tabular-${this.table}`;
@@ -7658,6 +8027,22 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
     this.db?.close();
   }
 }
+function compareWithOperator(a, op, b) {
+  const av = a;
+  const bv = b;
+  switch (op) {
+    case "=":
+      return av === bv;
+    case "<":
+      return av !== null && av !== undefined && av < bv;
+    case "<=":
+      return av !== null && av !== undefined && av <= bv;
+    case ">":
+      return av !== null && av !== undefined && av > bv;
+    case ">=":
+      return av !== null && av !== undefined && av >= bv;
+  }
+}
 
 // src/kv/IndexedDbKvStorage.ts
 var IDB_KV_REPOSITORY = createServiceToken28("storage.kvRepository.indexedDb");
@@ -8573,6 +8958,7 @@ class IndexedDbVectorStorage extends IndexedDbTabularStorage {
 export {
   traced,
   registerTabularRepository,
+  pickCoveringIndex,
   isSearchCondition,
   getVectorProperty,
   getTabularRepository,
@@ -8658,9 +9044,10 @@ export {
   EncryptedKvCredentialStore,
   DefaultKeyValueSchema,
   DefaultKeyValueKey,
+  CoveringIndexMissingError,
   CachedTabularStorage,
   CACHED_TABULAR_REPOSITORY,
   BaseTabularStorage
 };
 
-//# debugId=1E55C44CEF8B06B964756E2164756E21
+//# debugId=FEAE1DAF1DC5CDCE64756E2164756E21