@workglow/storage 0.2.22 → 0.2.24

package/dist/node.js

@@ -164,6 +164,13 @@ class BaseTabularStorage {
   waitOn(name) {
     return this.events.waitOn(name);
   }
+  async count(criteria) {
+    if (!criteria || Object.keys(criteria).length === 0) {
+      return await this.size();
+    }
+    const entities = await this.query(criteria);
+    return entities?.length ?? 0;
+  }
   async* records(pageSize = 100) {
     if (pageSize <= 0) {
       throw new RangeError(`pageSize must be greater than 0, got ${pageSize}`);
@@ -1191,6 +1198,9 @@ class TelemetryTabularStorage {
   size() {
     return traced("workglow.storage.tabular.size", this.storageName, () => this.inner.size());
   }
+  count(criteria) {
+    return traced("workglow.storage.tabular.count", this.storageName, () => this.inner.count(criteria));
+  }
   deleteSearch(criteria) {
     return traced("workglow.storage.tabular.deleteSearch", this.storageName, () => this.inner.deleteSearch(criteria));
   }
@@ -1552,6 +1562,19 @@ class InMemoryQueueStorage {
       this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
     }
   }
+  async release(id) {
+    await sleep(0);
+    const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
+    if (job) {
+      const oldJob = { ...job };
+      job.status = JobStatus.PENDING;
+      job.worker_id = null;
+      job.progress = 0;
+      job.progress_message = "";
+      job.progress_details = null;
+      this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
+    }
+  }
   async abort(id) {
     await sleep(0);
     const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
@@ -1651,6 +1674,9 @@ class TelemetryQueueStorage {
   complete(job) {
     return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
   }
+  release(id) {
+    return traced("workglow.storage.queue.release", this.storageName, () => this.inner.release(id));
+  }
   deleteAll() {
     return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
   }
@@ -3119,6 +3145,16 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
     const result = await db.query(`SELECT COUNT(*) FROM "${this.table}"`);
     return parseInt(result.rows[0].count, 10);
   }
+  async count(criteria) {
+    if (!criteria || Object.keys(criteria).length === 0) {
+      return await this.size();
+    }
+    this.validateQueryParams(criteria);
+    const db = this.db;
+    const { whereClause, params } = this.buildDeleteSearchWhere(criteria);
+    const result = await db.query(`SELECT COUNT(*) FROM "${this.table}" WHERE ${whereClause}`, params);
+    return parseInt(result.rows[0].count, 10);
+  }
   async getBulk(offset, limit) {
     const db = this.db;
     const orderByClause = this.primaryKeyColumns().map((col) => `"${String(col)}"`).join(", ");
@@ -3585,6 +3621,18 @@ class SqliteTabularStorage extends BaseSqlTabularStorage {
     `);
     return stmt.get()?.count || 0;
   }
+  async count(criteria) {
+    if (!criteria || Object.keys(criteria).length === 0) {
+      return await this.size();
+    }
+    this.validateQueryParams(criteria);
+    const db = this.db;
+    const { whereClause, params } = this.buildDeleteSearchWhere(criteria);
+    const stmt = db.prepare(`
+      SELECT COUNT(*) AS count FROM \`${this.table}\` WHERE ${whereClause}
+    `);
+    return stmt.get(...params)?.count || 0;
+  }
   async getBulk(offset, limit) {
     const db = this.db;
     const orderByClause = this.primaryKeyColumns().map((col) => `\`${String(col)}\``).join(", ");
@@ -4029,16 +4077,9 @@ class SupabaseTabularStorage extends BaseSqlTabularStorage {
     }
     return data;
   }
-  async deleteSearch(criteria) {
-    const criteriaKeys = Object.keys(criteria);
-    if (criteriaKeys.length === 0) {
-      return;
-    }
-    let query = this.client.from(this.table).delete();
-    for (const column of criteriaKeys) {
-      if (!(column in this.schema.properties)) {
-        throw new Error(`Schema must have a ${String(column)} field to use deleteSearch`);
-      }
+  applyCriteriaToFilter(query, criteria) {
+    let q = query;
+    for (const column of Object.keys(criteria)) {
       const criterion = criteria[column];
       let operator = "=";
       let value;
@@ -4050,32 +4091,45 @@ class SupabaseTabularStorage extends BaseSqlTabularStorage {
       }
       switch (operator) {
         case "=":
-          query = query.eq(String(column), value);
+          q = q.eq(String(column), value);
           break;
         case "<":
-          query = query.lt(String(column), value);
+          q = q.lt(String(column), value);
           break;
         case "<=":
-          query = query.lte(String(column), value);
+          q = q.lte(String(column), value);
           break;
         case ">":
-          query = query.gt(String(column), value);
+          q = q.gt(String(column), value);
           break;
         case ">=":
-          query = query.gte(String(column), value);
+          q = q.gte(String(column), value);
           break;
       }
     }
-    const { error } = await query;
+    return q;
+  }
+  async count(criteria) {
+    if (!criteria || Object.keys(criteria).length === 0) {
+      return await this.size();
+    }
+    this.validateQueryParams(criteria);
+    const query = this.applyCriteriaToFilter(this.client.from(this.table).select("*", { count: "exact", head: true }), criteria);
+    const { count, error } = await query;
     if (error)
       throw error;
-    this.events.emit("delete", criteriaKeys[0]);
+    return count ?? 0;
   }
-  async query(criteria, options) {
-    this.validateQueryParams(criteria, options);
+  async deleteSearch(criteria) {
     const criteriaKeys = Object.keys(criteria);
-    let query = this.client.from(this.table).select("*");
+    if (criteriaKeys.length === 0) {
+      return;
+    }
+    let query = this.client.from(this.table).delete();
     for (const column of criteriaKeys) {
+      if (!(column in this.schema.properties)) {
+        throw new Error(`Schema must have a ${String(column)} field to use deleteSearch`);
+      }
       const criterion = criteria[column];
       let operator = "=";
       let value;
@@ -4103,6 +4157,14 @@ class SupabaseTabularStorage extends BaseSqlTabularStorage {
           break;
       }
     }
+    const { error } = await query;
+    if (error)
+      throw error;
+    this.events.emit("delete", criteriaKeys[0]);
+  }
+  async query(criteria, options) {
+    this.validateQueryParams(criteria, options);
+    let query = this.applyCriteriaToFilter(this.client.from(this.table).select("*"), criteria);
     if (options?.orderBy) {
       for (const { column, direction } of options.orderBy) {
         query = query.order(String(column), { ascending: direction === "ASC" });
@@ -4609,6 +4671,17 @@ class PostgresQueueStorage {
       SET status = 'ABORTING' 
       WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
   }
+  async release(jobId) {
+    const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
+    await this.db.query(`
+      UPDATE ${this.tableName}
+        SET status = 'PENDING',
+            worker_id = NULL,
+            progress = 0,
+            progress_message = '',
+            progress_details = NULL
+        WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
+  }
   async getByRunId(job_run_id) {
     const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
     const result = await this.db.query(`
@@ -4821,6 +4894,20 @@ class SqliteQueueStorage {
     const stmt = this.db.prepare(AbortQuery);
     stmt.run(JobStatus.ABORTING, String(jobId), this.queueName, ...prefixParams);
   }
+  async release(jobId) {
+    const prefixConditions = this.buildPrefixWhereClause();
+    const prefixParams = this.getPrefixParamValues();
+    const ReleaseQuery = `
+      UPDATE ${this.tableName}
+        SET status = ?,
+            worker_id = NULL,
+            progress = 0,
+            progress_message = '',
+            progress_details = NULL
+        WHERE id = ? AND queue = ?${prefixConditions}`;
+    const stmt = this.db.prepare(ReleaseQuery);
+    stmt.run(JobStatus.PENDING, String(jobId), this.queueName, ...prefixParams);
+  }
   async getByRunId(job_run_id) {
     const prefixConditions = this.buildPrefixWhereClause();
     const prefixParams = this.getPrefixParamValues();
@@ -5304,6 +5391,19 @@ class SupabaseQueueStorage {
     if (error)
       throw error;
   }
+  async release(jobId) {
+    let query = this.client.from(this.tableName).update({
+      status: JobStatus.PENDING,
+      worker_id: null,
+      progress: 0,
+      progress_message: "",
+      progress_details: null
+    }).eq("id", jobId).eq("queue", this.queueName);
+    query = this.applyPrefixFilters(query);
+    const { error } = await query;
+    if (error)
+      throw error;
+  }
   async deleteAll() {
     let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
     query = this.applyPrefixFilters(query);
@@ -6966,6 +7066,7 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
   migrationOptions;
   hybridManager = null;
   hybridOptions;
+  cursorSafeIndexes;
   constructor(table = "tabular_store", schema, primaryKeyNames, indexes = [], migrationOptions = {}, clientProvidedKeys = "if-missing") {
     super(schema, primaryKeyNames, indexes, clientProvidedKeys);
     this.table = table;
@@ -7191,6 +7292,83 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
       request.onsuccess = () => resolve(request.result);
     });
   }
+  getCursorSafeIndexes() {
+    if (this.cursorSafeIndexes)
+      return this.cursorSafeIndexes;
+    const required = new Set(this.schema.required ?? []);
+    this.cursorSafeIndexes = this.indexes.filter((columns) => columns.every((column) => required.has(String(column))));
+    return this.cursorSafeIndexes;
+  }
+  createIndexedRange(store, criteria) {
+    const criteriaColumns = Object.keys(criteria);
+    if (criteriaColumns.length === 0)
+      return;
+    let best;
+    for (const indexColumns of this.getCursorSafeIndexes()) {
+      const prefixValues = [];
+      for (const column of indexColumns) {
+        const value = this.getEqualityCriterionValue(criteria, column);
+        if (value === undefined)
+          break;
+        prefixValues.push(value);
+      }
+      if (prefixValues.length === 0)
+        continue;
+      const indexedPrefix = indexColumns.slice(0, prefixValues.length);
+      const coversCriteria = criteriaColumns.every((column) => indexedPrefix.includes(column));
+      const better = !best || coversCriteria && !best.coversCriteria || coversCriteria === best.coversCriteria && prefixValues.length > best.prefixValues.length;
+      if (better) {
+        best = {
+          indexName: indexColumns.map((column) => String(column)).join("_"),
+          prefixValues,
+          fullMatch: prefixValues.length === indexColumns.length,
+          coversCriteria
+        };
+      }
+    }
+    if (!best)
+      return;
+    const range = best.fullMatch ? IDBKeyRange.only(best.prefixValues.length === 1 ? best.prefixValues[0] : best.prefixValues) : IDBKeyRange.bound(best.prefixValues, [...best.prefixValues, []]);
+    return {
+      source: store.index(best.indexName),
+      range,
+      coversCriteria: best.coversCriteria
+    };
+  }
+  async count(criteria) {
+    if (!criteria || Object.keys(criteria).length === 0) {
+      return await this.size();
+    }
+    this.validateQueryParams(criteria);
+    const db = await this.getDb();
+    return new Promise((resolve, reject) => {
+      const transaction = db.transaction(this.table, "readonly");
+      const store = transaction.objectStore(this.table);
+      const plan = this.createIndexedRange(store, criteria);
+      if (plan?.coversCriteria) {
+        const request2 = plan.source.count(plan.range);
+        request2.onerror = () => reject(request2.error);
+        request2.onsuccess = () => resolve(request2.result);
+        return;
+      }
+      const source = plan?.source ?? store;
+      const range = plan?.range;
+      let count = 0;
+      const request = source.openCursor(range);
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => {
+        const cursor = request.result;
+        if (!cursor) {
+          resolve(count);
+          return;
+        }
+        if (this.matchesCriteria(cursor.value, criteria)) {
+          count += 1;
+        }
+        cursor.continue();
+      };
+    });
+  }
   async getBulk(offset, limit) {
     if (offset < 0) {
       throw new RangeError(`offset must be non-negative, got ${offset}`);
@@ -7312,48 +7490,135 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
       }
     });
   }
+  getEqualityCriterionValue(criteria, column) {
+    const criterion = criteria[column];
+    if (criterion === undefined)
+      return;
+    if (isSearchCondition(criterion)) {
+      return criterion.operator === "=" ? criterion.value : undefined;
+    }
+    return criterion;
+  }
+  compareByOrder(a, b, options) {
+    if (!options?.orderBy)
+      return 0;
+    for (const { column, direction } of options.orderBy) {
+      const aVal = a[column];
+      const bVal = b[column];
+      if (aVal == null && bVal == null)
+        continue;
+      if (aVal == null)
+        return direction === "ASC" ? -1 : 1;
+      if (bVal == null)
+        return direction === "ASC" ? 1 : -1;
+      if (aVal < bVal)
+        return direction === "ASC" ? -1 : 1;
+      if (aVal > bVal)
+        return direction === "ASC" ? 1 : -1;
+    }
+    return 0;
+  }
+  createIndexedQuery(store, criteria, options) {
+    const orderBy = options?.orderBy ?? [];
+    let best;
+    for (const indexColumns of this.getCursorSafeIndexes()) {
+      const prefixValues = [];
+      for (const column of indexColumns) {
+        const value = this.getEqualityCriterionValue(criteria, column);
+        if (value === undefined)
+          break;
+        prefixValues.push(value);
+      }
+      if (prefixValues.length === 0)
+        continue;
+      const remainingColumns = indexColumns.slice(prefixValues.length);
+      let redundantOrderPrefixLength = 0;
+      while (redundantOrderPrefixLength < orderBy.length && redundantOrderPrefixLength < prefixValues.length && orderBy[redundantOrderPrefixLength]?.column === indexColumns[redundantOrderPrefixLength]) {
+        redundantOrderPrefixLength++;
+      }
+      const normalizedOrderBy = orderBy.slice(redundantOrderPrefixLength);
+      const satisfiesOrder = normalizedOrderBy.length === 0 || normalizedOrderBy.length <= remainingColumns.length && normalizedOrderBy.every((order, index) => order.column === remainingColumns[index]) && orderBy.every((order) => order.direction === orderBy[0]?.direction);
+      if (!satisfiesOrder && best)
+        continue;
+      if (!best || satisfiesOrder && !best.satisfiesOrder || prefixValues.length > best.prefixValues.length) {
+        best = {
+          indexName: indexColumns.map((column) => String(column)).join("_"),
+          prefixValues,
+          fullMatch: prefixValues.length === indexColumns.length,
+          satisfiesOrder,
+          direction: orderBy[0]?.direction === "DESC" ? "prev" : "next"
+        };
+      }
+    }
+    const appliedLimit = Boolean(best?.satisfiesOrder && options?.limit !== undefined);
+    const appliedOffset = Boolean(best?.satisfiesOrder && options?.offset !== undefined);
+    if (!best) {
+      return {
+        source: store,
+        range: undefined,
+        direction: orderBy[0]?.direction === "DESC" ? "prev" : "next",
+        satisfiesOrder: false,
+        appliedLimit: false,
+        appliedOffset: false,
+        skipRemaining: 0
+      };
+    }
+    const source = store.index(best.indexName);
+    const keyRange = best.fullMatch ? IDBKeyRange.only(best.prefixValues.length === 1 ? best.prefixValues[0] : best.prefixValues) : IDBKeyRange.bound(best.prefixValues, [...best.prefixValues, []]);
+    return {
+      source,
+      range: keyRange,
+      direction: best.direction,
+      satisfiesOrder: best.satisfiesOrder,
+      appliedLimit,
+      appliedOffset,
+      skipRemaining: appliedOffset ? options?.offset ?? 0 : 0
+    };
+  }
   async query(criteria, options) {
     this.validateQueryParams(criteria, options);
     const db = await this.getDb();
     return new Promise((resolve, reject) => {
       const transaction = db.transaction(this.table, "readonly");
       const store = transaction.objectStore(this.table);
-      const getAllRequest = store.getAll();
-      getAllRequest.onsuccess = () => {
-        const allRecords = getAllRequest.result;
-        let results = allRecords.filter((record) => this.matchesCriteria(record, criteria));
-        if (options?.orderBy && options.orderBy.length > 0) {
-          results.sort((a, b) => {
-            for (const { column, direction } of options.orderBy) {
-              const aVal = a[column];
-              const bVal = b[column];
-              if (aVal == null && bVal == null)
-                continue;
-              if (aVal == null)
-                return direction === "ASC" ? -1 : 1;
-              if (bVal == null)
-                return direction === "ASC" ? 1 : -1;
-              if (aVal < bVal)
-                return direction === "ASC" ? -1 : 1;
-              if (aVal > bVal)
-                return direction === "ASC" ? 1 : -1;
-            }
-            return 0;
-          });
-        }
-        if (options?.offset !== undefined) {
-          results = results.slice(options.offset);
+      const indexedQuery = this.createIndexedQuery(store, criteria, options);
+      const results = [];
+      const request = indexedQuery.source.openCursor(indexedQuery.range, indexedQuery.direction);
+      request.onsuccess = () => {
+        const cursor = request.result;
+        if (!cursor) {
+          let finalResults = results;
+          if (!indexedQuery.satisfiesOrder && options?.orderBy && options.orderBy.length > 0) {
+            finalResults = [...finalResults].sort((a, b) => this.compareByOrder(a, b, options));
+          }
+          if (!indexedQuery.appliedOffset && options?.offset !== undefined) {
+            finalResults = finalResults.slice(options.offset);
+          }
+          if (!indexedQuery.appliedLimit && options?.limit !== undefined) {
+            finalResults = finalResults.slice(0, options.limit);
+          }
+          const result = finalResults.length > 0 ? finalResults : undefined;
+          this.events.emit("query", criteria, result);
+          resolve(result);
+          return;
         }
-        if (options?.limit !== undefined) {
-          results = results.slice(0, options.limit);
+        const record = cursor.value;
+        if (this.matchesCriteria(record, criteria)) {
+          if (indexedQuery.skipRemaining > 0) {
+            indexedQuery.skipRemaining -= 1;
+          } else {
+            results.push(record);
+            if (indexedQuery.appliedLimit && results.length === options?.limit) {
+              const result = results.length > 0 ? results : undefined;
+              this.events.emit("query", criteria, result);
+              resolve(result);
+              return;
+            }
+          }
         }
-        const result = results.length > 0 ? results : undefined;
-        this.events.emit("query", criteria, result);
-        resolve(result);
-      };
-      getAllRequest.onerror = () => {
-        reject(getAllRequest.error);
+        cursor.continue();
       };
+      request.onerror = () => reject(request.error);
     });
   }
   getHybridManager() {
@@ -7788,7 +8053,7 @@ class IndexedDbQueueStorage {
     const prefixKeyValues = this.getPrefixKeyValues();
     const claimToken = workerId;
     const jobToReturn = await new Promise((resolve, reject) => {
-      const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, JobStatus.PENDING, ""], [...prefixKeyValues, this.queueName, JobStatus.PENDING, now], false, true));
+      const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, JobStatus.PENDING, ""], [...prefixKeyValues, this.queueName, JobStatus.PENDING, now], false, false));
       let claimedJob;
       let cursorStopped = false;
       cursorRequest.onsuccess = (e) => {
@@ -7891,6 +8156,17 @@ class IndexedDbQueueStorage {
       tx.onerror = () => reject(tx.error);
     });
   }
+  async release(id) {
+    const job = await this.get(id);
+    if (!job)
+      return;
+    job.status = JobStatus.PENDING;
+    job.worker_id = null;
+    job.progress = 0;
+    job.progress_message = "";
+    job.progress_details = null;
+    await this.put(job);
+  }
   async abort(id) {
     const job = await this.get(id);
     if (!job)
@@ -8387,4 +8663,4 @@ export {
   BaseTabularStorage
 };
 
-//# debugId=DE2802AC74ABE49264756E2164756E21
+//# debugId=1E55C44CEF8B06B964756E2164756E21