@workglow/storage 0.2.26 → 0.2.27

package/dist/browser.js

@@ -1574,6 +1574,7 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken9("jobqueue.storage.inMemory");
 
 class InMemoryQueueStorage {
   queueName;
+  scope = "process";
   prefixValues;
   events = new EventEmitter3;
   constructor(queueName, options) {
@@ -1783,6 +1784,9 @@ class TelemetryQueueStorage {
     this.storageName = storageName;
     this.inner = inner;
   }
+  get scope() {
+    return this.inner.scope;
+  }
   add(job) {
     return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
   }
@@ -1833,13 +1837,15 @@ class TelemetryQueueStorage {
   }
 }
 // src/queue-limiter/InMemoryRateLimiterStorage.ts
-import { createServiceToken as createServiceToken10, sleep as sleep2 } from "@workglow/util";
+import { createServiceToken as createServiceToken10, sleep as sleep2, uuid4 as uuid43 } from "@workglow/util";
 var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken10("ratelimiter.storage.inMemory");
 
 class InMemoryRateLimiterStorage {
+  scope = "process";
   prefixValues;
   executions = new Map;
   nextAvailableTimes = new Map;
+  reserveChains = new Map;
   constructor(options) {
     this.prefixValues = options?.prefixValues ?? {};
   }
@@ -1848,11 +1854,67 @@ class InMemoryRateLimiterStorage {
     return prefixPart ? `${prefixPart}|${queueName}` : queueName;
   }
   async setupDatabase() {}
+  async withKeyLock(key, fn) {
+    const previous = this.reserveChains.get(key) ?? Promise.resolve();
+    let release;
+    const next = new Promise((resolve) => {
+      release = resolve;
+    });
+    this.reserveChains.set(key, next);
+    try {
+      await previous;
+      return await fn();
+    } finally {
+      release(undefined);
+      if (this.reserveChains.get(key) === next) {
+        this.reserveChains.delete(key);
+      }
+    }
+  }
+  async tryReserveExecution(queueName, maxExecutions, windowMs) {
+    const key = this.makeKey(queueName);
+    return this.withKeyLock(key, () => {
+      const now = Date.now();
+      const windowStart = new Date(now - windowMs);
+      const executions = this.executions.get(key) ?? [];
+      const live = executions.filter((e) => e.executedAt > windowStart);
+      if (live.length >= maxExecutions) {
+        if (live.length !== executions.length) {
+          this.executions.set(key, live);
+        }
+        return null;
+      }
+      const next = this.nextAvailableTimes.get(key);
+      if (next && next.getTime() > now) {
+        return null;
+      }
+      const id = uuid43();
+      live.push({ id, queueName, executedAt: new Date(now) });
+      this.executions.set(key, live);
+      return id;
+    });
+  }
+  async releaseExecution(queueName, token) {
+    if (token === null || token === undefined)
+      return;
+    const key = this.makeKey(queueName);
+    await this.withKeyLock(key, () => {
+      const executions = this.executions.get(key);
+      if (!executions || executions.length === 0)
+        return;
+      const idx = executions.findIndex((e) => e.id === token);
+      if (idx === -1)
+        return;
+      executions.splice(idx, 1);
+      this.executions.set(key, executions);
+    });
+  }
   async recordExecution(queueName) {
     await sleep2(0);
     const key = this.makeKey(queueName);
     const executions = this.executions.get(key) ?? [];
     executions.push({
+      id: uuid43(),
       queueName,
       executedAt: new Date
     });
@@ -2434,7 +2496,7 @@ class LazyEncryptedCredentialStore {
   }
 }
 // src/tabular/IndexedDbTabularStorage.ts
-import { createServiceToken as createServiceToken12, deepEqual as deepEqual2, makeFingerprint as makeFingerprint5, uuid4 as uuid43 } from "@workglow/util";
+import { createServiceToken as createServiceToken12, deepEqual as deepEqual2, makeFingerprint as makeFingerprint5, uuid4 as uuid44 } from "@workglow/util";
 
 // src/util/IndexedDbTable.ts
 import { deepEqual } from "@workglow/util";
@@ -2825,7 +2887,7 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
   }
   generateKeyValue(columnName, strategy) {
     if (strategy === "uuid") {
-      return uuid43();
+      return uuid44();
     }
     throw new Error(`IndexedDB autoincrement keys are generated by the database, not client-side. Column: ${columnName}`);
   }
@@ -3398,13 +3460,15 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
         }
         let matches = true;
         for (const [col, crit] of Object.entries(criteria)) {
-          if (!isSearchCondition(crit))
-            continue;
           const pos = keyPathPositions.get(col);
           if (pos === undefined)
             continue;
+          if (pos < prefix.length)
+            continue;
           const valFromKey = Array.isArray(key) ? key[pos] : key;
-          if (!compareWithOperator(valFromKey, crit.operator, crit.value)) {
+          const op = isSearchCondition(crit) ? crit.operator : "=";
+          const val = isSearchCondition(crit) ? crit.value : crit;
+          if (!compareWithOperator(valFromKey, op, val)) {
             matches = false;
             break;
           }
@@ -3651,6 +3715,9 @@ class SharedInMemoryTabularStorage extends BaseTabularStorage {
   async query(criteria, options) {
     return await this.inMemoryRepo.query(criteria, options);
   }
+  async queryIndex(criteria, options) {
+    return await this.inMemoryRepo.queryIndex(criteria, options);
+  }
   async deleteSearch(criteria) {
     await this.inMemoryRepo.deleteSearch(criteria);
     this.broadcast({
@@ -4453,11 +4520,12 @@ class SupabaseKvStorage extends KvViaTabularStorage {
   }
 }
 // src/queue/IndexedDbQueueStorage.ts
-import { createServiceToken as createServiceToken17, deepEqual as deepEqual3, makeFingerprint as makeFingerprint6, uuid4 as uuid44 } from "@workglow/util";
+import { createServiceToken as createServiceToken17, deepEqual as deepEqual3, makeFingerprint as makeFingerprint6, uuid4 as uuid45 } from "@workglow/util";
 var INDEXED_DB_QUEUE_STORAGE = createServiceToken17("jobqueue.storage.indexedDb");
 
 class IndexedDbQueueStorage {
   queueName;
+  scope = "process";
   db;
   tableName;
   migrationOptions;
@@ -4534,8 +4602,8 @@ class IndexedDbQueueStorage {
     const db = await this.getDb();
     const now = new Date().toISOString();
     const jobWithPrefixes = job;
-    jobWithPrefixes.id = jobWithPrefixes.id ?? uuid44();
-    jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid44();
+    jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
+    jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
     jobWithPrefixes.queue = this.queueName;
     jobWithPrefixes.fingerprint = await makeFingerprint6(jobWithPrefixes.input);
     jobWithPrefixes.status = JobStatus.PENDING;
@@ -5033,11 +5101,12 @@ class IndexedDbQueueStorage {
   }
 }
 // src/queue/SupabaseQueueStorage.ts
-import { createServiceToken as createServiceToken18, deepEqual as deepEqual4, makeFingerprint as makeFingerprint7, uuid4 as uuid45 } from "@workglow/util";
+import { createServiceToken as createServiceToken18, deepEqual as deepEqual4, makeFingerprint as makeFingerprint7, uuid4 as uuid46 } from "@workglow/util";
 var SUPABASE_QUEUE_STORAGE = createServiceToken18("jobqueue.storage.supabase");
 
 class SupabaseQueueStorage {
   queueName;
+  scope = "cluster";
   client;
   prefixes;
   prefixValues;
@@ -5162,7 +5231,7 @@ class SupabaseQueueStorage {
   async add(job) {
     const now = new Date().toISOString();
     job.queue = this.queueName;
-    job.job_run_id = job.job_run_id ?? uuid45();
+    job.job_run_id = job.job_run_id ?? uuid46();
     job.fingerprint = await makeFingerprint7(job.input);
     job.status = JobStatus.PENDING;
     job.progress = 0;
@@ -5564,6 +5633,7 @@ import { createServiceToken as createServiceToken19 } from "@workglow/util";
 var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken19("ratelimiter.storage.indexedDb");
 
 class IndexedDbRateLimiterStorage {
+  scope = "process";
   executionDb;
   nextAvailableDb;
   executionTableName;
@@ -5632,6 +5702,72 @@ class IndexedDbRateLimiterStorage {
     ];
     this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
   }
+  async tryReserveExecution(queueName, maxExecutions, windowMs) {
+    const nextIso = await this.getNextAvailableTime(queueName);
+    if (nextIso && new Date(nextIso).getTime() > Date.now()) {
+      return null;
+    }
+    const execDb = await this.getExecutionDb();
+    const prefixKeyValues = this.getPrefixKeyValues();
+    const windowStartIso = new Date(Date.now() - windowMs).toISOString();
+    const execTx = execDb.transaction(this.executionTableName, "readwrite");
+    const execStore = execTx.objectStore(this.executionTableName);
+    const insertedId = crypto.randomUUID();
+    return new Promise((resolve, reject) => {
+      let liveCount = 0;
+      let didInsert = false;
+      const liveRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartIso], [...prefixKeyValues, queueName, "￿"], true, false);
+      const cursorReq = execStore.index("queue_executed_at").openCursor(liveRange);
+      cursorReq.onsuccess = (event) => {
+        const cursor = event.target.result;
+        if (cursor) {
+          const record2 = cursor.value;
+          if (this.matchesPrefixes(record2)) {
+            liveCount++;
+          }
+          cursor.continue();
+          return;
+        }
+        if (liveCount >= maxExecutions) {
+          execTx.abort();
+          return;
+        }
+        const record = {
+          id: insertedId,
+          queue_name: queueName,
+          executed_at: new Date().toISOString()
+        };
+        for (const [k, v] of Object.entries(this.prefixValues)) {
+          record[k] = v;
+        }
+        const addReq = execStore.add(record);
+        didInsert = true;
+        addReq.onerror = () => {
+          try {
+            execTx.abort();
+          } catch {}
+          reject(addReq.error);
+        };
+      };
+      cursorReq.onerror = () => reject(cursorReq.error);
+      execTx.oncomplete = () => resolve(didInsert ? insertedId : null);
+      execTx.onerror = () => reject(execTx.error);
+      execTx.onabort = () => resolve(null);
+    });
+  }
+  async releaseExecution(queueName, token) {
+    if (token === null || token === undefined)
+      return;
+    const db = await this.getExecutionDb();
+    const tx = db.transaction(this.executionTableName, "readwrite");
+    const store = tx.objectStore(this.executionTableName);
+    return new Promise((resolve, reject) => {
+      const req = store.delete(token);
+      req.onerror = () => reject(req.error);
+      tx.oncomplete = () => resolve();
+      tx.onerror = () => reject(tx.error);
+    });
+  }
   async recordExecution(queueName) {
     const db = await this.getExecutionDb();
     const tx = db.transaction(this.executionTableName, "readwrite");
@@ -5785,6 +5921,7 @@ import { createServiceToken as createServiceToken20 } from "@workglow/util";
 var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken20("ratelimiter.storage.supabase");
 
 class SupabaseRateLimiterStorage {
+  scope = "cluster";
   client;
   prefixes;
   prefixValues;
@@ -5867,6 +6004,75 @@ class SupabaseRateLimiterStorage {
     if (nextTableError && nextTableError.code !== "42P07") {
       throw nextTableError;
     }
+    const fnName = this.atomicReserveFunctionName();
+    const prefixSig = this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)}`).join(", ");
+    const prefixSigPrefix = prefixSig ? prefixSig + ", " : "";
+    const prefixWhere = this.prefixes.length > 0 ? " AND " + this.prefixes.map((p) => `${p.name} = _${p.name}`).join(" AND ") : "";
+    const prefixInsertCols = this.prefixes.length > 0 ? this.prefixes.map((p) => p.name).join(", ") + ", " : "";
+    const prefixInsertVals = this.prefixes.length > 0 ? this.prefixes.map((p) => `_${p.name}`).join(", ") + ", " : "";
+    const lockKeyParts = [`'${this.executionTableName}'`, ...this.prefixes.map((p) => `_${p.name}::text`), `_queue_name::text`];
+    const lockKeyExpr = `hashtextextended(${lockKeyParts.join(" || '|' || ")}, 0)`;
+    const createFnSql = `
+      CREATE OR REPLACE FUNCTION ${fnName}(
+        ${prefixSigPrefix}_queue_name TEXT, _window_start TIMESTAMPTZ, _max_exec INT
+      ) RETURNS BIGINT AS $fn$
+      DECLARE
+        _count INT;
+        _next TIMESTAMPTZ;
+        _new_id BIGINT;
+      BEGIN
+        PERFORM pg_advisory_xact_lock(${lockKeyExpr});
+        SELECT COUNT(*) INTO _count FROM ${this.executionTableName}
+          WHERE queue_name = _queue_name AND executed_at > _window_start${prefixWhere};
+        IF _count >= _max_exec THEN RETURN NULL; END IF;
+        SELECT next_available_at INTO _next FROM ${this.nextAvailableTableName}
+          WHERE queue_name = _queue_name${prefixWhere};
+        IF _next IS NOT NULL AND _next > NOW() THEN RETURN NULL; END IF;
+        INSERT INTO ${this.executionTableName} (${prefixInsertCols}queue_name)
+          VALUES (${prefixInsertVals}_queue_name)
+          RETURNING id INTO _new_id;
+        RETURN _new_id;
+      END;
+      $fn$ LANGUAGE plpgsql;
+    `;
+    const { error: fnError } = await this.client.rpc("exec_sql", { query: createFnSql });
+    if (fnError) {
+      throw fnError;
+    }
+  }
+  atomicReserveFunctionName() {
+    return `${this.executionTableName}_try_reserve`.slice(0, 63);
+  }
+  async tryReserveExecution(queueName, maxExecutions, windowMs) {
+    const args = {
+      _queue_name: queueName,
+      _window_start: new Date(Date.now() - windowMs).toISOString(),
+      _max_exec: maxExecutions
+    };
+    for (const p of this.prefixes) {
+      args[`_${p.name}`] = this.prefixValues[p.name];
+    }
+    const { data, error } = await this.client.rpc(this.atomicReserveFunctionName(), args);
+    if (error)
+      throw error;
+    if (data === null || data === undefined)
+      return null;
+    if (Array.isArray(data)) {
+      if (data.length === 0)
+        return null;
+      const first = Object.values(data[0])[0];
+      return first ?? null;
+    }
+    return data;
+  }
+  async releaseExecution(queueName, token) {
+    if (token === null || token === undefined)
+      return;
+    let del = this.client.from(this.executionTableName).delete().eq("id", token).eq("queue_name", queueName);
+    del = this.applyPrefixFilters(del);
+    const { error: delError } = await del;
+    if (delError)
+      throw delError;
   }
   async recordExecution(queueName) {
     const prefixInsertValues = this.getPrefixInsertValues();
@@ -6103,4 +6309,4 @@ export {
   BaseTabularStorage
 };
 
-//# debugId=2583E6527A9527C764756E2164756E21
+//# debugId=E2D716C3197417F164756E2164756E21