@tstdl/base 0.93.198 → 0.93.199

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tstdl/base",
-  "version": "0.93.198",
+  "version": "0.93.199",
   "author": "Patrick Hein",
   "publishConfig": {
     "access": "public"
@@ -153,8 +153,8 @@
     "type-fest": "^5.5"
   },
   "peerDependencies": {
-    "@aws-sdk/client-s3": "^3.1025",
-    "@aws-sdk/s3-request-presigner": "^3.1025",
+    "@aws-sdk/client-s3": "^3.1026",
+    "@aws-sdk/s3-request-presigner": "^3.1026",
     "@genkit-ai/google-genai": "^1.31",
     "@google-cloud/storage": "^7.19",
     "@toon-format/toon": "^2.1.0",

package/task-queue/postgres/task-queue.js

@@ -187,6 +187,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                 },
             }));
             const itemsWithIdempotency = entitiesWithIndex.filter((e) => isNotNull(e.entity.idempotencyKey));
+            itemsWithIdempotency.sort((a, b) => a.entity.idempotencyKey.localeCompare(b.entity.idempotencyKey));
             const itemsWithoutIdempotency = entitiesWithIndex.filter((e) => isNull(e.entity.idempotencyKey));
             const hasDependencies = itemsWithDistinctDependencies.some((item) => ((item.scheduleAfter?.length ?? 0) > 0) || ((item.completeAfter?.length ?? 0) > 0) || (isDefined(item.parentId) && (item.parentRequires != false) && !(isArray(item.parentRequires) && (item.parentRequires.length == 0))));
             const mustUseTransaction = (entitiesWithIndex.length > 1) || hasDependencies;
@@ -382,7 +383,6 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
             decrementsToApply.set(dep.taskId, current);
         }
         const values = [...decrementsToApply]
-            .toSorted(([idA], [idB]) => idA.localeCompare(idB))
             .map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
         const updates = session.$with('updates').as((qb) => qb
             .select({
@@ -391,8 +391,14 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
             completeIncrement: sql `(complete)::int`.as('complete_increment'),
         })
             .from(sql `(VALUES ${sql.join(values, sql `, `)}) AS t(id, schedule, complete)`));
+        const locked = session.$with('locked').as((qb) => qb
+            .select({ id: taskTable.id })
+            .from(taskTable)
+            .innerJoin(updates, eq(taskTable.id, updates.taskId))
+            .orderBy(asc(taskTable.id))
+            .for('update'));
         const updatedRows = await session
-            .with(updates)
+            .with(updates, locked)
             .update(taskTable)
             .set({
             unresolvedScheduleDependencies: sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`,
@@ -400,7 +406,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
             status: caseWhen(and(eq(taskTable.status, TaskStatus.Pending), gt(sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`, 0)), TaskStatus.Waiting).else(taskTable.status),
         })
             .from(updates)
-            .where(eq(taskTable.id, updates.taskId))
+            .where(and(eq(taskTable.id, updates.taskId), inArray(taskTable.id, session.select({ id: locked.id }).from(locked))))
             .returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
         const notifiedNamespaces = new Set();
         for (const row of updatedRows) {
@@ -755,6 +761,12 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
             updateState: sql `(state)::jsonb`.as('update_state'),
         })
             .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, progress, state)`));
+        const locked = session.$with('locked').as((qb) => qb
+            .select({ id: taskTable.id })
+            .from(taskTable)
+            .innerJoin(updates, eq(taskTable.id, updates.updateId))
+            .orderBy(asc(taskTable.id))
+            .for('update'));
         const updated = session.$with('updated').as(() => session
             .update(taskTable)
             .set({
@@ -763,10 +775,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
             state: coalesce(updates.updateState, taskTable.state),
         })
             .from(updates)
-            .where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.status, TaskStatus.Running), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
+            .where(and(eq(taskTable.id, updates.updateId), inArray(taskTable.id, session.select({ id: locked.id }).from(locked)), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.status, TaskStatus.Running), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
             .returning({ id: taskTable.id }));
         const result = await session
-            .with(updates, updated)
+            .with(updates, locked, updated)
             .select({ id: updated.id })
             .from(updated);
         return result.map((r) => r.id);
@@ -810,6 +822,12 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                 updateProgress: sql `(progress)::numeric`.as('update_progress'),
             })
                 .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, result, progress)`));
+            const locked = tx.pgTransaction.$with('locked').as((qb) => qb
+                .select({ id: taskTable.id })
+                .from(taskTable)
+                .innerJoin(updates, eq(taskTable.id, updates.updateId))
+                .orderBy(asc(taskTable.id))
+                .for('update'));
             const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
                 .update(taskTable)
                 .set({
@@ -821,10 +839,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                 visibilityDeadline: null,
             })
                 .from(updates)
-                .where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`))
+                .where(and(eq(taskTable.id, updates.updateId), inArray(taskTable.id, tx.pgTransaction.select({ id: locked.id }).from(locked)), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`))
                 .returning({ id: taskTable.id, status: taskTable.status }));
             const updatedRows = await tx.pgTransaction
-                .with(updates, updated)
+                .with(updates, locked, updated)
                 .select({ id: updated.id, status: updated.status })
                 .from(updated);
             if (updatedRows.length > 0) {
@@ -879,6 +897,12 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                 updateComplete: sql `(complete_timestamp)`.as('update_complete'),
             })
                 .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, tries, status, error, schedule_timestamp, complete_timestamp)`));
+            const locked = tx.pgTransaction.$with('locked').as((qb) => qb
+                .select({ id: taskTable.id })
+                .from(taskTable)
+                .innerJoin(updates, eq(taskTable.id, updates.updateId))
+                .orderBy(asc(taskTable.id))
+                .for('update'));
             const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
                 .update(taskTable)
                 .set({
@@ -890,10 +914,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                 completeTimestamp: sql `${updates.updateComplete}`,
             })
                 .from(updates)
-                .where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.tries, updates.updateTries)))
+                .where(and(eq(taskTable.id, updates.updateId), inArray(taskTable.id, tx.pgTransaction.select({ id: locked.id }).from(locked)), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.tries, updates.updateTries)))
                 .returning({ id: taskTable.id, status: taskTable.status }));
             const result = await tx.pgTransaction
-                .with(updates, updated)
+                .with(updates, locked, updated)
                 .select({ id: updated.id, status: updated.status })
                 .from(updated);
             if (result.length > 0) {
@@ -916,7 +940,9 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
         await this.resolveDependenciesMany([{ id, status, namespace: options?.namespace }], options);
     }
     async resolveDependenciesMany(tasks, options) {
-        const tasksToResolve = tasks.filter((t) => terminalStatuses.includes(t.status));
+        const tasksToResolve = tasks
+            .filter((t) => terminalStatuses.includes(t.status))
+            .toSorted((a, b) => a.id.localeCompare(b.id));
         if (tasksToResolve.length == 0) {
             return;
         }
@@ -1013,7 +1039,6 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
             }
             if (decrementsToApply.size > 0) {
                 const decrementValues = [...decrementsToApply]
-                    .toSorted(([idA], [idB]) => idA.localeCompare(idB))
                     .map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
                 const updates = tx.pgTransaction.$with('updates').as((qb) => qb
                     .select({
@@ -1022,8 +1047,14 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                     completeDecrement: sql `(complete)::int`.as('complete_decrement'),
                 })
                     .from(sql `(VALUES ${sql.join(decrementValues, sql `, `)}) AS t(id, schedule, complete)`));
+                const locked = tx.pgTransaction.$with('locked').as((qb) => qb
+                    .select({ id: taskTable.id })
+                    .from(taskTable)
+                    .innerJoin(updates, eq(taskTable.id, updates.taskId))
+                    .orderBy(asc(taskTable.id))
+                    .for('update'));
                 const updatedRows = await tx.pgTransaction
-                    .with(updates)
+                    .with(updates, locked)
                     .update(taskTable)
                     .set({
                     unresolvedScheduleDependencies: greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`),
@@ -1034,7 +1065,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                     token: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), null).else(taskTable.token),
                 })
                     .from(updates)
-                    .where(eq(taskTable.id, updates.taskId))
+                    .where(and(eq(taskTable.id, updates.taskId), inArray(taskTable.id, tx.pgTransaction.select({ id: locked.id }).from(locked))))
                     .returning({
                     id: taskTable.id,
                     namespace: taskTable.namespace,