anbaric-cloud-hosting 1.21.2 → 1.21.4

package/src/index.ts

@@ -52,6 +52,6 @@ export * from "./hosting/handlers/StateMachinesHandler";
 export * from "./hosting/handlers/auth/AuthorizeCliHandler";
 export * from "./hosting/handlers/auth/KeysHandler";
 export * from "./hosting/handlers/auth/WhoamiHandler";
-export * from "./queuing/ConfirmableQueue";
+export * from "./queuing/RemoteQueue";
 export * from "./queuing/ConsumerRegistry";
 export * from "./queuing/Dispatcher";

package/src/queuing/Dispatcher.ts

@@ -1,4 +1,5 @@
-import {Dequeue, QueueMessage} from "anbaric-tsapi";
+import {QueueMessage} from "anbaric-tsapi";
+import {RemoteQueue} from "./RemoteQueue";
 import {ConsumerRegistry} from "./ConsumerRegistry";
 
 class Dispatcher {
@@ -6,7 +7,7 @@ class Dispatcher {
     private ticker? : NodeJS.Timeout;
     private draining = false;
 
-    constructor(private queue : Dequeue, private registry : ConsumerRegistry,
+    constructor(private queue : RemoteQueue, private registry : ConsumerRegistry,
                 private dispatchIntervalMs : number = 1000) {}
 
     start() : void {
@@ -20,6 +21,13 @@ class Dispatcher {
         this.ticker = undefined;
     }
 
+    /* The queue leases each dequeued message and redelivers any it is not
+       `confirm`ed — the consumer confirms once it has processed one. So the
+       dispatcher never puts a message back: a failed push just leaves the
+       message to redeliver on its next lease. A message with no registered
+       listener yet is debounced (backed off), which lets a momentary
+       registration race resolve and cancels the message only if it never
+       finds a listener. */
     private async drain() : Promise<void> {
         if (this.draining) return;
         this.draining = true;
@@ -30,7 +38,7 @@ class Dispatcher {
             for (const message of messages) {
                 const url = this.registry.lookup(message.appId, message.workflowId);
                 if (!url) {
-                    await this.queue.enqueue(message.jobId, message.appId, message.workflowId);
+                    await this.queue.debounce(message);
                     continue;
                 }
                 byConsumerUrl.set(url, [...(byConsumerUrl.get(url) ?? []), message]);
@@ -53,9 +61,7 @@ class Dispatcher {
             });
             if (!response.ok) throw new Error(`Consumer at ${url} responded with status ${response.status}`);
         } catch {
-            for (const message of batch) {
-                await this.queue.enqueue(message.jobId, message.appId, message.workflowId);
-            }
+            // Left in the queue: the unconfirmed lease redelivers it for retry.
         }
     }
 

package/src/queuing/PostgresQueue.ts

@@ -1,11 +1,12 @@
 import {QueueMessage} from "anbaric-tsapi";
 import {Pool} from "pg";
-import {ConfirmableQueue} from "./ConfirmableQueue";
+import {RemoteQueue} from "./RemoteQueue";
 
 const DEQUEUE_BATCH_SIZE = 100;
 const LEASE_SECONDS = 30;
+const MAX_ATTEMPTS = 5;
 
-class PostgresQueue implements ConfirmableQueue {
+class PostgresQueue implements RemoteQueue {
 
     constructor(private pool : Pool) {}
 
@@ -23,6 +24,7 @@ class PostgresQueue implements ConfirmableQueue {
              WHERE position IN (
                  SELECT position FROM queue
                  WHERE (due IS NULL OR due <= now())
+                   AND (retry_at IS NULL OR retry_at <= now())
                    AND (leased_until IS NULL OR leased_until < now())
                  ORDER BY (due IS NOT NULL), due, position
                  LIMIT $1
@@ -47,6 +49,30 @@ class PostgresQueue implements ConfirmableQueue {
         await this.pool.query("DELETE FROM queue WHERE position = $1", [message.position]);
     }
 
+    async debounce(message : QueueMessage) : Promise<void> {
+        if (message.position === undefined) return;
+        // First bounce backs off 10s (a consumer that is mid-registration is
+        // ready by then); later bounces back off a minute. Clearing the lease
+        // makes the row eligible again the moment retry_at passes. After enough
+        // bounces the message is a genuine orphan, so it is cancelled.
+        const result = await this.pool.query(
+            `UPDATE queue
+             SET attempts = attempts + 1,
+                 retry_at = now() + (CASE WHEN attempts = 0 THEN interval '10 seconds' ELSE interval '1 minute' END),
+                 leased_until = NULL
+             WHERE position = $1
+             RETURNING attempts`,
+            [message.position],
+        );
+        if (Number(result.rows[0]?.attempts) >= MAX_ATTEMPTS) await this.cancel(message);
+    }
+
+    async cancel(message : QueueMessage) : Promise<void> {
+        console.warn(`Cancelling undeliverable queue message for job "${message.jobId}" (workflow "${message.workflowId}")`);
+        if (message.position === undefined) return;
+        await this.pool.query("DELETE FROM queue WHERE position = $1", [message.position]);
+    }
+
     async size() : Promise<number> {
         const result = await this.pool.query("SELECT count(*)::int AS count FROM queue");
         return result.rows[0].count;

package/src/queuing/RemoteQueue.ts

@@ -0,0 +1,17 @@
+import {Dequeue, QueueMessage} from "anbaric-tsapi";
+
+interface RemoteQueue extends Dequeue {
+
+    confirm(message : QueueMessage) : Promise<void>;
+    // Push a message that could not be delivered yet (e.g. its consumer has not
+    // registered) into the future with a growing back-off, so a momentary race
+    // resolves itself. After enough bounces the message is cancelled.
+    debounce(message : QueueMessage) : Promise<void>;
+    // Discard a message that can never be delivered. Removes it like confirm,
+    // but signals an error.
+    cancel(message : QueueMessage) : Promise<void>;
+    size() : Promise<number>;
+
+}
+
+export { RemoteQueue }

package/src/queuing/ConfirmableQueue.ts

@@ -1,10 +0,0 @@
-import {Dequeue, QueueMessage} from "anbaric-tsapi";
-
-interface ConfirmableQueue extends Dequeue {
-
-    confirm(message : QueueMessage) : Promise<void>;
-    size() : Promise<number>;
-
-}
-
-export { ConfirmableQueue }