anbaric-cloud-hosting 1.21.3 → 1.21.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anbaric-cloud-hosting",
-  "version": "1.21.3",
+  "version": "1.21.4",
   "description": "Anbaric Cloud hosting service: Postgres-backed job persistence and queuing exposed over an HTTP API",
   "license": "MIT",
   "author": "chris@anbaric.ai",
@@ -18,18 +18,18 @@
     "@aws-sdk/client-s3": "^3.1110.0",
     "@aws-sdk/client-secrets-manager": "^3.700.0",
     "@aws-sdk/client-servicediscovery": "^3.1110.0",
-    "anbaric-data-store": "^1.21.3",
-    "anbaric-plugins": "^1.21.3",
-    "anbaric-tsapi": "^1.21.3",
-    "anbaric-web": "^1.21.3",
+    "anbaric-data-store": "^1.21.4",
+    "anbaric-plugins": "^1.21.4",
+    "anbaric-tsapi": "^1.21.4",
+    "anbaric-web": "^1.21.4",
     "esbuild": "^0.28.2",
     "pg": "^8.16.0"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",
     "@types/pg": "^8.15.0",
-    "anbaric-impl-cloud": "^1.21.3",
-    "anbaric-state-machine": "^1.21.3",
+    "anbaric-impl-cloud": "^1.21.4",
+    "anbaric-state-machine": "^1.21.4",
     "tsx": "^4.20.0",
     "typescript": "^7.0.2"
   },

package/src/data-store/Schema.ts

@@ -72,6 +72,8 @@ const ensureSchema = async (pool : Pool) : Promise<void> => {
         )
     `);
     await pool.query("ALTER TABLE queue ADD COLUMN IF NOT EXISTS app_id TEXT");
+    await pool.query("ALTER TABLE queue ADD COLUMN IF NOT EXISTS retry_at TIMESTAMPTZ");
+    await pool.query("ALTER TABLE queue ADD COLUMN IF NOT EXISTS attempts INT NOT NULL DEFAULT 0");
     await pool.query(`UPDATE queue
         SET app_id = split_part(workflow_id, '/', 1),
             workflow_id = substring(workflow_id from position('/' in workflow_id) + 1)

package/src/hosting/HostingServer.ts

@@ -4,7 +4,7 @@ import {AuditRecordStore} from "../auditing/AuditRecordStore";
 import {Authenticator} from "../auth/Authenticator";
 import {CliAuthorizer} from "../auth/CliAuthorizer";
 import {TokenAuthenticator} from "../auth/TokenAuthenticator";
-import {ConfirmableQueue} from "../queuing/ConfirmableQueue";
+import {RemoteQueue} from "../queuing/RemoteQueue";
 import {ConsumerRegistry} from "../queuing/ConsumerRegistry";
 import {AppProxyHandler} from "./handlers/AppProxyHandler";
 import {AppLinkFallbackHandler} from "./handlers/AppLinkFallbackHandler";
@@ -42,7 +42,7 @@ class HostingServer {
     private publicServer : Server;
     private internalServer : Server;
 
-    constructor(persistence : JobPersistence, queue : ConfirmableQueue,
+    constructor(persistence : JobPersistence, queue : RemoteQueue,
                 registry : ConsumerRegistry = new ConsumerRegistry(),
                 buildLayer? : BuildLayer,
                 documentStoreFor? : (appId : string, collection : string) => JsonStore,

package/src/hosting/handlers/QueueHandler.ts

@@ -1,10 +1,10 @@
-import {ConfirmableQueue} from "../../queuing/ConfirmableQueue";
+import {RemoteQueue} from "../../queuing/RemoteQueue";
 import {Request} from "../Request";
 import {RequestHandler} from "../RequestHandler";
 
 class QueueHandler implements RequestHandler {
 
-    constructor(private queue : ConfirmableQueue) {}
+    constructor(private queue : RemoteQueue) {}
 
     async handle(request : Request) : Promise<void> {
         if (request.subresource) return request.notFound();

package/src/index.ts

@@ -52,6 +52,6 @@ export * from "./hosting/handlers/StateMachinesHandler";
 export * from "./hosting/handlers/auth/AuthorizeCliHandler";
 export * from "./hosting/handlers/auth/KeysHandler";
 export * from "./hosting/handlers/auth/WhoamiHandler";
-export * from "./queuing/ConfirmableQueue";
+export * from "./queuing/RemoteQueue";
 export * from "./queuing/ConsumerRegistry";
 export * from "./queuing/Dispatcher";

package/src/queuing/Dispatcher.ts

@@ -1,4 +1,5 @@
-import {Dequeue, QueueMessage} from "anbaric-tsapi";
+import {QueueMessage} from "anbaric-tsapi";
+import {RemoteQueue} from "./RemoteQueue";
 import {ConsumerRegistry} from "./ConsumerRegistry";
 
 class Dispatcher {
@@ -6,18 +7,8 @@ class Dispatcher {
     private ticker? : NodeJS.Timeout;
     private draining = false;
 
-    /* A confirmable queue leases each dequeued message and redelivers any it is
-       not `confirm`ed (the consumer confirms once it has processed it). So the
-       dispatcher must NOT put unroutable or failed messages back — the leased
-       row already redelivers, and re-enqueuing would add a fresh duplicate on
-       every tick, growing the queue without bound. A plain queue removes on
-       dequeue, so there those messages must be re-enqueued to be retried. */
-    private readonly redelivers : boolean;
-
-    constructor(private queue : Dequeue, private registry : ConsumerRegistry,
-                private dispatchIntervalMs : number = 1000) {
-        this.redelivers = typeof (this.queue as { confirm? : unknown }).confirm === "function";
-    }
+    constructor(private queue : RemoteQueue, private registry : ConsumerRegistry,
+                private dispatchIntervalMs : number = 1000) {}
 
     start() : void {
         if (this.ticker) return;
@@ -30,11 +21,16 @@ class Dispatcher {
         this.ticker = undefined;
     }
 
+    /* The queue leases each dequeued message and redelivers any it is not
+       `confirm`ed — the consumer confirms once it has processed one. So the
+       dispatcher never puts a message back: a failed push just leaves the
+       message to redeliver on its next lease. A message with no registered
+       listener yet is debounced (backed off), which lets a momentary
+       registration race resolve and cancels the message only if it never
+       finds a listener. */
     private async drain() : Promise<void> {
-
         if (this.draining) return;
         this.draining = true;
-
         try {
             const messages = await this.queue.dequeueSome();
             const byConsumerUrl = new Map<string, Array<QueueMessage>>();
@@ -42,7 +38,7 @@ class Dispatcher {
             for (const message of messages) {
                 const url = this.registry.lookup(message.appId, message.workflowId);
                 if (!url) {
-                    if (!this.redelivers) await this.queue.enqueue(message.jobId, message.appId, message.workflowId);
+                    await this.queue.debounce(message);
                     continue;
                 }
                 byConsumerUrl.set(url, [...(byConsumerUrl.get(url) ?? []), message]);
@@ -65,11 +61,7 @@ class Dispatcher {
             });
             if (!response.ok) throw new Error(`Consumer at ${url} responded with status ${response.status}`);
         } catch {
-            if (!this.redelivers) {
-                for (const message of batch) {
-                    await this.queue.enqueue(message.jobId, message.appId, message.workflowId);
-                }
-            }
+            // Left in the queue: the unconfirmed lease redelivers it for retry.
         }
     }
 

package/src/queuing/PostgresQueue.ts

@@ -1,11 +1,12 @@
 import {QueueMessage} from "anbaric-tsapi";
 import {Pool} from "pg";
-import {ConfirmableQueue} from "./ConfirmableQueue";
+import {RemoteQueue} from "./RemoteQueue";
 
 const DEQUEUE_BATCH_SIZE = 100;
 const LEASE_SECONDS = 30;
+const MAX_ATTEMPTS = 5;
 
-class PostgresQueue implements ConfirmableQueue {
+class PostgresQueue implements RemoteQueue {
 
     constructor(private pool : Pool) {}
 
@@ -23,6 +24,7 @@ class PostgresQueue implements ConfirmableQueue {
              WHERE position IN (
                  SELECT position FROM queue
                  WHERE (due IS NULL OR due <= now())
+                   AND (retry_at IS NULL OR retry_at <= now())
                    AND (leased_until IS NULL OR leased_until < now())
                  ORDER BY (due IS NOT NULL), due, position
                  LIMIT $1
@@ -47,6 +49,30 @@ class PostgresQueue implements ConfirmableQueue {
         await this.pool.query("DELETE FROM queue WHERE position = $1", [message.position]);
     }
 
+    async debounce(message : QueueMessage) : Promise<void> {
+        if (message.position === undefined) return;
+        // First bounce backs off 10s (a consumer that is mid-registration is
+        // ready by then); later bounces back off a minute. Clearing the lease
+        // makes the row eligible again the moment retry_at passes. After enough
+        // bounces the message is a genuine orphan, so it is cancelled.
+        const result = await this.pool.query(
+            `UPDATE queue
+             SET attempts = attempts + 1,
+                 retry_at = now() + (CASE WHEN attempts = 0 THEN interval '10 seconds' ELSE interval '1 minute' END),
+                 leased_until = NULL
+             WHERE position = $1
+             RETURNING attempts`,
+            [message.position],
+        );
+        if (Number(result.rows[0]?.attempts) >= MAX_ATTEMPTS) await this.cancel(message);
+    }
+
+    async cancel(message : QueueMessage) : Promise<void> {
+        console.warn(`Cancelling undeliverable queue message for job "${message.jobId}" (workflow "${message.workflowId}")`);
+        if (message.position === undefined) return;
+        await this.pool.query("DELETE FROM queue WHERE position = $1", [message.position]);
+    }
+
     async size() : Promise<number> {
         const result = await this.pool.query("SELECT count(*)::int AS count FROM queue");
         return result.rows[0].count;

package/src/queuing/RemoteQueue.ts

@@ -0,0 +1,17 @@
+import {Dequeue, QueueMessage} from "anbaric-tsapi";
+
+interface RemoteQueue extends Dequeue {
+
+    confirm(message : QueueMessage) : Promise<void>;
+    // Push a message that could not be delivered yet (e.g. its consumer has not
+    // registered) into the future with a growing back-off, so a momentary race
+    // resolves itself. After enough bounces the message is cancelled.
+    debounce(message : QueueMessage) : Promise<void>;
+    // Discard a message that can never be delivered. Removes it like confirm,
+    // but signals an error.
+    cancel(message : QueueMessage) : Promise<void>;
+    size() : Promise<number>;
+
+}
+
+export { RemoteQueue }

package/src/queuing/ConfirmableQueue.ts

@@ -1,10 +0,0 @@
-import {Dequeue, QueueMessage} from "anbaric-tsapi";
-
-interface ConfirmableQueue extends Dequeue {
-
-    confirm(message : QueueMessage) : Promise<void>;
-    size() : Promise<number>;
-
-}
-
-export { ConfirmableQueue }