anbaric-cloud-hosting 1.58.3 → 1.60.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anbaric-cloud-hosting",
-  "version": "1.58.3",
+  "version": "1.60.0",
   "description": "Anbaric Cloud hosting service: Postgres-backed job persistence and queuing exposed over an HTTP API",
   "license": "MIT",
   "author": "chris@anbaric.ai",
@@ -18,18 +18,18 @@
     "@aws-sdk/client-s3": "^3.1110.0",
     "@aws-sdk/client-secrets-manager": "^3.700.0",
     "@aws-sdk/client-servicediscovery": "^3.1110.0",
-    "anbaric-data-store": "^1.58.3",
-    "anbaric-plugins": "^1.58.3",
-    "anbaric-tsapi": "^1.58.3",
-    "anbaric-web": "^1.58.3",
+    "anbaric-data-store": "^1.60.0",
+    "anbaric-plugins": "^1.60.0",
+    "anbaric-tsapi": "^1.60.0",
+    "anbaric-web": "^1.60.0",
     "esbuild": "^0.28.2",
     "pg": "^8.16.0"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",
     "@types/pg": "^8.15.0",
-    "anbaric-impl-cloud": "^1.58.3",
-    "anbaric-state-machine": "^1.58.3",
+    "anbaric-impl-cloud": "^1.60.0",
+    "anbaric-state-machine": "^1.60.0",
     "tsx": "^4.20.0",
     "typescript": "^7.0.2"
   },

package/src/app-management/BaseBuildLayer.ts

@@ -15,6 +15,7 @@ type Deployment = {
     log : Array<string>,
     process? : ChildProcess,
     replaces? : Deployment,
+    draining? : { inFlight : number, since : number },
 };
 
 type Probe = (host : string, port : number) => Promise<boolean>;
@@ -23,6 +24,11 @@ const LOG_LIMIT = 200;
 const LIVENESS_TIMEOUT_MS = 30_000;
 const LIVENESS_PROBE_INTERVAL_MS = 250;
 const APP_ADMIN_PORT = 8791;
+/* How long a running app is given to finish the steps it has in hand before
+   it is replaced or removed regardless. Long enough for real work, short
+   enough that a step that will never finish cannot hold a deploy hostage. */
+const DRAIN_TIMEOUT_MS = 5 * 60_000;
+const DRAIN_POLL_INTERVAL_MS = 2_000;
 
 /* Liveness is the built-in admin server answering `ping` on the admin port, so
    an app that serves no HTTP still passes. */
@@ -32,12 +38,15 @@ abstract class BaseBuildLayer implements BuildLayer {
 
     protected deployments = new Map<string, Deployment>();
     protected docGenerator? : DocGenerator;
+    consumerUrlsFor : (appName : string) => Array<string> = () => [];
     private nextAppIndex = 0;
     private hydration? : Promise<void>;
 
     constructor(protected appsDir : string, private consumerPortBase : number = 8800,
                 private probe : Probe = adminProbe,
-                private livenessTimeoutMs : number = LIVENESS_TIMEOUT_MS) {}
+                private livenessTimeoutMs : number = LIVENESS_TIMEOUT_MS,
+                private drainTimeoutMs : number = DRAIN_TIMEOUT_MS,
+                private drainPollIntervalMs : number = DRAIN_POLL_INTERVAL_MS) {}
 
     /* The deployments map is rebuilt from the durable backend the first time the
        app registry is read, so apps survive a platform restart. A failed
@@ -95,16 +104,72 @@ abstract class BaseBuildLayer implements BuildLayer {
         yield* this.streamLogs(deployment, signal);
     }
 
-    async teardown(appName : string) : Promise<boolean> {
+    async teardown(appName : string) : Promise<DeploymentSummary | undefined> {
         const deployment = this.deployments.get(appName);
-        if (!deployment) return false;
+        if (!deployment) return undefined;
 
-        // stop() runs while the deployment is still the mapped one, so the
-        // Fargate guard lets it delete the service; then drop it from the map.
-        deployment.status = "stopped";
-        await this.stop(deployment);
-        this.deployments.delete(appName);
-        return true;
+        // An app with consumers to drain is drained in the background and the
+        // caller polls; one with nothing to drain is gone by the time this
+        // returns. Either way stop() runs while the deployment is still the
+        // mapped one, so the Fargate guard lets it delete the service.
+        const finish = async () => {
+            deployment.status = "stopped";
+            await this.stop(deployment);
+            if (this.deployments.get(appName) === deployment) this.deployments.delete(appName);
+        };
+
+        if (deployment.status === "running" && this.consumerUrlsFor(appName).length > 0) {
+            void this.drain(deployment, deployment).then(finish);
+            return this.summarize(deployment);
+        }
+
+        await finish();
+        return this.summarize(deployment);
+    }
+
+    /* Asks the running app to finish what it has in hand before it goes. The
+       app's consumers are told to stop accepting (the platform's pushes are
+       refused unconfirmed, so nothing is lost - it waits for the replacement)
+       and polled until nothing is in flight or the drain timeout passes;
+       `shown` is the deployment whose status the caller is watching. */
+    protected async drain(shown : Deployment, running : Deployment) : Promise<void> {
+        const urls = [...new Set(this.consumerUrlsFor(running.appName))];
+        if (urls.length === 0) return;
+
+        const previous = shown.status;
+        shown.status = "draining";
+        shown.draining = { inFlight: 0, since: Date.now() };
+        this.log(shown, `draining ${running.appName}: asking it to finish what it has in hand`);
+
+        await Promise.all(urls.map(url => fetch(`${url}/drain`, { method: "POST" }).catch(() => undefined)));
+
+        const deadline = Date.now() + this.drainTimeoutMs;
+        let inFlight = await this.inFlightAt(urls);
+        while (inFlight > 0 && Date.now() < deadline) {
+            shown.draining.inFlight = inFlight;
+            await new Promise(resolve => setTimeout(resolve, this.drainPollIntervalMs));
+            inFlight = await this.inFlightAt(urls);
+        }
+
+        this.log(shown, inFlight === 0
+            ? "drained: nothing in flight"
+            : `drain timed out after ${this.drainTimeoutMs / 1000}s with ${inFlight} step(s) in flight; going ahead`);
+        shown.draining = undefined;
+        if (shown.status === "draining") shown.status = previous;
+    }
+
+    // The steps in flight across the app's consumers; one that cannot be
+    // reached counts as idle, since nothing can be waited for there.
+    private async inFlightAt(urls : Array<string>) : Promise<number> {
+        const counts = await Promise.all(urls.map(async url => {
+            try {
+                const health = await (await fetch(`${url}/health`)).json() as { inFlight? : number };
+                return Number(health.inFlight ?? 0);
+            } catch {
+                return 0;
+            }
+        }));
+        return counts.reduce((total, count) => total + count, 0);
     }
 
     async cleanUp() : Promise<void> {
@@ -157,7 +222,13 @@ abstract class BaseBuildLayer implements BuildLayer {
         const manifest = JSON.parse(await readFile(join(appDir, "package.json"), "utf8"));
         await this.linkWorkspacePackages(appDir, manifest);
 
-        if (deployment.replaces) await this.stop(deployment.replaces);
+        // The version being replaced finishes its work first; the build above
+        // ran while it carried on, so the drain is the only pause it sees.
+        if (deployment.replaces) {
+            if (deployment.replaces.status === "running") await this.drain(deployment, deployment.replaces);
+            if (!this.isCurrent(deployment)) return;
+            await this.stop(deployment.replaces);
+        }
 
         if (!this.isCurrent(deployment)) return;
         await this.start(deployment, appDir, manifest.main);
@@ -245,6 +316,9 @@ abstract class BaseBuildLayer implements BuildLayer {
             status: deployment.status,
             appPort: deployment.appPort,
             appHost: deployment.appHost,
+            ...(deployment.draining
+                ? { draining: { inFlight: deployment.draining.inFlight, since: new Date(deployment.draining.since).toISOString() } }
+                : {}),
         };
     }
 

package/src/app-management/BuildLayer.ts

@@ -1,10 +1,19 @@
-type DeploymentStatus = "building" | "running" | "failed" | "stopped";
+/* building: the new version is being built; draining: the running version has
+   been told to stop taking on work and is finishing what it has in hand
+   before being replaced or removed; then running, failed or stopped. */
+type DeploymentStatus = "building" | "draining" | "running" | "failed" | "stopped";
+
+type Draining = {
+    inFlight : number,
+    since : string,
+};
 
 type DeploymentSummary = {
     appName : string,
     status : DeploymentStatus,
     appPort : number,
     appHost : string,
+    draining? : Draining,
 };
 
 interface BuildLayer {
@@ -15,12 +24,19 @@ interface BuildLayer {
     list() : Array<DeploymentSummary>;
     ping(appName : string) : Promise<boolean>;
     logs(appName : string, signal : AbortSignal) : AsyncIterable<string>;
-    teardown(appName : string) : Promise<boolean>;
+    /* Removes the app. One that is running its jobs is first drained, and the
+       call returns as soon as that has begun - the summary says "draining" -
+       so poll status until the app is gone. Returns undefined for an unknown
+       app. */
+    teardown(appName : string) : Promise<DeploymentSummary | undefined>;
     // Re-runs documentation generation for an already-deployed app from its
     // uploaded source, without redeploying it. Returns the number of docs written.
     regenerateDocs(appName : string) : Promise<number>;
     cleanUp() : Promise<void>;
+    // Where a running app's queue consumers can be reached, so it can be asked
+    // to drain; the hosting server supplies it from its consumer registry.
+    consumerUrlsFor : (appName : string) => Array<string>;
 
 }
 
-export type { BuildLayer, DeploymentStatus, DeploymentSummary };
+export type { BuildLayer, DeploymentStatus, DeploymentSummary, Draining };

package/src/app-management/FargateBuildLayer.ts

@@ -181,6 +181,9 @@ class FargateBuildLayer extends BaseBuildLayer {
                 name: deployment.appName,
                 image: imageUri,
                 essential: true,
+                // The most Fargate allows between SIGTERM and SIGKILL, so an
+                // app signalled with steps still in hand gets to finish them.
+                stopTimeout: 120,
                 portMappings: [
                     { containerPort: deployment.appPort, protocol: "tcp" },
                     { containerPort: deployment.adminPort, protocol: "tcp" },

package/src/data-store/PlatformMigrations.ts

@@ -230,6 +230,13 @@ const PLATFORM_MIGRATIONS : Array<Migration> = [
         "ALTER TABLE jobs DROP COLUMN properties",
     ),
 
+    /* A long-running step heartbeats its job; a job whose heartbeat has gone
+       quiet is marked Stalled. The index is what the stall sweep reads. */
+    statements("004-job-heartbeat",
+        "ALTER TABLE jobs ADD COLUMN heartbeat_at TIMESTAMPTZ",
+        "CREATE INDEX jobs_heartbeat ON jobs (heartbeat_at) WHERE heartbeat_at IS NOT NULL",
+    ),
+
 ];
 
 export { PLATFORM_MIGRATIONS };

package/src/data-store/PostgresJobPersistence.ts

@@ -13,6 +13,7 @@ type JobRow = {
     status : string,
     waiting_for? : string,
     await_metadata? : SerializedWaitForInput,
+    heartbeat_at? : Date,
 };
 
 /* The await a job is parked on is normalised into the awaits table: jobs
@@ -20,7 +21,7 @@ type JobRow = {
    job's awaitMetadata is rejoined on read. */
 const JOB_SELECT =
     `SELECT j.id, j.state, j.workflow_id, j.app_id, j.started_at, j.started_by, j.last_updated,
-            j.killed, j.status, j.waiting_for, a.metadata AS await_metadata
+            j.killed, j.status, j.waiting_for, j.heartbeat_at, a.metadata AS await_metadata
      FROM jobs j LEFT JOIN awaits a ON a.id = j.waiting_for`;
 
 /* Properties live one row each in job_properties, so a read fetches only the
@@ -153,6 +154,31 @@ class PostgresJobPersistence extends JobPersistence {
         return result.rows.map(row => this.deserializeRow(row, new JobProperties(properties.get(row.id) ?? new Map())));
     }
 
+    protected async heartbeatInternal(id : string, running : boolean) : Promise<void> {
+        // A heartbeat is proof the step is running, so one from a job marked
+        // Stalled - a process that came back, or a step slower than the
+        // sweep - returns it to active.
+        await this.pool.query(running
+            ? `UPDATE jobs SET heartbeat_at = now(),
+                   status = CASE WHEN status = $2 THEN $3 ELSE status END
+               WHERE id = $1`
+            : "UPDATE jobs SET heartbeat_at = NULL WHERE id = $1",
+            running ? [id, Job.Status.STALLED, Job.Status.ACTIVE] : [id]);
+    }
+
+    /* Marks Stalled every active job whose long-running step has not
+       heartbeated for `staleMs`. The heartbeat is left as it was, so the
+       record shows when the step was last heard from; a save that moves the
+       job on sets its status again. Returns how many were marked. */
+    async markStalled(staleMs : number = Job.STALL_AFTER_MS) : Promise<number> {
+        const result = await this.pool.query(
+            `UPDATE jobs SET status = $1, last_updated = now()
+             WHERE status = $2 AND killed = false AND heartbeat_at IS NOT NULL AND heartbeat_at < now() - ($3::text || ' milliseconds')::interval`,
+            [Job.Status.STALLED, Job.Status.ACTIVE, String(staleMs)],
+        );
+        return result.rowCount ?? 0;
+    }
+
     protected async killInternal(id : string) : Promise<void> {
         await this.pool.query("UPDATE jobs SET killed = true, last_updated = now() WHERE id = $1", [id]);
     }
@@ -185,9 +211,10 @@ class PostgresJobPersistence extends JobPersistence {
             status: row.status,
             waitingFor: row.waiting_for ?? undefined,
             awaitMetadata: row.await_metadata ?? undefined,
+            heartbeatAt: row.heartbeat_at?.toISOString(),
         });
         return new Job(shape.id, properties, shape.state, shape.workflowId, shape.appId, shape.startedBy, shape.startedAt,
-            shape.lastUpdated, shape.killed, shape.status, shape.awaitMetadata, shape.waitingFor);
+            shape.lastUpdated, shape.killed, shape.status, shape.awaitMetadata, shape.waitingFor, shape.heartbeatAt);
     }
 
 }

package/src/data-store/StallSweep.ts

@@ -0,0 +1,46 @@
+import {PostgresJobPersistence} from "./PostgresJobPersistence";
+
+const SWEEP_INTERVAL_MS = 60_000;
+
+/* Marks a job Stalled when the long-running step that was heartbeating it has
+   gone quiet - the process running it died, most likely mid-deploy or on a
+   crash. Marking is all it does: the platform never re-runs a long step on its
+   own, because only the app knows what a half-finished one has already done.
+   Someone sees the status and decides. */
+class StallSweep {
+
+    private ticker? : NodeJS.Timeout;
+    private sweeping = false;
+
+    constructor(private persistence : PostgresJobPersistence,
+                private intervalMs : number = SWEEP_INTERVAL_MS) {}
+
+    start() : void {
+        if (this.ticker) return;
+        this.ticker = setInterval(() => void this.sweep(), this.intervalMs);
+        this.ticker.unref();
+    }
+
+    async sweep() : Promise<number> {
+        if (this.sweeping) return 0;
+        this.sweeping = true;
+        try {
+            const stalled = await this.persistence.markStalled();
+            if (stalled > 0) console.warn(`[jobs] marked ${stalled} job(s) stalled: no heartbeat from a long-running step`);
+            return stalled;
+        } catch (error) {
+            console.error(`[jobs] the stall sweep failed: ${error instanceof Error ? error.message : error}`);
+            return 0;
+        } finally {
+            this.sweeping = false;
+        }
+    }
+
+    async cleanUp() : Promise<void> {
+        if (this.ticker) clearInterval(this.ticker);
+        this.ticker = undefined;
+    }
+
+}
+
+export { StallSweep }

package/src/hosting/HostingServer.ts

@@ -116,6 +116,10 @@ class HostingServer {
         }
         const appProxy = buildLayer ? new AppProxyHandler(buildLayer) : undefined;
         if (buildLayer && appProxy) {
+            // So a deploy can ask the running version to drain: its consumers
+            // are wherever they registered themselves.
+            buildLayer.consumerUrlsFor = appName =>
+                [...new Set(registry.list().filter(consumer => consumer.appId === appName).map(consumer => consumer.url))];
             publicRouter.registerApi("apps", new AppsHandler(buildLayer, tenant));
             publicRouter.register("app", appProxy);
         }

package/src/hosting/handlers/AppsHandler.ts

@@ -79,9 +79,9 @@ class AppsHandler implements RequestHandler {
                 return request.reply(200, { ...status, live });
             }
             case "DELETE": {
-                const torn = await this.buildLayer.teardown(appName);
-                if (!torn) return request.reply(404, { error: `No app named "${appName}"` });
-                return request.reply(200, { appName, status: "stopped" });
+                const outcome = await this.buildLayer.teardown(appName);
+                if (!outcome) return request.reply(404, { error: `No app named "${appName}"` });
+                return request.reply(200, outcome);
             }
         }
         request.notFound();

package/src/hosting/handlers/JobsHandler.ts

@@ -22,6 +22,12 @@ class JobsHandler implements RequestHandler {
             await this.persistence.kill(request.id, SystemActor.actor);
             return request.reply(204);
         }
+        if (request.id && request.subresource === "heartbeat") {
+            if (request.method !== "POST") return request.notFound();
+            const body = await request.body().catch(() => ({}));
+            await this.persistence.heartbeat(request.id, body?.running !== false);
+            return request.reply(204);
+        }
         if (request.subresource) return request.notFound();
         if (request.id) return this.handleJob(request, request.id);
         return this.handleCollection(request);

package/src/index.ts

@@ -2,6 +2,7 @@ export * from "./app-management/BuildLayer";
 export * from "./auditing/AuditRecordStore";
 export * from "./auditing/InMemoryAuditRecordStore";
 export * from "./data-store/PostgresAuditRecordStore";
+export * from "./data-store/StallSweep";
 export * from "./auth/Authenticator";
 export * from "./auth/CliAuthorizer";
 export * from "./auth/CliKey";

package/src/main.ts

@@ -17,6 +17,7 @@ import {SecretsManagerSecretStore} from "./data-store/SecretsManagerSecretStore"
 import {S3FileStorage} from "./data-store/S3FileStorage";
 import {PostgresJobRunSchedulePersistence} from "./data-store/PostgresJobRunSchedulePersistence";
 import {PostgresJobPersistence} from "./data-store/PostgresJobPersistence";
+import {StallSweep} from "./data-store/StallSweep";
 import {PostgresJsonStore} from "./data-store/PostgresJsonStore";
 import {PostgresQueue} from "./queuing/PostgresQueue";
 import {DockerBuildLayer} from "./app-management/DockerBuildLayer";
@@ -119,7 +120,8 @@ const fileStorageFor = (appId : string) : FileStorage =>
         ? new S3FileStorage(s3, filesBucket, appId ? `${appId}/` : "")
         : new LocalFileSystemStorage(join(process.env.ANBARIC_FILE_STORAGE_PATH ?? join(tmpdir(), "anbaric", "files"), appId));
 
-const server = new HostingServer(new PostgresJobPersistence(pool), queue, registry, buildLayer,
+const jobs = new PostgresJobPersistence(pool);
+const server = new HostingServer(jobs, queue, registry, buildLayer,
     (appId, collection) => new PostgresJsonStore(pool, appId, collection), secretStoreFor, authenticator, cliAuthorizer,
     tokenAuthenticator, process.env.ANBARIC_TENANT, new PostgresAuditRecordStore(pool), plugins,
     new PostgresJobRunSchedulePersistence(pool), await loadNotifier(process.env.ANBARIC_NOTIFIER_MODULE),
@@ -131,4 +133,7 @@ const internal = await server.listenInternal(internalPort);
 const dispatcher = new Dispatcher(queue, registry, Number(process.env.ANBARIC_DISPATCH_INTERVAL_MS ?? 1000));
 dispatcher.start();
 
+// Jobs whose long-running step stopped saying it was alive are marked, not touched.
+new StallSweep(jobs).start();
+
 console.log(`anbaric-cloud-hosting listening on port ${port}, internal entry point on ${internal}`);