anbaric-cloud-hosting 1.59.0 → 1.60.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anbaric-cloud-hosting",
-  "version": "1.59.0",
+  "version": "1.60.0",
   "description": "Anbaric Cloud hosting service: Postgres-backed job persistence and queuing exposed over an HTTP API",
   "license": "MIT",
   "author": "chris@anbaric.ai",
@@ -18,18 +18,18 @@
     "@aws-sdk/client-s3": "^3.1110.0",
     "@aws-sdk/client-secrets-manager": "^3.700.0",
     "@aws-sdk/client-servicediscovery": "^3.1110.0",
-    "anbaric-data-store": "^1.59.0",
-    "anbaric-plugins": "^1.59.0",
-    "anbaric-tsapi": "^1.59.0",
-    "anbaric-web": "^1.59.0",
+    "anbaric-data-store": "^1.60.0",
+    "anbaric-plugins": "^1.60.0",
+    "anbaric-tsapi": "^1.60.0",
+    "anbaric-web": "^1.60.0",
     "esbuild": "^0.28.2",
     "pg": "^8.16.0"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",
     "@types/pg": "^8.15.0",
-    "anbaric-impl-cloud": "^1.59.0",
-    "anbaric-state-machine": "^1.59.0",
+    "anbaric-impl-cloud": "^1.60.0",
+    "anbaric-state-machine": "^1.60.0",
     "tsx": "^4.20.0",
     "typescript": "^7.0.2"
   },

package/src/data-store/PlatformMigrations.ts

@@ -230,6 +230,13 @@ const PLATFORM_MIGRATIONS : Array<Migration> = [
         "ALTER TABLE jobs DROP COLUMN properties",
     ),
 
+    /* A long-running step heartbeats its job; a job whose heartbeat has gone
+       quiet is marked Stalled. The index is what the stall sweep reads. */
+    statements("004-job-heartbeat",
+        "ALTER TABLE jobs ADD COLUMN heartbeat_at TIMESTAMPTZ",
+        "CREATE INDEX jobs_heartbeat ON jobs (heartbeat_at) WHERE heartbeat_at IS NOT NULL",
+    ),
+
 ];
 
 export { PLATFORM_MIGRATIONS };

package/src/data-store/PostgresJobPersistence.ts

@@ -13,6 +13,7 @@ type JobRow = {
     status : string,
     waiting_for? : string,
     await_metadata? : SerializedWaitForInput,
+    heartbeat_at? : Date,
 };
 
 /* The await a job is parked on is normalised into the awaits table: jobs
@@ -20,7 +21,7 @@ type JobRow = {
    job's awaitMetadata is rejoined on read. */
 const JOB_SELECT =
     `SELECT j.id, j.state, j.workflow_id, j.app_id, j.started_at, j.started_by, j.last_updated,
-            j.killed, j.status, j.waiting_for, a.metadata AS await_metadata
+            j.killed, j.status, j.waiting_for, j.heartbeat_at, a.metadata AS await_metadata
      FROM jobs j LEFT JOIN awaits a ON a.id = j.waiting_for`;
 
 /* Properties live one row each in job_properties, so a read fetches only the
@@ -153,6 +154,31 @@ class PostgresJobPersistence extends JobPersistence {
         return result.rows.map(row => this.deserializeRow(row, new JobProperties(properties.get(row.id) ?? new Map())));
     }
 
+    protected async heartbeatInternal(id : string, running : boolean) : Promise<void> {
+        // A heartbeat is proof the step is running, so one from a job marked
+        // Stalled - a process that came back, or a step slower than the
+        // sweep - returns it to active.
+        await this.pool.query(running
+            ? `UPDATE jobs SET heartbeat_at = now(),
+                   status = CASE WHEN status = $2 THEN $3 ELSE status END
+               WHERE id = $1`
+            : "UPDATE jobs SET heartbeat_at = NULL WHERE id = $1",
+            running ? [id, Job.Status.STALLED, Job.Status.ACTIVE] : [id]);
+    }
+
+    /* Marks Stalled every active job whose long-running step has not
+       heartbeated for `staleMs`. The heartbeat is left as it was, so the
+       record shows when the step was last heard from; a save that moves the
+       job on sets its status again. Returns how many were marked. */
+    async markStalled(staleMs : number = Job.STALL_AFTER_MS) : Promise<number> {
+        const result = await this.pool.query(
+            `UPDATE jobs SET status = $1, last_updated = now()
+             WHERE status = $2 AND killed = false AND heartbeat_at IS NOT NULL AND heartbeat_at < now() - ($3::text || ' milliseconds')::interval`,
+            [Job.Status.STALLED, Job.Status.ACTIVE, String(staleMs)],
+        );
+        return result.rowCount ?? 0;
+    }
+
     protected async killInternal(id : string) : Promise<void> {
         await this.pool.query("UPDATE jobs SET killed = true, last_updated = now() WHERE id = $1", [id]);
     }
@@ -185,9 +211,10 @@ class PostgresJobPersistence extends JobPersistence {
             status: row.status,
             waitingFor: row.waiting_for ?? undefined,
             awaitMetadata: row.await_metadata ?? undefined,
+            heartbeatAt: row.heartbeat_at?.toISOString(),
         });
         return new Job(shape.id, properties, shape.state, shape.workflowId, shape.appId, shape.startedBy, shape.startedAt,
-            shape.lastUpdated, shape.killed, shape.status, shape.awaitMetadata, shape.waitingFor);
+            shape.lastUpdated, shape.killed, shape.status, shape.awaitMetadata, shape.waitingFor, shape.heartbeatAt);
     }
 
 }

package/src/data-store/StallSweep.ts

@@ -0,0 +1,46 @@
+import {PostgresJobPersistence} from "./PostgresJobPersistence";
+
+const SWEEP_INTERVAL_MS = 60_000;
+
+/* Marks a job Stalled when the long-running step that was heartbeating it has
+   gone quiet - the process running it died, most likely mid-deploy or on a
+   crash. Marking is all it does: the platform never re-runs a long step on its
+   own, because only the app knows what a half-finished one has already done.
+   Someone sees the status and decides. */
+class StallSweep {
+
+    private ticker? : NodeJS.Timeout;
+    private sweeping = false;
+
+    constructor(private persistence : PostgresJobPersistence,
+                private intervalMs : number = SWEEP_INTERVAL_MS) {}
+
+    start() : void {
+        if (this.ticker) return;
+        this.ticker = setInterval(() => void this.sweep(), this.intervalMs);
+        this.ticker.unref();
+    }
+
+    async sweep() : Promise<number> {
+        if (this.sweeping) return 0;
+        this.sweeping = true;
+        try {
+            const stalled = await this.persistence.markStalled();
+            if (stalled > 0) console.warn(`[jobs] marked ${stalled} job(s) stalled: no heartbeat from a long-running step`);
+            return stalled;
+        } catch (error) {
+            console.error(`[jobs] the stall sweep failed: ${error instanceof Error ? error.message : error}`);
+            return 0;
+        } finally {
+            this.sweeping = false;
+        }
+    }
+
+    async cleanUp() : Promise<void> {
+        if (this.ticker) clearInterval(this.ticker);
+        this.ticker = undefined;
+    }
+
+}
+
+export { StallSweep }

package/src/hosting/handlers/JobsHandler.ts

@@ -22,6 +22,12 @@ class JobsHandler implements RequestHandler {
             await this.persistence.kill(request.id, SystemActor.actor);
             return request.reply(204);
         }
+        if (request.id && request.subresource === "heartbeat") {
+            if (request.method !== "POST") return request.notFound();
+            const body = await request.body().catch(() => ({}));
+            await this.persistence.heartbeat(request.id, body?.running !== false);
+            return request.reply(204);
+        }
         if (request.subresource) return request.notFound();
         if (request.id) return this.handleJob(request, request.id);
         return this.handleCollection(request);

package/src/index.ts

@@ -2,6 +2,7 @@ export * from "./app-management/BuildLayer";
 export * from "./auditing/AuditRecordStore";
 export * from "./auditing/InMemoryAuditRecordStore";
 export * from "./data-store/PostgresAuditRecordStore";
+export * from "./data-store/StallSweep";
 export * from "./auth/Authenticator";
 export * from "./auth/CliAuthorizer";
 export * from "./auth/CliKey";

package/src/main.ts

@@ -17,6 +17,7 @@ import {SecretsManagerSecretStore} from "./data-store/SecretsManagerSecretStore"
 import {S3FileStorage} from "./data-store/S3FileStorage";
 import {PostgresJobRunSchedulePersistence} from "./data-store/PostgresJobRunSchedulePersistence";
 import {PostgresJobPersistence} from "./data-store/PostgresJobPersistence";
+import {StallSweep} from "./data-store/StallSweep";
 import {PostgresJsonStore} from "./data-store/PostgresJsonStore";
 import {PostgresQueue} from "./queuing/PostgresQueue";
 import {DockerBuildLayer} from "./app-management/DockerBuildLayer";
@@ -119,7 +120,8 @@ const fileStorageFor = (appId : string) : FileStorage =>
         ? new S3FileStorage(s3, filesBucket, appId ? `${appId}/` : "")
         : new LocalFileSystemStorage(join(process.env.ANBARIC_FILE_STORAGE_PATH ?? join(tmpdir(), "anbaric", "files"), appId));
 
-const server = new HostingServer(new PostgresJobPersistence(pool), queue, registry, buildLayer,
+const jobs = new PostgresJobPersistence(pool);
+const server = new HostingServer(jobs, queue, registry, buildLayer,
     (appId, collection) => new PostgresJsonStore(pool, appId, collection), secretStoreFor, authenticator, cliAuthorizer,
     tokenAuthenticator, process.env.ANBARIC_TENANT, new PostgresAuditRecordStore(pool), plugins,
     new PostgresJobRunSchedulePersistence(pool), await loadNotifier(process.env.ANBARIC_NOTIFIER_MODULE),
@@ -131,4 +133,7 @@ const internal = await server.listenInternal(internalPort);
 const dispatcher = new Dispatcher(queue, registry, Number(process.env.ANBARIC_DISPATCH_INTERVAL_MS ?? 1000));
 dispatcher.start();
 
+// Jobs whose long-running step stopped saying it was alive are marked, not touched.
+new StallSweep(jobs).start();
+
 console.log(`anbaric-cloud-hosting listening on port ${port}, internal entry point on ${internal}`);