anbaric-cloud-hosting 1.23.2 → 1.25.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anbaric-cloud-hosting",
-  "version": "1.23.2",
+  "version": "1.25.0",
   "description": "Anbaric Cloud hosting service: Postgres-backed job persistence and queuing exposed over an HTTP API",
   "license": "MIT",
   "author": "chris@anbaric.ai",
@@ -18,18 +18,18 @@
     "@aws-sdk/client-s3": "^3.1110.0",
     "@aws-sdk/client-secrets-manager": "^3.700.0",
     "@aws-sdk/client-servicediscovery": "^3.1110.0",
-    "anbaric-data-store": "^1.23.2",
-    "anbaric-plugins": "^1.23.2",
-    "anbaric-tsapi": "^1.23.2",
-    "anbaric-web": "^1.23.2",
+    "anbaric-data-store": "^1.25.0",
+    "anbaric-plugins": "^1.25.0",
+    "anbaric-tsapi": "^1.25.0",
+    "anbaric-web": "^1.25.0",
     "esbuild": "^0.28.2",
     "pg": "^8.16.0"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",
     "@types/pg": "^8.15.0",
-    "anbaric-impl-cloud": "^1.23.2",
-    "anbaric-state-machine": "^1.23.2",
+    "anbaric-impl-cloud": "^1.25.0",
+    "anbaric-state-machine": "^1.25.0",
     "tsx": "^4.20.0",
     "typescript": "^7.0.2"
   },

package/src/app-admin/launch.ts

@@ -9,6 +9,15 @@ import {AdminServer} from "./AdminServer";
    and its admin port stops answering `ping`. */
 const TSX = process.env.ANBARIC_TSX_BIN ?? "/anbaric/node_modules/.bin/tsx";
 
+/* A host can name a module to load before the app's own entry point, which is
+   how it registers implementations an app picks up through the factories -
+   anything the app should be able to use without knowing it is there. It runs
+   in the app's process, so registering from the launcher would be too early. */
+const preloadArgs = () => {
+    const preload = process.env.ANBARIC_APP_PRELOAD;
+    return preload ? ["--import", preload] : [];
+};
+
 const main = async () => {
     const entryPoint = process.argv[2];
     if (!entryPoint) throw new Error("Expected the app entry point as the first argument");
@@ -17,7 +26,7 @@ const main = async () => {
     const port = await admin.listen();
     console.log(`[anbaric-admin] listening on ${port}`);
 
-    const app = spawn(TSX, [entryPoint], { stdio: "inherit" });
+    const app = spawn(TSX, [...preloadArgs(), entryPoint], { stdio: "inherit" });
 
     const shutdown = (signal : NodeJS.Signals) => app.kill(signal);
     process.on("SIGTERM", shutdown);

package/src/data-store/PostgresJobPersistence.ts

@@ -49,9 +49,13 @@ class PostgresJobPersistence extends JobPersistence {
             await client.query(
                 `INSERT INTO jobs (id, state, properties, workflow_id, app_id, started_at, started_by, last_updated, killed, status, waiting_for)
                  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+                 -- killed is deliberately not updated here. Only kill() sets it,
+                 -- and a save carries whatever the job looked like when it was
+                 -- read: a pass that began before a kill would otherwise write
+                 -- killed=false straight back and bring the job back to life.
                  ON CONFLICT (id) DO UPDATE SET state = EXCLUDED.state, properties = EXCLUDED.properties,
                      workflow_id = EXCLUDED.workflow_id, app_id = EXCLUDED.app_id, last_updated = EXCLUDED.last_updated,
-                     killed = EXCLUDED.killed, status = EXCLUDED.status, waiting_for = EXCLUDED.waiting_for`,
+                     status = EXCLUDED.status, waiting_for = EXCLUDED.waiting_for`,
                 [job.id, job.state, Object.fromEntries(job.properties), job.workflowId, job.appId || null,
                     job.startedAt, job.startedBy, job.lastUpdated, job.killed, job.status, job.waitingFor ?? null],
             );

package/src/data-store/PostgresJobRunSchedulePersistence.ts

@@ -0,0 +1,56 @@
+import {JobRunSchedulePersistence, ScheduledRun} from "anbaric-tsapi";
+import {Pool} from "pg";
+
+const CLAIM_BATCH_SIZE = 100;
+
+/* The shared store behind the job run scheduler. Claiming is a single
+   UPDATE ... RETURNING over rows selected FOR UPDATE SKIP LOCKED, so several
+   schedulers can point at one database and each due run is handed to exactly
+   one of them - no run started twice, none lost to a scheduler that dies
+   mid-claim. */
+class PostgresJobRunSchedulePersistence implements JobRunSchedulePersistence {
+
+    constructor(private pool : Pool) {}
+
+    async highWaterMark(appId : string, workflowId : string) : Promise<Date | undefined> {
+        const result = await this.pool.query(
+            "SELECT max(run_at) AS latest FROM job_run_schedule WHERE app_id = $1 AND workflow_id = $2",
+            [appId, workflowId],
+        );
+        return result.rows[0]?.latest ?? undefined;
+    }
+
+    async plan(runs : Array<ScheduledRun>) : Promise<void> {
+        if (runs.length === 0) return;
+        await this.pool.query(
+            `INSERT INTO job_run_schedule (app_id, workflow_id, run_at)
+             SELECT * FROM unnest($1::text[], $2::text[], $3::timestamptz[])
+             ON CONFLICT (app_id, workflow_id, run_at) DO NOTHING`,
+            [runs.map(run => run.appId), runs.map(run => run.workflowId), runs.map(run => run.runAt)],
+        );
+    }
+
+    async claimDue(at : Date) : Promise<Array<ScheduledRun>> {
+        const result = await this.pool.query(
+            `UPDATE job_run_schedule SET claimed_at = now()
+             WHERE id IN (
+                 SELECT id FROM job_run_schedule
+                 WHERE claimed_at IS NULL AND run_at <= $1
+                 ORDER BY run_at
+                 LIMIT $2
+                 FOR UPDATE SKIP LOCKED
+             )
+             RETURNING app_id, workflow_id, run_at`,
+            [at, CLAIM_BATCH_SIZE],
+        );
+
+        return result.rows.map(row => ({
+            appId: row.app_id ?? "",
+            workflowId: row.workflow_id,
+            runAt: row.run_at,
+        }));
+    }
+
+}
+
+export { PostgresJobRunSchedulePersistence }

package/src/data-store/Schema.ts

@@ -78,6 +78,22 @@ const ensureSchema = async (pool : Pool) : Promise<void> => {
         SET app_id = split_part(workflow_id, '/', 1),
             workflow_id = substring(workflow_id from position('/' in workflow_id) + 1)
         WHERE app_id IS NULL AND workflow_id LIKE '%/%'`);
+    // Runs a scheduler has planned. Unique on (app_id, workflow_id, run_at) so
+    // two schedulers planning the same window converge on one row rather than
+    // starting a machine twice.
+    await pool.query(`
+        CREATE TABLE IF NOT EXISTS job_run_schedule (
+            id          BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+            app_id      TEXT NOT NULL DEFAULT '',
+            workflow_id TEXT NOT NULL,
+            run_at      TIMESTAMPTZ NOT NULL,
+            claimed_at  TIMESTAMPTZ
+        )
+    `);
+    await pool.query(`CREATE UNIQUE INDEX IF NOT EXISTS job_run_schedule_run
+        ON job_run_schedule (app_id, workflow_id, run_at)`);
+    await pool.query(`CREATE INDEX IF NOT EXISTS job_run_schedule_due
+        ON job_run_schedule (run_at) WHERE claimed_at IS NULL`);
     await pool.query("CREATE SCHEMA IF NOT EXISTS anbaric_system");
     await pool.query(`
         CREATE TABLE IF NOT EXISTS anbaric_system.cli_keys (

package/src/hosting/HostingServer.ts

@@ -1,4 +1,4 @@
-import {JobPersistence, JsonStore, SecretStore} from "anbaric-tsapi";
+import {JobPersistence, JobRunSchedulePersistence, JsonStore, SecretStore} from "anbaric-tsapi";
 import {BuildLayer} from "../app-management/BuildLayer";
 import {AuditRecordStore} from "../auditing/AuditRecordStore";
 import {Authenticator} from "../auth/Authenticator";
@@ -18,6 +18,7 @@ import {ConsumersHandler} from "./handlers/ConsumersHandler";
 import {DocumentsHandler} from "./handlers/DocumentsHandler";
 import {JobsHandler} from "./handlers/JobsHandler";
 import {FaviconHandler} from "./handlers/FaviconHandler";
+import {JobRunSchedulesHandler} from "./handlers/JobRunSchedulesHandler";
 import {LogoutHandler} from "./handlers/LogoutHandler";
 import {PagesHandler} from "./handlers/PagesHandler";
 import {PingHandler} from "./handlers/PingHandler";
@@ -54,7 +55,8 @@ class HostingServer {
                 tokenAuthenticator? : TokenAuthenticator,
                 tenant? : string,
                 auditRecords? : AuditRecordStore,
-                plugins : Array<LoadedPlugin> = []) {
+                plugins : Array<LoadedPlugin> = [],
+                jobRunSchedules? : JobRunSchedulePersistence) {
         const pages = new PagesHandler();
         const ping = new PingHandler(tenant);
         const jobs = new JobsHandler(persistence);
@@ -81,6 +83,7 @@ class HostingServer {
         if (documents) publicRouter.registerApi("documents", documents);
         if (secrets) publicRouter.registerApi("secrets", secrets);
         if (audits) publicRouter.registerApi("audits", audits);
+        if (jobRunSchedules) publicRouter.registerApi("job-run-schedules", new JobRunSchedulesHandler(jobRunSchedules));
         if (cliAuthorizer) {
             publicRouter.register("authorize-cli", new AuthorizeCliHandler(cliAuthorizer, pages, tenant));
             publicRouter.registerApi("keys", new KeysHandler(cliAuthorizer));
@@ -103,6 +106,7 @@ class HostingServer {
         if (documents) internalRouter.registerApi("documents", documents);
         if (secrets) internalRouter.registerApi("secrets", secrets);
         if (audits) internalRouter.registerApi("audits", audits);
+        if (jobRunSchedules) internalRouter.registerApi("job-run-schedules", new JobRunSchedulesHandler(jobRunSchedules));
 
         // The favicon is requested by the browser before anyone has signed in,
         // and by deployed apps' pages, so gating it behind a session would send

package/src/hosting/handlers/JobRunSchedulesHandler.ts

@@ -0,0 +1,46 @@
+import {JobRunSchedulePersistence} from "anbaric-tsapi";
+import {Request} from "../Request";
+import {RequestHandler} from "../RequestHandler";
+
+class JobRunSchedulesHandler implements RequestHandler {
+
+    constructor(private persistence : JobRunSchedulePersistence) {}
+
+    async handle(request : Request) : Promise<void> {
+        if (request.subresource) return request.notFound();
+
+        if (request.method === "GET" && request.id === "high-water-mark") {
+            const appId = request.url.searchParams.get("appId") ?? "";
+            const workflowId = request.url.searchParams.get("workflowId");
+            if (!workflowId) return request.reply(400, { error: "A workflowId is required" });
+
+            const latest = await this.persistence.highWaterMark(appId, workflowId);
+            return request.reply(200, { highWaterMark: latest?.toISOString() ?? null });
+        }
+
+        if (request.method !== "POST" || !request.id) return request.notFound();
+
+        switch (request.id) {
+            case "plan": {
+                const { runs } = await request.body();
+                await this.persistence.plan((runs ?? []).map((run : any) => ({
+                    appId: run.appId ?? "",
+                    workflowId: run.workflowId,
+                    runAt: new Date(run.runAt),
+                })));
+                return request.reply(204);
+            }
+            case "claim-due": {
+                const { at } = await request.body();
+                const claimed = await this.persistence.claimDue(at ? new Date(at) : new Date());
+                return request.reply(200, {
+                    runs: claimed.map(run => ({ ...run, runAt: run.runAt.toISOString() })),
+                });
+            }
+        }
+        request.notFound();
+    }
+
+}
+
+export { JobRunSchedulesHandler }