anbaric-cloud-hosting 1.8.1 → 1.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anbaric-cloud-hosting",
-  "version": "1.8.1",
+  "version": "1.10.0",
   "description": "Anbaric Cloud hosting service: Postgres-backed job persistence and queuing exposed over an HTTP API",
   "license": "MIT",
   "author": "chris@anbaric.ai",
@@ -12,22 +12,23 @@
     "test": "vitest run"
   },
   "dependencies": {
+    "@aws-sdk/client-cloudwatch-logs": "^3.1110.0",
     "@aws-sdk/client-codebuild": "^3.1110.0",
     "@aws-sdk/client-ecs": "^3.1110.0",
     "@aws-sdk/client-s3": "^3.1110.0",
     "@aws-sdk/client-secrets-manager": "^3.700.0",
     "@aws-sdk/client-servicediscovery": "^3.1110.0",
-    "anbaric-data-store": "^1.8.1",
-    "anbaric-plugins": "^1.8.1",
-    "anbaric-tsapi": "^1.8.1",
+    "anbaric-data-store": "^1.10.0",
+    "anbaric-plugins": "^1.10.0",
+    "anbaric-tsapi": "^1.10.0",
     "esbuild": "^0.28.2",
     "pg": "^8.16.0"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",
     "@types/pg": "^8.15.0",
-    "anbaric-impl-cloud": "^1.8.1",
-    "anbaric-state-machine": "^1.8.1",
+    "anbaric-impl-cloud": "^1.10.0",
+    "anbaric-state-machine": "^1.10.0",
     "tsx": "^4.20.0",
     "typescript": "^7.0.2"
   },

package/src/app-admin/AdminServer.ts

@@ -0,0 +1,64 @@
+import {createServer, Server, Socket} from "node:net";
+
+type AdminCommand = (argument : string) => string | Promise<string>;
+
+/* The socket-level admin server baked into every app image. It listens on the
+   admin port and answers newline-terminated commands, writing one line back per
+   command. `ping` (answering `ok`) is built in and is what the platform probes
+   and reports as the app being up; further commands are registered with
+   `handle`. Kept deliberately transport-light (raw TCP, no HTTP) so an app that
+   serves no web traffic still has a liveness surface. */
+class AdminServer {
+
+    private commands = new Map<string, AdminCommand>();
+    private server? : Server;
+
+    constructor(private port : number = Number(process.env.ANBARIC_ADMIN_PORT ?? 8791)) {
+        this.handle("ping", () => "ok");
+    }
+
+    handle(command : string, run : AdminCommand) : this {
+        this.commands.set(command, run);
+        return this;
+    }
+
+    listen() : Promise<number> {
+        this.server = createServer(socket => this.serve(socket));
+        return new Promise(resolve => this.server!.listen(this.port, () =>
+            resolve((this.server!.address() as { port : number }).port)));
+    }
+
+    async close() : Promise<void> {
+        await new Promise<void>(resolve => this.server ? this.server.close(() => resolve()) : resolve());
+    }
+
+    private serve(socket : Socket) : void {
+        let buffer = "";
+        socket.setEncoding("utf8");
+        socket.on("data", async chunk => {
+            buffer += chunk;
+            let newline : number;
+            while ((newline = buffer.indexOf("\n")) >= 0) {
+                const line = buffer.slice(0, newline).trim();
+                buffer = buffer.slice(newline + 1);
+                if (line) await this.respond(socket, line);
+            }
+        });
+        socket.on("error", () => socket.destroy());
+    }
+
+    private async respond(socket : Socket, line : string) : Promise<void> {
+        const [command, ...rest] = line.split(" ");
+        const run = this.commands.get(command);
+        if (!run) return void socket.write(`error unknown command "${command}"\n`);
+        try {
+            socket.write(`${await run(rest.join(" "))}\n`);
+        } catch (error) {
+            socket.write(`error ${error instanceof Error ? error.message : error}\n`);
+        }
+    }
+
+}
+
+export { AdminServer };
+export type { AdminCommand };

package/src/app-admin/adminPing.ts

@@ -0,0 +1,21 @@
+import {connect} from "node:net";
+
+/* Opens an app's admin port, sends `ping`, and resolves true when it answers
+   `ok` within the timeout - false on any timeout, refusal or other error. Used
+   both by the liveness probe and by on-demand status reporting. */
+const adminPing = (host : string, port : number, timeoutMs : number = 1000) : Promise<boolean> =>
+    new Promise(resolve => {
+        const socket = connect({ host, port });
+        let answered = "";
+        const done = (result : boolean) => { socket.destroy(); resolve(result); };
+        socket.setTimeout(timeoutMs, () => done(false));
+        socket.setEncoding("utf8");
+        socket.on("connect", () => socket.write("ping\n"));
+        socket.on("data", chunk => {
+            answered += chunk;
+            if (answered.includes("\n")) done(answered.trim().startsWith("ok"));
+        });
+        socket.on("error", () => done(false));
+    });
+
+export { adminPing };

package/src/app-admin/launch.ts

@@ -0,0 +1,38 @@
+import {spawn} from "node:child_process";
+import {AdminServer} from "./AdminServer";
+
+/* The entry point every deployed app image runs (its Dockerfile CMD). It starts
+   the built-in admin server - a separate concern from the app itself: liveness
+   now, more later - and then runs the app as its own child process. The
+   container's lifecycle follows the app's: when the app exits, so does this
+   launcher (and the admin port with it), so a failed app is a failed container
+   and its admin port stops answering `ping`. */
+const TSX = process.env.ANBARIC_TSX_BIN ?? "/anbaric/node_modules/.bin/tsx";
+
+const main = async () => {
+    const entryPoint = process.argv[2];
+    if (!entryPoint) throw new Error("Expected the app entry point as the first argument");
+
+    const admin = new AdminServer();
+    const port = await admin.listen();
+    console.log(`[anbaric-admin] listening on ${port}`);
+
+    const app = spawn(TSX, [entryPoint], { stdio: "inherit" });
+
+    const shutdown = (signal : NodeJS.Signals) => app.kill(signal);
+    process.on("SIGTERM", shutdown);
+    process.on("SIGINT", shutdown);
+
+    app.on("error", error => {
+        console.error(`[anbaric-admin] failed to start the app: ${error.message}`);
+        admin.close().finally(() => process.exit(1));
+    });
+    app.on("exit", (code, signal) => {
+        admin.close().finally(() => process.exit(code ?? (signal ? 1 : 0)));
+    });
+};
+
+main().catch(error => {
+    console.error(`[anbaric-admin] ${error instanceof Error ? error.message : error}`);
+    process.exit(1);
+});

package/src/app-management/BaseBuildLayer.ts

@@ -1,6 +1,7 @@
 import {ChildProcess, spawn} from "node:child_process";
 import {mkdir, readFile, rm, symlink, writeFile} from "node:fs/promises";
 import {join} from "node:path";
+import {adminPing} from "../app-admin/adminPing";
 import {BuildLayer, DeploymentStatus, DeploymentSummary} from "./BuildLayer";
 
 type Deployment = {
@@ -8,6 +9,7 @@ type Deployment = {
     status : DeploymentStatus,
     appPort : number,
     appHost : string,
+    adminPort : number,
     consumerPort : number,
     log : Array<string>,
     process? : ChildProcess,
@@ -19,15 +21,11 @@ type Probe = (host : string, port : number) => Promise<boolean>;
 const LOG_LIMIT = 200;
 const LIVENESS_TIMEOUT_MS = 30_000;
 const LIVENESS_PROBE_INTERVAL_MS = 250;
+const APP_ADMIN_PORT = 8791;
 
-const httpProbe : Probe = async (host, port) => {
-    try {
-        await fetch(`http://${host}:${port}/`, { signal: AbortSignal.timeout(LIVENESS_PROBE_INTERVAL_MS) });
-        return true;
-    } catch {
-        return false;
-    }
-};
+/* Liveness is the built-in admin server answering `ping` on the admin port, so
+   an app that serves no HTTP still passes. */
+const adminProbe : Probe = (host, port) => adminPing(host, port, LIVENESS_PROBE_INTERVAL_MS);
 
 abstract class BaseBuildLayer implements BuildLayer {
 
@@ -35,7 +33,7 @@ abstract class BaseBuildLayer implements BuildLayer {
     private nextAppIndex = 0;
 
     constructor(protected appsDir : string, private consumerPortBase : number = 8800,
-                private probe : Probe = httpProbe,
+                private probe : Probe = adminProbe,
                 private livenessTimeoutMs : number = LIVENESS_TIMEOUT_MS) {}
 
     deploy(appName : string, appPort : number, tarball : Buffer) : DeploymentSummary {
@@ -46,6 +44,7 @@ abstract class BaseBuildLayer implements BuildLayer {
             status: "building",
             appPort,
             appHost: this.appHostFor(appName),
+            adminPort: APP_ADMIN_PORT,
             consumerPort: existing?.consumerPort ?? this.consumerPortBase + this.nextAppIndex++,
             log: [],
             replaces: existing,
@@ -69,6 +68,30 @@ abstract class BaseBuildLayer implements BuildLayer {
         return Array.from(this.deployments.values(), deployment => this.summarize(deployment));
     }
 
+    async ping(appName : string) : Promise<boolean> {
+        const deployment = this.deployments.get(appName);
+        if (!deployment || deployment.status !== "running") return false;
+        return adminPing(deployment.appHost, deployment.adminPort, LIVENESS_PROBE_INTERVAL_MS);
+    }
+
+    async *logs(appName : string, signal : AbortSignal) : AsyncGenerator<string> {
+        const deployment = this.deployments.get(appName);
+        if (!deployment) throw new Error(`No app named "${appName}"`);
+        yield* this.streamLogs(deployment, signal);
+    }
+
+    async teardown(appName : string) : Promise<boolean> {
+        const deployment = this.deployments.get(appName);
+        if (!deployment) return false;
+
+        // stop() runs while the deployment is still the mapped one, so the
+        // Fargate guard lets it delete the service; then drop it from the map.
+        deployment.status = "stopped";
+        await this.stop(deployment);
+        this.deployments.delete(appName);
+        return true;
+    }
+
     async cleanUp() : Promise<void> {
         await Promise.all(Array.from(this.deployments.values(), deployment => {
             deployment.status = "stopped";
@@ -79,6 +102,7 @@ abstract class BaseBuildLayer implements BuildLayer {
     protected abstract appHostFor(appName : string) : string;
     protected abstract start(deployment : Deployment, appDir : string, entryPoint : string) : Promise<void>;
     protected abstract stop(deployment : Deployment) : Promise<void>;
+    protected abstract streamLogs(deployment : Deployment, signal : AbortSignal) : AsyncIterable<string>;
 
     private async buildAndStart(deployment : Deployment, tarball : Buffer) : Promise<void> {
         const appDir = join(this.appsDir, deployment.appName);
@@ -110,9 +134,9 @@ abstract class BaseBuildLayer implements BuildLayer {
 
         while (Date.now() < deadline) {
             if (deployment.status !== "building") return;
-            if (await this.probe(deployment.appHost, deployment.appPort)) {
+            if (await this.probe(deployment.appHost, deployment.adminPort)) {
                 deployment.status = "running";
-                this.log(deployment, `app is live at ${deployment.appHost}:${deployment.appPort}`);
+                this.log(deployment, `app is live (admin port ${deployment.adminPort})`);
                 return;
             }
             await new Promise(resolve => setTimeout(resolve, LIVENESS_PROBE_INTERVAL_MS));
@@ -120,7 +144,7 @@ abstract class BaseBuildLayer implements BuildLayer {
 
         await this.stop(deployment);
         deployment.status = "failed";
-        this.log(deployment, `app did not respond at ${deployment.appHost}:${deployment.appPort} within ${this.livenessTimeoutMs / 1000}s`);
+        this.log(deployment, `app admin port ${deployment.adminPort} did not answer within ${this.livenessTimeoutMs / 1000}s`);
     }
 
     private async linkWorkspacePackages(appDir : string, manifest : { dependencies? : Record<string, string> }) : Promise<void> {
@@ -174,5 +198,5 @@ abstract class BaseBuildLayer implements BuildLayer {
 
 }
 
-export { BaseBuildLayer, httpProbe };
+export { BaseBuildLayer, adminProbe };
 export type { Deployment, Probe };

package/src/app-management/BuildLayer.ts

@@ -12,6 +12,9 @@ interface BuildLayer {
     deploy(appName : string, appPort : number, tarball : Buffer) : DeploymentSummary;
     status(appName : string) : (DeploymentSummary & { log : Array<string> }) | undefined;
     list() : Array<DeploymentSummary>;
+    ping(appName : string) : Promise<boolean>;
+    logs(appName : string, signal : AbortSignal) : AsyncIterable<string>;
+    teardown(appName : string) : Promise<boolean>;
     cleanUp() : Promise<void>;
 
 }

package/src/app-management/DockerBuildLayer.ts

@@ -2,11 +2,19 @@ import {spawn} from "node:child_process";
 import {writeFile} from "node:fs/promises";
 import {join} from "node:path";
 import {BaseBuildLayer, Deployment, Probe} from "./BaseBuildLayer";
+import {childLines} from "./childLines";
+
+type LogStreamer = (container : string, signal : AbortSignal) => AsyncIterable<string>;
+
+const dockerLogStreamer : LogStreamer = (container, signal) =>
+    childLines(spawn("docker", ["logs", "--follow", "--tail", "50", container]), signal);
 
 type DockerBuildLayerOptions = {
     baseImage : string,
     network : string,
     platformUrl : string,
+    sqlDatabaseUrl? : string,
+    sqlSchema? : string,
 };
 
 type CommandRunner = (command : string, args : Array<string>, onOutput : (line : string) => void) => Promise<void>;
@@ -33,14 +41,15 @@ const dockerfileFor = (baseImage : string, entryPoint : string) : string => `FRO
 COPY . /anbaric-app
 WORKDIR /anbaric-app
 RUN npm install --omit=dev --no-audit --no-fund
-CMD ["/anbaric/node_modules/.bin/tsx", "${entryPoint}"]
+CMD ["/anbaric/node_modules/.bin/tsx", "/anbaric/node_modules/anbaric-cloud-hosting/src/app-admin/launch.ts", "${entryPoint}"]
 `;
 
 class DockerBuildLayer extends BaseBuildLayer {
 
     constructor(appsDir : string, private options : DockerBuildLayerOptions,
                 consumerPortBase? : number,
-                private runCommand : CommandRunner = spawnRunner, probe? : Probe) {
+                private runCommand : CommandRunner = spawnRunner, probe? : Probe,
+                private logStreamer : LogStreamer = dockerLogStreamer) {
         super(appsDir, consumerPortBase, probe);
     }
 
@@ -48,6 +57,10 @@ class DockerBuildLayer extends BaseBuildLayer {
         return `anbaric-app-${appName}`;
     }
 
+    protected streamLogs(deployment : Deployment, signal : AbortSignal) : AsyncIterable<string> {
+        return this.logStreamer(this.appHostFor(deployment.appName), signal);
+    }
+
     protected async start(deployment : Deployment, appDir : string, entryPoint : string) : Promise<void> {
         const image = this.appHostFor(deployment.appName);
         const container = this.appHostFor(deployment.appName);
@@ -58,10 +71,17 @@ class DockerBuildLayer extends BaseBuildLayer {
 
         await this.removeContainer(deployment, container);
 
+        const sqlEnv = this.options.sqlDatabaseUrl ? [
+            "--env", "ANBARIC_SQL_STORE_TYPE=cloud",
+            "--env", `ANBARIC_SQL_DATABASE_URL=${this.options.sqlDatabaseUrl}`,
+            "--env", `ANBARIC_SQL_SCHEMA=${this.options.sqlSchema ?? "anbaric_app_data"}`,
+        ] : [];
+
         this.log(deployment, `starting container ${container}`);
         await this.docker(deployment, ["run", "--detach", "--name", container,
             "--network", this.options.network,
             "--env", `PORT=${deployment.appPort}`,
+            "--env", `ANBARIC_ADMIN_PORT=${deployment.adminPort}`,
             "--env", `ANBARIC_CLOUD_URL=${this.options.platformUrl}`,
             "--env", "ANBARIC_JOB_PERSISTENCE_TYPE=cloud",
             "--env", "ANBARIC_QUEUE_TYPE=cloud",
@@ -70,6 +90,7 @@ class DockerBuildLayer extends BaseBuildLayer {
             "--env", "ANBARIC_AUDITOR_TYPE=cloud",
             "--env", `ANBARIC_CONSUMER_PORT=${deployment.consumerPort}`,
             "--env", `ANBARIC_CONSUMER_URL=http://${container}:${deployment.consumerPort}`,
+            ...sqlEnv,
             image]);
 
         void this.docker(deployment, ["logs", "--follow", container]).catch(() => {});
@@ -90,4 +111,4 @@ class DockerBuildLayer extends BaseBuildLayer {
 }
 
 export { DockerBuildLayer, dockerfileFor };
-export type { CommandRunner, DockerBuildLayerOptions };
+export type { CommandRunner, DockerBuildLayerOptions, LogStreamer };

package/src/app-management/FargateBuildLayer.ts

@@ -6,6 +6,7 @@ import {ECSClient, CreateServiceCommand, DeleteServiceCommand, DescribeServicesC
 import {S3Client, PutObjectCommand} from "@aws-sdk/client-s3";
 import {ServiceDiscoveryClient, CreateServiceCommand as CreateDiscoveryServiceCommand,
     ListServicesCommand} from "@aws-sdk/client-servicediscovery";
+import {CloudWatchLogsClient, StartLiveTailCommand} from "@aws-sdk/client-cloudwatch-logs";
 import {BaseBuildLayer, Deployment, Probe} from "./BaseBuildLayer";
 import {dockerfileFor} from "./DockerBuildLayer";
 
@@ -22,9 +23,12 @@ type FargateBuildLayerOptions = {
     baseImage : string,
     appExecutionRoleArn : string,
     appsLogGroup : string,
+    appsLogGroupArn : string,
     platformUrl : string,
     servicesUrl? : string,
     servicesApiKey? : string,
+    sqlDatabaseUrlSecretArn? : string,
+    sqlSchema? : string,
 };
 
 type AwsClients = {
@@ -32,6 +36,7 @@ type AwsClients = {
     codeBuild : { send(command : any) : Promise<any> },
     ecs : { send(command : any) : Promise<any> },
     serviceDiscovery : { send(command : any) : Promise<any> },
+    cloudWatchLogs : { send(command : any, options? : any) : Promise<any> },
 };
 
 const defaultClients = (region : string) : AwsClients => ({
@@ -39,6 +44,7 @@ const defaultClients = (region : string) : AwsClients => ({
     codeBuild: new CodeBuildClient({ region }),
     ecs: new ECSClient({ region }),
     serviceDiscovery: new ServiceDiscoveryClient({ region }),
+    cloudWatchLogs: new CloudWatchLogsClient({ region }),
 });
 
 const BUILD_TIMEOUT_MS = 900_000;
@@ -60,6 +66,23 @@ class FargateBuildLayer extends BaseBuildLayer {
         return `${appName}.${this.options.namespaceName}`;
     }
 
+    protected async *streamLogs(deployment : Deployment, signal : AbortSignal) : AsyncIterable<string> {
+        const response = await this.aws.cloudWatchLogs.send(new StartLiveTailCommand({
+            logGroupIdentifiers: [this.options.appsLogGroupArn],
+            logStreamNamePrefixes: [deployment.appName],
+        }), { abortSignal: signal });
+
+        try {
+            for await (const event of response.responseStream) {
+                for (const result of event.sessionUpdate?.sessionResults ?? []) {
+                    if (result.message !== undefined) yield result.message;
+                }
+            }
+        } catch (error) {
+            if (!signal.aborted) throw error;
+        }
+    }
+
     protected async start(deployment : Deployment, appDir : string, entryPoint : string) : Promise<void> {
         const imageUri = `${this.options.appsRepositoryUrl}:${deployment.appName}`;
 
@@ -141,10 +164,12 @@ class FargateBuildLayer extends BaseBuildLayer {
                 essential: true,
                 portMappings: [
                     { containerPort: deployment.appPort, protocol: "tcp" },
+                    { containerPort: deployment.adminPort, protocol: "tcp" },
                     { containerPort: deployment.consumerPort, protocol: "tcp" },
                 ],
                 environment: [
                     { name: "PORT", value: String(deployment.appPort) },
+                    { name: "ANBARIC_ADMIN_PORT", value: String(deployment.adminPort) },
                     { name: "ANBARIC_CLOUD_URL", value: this.options.platformUrl },
                     { name: "ANBARIC_JOB_PERSISTENCE_TYPE", value: "cloud" },
                     { name: "ANBARIC_QUEUE_TYPE", value: "cloud" },
@@ -155,7 +180,14 @@ class FargateBuildLayer extends BaseBuildLayer {
                     { name: "ANBARIC_CONSUMER_URL", value: `http://${deployment.appHost}:${deployment.consumerPort}` },
                     ...(this.options.servicesUrl ? [{ name: "ANBARIC_SERVICES_URL", value: this.options.servicesUrl }] : []),
                     ...(this.options.servicesApiKey ? [{ name: "ANBARIC_SERVICES_API_KEY", value: this.options.servicesApiKey }] : []),
+                    ...(this.options.sqlDatabaseUrlSecretArn ? [
+                        { name: "ANBARIC_SQL_STORE_TYPE", value: "cloud" },
+                        { name: "ANBARIC_SQL_SCHEMA", value: this.options.sqlSchema ?? "anbaric_app_data" },
+                    ] : []),
                 ],
+                secrets: this.options.sqlDatabaseUrlSecretArn
+                    ? [{ name: "ANBARIC_SQL_DATABASE_URL", valueFrom: this.options.sqlDatabaseUrlSecretArn }]
+                    : undefined,
                 logConfiguration: {
                     logDriver: "awslogs",
                     options: {

package/src/app-management/childLines.ts

@@ -0,0 +1,42 @@
+import {ChildProcessWithoutNullStreams} from "node:child_process";
+
+/* Yields the child's stdout and stderr merged, one line at a time, until the
+   process ends or the signal aborts (which kills it). Backs the docker-logs
+   follow stream. */
+async function* childLines(child : ChildProcessWithoutNullStreams, signal : AbortSignal) : AsyncGenerator<string> {
+    const queue : Array<string> = [];
+    let wake : (() => void) | undefined;
+    let ended = false;
+
+    const push = (line : string) => { queue.push(line); wake?.(); wake = undefined; };
+    const finish = () => { ended = true; wake?.(); wake = undefined; };
+
+    for (const stream of [child.stdout, child.stderr]) {
+        let buffer = "";
+        stream.setEncoding("utf8");
+        stream.on("data", chunk => {
+            buffer += chunk;
+            let newline : number;
+            while ((newline = buffer.indexOf("\n")) >= 0) {
+                push(buffer.slice(0, newline));
+                buffer = buffer.slice(newline + 1);
+            }
+        });
+    }
+
+    child.on("close", finish);
+    child.on("error", finish);
+    const abort = () => { child.kill(); finish(); };
+    signal.addEventListener("abort", abort);
+
+    try {
+        while (!ended || queue.length > 0) {
+            if (queue.length > 0) yield queue.shift()!;
+            else await new Promise<void>(resolve => { wake = resolve; });
+        }
+    } finally {
+        signal.removeEventListener("abort", abort);
+    }
+}
+
+export { childLines };

package/src/auditing/AuditRecordStore.ts

@@ -1,10 +1,10 @@
-import {AuditInteraction, AuditRecord} from "anbaric-tsapi";
+import {AuditRecord} from "anbaric-tsapi";
 
 type AuditFilter = {
     resourceType? : string,
     resourceId? : string,
     actorId? : string,
-    interaction? : AuditInteraction,
+    interaction? : string,
     search? : string,
     pageSize? : number,
     page? : number,

package/src/data-store/PostgresAuditRecordStore.ts

@@ -1,10 +1,10 @@
-import {AuditInteraction, AuditRecord} from "anbaric-tsapi";
+import {AuditRecord} from "anbaric-tsapi";
 import {Pool} from "pg";
 import {AuditFilter, AuditRecordStore} from "../auditing/AuditRecordStore";
 
-const DEFAULT_WRITE_MASK : Array<AuditInteraction> = [
-    AuditInteraction.CREATE, AuditInteraction.UPDATE_PROPERTIES, AuditInteraction.CHANGE_STATE, AuditInteraction.DELETE,
-];
+/* The writes worth keeping in the durable trail; reads (READ, LIST, QUERY) are
+   dropped by default. Interactions are plain strings chosen by each store. */
+const DEFAULT_WRITE_MASK : Array<string> = ["CREATE", "SAVE", "UPDATE_PROPERTIES", "CHANGE_STATE", "DELETE", "EXECUTE"];
 
 /* Persists audit records, subject to a write mask: only the interactions in
    the mask are stored, so high-volume reads can be audited at the call site
@@ -12,16 +12,16 @@ const DEFAULT_WRITE_MASK : Array<AuditInteraction> = [
    overridable via ANBARIC_AUDIT_INTERACTIONS. */
 class PostgresAuditRecordStore implements AuditRecordStore {
 
-    private writeMask : Set<AuditInteraction>;
+    private writeMask : Set<string>;
 
-    constructor(private pool : Pool, writeMask : Set<AuditInteraction> = PostgresAuditRecordStore.maskFromEnvironment()) {
+    constructor(private pool : Pool, writeMask : Set<string> = PostgresAuditRecordStore.maskFromEnvironment()) {
         this.writeMask = writeMask;
     }
 
-    static maskFromEnvironment() : Set<AuditInteraction> {
+    static maskFromEnvironment() : Set<string> {
         const configured = process.env.ANBARIC_AUDIT_INTERACTIONS;
         if (!configured) return new Set(DEFAULT_WRITE_MASK);
-        return new Set(configured.split(",").map(entry => entry.trim()).filter(Boolean) as Array<AuditInteraction>);
+        return new Set(configured.split(",").map(entry => entry.trim()).filter(Boolean) );
     }
 
     async save(record : AuditRecord) : Promise<void> {
@@ -30,7 +30,7 @@ class PostgresAuditRecordStore implements AuditRecordStore {
         await this.pool.query(
             `INSERT INTO anbaric_system.audit_records
                 (resource_type, resource_id, actor_id, actor_type, interaction, description, details)
-             VALUES ($1, $2, $3, $4, $5::anbaric_system.audit_interaction[], $6, $7)`,
+             VALUES ($1, $2, $3, $4, $5::text[], $6, $7)`,
             [record.resourceType, record.resourceId, record.actorId, record.actorType, record.interaction,
                 record.description, JSON.stringify(record.details ?? null)],
         );
@@ -54,7 +54,7 @@ class PostgresAuditRecordStore implements AuditRecordStore {
         }
         if (filter.interaction) {
             parameters.push(filter.interaction);
-            conditions.push(`$${parameters.length}::anbaric_system.audit_interaction = ANY(interaction)`);
+            conditions.push(`$${parameters.length}::text = ANY(interaction)`);
         }
         if (filter.search) {
             parameters.push(`%${filter.search}%`);

package/src/data-store/Schema.ts

@@ -66,7 +66,7 @@ const ensureSchema = async (pool : Pool) : Promise<void> => {
             resource_id   TEXT NOT NULL,
             actor_id      TEXT NOT NULL,
             actor_type    TEXT NOT NULL,
-            interaction   anbaric_system.audit_interaction[] NOT NULL,
+            interaction   TEXT[] NOT NULL,
             description   TEXT NOT NULL,
             details       JSONB,
             at            TIMESTAMPTZ NOT NULL DEFAULT now()
@@ -74,7 +74,7 @@ const ensureSchema = async (pool : Pool) : Promise<void> => {
     `);
     await pool.query("ALTER TABLE anbaric_system.audit_records ADD COLUMN IF NOT EXISTS resource_type TEXT");
     await pool.query("ALTER TABLE anbaric_system.audit_records ADD COLUMN IF NOT EXISTS resource_id TEXT");
-    await pool.query("ALTER TABLE anbaric_system.audit_records ADD COLUMN IF NOT EXISTS interaction anbaric_system.audit_interaction[]");
+    await pool.query("ALTER TABLE anbaric_system.audit_records ADD COLUMN IF NOT EXISTS interaction TEXT[]");
     await pool.query(`
         DO $$ BEGIN
             IF EXISTS (SELECT 1 FROM information_schema.columns
@@ -100,13 +100,46 @@ const ensureSchema = async (pool : Pool) : Promise<void> => {
             END IF;
         END $$
     `);
+    await pool.query(`
+        DO $$ BEGIN
+            IF EXISTS (SELECT 1 FROM information_schema.columns
+                       WHERE table_schema = 'anbaric_system' AND table_name = 'audit_records'
+                         AND column_name = 'interaction' AND udt_name = '_audit_interaction') THEN
+                ALTER TABLE anbaric_system.audit_records
+                    ALTER COLUMN interaction TYPE TEXT[] USING interaction::text[];
+            END IF;
+        END $$
+    `);
     await pool.query("UPDATE anbaric_system.audit_records SET resource_type = 'job' WHERE resource_type IS NULL");
     await pool.query("UPDATE anbaric_system.audit_records SET resource_id = 'unknown' WHERE resource_id IS NULL");
-    await pool.query("UPDATE anbaric_system.audit_records SET interaction = ARRAY['UPDATE_PROPERTIES']::anbaric_system.audit_interaction[] WHERE interaction IS NULL");
+    await pool.query("UPDATE anbaric_system.audit_records SET interaction = ARRAY['UPDATE_PROPERTIES']::text[] WHERE interaction IS NULL");
     await pool.query("ALTER TABLE anbaric_system.audit_records ALTER COLUMN resource_type SET NOT NULL");
     await pool.query("ALTER TABLE anbaric_system.audit_records ALTER COLUMN resource_id SET NOT NULL");
     await pool.query("ALTER TABLE anbaric_system.audit_records ALTER COLUMN interaction SET NOT NULL");
     await pool.query("CREATE INDEX IF NOT EXISTS audit_records_resource ON anbaric_system.audit_records (resource_type, resource_id)");
+
+    // The shared SQL store for apps: a dedicated schema plus a login role scoped
+    // to it, so deployed apps reach only anbaric_app_data - never the platform's
+    // public (jobs/documents/queue) or anbaric_system schemas.
+    const appSqlSchema = process.env.ANBARIC_SQL_SCHEMA ?? "anbaric_app_data";
+    if (!/^[a-z_][a-z0-9_]*$/i.test(appSqlSchema)) throw new Error(`Invalid ANBARIC_SQL_SCHEMA "${appSqlSchema}"`);
+    await pool.query(`CREATE SCHEMA IF NOT EXISTS ${appSqlSchema}`);
+
+    const appDbPassword = process.env.ANBARIC_APP_DB_PASSWORD;
+    if (appDbPassword) {
+        await pool.query(`DO $$ BEGIN
+            IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'anbaric_app') THEN CREATE ROLE anbaric_app LOGIN; END IF;
+        END $$`);
+        await pool.query(`ALTER ROLE anbaric_app LOGIN PASSWORD '${appDbPassword.replace(/'/g, "''")}'`);
+        await pool.query(`ALTER ROLE anbaric_app SET search_path TO ${appSqlSchema}`);
+        await pool.query(`GRANT USAGE, CREATE ON SCHEMA ${appSqlSchema} TO anbaric_app`);
+        await pool.query(`GRANT ALL ON ALL TABLES IN SCHEMA ${appSqlSchema} TO anbaric_app`);
+        await pool.query(`GRANT ALL ON ALL SEQUENCES IN SCHEMA ${appSqlSchema} TO anbaric_app`);
+        await pool.query(`ALTER DEFAULT PRIVILEGES IN SCHEMA ${appSqlSchema} GRANT ALL ON TABLES TO anbaric_app`);
+        await pool.query(`ALTER DEFAULT PRIVILEGES IN SCHEMA ${appSqlSchema} GRANT ALL ON SEQUENCES TO anbaric_app`);
+        await pool.query("REVOKE ALL ON SCHEMA public FROM anbaric_app");
+        await pool.query("REVOKE ALL ON SCHEMA anbaric_system FROM anbaric_app");
+    }
 };
 
 export { ensureSchema }

package/src/hosting/handlers/AppsHandler.ts

@@ -11,6 +11,9 @@ class AppsHandler implements RequestHandler {
             case "deploy":
                 if (request.id) return this.handleDeploy(request, request.id);
                 break;
+            case "logs":
+                if (request.id) return this.handleLogs(request, request.id);
+                break;
             case undefined:
                 if (request.id) return this.handleApp(request, request.id);
                 return this.handleCollection(request);
@@ -38,12 +41,58 @@ class AppsHandler implements RequestHandler {
             case "GET": {
                 const status = this.buildLayer.status(appName);
                 if (!status) return request.reply(404, { error: `No app named "${appName}"` });
-                return request.reply(200, status);
+                const live = await this.buildLayer.ping(appName);
+                return request.reply(200, { ...status, live });
+            }
+            case "DELETE": {
+                const torn = await this.buildLayer.teardown(appName);
+                if (!torn) return request.reply(404, { error: `No app named "${appName}"` });
+                return request.reply(200, { appName, status: "stopped" });
             }
         }
         request.notFound();
     }
 
+    private async handleLogs(request : Request, appName : string) : Promise<void> {
+        if (request.method !== "GET") return request.notFound();
+        if (!this.buildLayer.status(appName)) return request.reply(404, { error: `No app named "${appName}"` });
+
+        const response = request.rawResponse;
+        response.writeHead(200, {
+            "content-type": "text/plain; charset=utf-8",
+            "cache-control": "no-cache",
+            "x-content-type-options": "nosniff",
+        });
+
+        const controller = new AbortController();
+        request.raw.on("close", () => controller.abort());
+
+        // A newline heartbeat after each idle interval keeps the streamed
+        // connection alive through the edge (CloudFront/ALB) read timeouts when
+        // the app is producing no output.
+        const beat = () : ReturnType<typeof setTimeout> => setTimeout(() => {
+            if (!response.writableEnded) { response.write("\n"); heartbeat = beat(); }
+        }, 15_000);
+        let heartbeat = beat();
+
+        try {
+            for await (const line of this.buildLayer.logs(appName, controller.signal)) {
+                clearTimeout(heartbeat);
+                if (!response.write(`${line}\n`)) {
+                    await new Promise<void>(resolve => response.once("drain", resolve));
+                }
+                heartbeat = beat();
+            }
+        } catch (error) {
+            if (!controller.signal.aborted) {
+                response.write(`[log stream error: ${error instanceof Error ? error.message : error}]\n`);
+            }
+        } finally {
+            clearTimeout(heartbeat);
+            response.end();
+        }
+    }
+
     private async handleCollection(request : Request) : Promise<void> {
         switch (request.method) {
             case "GET":

package/src/hosting/handlers/AuditsHandler.ts

@@ -1,10 +1,7 @@
-import {AuditInteraction} from "anbaric-tsapi";
 import {AuditRecordStore} from "../../auditing/AuditRecordStore";
 import {Request} from "../Request";
 import {RequestHandler} from "../RequestHandler";
 
-const AUDIT_INTERACTIONS = new Set(["CREATE", "UPDATE_PROPERTIES", "CHANGE_STATE", "DELETE", "READ", "LIST"]);
-
 class AuditsHandler implements RequestHandler {
 
     constructor(private auditRecords : AuditRecordStore) {}
@@ -27,7 +24,7 @@ class AuditsHandler implements RequestHandler {
             || typeof record?.description !== "string" || typeof record?.actorId !== "string"
             || typeof record?.actorType !== "string"
             || !Array.isArray(record?.interaction) || record.interaction.length === 0
-            || !record.interaction.every((interaction : unknown) => typeof interaction === "string" && AUDIT_INTERACTIONS.has(interaction))) {
+            || !record.interaction.every((interaction : unknown) => typeof interaction === "string" && interaction.length > 0)) {
             return request.reply(400, { error: "Expected a body of { resourceType, resourceId, actorId, actorType, interaction: [...], description, ... }" });
         }
         await this.auditRecords.save(record);
@@ -40,7 +37,7 @@ class AuditsHandler implements RequestHandler {
             resourceType: request.query("resourceType"),
             resourceId: request.query("resourceId"),
             actorId: request.query("actorId"),
-            interaction: interaction && AUDIT_INTERACTIONS.has(interaction) ? interaction as AuditInteraction : undefined,
+            interaction: interaction || undefined,
             search: request.query("search"),
             pageSize: request.query("pageSize") === undefined ? undefined : Number(request.query("pageSize")),
             page: request.query("page") === undefined ? undefined : Number(request.query("page")),

package/src/hosting/handlers/QueueHandler.ts

@@ -23,8 +23,8 @@ class QueueHandler implements RequestHandler {
             case "dequeue":
                 return request.reply(200, { messages: await this.queue.dequeueSome() });
             case "confirm": {
-                const { jobId, workflowId } = await request.body();
-                await this.queue.confirm({ jobId, workflowId });
+                const { jobId, workflowId, position } = await request.body();
+                await this.queue.confirm({ jobId, workflowId, position });
                 return request.reply(204);
             }
         }

package/src/index.ts

@@ -17,6 +17,8 @@ export * from "./auth/TokenAuthenticator";
 export * from "./auth/User";
 export * from "./app-management/DockerBuildLayer";
 export * from "./app-management/FargateBuildLayer";
+export * from "./app-admin/AdminServer";
+export * from "./app-admin/adminPing";
 export * from "./data-store/PostgresCliKeyStore";
 export * from "./data-store/PostgresJobPersistence";
 export * from "./data-store/PostgresJsonStore";

package/src/main.ts

@@ -35,6 +35,8 @@ const buildLayer = process.env.ANBARIC_BUILD_LAYER === "docker"
         baseImage: process.env.ANBARIC_APP_BASE_IMAGE ?? "anbaric-v2-platform:local",
         network: process.env.ANBARIC_DOCKER_NETWORK ?? "anbaric-v2-local",
         platformUrl: process.env.ANBARIC_PLATFORM_INTERNAL_URL ?? `http://localhost:${internalPort}`,
+        sqlDatabaseUrl: process.env.ANBARIC_APP_SQL_DATABASE_URL,
+        sqlSchema: process.env.ANBARIC_SQL_SCHEMA,
     })
     : process.env.ANBARIC_BUILD_LAYER === "fargate"
     ? new FargateBuildLayer(appsDir, {
@@ -50,9 +52,12 @@ const buildLayer = process.env.ANBARIC_BUILD_LAYER === "docker"
         baseImage: process.env.ANBARIC_AWS_BASE_IMAGE!,
         appExecutionRoleArn: process.env.ANBARIC_AWS_APP_EXECUTION_ROLE!,
         appsLogGroup: process.env.ANBARIC_AWS_APPS_LOG_GROUP!,
+        appsLogGroupArn: process.env.ANBARIC_AWS_APPS_LOG_GROUP_ARN!,
         platformUrl: process.env.ANBARIC_PLATFORM_INTERNAL_URL!,
         servicesUrl: process.env.ANBARIC_SERVICES_URL,
         servicesApiKey: process.env.ANBARIC_SERVICES_API_KEY,
+        sqlDatabaseUrlSecretArn: process.env.ANBARIC_AWS_APP_SQL_URL_SECRET,
+        sqlSchema: process.env.ANBARIC_SQL_SCHEMA,
     })
     : undefined;
 

package/src/queuing/PostgresQueue.ts

@@ -39,14 +39,12 @@ class PostgresQueue implements ConfirmableQueue {
                 if (!b.due) return 1;
                 return a.due.getTime() - b.due.getTime() || a.position - b.position;
             })
-            .map(row => ({ jobId: row.job_id, workflowId: row.workflow_id }));
+            .map(row => ({ jobId: row.job_id, workflowId: row.workflow_id, position: Number(row.position) }));
     }
 
     async confirm(message : QueueMessage) : Promise<void> {
-        await this.pool.query(
-            "DELETE FROM queue WHERE job_id = $1 AND workflow_id = $2",
-            [message.jobId, message.workflowId],
-        );
+        if (message.position === undefined) return;
+        await this.pool.query("DELETE FROM queue WHERE position = $1", [message.position]);
     }
 
 }