anbaric-cloud-hosting 1.68.1 → 1.68.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anbaric-cloud-hosting",
-  "version": "1.68.1",
+  "version": "1.68.2",
   "description": "Anbaric Cloud hosting service: Postgres-backed job persistence and queuing exposed over an HTTP API",
   "license": "MIT",
   "author": "chris@anbaric.ai",
@@ -18,18 +18,18 @@
     "@aws-sdk/client-s3": "^3.1110.0",
     "@aws-sdk/client-secrets-manager": "^3.700.0",
     "@aws-sdk/client-servicediscovery": "^3.1110.0",
-    "anbaric-data-store": "^1.68.1",
-    "anbaric-plugins": "^1.68.1",
-    "anbaric-tsapi": "^1.68.1",
-    "anbaric-web": "^1.68.1",
+    "anbaric-data-store": "^1.68.2",
+    "anbaric-plugins": "^1.68.2",
+    "anbaric-tsapi": "^1.68.2",
+    "anbaric-web": "^1.68.2",
     "esbuild": "^0.28.2",
     "pg": "^8.16.0"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",
     "@types/pg": "^8.15.0",
-    "anbaric-impl-cloud": "^1.68.1",
-    "anbaric-state-machine": "^1.68.1",
+    "anbaric-impl-cloud": "^1.68.2",
+    "anbaric-state-machine": "^1.68.2",
     "tsx": "^4.20.0",
     "typescript": "^7.0.2"
   },

package/src/hosting/FatalErrors.ts

@@ -0,0 +1,33 @@
+type Exit = (code : number) => void;
+type Log = (line : string) => void;
+
+/* The platform's last words. An error nobody caught ends the process - Node
+   gives no choice there - but it should end it saying what happened, in one
+   line a log search can find, rather than a stack trace that only a person
+   reading the stream would notice. ECS restarts the task either way; this is
+   what makes the restart explicable afterwards. */
+class FatalErrors {
+
+    constructor(private log : Log = line => console.error(line), private exit : Exit = code => process.exit(code)) {}
+
+    install(target : NodeJS.Process = process) : void {
+        target.on("uncaughtException", error => this.die("uncaughtException", error));
+        target.on("unhandledRejection", reason => this.die("unhandledRejection", reason));
+        target.on("SIGTERM", () => this.log(this.line("platform.stopping", { signal: "SIGTERM" })));
+    }
+
+    private die(kind : string, error : unknown) : void {
+        const detail = error instanceof Error
+            ? { message: error.message, name: error.name, code: (error as { code? : string }).code, stack: error.stack }
+            : { message: String(error) };
+        this.log(this.line("platform.crash", { kind, ...detail }));
+        this.exit(1);
+    }
+
+    private line(event : string, fields : Record<string, unknown>) : string {
+        return JSON.stringify({ level: "fatal", event, at: new Date().toISOString(), ...fields });
+    }
+
+}
+
+export { FatalErrors };

package/src/hosting/Server.ts

@@ -17,14 +17,32 @@ class Server {
                 const message = error instanceof Error ? error.message : "Internal error";
                 const notFound = /^No .+ found/.test(message);
                 if (! notFound) console.error(`[server] ${incoming.method} ${incoming.url} failed (ray ${response.getHeader("x-anbaric-ray") ?? "-"}):`, error);
-                if (!response.writableEnded) {
-                    response.writeHead(notFound ? 404 : 500, { "content-type": "application/json" });
-                    response.end(JSON.stringify({ error: notFound ? message : Server.internalErrorMessage(response) }));
-                }
+                Server.failResponse(response, notFound ? 404 : 500, notFound ? message : Server.internalErrorMessage(response));
             });
         });
     }
 
+    /* The last word on a request that failed. A response that has not started
+       gets a status and a body; one that was already streaming when it failed
+       cannot be given either, so its connection is cut instead - the client
+       sees a truncated body, which is the truth. Writing headers twice would
+       throw inside the error handler itself, with nothing left to catch it,
+       and take the whole platform down with one dropped connection. */
+    private static failResponse(response : ServerResponse, status : number, error : string) : void {
+        if (response.writableEnded) return;
+
+        if (response.headersSent) {
+            response.destroy();
+            return;
+        }
+        try {
+            response.writeHead(status, { "content-type": "application/json" });
+            response.end(JSON.stringify({ error }));
+        } catch {
+            response.destroy();
+        }
+    }
+
     /* What a failure inside the platform looks like from outside: never the
        underlying error, which names infrastructure and credentials, only a
        reference the logs can be searched for. */

package/src/hosting/handlers/AppProxyHandler.ts

@@ -54,7 +54,20 @@ class AppProxyHandler implements RequestHandler {
 
         const body = request.method === "GET" || request.method === "HEAD" ? undefined : await request.rawBody();
 
+        /* Once the app's response has started, whatever ends it ends the
+           request: a browser that closed the tab mid-stream, or an app that
+           dropped the connection. Neither is a failure of the platform's,
+           and there is nothing left to say to the client, so the upstream is
+           let go and the request is over. Only a failure before the first
+           byte is an error worth answering. */
         await new Promise<void>((resolve, reject) => {
+            const client = request.rawResponse;
+            const settle = (error : unknown) => {
+                if (! client.headersSent) return reject(error);
+                client.destroy();
+                resolve();
+            };
+
             const upstream = httpRequest({
                 host: app.appHost,
                 port: app.appPort,
@@ -62,12 +75,18 @@ class AppProxyHandler implements RequestHandler {
                 path: `${appPath}${request.url.search}`,
                 headers: this.forwardHeaders(request, prefix),
             }, response => {
-                request.rawResponse.writeHead(response.statusCode ?? 502, this.passThrough(response.headers));
-                response.pipe(request.rawResponse);
+                client.writeHead(response.statusCode ?? 502, this.passThrough(response.headers));
+                response.pipe(client);
                 response.on("end", resolve);
-                response.on("error", reject);
+                response.on("error", settle);
+                response.on("close", () => { if (! response.complete) settle(new Error("The app closed the connection")); });
+            });
+            client.on("close", () => {
+                if (client.writableFinished) return;
+                upstream.destroy();
+                resolve();
             });
-            upstream.on("error", reject);
+            upstream.on("error", settle);
             if (body && body.length > 0) upstream.write(body);
             upstream.end();
         });

package/src/main.ts

@@ -33,11 +33,14 @@ import {PostgresPromptManager} from "./data-store/PostgresPromptManager";
 import {ConsumerRegistry} from "./queuing/ConsumerRegistry";
 import {Dispatcher} from "./queuing/Dispatcher";
 import {HostingServer} from "./hosting/HostingServer";
+import {FatalErrors} from "./hosting/FatalErrors";
 import {PluginLoader} from "./plugins/PluginLoader";
 import {UsageReporter} from "./usage/UsageReporter";
 import {HttpSubdomains} from "./subdomains/HttpSubdomains";
 import {CentralBilling} from "./billing/CentralBilling";
 
+new FatalErrors().install();
+
 const pool = new Pool({ connectionString: process.env.ANBARIC_DATABASE_URL });
 await ensureSchema(pool);