anbaric-cloud-hosting 1.59.0 → 1.61.0

package/src/index.ts

@@ -2,6 +2,7 @@ export * from "./app-management/BuildLayer";
 export * from "./auditing/AuditRecordStore";
 export * from "./auditing/InMemoryAuditRecordStore";
 export * from "./data-store/PostgresAuditRecordStore";
+export * from "./data-store/StallSweep";
 export * from "./auth/Authenticator";
 export * from "./auth/CliAuthorizer";
 export * from "./auth/CliKey";

package/src/main.ts

@@ -17,6 +17,7 @@ import {SecretsManagerSecretStore} from "./data-store/SecretsManagerSecretStore"
 import {S3FileStorage} from "./data-store/S3FileStorage";
 import {PostgresJobRunSchedulePersistence} from "./data-store/PostgresJobRunSchedulePersistence";
 import {PostgresJobPersistence} from "./data-store/PostgresJobPersistence";
+import {StallSweep} from "./data-store/StallSweep";
 import {PostgresJsonStore} from "./data-store/PostgresJsonStore";
 import {PostgresQueue} from "./queuing/PostgresQueue";
 import {DockerBuildLayer} from "./app-management/DockerBuildLayer";
@@ -32,6 +33,8 @@ import {ConsumerRegistry} from "./queuing/ConsumerRegistry";
 import {Dispatcher} from "./queuing/Dispatcher";
 import {HostingServer} from "./hosting/HostingServer";
 import {PluginLoader} from "./plugins/PluginLoader";
+import {UsageReporter} from "./usage/UsageReporter";
+import {HttpSubdomains} from "./subdomains/HttpSubdomains";
 
 const pool = new Pool({ connectionString: process.env.ANBARIC_DATABASE_URL });
 await ensureSchema(pool);
@@ -107,6 +110,13 @@ const memberships = process.env.ANBARIC_CLI_KEY_LOOKUP_URL && process.env.ANBARI
     ? new HttpMembershipService(process.env.ANBARIC_CLI_KEY_LOOKUP_URL, process.env.ANBARIC_CLI_KEY_LOOKUP_SECRET ?? "", process.env.ANBARIC_TENANT)
     : undefined;
 
+/* Where this tenant's apps answer. The addresses are the control plane's to
+   keep, because one has to be unique across every tenant; a platform that has
+   no control plane to ask has no addresses and serves its apps by path. */
+const subdomains = process.env.ANBARIC_CLI_KEY_LOOKUP_URL && process.env.ANBARIC_TENANT
+    ? new HttpSubdomains(process.env.ANBARIC_CLI_KEY_LOOKUP_URL, process.env.ANBARIC_CLI_KEY_LOOKUP_SECRET ?? "", process.env.ANBARIC_TENANT)
+    : undefined;
+
 /* Files are owned by an app, like secrets: on the hosted platform each app
    gets its own key prefix in the tenant's bucket, and locally its own folder
    under the storage root. A caller with no app - the console - gets the whole
@@ -119,16 +129,44 @@ const fileStorageFor = (appId : string) : FileStorage =>
         ? new S3FileStorage(s3, filesBucket, appId ? `${appId}/` : "")
         : new LocalFileSystemStorage(join(process.env.ANBARIC_FILE_STORAGE_PATH ?? join(tmpdir(), "anbaric", "files"), appId));
 
-const server = new HostingServer(new PostgresJobPersistence(pool), queue, registry, buildLayer,
+const jobs = new PostgresJobPersistence(pool);
+const server = new HostingServer(jobs, queue, registry, buildLayer,
     (appId, collection) => new PostgresJsonStore(pool, appId, collection), secretStoreFor, authenticator, cliAuthorizer,
     tokenAuthenticator, process.env.ANBARIC_TENANT, new PostgresAuditRecordStore(pool), plugins,
     new PostgresJobRunSchedulePersistence(pool), await loadNotifier(process.env.ANBARIC_NOTIFIER_MODULE),
     new PostgresEntitlementStore(pool), new PostgresUserDirectory(pool), memberships,
-    (appId) => new PostgresPromptManager(pool, appId), fileStorageFor);
+    (appId) => new PostgresPromptManager(pool, appId), fileStorageFor, subdomains);
 const port = await server.listen(hostingPort);
 const internal = await server.listenInternal(internalPort);
 
 const dispatcher = new Dispatcher(queue, registry, Number(process.env.ANBARIC_DISPATCH_INTERVAL_MS ?? 1000));
 dispatcher.start();
 
+/* Tells the control plane how many apps this tenant runs, which is what it is
+   billed for. Only a deployed tenant has anywhere to report to; a platform run
+   locally simply does not. Apps that are running or draining count - one being
+   built has not run yet, and one that is stopped or failed is not running. */
+if (buildLayer && process.env.ANBARIC_TENANT && process.env.ANBARIC_CLI_KEY_LOOKUP_URL) {
+    new UsageReporter(
+        process.env.ANBARIC_CLI_KEY_LOOKUP_URL,
+        process.env.ANBARIC_CLI_KEY_LOOKUP_SECRET ?? "",
+        process.env.ANBARIC_TENANT,
+        () => buildLayer.list().filter(app => app.status === "running" || app.status === "draining").length,
+    ).start();
+}
+
+/* Makes sure every app this platform is running has an address. Allocation is
+   idempotent, so this costs one call per app and changes nothing in the
+   ordinary case - but it is what gives an address to an app deployed before
+   addresses existed, and what repairs one the control plane never managed to
+   publish to the edge. Cheaper and harder to forget than a migration script. */
+if (buildLayer && subdomains) {
+    void buildLayer.ensureHydrated()
+        .then(() => Promise.all(buildLayer.list().map(app => subdomains.allocate(app.appName))))
+        .catch(error => console.warn(`[subdomains] could not reconcile app addresses: ${error instanceof Error ? error.message : error}`));
+}
+
+// Jobs whose long-running step stopped saying it was alive are marked, not touched.
+new StallSweep(jobs).start();
+
 console.log(`anbaric-cloud-hosting listening on port ${port}, internal entry point on ${internal}`);

package/src/subdomains/HttpSubdomains.ts

@@ -0,0 +1,91 @@
+import {AppAddress, Subdomains} from "./Subdomains";
+
+type FetchFn = (url : string, init : RequestInit) => Promise<Response>;
+
+/* Asks the control plane for this tenant's app addresses, with the same
+   central api key the platform already holds for key lookups and memberships.
+
+   Allocation is idempotent at the other end, so asking twice for one app is
+   not a mistake - it is how the platform makes sure every app it is running
+   has an address, every time it boots. A failure to allocate is reported as
+   nothing rather than thrown: an app that is running and reachable by path
+   should not be held up because its address could not be arranged, and the
+   next boot will ask again. Setting one, which a person did on purpose and is
+   waiting on, does throw. */
+const LIST_CACHE_TTL_MS = 60_000;
+
+class HttpSubdomains implements Subdomains {
+
+    private listed? : { addresses : Array<AppAddress>, expires : number };
+
+    constructor(private centralUrl : string, private secret : string, private tenantSlug : string,
+                private fetchFn : FetchFn = (url, init) => fetch(url, init),
+                private cacheTtlMs : number = LIST_CACHE_TTL_MS) {}
+
+    async allocate(appName : string) : Promise<AppAddress | undefined> {
+        try {
+            const response = await this.call("POST", `/${encodeURIComponent(appName)}/subdomain`);
+            this.listed = undefined;
+            return await response.json() as AppAddress;
+        } catch (error) {
+            console.warn(`[subdomains] could not get an address for "${appName}": ${this.because(error)}`);
+            return undefined;
+        }
+    }
+
+    async set(appName : string, subdomain : string) : Promise<AppAddress> {
+        const response = await this.call("PUT", `/${encodeURIComponent(appName)}/subdomain`, { subdomain });
+        this.listed = undefined;
+        return await response.json() as AppAddress;
+    }
+
+    async release(appName : string) : Promise<void> {
+        try {
+            await this.call("DELETE", `/${encodeURIComponent(appName)}/subdomain`, undefined, [404]);
+        } catch (error) {
+            console.warn(`[subdomains] could not release the address for "${appName}": ${this.because(error)}`);
+        }
+        this.listed = undefined;
+    }
+
+    /* The console asks for the app list on every page load, and an address
+       only changes when this platform changes it - so the answer is held until
+       it does, or until a minute has passed and another platform of the same
+       tenant might have. */
+    async all() : Promise<Array<AppAddress>> {
+        if (this.listed && this.listed.expires > Date.now()) return this.listed.addresses;
+
+        try {
+            const response = await this.call("GET", "");
+            const addresses = await response.json() as Array<AppAddress>;
+            this.listed = { addresses, expires: Date.now() + this.cacheTtlMs };
+            return addresses;
+        } catch (error) {
+            console.warn(`[subdomains] could not list app addresses: ${this.because(error)}`);
+            return this.listed?.addresses ?? [];
+        }
+    }
+
+    private because(error : unknown) : string {
+        return error instanceof Error ? error.message : String(error);
+    }
+
+    private async call(method : string, path : string, body? : unknown,
+                       tolerated : Array<number> = []) : Promise<Response> {
+        const headers : Record<string, string> = { "x-anbaric-central-key": this.secret };
+        if (body !== undefined) headers["content-type"] = "application/json";
+
+        const response = await this.fetchFn(
+            `${this.centralUrl.replace(/\/$/, "")}/tenants/${encodeURIComponent(this.tenantSlug)}/apps${path}`,
+            { method, headers, body: body === undefined ? undefined : JSON.stringify(body) });
+
+        if (! response.ok && ! tolerated.includes(response.status)) {
+            const problem = await response.json().catch(() => ({})) as { error? : string };
+            throw new Error(problem.error ?? `The request to the central login failed with status ${response.status}`);
+        }
+        return response;
+    }
+
+}
+
+export { HttpSubdomains };

package/src/subdomains/Subdomains.ts

@@ -0,0 +1,30 @@
+type AppAddress = {
+
+    appName : string,
+    subdomain : string,
+    published : boolean,
+
+};
+
+/* Where this tenant's apps answer. The addresses themselves belong to the
+   control plane, because one has to be unique across every tenant and no
+   single platform can know that; a platform asks for one and is told.
+
+   A platform that is not part of Anbaric Cloud has no control plane to ask, so
+   it has no addresses either and serves its apps by path. That is the whole
+   difference, and it is why this is an interface with a do-nothing
+   implementation rather than something the rest of the platform must check
+   for. */
+interface Subdomains {
+
+    allocate(appName : string) : Promise<AppAddress | undefined>;
+
+    set(appName : string, subdomain : string) : Promise<AppAddress>;
+
+    release(appName : string) : Promise<void>;
+
+    all() : Promise<Array<AppAddress>>;
+
+}
+
+export type { Subdomains, AppAddress };

package/src/usage/UsageReporter.ts

@@ -0,0 +1,63 @@
+type FetchFn = (url : string, init : RequestInit) => Promise<Response>;
+
+type CountApps = () => number;
+
+const REPORT_INTERVAL_MS = 60 * 60_000;
+
+/* Tells the control plane how many apps this tenant is running, hourly.
+   Hourly rather than daily because a day is recorded as the most apps seen on
+   it: an app deployed at noon has still run that day, and a platform that only
+   spoke at midnight would never have seen it.
+
+   It reports a number and learns nothing back. What an app costs, what the
+   tenant's allowance covers and when it is billed are the control plane's
+   business - so pricing can change without redeploying a single customer's
+   platform. A report that fails is simply the next hour's problem: the day is
+   the highest count that reached central, so one lost hour costs nothing
+   unless every hour of the day is lost. */
+class UsageReporter {
+
+    private ticker? : NodeJS.Timeout;
+
+    constructor(private centralUrl : string, private secret : string, private tenantSlug : string,
+                private countApps : CountApps,
+                private fetchFn : FetchFn = (url, init) => fetch(url, init),
+                private intervalMs : number = REPORT_INTERVAL_MS) {}
+
+    start() : void {
+        if (this.ticker) return;
+        void this.report();
+        this.ticker = setInterval(() => void this.report(), this.intervalMs);
+        this.ticker.unref?.();
+    }
+
+    async report() : Promise<boolean> {
+        try {
+            const response = await this.fetchFn(`${this.centralUrl}/usage`, {
+                method: "POST",
+                headers: { "content-type": "application/json", "x-anbaric-central-key": this.secret },
+                body: JSON.stringify({ slug: this.tenantSlug, apps: this.countApps(), day: this.today() }),
+            });
+            if (! response.ok) console.warn(`[usage] central answered ${response.status} to the app count`);
+            return response.ok;
+        } catch (error) {
+            console.warn(`[usage] could not report the app count: ${error instanceof Error ? error.message : error}`);
+            return false;
+        }
+    }
+
+    // The day in UTC, so every tenant's day starts at the same moment however
+    // the platform's clock is set.
+    private today() : string {
+        return new Date().toISOString().slice(0, 10);
+    }
+
+    async cleanUp() : Promise<void> {
+        if (this.ticker) clearInterval(this.ticker);
+        this.ticker = undefined;
+    }
+
+}
+
+export { UsageReporter };
+export type { CountApps };