anbaric-cloud-hosting 1.0.0

package/src/index.ts

@@ -0,0 +1,24 @@
+export * from "./app-management/BuildLayer";
+export * from "./auth/Authenticator";
+export * from "./auth/CliAuthorizer";
+export * from "./auth/CliKey";
+export * from "./auth/CliKeyStore";
+export * from "./auth/InMemoryCliKeyStore";
+export * from "./auth/AuthenticatorLoader";
+export * from "./auth/KeyPair";
+export * from "./auth/Role";
+export * from "./auth/Tenant";
+export * from "./auth/TokenAuthenticator";
+export * from "./auth/User";
+export * from "./app-management/DockerBuildLayer";
+export * from "./app-management/FargateBuildLayer";
+export * from "./data-store/PostgresCliKeyStore";
+export * from "./data-store/PostgresJobPersistence";
+export * from "./data-store/PostgresJsonStore";
+export * from "./data-store/Schema";
+export * from "./data-store/SecretsManagerSecretStore";
+export * from "./hosting/HostingServer";
+export * from "./hosting/Router";
+export * from "./queuing/ConfirmableQueue";
+export * from "./queuing/ConsumerRegistry";
+export * from "./queuing/Dispatcher";

package/src/main.ts

@@ -0,0 +1,72 @@
+import {SecretsManagerClient} from "@aws-sdk/client-secrets-manager";
+import {SecretStore} from "anbaric-tsapi";
+import {InMemorySecretStore} from "anbaric-data-store";
+import {Pool} from "pg";
+import {loadAuthenticator} from "./auth/AuthenticatorLoader";
+import {CliAuthorizer} from "./auth/CliAuthorizer";
+import {TokenAuthenticator} from "./auth/TokenAuthenticator";
+import {PostgresCliKeyStore} from "./data-store/PostgresCliKeyStore";
+import {ensureSchema} from "./data-store/Schema";
+import {SecretsManagerSecretStore} from "./data-store/SecretsManagerSecretStore";
+import {PostgresJobPersistence} from "./data-store/PostgresJobPersistence";
+import {PostgresJsonStore} from "./data-store/PostgresJsonStore";
+import {PostgresQueue} from "./queuing/PostgresQueue";
+import {DockerBuildLayer} from "./app-management/DockerBuildLayer";
+import {FargateBuildLayer} from "./app-management/FargateBuildLayer";
+import {ConsumerRegistry} from "./queuing/ConsumerRegistry";
+import {Dispatcher} from "./queuing/Dispatcher";
+import {HostingServer} from "./hosting/HostingServer";
+
+const pool = new Pool({ connectionString: process.env.ANBARIC_DATABASE_URL });
+await ensureSchema(pool);
+
+const queue = new PostgresQueue(pool);
+const registry = new ConsumerRegistry();
+
+const hostingPort = Number(process.env.ANBARIC_HOSTING_PORT ?? 8787);
+const internalPort = Number(process.env.ANBARIC_INTERNAL_PORT ?? 8788);
+const appsDir = process.env.ANBARIC_APPS_DIR ?? "/tmp/anbaric-apps";
+
+const buildLayer = process.env.ANBARIC_BUILD_LAYER === "docker"
+    ? new DockerBuildLayer(appsDir, {
+        baseImage: process.env.ANBARIC_APP_BASE_IMAGE ?? "anbaric-v2-platform:local",
+        network: process.env.ANBARIC_DOCKER_NETWORK ?? "anbaric-v2-local",
+        platformUrl: process.env.ANBARIC_PLATFORM_INTERNAL_URL ?? `http://localhost:${internalPort}`,
+    })
+    : process.env.ANBARIC_BUILD_LAYER === "fargate"
+    ? new FargateBuildLayer(appsDir, {
+        awsRegion: process.env.AWS_REGION!,
+        cluster: process.env.ANBARIC_AWS_CLUSTER!,
+        subnets: (process.env.ANBARIC_AWS_SUBNETS ?? "").split(","),
+        appSecurityGroup: process.env.ANBARIC_AWS_APP_SECURITY_GROUP!,
+        namespaceId: process.env.ANBARIC_AWS_NAMESPACE_ID!,
+        namespaceName: process.env.ANBARIC_AWS_NAMESPACE_NAME!,
+        appsRepositoryUrl: process.env.ANBARIC_AWS_APPS_REPOSITORY!,
+        buildBucket: process.env.ANBARIC_AWS_BUILD_BUCKET!,
+        buildProject: process.env.ANBARIC_AWS_BUILD_PROJECT!,
+        baseImage: process.env.ANBARIC_AWS_BASE_IMAGE!,
+        appExecutionRoleArn: process.env.ANBARIC_AWS_APP_EXECUTION_ROLE!,
+        appsLogGroup: process.env.ANBARIC_AWS_APPS_LOG_GROUP!,
+        platformUrl: process.env.ANBARIC_PLATFORM_INTERNAL_URL!,
+    })
+    : undefined;
+
+const secretStore : SecretStore = process.env.AWS_REGION
+    ? new SecretsManagerSecretStore(new SecretsManagerClient({}))
+    : new InMemorySecretStore();
+
+const authenticator = await loadAuthenticator(process.env.ANBARIC_AUTHENTICATOR);
+const cliKeyStore = new PostgresCliKeyStore(pool);
+const cliAuthorizer = new CliAuthorizer(cliKeyStore);
+const tokenAuthenticator = new TokenAuthenticator(cliKeyStore);
+
+const server = new HostingServer(new PostgresJobPersistence(pool), queue, registry, buildLayer,
+    (collection) => new PostgresJsonStore(pool, collection), secretStore, authenticator, cliAuthorizer,
+    tokenAuthenticator, process.env.ANBARIC_TENANT);
+const port = await server.listen(hostingPort);
+const internal = await server.listenInternal(internalPort);
+
+const dispatcher = new Dispatcher(queue, registry, Number(process.env.ANBARIC_DISPATCH_INTERVAL_MS ?? 1000));
+dispatcher.start();
+
+console.log(`anbaric-cloud-hosting listening on port ${port}, internal entry point on ${internal}`);

package/src/queuing/ConfirmableQueue.ts

@@ -0,0 +1,9 @@
+import {Dequeue, QueueMessage} from "anbaric-tsapi";
+
+interface ConfirmableQueue extends Dequeue {
+
+    confirm(message : QueueMessage) : Promise<void>;
+
+}
+
+export { ConfirmableQueue }

package/src/queuing/ConsumerRegistry.ts

@@ -0,0 +1,19 @@
+class ConsumerRegistry {
+
+    private consumers = new Map<string, string>();
+
+    register(workflowId : string, url : string) : void {
+        this.consumers.set(workflowId, url);
+    }
+
+    lookup(workflowId : string) : string | undefined {
+        return this.consumers.get(workflowId);
+    }
+
+    list() : Array<{ workflowId : string, url : string }> {
+        return Array.from(this.consumers, ([workflowId, url]) => ({ workflowId, url }));
+    }
+
+}
+
+export { ConsumerRegistry }

package/src/queuing/Dispatcher.ts

@@ -0,0 +1,64 @@
+import {Dequeue, QueueMessage} from "anbaric-tsapi";
+import {ConsumerRegistry} from "./ConsumerRegistry";
+
+class Dispatcher {
+
+    private ticker? : NodeJS.Timeout;
+    private draining = false;
+
+    constructor(private queue : Dequeue, private registry : ConsumerRegistry,
+                private dispatchIntervalMs : number = 1000) {}
+
+    start() : void {
+        if (this.ticker) return;
+        this.ticker = setInterval(() => void this.drain(), this.dispatchIntervalMs);
+        this.ticker.unref();
+    }
+
+    async cleanUp() : Promise<void> {
+        if (this.ticker) clearInterval(this.ticker);
+        this.ticker = undefined;
+    }
+
+    private async drain() : Promise<void> {
+        if (this.draining) return;
+        this.draining = true;
+        try {
+            const messages = await this.queue.dequeueSome();
+            const byConsumerUrl = new Map<string, Array<QueueMessage>>();
+
+            for (const message of messages) {
+                const url = this.registry.lookup(message.workflowId);
+                if (!url) {
+                    await this.queue.enqueue(message.jobId, message.workflowId);
+                    continue;
+                }
+                byConsumerUrl.set(url, [...(byConsumerUrl.get(url) ?? []), message]);
+            }
+
+            for (const [url, batch] of byConsumerUrl) {
+                await this.push(url, batch);
+            }
+        } finally {
+            this.draining = false;
+        }
+    }
+
+    private async push(url : string, batch : Array<QueueMessage>) : Promise<void> {
+        try {
+            const response = await fetch(`${url}/process`, {
+                method: "POST",
+                headers: { "content-type": "application/json" },
+                body: JSON.stringify({ messages: batch }),
+            });
+            if (!response.ok) throw new Error(`Consumer at ${url} responded with status ${response.status}`);
+        } catch {
+            for (const message of batch) {
+                await this.queue.enqueue(message.jobId, message.workflowId);
+            }
+        }
+    }
+
+}
+
+export { Dispatcher }

package/src/queuing/PostgresQueue.ts

@@ -0,0 +1,54 @@
+import {QueueMessage} from "anbaric-tsapi";
+import {Pool} from "pg";
+import {ConfirmableQueue} from "./ConfirmableQueue";
+
+const DEQUEUE_BATCH_SIZE = 100;
+const LEASE_SECONDS = 30;
+
+class PostgresQueue implements ConfirmableQueue {
+
+    constructor(private pool : Pool) {}
+
+    async enqueue(jobId : string, workflowId : string) : Promise<void> {
+        await this.pool.query("INSERT INTO queue (job_id, workflow_id) VALUES ($1, $2)", [jobId, workflowId]);
+    }
+
+    async schedule(jobId : string, workflowId : string, due : Date) : Promise<void> {
+        await this.pool.query("INSERT INTO queue (job_id, workflow_id, due) VALUES ($1, $2, $3)", [jobId, workflowId, due]);
+    }
+
+    async dequeueSome() : Promise<Array<QueueMessage>> {
+        const result = await this.pool.query(
+            `UPDATE queue SET leased_until = now() + interval '${LEASE_SECONDS} seconds'
+             WHERE position IN (
+                 SELECT position FROM queue
+                 WHERE (due IS NULL OR due <= now())
+                   AND (leased_until IS NULL OR leased_until < now())
+                 ORDER BY (due IS NOT NULL), due, position
+                 LIMIT $1
+                 FOR UPDATE SKIP LOCKED
+             )
+             RETURNING job_id, workflow_id, due, position`,
+            [DEQUEUE_BATCH_SIZE],
+        );
+
+        return result.rows
+            .sort((a, b) => {
+                if (!a.due && !b.due) return a.position - b.position;
+                if (!a.due) return -1;
+                if (!b.due) return 1;
+                return a.due.getTime() - b.due.getTime() || a.position - b.position;
+            })
+            .map(row => ({ jobId: row.job_id, workflowId: row.workflow_id }));
+    }
+
+    async confirm(message : QueueMessage) : Promise<void> {
+        await this.pool.query(
+            "DELETE FROM queue WHERE job_id = $1 AND workflow_id = $2",
+            [message.jobId, message.workflowId],
+        );
+    }
+
+}
+
+export { PostgresQueue }