@formancy/server 0.4.0 → 0.5.0

package/dist/main.mjs

@@ -1,11 +1,11 @@
-import { a as bootstrapSchema, i as createPostgresStorage, n as createLocalFileStore, o as createApp, t as startFileCollector } from "./file-collector-CGSJAtGO.mjs";
+import { a as createLocalFileStore, c as createPostgresStorage, d as LOG_LEVELS, f as createServerLog, h as maxFileBytesFrom, i as startFileCollector, l as bootstrapSchema, m as workerLog, p as databaseNotices, r as modelSettings, s as createPostgresRateLimitStore, t as createCompleter, u as createApp } from "./completers-DCFoZpo3.mjs";
 import { createHash, createHmac, randomBytes } from "node:crypto";
 import { deliveryHeaders, drainOutbox, isIpLiteral, isPrivateAddress } from "@formancy/server-core";
 import postgres from "postgres";
 import { lookup } from "node:dns/promises";
 import { Agent, fetch as fetch$1 } from "undici";
 import { Readable } from "node:stream";
-import { connect } from "node:net";
+import { connect, isIP } from "node:net";
 //#region src/webhook-url.ts
 /**
 * What is wrong with this webhook URL, or undefined when nothing is.
@@ -132,6 +132,7 @@ async function drain(response, cap) {
 //#region src/outbox-worker.ts
 function startOutboxWorker(storage, options = {}) {
 	const interval = options.intervalMs ?? 5e3;
+	const log = workerLog(options.log);
 	let stopped = false;
 	let running = false;
 	const tick = async () => {
@@ -155,7 +156,10 @@ function startOutboxWorker(storage, options = {}) {
 				})
 			});
 		} catch (error) {
-			console.error("outbox: pass failed", error);
+			log.error({
+				event: "outbox.failed",
+				err: error
+			});
 		} finally {
 			running = false;
 		}
@@ -170,6 +174,7 @@ function startOutboxWorker(storage, options = {}) {
 //#endregion
 //#region src/challenge-sweeper.ts
 function startChallengeSweeper(storage, options = {}) {
+	const log = workerLog(options.log);
 	const interval = options.intervalMs ?? 36e5;
 	let stopped = false;
 	let running = false;
@@ -179,7 +184,10 @@ function startChallengeSweeper(storage, options = {}) {
 		try {
 			await storage.forgetExpiredChallenges((/* @__PURE__ */ new Date()).toISOString());
 		} catch (error) {
-			console.error("challenges: a sweep failed", error);
+			log.error({
+				event: "sweeper.failed",
+				err: error
+			});
 		} finally {
 			running = false;
 		}
@@ -310,6 +318,27 @@ function createS3FileStore(config) {
 		const path = key.split("/").map(encodeSegment).join("/");
 		return new URL(`${endpoint}/${encodeSegment(config.bucket)}/${path}`);
 	};
+	/**
+	* One request, and the clock that abandons it.
+	*
+	* **The clock measures how long the store keeps us waiting.** It starts when the
+	* request is sent and stops when the headers arrive -- which, for a PUT, is after
+	* the whole body has gone. It starts again at once, because the store now owes the
+	* body, stops when a chunk arrives, and starts again only when the caller asks for
+	* the next one -- so a chunk the caller is still holding costs the store nothing.
+	*
+	* **Not `AbortSignal.timeout`, which is what this replaced.** That bounds the whole
+	* exchange, body included, so a download still streaming when it ran out was cut
+	* off mid-file however steadily it was arriving. And how long a download streams
+	* is set by the person downloading it more than by the store: backpressure carries
+	* a slow connection all the way back to this socket, so a bound on the whole
+	* request was a bound on the downloader's bandwidth.
+	*
+	* The clock restarts at the headers rather than at the first read so that a body
+	* nobody reads -- the error document on a path that throws -- is still abandoned
+	* after `timeoutMs`, rather than holding its connection for however long something
+	* else takes to close it.
+	*/
 	const send = async (method, key, body) => {
 		const url = urlFor(key);
 		const payloadHash = body === void 0 ? EMPTY_PAYLOAD_SHA256 : createHash("sha256").update(body).digest("hex");
@@ -324,12 +353,30 @@ function createS3FileStore(config) {
 			secretAccessKey: config.secretAccessKey,
 			now: /* @__PURE__ */ new Date()
 		});
-		return fetch(url, {
+		const waiting = clock(timeoutMs, `S3 ${method} of ${key}`);
+		waiting.start("no answer");
+		const response = await fetch(url, {
 			method,
 			headers,
 			...body === void 0 ? {} : { body: new Uint8Array(body) },
-			signal: AbortSignal.timeout(timeoutMs)
-		});
+			signal: waiting.signal
+		}).finally(waiting.stop);
+		const { ok, status, statusText } = response;
+		const answer = {
+			ok,
+			status,
+			statusText,
+			headers: response.headers
+		};
+		if (response.body === null) return {
+			...answer,
+			body: null
+		};
+		waiting.start("no data");
+		return {
+			...answer,
+			body: timed(response.body, waiting)
+		};
 	};
 	/**
 	* A status that means "there is no such object", as opposed to one that means
@@ -351,21 +398,21 @@ function createS3FileStore(config) {
 		async put(key, bytes) {
 			const response = await send("PUT", key, bytes);
 			if (!response.ok) throw failed("PUT", key, response);
-			await response.arrayBuffer();
+			await discard(response.body);
 		},
 		async open(key) {
 			const response = await send("GET", key);
 			if (isAbsent(response)) {
-				await response.arrayBuffer();
+				await discard(response.body);
 				return;
 			}
 			if (!response.ok || response.body === null) throw failed("GET", key, response);
-			return Readable.fromWeb(response.body);
+			return Readable.from(response.body, { objectMode: false });
 		},
 		async remove(key) {
 			const response = await send("DELETE", key);
 			if (response.ok || isAbsent(response)) {
-				await response.arrayBuffer();
+				await discard(response.body);
 				return;
 			}
 			throw failed("DELETE", key, response);
@@ -381,6 +428,55 @@ function createS3FileStore(config) {
 	};
 }
 /**
+* A clock for one request: started while the store owes something, stopped while it
+* does not, and aborting the request if it ever runs out. `owed` names what the
+* store failed to send, for the error that says so.
+*
+* Unref'd, as `AbortSignal.timeout` was: a deadline is not work. One is left running
+* under every body nobody reads, and a referenced timer would hold the process it
+* lives in for up to `timeoutMs` after everything else had finished. It still fires
+* when it matters: a request still in flight holds the process up by its connection.
+*/
+function clock(timeoutMs, request) {
+	const controller = new AbortController();
+	let timer;
+	const stop = () => clearTimeout(timer);
+	const start = (owed) => {
+		stop();
+		timer = setTimeout(() => {
+			controller.abort(/* @__PURE__ */ new Error(`${request}: ${owed} within ${String(timeoutMs)} ms`));
+		}, timeoutMs);
+		timer.unref();
+	};
+	return {
+		signal: controller.signal,
+		start,
+		stop
+	};
+}
+/**
+* A body chunk by chunk, with the clock running only while the next chunk is awaited.
+*
+* An abort after the headers errors the body with the clock's reason, so a body the
+* clock ran out on fails rather than ending early and looking complete.
+*/
+async function* timed(stream, waiting) {
+	try {
+		for await (const chunk of stream) {
+			waiting.stop();
+			yield chunk;
+			waiting.start("no data");
+		}
+	} finally {
+		waiting.stop();
+	}
+}
+/** Reads a body to its end and drops it, which hands the connection back to the pool. */
+async function discard(body) {
+	if (body === null) return;
+	for await (const _chunk of body);
+}
+/**
 * AWS's `UriEncode` for one path segment.
 *
 * `encodeURIComponent` leaves `!'()*` alone and S3 wants them encoded, and its
@@ -393,6 +489,60 @@ function encodeSegment(value) {
 	return encodeURIComponent(value).replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`);
 }
 //#endregion
+//#region src/file-store-settings.ts
+/**
+* Absent — or empty, see `setting` — means this deployment accepts no files,
+* and that is a supported state rather than a misconfiguration: a form with a
+* file field still renders and still submits, and the field says plainly that
+* there is nowhere to put one. Turning uploads on is naming a directory, which
+* in a container is naming a volume — and a volume is the one thing a
+* self-hoster has to think about, so it is not defaulted here.
+*
+* Throws on a configuration that names two stores, or an object store without
+* everything it needs, so the server refuses to start rather than serving one.
+*/
+function fileStoreSettings(env) {
+	const directory = setting(env, "FORMANCY_FILES_DIR");
+	const endpoint = setting(env, "FORMANCY_S3_ENDPOINT");
+	if (directory !== void 0 && endpoint !== void 0) throw new Error("Set FORMANCY_FILES_DIR or FORMANCY_S3_ENDPOINT, not both: two stores means files land in one and are looked for in the other.");
+	if (endpoint !== void 0) {
+		const required = (name) => {
+			const value = env[name];
+			if (value === void 0 || value === "") throw new Error(`${name} is required when FORMANCY_S3_ENDPOINT is set.`);
+			return value;
+		};
+		return {
+			kind: "s3",
+			config: {
+				endpoint,
+				bucket: required("FORMANCY_S3_BUCKET"),
+				region: required("FORMANCY_S3_REGION"),
+				accessKeyId: required("FORMANCY_S3_ACCESS_KEY_ID"),
+				secretAccessKey: required("FORMANCY_S3_SECRET_ACCESS_KEY")
+			}
+		};
+	}
+	return directory === void 0 ? { kind: "none" } : {
+		kind: "local",
+		directory
+	};
+}
+/**
+* A setting, with the empty string read as unset.
+*
+* Because compose cannot remove a variable, only blank it. Both compose files
+* point FORMANCY_FILES_DIR at their volume, and `.env` switching to the object
+* store does so by setting it to `""` — which, read as a directory, is two
+* stores and a server that refuses to start. Read as a path it is no better:
+* `createLocalFileStore('')` resolves to the working directory, which switches
+* uploads on in a directory nobody chose — in the published image, `/app`,
+* which its user cannot write.
+*/
+function setting(env, name) {
+	const value = env[name];
+	return value === "" ? void 0 : value;
+}
+//#endregion
 //#region src/clamd-scanner.ts
 /** How much is sent per INSTREAM chunk. clamd takes chunks of any size up to its stream limit. */
 const CHUNK = 65536;
@@ -422,6 +572,11 @@ const CLAMD_DEFAULT_MAX_BYTES = 104857600;
 * as a scanner that is down and would tell the person to try again forever; the real clamd
 * measured here answered intact. Checked first, the outcome does not depend on which.
 * Set it to what clamd is configured with.
+*
+* **`timeoutMs` is for silence, not for the whole scan.** It is the socket's idle timeout,
+* restarted by any traffic on the connection, so it abandons a clamd that has stopped
+* answering and not a scan that is still moving. Nothing here bounds how long a scan takes,
+* which is why the lease a request holds a file under can be outlasted (0153).
 */
 function createClamdScanner({ host, port, maxBytes = CLAMD_DEFAULT_MAX_BYTES, timeoutMs = 3e4 }) {
 	return { scan: (_file, bytes) => new Promise((resolve, reject) => {
@@ -479,6 +634,87 @@ function verdictOf(reply) {
 	};
 }
 //#endregion
+//#region src/trust-proxy.ts
+const WHAT_TO_WRITE = "Write the address of the reverse proxy in front of this server, e.g. 10.0.0.2, or a comma-separated list of addresses and CIDR ranges, e.g. 10.0.0.2,172.18.0.0/16 — or leave it unset to trust no proxy.";
+/**
+* The proxies whose `X-Forwarded-For` this server believes, read from
+* `FORMANCY_TRUST_PROXY`.
+*
+* Every limit on the public plane counts the client's address. Behind a reverse
+* proxy the socket's address is the proxy's, so trusting nothing gives every
+* respondent one shared budget; trusting the wrong thing lets a client write its
+* own `X-Forwarded-For` and choose a fresh budget per request. So the setting
+* names the proxies, and only addresses and ranges:
+*
+* - **No hop count.** Fastify's option takes a number, and the Fastify this
+*   server depends on treats one as "trust nothing", because a count cannot
+*   tell the proxy from a client that connects directly. Accepting it here
+*   would be a setting that reads as configured and does nothing;
+*   `trust-proxy.test.ts` checks that Fastify still ignores one.
+* - **No `true`,** which Fastify also takes, and which believes anybody.
+* - **No names,** such as `uniquelocal`, which Fastify's `proxy-addr` expands
+*   into ranges nobody reading the setting can see — that one is every private
+*   address, a Docker network's gateway with it, and the gateway is where
+*   Docker hands published-port connections over from. Refused by the address
+*   check below, as a hostname is.
+*
+* Decision 0156 has the reasoning, and what was measured against Docker.
+*
+* Absent or empty returns undefined, which trusts nothing — the behaviour before
+* this setting existed, and the right one with no proxy in front. Anything else
+* that is not an address or a range is refused here, with a sentence saying what
+* to write, rather than left to Fastify's parser, whose TypeError names neither
+* the variable nor the fix.
+*/
+function trustProxyFrom(value) {
+	if (value === void 0 || value.trim() === "") return void 0;
+	const trimmed = value.trim();
+	if (/^\d+$/.test(trimmed)) throw new Error(`FORMANCY_TRUST_PROXY is "${trimmed}", a hop count, which Fastify ignores: a count cannot tell the proxy from a client that connects directly. ${WHAT_TO_WRITE}`);
+	if (trimmed.toLowerCase() === "true") throw new Error(`FORMANCY_TRUST_PROXY is "true", which would believe X-Forwarded-For from anybody and let every client choose the address its rate limits count. ${WHAT_TO_WRITE}`);
+	const entries = trimmed.split(",").map((entry) => entry.trim());
+	for (const entry of entries) if (!isAddressOrRange(entry)) throw new Error(`FORMANCY_TRUST_PROXY has "${entry}", which is not an IP address or a CIDR range. ${WHAT_TO_WRITE}`);
+	return entries;
+}
+/**
+* An address, or an address with a prefix length its family allows.
+*
+* A zero prefix is refused, as Fastify's parser refuses it: `/0` is every
+* address, which is `true` spelled differently. A netmask after the slash is
+* refused too, though Fastify reads one — a second spelling of the same thing
+* is a second thing to get wrong, and a prefix length says it.
+*/
+function isAddressOrRange(entry) {
+	const slash = entry.indexOf("/");
+	if (slash === -1) return isIP(entry) !== 0;
+	const family = isIP(entry.slice(0, slash));
+	const prefix = entry.slice(slash + 1);
+	if (family === 0 || !/^\d{1,3}$/.test(prefix)) return false;
+	const bits = Number(prefix);
+	return bits >= 1 && bits <= (family === 4 ? 32 : 128);
+}
+//#endregion
+//#region src/log-settings.ts
+/**
+* How much the server logs, read from `FORMANCY_LOG_LEVEL`
+* ([0168](../../../docs/decisions/0168-the-log-is-built-from-a-list-of-fields.md)).
+*
+* Unset or empty is `info`: a line per request and per error. The log is on unless somebody
+* turns it off, because off is the state this setting replaced, where nothing said why a
+* request failed. `off` keeps no log at all; anything else that is not one of pino's level
+* names, spelled as pino spells them, stops the server at startup — a server that started
+* anyway would log at a level nobody chose, or not at all while its operator believed it
+* did. Its own module because `main.ts` connects to a database when it is imported, so
+* nothing in it can be tested.
+*/
+function logLevelFrom(env) {
+	const value = env["FORMANCY_LOG_LEVEL"]?.trim() ?? "";
+	if (value === "") return "info";
+	if (value === "off") return value;
+	const level = LOG_LEVELS.find((known) => known === value);
+	if (level === void 0) throw new Error(`FORMANCY_LOG_LEVEL is "${value}", which is not a level this server has. Write one of off, ${LOG_LEVELS.join(", ")} — or leave it unset for info.`);
+	return level;
+}
+//#endregion
 //#region src/main.ts
 process.env["RECHECK_BACKEND"] ??= "pure";
 /**
@@ -495,44 +731,28 @@ if (databaseUrl === void 0 || databaseUrl === "") {
 	process.exit(1);
 }
 const port = Number(process.env["PORT"] ?? 4380);
-const sql = postgres(databaseUrl);
+/**
+* The server's log, on standard output: a JSON line per request and per error, each built
+* from a list of fields, so no body, header, query string, answer or credential has anywhere
+* to go (0168). `info` unless `FORMANCY_LOG_LEVEL` says otherwise; `off` keeps none, and a
+* level the server does not have stops it here. Read first, so the database's notices on the
+* first connection go through it too rather than being printed whole by the driver, and so
+* does the rate limits' counter, which is made before the app whose log it would otherwise use.
+*/
+const logLevel = logLevelFrom(process.env);
+const log = logLevel === "off" ? void 0 : createServerLog(process.stdout, logLevel);
+const sql = postgres(databaseUrl, { onnotice: databaseNotices(log) });
 let authSecret = process.env["FORMANCY_AUTH_SECRET"];
 if (authSecret === void 0 || authSecret === "") {
 	authSecret = randomBytes(33).toString("base64url");
-	console.warn("FORMANCY_AUTH_SECRET is not set: generated an ephemeral one, sessions will not survive a restart.");
+	console.warn("FORMANCY_AUTH_SECRET is not set: generated an ephemeral one, so sessions will not survive a restart, nor will drafts or forms being filled in.");
 }
 const adminEmail = process.env["FORMANCY_ADMIN_EMAIL"];
 const adminPassword = process.env["FORMANCY_ADMIN_PASSWORD"];
-/**
-* Where uploaded bytes go.
-*
-* Absent means this deployment accepts no files, and that is a supported
-* state rather than a misconfiguration: a form with a file field still renders
-* and still submits, and the field says plainly that there is nowhere to put
-* one. Turning uploads on is naming a directory, which in a container is
-* naming a volume — and a volume is the one thing a self-hoster has to think
-* about, so it is not defaulted.
-*/
-const filesDirectory = process.env["FORMANCY_FILES_DIR"];
-const s3Endpoint = process.env["FORMANCY_S3_ENDPOINT"];
-if (filesDirectory !== void 0 && s3Endpoint !== void 0) throw new Error("Set FORMANCY_FILES_DIR or FORMANCY_S3_ENDPOINT, not both: two stores means files land in one and are looked for in the other.");
-const fileStore = (() => {
-	if (s3Endpoint === void 0) return void 0;
-	const required = (name) => {
-		const value = process.env[name];
-		if (value === void 0 || value === "") throw new Error(`${name} is required when FORMANCY_S3_ENDPOINT is set.`);
-		return value;
-	};
-	return createS3FileStore({
-		endpoint: s3Endpoint,
-		bucket: required("FORMANCY_S3_BUCKET"),
-		region: required("FORMANCY_S3_REGION"),
-		accessKeyId: required("FORMANCY_S3_ACCESS_KEY_ID"),
-		secretAccessKey: required("FORMANCY_S3_SECRET_ACCESS_KEY")
-	});
-})() ?? (filesDirectory === void 0 ? void 0 : createLocalFileStore(filesDirectory));
-const maxFileBytes = Number(process.env["FORMANCY_MAX_FILE_BYTES"] ?? 10485760);
-if (!Number.isFinite(maxFileBytes) || maxFileBytes <= 0) throw new Error("FORMANCY_MAX_FILE_BYTES must be a positive number of bytes.");
+/** Where uploaded bytes go: the rules are in `fileStoreSettings`, which throws. */
+const storeSettings = fileStoreSettings(process.env);
+const fileStore = storeSettings.kind === "s3" ? createS3FileStore(storeSettings.config) : storeSettings.kind === "local" ? createLocalFileStore(storeSettings.directory) : void 0;
+const maxFileBytes = maxFileBytesFrom(process.env);
 /**
 * A ClamAV daemon to ask about every upload before its bytes are kept.
 *
@@ -569,10 +789,33 @@ const scanner = clamdHost === void 0 || clamdHost === "" ? void 0 : createClamdS
 */
 const challengeSecret = process.env["FORMANCY_CHALLENGE_SECRET"];
 if (challengeSecret !== void 0 && challengeSecret.length < 32) throw new Error("FORMANCY_CHALLENGE_SECRET must be at least 32 characters, or unset.");
+/**
+* The reverse proxies whose X-Forwarded-For names the client. Unset trusts none,
+* which is right with nothing in front and wrong behind a proxy: every
+* respondent would share its one rate-limit budget. Refused at startup when it
+* is not addresses and ranges, since a server that started anyway would count
+* the wrong client and nothing would say so.
+*/
+const trustProxy = trustProxyFrom(process.env["FORMANCY_TRUST_PROXY"]);
+/**
+* The model the builders ask through this server, with its key kept here rather than in a
+* browser (0165). Unset is a supported state and the default: no model, no prompt pane in
+* the admin, and no form sent anywhere. Half a configuration — a key with no provider, a
+* provider with no model — is refused here, because a server that started anyway would
+* have no model while the operator believed it had one. The key is never logged.
+*/
+const modelChoice = modelSettings(process.env);
+const model = modelChoice.kind === "none" ? void 0 : {
+	provider: modelChoice.provider,
+	model: modelChoice.model,
+	completer: createCompleter(modelChoice)
+};
 await bootstrapSchema(sql);
 const storage = createPostgresStorage(sql);
+const rateLimits = createPostgresRateLimitStore(databaseUrl, log === void 0 ? {} : { log });
 const app = await createApp(storage, {
 	authSecret,
+	rateLimitStore: rateLimits.store,
 	...adminEmail !== void 0 && adminPassword !== void 0 ? { bootstrapAdmin: {
 		email: adminEmail,
 		password: adminPassword
@@ -580,14 +823,21 @@ const app = await createApp(storage, {
 	...fileStore === void 0 ? {} : { fileStore },
 	...challengeSecret === void 0 ? {} : { challengeSecret },
 	...scanner === void 0 ? {} : { scanner },
+	...trustProxy === void 0 ? {} : { trustProxy },
+	...model === void 0 ? {} : { model },
+	...logLevel === "off" ? {} : { log: {
+		sink: process.stdout,
+		level: logLevel
+	} },
 	maxFileBytes
 });
 const outbox = startOutboxWorker(storage, {
+	log: app.log,
 	...process.env["FORMANCY_WEBHOOK_ALLOW_HTTP"] === "true" ? { allowHttp: true } : {},
 	...process.env["FORMANCY_WEBHOOK_ALLOW_PRIVATE"] === "true" ? { allowPrivateAddresses: true } : {}
 });
-const collector = fileStore === void 0 ? void 0 : startFileCollector(storage, fileStore);
-const sweeper = challengeSecret === void 0 ? void 0 : startChallengeSweeper(storage);
+const collector = fileStore === void 0 ? void 0 : startFileCollector(storage, fileStore, { log: app.log });
+const sweeper = challengeSecret === void 0 ? void 0 : startChallengeSweeper(storage, { log: app.log });
 for (const signal of ["SIGINT", "SIGTERM"]) process.once(signal, () => {
 	outbox.stop();
 	collector?.stop();
@@ -599,8 +849,9 @@ await app.listen({
 	host: process.env["HOST"] ?? "0.0.0.0"
 });
 console.log(`formancy server listening on :${port}`);
+console.log(model === void 0 ? "No model configured." : `Builders ask ${model.provider}'s ${model.model}.`);
 for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, () => {
-	app.close().then(() => sql.end()).then(() => process.exit(0));
+	app.close().then(() => rateLimits.end()).then(() => sql.end()).then(() => process.exit(0));
 });
 //#endregion
 export {};