@formancy/server 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/completers-DCFoZpo3.mjs +3110 -0
- package/dist/completers-DCFoZpo3.mjs.map +1 -0
- package/dist/index.d.mts +182 -9
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +2 -2
- package/dist/main.mjs +401 -42
- package/dist/main.mjs.map +1 -1
- package/package.json +8 -4
- package/dist/file-collector-LmfD8Y_d.mjs +0 -1504
- package/dist/file-collector-LmfD8Y_d.mjs.map +0 -1
package/dist/main.mjs
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { a as
|
|
1
|
+
import { a as createLocalFileStore, c as createPostgresStorage, d as LOG_LEVELS, f as createServerLog, h as maxFileBytesFrom, i as startFileCollector, l as bootstrapSchema, m as workerLog, p as databaseNotices, r as modelSettings, s as createPostgresRateLimitStore, t as createCompleter, u as createApp } from "./completers-DCFoZpo3.mjs";
|
|
2
2
|
import { createHash, createHmac, randomBytes } from "node:crypto";
|
|
3
3
|
import { deliveryHeaders, drainOutbox, isIpLiteral, isPrivateAddress } from "@formancy/server-core";
|
|
4
4
|
import postgres from "postgres";
|
|
5
5
|
import { lookup } from "node:dns/promises";
|
|
6
6
|
import { Agent, fetch as fetch$1 } from "undici";
|
|
7
7
|
import { Readable } from "node:stream";
|
|
8
|
+
import { connect, isIP } from "node:net";
|
|
8
9
|
//#region src/webhook-url.ts
|
|
9
10
|
/**
|
|
10
11
|
* What is wrong with this webhook URL, or undefined when nothing is.
|
|
@@ -131,6 +132,7 @@ async function drain(response, cap) {
|
|
|
131
132
|
//#region src/outbox-worker.ts
|
|
132
133
|
function startOutboxWorker(storage, options = {}) {
|
|
133
134
|
const interval = options.intervalMs ?? 5e3;
|
|
135
|
+
const log = workerLog(options.log);
|
|
134
136
|
let stopped = false;
|
|
135
137
|
let running = false;
|
|
136
138
|
const tick = async () => {
|
|
@@ -154,7 +156,10 @@ function startOutboxWorker(storage, options = {}) {
|
|
|
154
156
|
})
|
|
155
157
|
});
|
|
156
158
|
} catch (error) {
|
|
157
|
-
|
|
159
|
+
log.error({
|
|
160
|
+
event: "outbox.failed",
|
|
161
|
+
err: error
|
|
162
|
+
});
|
|
158
163
|
} finally {
|
|
159
164
|
running = false;
|
|
160
165
|
}
|
|
@@ -169,6 +174,7 @@ function startOutboxWorker(storage, options = {}) {
|
|
|
169
174
|
//#endregion
|
|
170
175
|
//#region src/challenge-sweeper.ts
|
|
171
176
|
function startChallengeSweeper(storage, options = {}) {
|
|
177
|
+
const log = workerLog(options.log);
|
|
172
178
|
const interval = options.intervalMs ?? 36e5;
|
|
173
179
|
let stopped = false;
|
|
174
180
|
let running = false;
|
|
@@ -178,7 +184,10 @@ function startChallengeSweeper(storage, options = {}) {
|
|
|
178
184
|
try {
|
|
179
185
|
await storage.forgetExpiredChallenges((/* @__PURE__ */ new Date()).toISOString());
|
|
180
186
|
} catch (error) {
|
|
181
|
-
|
|
187
|
+
log.error({
|
|
188
|
+
event: "sweeper.failed",
|
|
189
|
+
err: error
|
|
190
|
+
});
|
|
182
191
|
} finally {
|
|
183
192
|
running = false;
|
|
184
193
|
}
|
|
@@ -309,6 +318,27 @@ function createS3FileStore(config) {
|
|
|
309
318
|
const path = key.split("/").map(encodeSegment).join("/");
|
|
310
319
|
return new URL(`${endpoint}/${encodeSegment(config.bucket)}/${path}`);
|
|
311
320
|
};
|
|
321
|
+
/**
|
|
322
|
+
* One request, and the clock that abandons it.
|
|
323
|
+
*
|
|
324
|
+
* **The clock measures how long the store keeps us waiting.** It starts when the
|
|
325
|
+
* request is sent and stops when the headers arrive -- which, for a PUT, is after
|
|
326
|
+
* the whole body has gone. It starts again at once, because the store now owes the
|
|
327
|
+
* body, stops when a chunk arrives, and starts again only when the caller asks for
|
|
328
|
+
* the next one -- so a chunk the caller is still holding costs the store nothing.
|
|
329
|
+
*
|
|
330
|
+
* **Not `AbortSignal.timeout`, which is what this replaced.** That bounds the whole
|
|
331
|
+
* exchange, body included, so a download still streaming when it ran out was cut
|
|
332
|
+
* off mid-file however steadily it was arriving. And how long a download streams
|
|
333
|
+
* is set by the person downloading it more than by the store: backpressure carries
|
|
334
|
+
* a slow connection all the way back to this socket, so a bound on the whole
|
|
335
|
+
* request was a bound on the downloader's bandwidth.
|
|
336
|
+
*
|
|
337
|
+
* The clock restarts at the headers rather than at the first read so that a body
|
|
338
|
+
* nobody reads -- the error document on a path that throws -- is still abandoned
|
|
339
|
+
* after `timeoutMs`, rather than holding its connection for however long something
|
|
340
|
+
* else takes to close it.
|
|
341
|
+
*/
|
|
312
342
|
const send = async (method, key, body) => {
|
|
313
343
|
const url = urlFor(key);
|
|
314
344
|
const payloadHash = body === void 0 ? EMPTY_PAYLOAD_SHA256 : createHash("sha256").update(body).digest("hex");
|
|
@@ -323,12 +353,30 @@ function createS3FileStore(config) {
|
|
|
323
353
|
secretAccessKey: config.secretAccessKey,
|
|
324
354
|
now: /* @__PURE__ */ new Date()
|
|
325
355
|
});
|
|
326
|
-
|
|
356
|
+
const waiting = clock(timeoutMs, `S3 ${method} of ${key}`);
|
|
357
|
+
waiting.start("no answer");
|
|
358
|
+
const response = await fetch(url, {
|
|
327
359
|
method,
|
|
328
360
|
headers,
|
|
329
361
|
...body === void 0 ? {} : { body: new Uint8Array(body) },
|
|
330
|
-
signal:
|
|
331
|
-
});
|
|
362
|
+
signal: waiting.signal
|
|
363
|
+
}).finally(waiting.stop);
|
|
364
|
+
const { ok, status, statusText } = response;
|
|
365
|
+
const answer = {
|
|
366
|
+
ok,
|
|
367
|
+
status,
|
|
368
|
+
statusText,
|
|
369
|
+
headers: response.headers
|
|
370
|
+
};
|
|
371
|
+
if (response.body === null) return {
|
|
372
|
+
...answer,
|
|
373
|
+
body: null
|
|
374
|
+
};
|
|
375
|
+
waiting.start("no data");
|
|
376
|
+
return {
|
|
377
|
+
...answer,
|
|
378
|
+
body: timed(response.body, waiting)
|
|
379
|
+
};
|
|
332
380
|
};
|
|
333
381
|
/**
|
|
334
382
|
* A status that means "there is no such object", as opposed to one that means
|
|
@@ -350,21 +398,21 @@ function createS3FileStore(config) {
|
|
|
350
398
|
async put(key, bytes) {
|
|
351
399
|
const response = await send("PUT", key, bytes);
|
|
352
400
|
if (!response.ok) throw failed("PUT", key, response);
|
|
353
|
-
await response.
|
|
401
|
+
await discard(response.body);
|
|
354
402
|
},
|
|
355
403
|
async open(key) {
|
|
356
404
|
const response = await send("GET", key);
|
|
357
405
|
if (isAbsent(response)) {
|
|
358
|
-
await response.
|
|
406
|
+
await discard(response.body);
|
|
359
407
|
return;
|
|
360
408
|
}
|
|
361
409
|
if (!response.ok || response.body === null) throw failed("GET", key, response);
|
|
362
|
-
return Readable.
|
|
410
|
+
return Readable.from(response.body, { objectMode: false });
|
|
363
411
|
},
|
|
364
412
|
async remove(key) {
|
|
365
413
|
const response = await send("DELETE", key);
|
|
366
414
|
if (response.ok || isAbsent(response)) {
|
|
367
|
-
await response.
|
|
415
|
+
await discard(response.body);
|
|
368
416
|
return;
|
|
369
417
|
}
|
|
370
418
|
throw failed("DELETE", key, response);
|
|
@@ -380,6 +428,55 @@ function createS3FileStore(config) {
|
|
|
380
428
|
};
|
|
381
429
|
}
|
|
382
430
|
/**
|
|
431
|
+
* A clock for one request: started while the store owes something, stopped while it
|
|
432
|
+
* does not, and aborting the request if it ever runs out. `owed` names what the
|
|
433
|
+
* store failed to send, for the error that says so.
|
|
434
|
+
*
|
|
435
|
+
* Unref'd, as `AbortSignal.timeout` was: a deadline is not work. One is left running
|
|
436
|
+
* under every body nobody reads, and a referenced timer would hold the process it
|
|
437
|
+
* lives in for up to `timeoutMs` after everything else had finished. It still fires
|
|
438
|
+
* when it matters: a request still in flight holds the process up by its connection.
|
|
439
|
+
*/
|
|
440
|
+
function clock(timeoutMs, request) {
|
|
441
|
+
const controller = new AbortController();
|
|
442
|
+
let timer;
|
|
443
|
+
const stop = () => clearTimeout(timer);
|
|
444
|
+
const start = (owed) => {
|
|
445
|
+
stop();
|
|
446
|
+
timer = setTimeout(() => {
|
|
447
|
+
controller.abort(/* @__PURE__ */ new Error(`${request}: ${owed} within ${String(timeoutMs)} ms`));
|
|
448
|
+
}, timeoutMs);
|
|
449
|
+
timer.unref();
|
|
450
|
+
};
|
|
451
|
+
return {
|
|
452
|
+
signal: controller.signal,
|
|
453
|
+
start,
|
|
454
|
+
stop
|
|
455
|
+
};
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* A body chunk by chunk, with the clock running only while the next chunk is awaited.
|
|
459
|
+
*
|
|
460
|
+
* An abort after the headers errors the body with the clock's reason, so a body the
|
|
461
|
+
* clock ran out on fails rather than ending early and looking complete.
|
|
462
|
+
*/
|
|
463
|
+
async function* timed(stream, waiting) {
|
|
464
|
+
try {
|
|
465
|
+
for await (const chunk of stream) {
|
|
466
|
+
waiting.stop();
|
|
467
|
+
yield chunk;
|
|
468
|
+
waiting.start("no data");
|
|
469
|
+
}
|
|
470
|
+
} finally {
|
|
471
|
+
waiting.stop();
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
/** Reads a body to its end and drops it, which hands the connection back to the pool. */
|
|
475
|
+
async function discard(body) {
|
|
476
|
+
if (body === null) return;
|
|
477
|
+
for await (const _chunk of body);
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
383
480
|
* AWS's `UriEncode` for one path segment.
|
|
384
481
|
*
|
|
385
482
|
* `encodeURIComponent` leaves `!'()*` alone and S3 wants them encoded, and its
|
|
@@ -392,6 +489,232 @@ function encodeSegment(value) {
|
|
|
392
489
|
return encodeURIComponent(value).replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
393
490
|
}
|
|
394
491
|
//#endregion
|
|
492
|
+
//#region src/file-store-settings.ts
|
|
493
|
+
/**
|
|
494
|
+
* Absent — or empty, see `setting` — means this deployment accepts no files,
|
|
495
|
+
* and that is a supported state rather than a misconfiguration: a form with a
|
|
496
|
+
* file field still renders and still submits, and the field says plainly that
|
|
497
|
+
* there is nowhere to put one. Turning uploads on is naming a directory, which
|
|
498
|
+
* in a container is naming a volume — and a volume is the one thing a
|
|
499
|
+
* self-hoster has to think about, so it is not defaulted here.
|
|
500
|
+
*
|
|
501
|
+
* Throws on a configuration that names two stores, or an object store without
|
|
502
|
+
* everything it needs, so the server refuses to start rather than serving one.
|
|
503
|
+
*/
|
|
504
|
+
function fileStoreSettings(env) {
|
|
505
|
+
const directory = setting(env, "FORMANCY_FILES_DIR");
|
|
506
|
+
const endpoint = setting(env, "FORMANCY_S3_ENDPOINT");
|
|
507
|
+
if (directory !== void 0 && endpoint !== void 0) throw new Error("Set FORMANCY_FILES_DIR or FORMANCY_S3_ENDPOINT, not both: two stores means files land in one and are looked for in the other.");
|
|
508
|
+
if (endpoint !== void 0) {
|
|
509
|
+
const required = (name) => {
|
|
510
|
+
const value = env[name];
|
|
511
|
+
if (value === void 0 || value === "") throw new Error(`${name} is required when FORMANCY_S3_ENDPOINT is set.`);
|
|
512
|
+
return value;
|
|
513
|
+
};
|
|
514
|
+
return {
|
|
515
|
+
kind: "s3",
|
|
516
|
+
config: {
|
|
517
|
+
endpoint,
|
|
518
|
+
bucket: required("FORMANCY_S3_BUCKET"),
|
|
519
|
+
region: required("FORMANCY_S3_REGION"),
|
|
520
|
+
accessKeyId: required("FORMANCY_S3_ACCESS_KEY_ID"),
|
|
521
|
+
secretAccessKey: required("FORMANCY_S3_SECRET_ACCESS_KEY")
|
|
522
|
+
}
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
return directory === void 0 ? { kind: "none" } : {
|
|
526
|
+
kind: "local",
|
|
527
|
+
directory
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* A setting, with the empty string read as unset.
|
|
532
|
+
*
|
|
533
|
+
* Because compose cannot remove a variable, only blank it. Both compose files
|
|
534
|
+
* point FORMANCY_FILES_DIR at their volume, and `.env` switching to the object
|
|
535
|
+
* store does so by setting it to `""` — which, read as a directory, is two
|
|
536
|
+
* stores and a server that refuses to start. Read as a path it is no better:
|
|
537
|
+
* `createLocalFileStore('')` resolves to the working directory, which switches
|
|
538
|
+
* uploads on in a directory nobody chose — in the published image, `/app`,
|
|
539
|
+
* which its user cannot write.
|
|
540
|
+
*/
|
|
541
|
+
function setting(env, name) {
|
|
542
|
+
const value = env[name];
|
|
543
|
+
return value === "" ? void 0 : value;
|
|
544
|
+
}
|
|
545
|
+
//#endregion
|
|
546
|
+
//#region src/clamd-scanner.ts
|
|
547
|
+
/** How much is sent per INSTREAM chunk. clamd takes chunks of any size up to its stream limit. */
|
|
548
|
+
const CHUNK = 65536;
|
|
549
|
+
/**
|
|
550
|
+
* clamd's own default `StreamMaxLength`, 100 MiB — as the `clamd.conf` ClamAV 1.5.4 ships
|
|
551
|
+
* says, and as measured against it: a 90 MiB stream was answered, a 110 MiB one refused.
|
|
552
|
+
*/
|
|
553
|
+
const CLAMD_DEFAULT_MAX_BYTES = 104857600;
|
|
554
|
+
/**
|
|
555
|
+
* ClamAV's daemon, asked over its INSTREAM command
|
|
556
|
+
* ([0131](../../../docs/decisions/0131-an-upload-is-scanned-before-it-is-kept.md)).
|
|
557
|
+
*
|
|
558
|
+
* The protocol is clamd's own and small enough to speak directly — the command, the bytes in
|
|
559
|
+
* length-prefixed chunks, a zero-length chunk to end them, and one line back — so this adds
|
|
560
|
+
* no dependency to the server. Its tests run against a stand-in that speaks the protocol as
|
|
561
|
+
* clamd's manual describes it; it was run once against ClamAV 1.5.4 itself, and the
|
|
562
|
+
* documentation says what that showed and what it did not.
|
|
563
|
+
*
|
|
564
|
+
* **Only `stream: OK` is clean.** A finding is refused by its name. A file over clamd's size
|
|
565
|
+
* limit is refused too, because sending it again changes nothing. Anything else — an answer
|
|
566
|
+
* this does not understand, no answer, no daemon — rejects, which the use-case reads as a
|
|
567
|
+
* scanner that cannot be asked, and refuses the file for.
|
|
568
|
+
*
|
|
569
|
+
* **`maxBytes` is clamd's `StreamMaxLength`, and is checked before a byte is sent.** Past
|
|
570
|
+
* it, clamd says so and closes the connection while bytes may still be arriving. Against the
|
|
571
|
+
* stand-in the bytes in flight reset it and the answer was lost to `ECONNRESET`, which reads
|
|
572
|
+
* as a scanner that is down and would tell the person to try again forever; the real clamd
|
|
573
|
+
* measured here answered intact. Checked first, the outcome does not depend on which.
|
|
574
|
+
* Set it to what clamd is configured with.
|
|
575
|
+
*
|
|
576
|
+
* **`timeoutMs` is for silence, not for the whole scan.** It is the socket's idle timeout,
|
|
577
|
+
* restarted by any traffic on the connection, so it abandons a clamd that has stopped
|
|
578
|
+
* answering and not a scan that is still moving. Nothing here bounds how long a scan takes,
|
|
579
|
+
* which is why the lease a request holds a file under can be outlasted (0153).
|
|
580
|
+
*/
|
|
581
|
+
function createClamdScanner({ host, port, maxBytes = CLAMD_DEFAULT_MAX_BYTES, timeoutMs = 3e4 }) {
|
|
582
|
+
return { scan: (_file, bytes) => new Promise((resolve, reject) => {
|
|
583
|
+
if (bytes.byteLength > maxBytes) {
|
|
584
|
+
resolve({
|
|
585
|
+
clean: false,
|
|
586
|
+
finding: `could not be scanned: larger than the scanner accepts (${String(maxBytes)} bytes)`
|
|
587
|
+
});
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
590
|
+
const socket = connect({
|
|
591
|
+
host,
|
|
592
|
+
port
|
|
593
|
+
});
|
|
594
|
+
const answer = [];
|
|
595
|
+
let failure;
|
|
596
|
+
socket.setTimeout(timeoutMs, () => socket.destroy(/* @__PURE__ */ new Error(`clamd at ${host}:${String(port)} did not answer within ${String(timeoutMs)} ms`)));
|
|
597
|
+
socket.on("connect", () => {
|
|
598
|
+
socket.write("zINSTREAM\0");
|
|
599
|
+
for (let at = 0; at < bytes.byteLength; at += CHUNK) {
|
|
600
|
+
const piece = bytes.subarray(at, at + CHUNK);
|
|
601
|
+
const length = Buffer.alloc(4);
|
|
602
|
+
length.writeUInt32BE(piece.byteLength);
|
|
603
|
+
socket.write(length);
|
|
604
|
+
socket.write(piece);
|
|
605
|
+
}
|
|
606
|
+
socket.write(Buffer.alloc(4));
|
|
607
|
+
});
|
|
608
|
+
socket.on("data", (data) => answer.push(data));
|
|
609
|
+
socket.on("error", (error) => {
|
|
610
|
+
failure = error;
|
|
611
|
+
});
|
|
612
|
+
socket.on("close", () => {
|
|
613
|
+
const reply = Buffer.concat(answer).toString("utf8").replace(/\0+$/, "").trim();
|
|
614
|
+
if (reply === "") {
|
|
615
|
+
reject(failure ?? /* @__PURE__ */ new Error(`clamd at ${host}:${String(port)} closed without answering`));
|
|
616
|
+
return;
|
|
617
|
+
}
|
|
618
|
+
const verdict = verdictOf(reply);
|
|
619
|
+
if (verdict === void 0) reject(/* @__PURE__ */ new Error(`clamd answered something unexpected: ${reply}`));
|
|
620
|
+
else resolve(verdict);
|
|
621
|
+
});
|
|
622
|
+
}) };
|
|
623
|
+
}
|
|
624
|
+
function verdictOf(reply) {
|
|
625
|
+
if (reply === "stream: OK") return { clean: true };
|
|
626
|
+
const found = /^stream: (.+) FOUND$/.exec(reply);
|
|
627
|
+
if (found !== null) return {
|
|
628
|
+
clean: false,
|
|
629
|
+
finding: found[1]
|
|
630
|
+
};
|
|
631
|
+
if (/size limit exceeded/i.test(reply)) return {
|
|
632
|
+
clean: false,
|
|
633
|
+
finding: `could not be scanned: ${reply.replace(/\.?\s*ERROR$/, "")}`
|
|
634
|
+
};
|
|
635
|
+
}
|
|
636
|
+
//#endregion
|
|
637
|
+
//#region src/trust-proxy.ts
|
|
638
|
+
const WHAT_TO_WRITE = "Write the address of the reverse proxy in front of this server, e.g. 10.0.0.2, or a comma-separated list of addresses and CIDR ranges, e.g. 10.0.0.2,172.18.0.0/16 — or leave it unset to trust no proxy.";
|
|
639
|
+
/**
|
|
640
|
+
* The proxies whose `X-Forwarded-For` this server believes, read from
|
|
641
|
+
* `FORMANCY_TRUST_PROXY`.
|
|
642
|
+
*
|
|
643
|
+
* Every limit on the public plane counts the client's address. Behind a reverse
|
|
644
|
+
* proxy the socket's address is the proxy's, so trusting nothing gives every
|
|
645
|
+
* respondent one shared budget; trusting the wrong thing lets a client write its
|
|
646
|
+
* own `X-Forwarded-For` and choose a fresh budget per request. So the setting
|
|
647
|
+
* names the proxies, and only addresses and ranges:
|
|
648
|
+
*
|
|
649
|
+
* - **No hop count.** Fastify's option takes a number, and the Fastify this
|
|
650
|
+
* server depends on treats one as "trust nothing", because a count cannot
|
|
651
|
+
* tell the proxy from a client that connects directly. Accepting it here
|
|
652
|
+
* would be a setting that reads as configured and does nothing;
|
|
653
|
+
* `trust-proxy.test.ts` checks that Fastify still ignores one.
|
|
654
|
+
* - **No `true`,** which Fastify also takes, and which believes anybody.
|
|
655
|
+
* - **No names,** such as `uniquelocal`, which Fastify's `proxy-addr` expands
|
|
656
|
+
* into ranges nobody reading the setting can see — that one is every private
|
|
657
|
+
* address, a Docker network's gateway with it, and the gateway is where
|
|
658
|
+
* Docker hands published-port connections over from. Refused by the address
|
|
659
|
+
* check below, as a hostname is.
|
|
660
|
+
*
|
|
661
|
+
* Decision 0156 has the reasoning, and what was measured against Docker.
|
|
662
|
+
*
|
|
663
|
+
* Absent or empty returns undefined, which trusts nothing — the behaviour before
|
|
664
|
+
* this setting existed, and the right one with no proxy in front. Anything else
|
|
665
|
+
* that is not an address or a range is refused here, with a sentence saying what
|
|
666
|
+
* to write, rather than left to Fastify's parser, whose TypeError names neither
|
|
667
|
+
* the variable nor the fix.
|
|
668
|
+
*/
|
|
669
|
+
function trustProxyFrom(value) {
|
|
670
|
+
if (value === void 0 || value.trim() === "") return void 0;
|
|
671
|
+
const trimmed = value.trim();
|
|
672
|
+
if (/^\d+$/.test(trimmed)) throw new Error(`FORMANCY_TRUST_PROXY is "${trimmed}", a hop count, which Fastify ignores: a count cannot tell the proxy from a client that connects directly. ${WHAT_TO_WRITE}`);
|
|
673
|
+
if (trimmed.toLowerCase() === "true") throw new Error(`FORMANCY_TRUST_PROXY is "true", which would believe X-Forwarded-For from anybody and let every client choose the address its rate limits count. ${WHAT_TO_WRITE}`);
|
|
674
|
+
const entries = trimmed.split(",").map((entry) => entry.trim());
|
|
675
|
+
for (const entry of entries) if (!isAddressOrRange(entry)) throw new Error(`FORMANCY_TRUST_PROXY has "${entry}", which is not an IP address or a CIDR range. ${WHAT_TO_WRITE}`);
|
|
676
|
+
return entries;
|
|
677
|
+
}
|
|
678
|
+
/**
|
|
679
|
+
* An address, or an address with a prefix length its family allows.
|
|
680
|
+
*
|
|
681
|
+
* A zero prefix is refused, as Fastify's parser refuses it: `/0` is every
|
|
682
|
+
* address, which is `true` spelled differently. A netmask after the slash is
|
|
683
|
+
* refused too, though Fastify reads one — a second spelling of the same thing
|
|
684
|
+
* is a second thing to get wrong, and a prefix length says it.
|
|
685
|
+
*/
|
|
686
|
+
function isAddressOrRange(entry) {
|
|
687
|
+
const slash = entry.indexOf("/");
|
|
688
|
+
if (slash === -1) return isIP(entry) !== 0;
|
|
689
|
+
const family = isIP(entry.slice(0, slash));
|
|
690
|
+
const prefix = entry.slice(slash + 1);
|
|
691
|
+
if (family === 0 || !/^\d{1,3}$/.test(prefix)) return false;
|
|
692
|
+
const bits = Number(prefix);
|
|
693
|
+
return bits >= 1 && bits <= (family === 4 ? 32 : 128);
|
|
694
|
+
}
|
|
695
|
+
//#endregion
|
|
696
|
+
//#region src/log-settings.ts
|
|
697
|
+
/**
|
|
698
|
+
* How much the server logs, read from `FORMANCY_LOG_LEVEL`
|
|
699
|
+
* ([0168](../../../docs/decisions/0168-the-log-is-built-from-a-list-of-fields.md)).
|
|
700
|
+
*
|
|
701
|
+
* Unset or empty is `info`: a line per request and per error. The log is on unless somebody
|
|
702
|
+
* turns it off, because off is the state this setting replaced, where nothing said why a
|
|
703
|
+
* request failed. `off` keeps no log at all; anything else that is not one of pino's level
|
|
704
|
+
* names, spelled as pino spells them, stops the server at startup — a server that started
|
|
705
|
+
* anyway would log at a level nobody chose, or not at all while its operator believed it
|
|
706
|
+
* did. Its own module because `main.ts` connects to a database when it is imported, so
|
|
707
|
+
* nothing in it can be tested.
|
|
708
|
+
*/
|
|
709
|
+
function logLevelFrom(env) {
|
|
710
|
+
const value = env["FORMANCY_LOG_LEVEL"]?.trim() ?? "";
|
|
711
|
+
if (value === "") return "info";
|
|
712
|
+
if (value === "off") return value;
|
|
713
|
+
const level = LOG_LEVELS.find((known) => known === value);
|
|
714
|
+
if (level === void 0) throw new Error(`FORMANCY_LOG_LEVEL is "${value}", which is not a level this server has. Write one of off, ${LOG_LEVELS.join(", ")} — or leave it unset for info.`);
|
|
715
|
+
return level;
|
|
716
|
+
}
|
|
717
|
+
//#endregion
|
|
395
718
|
//#region src/main.ts
|
|
396
719
|
process.env["RECHECK_BACKEND"] ??= "pure";
|
|
397
720
|
/**
|
|
@@ -408,44 +731,48 @@ if (databaseUrl === void 0 || databaseUrl === "") {
|
|
|
408
731
|
process.exit(1);
|
|
409
732
|
}
|
|
410
733
|
const port = Number(process.env["PORT"] ?? 4380);
|
|
411
|
-
|
|
734
|
+
/**
|
|
735
|
+
* The server's log, on standard output: a JSON line per request and per error, each built
|
|
736
|
+
* from a list of fields, so no body, header, query string, answer or credential has anywhere
|
|
737
|
+
* to go (0168). `info` unless `FORMANCY_LOG_LEVEL` says otherwise; `off` keeps none, and a
|
|
738
|
+
* level the server does not have stops it here. Read first, so the database's notices on the
|
|
739
|
+
* first connection go through it too rather than being printed whole by the driver, and so
|
|
740
|
+
* does the rate limits' counter, which is made before the app whose log it would otherwise use.
|
|
741
|
+
*/
|
|
742
|
+
const logLevel = logLevelFrom(process.env);
|
|
743
|
+
const log = logLevel === "off" ? void 0 : createServerLog(process.stdout, logLevel);
|
|
744
|
+
const sql = postgres(databaseUrl, { onnotice: databaseNotices(log) });
|
|
412
745
|
let authSecret = process.env["FORMANCY_AUTH_SECRET"];
|
|
413
746
|
if (authSecret === void 0 || authSecret === "") {
|
|
414
747
|
authSecret = randomBytes(33).toString("base64url");
|
|
415
|
-
console.warn("FORMANCY_AUTH_SECRET is not set: generated an ephemeral one, sessions will not survive a restart.");
|
|
748
|
+
console.warn("FORMANCY_AUTH_SECRET is not set: generated an ephemeral one, so sessions will not survive a restart, nor will drafts or forms being filled in.");
|
|
416
749
|
}
|
|
417
750
|
const adminEmail = process.env["FORMANCY_ADMIN_EMAIL"];
|
|
418
751
|
const adminPassword = process.env["FORMANCY_ADMIN_PASSWORD"];
|
|
752
|
+
/** Where uploaded bytes go: the rules are in `fileStoreSettings`, which throws. */
|
|
753
|
+
const storeSettings = fileStoreSettings(process.env);
|
|
754
|
+
const fileStore = storeSettings.kind === "s3" ? createS3FileStore(storeSettings.config) : storeSettings.kind === "local" ? createLocalFileStore(storeSettings.directory) : void 0;
|
|
755
|
+
const maxFileBytes = maxFileBytesFrom(process.env);
|
|
419
756
|
/**
|
|
420
|
-
*
|
|
757
|
+
* A ClamAV daemon to ask about every upload before its bytes are kept.
|
|
421
758
|
*
|
|
422
|
-
*
|
|
423
|
-
*
|
|
424
|
-
*
|
|
425
|
-
*
|
|
426
|
-
*
|
|
427
|
-
*
|
|
759
|
+
* Unset is a supported state: no file is scanned, which the documentation says. Set, a
|
|
760
|
+
* file is kept only once clamd has called it clean, and refused when clamd cannot be
|
|
761
|
+
* reached — the point of configuring a scanner is that nothing unscanned gets in
|
|
762
|
+
* ([0131](../../../docs/decisions/0131-an-upload-is-scanned-before-it-is-kept.md)).
|
|
763
|
+
* Not checked at startup: clamd takes minutes to load its signatures, and a server that
|
|
764
|
+
* refused to start until then would turn a slow scanner into an outage of everything.
|
|
428
765
|
*/
|
|
429
|
-
const
|
|
430
|
-
const
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
return createS3FileStore({
|
|
440
|
-
endpoint: s3Endpoint,
|
|
441
|
-
bucket: required("FORMANCY_S3_BUCKET"),
|
|
442
|
-
region: required("FORMANCY_S3_REGION"),
|
|
443
|
-
accessKeyId: required("FORMANCY_S3_ACCESS_KEY_ID"),
|
|
444
|
-
secretAccessKey: required("FORMANCY_S3_SECRET_ACCESS_KEY")
|
|
445
|
-
});
|
|
446
|
-
})() ?? (filesDirectory === void 0 ? void 0 : createLocalFileStore(filesDirectory));
|
|
447
|
-
const maxFileBytes = Number(process.env["FORMANCY_MAX_FILE_BYTES"] ?? 10485760);
|
|
448
|
-
if (!Number.isFinite(maxFileBytes) || maxFileBytes <= 0) throw new Error("FORMANCY_MAX_FILE_BYTES must be a positive number of bytes.");
|
|
766
|
+
const clamdHost = process.env["FORMANCY_CLAMD_HOST"];
|
|
767
|
+
const clamdPort = Number(process.env["FORMANCY_CLAMD_PORT"] ?? 3310);
|
|
768
|
+
const clamdMaxBytes = Number(process.env["FORMANCY_CLAMD_MAX_BYTES"] ?? 104857600);
|
|
769
|
+
if (!Number.isInteger(clamdPort) || clamdPort <= 0 || clamdPort > 65535) throw new Error("FORMANCY_CLAMD_PORT must be a port number.");
|
|
770
|
+
if (!Number.isFinite(clamdMaxBytes) || clamdMaxBytes <= 0) throw new Error("FORMANCY_CLAMD_MAX_BYTES must be a positive number of bytes: clamd's StreamMaxLength.");
|
|
771
|
+
const scanner = clamdHost === void 0 || clamdHost === "" ? void 0 : createClamdScanner({
|
|
772
|
+
host: clamdHost,
|
|
773
|
+
port: clamdPort,
|
|
774
|
+
maxBytes: clamdMaxBytes
|
|
775
|
+
});
|
|
449
776
|
/**
|
|
450
777
|
* Turns the proof-of-work challenge on for anonymous submissions.
|
|
451
778
|
*
|
|
@@ -462,24 +789,55 @@ if (!Number.isFinite(maxFileBytes) || maxFileBytes <= 0) throw new Error("FORMAN
|
|
|
462
789
|
*/
|
|
463
790
|
const challengeSecret = process.env["FORMANCY_CHALLENGE_SECRET"];
|
|
464
791
|
if (challengeSecret !== void 0 && challengeSecret.length < 32) throw new Error("FORMANCY_CHALLENGE_SECRET must be at least 32 characters, or unset.");
|
|
792
|
+
/**
|
|
793
|
+
* The reverse proxies whose X-Forwarded-For names the client. Unset trusts none,
|
|
794
|
+
* which is right with nothing in front and wrong behind a proxy: every
|
|
795
|
+
* respondent would share its one rate-limit budget. Refused at startup when it
|
|
796
|
+
* is not addresses and ranges, since a server that started anyway would count
|
|
797
|
+
* the wrong client and nothing would say so.
|
|
798
|
+
*/
|
|
799
|
+
const trustProxy = trustProxyFrom(process.env["FORMANCY_TRUST_PROXY"]);
|
|
800
|
+
/**
|
|
801
|
+
* The model the builders ask through this server, with its key kept here rather than in a
|
|
802
|
+
* browser (0165). Unset is a supported state and the default: no model, no prompt pane in
|
|
803
|
+
* the admin, and no form sent anywhere. Half a configuration — a key with no provider, a
|
|
804
|
+
* provider with no model — is refused here, because a server that started anyway would
|
|
805
|
+
* have no model while the operator believed it had one. The key is never logged.
|
|
806
|
+
*/
|
|
807
|
+
const modelChoice = modelSettings(process.env);
|
|
808
|
+
const model = modelChoice.kind === "none" ? void 0 : {
|
|
809
|
+
provider: modelChoice.provider,
|
|
810
|
+
model: modelChoice.model,
|
|
811
|
+
completer: createCompleter(modelChoice)
|
|
812
|
+
};
|
|
465
813
|
await bootstrapSchema(sql);
|
|
466
814
|
const storage = createPostgresStorage(sql);
|
|
815
|
+
const rateLimits = createPostgresRateLimitStore(databaseUrl, log === void 0 ? {} : { log });
|
|
467
816
|
const app = await createApp(storage, {
|
|
468
817
|
authSecret,
|
|
818
|
+
rateLimitStore: rateLimits.store,
|
|
469
819
|
...adminEmail !== void 0 && adminPassword !== void 0 ? { bootstrapAdmin: {
|
|
470
820
|
email: adminEmail,
|
|
471
821
|
password: adminPassword
|
|
472
822
|
} } : {},
|
|
473
823
|
...fileStore === void 0 ? {} : { fileStore },
|
|
474
824
|
...challengeSecret === void 0 ? {} : { challengeSecret },
|
|
825
|
+
...scanner === void 0 ? {} : { scanner },
|
|
826
|
+
...trustProxy === void 0 ? {} : { trustProxy },
|
|
827
|
+
...model === void 0 ? {} : { model },
|
|
828
|
+
...logLevel === "off" ? {} : { log: {
|
|
829
|
+
sink: process.stdout,
|
|
830
|
+
level: logLevel
|
|
831
|
+
} },
|
|
475
832
|
maxFileBytes
|
|
476
833
|
});
|
|
477
834
|
const outbox = startOutboxWorker(storage, {
|
|
835
|
+
log: app.log,
|
|
478
836
|
...process.env["FORMANCY_WEBHOOK_ALLOW_HTTP"] === "true" ? { allowHttp: true } : {},
|
|
479
837
|
...process.env["FORMANCY_WEBHOOK_ALLOW_PRIVATE"] === "true" ? { allowPrivateAddresses: true } : {}
|
|
480
838
|
});
|
|
481
|
-
const collector = fileStore === void 0 ? void 0 : startFileCollector(storage, fileStore);
|
|
482
|
-
const sweeper = challengeSecret === void 0 ? void 0 : startChallengeSweeper(storage);
|
|
839
|
+
const collector = fileStore === void 0 ? void 0 : startFileCollector(storage, fileStore, { log: app.log });
|
|
840
|
+
const sweeper = challengeSecret === void 0 ? void 0 : startChallengeSweeper(storage, { log: app.log });
|
|
483
841
|
for (const signal of ["SIGINT", "SIGTERM"]) process.once(signal, () => {
|
|
484
842
|
outbox.stop();
|
|
485
843
|
collector?.stop();
|
|
@@ -491,8 +849,9 @@ await app.listen({
|
|
|
491
849
|
host: process.env["HOST"] ?? "0.0.0.0"
|
|
492
850
|
});
|
|
493
851
|
console.log(`formancy server listening on :${port}`);
|
|
852
|
+
console.log(model === void 0 ? "No model configured." : `Builders ask ${model.provider}'s ${model.model}.`);
|
|
494
853
|
for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, () => {
|
|
495
|
-
app.close().then(() => sql.end()).then(() => process.exit(0));
|
|
854
|
+
app.close().then(() => rateLimits.end()).then(() => sql.end()).then(() => process.exit(0));
|
|
496
855
|
});
|
|
497
856
|
//#endregion
|
|
498
857
|
export {};
|