@formancy/server 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/completers-DCFoZpo3.mjs +3110 -0
- package/dist/completers-DCFoZpo3.mjs.map +1 -0
- package/dist/index.d.mts +180 -9
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +2 -2
- package/dist/main.mjs +297 -46
- package/dist/main.mjs.map +1 -1
- package/package.json +8 -4
- package/dist/file-collector-CGSJAtGO.mjs +0 -1545
- package/dist/file-collector-CGSJAtGO.mjs.map +0 -1
|
@@ -0,0 +1,3110 @@
|
|
|
1
|
+
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
2
|
+
import { AUDIT_ACTIONS, auditedBy, authenticateApiKey, authenticateLocal, can, collectAbandonedFiles, completeBuilderRequest, createApiKey, createLocalUser, createSubmission, decodeSolution, exportCsv, formToFill, healthOf, keepExamples, listForms, listSubmissions, listVersions, mintChallenge, offerUpload, publishForm, readExamples, replayDelivery, resolveForm, resumeDraft, saveDraft, screenUpload, setFormAccess, startDraft, verifySolution } from "@formancy/server-core";
|
|
3
|
+
import { MODEL_REQUEST_KINDS } from "@formancy/builder-core";
|
|
4
|
+
import Fastify, { LogController } from "fastify";
|
|
5
|
+
import rateLimit from "@fastify/rate-limit";
|
|
6
|
+
import { hash, verify } from "@node-rs/argon2";
|
|
7
|
+
import { SignJWT, jwtVerify } from "jose";
|
|
8
|
+
import { METHODS, STATUS_CODES, validateHeaderName, validateHeaderValue } from "node:http";
|
|
9
|
+
import { and, desc, eq, inArray, isNull, lt, lte, max, ne, or } from "drizzle-orm";
|
|
10
|
+
import { drizzle } from "drizzle-orm/postgres-js";
|
|
11
|
+
import { integer, jsonb, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core";
|
|
12
|
+
import postgres from "postgres";
|
|
13
|
+
import { createReadStream } from "node:fs";
|
|
14
|
+
import { mkdir, rm, stat, writeFile } from "node:fs/promises";
|
|
15
|
+
import { dirname, join, resolve, sep } from "node:path";
|
|
16
|
+
import Anthropic, { APIError } from "@anthropic-ai/sdk";
|
|
17
|
+
import OpenAI, { APIError as APIError$1 } from "openai";
|
|
18
|
+
//#region src/header-values.ts
|
|
19
|
+
/**
|
|
20
|
+
* A response header's value made from data: a name somebody else chose
|
|
21
|
+
* ([0175](../../../docs/decisions/0175-a-name-goes-into-a-header-encoded.md)).
|
|
22
|
+
*
|
|
23
|
+
* Node writes a header only when every character of its value is a tab, printable ASCII or one
|
|
24
|
+
* of the bytes 0x80 to 0xFF, and throws `ERR_INVALID_CHAR` for anything else — when the reply is
|
|
25
|
+
* written, after the route has answered. Nothing about a file's name or a form's path keeps it
|
|
26
|
+
* inside that: a respondent names a file `日本 €.pdf`, an author publishes a path with a line
|
|
27
|
+
* break in it. And the bytes 0x80 to 0xFF it does accept are no better: the server sent `é` as one
|
|
28
|
+
* byte before a stream and as two, its UTF-8, before a string body, so a client read `café` on a
|
|
29
|
+
* download and `café` on an export.
|
|
30
|
+
*
|
|
31
|
+
* So a value is made here, from the data, in printable ASCII only, and a route writes what this
|
|
32
|
+
* returns rather than a template of its own. `header-values.test.ts` holds every export to
|
|
33
|
+
* every UTF-16 code unit, and reads the server's sources for a header written any other way.
|
|
34
|
+
*/
|
|
35
|
+
/**
|
|
36
|
+
* `content-disposition` for a download, by RFC 6266: `attachment`, and the name twice when
|
|
37
|
+
* ASCII cannot carry it — once readable, once exact.
|
|
38
|
+
*
|
|
39
|
+
* `filename` is a quoted string of printable ASCII, with every character it cannot hold as
|
|
40
|
+
* itself made `_`: anything outside it, the quote that would end it, the backslash a reader may
|
|
41
|
+
* or may not take as an escape, and the percent sign RFC 6266 (appendix D) says some readers
|
|
42
|
+
* decode. `filename*` is RFC 8187's extended value, the name's UTF-8 percent-encoded, which
|
|
43
|
+
* RFC 6266 has a reader prefer when both are there. It is added only when the readable one
|
|
44
|
+
* lost something, so an ordinary name keeps the header it always had.
|
|
45
|
+
*
|
|
46
|
+
* The name is not cleaned of anything: a control character or a path separator goes to the
|
|
47
|
+
* reader's browser encoded, and making a file name its system can hold is the browser's, as it
|
|
48
|
+
* is for every download.
|
|
49
|
+
*/
|
|
50
|
+
function attachment(filename) {
|
|
51
|
+
const readable = filename.replace(/[^\x20-\x7e]|["\\%]/gu, "_");
|
|
52
|
+
if (readable === filename) return `attachment; filename="${readable}"`;
|
|
53
|
+
return `attachment; filename="${readable}"; filename*=UTF-8''${percentEncoded(filename)}`;
|
|
54
|
+
}
|
|
55
|
+
/** RFC 8187's attr-char: what an extended value may hold as itself. */
|
|
56
|
+
const ATTR_CHAR = /^[A-Za-z0-9!#$&+\-.^_`|~]$/;
|
|
57
|
+
/**
|
|
58
|
+
* `text` as UTF-8, each byte that is not an attr-char written `%XX`.
|
|
59
|
+
*
|
|
60
|
+
* Through a `Buffer` rather than `encodeURIComponent`, which leaves `'`, `(`, `)` and `*` as they
|
|
61
|
+
* are — and a `'` ends an extended value's charset — and throws on a lone surrogate, which a
|
|
62
|
+
* JSON body can carry. A `Buffer` writes one as U+FFFD.
|
|
63
|
+
*/
|
|
64
|
+
function percentEncoded(text) {
|
|
65
|
+
let encoded = "";
|
|
66
|
+
for (const byte of Buffer.from(text, "utf8")) {
|
|
67
|
+
const character = String.fromCharCode(byte);
|
|
68
|
+
encoded += ATTR_CHAR.test(character) ? character : `%${byte.toString(16).toUpperCase().padStart(2, "0")}`;
|
|
69
|
+
}
|
|
70
|
+
return encoded;
|
|
71
|
+
}
|
|
72
|
+
//#endregion
|
|
73
|
+
//#region src/rate-limits.ts
|
|
74
|
+
/**
|
|
75
|
+
* A route's `config.rateLimit`.
|
|
76
|
+
*
|
|
77
|
+
* The failure mode is an argument, so no limit can be written without saying it; left to the
|
|
78
|
+
* plugin, it is an error, which a route answers as a 500 nobody chose. And the window is
|
|
79
|
+
* translated here, once: `timeWindowMs` is this app's name for it and the plugin does not
|
|
80
|
+
* read it, which one route found out by silently taking the global window instead.
|
|
81
|
+
*/
|
|
82
|
+
function limited(budget, whenUncounted) {
|
|
83
|
+
return {
|
|
84
|
+
max: budget.max,
|
|
85
|
+
timeWindow: budget.timeWindowMs,
|
|
86
|
+
skipOnError: whenUncounted === "admit"
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
//#endregion
|
|
90
|
+
//#region src/routes/files.ts
|
|
91
|
+
/**
|
|
92
|
+
* How long one request holds a file while its bytes are scanned and written (0153).
|
|
93
|
+
*
|
|
94
|
+
* A judgement, not a bound the adapters guarantee. The object store this package supplies
|
|
95
|
+
* abandons a PUT after 30 seconds, as a whole; its clamd adapter's 30 seconds are of silence
|
|
96
|
+
* — any traffic on the connection restarts them — so nothing here limits how long a scan
|
|
97
|
+
* takes. Longer makes an overrun rarer, and keeps a PUT of the same file busy for longer
|
|
98
|
+
* after a request that died holding it. A request that does overrun it can still replace a
|
|
99
|
+
* claimed file's bytes, and `docs/architecture/11-risks-and-debt.md` says what that costs.
|
|
100
|
+
*/
|
|
101
|
+
const RECEIVE_LEASE_MS = 12e4;
|
|
102
|
+
/** A request whose lease ran out is not accepted, whichever check found it. */
|
|
103
|
+
const OUTLASTED = {
|
|
104
|
+
status: 409,
|
|
105
|
+
body: {
|
|
106
|
+
error: "busy",
|
|
107
|
+
message: "This upload took longer than the server holds a file for one request, so it was not accepted. Try again."
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
/**
|
|
111
|
+
* A form's files: offering one, receiving its bytes, and serving it back.
|
|
112
|
+
*
|
|
113
|
+
* One route family per plugin, the seam `app.ts`'s size budget names. It moved when
|
|
114
|
+
* scanning an upload's bytes needed lines that file had no room for.
|
|
115
|
+
*/
|
|
116
|
+
async function fileRoutes(app, { deps, fileStore, maxFileBytes, limit, actorOf }) {
|
|
117
|
+
const { storage } = deps;
|
|
118
|
+
const submissionLimit = limit;
|
|
119
|
+
/**
|
|
120
|
+
* Ask where to put a file.
|
|
121
|
+
*
|
|
122
|
+
* Refusing here, before a byte is sent, is the difference between rejecting
|
|
123
|
+
* a 2 GB upload and receiving one first. The reply carries the id the
|
|
124
|
+
* submission will reference and the URL to PUT the bytes to.
|
|
125
|
+
*/
|
|
126
|
+
app.post("/f/:path/files", { config: { rateLimit: limited(submissionLimit, "admit") } }, async (request, reply) => {
|
|
127
|
+
if (fileStore === void 0) return reply.code(501).send({
|
|
128
|
+
error: "uploads_unavailable",
|
|
129
|
+
message: "This deployment has no file store configured, so it cannot accept uploads."
|
|
130
|
+
});
|
|
131
|
+
const { path } = request.params;
|
|
132
|
+
const resolved = await resolveForm(deps, path);
|
|
133
|
+
if (resolved === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
134
|
+
const form = await storage.getFormByPath(path);
|
|
135
|
+
if (form === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
136
|
+
const actor = await actorOf(request);
|
|
137
|
+
const origin = request.headers.origin;
|
|
138
|
+
const body = request.body ?? {};
|
|
139
|
+
const field = typeof body.field === "string" ? body.field : void 0;
|
|
140
|
+
const name = typeof body.name === "string" ? body.name : void 0;
|
|
141
|
+
const size = typeof body.size === "number" ? body.size : void 0;
|
|
142
|
+
const contentType = typeof body.contentType === "string" ? body.contentType : "application/octet-stream";
|
|
143
|
+
if (field === void 0 || name === void 0 || size === void 0) return reply.code(400).send({
|
|
144
|
+
error: "invalid_request",
|
|
145
|
+
message: "Send the field, the name and the size of the file you want to upload."
|
|
146
|
+
});
|
|
147
|
+
if (!Number.isSafeInteger(size) || size < 0) return reply.code(400).send({
|
|
148
|
+
error: "invalid_request",
|
|
149
|
+
message: "The size of a file is a whole number of bytes, zero or more."
|
|
150
|
+
});
|
|
151
|
+
if (size > maxFileBytes) return reply.code(413).send({
|
|
152
|
+
error: "too_large",
|
|
153
|
+
message: `This deployment accepts files up to ${String(maxFileBytes)} bytes.`
|
|
154
|
+
});
|
|
155
|
+
const outcome = await offerUpload({
|
|
156
|
+
storage,
|
|
157
|
+
now: () => /* @__PURE__ */ new Date(),
|
|
158
|
+
newId: () => randomUUID()
|
|
159
|
+
}, {
|
|
160
|
+
schema: resolved.schema,
|
|
161
|
+
form,
|
|
162
|
+
actor: actor === void 0 ? "anonymous" : "authenticated",
|
|
163
|
+
origin: typeof origin === "string" ? origin : void 0,
|
|
164
|
+
field,
|
|
165
|
+
name,
|
|
166
|
+
size,
|
|
167
|
+
contentType
|
|
168
|
+
});
|
|
169
|
+
if (!outcome.ok) {
|
|
170
|
+
const status = outcome.reason === "not_allowed" ? 403 : outcome.reason === "too_large" ? 413 : 400;
|
|
171
|
+
return reply.code(status).send({ error: outcome.reason });
|
|
172
|
+
}
|
|
173
|
+
return reply.code(201).send({
|
|
174
|
+
id: outcome.file.id,
|
|
175
|
+
name: outcome.file.name,
|
|
176
|
+
size: outcome.file.size,
|
|
177
|
+
contentType: outcome.file.contentType,
|
|
178
|
+
storageKey: outcome.file.storageKey,
|
|
179
|
+
uploadUrl: `/f/${encodeURIComponent(path)}/files/${outcome.file.id}`
|
|
180
|
+
});
|
|
181
|
+
});
|
|
182
|
+
/**
|
|
183
|
+
* Receive the bytes for a file that was offered.
|
|
184
|
+
*
|
|
185
|
+
* In a context of its own, with every body parser removed and one put back that hands
|
|
186
|
+
* any body over as bytes. A file is whatever the reader had — the form's `accept` list
|
|
187
|
+
* decides whether it is allowed, at the offer — and Fastify parses `text/plain` and
|
|
188
|
+
* `application/json` itself: a catch-all added beside those only ever saw the types with
|
|
189
|
+
* no parser of their own, so a `.txt` or `.json` file arrived here as a string or an
|
|
190
|
+
* object and was refused as no body. Encapsulated, so that no other route is handed raw
|
|
191
|
+
* bytes, nor a body as large as a file past the request body cap.
|
|
192
|
+
*/
|
|
193
|
+
await app.register(async (byteRoutes) => {
|
|
194
|
+
byteRoutes.removeAllContentTypeParsers();
|
|
195
|
+
byteRoutes.addContentTypeParser("*", {
|
|
196
|
+
parseAs: "buffer",
|
|
197
|
+
bodyLimit: maxFileBytes
|
|
198
|
+
}, (_request, body, done) => {
|
|
199
|
+
done(null, body);
|
|
200
|
+
});
|
|
201
|
+
byteRoutes.put("/f/:path/files/:id", { config: { rateLimit: limited(submissionLimit, "admit") } }, async (request, reply) => {
|
|
202
|
+
if (fileStore === void 0) return reply.code(501).send({ error: "uploads_unavailable" });
|
|
203
|
+
const { path, id } = request.params;
|
|
204
|
+
const form = await storage.getFormByPath(path);
|
|
205
|
+
if (form === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
206
|
+
const file = await storage.getFile(id);
|
|
207
|
+
if (file === void 0 || file.formId !== form.id) return reply.code(404).send({ error: "unknown_file" });
|
|
208
|
+
if (file.state !== "offered") return reply.code(409).send({ error: "already_uploaded" });
|
|
209
|
+
const bytes = request.body;
|
|
210
|
+
if (!Buffer.isBuffer(bytes)) return reply.code(400).send({ error: "no_body" });
|
|
211
|
+
if (bytes.byteLength > maxFileBytes) return reply.code(413).send({ error: "too_large" });
|
|
212
|
+
if (bytes.byteLength !== file.size) return reply.code(400).send({
|
|
213
|
+
error: "size_mismatch",
|
|
214
|
+
message: `This upload was offered as ${String(file.size)} bytes and ${String(bytes.byteLength)} arrived.`
|
|
215
|
+
});
|
|
216
|
+
const now = deps.nowIso();
|
|
217
|
+
const lease = new Date(Date.parse(now) + RECEIVE_LEASE_MS).toISOString();
|
|
218
|
+
if (!await storage.leaseFile(file.id, now, lease)) return reply.code(409).send({
|
|
219
|
+
error: "busy",
|
|
220
|
+
message: "Another request is sending this file's bytes, so these were not kept. Try again in a moment."
|
|
221
|
+
});
|
|
222
|
+
const answer = await receive(fileStore, file, bytes, lease, request.log).catch(async (error) => {
|
|
223
|
+
await giveBack(file.id, lease, request.log);
|
|
224
|
+
throw error;
|
|
225
|
+
});
|
|
226
|
+
if (answer.status !== 204) await giveBack(file.id, lease, request.log);
|
|
227
|
+
return reply.code(answer.status).send(answer.body);
|
|
228
|
+
});
|
|
229
|
+
});
|
|
230
|
+
/**
|
|
231
|
+
* Give a file back after a request that kept nothing, without letting the release decide
|
|
232
|
+
* the reply.
|
|
233
|
+
*
|
|
234
|
+
* A release that fails is most likely the database the request has already failed on, and
|
|
235
|
+
* its error would stand in for the one that says why the bytes were not kept — or turn a
|
|
236
|
+
* scanner's refusal into a 500. The lease runs out on its own, so all a failed release
|
|
237
|
+
* costs is a `PUT` of this same file being busy until it does.
|
|
238
|
+
*/
|
|
239
|
+
async function giveBack(id, lease, log) {
|
|
240
|
+
await storage.releaseFile(id, lease).catch((cause) => {
|
|
241
|
+
log.error({
|
|
242
|
+
event: "upload.unreleased",
|
|
243
|
+
err: cause
|
|
244
|
+
});
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Scan a file's bytes, write them and settle its row, for a request holding its lease.
|
|
249
|
+
*
|
|
250
|
+
* Returns the answer rather than sending it, so the route can give the lease back first.
|
|
251
|
+
*/
|
|
252
|
+
async function receive(store, file, bytes, lease, log) {
|
|
253
|
+
const screened = await screenUpload(deps.scanner, file, bytes);
|
|
254
|
+
if (!screened.ok) {
|
|
255
|
+
if (screened.reason === "refused") {
|
|
256
|
+
log.warn({ event: "upload.refused" });
|
|
257
|
+
return {
|
|
258
|
+
status: 422,
|
|
259
|
+
body: {
|
|
260
|
+
error: "refused_by_scanner",
|
|
261
|
+
message: `This file was refused by the deployment's virus scanner (${screened.finding}).`
|
|
262
|
+
}
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
log.error({ event: "scanner.unreachable" });
|
|
266
|
+
return {
|
|
267
|
+
status: 503,
|
|
268
|
+
body: {
|
|
269
|
+
error: "scanner_unavailable",
|
|
270
|
+
message: "This file could not be scanned just now, so it was not kept. Try again."
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
if ((await storage.getFile(file.id))?.receivingUntil !== lease || Date.parse(deps.nowIso()) >= Date.parse(lease)) return OUTLASTED;
|
|
275
|
+
await store.put(file.storageKey, bytes);
|
|
276
|
+
if (await storage.settleFile(file.id, lease)) return { status: 204 };
|
|
277
|
+
return OUTLASTED;
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Serve a file back.
|
|
281
|
+
*
|
|
282
|
+
* Always as an attachment, never inline, and `nosniff` alongside it. Stored
|
|
283
|
+
* cross-site scripting through an uploaded HTML or SVG file is the most
|
|
284
|
+
* commonly exploited vulnerability in this product category, and the real
|
|
285
|
+
* answer is a separate hostname — which a single-container deployment does
|
|
286
|
+
* not have. Forcing a download is the accommodation, and it is the one place
|
|
287
|
+
* this deployment model genuinely costs something.
|
|
288
|
+
*/
|
|
289
|
+
app.get("/f/:path/files/:id", async (request, reply) => {
|
|
290
|
+
if (fileStore === void 0) return reply.code(501).send({ error: "uploads_unavailable" });
|
|
291
|
+
const { path, id } = request.params;
|
|
292
|
+
if (await actorOf(request) === void 0) return reply.code(401).send({ error: "unauthenticated" });
|
|
293
|
+
const form = await storage.getFormByPath(path);
|
|
294
|
+
const file = await storage.getFile(id);
|
|
295
|
+
if (form === void 0 || file === void 0 || file.formId !== form.id) return reply.code(404).send({ error: "unknown_file" });
|
|
296
|
+
const stream = await fileStore.open(file.storageKey);
|
|
297
|
+
if (stream === void 0) return reply.code(404).send({ error: "unknown_file" });
|
|
298
|
+
return reply.header("content-type", "application/octet-stream").header("x-content-type-options", "nosniff").header("content-security-policy", "default-src 'none'; sandbox").header("content-disposition", attachment(file.name)).send(stream);
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
//#endregion
|
|
302
|
+
//#region src/headers.ts
|
|
303
|
+
/**
|
|
304
|
+
* The header a client declares what it was looking at with.
|
|
305
|
+
*
|
|
306
|
+
* One name, read by the submission route and by the publish plugin: a
|
|
307
|
+
* submission says which version it rendered, and a publish says which version it
|
|
308
|
+
* opened. Two spellings of one header is a bug nobody sees until a client sends
|
|
309
|
+
* the other one.
|
|
310
|
+
*/
|
|
311
|
+
const SCHEMA_HASH_HEADER = "x-formancy-schema-hash";
|
|
312
|
+
/** A solved proof-of-work challenge, as base64 JSON: the shape an ALTCHA client already produces. */
|
|
313
|
+
const CHALLENGE_HEADER = "x-formancy-challenge";
|
|
314
|
+
/**
|
|
315
|
+
* The token a response is sent with: handed out with the form, and with a draft, naming the id
|
|
316
|
+
* the response is stored under (0169). Required of an anonymous submission.
|
|
317
|
+
*/
|
|
318
|
+
const SUBMISSION_TOKEN_HEADER = "x-formancy-submission-token";
|
|
319
|
+
//#endregion
|
|
320
|
+
//#region src/routes/publish.ts
|
|
321
|
+
/**
|
|
322
|
+
* Publishing a form, as a plugin of its own.
|
|
323
|
+
*
|
|
324
|
+
* One route family per plugin, which is Fastify's own unit and what `app.ts`'s
|
|
325
|
+
* size budget named as the seam. It moved when the budget refused the next thing
|
|
326
|
+
* added to it rather than letting the number drift.
|
|
327
|
+
*/
|
|
328
|
+
async function publishRoutes(app, { deps, requires }) {
|
|
329
|
+
app.post("/forms", { preHandler: requires("form.publish") }, async (request, reply) => {
|
|
330
|
+
const body = request.body;
|
|
331
|
+
if (body === null || typeof body.path !== "string" || body.path === "" || body.schema === void 0) return reply.code(400).send({
|
|
332
|
+
error: "bad_request",
|
|
333
|
+
message: "Body needs { path, schema }."
|
|
334
|
+
});
|
|
335
|
+
const basedOn = request.headers[SCHEMA_HASH_HEADER];
|
|
336
|
+
const outcome = await publishForm(deps, {
|
|
337
|
+
path: body.path,
|
|
338
|
+
schema: body.schema,
|
|
339
|
+
actor: request.actor,
|
|
340
|
+
requestId: request.id,
|
|
341
|
+
...typeof basedOn === "string" && basedOn !== "" ? { basedOnSchemaHash: basedOn } : {}
|
|
342
|
+
});
|
|
343
|
+
if (!outcome.ok) switch (outcome.kind) {
|
|
344
|
+
case "invalid_schema": return reply.code(422).send({
|
|
345
|
+
error: outcome.kind,
|
|
346
|
+
errors: outcome.errors
|
|
347
|
+
});
|
|
348
|
+
case "unsafe_pattern": return reply.code(422).send({
|
|
349
|
+
error: outcome.kind,
|
|
350
|
+
patterns: outcome.patterns
|
|
351
|
+
});
|
|
352
|
+
case "invalid_logic": return reply.code(422).send({
|
|
353
|
+
error: outcome.kind,
|
|
354
|
+
message: outcome.message
|
|
355
|
+
});
|
|
356
|
+
case "unknown_options_source": return reply.code(422).send({
|
|
357
|
+
error: outcome.kind,
|
|
358
|
+
sources: outcome.sources
|
|
359
|
+
});
|
|
360
|
+
case "already_published": return reply.code(409).send({
|
|
361
|
+
error: outcome.kind,
|
|
362
|
+
version: outcome.version
|
|
363
|
+
});
|
|
364
|
+
case "version_changed": return reply.code(409).send({
|
|
365
|
+
error: "FORM_VERSION_CHANGED",
|
|
366
|
+
current: outcome.current === void 0 ? void 0 : {
|
|
367
|
+
version: outcome.current.version,
|
|
368
|
+
schemaHash: outcome.current.schemaHash,
|
|
369
|
+
schema: outcome.current.schema
|
|
370
|
+
}
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
return reply.code(201).send({
|
|
374
|
+
version: outcome.version,
|
|
375
|
+
schemaHash: outcome.schemaHash,
|
|
376
|
+
...outcome.warnings.length > 0 ? { warnings: outcome.warnings } : {}
|
|
377
|
+
});
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
//#endregion
|
|
381
|
+
//#region src/routes/access.ts
|
|
382
|
+
/**
|
|
383
|
+
* Who may submit a form, and from where: a property of the deployment, not of the document,
|
|
384
|
+
* so changing it mints no version and leaves every client's schema hash standing
|
|
385
|
+
* ([0044](../../../../docs/decisions/0044-access-outside-the-document.md)).
|
|
386
|
+
*
|
|
387
|
+
* One route family per plugin, the seam `app.ts`'s size budget names. It left `app.ts` when
|
|
388
|
+
* the examples kept beside a form needed a plugin registered there (0166).
|
|
389
|
+
*/
|
|
390
|
+
async function accessRoutes(app, { deps, requires, audit }) {
|
|
391
|
+
app.put("/f/:path/access", { preHandler: requires("form.publish") }, async (request, reply) => {
|
|
392
|
+
const { path } = request.params;
|
|
393
|
+
const body = request.body;
|
|
394
|
+
const submit = body?.submit;
|
|
395
|
+
if (submit !== "authenticated" && submit !== "public") return reply.code(400).send({
|
|
396
|
+
error: "invalid_access",
|
|
397
|
+
message: "submit must be \"authenticated\" or \"public\"."
|
|
398
|
+
});
|
|
399
|
+
const origins = body?.allowedOrigins;
|
|
400
|
+
if (origins !== void 0 && !(Array.isArray(origins) && origins.every((o) => typeof o === "string"))) return reply.code(400).send({
|
|
401
|
+
error: "invalid_access",
|
|
402
|
+
message: "allowedOrigins must be an array of strings, or absent for no allowlist."
|
|
403
|
+
});
|
|
404
|
+
if (!(await setFormAccess(deps, {
|
|
405
|
+
path,
|
|
406
|
+
submit,
|
|
407
|
+
...origins === void 0 ? {} : { allowedOrigins: origins }
|
|
408
|
+
})).ok) return reply.code(404).send({ error: "unknown_form" });
|
|
409
|
+
await audit(request, {
|
|
410
|
+
action: "form.access.changed",
|
|
411
|
+
subject: path,
|
|
412
|
+
detail: {
|
|
413
|
+
submit,
|
|
414
|
+
allowedOrigins: origins === void 0 ? "unchanged" : origins.join(" ")
|
|
415
|
+
}
|
|
416
|
+
});
|
|
417
|
+
return reply.code(204).send();
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
//#endregion
|
|
421
|
+
//#region src/routes/examples.ts
|
|
422
|
+
/**
|
|
423
|
+
* A form's examples, and the sample they start from, kept beside the form (0166).
|
|
424
|
+
*
|
|
425
|
+
* `GET /f/:path/examples` answers `{ scenarios, sample? }`, with `unreadable` beside them when
|
|
426
|
+
* something kept is not an example, and `PUT` replaces both with `{ scenarios, sample? }`. The routes authenticate and carry: who may read and change them, what an
|
|
427
|
+
* example is, and what a publish says about them are `server-core`'s, so the 403 here is the
|
|
428
|
+
* use-case's answer and not a `requires` of the route's own. Publishing needs nothing from
|
|
429
|
+
* here — `publishForm` reads the examples itself, and its warnings ride on the `201`.
|
|
430
|
+
*
|
|
431
|
+
* One route family per plugin, the seam `app.ts`'s size budget names.
|
|
432
|
+
*/
|
|
433
|
+
async function exampleRoutes(app, { deps, actorOf }) {
|
|
434
|
+
app.get("/f/:path/examples", async (request, reply) => {
|
|
435
|
+
const actor = await actorOf(request);
|
|
436
|
+
if (actor === void 0) return reply.code(401).send({ error: "unauthenticated" });
|
|
437
|
+
const { path } = request.params;
|
|
438
|
+
return answer$1(reply, await readExamples(deps, {
|
|
439
|
+
path,
|
|
440
|
+
actor
|
|
441
|
+
}));
|
|
442
|
+
});
|
|
443
|
+
app.put("/f/:path/examples", async (request, reply) => {
|
|
444
|
+
const actor = await actorOf(request);
|
|
445
|
+
if (actor === void 0) return reply.code(401).send({ error: "unauthenticated" });
|
|
446
|
+
const { path } = request.params;
|
|
447
|
+
return answer$1(reply, await keepExamples(deps, {
|
|
448
|
+
path,
|
|
449
|
+
actor,
|
|
450
|
+
examples: request.body,
|
|
451
|
+
requestId: request.id
|
|
452
|
+
}));
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
/** An outcome as a response: the examples as kept, or the refusal, in the plane's shapes. */
|
|
456
|
+
function answer$1(reply, outcome) {
|
|
457
|
+
if (outcome.ok) return reply.send(outcome.examples);
|
|
458
|
+
switch (outcome.kind) {
|
|
459
|
+
case "forbidden": return reply.code(403).send({
|
|
460
|
+
error: "forbidden",
|
|
461
|
+
action: "form.publish"
|
|
462
|
+
});
|
|
463
|
+
case "unknown_form": return reply.code(404).send({ error: "unknown_form" });
|
|
464
|
+
case "invalid_examples": return reply.code(422).send({
|
|
465
|
+
error: outcome.kind,
|
|
466
|
+
problems: outcome.problems
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
//#endregion
|
|
471
|
+
//#region src/routes/deliveries.ts
|
|
472
|
+
/**
|
|
473
|
+
* Webhooks as an operator sees them: their health, the deliveries that died, and
|
|
474
|
+
* sending one of those again.
|
|
475
|
+
*
|
|
476
|
+
* One route family per plugin, the seam `app.ts`'s size budget names. It moved
|
|
477
|
+
* when that file needed room to say which proxy it believes about a client.
|
|
478
|
+
*/
|
|
479
|
+
async function deliveryRoutes(app, { deps, requires, audit }) {
|
|
480
|
+
/**
|
|
481
|
+
* Webhook health, and the deliveries that died.
|
|
482
|
+
*
|
|
483
|
+
* The reason the breaker keeps its counters on the row rather than in the
|
|
484
|
+
* worker's memory. A self-hoster has no operations team watching a
|
|
485
|
+
* dashboard, so a destination refusing deliveries since Tuesday has to be
|
|
486
|
+
* answerable from the product — otherwise it is found when somebody asks
|
|
487
|
+
* why the CRM has no leads this week.
|
|
488
|
+
*/
|
|
489
|
+
app.get("/webhooks", { preHandler: requires("form.publish") }, async (_request, reply) => {
|
|
490
|
+
const now = new Date(deps.nowIso());
|
|
491
|
+
const hooks = await deps.storage.listWebhooks();
|
|
492
|
+
return reply.send({ webhooks: hooks.map((hook) => healthOf(hook, now)) });
|
|
493
|
+
});
|
|
494
|
+
app.get("/deliveries/dead", { preHandler: requires("form.publish") }, async (request, reply) => {
|
|
495
|
+
const raw = request.query?.limit;
|
|
496
|
+
const asked = typeof raw === "string" ? Number.parseInt(raw, 10) : 50;
|
|
497
|
+
const limit = Number.isFinite(asked) ? Math.min(Math.max(asked, 1), 200) : 50;
|
|
498
|
+
const dead = await deps.storage.deadDeliveries(limit);
|
|
499
|
+
return reply.send({ deliveries: dead.map((delivery) => ({
|
|
500
|
+
id: delivery.id,
|
|
501
|
+
webhookId: delivery.webhookId,
|
|
502
|
+
submissionId: delivery.submissionId,
|
|
503
|
+
eventId: delivery.eventId,
|
|
504
|
+
attempt: delivery.attempt,
|
|
505
|
+
lastError: delivery.lastError
|
|
506
|
+
})) });
|
|
507
|
+
});
|
|
508
|
+
app.post("/deliveries/:id/replay", { preHandler: requires("form.publish") }, async (request, reply) => {
|
|
509
|
+
const { id } = request.params;
|
|
510
|
+
const outcome = await replayDelivery({
|
|
511
|
+
storage: deps.storage,
|
|
512
|
+
now: () => new Date(deps.nowIso())
|
|
513
|
+
}, id);
|
|
514
|
+
if (!outcome.ok) return outcome.kind === "unknown" ? reply.code(404).send({ error: "unknown_delivery" }) : reply.code(409).send({
|
|
515
|
+
error: "not_dead",
|
|
516
|
+
state: outcome.state,
|
|
517
|
+
message: "Only a dead delivery can be replayed. A pending one is already queued, and a delivered one would be sent twice."
|
|
518
|
+
});
|
|
519
|
+
await audit(request, {
|
|
520
|
+
action: "delivery.replayed",
|
|
521
|
+
subject: id,
|
|
522
|
+
detail: { webhookId: outcome.delivery.webhookId }
|
|
523
|
+
});
|
|
524
|
+
return reply.code(202).send({
|
|
525
|
+
id,
|
|
526
|
+
state: outcome.delivery.state
|
|
527
|
+
});
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
//#endregion
|
|
531
|
+
//#region src/routes/drafts.ts
|
|
532
|
+
/** Proves the bearer started this draft. Lower-case: Fastify normalises. */
|
|
533
|
+
const DRAFT_TOKEN_HEADER = "x-formancy-draft-token";
|
|
534
|
+
/**
|
|
535
|
+
* A respondent's draft: started, written and read back, each with the key only its starter
|
|
536
|
+
* holds ([0062](../../../../docs/decisions/0062-a-draft-carries-its-own-key.md)).
|
|
537
|
+
*
|
|
538
|
+
* One route family per plugin, the seam `app.ts`'s size budget names. It moved when the
|
|
539
|
+
* deployment's model needed lines that file had no room for.
|
|
540
|
+
*/
|
|
541
|
+
async function draftRoutes(app, { deps, limit }) {
|
|
542
|
+
/**
|
|
543
|
+
* Start a draft, and get the only key to it.
|
|
544
|
+
*
|
|
545
|
+
* The public plane is anonymous, so a draft has no account behind it. It was
|
|
546
|
+
* previously addressed by an id the CALLER chose, with no check on either
|
|
547
|
+
* route: anybody who knew or guessed an id could read a stranger's part-filled
|
|
548
|
+
* form, and overwrite it — after which that person submits the substituted
|
|
549
|
+
* content under their own name and nothing says otherwise
|
|
550
|
+
* ([0062](../../../../docs/decisions/0062-a-draft-carries-its-own-key.md)).
|
|
551
|
+
*
|
|
552
|
+
* So the server picks the id and signs it, and the token comes back exactly
|
|
553
|
+
* once. Losing it means losing the draft, which is the correct trade for the
|
|
554
|
+
* alternative.
|
|
555
|
+
*/
|
|
556
|
+
app.post("/f/:path/drafts", { config: { rateLimit: limited(limit, "admit") } }, async (request, reply) => {
|
|
557
|
+
const { path } = request.params;
|
|
558
|
+
const started = await startDraft(deps, { path });
|
|
559
|
+
if (started === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
560
|
+
return reply.code(201).send(started);
|
|
561
|
+
});
|
|
562
|
+
app.put("/f/:path/drafts/:draftId", { config: { rateLimit: limited(limit, "admit") } }, async (request, reply) => {
|
|
563
|
+
const { path, draftId } = request.params;
|
|
564
|
+
const token = request.headers[DRAFT_TOKEN_HEADER];
|
|
565
|
+
if (typeof token !== "string") return reply.code(401).send({ error: "draft_token_required" });
|
|
566
|
+
const saved = await saveDraft(deps, {
|
|
567
|
+
path,
|
|
568
|
+
draftId,
|
|
569
|
+
token,
|
|
570
|
+
data: request.body ?? {}
|
|
571
|
+
});
|
|
572
|
+
if (saved === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
573
|
+
if (!saved.saved) return reply.code(403).send({ error: "draft_token_invalid" });
|
|
574
|
+
return reply.send({ version: saved.version });
|
|
575
|
+
});
|
|
576
|
+
app.get("/f/:path/drafts/:draftId", { config: { rateLimit: limited(limit, "admit") } }, async (request, reply) => {
|
|
577
|
+
const { path, draftId } = request.params;
|
|
578
|
+
const token = request.headers[DRAFT_TOKEN_HEADER];
|
|
579
|
+
if (typeof token !== "string") return reply.code(401).send({ error: "draft_token_required" });
|
|
580
|
+
const resumed = await resumeDraft(deps, {
|
|
581
|
+
path,
|
|
582
|
+
draftId,
|
|
583
|
+
token
|
|
584
|
+
});
|
|
585
|
+
if (resumed === void 0) return reply.code(404).send({ error: "unknown_draft" });
|
|
586
|
+
return reply.send(resumed);
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
//#endregion
|
|
590
|
+
//#region src/routes/model.ts
|
|
591
|
+
/**
|
|
592
|
+
* The largest request the route reads, in bytes, set on the route rather than inherited.
|
|
593
|
+
*
|
|
594
|
+
* Every byte is an input token the operator pays for, so the cap is the route's own:
|
|
595
|
+
* raising the server's limit for something else does not raise what an editor can send a
|
|
596
|
+
* model. It has to hold what the builders send about a form the server publishes, and the
|
|
597
|
+
* largest of that is a translation, not the form: each answer's row repeats the question
|
|
598
|
+
* it answers as its context, pretty-printed and escaped again as a string.
|
|
599
|
+
* `model-route.test.ts` sends all three requests about the largest questionnaire the server's
|
|
600
|
+
* default limit publishes, its questions a sentence long, and they are taken. Longer
|
|
601
|
+
* questions over many answers make a translation past this, refused before it is read (0165).
|
|
602
|
+
*/
|
|
603
|
+
const MODEL_BODY_LIMIT_BYTES = 1048576;
|
|
604
|
+
/**
|
|
605
|
+
* The builders' requests to the deployment's model (0165).
|
|
606
|
+
*
|
|
607
|
+
* `GET /model` says whether there is one, and which, to somebody who may edit a form: the
|
|
608
|
+
* admin draws the prompt pane only when there is. `POST /model/complete` takes
|
|
609
|
+
* `{ kind, user }` and answers `{ text }` — the model's answer, for the browser to check
|
|
610
|
+
* — or `{ declined }` when the provider refused, which the admin's `AskModel` hands back as
|
|
611
|
+
* a decline (0158). Everything else is an error with a sentence for the person.
|
|
612
|
+
*
|
|
613
|
+
* One route family per plugin, the seam `app.ts`'s size budget names.
|
|
614
|
+
*/
|
|
615
|
+
async function modelRoutes(app, { model, requires, audit, limit }) {
|
|
616
|
+
app.get("/model", { preHandler: requires("form.publish") }, async (_request, reply) => {
|
|
617
|
+
if (model === void 0) return reply.code(404).send({ error: "model_not_configured" });
|
|
618
|
+
return reply.send({
|
|
619
|
+
provider: model.provider,
|
|
620
|
+
model: model.model
|
|
621
|
+
});
|
|
622
|
+
});
|
|
623
|
+
app.post("/model/complete", {
|
|
624
|
+
preHandler: requires("form.publish"),
|
|
625
|
+
bodyLimit: MODEL_BODY_LIMIT_BYTES,
|
|
626
|
+
config: { rateLimit: {
|
|
627
|
+
...limited(limit, "refuse"),
|
|
628
|
+
hook: "preHandler",
|
|
629
|
+
keyGenerator: (request) => {
|
|
630
|
+
const actor = request.actor;
|
|
631
|
+
return actor === void 0 ? `ip:${request.ip}` : `${actor.kind}:${actor.id}`;
|
|
632
|
+
}
|
|
633
|
+
} }
|
|
634
|
+
}, async (request, reply) => {
|
|
635
|
+
if (model === void 0) return reply.code(404).send({ error: "model_not_configured" });
|
|
636
|
+
const body = request.body ?? {};
|
|
637
|
+
const outcome = await completeBuilderRequest(model.completer, body, cancelledWith(reply));
|
|
638
|
+
if (outcome.ok || outcome.failure !== "unknown_kind" && outcome.failure !== "no_user") await audit(request, {
|
|
639
|
+
action: "model.asked",
|
|
640
|
+
subject: String(body.kind),
|
|
641
|
+
detail: {
|
|
642
|
+
provider: model.provider,
|
|
643
|
+
model: model.model,
|
|
644
|
+
characters: String(body.user).length,
|
|
645
|
+
outcome: outcome.ok ? "answered" : outcome.failure,
|
|
646
|
+
...!outcome.ok && outcome.failure === "unavailable" && outcome.status !== void 0 ? { status: outcome.status } : {}
|
|
647
|
+
}
|
|
648
|
+
});
|
|
649
|
+
return answer(reply, request, outcome);
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
/**
|
|
653
|
+
* The browser going away, as a cancellation: the response closing before it was written.
|
|
654
|
+
*
|
|
655
|
+
* Every response closes, so one that finished is not a browser that left; only a close
|
|
656
|
+
* with nothing written yet tells the adapter to abandon the request at the provider.
|
|
657
|
+
*
|
|
658
|
+
* A browser can also have gone before the handler ran, while the session or an API key was
|
|
659
|
+
* still being checked. That response has closed already and will not close again, so it
|
|
660
|
+
* is read as gone from the start, and the adapter is told at once — before it sends.
|
|
661
|
+
*/
|
|
662
|
+
function cancelledWith(reply) {
|
|
663
|
+
let gone = reply.raw.destroyed && !reply.raw.writableFinished;
|
|
664
|
+
const listeners = [];
|
|
665
|
+
reply.raw.once("close", () => {
|
|
666
|
+
if (reply.raw.writableFinished) return;
|
|
667
|
+
gone = true;
|
|
668
|
+
for (const listener of listeners.splice(0)) listener();
|
|
669
|
+
});
|
|
670
|
+
return { onCancel(listener) {
|
|
671
|
+
if (gone) listener();
|
|
672
|
+
else listeners.push(listener);
|
|
673
|
+
} };
|
|
674
|
+
}
|
|
675
|
+
/** An outcome as a response, every failure with a sentence for the person reading it. */
|
|
676
|
+
function answer(reply, request, outcome) {
|
|
677
|
+
if (outcome.ok) return reply.send({ text: outcome.text });
|
|
678
|
+
switch (outcome.failure) {
|
|
679
|
+
case "unknown_kind": return reply.code(400).send({
|
|
680
|
+
error: "unknown_kind",
|
|
681
|
+
message: `kind must be one of ${MODEL_REQUEST_KINDS.join(", ")}: the requests formancy makes of a model.`
|
|
682
|
+
});
|
|
683
|
+
case "no_user": return reply.code(400).send({
|
|
684
|
+
error: "bad_request",
|
|
685
|
+
message: "Body needs { kind, user }, user being the text to ask."
|
|
686
|
+
});
|
|
687
|
+
case "refused": return reply.send({ declined: outcome.reason });
|
|
688
|
+
case "truncated": return reply.code(502).send({
|
|
689
|
+
error: "model_truncated",
|
|
690
|
+
message: "The model’s answer reached its length limit and was cut off, so it was not used."
|
|
691
|
+
});
|
|
692
|
+
case "cancelled": return reply.code(499).send({ error: "cancelled" });
|
|
693
|
+
case "unavailable":
|
|
694
|
+
request.log.error({
|
|
695
|
+
event: "model.unreachable",
|
|
696
|
+
upstream: outcome.status
|
|
697
|
+
});
|
|
698
|
+
return reply.code(502).send({
|
|
699
|
+
error: "model_unavailable",
|
|
700
|
+
message: unavailableSentence(outcome.status)
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
/** What a provider's status means for the person whose turn it ended. */
|
|
705
|
+
function unavailableSentence(status) {
|
|
706
|
+
if (status === 401 || status === 403) return "The model service refused this server’s key.";
|
|
707
|
+
if (status === 429) return "The model service is limiting this server’s requests. Try again in a while.";
|
|
708
|
+
if (status === 400 || status === 404) return "The model service refused the request: the model this server names may not exist, or may not take it.";
|
|
709
|
+
return "The model service could not be reached, or answered with an error.";
|
|
710
|
+
}
|
|
711
|
+
//#endregion
|
|
712
|
+
//#region src/routes/submissions.ts
|
|
713
|
+
/**
|
|
714
|
+
* What a respondent's browser does with a form: read it, ask for a challenge, and submit it.
|
|
715
|
+
*
|
|
716
|
+
* One route family per plugin, the seam `app.ts`'s size budget names. It moved when the
|
|
717
|
+
* submission token needed lines that file had no room for.
|
|
718
|
+
*/
|
|
719
|
+
async function submissionRoutes(app, { deps, challengeSecret, limit, actorOf }) {
|
|
720
|
+
const submissionLimit = limit;
|
|
721
|
+
/**
|
|
722
|
+
* Fetch a published form. The one public route deliberately NOT rate-limited.
|
|
723
|
+
*
|
|
724
|
+
* Every other unauthenticated route is: submitting, starting a draft, writing
|
|
725
|
+
* one, reading one, minting a challenge, asking for an upload target. Those are
|
|
726
|
+
* writes, or guesses, or work somebody can demand of the server. This is the
|
|
727
|
+
* read every visitor has to make before they can do anything at all.
|
|
728
|
+
*
|
|
729
|
+
* Limiting it by IP would refuse the form to real people sharing an address —
|
|
730
|
+
* an office, a school, a phone network behind CGNAT — and the failure would
|
|
731
|
+
* look like the form being broken rather than like a limit. That is a worse
|
|
732
|
+
* outcome than the cheap read it would prevent.
|
|
733
|
+
*
|
|
734
|
+
* Written down because the asymmetry is deliberate and looks like an omission.
|
|
735
|
+
*
|
|
736
|
+
* It also hands out the token the response is sent with, and that changes neither half of
|
|
737
|
+
* the argument: minting one is an HMAC and writes nothing, and it is not a defence against
|
|
738
|
+
* anybody who can read the form, so limiting it would buy nothing (0169). What it does change
|
|
739
|
+
* is that the reply is per reading: `no-store`, because a cache that kept it would hand one
|
|
740
|
+
* token to everybody behind it, and only the first of them could send a response.
|
|
741
|
+
*/
|
|
742
|
+
app.get("/f/:path", async (request, reply) => {
|
|
743
|
+
const { path } = request.params;
|
|
744
|
+
const form = await formToFill(deps, path);
|
|
745
|
+
if (form === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
746
|
+
return reply.header("cache-control", "no-store").send({
|
|
747
|
+
version: form.version,
|
|
748
|
+
schemaHash: form.schemaHash,
|
|
749
|
+
schema: form.schema,
|
|
750
|
+
submissionToken: form.submissionToken
|
|
751
|
+
});
|
|
752
|
+
});
|
|
753
|
+
/**
|
|
754
|
+
* A puzzle for an anonymous visitor to solve before submitting.
|
|
755
|
+
*
|
|
756
|
+
* Public because the form is: requiring a session to obtain a challenge for
|
|
757
|
+
* an unauthenticated submission would be a circle. It is cheap to mint and
|
|
758
|
+
* stateless, so handing one out costs this server a hash.
|
|
759
|
+
*/
|
|
760
|
+
app.get("/f/:path/challenge", { config: { rateLimit: limited(submissionLimit, "admit") } }, async (request, reply) => {
|
|
761
|
+
if (challengeSecret === void 0) return reply.code(404).send({ error: "challenge_not_enabled" });
|
|
762
|
+
const { path } = request.params;
|
|
763
|
+
if (await deps.storage.getFormByPath(path) === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
764
|
+
return reply.send(await mintChallenge({
|
|
765
|
+
secret: challengeSecret,
|
|
766
|
+
nowSeconds: Math.floor(new Date(deps.nowIso()).getTime() / 1e3)
|
|
767
|
+
}));
|
|
768
|
+
});
|
|
769
|
+
app.post("/f/:path/submissions", { config: { rateLimit: limited(submissionLimit, "admit") } }, async (request, reply) => {
|
|
770
|
+
const { path } = request.params;
|
|
771
|
+
const declaredSchemaHash = request.headers[SCHEMA_HASH_HEADER];
|
|
772
|
+
if (typeof declaredSchemaHash !== "string" || declaredSchemaHash === "") return reply.code(400).send({
|
|
773
|
+
error: "missing_schema_hash",
|
|
774
|
+
message: `Send the schema hash you rendered in the ${SCHEMA_HASH_HEADER} header.`
|
|
775
|
+
});
|
|
776
|
+
const actor = await actorOf(request);
|
|
777
|
+
const origin = request.headers.origin;
|
|
778
|
+
const token = request.headers[SUBMISSION_TOKEN_HEADER];
|
|
779
|
+
if (challengeSecret !== void 0 && actor === void 0) {
|
|
780
|
+
const refusal = await checkChallenge(deps, challengeSecret, request);
|
|
781
|
+
if (refusal !== void 0) return reply.code(400).send(refusal);
|
|
782
|
+
}
|
|
783
|
+
const outcome = await createSubmission(deps, {
|
|
784
|
+
path,
|
|
785
|
+
declaredSchemaHash,
|
|
786
|
+
requestId: request.id,
|
|
787
|
+
data: request.body ?? {},
|
|
788
|
+
actor: actor === void 0 ? "anonymous" : "authenticated",
|
|
789
|
+
...typeof origin === "string" ? { origin } : {},
|
|
790
|
+
...typeof token === "string" ? { token } : {}
|
|
791
|
+
});
|
|
792
|
+
if (outcome.ok) return reply.code(201).send({
|
|
793
|
+
id: outcome.id,
|
|
794
|
+
data: outcome.canonicalData
|
|
795
|
+
});
|
|
796
|
+
switch (outcome.kind) {
|
|
797
|
+
case "unknown_form": return reply.code(404).send({ error: "unknown_form" });
|
|
798
|
+
case "version_changed": return reply.code(409).send({
|
|
799
|
+
error: "FORM_VERSION_CHANGED",
|
|
800
|
+
current: outcome.current === void 0 ? void 0 : {
|
|
801
|
+
version: outcome.current.version,
|
|
802
|
+
schemaHash: outcome.current.schemaHash,
|
|
803
|
+
schema: outcome.current.schema
|
|
804
|
+
}
|
|
805
|
+
});
|
|
806
|
+
case "invalid": return reply.code(422).send({
|
|
807
|
+
error: "invalid",
|
|
808
|
+
errors: outcome.errors
|
|
809
|
+
});
|
|
810
|
+
case "forbidden": return reply.code(403).send({ error: "forbidden" });
|
|
811
|
+
case "source_unavailable": return reply.code(503).send({
|
|
812
|
+
error: "source_unavailable",
|
|
813
|
+
source: outcome.source
|
|
814
|
+
});
|
|
815
|
+
case "token_required": return reply.code(400).send({
|
|
816
|
+
error: "submission_token_required",
|
|
817
|
+
message: `This form was sent without the token it was handed out with. Read the form again and send its submissionToken in ${SUBMISSION_TOKEN_HEADER}.`
|
|
818
|
+
});
|
|
819
|
+
case "token_invalid": return reply.code(400).send({
|
|
820
|
+
error: "submission_token_invalid",
|
|
821
|
+
message: "This form was sent with a token the server does not recognise. Read the form again for a new one, and send the answers with it."
|
|
822
|
+
});
|
|
823
|
+
case "token_spent": return reply.code(409).send({
|
|
824
|
+
error: "submission_token_spent",
|
|
825
|
+
id: outcome.id,
|
|
826
|
+
message: "This response has already been sent, and is stored. It was not stored again."
|
|
827
|
+
});
|
|
828
|
+
}
|
|
829
|
+
});
|
|
830
|
+
}
|
|
831
|
+
/**
|
|
832
|
+
* Check a submitted solution, and spend it.
|
|
833
|
+
*
|
|
834
|
+
* Returns the refusal to send, or undefined when it passed. The spend is
|
|
835
|
+
* separate from the verification and comes last: the stateless checks say
|
|
836
|
+
* the solution is correct, and only the database can say it has not been
|
|
837
|
+
* used — a correct solution stays correct, so without this one puzzle
|
|
838
|
+
* would buy a thousand submissions.
|
|
839
|
+
*/
|
|
840
|
+
async function checkChallenge(deps, challengeSecret, request) {
|
|
841
|
+
const header = request.headers[CHALLENGE_HEADER];
|
|
842
|
+
if (typeof header !== "string" || header === "") return {
|
|
843
|
+
error: "challenge_required",
|
|
844
|
+
message: `This form needs a solved challenge. Ask GET /f/:path/challenge for one and return it in ${CHALLENGE_HEADER}.`
|
|
845
|
+
};
|
|
846
|
+
const solution = decodeSolution(header);
|
|
847
|
+
if (solution === void 0) return {
|
|
848
|
+
error: "challenge_malformed",
|
|
849
|
+
message: "The challenge header is not base64 JSON."
|
|
850
|
+
};
|
|
851
|
+
const nowSeconds = Math.floor(new Date(deps.nowIso()).getTime() / 1e3);
|
|
852
|
+
const verified = await verifySolution(challengeSecret, solution, nowSeconds);
|
|
853
|
+
if (!verified.ok) return {
|
|
854
|
+
error: `challenge_${verified.reason}`,
|
|
855
|
+
message: "The challenge was not solved. Ask for a new one and try again."
|
|
856
|
+
};
|
|
857
|
+
if (!await deps.storage.spendChallenge(verified.challenge, (/* @__PURE__ */ new Date(verified.expiresAtSeconds * 1e3)).toISOString())) return {
|
|
858
|
+
error: "challenge_spent",
|
|
859
|
+
message: "That challenge has already been used. Each one is good for one submission."
|
|
860
|
+
};
|
|
861
|
+
}
|
|
862
|
+
//#endregion
|
|
863
|
+
//#region src/auth-runtime.ts
|
|
864
|
+
/**
|
|
865
|
+
* The real cryptography, confined to the composition root: argon2id for
|
|
866
|
+
* secrets at rest, an HMAC-signed short-lived access token for sessions, and
|
|
867
|
+
* a CSPRNG for API keys. server-core never sees any of this — it is handed
|
|
868
|
+
* hash/verify/random functions and stays deterministic under test.
|
|
869
|
+
*
|
|
870
|
+
* HS256 rather than EdDSA for the session token, deliberately for this phase:
|
|
871
|
+
* one secret in one env var is the self-hosting story a single-container
|
|
872
|
+
* deployment can operate, and the signing scheme is an implementation detail
|
|
873
|
+
* of a token that lives fifteen minutes. Revisit with key rotation.
|
|
874
|
+
*/
|
|
875
|
+
function realSecretHashing() {
|
|
876
|
+
return {
|
|
877
|
+
hashSecret: (secret) => hash(secret),
|
|
878
|
+
verifySecret: async (secret, digest) => {
|
|
879
|
+
try {
|
|
880
|
+
return await verify(digest, secret);
|
|
881
|
+
} catch {
|
|
882
|
+
return false;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
};
|
|
886
|
+
}
|
|
887
|
+
function realRandomToken() {
|
|
888
|
+
return randomBytes(30).toString("base64url");
|
|
889
|
+
}
|
|
890
|
+
const TOKEN_LIFETIME = "15m";
|
|
891
|
+
function createSessionTokens(secret) {
|
|
892
|
+
if (secret.length < 32) throw new Error("FORMANCY_AUTH_SECRET must be at least 32 characters. Generate one: openssl rand -base64 33");
|
|
893
|
+
const key = new TextEncoder().encode(secret);
|
|
894
|
+
return {
|
|
895
|
+
async issue(actor) {
|
|
896
|
+
return new SignJWT({
|
|
897
|
+
role: actor.role,
|
|
898
|
+
kind: actor.kind
|
|
899
|
+
}).setProtectedHeader({ alg: "HS256" }).setSubject(actor.id).setIssuedAt().setExpirationTime(TOKEN_LIFETIME).sign(key);
|
|
900
|
+
},
|
|
901
|
+
async verify(token) {
|
|
902
|
+
try {
|
|
903
|
+
const { payload } = await jwtVerify(token, key, { algorithms: ["HS256"] });
|
|
904
|
+
const role = payload["role"];
|
|
905
|
+
const kind = payload["kind"];
|
|
906
|
+
if (typeof payload.sub !== "string" || role !== "admin" && role !== "editor" && role !== "viewer" || kind !== "user" && kind !== "apiKey") return;
|
|
907
|
+
return {
|
|
908
|
+
id: payload.sub,
|
|
909
|
+
role,
|
|
910
|
+
kind
|
|
911
|
+
};
|
|
912
|
+
} catch {
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
};
|
|
917
|
+
}
|
|
918
|
+
//#endregion
|
|
919
|
+
//#region src/upload-settings.ts
|
|
920
|
+
/**
|
|
921
|
+
* How large a file this deployment accepts.
|
|
922
|
+
*
|
|
923
|
+
* Its own module because `main.ts` is a composition root that connects to a database
|
|
924
|
+
* when it is imported, so nothing in it can be tested — and this is a setting whose wrong
|
|
925
|
+
* values all look plausible.
|
|
926
|
+
*/
|
|
927
|
+
/** Ten megabytes: what the documentation and `.env.example` give as the default. */
|
|
928
|
+
const DEFAULT_MAX_FILE_BYTES = 10485760;
|
|
929
|
+
/**
|
|
930
|
+
* The largest ceiling a deployment may set: the largest number `files.size` can hold.
|
|
931
|
+
*
|
|
932
|
+
* That column is a PostgreSQL `integer` (`db.ts`), a signed 32-bit number. Every size an
|
|
933
|
+
* offer accepts is written to it, so a ceiling above this would let an offer through the
|
|
934
|
+
* route and fail it at the insert. Raising it means widening the column first;
|
|
935
|
+
* `server.integration.test.ts` reads the column's precision from the database and fails
|
|
936
|
+
* when this number and the column disagree.
|
|
937
|
+
*/
|
|
938
|
+
const LARGEST_MAX_FILE_BYTES = 2147483647;
|
|
939
|
+
/**
|
|
940
|
+
* `value`, once it is a ceiling the server can honour: a whole number of bytes, at least
|
|
941
|
+
* one, and no more than `files.size` holds. Throws naming `setting`, because the message
|
|
942
|
+
* is all an operator has to go on.
|
|
943
|
+
*/
|
|
944
|
+
function checkedMaxFileBytes(value, setting) {
|
|
945
|
+
if (!Number.isInteger(value) || value < 1 || value > 2147483647) throw new Error(`${setting} must be a whole number of bytes from 1 to ${String(LARGEST_MAX_FILE_BYTES)}, the largest size the files table holds.`);
|
|
946
|
+
return value;
|
|
947
|
+
}
|
|
948
|
+
/**
|
|
949
|
+
* The ceiling `FORMANCY_MAX_FILE_BYTES` sets, or the default when it is unset.
|
|
950
|
+
*
|
|
951
|
+
* Empty is refused rather than read as unset: the compose files already substitute the
|
|
952
|
+
* default for an empty value, so an empty one here was set by hand, to something.
|
|
953
|
+
*/
|
|
954
|
+
function maxFileBytesFrom(env) {
|
|
955
|
+
const raw = env["FORMANCY_MAX_FILE_BYTES"];
|
|
956
|
+
if (raw === void 0) return DEFAULT_MAX_FILE_BYTES;
|
|
957
|
+
return checkedMaxFileBytes(Number(raw), "FORMANCY_MAX_FILE_BYTES");
|
|
958
|
+
}
|
|
959
|
+
//#endregion
|
|
960
|
+
//#region src/server-log.ts
|
|
961
|
+
/**
|
|
962
|
+
* The server's log: a JSON line for every request, one for every error that answered one, and
|
|
963
|
+
* one for each thing a route or a background worker has to say
|
|
964
|
+
* ([0168](../../../docs/decisions/0168-the-log-is-built-from-a-list-of-fields.md)).
|
|
965
|
+
*
|
|
966
|
+
* **What keeps everything else out is construction, not a filter over text.** A line is made
|
|
967
|
+
* from the fields `LOG_FIELDS` names, each kept only when its value is of that field's kind,
|
|
968
|
+
* and from nothing else. A key a call hands over that is not on the list is never read, so a
|
|
969
|
+
* body, a header, a query string, an answer, a password, a draft's key or a file's name has no
|
|
970
|
+
* field to go in. The words of a call are never written at all — pino's message argument,
|
|
971
|
+
* Fastify's `error.message`, its `Route GET:/the/raw/url not found` — so neither are the
|
|
972
|
+
* parameters a database error quotes: a line says what happened with an `event` from
|
|
973
|
+
* `LOG_EVENTS`, and a thrown error is written as its kind and its code.
|
|
974
|
+
*
|
|
975
|
+
* Its own logger rather than pino's options, because pino's rule for what to keep would be
|
|
976
|
+
* three hooks — a log formatter, a bindings formatter and a method hook to drop the message —
|
|
977
|
+
* whose order is pino's internals; here it is one function, `written`, and a list.
|
|
978
|
+
*/
|
|
979
|
+
/** Most severe first: `fatal` writes the least, `trace` the most. The names are pino's. */
|
|
980
|
+
const LOG_LEVELS = [
|
|
981
|
+
"fatal",
|
|
982
|
+
"error",
|
|
983
|
+
"warn",
|
|
984
|
+
"info",
|
|
985
|
+
"debug",
|
|
986
|
+
"trace"
|
|
987
|
+
];
|
|
988
|
+
const SEVERITY = {
|
|
989
|
+
fatal: 60,
|
|
990
|
+
error: 50,
|
|
991
|
+
warn: 40,
|
|
992
|
+
info: 30,
|
|
993
|
+
debug: 20,
|
|
994
|
+
trace: 10
|
|
995
|
+
};
|
|
996
|
+
/** What a line can say happened. Anything else is `unlisted`, or not written. */
|
|
997
|
+
const LOG_EVENTS = [
|
|
998
|
+
"request",
|
|
999
|
+
"request.abandoned",
|
|
1000
|
+
"request.refused",
|
|
1001
|
+
"request.failed",
|
|
1002
|
+
"audit.unwritten",
|
|
1003
|
+
"upload.refused",
|
|
1004
|
+
"upload.unreleased",
|
|
1005
|
+
"scanner.unreachable",
|
|
1006
|
+
"model.unreachable",
|
|
1007
|
+
"outbox.failed",
|
|
1008
|
+
"collector.failed",
|
|
1009
|
+
"sweeper.failed",
|
|
1010
|
+
"ratelimit.unanswered",
|
|
1011
|
+
"ratelimit.answering",
|
|
1012
|
+
"ratelimit.sweep.failed",
|
|
1013
|
+
"database.notice",
|
|
1014
|
+
"unlisted"
|
|
1015
|
+
];
|
|
1016
|
+
/**
|
|
1017
|
+
* Every field a line can have, in the order it is written, each with the kind of value it
|
|
1018
|
+
* holds. A field is kept only when its value is of its kind, so a string in a number's place,
|
|
1019
|
+
* or a sentence in an identifier's, is dropped rather than written.
|
|
1020
|
+
*/
|
|
1021
|
+
const KINDS = {
|
|
1022
|
+
time: (value) => typeof value === "string",
|
|
1023
|
+
level: (value) => LOG_LEVELS.includes(value),
|
|
1024
|
+
event: (value) => LOG_EVENTS.includes(value),
|
|
1025
|
+
reqId: (value) => typeof value === "string" && /^[\w-]{1,64}$/.test(value),
|
|
1026
|
+
method: (value) => typeof value === "string" && METHODS.includes(value),
|
|
1027
|
+
route: (value) => typeof value === "string" && /^\/[\w\-./:*]*$/.test(value),
|
|
1028
|
+
status: isStatus,
|
|
1029
|
+
ms: (value) => typeof value === "number" && Number.isFinite(value) && value >= 0,
|
|
1030
|
+
kind: isIdentifier,
|
|
1031
|
+
code: isCode,
|
|
1032
|
+
action: (value) => AUDIT_ACTIONS.includes(value),
|
|
1033
|
+
upstream: isStatus
|
|
1034
|
+
};
|
|
1035
|
+
const LOG_FIELDS = Object.keys(KINDS);
|
|
1036
|
+
/**
|
|
1037
|
+
* The one place a line becomes text: dropped below `level`, and otherwise each listed field
|
|
1038
|
+
* whose value is of its kind, in the listed order.
|
|
1039
|
+
*/
|
|
1040
|
+
function written(sink, level) {
|
|
1041
|
+
const threshold = SEVERITY[level];
|
|
1042
|
+
return (line) => {
|
|
1043
|
+
if (SEVERITY[line.level] < threshold) return;
|
|
1044
|
+
const kept = {};
|
|
1045
|
+
const given = {
|
|
1046
|
+
...line,
|
|
1047
|
+
time: (/* @__PURE__ */ new Date()).toISOString()
|
|
1048
|
+
};
|
|
1049
|
+
for (const field of LOG_FIELDS) if (KINDS[field](given[field])) kept[field] = given[field];
|
|
1050
|
+
sink.write(`${JSON.stringify(kept)}\n`);
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
1053
|
+
/** A log whose every line is built by the rule above. What `createApp` hands Fastify. */
|
|
1054
|
+
function createServerLog(sink, level) {
|
|
1055
|
+
return logger(written(sink, level), level, void 0);
|
|
1056
|
+
}
|
|
1057
|
+
/**
|
|
1058
|
+
* Fastify's logger interface over the rule: each call contributes its event, its error, and
|
|
1059
|
+
* the status, code, action and upstream status a caller may name, and nothing else it was
|
|
1060
|
+
* handed. A
|
|
1061
|
+
* child keeps the request id it is bound to, which is how Fastify gives a request's lines its
|
|
1062
|
+
* id, and nothing else bound beside it.
|
|
1063
|
+
*/
|
|
1064
|
+
function logger(write, level, reqId) {
|
|
1065
|
+
const at = (lineLevel) => (first) => {
|
|
1066
|
+
const given = first instanceof Error ? { err: first } : isRecord$1(first) ? first : {};
|
|
1067
|
+
const event = given["event"] !== "unlisted" && KINDS.event(given["event"]) ? given["event"] : void 0;
|
|
1068
|
+
if (event === void 0 && SEVERITY[lineLevel] < SEVERITY.warn) return;
|
|
1069
|
+
write({
|
|
1070
|
+
level: lineLevel,
|
|
1071
|
+
event: event ?? "unlisted",
|
|
1072
|
+
reqId,
|
|
1073
|
+
status: given["status"],
|
|
1074
|
+
code: given["code"],
|
|
1075
|
+
...described(given["err"]),
|
|
1076
|
+
action: given["action"],
|
|
1077
|
+
upstream: given["upstream"]
|
|
1078
|
+
});
|
|
1079
|
+
};
|
|
1080
|
+
return {
|
|
1081
|
+
level,
|
|
1082
|
+
fatal: at("fatal"),
|
|
1083
|
+
error: at("error"),
|
|
1084
|
+
warn: at("warn"),
|
|
1085
|
+
info: at("info"),
|
|
1086
|
+
debug: at("debug"),
|
|
1087
|
+
trace: at("trace"),
|
|
1088
|
+
silent: () => void 0,
|
|
1089
|
+
child: (bindings) => logger(write, level, KINDS.reqId(bindings["reqId"]) ? bindings["reqId"] : reqId)
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
/**
|
|
1093
|
+
* When Fastify writes about a request, written as this log's lines: Fastify's own extension
|
|
1094
|
+
* point for its internal lines, rather than hooks beside a logger that also writes them.
|
|
1095
|
+
*
|
|
1096
|
+
* One line when a request is answered, not one when it arrives as well — half the lines,
|
|
1097
|
+
* and the one that has the status. A route nobody registered gets no line of its own: its
|
|
1098
|
+
* request line says 404 and names no route.
|
|
1099
|
+
*
|
|
1100
|
+
* **Every request, including the ones Fastify answers past its own request line.** Fastify
|
|
1101
|
+
* writes that line when an answer has been sent, so three kinds left none: a request whose
|
|
1102
|
+
* client went away first, whose answer is never sent; one refused for an unreadable URL or an
|
|
1103
|
+
* over-long parameter, answered by hand before routing unless `frameworkErrors` is set; and
|
|
1104
|
+
* one that arrives while the server closes, refused with a `503` before routing and said at
|
|
1105
|
+
* `info` in words, which this log drops. Each is written here as well.
|
|
1106
|
+
*/
|
|
1107
|
+
var RequestLines = class extends LogController {
|
|
1108
|
+
#write;
|
|
1109
|
+
constructor(write) {
|
|
1110
|
+
super();
|
|
1111
|
+
this.#write = write;
|
|
1112
|
+
}
|
|
1113
|
+
/**
|
|
1114
|
+
* Nothing written on arrival; but a response that closes before it has finished sending is
|
|
1115
|
+
* a client that left. Read from the response's own `finish`, which a socket and an injected
|
|
1116
|
+
* request both emit, rather than `writableFinished`, which an injected one never sets.
|
|
1117
|
+
*/
|
|
1118
|
+
incomingRequest(request, reply) {
|
|
1119
|
+
let sent = false;
|
|
1120
|
+
reply.raw.once("finish", () => sent = true);
|
|
1121
|
+
reply.raw.once("close", () => {
|
|
1122
|
+
if (sent) return;
|
|
1123
|
+
this.#write({
|
|
1124
|
+
level: "info",
|
|
1125
|
+
event: "request.abandoned",
|
|
1126
|
+
...where(request),
|
|
1127
|
+
ms: Math.round(reply.elapsedTime)
|
|
1128
|
+
});
|
|
1129
|
+
});
|
|
1130
|
+
}
|
|
1131
|
+
routeNotFound() {}
|
|
1132
|
+
/** At `error` for a 5xx, thrown or answered by a route, so a level kept for failures keeps it. */
|
|
1133
|
+
requestCompleted(error, request, reply) {
|
|
1134
|
+
this.#write({
|
|
1135
|
+
level: error != null || reply.statusCode >= 500 ? "error" : "info",
|
|
1136
|
+
event: "request",
|
|
1137
|
+
...where(request),
|
|
1138
|
+
status: reply.statusCode,
|
|
1139
|
+
ms: Math.round(reply.elapsedTime),
|
|
1140
|
+
...described(error)
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
/**
|
|
1144
|
+
* A request refused before it was routed, because its URL cannot be decoded or a parameter
|
|
1145
|
+
* is longer than the router takes: answered through the reply, so the error handler names it
|
|
1146
|
+
* as it names any refusal, and its request line written once the answer has gone. Fastify
|
|
1147
|
+
* calls this `frameworkErrors` and starts no clock for it, so `ms` is 0.
|
|
1148
|
+
*/
|
|
1149
|
+
refusedBeforeRouting = (error, request, reply) => {
|
|
1150
|
+
reply.raw.once("finish", () => this.requestCompleted(null, request, reply));
|
|
1151
|
+
reply.send(error);
|
|
1152
|
+
};
|
|
1153
|
+
/** Words and no event from Fastify, so written here: the `503` and the request's id. */
|
|
1154
|
+
serviceUnavailable(logger) {
|
|
1155
|
+
logger.error({
|
|
1156
|
+
event: "request",
|
|
1157
|
+
status: 503
|
|
1158
|
+
});
|
|
1159
|
+
}
|
|
1160
|
+
defaultErrorLog(error, request, reply) {
|
|
1161
|
+
const failed = reply.statusCode >= 500;
|
|
1162
|
+
this.#write({
|
|
1163
|
+
level: failed ? "error" : "info",
|
|
1164
|
+
event: failed ? "request.failed" : "request.refused",
|
|
1165
|
+
...where(request),
|
|
1166
|
+
status: reply.statusCode,
|
|
1167
|
+
...described(error)
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1170
|
+
};
|
|
1171
|
+
function where(request) {
|
|
1172
|
+
return {
|
|
1173
|
+
reqId: request.id,
|
|
1174
|
+
method: request.method,
|
|
1175
|
+
route: request.routeOptions.url
|
|
1176
|
+
};
|
|
1177
|
+
}
|
|
1178
|
+
/**
|
|
1179
|
+
* Fastify's options for a server that keeps this log, or none for one that keeps no log —
|
|
1180
|
+
* `createApp` given no `log`, which is how a host embedding it keeps its own output (0115).
|
|
1181
|
+
* Without one, Fastify answers what it refuses before routing by hand, as it always has; with
|
|
1182
|
+
* one, through the reply so the refusal can be written — the same status and the same fields.
|
|
1183
|
+
*/
|
|
1184
|
+
function serverLogOptions(log) {
|
|
1185
|
+
if (log === void 0) return {};
|
|
1186
|
+
const write = written(log.sink, log.level);
|
|
1187
|
+
const lines = new RequestLines(write);
|
|
1188
|
+
return {
|
|
1189
|
+
loggerInstance: logger(write, log.level, void 0),
|
|
1190
|
+
logController: lines,
|
|
1191
|
+
frameworkErrors: lines.refusedBeforeRouting
|
|
1192
|
+
};
|
|
1193
|
+
}
|
|
1194
|
+
/**
|
|
1195
|
+
* What a thrown thing is, from what it calls itself: its class, and the first code along its
|
|
1196
|
+
* causes — Drizzle wraps PostgreSQL's error, whose SQLSTATE is the useful part. Never its
|
|
1197
|
+
* message or its stack, whose first line is the message.
|
|
1198
|
+
*/
|
|
1199
|
+
function described(error) {
|
|
1200
|
+
if (!(error instanceof Error)) return {};
|
|
1201
|
+
let code;
|
|
1202
|
+
let at = error;
|
|
1203
|
+
for (let depth = 0; code === void 0 && at instanceof Error && depth < 4; depth += 1) {
|
|
1204
|
+
const own = at.code;
|
|
1205
|
+
if (isCode(own)) code = own;
|
|
1206
|
+
at = at.cause;
|
|
1207
|
+
}
|
|
1208
|
+
return {
|
|
1209
|
+
kind: kindOf(error),
|
|
1210
|
+
...code === void 0 ? {} : { code }
|
|
1211
|
+
};
|
|
1212
|
+
}
|
|
1213
|
+
/** `name`, which most errors set; a class that leaves it as `Error` by its constructor. */
|
|
1214
|
+
function kindOf(error) {
|
|
1215
|
+
if (error.name !== "Error" && isIdentifier(error.name)) return error.name;
|
|
1216
|
+
const constructed = error.constructor.name;
|
|
1217
|
+
return isIdentifier(constructed) ? constructed : "Error";
|
|
1218
|
+
}
|
|
1219
|
+
function isIdentifier(value) {
|
|
1220
|
+
return typeof value === "string" && /^[A-Za-z_$][\w$]{0,63}$/.test(value);
|
|
1221
|
+
}
|
|
1222
|
+
/**
|
|
1223
|
+
* `ECONNREFUSED`, `22P02`, `FST_ERR_CTP_BODY_TOO_LARGE`: capitals, digits and underscores. The
|
|
1224
|
+
* one rule for what a code looks like, which the reply to a 5xx keeps a code by as well (0172).
|
|
1225
|
+
*/
|
|
1226
|
+
function isCode(value) {
|
|
1227
|
+
return typeof value === "string" && /^[A-Z0-9_]{1,64}$/.test(value);
|
|
1228
|
+
}
|
|
1229
|
+
function isStatus(value) {
|
|
1230
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 599;
|
|
1231
|
+
}
|
|
1232
|
+
function isRecord$1(value) {
|
|
1233
|
+
return typeof value === "object" && value !== null;
|
|
1234
|
+
}
|
|
1235
|
+
/**
|
|
1236
|
+
* The log a worker was given, or one writing its failures to standard error by the same rule:
|
|
1237
|
+
* where they went before there was a log, and so where a host that starts a worker without
|
|
1238
|
+
* one still finds them.
|
|
1239
|
+
*/
|
|
1240
|
+
function workerLog(given) {
|
|
1241
|
+
return given ?? createServerLog(process.stderr, "error");
|
|
1242
|
+
}
|
|
1243
|
+
/**
|
|
1244
|
+
* What the database says beside its answers, as lines by the same rule: its SQLSTATE, never
|
|
1245
|
+
* its words. postgres.js prints each notice whole to standard output unless it is handed
|
|
1246
|
+
* somewhere else to put it, and the bootstrap draws one on every start for each table, column
|
|
1247
|
+
* and trigger that already exists. Those are `debug`; a `WARNING` is `warn`. With no log,
|
|
1248
|
+
* nowhere.
|
|
1249
|
+
*/
|
|
1250
|
+
function databaseNotices(log) {
|
|
1251
|
+
return (notice) => {
|
|
1252
|
+
if (log === void 0) return;
|
|
1253
|
+
const line = {
|
|
1254
|
+
event: "database.notice",
|
|
1255
|
+
code: notice["code"]
|
|
1256
|
+
};
|
|
1257
|
+
if (notice["severity"] === "WARNING") log.warn(line);
|
|
1258
|
+
else log.debug(line);
|
|
1259
|
+
};
|
|
1260
|
+
}
|
|
1261
|
+
//#endregion
|
|
1262
|
+
//#region src/error-replies.ts
|
|
1263
|
+
/**
|
|
1264
|
+
* What an error answers its client with
|
|
1265
|
+
* ([0172](../../../docs/decisions/0172-a-5xx-says-nothing-of-what-was-thrown.md)).
|
|
1266
|
+
*
|
|
1267
|
+
* Fastify answers an error with `{ statusCode, code, error, message }`, the message being the
|
|
1268
|
+
* error's own. For an error that names its own 4xx that is a sentence written for the client —
|
|
1269
|
+
* the body did not parse, the file is too large — and it stays. For a 5xx it is whatever was
|
|
1270
|
+
* thrown: a database error's message is the failing query and its parameters, a disk's names
|
|
1271
|
+
* the path it could not write. So a 5xx keeps the shape and says only its status: the status's
|
|
1272
|
+
* own name as `error` and as `message`, and a `code` only when the error named its own status —
|
|
1273
|
+
* made to be answered, as `RATE_LIMIT_UNAVAILABLE` is (0170) — and only in the shape the log
|
|
1274
|
+
* keeps a code in. A fault's code, `ENOSPC` or an SQLSTATE, is the operator's, in the log. And a
|
|
1275
|
+
* fault answered under a 4xx — Fastify keeps a status of 400 or more that the reply already had
|
|
1276
|
+
* — says only that status too (0175): its words were no more written for the client there.
|
|
1277
|
+
*
|
|
1278
|
+
* **Two hooks rather than an error handler**, because Fastify writes its line about an error —
|
|
1279
|
+
* `request.failed` and `request.refused`, with the error's kind and code (`server-log.ts`) —
|
|
1280
|
+
* only from its own default handler. One set with `setErrorHandler` replaces it, and the line
|
|
1281
|
+
* would go with it. So the default handler still sets the status and writes the line, and the
|
|
1282
|
+
* body is replaced on its way out, in `onSend`, where the status is final. `onError` is how
|
|
1283
|
+
* `onSend` knows the body is an error's: a route that answers a 5xx itself — no file store, a
|
|
1284
|
+
* scanner that could not be asked — chose its words for the person reading them.
|
|
1285
|
+
*
|
|
1286
|
+
* **An error's reply is written without a header Node cannot write**
|
|
1287
|
+
* ([0175](../../../docs/decisions/0175-a-name-goes-into-a-header-encoded.md)). Node checks a
|
|
1288
|
+
* header's value when the reply is written, after the route has answered, and throws. Fastify
|
|
1289
|
+
* answers that error with the headers the reply already had, so its own reply failed the same
|
|
1290
|
+
* way, and Fastify's last resort — which runs no hook — wrote it with Node's code and sentence,
|
|
1291
|
+
* under whatever status the reply had. Dropping the header here is what lets this reply be the
|
|
1292
|
+
* one that goes out, because no `onSend` hook runs after this one but Fastify's own on a `HEAD`
|
|
1293
|
+
* route, which sets only a length — `error-replies.test.ts` puts such a header on a request to
|
|
1294
|
+
* every route and reads each answer. A route's own value is built so it cannot be refused
|
|
1295
|
+
* (`header-values.ts`); this is for the one that was not.
|
|
1296
|
+
*
|
|
1297
|
+
* On the root instance, before any route is registered, so every route and every plugin's
|
|
1298
|
+
* context has both. Not tied to the log: an app given none withholds the same words.
|
|
1299
|
+
*/
|
|
1300
|
+
function withholdServerErrors(app) {
|
|
1301
|
+
const thrown = /* @__PURE__ */ new WeakMap();
|
|
1302
|
+
app.addHook("onError", async (_request, reply, error) => {
|
|
1303
|
+
thrown.set(reply, error);
|
|
1304
|
+
});
|
|
1305
|
+
app.addHook("onSend", async (_request, reply, payload) => {
|
|
1306
|
+
if (!thrown.has(reply)) return payload;
|
|
1307
|
+
dropUnwritable(reply);
|
|
1308
|
+
const error = thrown.get(reply);
|
|
1309
|
+
if (reply.statusCode < 500 && answerable(error)) return payload;
|
|
1310
|
+
return JSON.stringify(withheld(reply.statusCode, error));
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
/** Remove every header Node would refuse to write, by its own checks rather than a copy of them. */
|
|
1314
|
+
function dropUnwritable(reply) {
|
|
1315
|
+
for (const [name, value] of Object.entries(reply.getHeaders())) if (!writable(name, value)) reply.removeHeader(name);
|
|
1316
|
+
}
|
|
1317
|
+
/**
|
|
1318
|
+
* A header as Node writes it: a number as its digits, and a list one value at a time. Fastify
|
|
1319
|
+
* keeps no header without a value — `reply.header` defaults one to `''` — though `getHeaders`
|
|
1320
|
+
* is typed as if it might.
|
|
1321
|
+
*/
|
|
1322
|
+
function writable(name, value) {
|
|
1323
|
+
try {
|
|
1324
|
+
validateHeaderName(name);
|
|
1325
|
+
for (const each of [value ?? []].flat()) validateHeaderValue(name, String(each));
|
|
1326
|
+
return true;
|
|
1327
|
+
} catch {
|
|
1328
|
+
return false;
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
/**
|
|
1332
|
+
* Whether an error names its own status: made to be answered, by this server or by Fastify, so
|
|
1333
|
+
* its code is part of the answer and, below 500, its sentence too. One that names none is a
|
|
1334
|
+
* fault from below, whatever status the reply already had when it was thrown.
|
|
1335
|
+
*/
|
|
1336
|
+
function answerable(error) {
|
|
1337
|
+
return isRecord(error) && (typeof error["statusCode"] === "number" || typeof error["status"] === "number");
|
|
1338
|
+
}
|
|
1339
|
+
/** Fastify's shape for an error's reply, with nothing in it the thrown error said. */
|
|
1340
|
+
function withheld(status, error) {
|
|
1341
|
+
const name = STATUS_CODES[status] ?? "Server Error";
|
|
1342
|
+
const own = isRecord(error) ? error : {};
|
|
1343
|
+
const code = answerable(own) && isCode(own["code"]) ? own["code"] : void 0;
|
|
1344
|
+
return {
|
|
1345
|
+
statusCode: status,
|
|
1346
|
+
...code === void 0 ? {} : { code },
|
|
1347
|
+
error: name,
|
|
1348
|
+
message: name
|
|
1349
|
+
};
|
|
1350
|
+
}
|
|
1351
|
+
function isRecord(value) {
|
|
1352
|
+
return typeof value === "object" && value !== null;
|
|
1353
|
+
}
|
|
1354
|
+
//#endregion
|
|
1355
|
+
//#region src/nul-characters.ts
|
|
1356
|
+
/**
|
|
1357
|
+
* A body with a NUL character in it is refused as malformed
|
|
1358
|
+
* ([0173](../../../docs/decisions/0173-a-nul-character-names-nothing-and-a-body-with-one-is-refused.md)).
|
|
1359
|
+
*
|
|
1360
|
+
* PostgreSQL holds no U+0000 in `text`, `22021`, nor in `jsonb`, `22P05`, so a string with one
|
|
1361
|
+
* can be kept nowhere this server keeps anything: an answer, a draft, a schema, an example, a
|
|
1362
|
+
* file's name or type, an origin, an email, a key's name. Each was a `500` where its route
|
|
1363
|
+
* wrote it, and a line at `error`. Refused here, once, on the root instance before any route,
|
|
1364
|
+
* so every route and every plugin's context has it and a route added later needs nothing of
|
|
1365
|
+
* its own.
|
|
1366
|
+
*
|
|
1367
|
+
* In `preValidation`, which is after the body is parsed — by whichever parser the request's
|
|
1368
|
+
* type chose, so a `text/plain` string is looked at as well as JSON — and before any route's
|
|
1369
|
+
* own `preHandler`. That makes it what a body that does not parse is: answered by the body
|
|
1370
|
+
* alone, whoever sent it and whatever accounts exist, before a session is asked for. A file's
|
|
1371
|
+
* bytes arrive as a `Buffer` and are not looked at; a NUL byte is ordinary in a file.
|
|
1372
|
+
*
|
|
1373
|
+
* The answer is the route plane's shape for a refusal, with the JSON Pointer (RFC 6901) to the
|
|
1374
|
+
* string or the member's name the NUL is in, so a client can say which answer to correct.
|
|
1375
|
+
*/
|
|
1376
|
+
function refuseNulCharacters(app) {
|
|
1377
|
+
app.addHook("preValidation", async (request, reply) => {
|
|
1378
|
+
const pointer = nulAt(request.body);
|
|
1379
|
+
if (pointer === void 0) return;
|
|
1380
|
+
return reply.code(400).send({
|
|
1381
|
+
error: "nul_character",
|
|
1382
|
+
pointer,
|
|
1383
|
+
message: `The NUL character (U+0000) is in no string this server accepts, and this request's body has one at ${pointer === "" ? "the body itself" : pointer}.`
|
|
1384
|
+
});
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
/**
|
|
1388
|
+
* The JSON Pointer to the first string or member name in `body` with a NUL in it, in the order
|
|
1389
|
+
* the body was written, or `undefined` when there is none.
|
|
1390
|
+
*
|
|
1391
|
+
* A loop over a stack of its own rather than a recursion: `JSON.parse` takes a body nested
|
|
1392
|
+
* deeper than the call stack goes, and a walk that threw on it would make that body a `500` on
|
|
1393
|
+
* every route. And every body is walked, so nothing is built for a value on the way — a step
|
|
1394
|
+
* for each container, and the pointer only for the string it finds. Building a pointer for
|
|
1395
|
+
* every value made the walk of a body at the cap take longer than parsing it.
|
|
1396
|
+
*/
|
|
1397
|
+
function nulAt(body) {
|
|
1398
|
+
const values = [body];
|
|
1399
|
+
const containers = [void 0];
|
|
1400
|
+
const keys = [void 0];
|
|
1401
|
+
while (values.length > 0) {
|
|
1402
|
+
const value = values.pop();
|
|
1403
|
+
const container = containers.pop();
|
|
1404
|
+
const key = keys.pop();
|
|
1405
|
+
if (typeof value === "string") {
|
|
1406
|
+
if (value.includes("\0")) return pointer(container, key);
|
|
1407
|
+
continue;
|
|
1408
|
+
}
|
|
1409
|
+
const here = key === void 0 ? container : {
|
|
1410
|
+
up: container,
|
|
1411
|
+
key
|
|
1412
|
+
};
|
|
1413
|
+
if (Array.isArray(value)) for (let index = value.length - 1; index >= 0; index -= 1) {
|
|
1414
|
+
values.push(value[index]);
|
|
1415
|
+
containers.push(here);
|
|
1416
|
+
keys.push(index);
|
|
1417
|
+
}
|
|
1418
|
+
else if (isPlainObject(value)) {
|
|
1419
|
+
const names = Object.keys(value);
|
|
1420
|
+
for (let index = names.length - 1; index >= 0; index -= 1) {
|
|
1421
|
+
const name = names[index];
|
|
1422
|
+
values.push(value[name]);
|
|
1423
|
+
containers.push(here);
|
|
1424
|
+
keys.push(name);
|
|
1425
|
+
if (name.includes("\0")) {
|
|
1426
|
+
values.push(name);
|
|
1427
|
+
containers.push(here);
|
|
1428
|
+
keys.push(name);
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
/** RFC 6901's spelling of the place `key` names in `container`: '' for the body itself. */
|
|
1435
|
+
function pointer(container, key) {
|
|
1436
|
+
const steps = [];
|
|
1437
|
+
for (let at = key === void 0 ? container : {
|
|
1438
|
+
up: container,
|
|
1439
|
+
key
|
|
1440
|
+
}; at !== void 0; at = at.up) steps.push(at.key);
|
|
1441
|
+
return steps.reverse().map((step) => `/${typeof step === "number" ? String(step) : token(step)}`).join("");
|
|
1442
|
+
}
|
|
1443
|
+
/** A member's name as one step of a JSON Pointer: `~` and `/` escaped, as RFC 6901 has them. */
|
|
1444
|
+
function token(name) {
|
|
1445
|
+
return name.replaceAll("~", "~0").replaceAll("/", "~1");
|
|
1446
|
+
}
|
|
1447
|
+
/** What a parser makes of a JSON object, and nothing else — not a `Buffer`, not a stream. */
|
|
1448
|
+
function isPlainObject(value) {
|
|
1449
|
+
if (typeof value !== "object" || value === null) return false;
|
|
1450
|
+
const prototype = Object.getPrototypeOf(value);
|
|
1451
|
+
return prototype === Object.prototype || prototype === null;
|
|
1452
|
+
}
|
|
1453
|
+
//#endregion
|
|
1454
|
+
//#region src/audit-trail.ts
|
|
1455
|
+
/**
|
|
1456
|
+
* Append an audit row for something that has already happened.
|
|
1457
|
+
*
|
|
1458
|
+
* Never throws. An export that succeeded and an audit row that did not is
|
|
1459
|
+
* bad; an export that 500s because the audit row failed, after the rows have
|
|
1460
|
+
* already been written to the response, is worse and helps nobody. So the
|
|
1461
|
+
* failure is written to the log at error level, as `audit.unwritten` with the
|
|
1462
|
+
* row's action and the error's kind — never its subject, which for a failed
|
|
1463
|
+
* login is the email somebody tried — and the request stands.
|
|
1464
|
+
*
|
|
1465
|
+
* For a mutation that HAS a transaction, do not use this — pass the entry
|
|
1466
|
+
* into that call so the two commit together. A publish, a change to a form's
|
|
1467
|
+
* examples and a submission are those: their use-cases build the row, and
|
|
1468
|
+
* the route hands them `request.id` so it names the request as this one does.
|
|
1469
|
+
*
|
|
1470
|
+
* Its own module because what happens when a row cannot be written is the log's
|
|
1471
|
+
* business (0168), and `app.ts` had no room for it.
|
|
1472
|
+
*/
|
|
1473
|
+
function auditTrail(deps) {
|
|
1474
|
+
return async (request, draft) => {
|
|
1475
|
+
const actor = request.actor;
|
|
1476
|
+
try {
|
|
1477
|
+
await deps.storage.recordAudit({
|
|
1478
|
+
id: deps.newId(),
|
|
1479
|
+
at: deps.nowIso(),
|
|
1480
|
+
requestId: request.id,
|
|
1481
|
+
...auditedBy(actor, draft)
|
|
1482
|
+
});
|
|
1483
|
+
} catch (error) {
|
|
1484
|
+
request.log.error({
|
|
1485
|
+
event: "audit.unwritten",
|
|
1486
|
+
action: draft.action,
|
|
1487
|
+
err: error
|
|
1488
|
+
});
|
|
1489
|
+
}
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
1492
|
+
//#endregion
|
|
1493
|
+
//#region src/app.ts
|
|
1494
|
+
const API_KEY_HEADER = "x-formancy-api-key";
|
|
1495
|
+
/**
|
|
1496
|
+
* The HTTP surface, split into two planes.
|
|
1497
|
+
*
|
|
1498
|
+
* PUBLIC — what a person filling a form needs: resolve, submit, drafts. These
|
|
1499
|
+
* stay unauthenticated by design; anonymous-submission hardening is its own
|
|
1500
|
+
* upcoming step and is opt-in per form.
|
|
1501
|
+
*
|
|
1502
|
+
* MANAGEMENT — everything an operator does: publish, catalog, submissions,
|
|
1503
|
+
* export, users, keys. Authenticated via a session token (login) or an API
|
|
1504
|
+
* key header, authorized through server-core's one-table can().
|
|
1505
|
+
*
|
|
1506
|
+
* This file is the composition root, so THIS is where ambient reality enters:
|
|
1507
|
+
* randomUUID, the real clock, argon2 and the CSPRNG. Everything below stays
|
|
1508
|
+
* injected and replayable.
|
|
1509
|
+
*/
|
|
1510
|
+
async function createApp(storage, options) {
|
|
1511
|
+
const app = Fastify({
|
|
1512
|
+
...serverLogOptions(options.log),
|
|
1513
|
+
bodyLimit: options.bodyLimitBytes ?? 262144,
|
|
1514
|
+
trustProxy: options.trustProxy ?? false
|
|
1515
|
+
});
|
|
1516
|
+
withholdServerErrors(app);
|
|
1517
|
+
refuseNulCharacters(app);
|
|
1518
|
+
const maxFileBytes = checkedMaxFileBytes(options.maxFileBytes ?? 10485760, "maxFileBytes");
|
|
1519
|
+
const fileStore = options.fileStore;
|
|
1520
|
+
const submissionLimit = options.submissionRateLimit ?? {
|
|
1521
|
+
max: 30,
|
|
1522
|
+
timeWindowMs: 6e4
|
|
1523
|
+
};
|
|
1524
|
+
const loginLimit = options.loginRateLimit ?? {
|
|
1525
|
+
max: 10,
|
|
1526
|
+
timeWindowMs: 6e4
|
|
1527
|
+
};
|
|
1528
|
+
const challengeSecret = options.challengeSecret;
|
|
1529
|
+
await app.register(rateLimit, {
|
|
1530
|
+
global: false,
|
|
1531
|
+
max: submissionLimit.max,
|
|
1532
|
+
timeWindow: submissionLimit.timeWindowMs,
|
|
1533
|
+
...options.rateLimitStore === void 0 ? {} : { store: options.rateLimitStore }
|
|
1534
|
+
});
|
|
1535
|
+
const deps = {
|
|
1536
|
+
storage,
|
|
1537
|
+
newId: () => randomUUID(),
|
|
1538
|
+
nowIso: () => (/* @__PURE__ */ new Date()).toISOString(),
|
|
1539
|
+
draftSecret: options.authSecret,
|
|
1540
|
+
...options.optionsSources === void 0 ? {} : { optionsSources: options.optionsSources },
|
|
1541
|
+
...options.scanner === void 0 ? {} : { scanner: options.scanner },
|
|
1542
|
+
capabilities: {
|
|
1543
|
+
now: () => Date.now(),
|
|
1544
|
+
today: () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 10),
|
|
1545
|
+
random: () => Math.random()
|
|
1546
|
+
}
|
|
1547
|
+
};
|
|
1548
|
+
const authDeps = {
|
|
1549
|
+
storage,
|
|
1550
|
+
newId: deps.newId,
|
|
1551
|
+
nowIso: deps.nowIso,
|
|
1552
|
+
randomToken: realRandomToken,
|
|
1553
|
+
...realSecretHashing()
|
|
1554
|
+
};
|
|
1555
|
+
const sessions = createSessionTokens(options.authSecret);
|
|
1556
|
+
if (options.bootstrapAdmin !== void 0) {
|
|
1557
|
+
if (await storage.getUserByEmail(options.bootstrapAdmin.email) === void 0) {
|
|
1558
|
+
const outcome = await createLocalUser(authDeps, {
|
|
1559
|
+
...options.bootstrapAdmin,
|
|
1560
|
+
role: "admin"
|
|
1561
|
+
});
|
|
1562
|
+
if (!outcome.ok) throw new Error(`Bootstrap admin rejected: ${outcome.message}`);
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
async function actorOf(request) {
|
|
1566
|
+
const bearer = request.headers.authorization;
|
|
1567
|
+
if (typeof bearer === "string" && bearer.startsWith("Bearer ")) return sessions.verify(bearer.slice(7));
|
|
1568
|
+
const apiKey = request.headers[API_KEY_HEADER];
|
|
1569
|
+
if (typeof apiKey === "string" && apiKey !== "") {
|
|
1570
|
+
const outcome = await authenticateApiKey(authDeps, apiKey);
|
|
1571
|
+
return outcome.ok ? outcome.actor : void 0;
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
/** 401 without an identity, 403 with one that lacks the permission — the
|
|
1575
|
+
* difference tells a client whether to log in or to give up. */
|
|
1576
|
+
function requires(action) {
|
|
1577
|
+
return async (request, reply) => {
|
|
1578
|
+
const actor = await actorOf(request);
|
|
1579
|
+
if (actor === void 0) return reply.code(401).send({ error: "unauthenticated" });
|
|
1580
|
+
if (!can(actor, action)) return reply.code(403).send({
|
|
1581
|
+
error: "forbidden",
|
|
1582
|
+
action
|
|
1583
|
+
});
|
|
1584
|
+
request.actor = actor;
|
|
1585
|
+
};
|
|
1586
|
+
}
|
|
1587
|
+
const audit = auditTrail(deps);
|
|
1588
|
+
app.post("/auth/login", { config: { rateLimit: limited(loginLimit, "refuse") } }, async (request, reply) => {
|
|
1589
|
+
const body = request.body;
|
|
1590
|
+
if (body === null || typeof body.email !== "string" || typeof body.password !== "string") return reply.code(400).send({
|
|
1591
|
+
error: "bad_request",
|
|
1592
|
+
message: "Body needs { email, password }."
|
|
1593
|
+
});
|
|
1594
|
+
const outcome = await authenticateLocal(authDeps, {
|
|
1595
|
+
email: body.email,
|
|
1596
|
+
password: body.password
|
|
1597
|
+
});
|
|
1598
|
+
if (!outcome.ok) {
|
|
1599
|
+
await audit(request, {
|
|
1600
|
+
action: "auth.login.failed",
|
|
1601
|
+
subject: body.email
|
|
1602
|
+
});
|
|
1603
|
+
return reply.code(401).send({ error: "invalid_credentials" });
|
|
1604
|
+
}
|
|
1605
|
+
await audit(request, {
|
|
1606
|
+
action: "auth.login",
|
|
1607
|
+
subject: body.email,
|
|
1608
|
+
actorKind: "user",
|
|
1609
|
+
actorId: outcome.actor.id
|
|
1610
|
+
});
|
|
1611
|
+
return reply.send({
|
|
1612
|
+
token: await sessions.issue(outcome.actor),
|
|
1613
|
+
role: outcome.actor.role
|
|
1614
|
+
});
|
|
1615
|
+
});
|
|
1616
|
+
app.post("/users", { preHandler: requires("user.create") }, async (request, reply) => {
|
|
1617
|
+
const body = request.body;
|
|
1618
|
+
if (body === null || typeof body.email !== "string" || typeof body.password !== "string" || body.role !== "admin" && body.role !== "editor" && body.role !== "viewer") return reply.code(400).send({
|
|
1619
|
+
error: "bad_request",
|
|
1620
|
+
message: "Body needs { email, password, role }."
|
|
1621
|
+
});
|
|
1622
|
+
const outcome = await createLocalUser(authDeps, {
|
|
1623
|
+
email: body.email,
|
|
1624
|
+
password: body.password,
|
|
1625
|
+
role: body.role
|
|
1626
|
+
});
|
|
1627
|
+
if (!outcome.ok) return reply.code(422).send({
|
|
1628
|
+
error: "rejected",
|
|
1629
|
+
message: outcome.message
|
|
1630
|
+
});
|
|
1631
|
+
return reply.code(201).send({ id: outcome.id });
|
|
1632
|
+
});
|
|
1633
|
+
app.post("/api-keys", { preHandler: requires("apiKey.create") }, async (request, reply) => {
|
|
1634
|
+
const body = request.body;
|
|
1635
|
+
if (body === null || typeof body.name !== "string" || body.role !== "admin" && body.role !== "editor" && body.role !== "viewer") return reply.code(400).send({
|
|
1636
|
+
error: "bad_request",
|
|
1637
|
+
message: "Body needs { name, role }."
|
|
1638
|
+
});
|
|
1639
|
+
const outcome = await createApiKey(authDeps, {
|
|
1640
|
+
name: body.name,
|
|
1641
|
+
role: body.role
|
|
1642
|
+
});
|
|
1643
|
+
if (!outcome.ok) return reply.code(422).send({
|
|
1644
|
+
error: "rejected",
|
|
1645
|
+
message: outcome.message
|
|
1646
|
+
});
|
|
1647
|
+
return reply.code(201).send({
|
|
1648
|
+
id: outcome.id,
|
|
1649
|
+
secret: outcome.secret
|
|
1650
|
+
});
|
|
1651
|
+
});
|
|
1652
|
+
await app.register(publishRoutes, {
|
|
1653
|
+
deps,
|
|
1654
|
+
requires
|
|
1655
|
+
});
|
|
1656
|
+
await app.register(accessRoutes, {
|
|
1657
|
+
deps,
|
|
1658
|
+
requires,
|
|
1659
|
+
audit
|
|
1660
|
+
});
|
|
1661
|
+
await app.register(exampleRoutes, {
|
|
1662
|
+
deps,
|
|
1663
|
+
actorOf
|
|
1664
|
+
});
|
|
1665
|
+
/**
|
|
1666
|
+
* The log itself.
|
|
1667
|
+
*
|
|
1668
|
+
* Admin only, and reading it is NOT audited. A log that records its own
|
|
1669
|
+
* reads grows without bound from a dashboard that polls it, and the entry
|
|
1670
|
+
* would say nothing an access log does not — the events worth recording
|
|
1671
|
+
* are the ones that touch somebody else's data.
|
|
1672
|
+
*/
|
|
1673
|
+
app.get("/audit", { preHandler: requires("user.create") }, async (request, reply) => {
|
|
1674
|
+
const raw = request.query?.limit;
|
|
1675
|
+
const asked = typeof raw === "string" ? Number.parseInt(raw, 10) : 100;
|
|
1676
|
+
const limit = Number.isFinite(asked) ? Math.min(Math.max(asked, 1), 500) : 100;
|
|
1677
|
+
return reply.send({ entries: await deps.storage.listAudit(limit) });
|
|
1678
|
+
});
|
|
1679
|
+
await app.register(deliveryRoutes, {
|
|
1680
|
+
deps,
|
|
1681
|
+
requires,
|
|
1682
|
+
audit
|
|
1683
|
+
});
|
|
1684
|
+
const modelLimit = options.modelRateLimit ?? {
|
|
1685
|
+
max: 10,
|
|
1686
|
+
timeWindowMs: 6e4
|
|
1687
|
+
};
|
|
1688
|
+
await app.register(modelRoutes, {
|
|
1689
|
+
model: options.model,
|
|
1690
|
+
requires,
|
|
1691
|
+
audit,
|
|
1692
|
+
limit: modelLimit
|
|
1693
|
+
});
|
|
1694
|
+
app.get("/forms", { preHandler: requires("form.read") }, async (_request, reply) => {
|
|
1695
|
+
return reply.send({ forms: await listForms(deps) });
|
|
1696
|
+
});
|
|
1697
|
+
app.get("/f/:path/versions", { preHandler: requires("form.read") }, async (request, reply) => {
|
|
1698
|
+
const { path } = request.params;
|
|
1699
|
+
const versions = await listVersions(deps, path);
|
|
1700
|
+
if (versions === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
1701
|
+
return reply.send({ versions });
|
|
1702
|
+
});
|
|
1703
|
+
app.get("/f/:path/submissions", { preHandler: requires("submission.read") }, async (request, reply) => {
|
|
1704
|
+
const { path } = request.params;
|
|
1705
|
+
const listed = await listSubmissions(deps, path);
|
|
1706
|
+
if (listed === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
1707
|
+
await audit(request, {
|
|
1708
|
+
action: "submission.read",
|
|
1709
|
+
subject: path,
|
|
1710
|
+
detail: { count: listed.length }
|
|
1711
|
+
});
|
|
1712
|
+
return reply.send({ submissions: listed });
|
|
1713
|
+
});
|
|
1714
|
+
app.get("/f/:path/submissions/export.csv", { preHandler: requires("submission.export") }, async (request, reply) => {
|
|
1715
|
+
const { path } = request.params;
|
|
1716
|
+
const csv = await exportCsv(deps, path);
|
|
1717
|
+
if (csv === void 0) return reply.code(404).send({ error: "unknown_form" });
|
|
1718
|
+
await audit(request, {
|
|
1719
|
+
action: "submission.exported",
|
|
1720
|
+
subject: path,
|
|
1721
|
+
detail: { bytes: Buffer.byteLength(csv, "utf8") }
|
|
1722
|
+
});
|
|
1723
|
+
return reply.header("content-type", "text/csv; charset=utf-8").header("content-disposition", attachment(`${path}-submissions.csv`)).send(csv);
|
|
1724
|
+
});
|
|
1725
|
+
await app.register(submissionRoutes, {
|
|
1726
|
+
deps,
|
|
1727
|
+
challengeSecret,
|
|
1728
|
+
limit: submissionLimit,
|
|
1729
|
+
actorOf
|
|
1730
|
+
});
|
|
1731
|
+
await app.register(draftRoutes, {
|
|
1732
|
+
deps,
|
|
1733
|
+
limit: submissionLimit
|
|
1734
|
+
});
|
|
1735
|
+
await app.register(fileRoutes, {
|
|
1736
|
+
deps,
|
|
1737
|
+
fileStore,
|
|
1738
|
+
maxFileBytes,
|
|
1739
|
+
limit: submissionLimit,
|
|
1740
|
+
actorOf
|
|
1741
|
+
});
|
|
1742
|
+
return app;
|
|
1743
|
+
}
|
|
1744
|
+
//#endregion
|
|
1745
|
+
//#region src/db.ts
|
|
1746
|
+
/**
|
|
1747
|
+
* The thin slice's tables. Three rules from the design spec are enforced HERE,
|
|
1748
|
+
* in the database, because application code can be bypassed and these must not
|
|
1749
|
+
* be:
|
|
1750
|
+
*
|
|
1751
|
+
* - published versions are immutable (a trigger refuses UPDATE),
|
|
1752
|
+
* - a submission binds to its exact version by real foreign key,
|
|
1753
|
+
* - that key is ON DELETE RESTRICT, so orphaning a submission from the schema
|
|
1754
|
+
* that produced it is structurally impossible.
|
|
1755
|
+
*/
|
|
1756
|
+
const forms = pgTable("forms", {
|
|
1757
|
+
id: uuid("id").primaryKey(),
|
|
1758
|
+
path: text("path").notNull().unique(),
|
|
1759
|
+
currentVersionId: uuid("current_version_id"),
|
|
1760
|
+
accessSubmit: text("access_submit").notNull().default("authenticated"),
|
|
1761
|
+
allowedOrigins: text("allowed_origins").array(),
|
|
1762
|
+
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow()
|
|
1763
|
+
});
|
|
1764
|
+
const formVersions = pgTable("form_versions", {
|
|
1765
|
+
id: uuid("id").primaryKey(),
|
|
1766
|
+
formId: uuid("form_id").notNull().references(() => forms.id),
|
|
1767
|
+
version: integer("version").notNull(),
|
|
1768
|
+
schema: jsonb("schema").notNull(),
|
|
1769
|
+
schemaHash: text("schema_hash").notNull(),
|
|
1770
|
+
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow()
|
|
1771
|
+
});
|
|
1772
|
+
const users = pgTable("users", {
|
|
1773
|
+
id: uuid("id").primaryKey(),
|
|
1774
|
+
email: text("email").notNull().unique(),
|
|
1775
|
+
passwordHash: text("password_hash").notNull(),
|
|
1776
|
+
role: text("role").notNull(),
|
|
1777
|
+
createdAt: timestamp("created_at", { withTimezone: true }).notNull()
|
|
1778
|
+
});
|
|
1779
|
+
const apiKeys = pgTable("api_keys", {
|
|
1780
|
+
id: uuid("id").primaryKey(),
|
|
1781
|
+
name: text("name").notNull(),
|
|
1782
|
+
prefix: text("prefix").notNull(),
|
|
1783
|
+
secretHash: text("secret_hash").notNull(),
|
|
1784
|
+
role: text("role").notNull(),
|
|
1785
|
+
createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
|
|
1786
|
+
revokedAt: timestamp("revoked_at", { withTimezone: true })
|
|
1787
|
+
});
|
|
1788
|
+
const drafts = pgTable("drafts", {
|
|
1789
|
+
id: text("id").notNull(),
|
|
1790
|
+
formId: uuid("form_id").notNull().references(() => forms.id),
|
|
1791
|
+
formVersionId: uuid("form_version_id").notNull().references(() => formVersions.id),
|
|
1792
|
+
data: jsonb("data").notNull(),
|
|
1793
|
+
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull()
|
|
1794
|
+
});
|
|
1795
|
+
const webhooks = pgTable("webhooks", {
|
|
1796
|
+
id: uuid("id").primaryKey(),
|
|
1797
|
+
formId: uuid("form_id").notNull().references(() => forms.id),
|
|
1798
|
+
url: text("url").notNull(),
|
|
1799
|
+
secret: text("secret").notNull(),
|
|
1800
|
+
consecutiveFailures: integer("consecutive_failures").notNull().default(0),
|
|
1801
|
+
openedAt: timestamp("opened_at", { withTimezone: true })
|
|
1802
|
+
});
|
|
1803
|
+
/**
|
|
1804
|
+
* The outbox. A row here and the submission that caused it are written by the
|
|
1805
|
+
* same transaction, which is what makes "accepted" and "will be delivered" one
|
|
1806
|
+
* decision rather than two.
|
|
1807
|
+
*/
|
|
1808
|
+
const deliveries = pgTable("deliveries", {
|
|
1809
|
+
id: uuid("id").primaryKey(),
|
|
1810
|
+
webhookId: uuid("webhook_id").notNull(),
|
|
1811
|
+
submissionId: uuid("submission_id").notNull(),
|
|
1812
|
+
eventId: uuid("event_id").notNull(),
|
|
1813
|
+
body: text("body").notNull(),
|
|
1814
|
+
attempt: integer("attempt").notNull().default(0),
|
|
1815
|
+
nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }).notNull(),
|
|
1816
|
+
state: text("state").notNull().default("pending"),
|
|
1817
|
+
lastError: text("last_error")
|
|
1818
|
+
});
|
|
1819
|
+
const spentChallenges = pgTable("spent_challenges", {
|
|
1820
|
+
challenge: text("challenge").primaryKey(),
|
|
1821
|
+
expiresAt: timestamp("expires_at", { withTimezone: true }).notNull()
|
|
1822
|
+
});
|
|
1823
|
+
const auditLog = pgTable("audit_log", {
|
|
1824
|
+
id: uuid("id").primaryKey(),
|
|
1825
|
+
at: timestamp("at", { withTimezone: true }).notNull(),
|
|
1826
|
+
action: text("action").notNull(),
|
|
1827
|
+
actorKind: text("actor_kind"),
|
|
1828
|
+
actorId: text("actor_id"),
|
|
1829
|
+
subject: text("subject"),
|
|
1830
|
+
requestId: text("request_id"),
|
|
1831
|
+
detail: jsonb("detail")
|
|
1832
|
+
});
|
|
1833
|
+
/**
|
|
1834
|
+
* A form's examples and the sample they start from, one row per form, beside it and never in
|
|
1835
|
+
* a version (0166). The examples change while the form does not, and a version cannot.
|
|
1836
|
+
*/
|
|
1837
|
+
const formExamples = pgTable("form_examples", {
|
|
1838
|
+
formId: uuid("form_id").primaryKey().references(() => forms.id),
|
|
1839
|
+
scenarios: jsonb("scenarios").notNull(),
|
|
1840
|
+
sample: jsonb("sample"),
|
|
1841
|
+
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull()
|
|
1842
|
+
});
|
|
1843
|
+
/**
|
|
1844
|
+
* An audit entry as the table holds it.
|
|
1845
|
+
*
|
|
1846
|
+
* Absent and null are the same thing here, and the conversion is in one place, beside the
|
|
1847
|
+
* table, so the storage's call sites — a publish, a submission, a form's examples — cannot
|
|
1848
|
+
* disagree about which they write.
|
|
1849
|
+
*/
|
|
1850
|
+
function auditRow(entry) {
|
|
1851
|
+
return {
|
|
1852
|
+
id: entry.id,
|
|
1853
|
+
at: new Date(entry.at),
|
|
1854
|
+
action: entry.action,
|
|
1855
|
+
actorKind: entry.actorKind ?? null,
|
|
1856
|
+
actorId: entry.actorId ?? null,
|
|
1857
|
+
subject: entry.subject ?? null,
|
|
1858
|
+
requestId: entry.requestId ?? null,
|
|
1859
|
+
detail: entry.detail ?? null
|
|
1860
|
+
};
|
|
1861
|
+
}
|
|
1862
|
+
const files = pgTable("files", {
|
|
1863
|
+
id: uuid("id").primaryKey(),
|
|
1864
|
+
formId: uuid("form_id").notNull(),
|
|
1865
|
+
name: text("name").notNull(),
|
|
1866
|
+
size: integer("size").notNull(),
|
|
1867
|
+
contentType: text("content_type").notNull(),
|
|
1868
|
+
storageKey: text("storage_key").notNull(),
|
|
1869
|
+
state: text("state").notNull().default("offered"),
|
|
1870
|
+
createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
|
|
1871
|
+
submissionId: uuid("submission_id"),
|
|
1872
|
+
receivingUntil: timestamp("receiving_until", { withTimezone: true })
|
|
1873
|
+
});
|
|
1874
|
+
const submissions = pgTable("submissions", {
|
|
1875
|
+
id: uuid("id").primaryKey(),
|
|
1876
|
+
formId: uuid("form_id").notNull().references(() => forms.id),
|
|
1877
|
+
formVersionId: uuid("form_version_id").notNull().references(() => formVersions.id, { onDelete: "restrict" }),
|
|
1878
|
+
data: jsonb("data").notNull(),
|
|
1879
|
+
submittedAt: timestamp("submitted_at", { withTimezone: true }).notNull()
|
|
1880
|
+
});
|
|
1881
|
+
/**
|
|
1882
|
+
* Whether `id` can be compared with a `uuid` column: the hyphenated form every id this server
|
|
1883
|
+
* mints has. PostgreSQL refuses anything it cannot read as a uuid with an error, `22P02`, where
|
|
1884
|
+
* a `text` column — or the in-memory storage — finds no row; so a lookup by an id from a URL
|
|
1885
|
+
* that is not one answers no row without asking (0172). A spelling PostgreSQL would also read,
|
|
1886
|
+
* braced or without hyphens, names no row either: no id was handed out in one.
|
|
1887
|
+
*/
|
|
1888
|
+
function isUuid(id) {
|
|
1889
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id);
|
|
1890
|
+
}
|
|
1891
|
+
/**
|
|
1892
|
+
* Whether `value` can be compared with a `text` column: PostgreSQL holds no NUL character in
|
|
1893
|
+
* text and refuses one with an error, `22021`, where the in-memory storage finds no row. So a
|
|
1894
|
+
* lookup by a value with one — a form's path from a URL, an API key's prefix from a header —
|
|
1895
|
+
* answers no row without asking, as `isUuid` does for its column (0173). No row was ever
|
|
1896
|
+
* written with one, since no write of one ever succeeded.
|
|
1897
|
+
*/
|
|
1898
|
+
function fitsText(value) {
|
|
1899
|
+
return !value.includes("\0");
|
|
1900
|
+
}
|
|
1901
|
+
/**
|
|
1902
|
+
* Bootstrap DDL for the thin slice — drizzle-kit migrations arrive with the
|
|
1903
|
+
* lifecycle work; a self-hosted skeleton that creates its own tables on first
|
|
1904
|
+
* start is worth more right now than a migration pipeline.
|
|
1905
|
+
*/
|
|
1906
|
+
async function bootstrapSchema(sql) {
|
|
1907
|
+
await sql`
|
|
1908
|
+
CREATE TABLE IF NOT EXISTS forms (
|
|
1909
|
+
id uuid PRIMARY KEY,
|
|
1910
|
+
path text NOT NULL UNIQUE,
|
|
1911
|
+
current_version_id uuid,
|
|
1912
|
+
-- Private by default in the DATABASE, not only in the code, so a row
|
|
1913
|
+
-- inserted by a migration or a fixture is not accidentally open.
|
|
1914
|
+
access_submit text NOT NULL DEFAULT 'authenticated'
|
|
1915
|
+
CHECK (access_submit IN ('authenticated', 'public')),
|
|
1916
|
+
-- NULL: no allowlist in force. Empty array: nothing allowed. Different
|
|
1917
|
+
-- things, which is why this is nullable rather than defaulting to '{}'.
|
|
1918
|
+
allowed_origins text[],
|
|
1919
|
+
created_at timestamptz NOT NULL DEFAULT now()
|
|
1920
|
+
)`;
|
|
1921
|
+
await sql`
|
|
1922
|
+
ALTER TABLE forms
|
|
1923
|
+
ADD COLUMN IF NOT EXISTS access_submit text NOT NULL DEFAULT 'authenticated'`;
|
|
1924
|
+
await sql`ALTER TABLE forms ADD COLUMN IF NOT EXISTS allowed_origins text[]`;
|
|
1925
|
+
await sql`
|
|
1926
|
+
CREATE TABLE IF NOT EXISTS form_versions (
|
|
1927
|
+
id uuid PRIMARY KEY,
|
|
1928
|
+
form_id uuid NOT NULL REFERENCES forms(id),
|
|
1929
|
+
version integer NOT NULL,
|
|
1930
|
+
schema jsonb NOT NULL,
|
|
1931
|
+
schema_hash text NOT NULL,
|
|
1932
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1933
|
+
UNIQUE (form_id, version),
|
|
1934
|
+
UNIQUE (form_id, schema_hash)
|
|
1935
|
+
)`;
|
|
1936
|
+
await sql`
|
|
1937
|
+
CREATE TABLE IF NOT EXISTS submissions (
|
|
1938
|
+
id uuid PRIMARY KEY,
|
|
1939
|
+
form_id uuid NOT NULL REFERENCES forms(id),
|
|
1940
|
+
form_version_id uuid NOT NULL REFERENCES form_versions(id) ON DELETE RESTRICT,
|
|
1941
|
+
data jsonb NOT NULL,
|
|
1942
|
+
submitted_at timestamptz NOT NULL
|
|
1943
|
+
)`;
|
|
1944
|
+
await sql`
|
|
1945
|
+
CREATE TABLE IF NOT EXISTS webhooks (
|
|
1946
|
+
id uuid PRIMARY KEY,
|
|
1947
|
+
form_id uuid NOT NULL REFERENCES forms(id) ON DELETE CASCADE,
|
|
1948
|
+
url text NOT NULL,
|
|
1949
|
+
secret text NOT NULL
|
|
1950
|
+
)`;
|
|
1951
|
+
await sql`
|
|
1952
|
+
ALTER TABLE webhooks
|
|
1953
|
+
ADD COLUMN IF NOT EXISTS consecutive_failures integer NOT NULL DEFAULT 0,
|
|
1954
|
+
-- When the breaker opened, null while the destination is answering. On
|
|
1955
|
+
-- the row rather than in the worker's memory: memory does not survive a
|
|
1956
|
+
-- restart, and a self-hoster with no operations team needs a failing
|
|
1957
|
+
-- webhook to be visible on a screen rather than in a log nobody tails.
|
|
1958
|
+
ADD COLUMN IF NOT EXISTS opened_at timestamptz`;
|
|
1959
|
+
await sql`
|
|
1960
|
+
CREATE TABLE IF NOT EXISTS deliveries (
|
|
1961
|
+
id uuid PRIMARY KEY,
|
|
1962
|
+
webhook_id uuid NOT NULL REFERENCES webhooks(id) ON DELETE CASCADE,
|
|
1963
|
+
-- RESTRICT, not CASCADE: a delivery is the record that something was
|
|
1964
|
+
-- sent about this submission, and deleting the submission must not
|
|
1965
|
+
-- quietly erase it.
|
|
1966
|
+
submission_id uuid NOT NULL REFERENCES submissions(id) ON DELETE RESTRICT,
|
|
1967
|
+
event_id uuid NOT NULL,
|
|
1968
|
+
body text NOT NULL,
|
|
1969
|
+
attempt integer NOT NULL DEFAULT 0,
|
|
1970
|
+
next_attempt_at timestamptz NOT NULL,
|
|
1971
|
+
state text NOT NULL DEFAULT 'pending'
|
|
1972
|
+
CHECK (state IN ('pending', 'delivered', 'dead')),
|
|
1973
|
+
last_error text
|
|
1974
|
+
)`;
|
|
1975
|
+
await sql`
|
|
1976
|
+
CREATE INDEX IF NOT EXISTS deliveries_due
|
|
1977
|
+
ON deliveries (next_attempt_at) WHERE state = 'pending'`;
|
|
1978
|
+
await sql`
|
|
1979
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
1980
|
+
id uuid PRIMARY KEY,
|
|
1981
|
+
form_id uuid NOT NULL REFERENCES forms(id) ON DELETE RESTRICT,
|
|
1982
|
+
-- What the reader called it. Display only: the storage key is minted by
|
|
1983
|
+
-- the server, because a key built from a submitted filename is a path
|
|
1984
|
+
-- traversal waiting for somebody to try it.
|
|
1985
|
+
name text NOT NULL,
|
|
1986
|
+
size integer NOT NULL,
|
|
1987
|
+
content_type text NOT NULL,
|
|
1988
|
+
storage_key text NOT NULL,
|
|
1989
|
+
state text NOT NULL DEFAULT 'offered'
|
|
1990
|
+
CHECK (state IN ('offered', 'stored', 'claimed')),
|
|
1991
|
+
created_at timestamptz NOT NULL,
|
|
1992
|
+
-- RESTRICT for the same reason a delivery uses it: the row is the record
|
|
1993
|
+
-- that something was attached, and deleting the submission must not
|
|
1994
|
+
-- quietly erase what it carried.
|
|
1995
|
+
submission_id uuid REFERENCES submissions(id) ON DELETE RESTRICT,
|
|
1996
|
+
-- One file, one submission. Enforced here rather than in application
|
|
1997
|
+
-- code, because two concurrent submissions claiming the same file is
|
|
1998
|
+
-- exactly the race application code loses — and the loser would be a
|
|
1999
|
+
-- submission referencing bytes another submission owns.
|
|
2000
|
+
CONSTRAINT files_one_submission UNIQUE (id, submission_id)
|
|
2001
|
+
)`;
|
|
2002
|
+
await sql`ALTER TABLE files ADD COLUMN IF NOT EXISTS receiving_until timestamptz`;
|
|
2003
|
+
await sql`
|
|
2004
|
+
CREATE INDEX IF NOT EXISTS files_abandoned
|
|
2005
|
+
ON files (created_at) WHERE state <> 'claimed'`;
|
|
2006
|
+
await sql`
|
|
2007
|
+
CREATE TABLE IF NOT EXISTS users (
|
|
2008
|
+
id uuid PRIMARY KEY,
|
|
2009
|
+
email text NOT NULL UNIQUE,
|
|
2010
|
+
password_hash text NOT NULL,
|
|
2011
|
+
role text NOT NULL,
|
|
2012
|
+
created_at timestamptz NOT NULL
|
|
2013
|
+
)`;
|
|
2014
|
+
await sql`
|
|
2015
|
+
CREATE TABLE IF NOT EXISTS api_keys (
|
|
2016
|
+
id uuid PRIMARY KEY,
|
|
2017
|
+
name text NOT NULL,
|
|
2018
|
+
prefix text NOT NULL,
|
|
2019
|
+
secret_hash text NOT NULL,
|
|
2020
|
+
role text NOT NULL,
|
|
2021
|
+
created_at timestamptz NOT NULL,
|
|
2022
|
+
revoked_at timestamptz
|
|
2023
|
+
)`;
|
|
2024
|
+
await sql`CREATE INDEX IF NOT EXISTS api_keys_prefix ON api_keys (prefix)`;
|
|
2025
|
+
await sql`
|
|
2026
|
+
CREATE TABLE IF NOT EXISTS spent_challenges (
|
|
2027
|
+
-- The primary key IS the mechanism. Two requests arriving with one
|
|
2028
|
+
-- solution both pass every stateless check — the signature is ours
|
|
2029
|
+
-- and the hash is right — and only the database can decide which of
|
|
2030
|
+
-- them spends it. A read-then-write would race exactly where somebody
|
|
2031
|
+
-- is looking.
|
|
2032
|
+
challenge text PRIMARY KEY,
|
|
2033
|
+
expires_at timestamptz NOT NULL
|
|
2034
|
+
)`;
|
|
2035
|
+
await sql`CREATE INDEX IF NOT EXISTS spent_challenges_expiry ON spent_challenges (expires_at)`;
|
|
2036
|
+
await sql`
|
|
2037
|
+
CREATE UNLOGGED TABLE IF NOT EXISTS rate_limit_counters (
|
|
2038
|
+
key text PRIMARY KEY,
|
|
2039
|
+
hits integer NOT NULL,
|
|
2040
|
+
resets_at timestamptz NOT NULL
|
|
2041
|
+
)`;
|
|
2042
|
+
await sql`
|
|
2043
|
+
CREATE TABLE IF NOT EXISTS audit_log (
|
|
2044
|
+
id uuid PRIMARY KEY,
|
|
2045
|
+
at timestamptz NOT NULL,
|
|
2046
|
+
action text NOT NULL,
|
|
2047
|
+
-- Null for an anonymous public submission, which is an answer rather
|
|
2048
|
+
-- than a gap: the form was open and nobody was signed in.
|
|
2049
|
+
actor_kind text CHECK (actor_kind IN ('user', 'apiKey')),
|
|
2050
|
+
actor_id text,
|
|
2051
|
+
subject text,
|
|
2052
|
+
request_id text,
|
|
2053
|
+
-- Identifiers and counts. NEVER a submission's answers: an audit log is
|
|
2054
|
+
-- read by more people and kept longer than the data it describes, so
|
|
2055
|
+
-- data inside it is a second copy of the thing being protected.
|
|
2056
|
+
detail jsonb
|
|
2057
|
+
)`;
|
|
2058
|
+
await sql`CREATE INDEX IF NOT EXISTS audit_log_at ON audit_log (at DESC)`;
|
|
2059
|
+
await sql`CREATE INDEX IF NOT EXISTS audit_log_subject ON audit_log (subject, at DESC)`;
|
|
2060
|
+
await sql`
|
|
2061
|
+
CREATE TABLE IF NOT EXISTS drafts (
|
|
2062
|
+
id text NOT NULL,
|
|
2063
|
+
form_id uuid NOT NULL REFERENCES forms(id),
|
|
2064
|
+
form_version_id uuid NOT NULL REFERENCES form_versions(id),
|
|
2065
|
+
data jsonb NOT NULL,
|
|
2066
|
+
updated_at timestamptz NOT NULL,
|
|
2067
|
+
PRIMARY KEY (form_id, id)
|
|
2068
|
+
)`;
|
|
2069
|
+
await sql`
|
|
2070
|
+
CREATE TABLE IF NOT EXISTS form_examples (
|
|
2071
|
+
form_id uuid PRIMARY KEY REFERENCES forms(id),
|
|
2072
|
+
scenarios jsonb NOT NULL,
|
|
2073
|
+
sample jsonb,
|
|
2074
|
+
updated_at timestamptz NOT NULL
|
|
2075
|
+
)`;
|
|
2076
|
+
await sql`
|
|
2077
|
+
CREATE OR REPLACE FUNCTION refuse_version_update() RETURNS trigger AS $$
|
|
2078
|
+
BEGIN
|
|
2079
|
+
RAISE EXCEPTION 'form_versions rows are immutable; publish a new version instead';
|
|
2080
|
+
END
|
|
2081
|
+
$$ LANGUAGE plpgsql`;
|
|
2082
|
+
await sql`DROP TRIGGER IF EXISTS form_versions_immutable ON form_versions`;
|
|
2083
|
+
await sql`
|
|
2084
|
+
CREATE TRIGGER form_versions_immutable
|
|
2085
|
+
BEFORE UPDATE ON form_versions
|
|
2086
|
+
FOR EACH ROW EXECUTE FUNCTION refuse_version_update()`;
|
|
2087
|
+
await sql`
|
|
2088
|
+
CREATE OR REPLACE FUNCTION refuse_audit_change() RETURNS trigger AS $$
|
|
2089
|
+
BEGIN
|
|
2090
|
+
RAISE EXCEPTION 'audit_log is append-only';
|
|
2091
|
+
END
|
|
2092
|
+
$$ LANGUAGE plpgsql`;
|
|
2093
|
+
await sql`DROP TRIGGER IF EXISTS audit_log_append_only ON audit_log`;
|
|
2094
|
+
await sql`
|
|
2095
|
+
CREATE TRIGGER audit_log_append_only
|
|
2096
|
+
BEFORE UPDATE OR DELETE ON audit_log
|
|
2097
|
+
FOR EACH ROW EXECUTE FUNCTION refuse_audit_change()`;
|
|
2098
|
+
}
|
|
2099
|
+
//#endregion
|
|
2100
|
+
//#region src/postgres-examples.ts
|
|
2101
|
+
/**
|
|
2102
|
+
* A form's examples over PostgreSQL: one row per form in `form_examples`, replaced whole
|
|
2103
|
+
* (0166).
|
|
2104
|
+
*
|
|
2105
|
+
* Its own file because it changes for its own reason, and because `postgres-storage.ts`
|
|
2106
|
+
* would otherwise have crossed the size budget with it.
|
|
2107
|
+
*/
|
|
2108
|
+
function examplesStorage(db) {
|
|
2109
|
+
return {
|
|
2110
|
+
async getExamples(formId) {
|
|
2111
|
+
const row = (await db.select().from(formExamples).where(eq(formExamples.formId, formId)).limit(1))[0];
|
|
2112
|
+
if (row === void 0) return void 0;
|
|
2113
|
+
return {
|
|
2114
|
+
formId: row.formId,
|
|
2115
|
+
scenarios: row.scenarios,
|
|
2116
|
+
sample: row.sample,
|
|
2117
|
+
updatedAt: row.updatedAt.toISOString()
|
|
2118
|
+
};
|
|
2119
|
+
},
|
|
2120
|
+
async keepExamples(record, audit) {
|
|
2121
|
+
await db.transaction(async (tx) => {
|
|
2122
|
+
const row = {
|
|
2123
|
+
formId: record.formId,
|
|
2124
|
+
scenarios: record.scenarios,
|
|
2125
|
+
sample: record.sample,
|
|
2126
|
+
updatedAt: new Date(record.updatedAt)
|
|
2127
|
+
};
|
|
2128
|
+
await tx.insert(formExamples).values(row).onConflictDoUpdate({
|
|
2129
|
+
target: formExamples.formId,
|
|
2130
|
+
set: {
|
|
2131
|
+
scenarios: row.scenarios,
|
|
2132
|
+
sample: row.sample,
|
|
2133
|
+
updatedAt: row.updatedAt
|
|
2134
|
+
}
|
|
2135
|
+
});
|
|
2136
|
+
if (audit !== void 0) await tx.insert(auditLog).values(auditRow(audit));
|
|
2137
|
+
});
|
|
2138
|
+
}
|
|
2139
|
+
};
|
|
2140
|
+
}
|
|
2141
|
+
//#endregion
|
|
2142
|
+
//#region src/postgres-identities.ts
|
|
2143
|
+
/**
|
|
2144
|
+
* Who may use the management plane, over PostgreSQL: users with a password and API keys, each
|
|
2145
|
+
* stored as a hash and looked up by what a request presents — an email, a key's prefix.
|
|
2146
|
+
*
|
|
2147
|
+
* Its own file because it changes for its own reason — the way somebody is let in, not the
|
|
2148
|
+
* forms and responses everything else here stores — and because `postgres-storage.ts` would
|
|
2149
|
+
* otherwise have crossed the size budget when its lookups learned to find nothing for a NUL
|
|
2150
|
+
* (0173).
|
|
2151
|
+
*/
|
|
2152
|
+
function identitiesStorage(db) {
|
|
2153
|
+
return {
|
|
2154
|
+
async insertUser(record) {
|
|
2155
|
+
await db.insert(users).values({
|
|
2156
|
+
id: record.id,
|
|
2157
|
+
email: record.email,
|
|
2158
|
+
passwordHash: record.passwordHash,
|
|
2159
|
+
role: record.role,
|
|
2160
|
+
createdAt: new Date(record.createdAt)
|
|
2161
|
+
});
|
|
2162
|
+
},
|
|
2163
|
+
async getUserByEmail(email) {
|
|
2164
|
+
if (!fitsText(email)) return void 0;
|
|
2165
|
+
const row = (await db.select().from(users).where(eq(users.email, email)).limit(1))[0];
|
|
2166
|
+
if (row === void 0) return void 0;
|
|
2167
|
+
return {
|
|
2168
|
+
id: row.id,
|
|
2169
|
+
email: row.email,
|
|
2170
|
+
passwordHash: row.passwordHash,
|
|
2171
|
+
role: row.role,
|
|
2172
|
+
createdAt: row.createdAt.toISOString()
|
|
2173
|
+
};
|
|
2174
|
+
},
|
|
2175
|
+
async insertApiKey(record) {
|
|
2176
|
+
await db.insert(apiKeys).values({
|
|
2177
|
+
id: record.id,
|
|
2178
|
+
name: record.name,
|
|
2179
|
+
prefix: record.prefix,
|
|
2180
|
+
secretHash: record.secretHash,
|
|
2181
|
+
role: record.role,
|
|
2182
|
+
createdAt: new Date(record.createdAt),
|
|
2183
|
+
revokedAt: record.revokedAt === null ? null : new Date(record.revokedAt)
|
|
2184
|
+
});
|
|
2185
|
+
},
|
|
2186
|
+
async listApiKeys() {
|
|
2187
|
+
return (await db.select().from(apiKeys)).map(toApiKeyRecord);
|
|
2188
|
+
},
|
|
2189
|
+
async findApiKeysByPrefix(prefix) {
|
|
2190
|
+
if (!fitsText(prefix)) return [];
|
|
2191
|
+
return (await db.select().from(apiKeys).where(eq(apiKeys.prefix, prefix))).map(toApiKeyRecord);
|
|
2192
|
+
},
|
|
2193
|
+
async revokeApiKey(id, atIso) {
|
|
2194
|
+
await db.update(apiKeys).set({ revokedAt: new Date(atIso) }).where(eq(apiKeys.id, id));
|
|
2195
|
+
}
|
|
2196
|
+
};
|
|
2197
|
+
}
|
|
2198
|
+
/** An API key's row as the port describes it: timestamps as ISO strings. */
|
|
2199
|
+
function toApiKeyRecord(row) {
|
|
2200
|
+
return {
|
|
2201
|
+
id: row.id,
|
|
2202
|
+
name: row.name,
|
|
2203
|
+
prefix: row.prefix,
|
|
2204
|
+
secretHash: row.secretHash,
|
|
2205
|
+
role: row.role,
|
|
2206
|
+
createdAt: row.createdAt.toISOString(),
|
|
2207
|
+
revokedAt: row.revokedAt === null ? null : row.revokedAt.toISOString()
|
|
2208
|
+
};
|
|
2209
|
+
}
|
|
2210
|
+
//#endregion
|
|
2211
|
+
//#region src/postgres-storage.ts
|
|
2212
|
+
/** The Storage port over Postgres — the mirror of server-core's in-memory one. */
|
|
2213
|
+
function createPostgresStorage(sql) {
|
|
2214
|
+
const db = drizzle(sql);
|
|
2215
|
+
return {
|
|
2216
|
+
...examplesStorage(db),
|
|
2217
|
+
...identitiesStorage(db),
|
|
2218
|
+
async getFormByPath(path) {
|
|
2219
|
+
if (!fitsText(path)) return void 0;
|
|
2220
|
+
const row = (await db.select().from(forms).where(eq(forms.path, path)).limit(1))[0];
|
|
2221
|
+
if (row === void 0) return void 0;
|
|
2222
|
+
return {
|
|
2223
|
+
id: row.id,
|
|
2224
|
+
path: row.path,
|
|
2225
|
+
currentVersionId: row.currentVersionId,
|
|
2226
|
+
accessSubmit: row.accessSubmit === "public" ? "public" : "authenticated",
|
|
2227
|
+
allowedOrigins: row.allowedOrigins
|
|
2228
|
+
};
|
|
2229
|
+
},
|
|
2230
|
+
async listForms() {
|
|
2231
|
+
return (await db.select().from(forms)).map((row) => ({
|
|
2232
|
+
id: row.id,
|
|
2233
|
+
path: row.path,
|
|
2234
|
+
currentVersionId: row.currentVersionId,
|
|
2235
|
+
accessSubmit: row.accessSubmit === "public" ? "public" : "authenticated",
|
|
2236
|
+
allowedOrigins: row.allowedOrigins
|
|
2237
|
+
}));
|
|
2238
|
+
},
|
|
2239
|
+
async createForm(record) {
|
|
2240
|
+
await db.insert(forms).values({
|
|
2241
|
+
id: record.id,
|
|
2242
|
+
path: record.path,
|
|
2243
|
+
currentVersionId: record.currentVersionId,
|
|
2244
|
+
accessSubmit: record.accessSubmit,
|
|
2245
|
+
allowedOrigins: record.allowedOrigins
|
|
2246
|
+
});
|
|
2247
|
+
},
|
|
2248
|
+
async updateFormAccess(formId, access) {
|
|
2249
|
+
await db.update(forms).set({
|
|
2250
|
+
accessSubmit: access.accessSubmit,
|
|
2251
|
+
allowedOrigins: access.allowedOrigins
|
|
2252
|
+
}).where(eq(forms.id, formId));
|
|
2253
|
+
},
|
|
2254
|
+
async setCurrentVersion(formId, versionId) {
|
|
2255
|
+
await db.update(forms).set({ currentVersionId: versionId }).where(eq(forms.id, formId));
|
|
2256
|
+
},
|
|
2257
|
+
async insertVersion(record) {
|
|
2258
|
+
await db.insert(formVersions).values({
|
|
2259
|
+
id: record.id,
|
|
2260
|
+
formId: record.formId,
|
|
2261
|
+
version: record.version,
|
|
2262
|
+
schema: record.schema,
|
|
2263
|
+
schemaHash: record.schemaHash
|
|
2264
|
+
});
|
|
2265
|
+
},
|
|
2266
|
+
async publishVersion({ form, version, audit }) {
|
|
2267
|
+
await db.transaction(async (tx) => {
|
|
2268
|
+
if (form !== void 0) await tx.insert(forms).values({
|
|
2269
|
+
id: form.id,
|
|
2270
|
+
path: form.path,
|
|
2271
|
+
currentVersionId: form.currentVersionId,
|
|
2272
|
+
accessSubmit: form.accessSubmit,
|
|
2273
|
+
allowedOrigins: form.allowedOrigins
|
|
2274
|
+
});
|
|
2275
|
+
await tx.insert(formVersions).values({
|
|
2276
|
+
id: version.id,
|
|
2277
|
+
formId: version.formId,
|
|
2278
|
+
version: version.version,
|
|
2279
|
+
schema: version.schema,
|
|
2280
|
+
schemaHash: version.schemaHash
|
|
2281
|
+
});
|
|
2282
|
+
if ((await tx.update(forms).set({ currentVersionId: version.id }).where(eq(forms.id, version.formId)).returning({ id: forms.id })).length !== 1) throw new Error(`Publishing to form ${version.formId} matched no form row. Rolling back, so there is no version pointing at a form that is not there.`);
|
|
2283
|
+
if (audit !== void 0) await tx.insert(auditLog).values(auditRow(audit));
|
|
2284
|
+
});
|
|
2285
|
+
},
|
|
2286
|
+
async getVersionById(id) {
|
|
2287
|
+
const row = (await db.select().from(formVersions).where(eq(formVersions.id, id)).limit(1))[0];
|
|
2288
|
+
if (row === void 0) return void 0;
|
|
2289
|
+
return {
|
|
2290
|
+
id: row.id,
|
|
2291
|
+
formId: row.formId,
|
|
2292
|
+
version: row.version,
|
|
2293
|
+
schema: row.schema,
|
|
2294
|
+
schemaHash: row.schemaHash
|
|
2295
|
+
};
|
|
2296
|
+
},
|
|
2297
|
+
async findVersionByHash(formId, schemaHash) {
|
|
2298
|
+
const row = (await db.select().from(formVersions).where(eq(formVersions.formId, formId))).find((candidate) => candidate.schemaHash === schemaHash);
|
|
2299
|
+
if (row === void 0) return void 0;
|
|
2300
|
+
return {
|
|
2301
|
+
id: row.id,
|
|
2302
|
+
formId: row.formId,
|
|
2303
|
+
version: row.version,
|
|
2304
|
+
schema: row.schema,
|
|
2305
|
+
schemaHash: row.schemaHash
|
|
2306
|
+
};
|
|
2307
|
+
},
|
|
2308
|
+
async latestVersionNumber(formId) {
|
|
2309
|
+
return (await db.select({ latest: max(formVersions.version) }).from(formVersions).where(eq(formVersions.formId, formId)))[0]?.latest ?? 0;
|
|
2310
|
+
},
|
|
2311
|
+
async insertSubmission(record, queued, claimFileIds, audit) {
|
|
2312
|
+
return db.transaction(async (tx) => {
|
|
2313
|
+
if ((await tx.insert(submissions).values({
|
|
2314
|
+
id: record.id,
|
|
2315
|
+
formId: record.formId,
|
|
2316
|
+
formVersionId: record.formVersionId,
|
|
2317
|
+
data: record.data,
|
|
2318
|
+
submittedAt: new Date(record.submittedAt)
|
|
2319
|
+
}).onConflictDoNothing({ target: submissions.id }).returning({ id: submissions.id })).length === 0) return false;
|
|
2320
|
+
if (queued !== void 0 && queued.length > 0) await tx.insert(deliveries).values(queued.map((delivery) => ({
|
|
2321
|
+
id: delivery.id,
|
|
2322
|
+
webhookId: delivery.webhookId,
|
|
2323
|
+
submissionId: delivery.submissionId,
|
|
2324
|
+
eventId: delivery.eventId,
|
|
2325
|
+
body: delivery.body,
|
|
2326
|
+
attempt: delivery.attempt,
|
|
2327
|
+
nextAttemptAt: new Date(delivery.nextAttemptAt),
|
|
2328
|
+
state: delivery.state,
|
|
2329
|
+
lastError: delivery.lastError
|
|
2330
|
+
})));
|
|
2331
|
+
if (claimFileIds !== void 0 && claimFileIds.length > 0) {
|
|
2332
|
+
if ((await tx.update(files).set({
|
|
2333
|
+
state: "claimed",
|
|
2334
|
+
submissionId: record.id
|
|
2335
|
+
}).where(and(inArray(files.id, [...claimFileIds]), eq(files.state, "stored"))).returning({ id: files.id })).length !== claimFileIds.length) throw new Error("A file this submission names was claimed by another one first. Rolling back, so the submission does not reference bytes it does not own.");
|
|
2336
|
+
}
|
|
2337
|
+
if (audit !== void 0) await tx.insert(auditLog).values(auditRow(audit));
|
|
2338
|
+
return true;
|
|
2339
|
+
});
|
|
2340
|
+
},
|
|
2341
|
+
async hasSubmission(id) {
|
|
2342
|
+
return (await db.select({ id: submissions.id }).from(submissions).where(eq(submissions.id, id)).limit(1)).length === 1;
|
|
2343
|
+
},
|
|
2344
|
+
async spendChallenge(challenge, expiresAtIso) {
|
|
2345
|
+
return (await db.insert(spentChallenges).values({
|
|
2346
|
+
challenge,
|
|
2347
|
+
expiresAt: new Date(expiresAtIso)
|
|
2348
|
+
}).onConflictDoNothing().returning({ challenge: spentChallenges.challenge })).length === 1;
|
|
2349
|
+
},
|
|
2350
|
+
async forgetExpiredChallenges(beforeIso) {
|
|
2351
|
+
return (await db.delete(spentChallenges).where(lt(spentChallenges.expiresAt, new Date(beforeIso))).returning({ challenge: spentChallenges.challenge })).length;
|
|
2352
|
+
},
|
|
2353
|
+
async recordAudit(entry) {
|
|
2354
|
+
await db.insert(auditLog).values(auditRow(entry));
|
|
2355
|
+
},
|
|
2356
|
+
async listAudit(limit) {
|
|
2357
|
+
return (await db.select().from(auditLog).orderBy(desc(auditLog.at)).limit(limit)).map((row) => ({
|
|
2358
|
+
id: row.id,
|
|
2359
|
+
at: row.at.toISOString(),
|
|
2360
|
+
action: row.action,
|
|
2361
|
+
...row.actorKind === null ? {} : { actorKind: row.actorKind },
|
|
2362
|
+
...row.actorId === null ? {} : { actorId: row.actorId },
|
|
2363
|
+
...row.subject === null ? {} : { subject: row.subject },
|
|
2364
|
+
...row.requestId === null ? {} : { requestId: row.requestId },
|
|
2365
|
+
...row.detail === null ? {} : { detail: row.detail }
|
|
2366
|
+
}));
|
|
2367
|
+
},
|
|
2368
|
+
async insertFile(record) {
|
|
2369
|
+
await db.insert(files).values({
|
|
2370
|
+
...record,
|
|
2371
|
+
createdAt: new Date(record.createdAt),
|
|
2372
|
+
receivingUntil: record.receivingUntil === null ? null : new Date(record.receivingUntil)
|
|
2373
|
+
});
|
|
2374
|
+
},
|
|
2375
|
+
async getFile(id) {
|
|
2376
|
+
if (!isUuid(id)) return void 0;
|
|
2377
|
+
const [row] = await db.select().from(files).where(eq(files.id, id)).limit(1);
|
|
2378
|
+
return row === void 0 ? void 0 : toFileRecord(row);
|
|
2379
|
+
},
|
|
2380
|
+
async leaseFile(id, nowIso, untilIso) {
|
|
2381
|
+
const free = or(isNull(files.receivingUntil), lte(files.receivingUntil, new Date(nowIso)));
|
|
2382
|
+
return (await db.update(files).set({ receivingUntil: new Date(untilIso) }).where(and(eq(files.id, id), eq(files.state, "offered"), free)).returning({ id: files.id })).length === 1;
|
|
2383
|
+
},
|
|
2384
|
+
async settleFile(id, untilIso) {
|
|
2385
|
+
const held = eq(files.receivingUntil, new Date(untilIso));
|
|
2386
|
+
return (await db.update(files).set({
|
|
2387
|
+
state: "stored",
|
|
2388
|
+
receivingUntil: null
|
|
2389
|
+
}).where(and(eq(files.id, id), eq(files.state, "offered"), held)).returning({ id: files.id })).length === 1;
|
|
2390
|
+
},
|
|
2391
|
+
async releaseFile(id, untilIso) {
|
|
2392
|
+
return (await db.update(files).set({ receivingUntil: null }).where(and(eq(files.id, id), eq(files.receivingUntil, new Date(untilIso)))).returning({ id: files.id })).length === 1;
|
|
2393
|
+
},
|
|
2394
|
+
async abandonedFiles(beforeIso) {
|
|
2395
|
+
return (await db.select().from(files).where(and(ne(files.state, "claimed"), lt(files.createdAt, new Date(beforeIso))))).map(toFileRecord);
|
|
2396
|
+
},
|
|
2397
|
+
async deleteFiles(ids) {
|
|
2398
|
+
if (ids.length === 0) return;
|
|
2399
|
+
await db.delete(files).where(inArray(files.id, [...ids]));
|
|
2400
|
+
},
|
|
2401
|
+
async webhooksForForm(formId) {
|
|
2402
|
+
return (await db.select().from(webhooks).where(eq(webhooks.formId, formId))).map(toWebhookRecord);
|
|
2403
|
+
},
|
|
2404
|
+
async insertWebhook(record) {
|
|
2405
|
+
await db.insert(webhooks).values({
|
|
2406
|
+
id: record.id,
|
|
2407
|
+
formId: record.formId,
|
|
2408
|
+
url: record.url,
|
|
2409
|
+
secret: record.secret,
|
|
2410
|
+
consecutiveFailures: record.consecutiveFailures,
|
|
2411
|
+
openedAt: record.openedAt === null ? null : new Date(record.openedAt)
|
|
2412
|
+
});
|
|
2413
|
+
},
|
|
2414
|
+
async updateWebhook(record) {
|
|
2415
|
+
await db.update(webhooks).set({
|
|
2416
|
+
consecutiveFailures: record.consecutiveFailures,
|
|
2417
|
+
openedAt: record.openedAt === null ? null : new Date(record.openedAt)
|
|
2418
|
+
}).where(eq(webhooks.id, record.id));
|
|
2419
|
+
},
|
|
2420
|
+
async listWebhooks() {
|
|
2421
|
+
return (await db.select().from(webhooks)).map(toWebhookRecord);
|
|
2422
|
+
},
|
|
2423
|
+
async deadDeliveries(limit) {
|
|
2424
|
+
return (await db.select().from(deliveries).where(eq(deliveries.state, "dead")).orderBy(desc(deliveries.nextAttemptAt)).limit(limit)).map(toDeliveryRecord);
|
|
2425
|
+
},
|
|
2426
|
+
async getDelivery(id) {
|
|
2427
|
+
if (!isUuid(id)) return void 0;
|
|
2428
|
+
const [row] = await db.select().from(deliveries).where(eq(deliveries.id, id)).limit(1);
|
|
2429
|
+
return row === void 0 ? void 0 : toDeliveryRecord(row);
|
|
2430
|
+
},
|
|
2431
|
+
async claimDueDeliveries(nowIso, limit) {
|
|
2432
|
+
return (await db.select().from(deliveries).where(and(eq(deliveries.state, "pending"), lte(deliveries.nextAttemptAt, new Date(nowIso)))).orderBy(deliveries.nextAttemptAt).limit(limit)).map(toDeliveryRecord);
|
|
2433
|
+
},
|
|
2434
|
+
async updateDelivery(record) {
|
|
2435
|
+
await db.update(deliveries).set({
|
|
2436
|
+
attempt: record.attempt,
|
|
2437
|
+
nextAttemptAt: new Date(record.nextAttemptAt),
|
|
2438
|
+
state: record.state,
|
|
2439
|
+
lastError: record.lastError
|
|
2440
|
+
}).where(eq(deliveries.id, record.id));
|
|
2441
|
+
},
|
|
2442
|
+
async listSubmissions() {
|
|
2443
|
+
return (await db.select().from(submissions)).map(toSubmissionRecord);
|
|
2444
|
+
},
|
|
2445
|
+
async listSubmissionsByForm(formId) {
|
|
2446
|
+
return (await db.select().from(submissions).where(eq(submissions.formId, formId)).orderBy(desc(submissions.submittedAt), desc(submissions.id))).map(toSubmissionRecord);
|
|
2447
|
+
},
|
|
2448
|
+
async listVersionsByForm(formId) {
|
|
2449
|
+
return (await db.select().from(formVersions).where(eq(formVersions.formId, formId)).orderBy(desc(formVersions.version))).map((row) => ({
|
|
2450
|
+
id: row.id,
|
|
2451
|
+
formId: row.formId,
|
|
2452
|
+
version: row.version,
|
|
2453
|
+
schema: row.schema,
|
|
2454
|
+
schemaHash: row.schemaHash
|
|
2455
|
+
}));
|
|
2456
|
+
},
|
|
2457
|
+
async upsertDraft(record) {
|
|
2458
|
+
await db.insert(drafts).values({
|
|
2459
|
+
id: record.id,
|
|
2460
|
+
formId: record.formId,
|
|
2461
|
+
formVersionId: record.formVersionId,
|
|
2462
|
+
data: record.data,
|
|
2463
|
+
updatedAt: new Date(record.updatedAt)
|
|
2464
|
+
}).onConflictDoUpdate({
|
|
2465
|
+
target: [drafts.formId, drafts.id],
|
|
2466
|
+
set: {
|
|
2467
|
+
formVersionId: record.formVersionId,
|
|
2468
|
+
data: record.data,
|
|
2469
|
+
updatedAt: new Date(record.updatedAt)
|
|
2470
|
+
}
|
|
2471
|
+
});
|
|
2472
|
+
},
|
|
2473
|
+
async getDraft(formId, draftId) {
|
|
2474
|
+
if (!fitsText(draftId)) return void 0;
|
|
2475
|
+
const row = (await db.select().from(drafts).where(and(eq(drafts.formId, formId), eq(drafts.id, draftId))).limit(1))[0];
|
|
2476
|
+
if (row === void 0) return void 0;
|
|
2477
|
+
return {
|
|
2478
|
+
id: row.id,
|
|
2479
|
+
formId: row.formId,
|
|
2480
|
+
formVersionId: row.formVersionId,
|
|
2481
|
+
data: row.data,
|
|
2482
|
+
updatedAt: row.updatedAt.toISOString()
|
|
2483
|
+
};
|
|
2484
|
+
}
|
|
2485
|
+
};
|
|
2486
|
+
}
|
|
2487
|
+
function toSubmissionRecord(row) {
|
|
2488
|
+
return {
|
|
2489
|
+
id: row.id,
|
|
2490
|
+
formId: row.formId,
|
|
2491
|
+
formVersionId: row.formVersionId,
|
|
2492
|
+
data: row.data,
|
|
2493
|
+
submittedAt: row.submittedAt.toISOString()
|
|
2494
|
+
};
|
|
2495
|
+
}
|
|
2496
|
+
/** A delivery row as the port describes it. */
|
|
2497
|
+
function toDeliveryRecord(row) {
|
|
2498
|
+
return {
|
|
2499
|
+
id: row.id,
|
|
2500
|
+
webhookId: row.webhookId,
|
|
2501
|
+
submissionId: row.submissionId,
|
|
2502
|
+
eventId: row.eventId,
|
|
2503
|
+
body: row.body,
|
|
2504
|
+
attempt: row.attempt,
|
|
2505
|
+
nextAttemptAt: row.nextAttemptAt.toISOString(),
|
|
2506
|
+
state: row.state,
|
|
2507
|
+
lastError: row.lastError
|
|
2508
|
+
};
|
|
2509
|
+
}
|
|
2510
|
+
/** A files row as the port describes it: timestamps as ISO strings. */
|
|
2511
|
+
function toFileRecord(row) {
|
|
2512
|
+
return {
|
|
2513
|
+
id: row.id,
|
|
2514
|
+
formId: row.formId,
|
|
2515
|
+
name: row.name,
|
|
2516
|
+
size: row.size,
|
|
2517
|
+
contentType: row.contentType,
|
|
2518
|
+
storageKey: row.storageKey,
|
|
2519
|
+
state: row.state,
|
|
2520
|
+
createdAt: row.createdAt.toISOString(),
|
|
2521
|
+
submissionId: row.submissionId,
|
|
2522
|
+
receivingUntil: row.receivingUntil === null ? null : row.receivingUntil.toISOString()
|
|
2523
|
+
};
|
|
2524
|
+
}
|
|
2525
|
+
/** A webhook row as the port describes it, health included. */
|
|
2526
|
+
function toWebhookRecord(row) {
|
|
2527
|
+
return {
|
|
2528
|
+
id: row.id,
|
|
2529
|
+
formId: row.formId,
|
|
2530
|
+
url: row.url,
|
|
2531
|
+
secret: row.secret,
|
|
2532
|
+
consecutiveFailures: row.consecutiveFailures,
|
|
2533
|
+
openedAt: row.openedAt === null ? null : row.openedAt.toISOString()
|
|
2534
|
+
};
|
|
2535
|
+
}
|
|
2536
|
+
/**
|
|
2537
|
+
* `@fastify/rate-limit`'s store over the PostgreSQL every replica already shares (0170).
|
|
2538
|
+
*
|
|
2539
|
+
* The plugin's own store keeps its counts in the process, so behind N replicas each limit
|
|
2540
|
+
* allowed N times what it said. This one keeps them in `rate_limit_counters`, one row per
|
|
2541
|
+
* route and client, and counts with one statement: the upsert takes the row's lock, so two
|
|
2542
|
+
* replicas counting the same client at once are counted one after the other and neither
|
|
2543
|
+
* increment is lost.
|
|
2544
|
+
*
|
|
2545
|
+
* A window starts at a key's first request and lasts the route's `timeWindow`, as the plugin's
|
|
2546
|
+
* own stores do it, on the database's clock — one clock for every replica, where each
|
|
2547
|
+
* process's own would end the window at a different moment on each.
|
|
2548
|
+
*
|
|
2549
|
+
* It opens its own connections from the database's address rather than taking a pool, because
|
|
2550
|
+
* which connections it counts on is what keeps a counter that cannot answer from stopping
|
|
2551
|
+
* everything else: handed the pool storage queries on, it would count correctly and bring that
|
|
2552
|
+
* outage back. A count that does not come back within the bound is abandoned: one still waiting
|
|
2553
|
+
* here to be sent is dropped unsent, and one already sent is ended by the database once it runs,
|
|
2554
|
+
* by the `statement_timeout` of these connections, with no cancel request — a connection more —
|
|
2555
|
+
* needed for it. The error the plugin is handed says nothing about the database, because a route
|
|
2556
|
+
* that refuses without a count sends it to whoever asked; what each route does then is its own
|
|
2557
|
+
* declaration (`rate-limits.ts`).
|
|
2558
|
+
*
|
|
2559
|
+
* The plugin's in-memory store is the second implementation of the same interface, and what
|
|
2560
|
+
* `createApp` counts in when it is given none.
|
|
2561
|
+
*/
|
|
2562
|
+
function createPostgresRateLimitStore(databaseUrl, options = {}) {
|
|
2563
|
+
const timeoutMs = options.timeoutMs ?? 1e3;
|
|
2564
|
+
const sweepEveryMs = options.sweepEveryMs ?? 6e5;
|
|
2565
|
+
const log = options.log;
|
|
2566
|
+
const sql = postgres(databaseUrl, {
|
|
2567
|
+
max: 4,
|
|
2568
|
+
connection: { statement_timeout: timeoutMs },
|
|
2569
|
+
onnotice: databaseNotices(log)
|
|
2570
|
+
});
|
|
2571
|
+
let answering = true;
|
|
2572
|
+
let sweptAt = Date.now();
|
|
2573
|
+
let sweeping = false;
|
|
2574
|
+
let running = 0;
|
|
2575
|
+
const waiting = [];
|
|
2576
|
+
function done() {
|
|
2577
|
+
running--;
|
|
2578
|
+
waiting.shift()?.();
|
|
2579
|
+
}
|
|
2580
|
+
function count(key, windowMs) {
|
|
2581
|
+
return new Promise((resolve, reject) => {
|
|
2582
|
+
const send = () => {
|
|
2583
|
+
running++;
|
|
2584
|
+
sql`
|
|
2585
|
+
INSERT INTO rate_limit_counters AS counter (key, hits, resets_at)
|
|
2586
|
+
VALUES (${key}, 1, now() + ${windowMs}::double precision * interval '1 millisecond')
|
|
2587
|
+
ON CONFLICT (key) DO UPDATE SET
|
|
2588
|
+
hits = CASE WHEN counter.resets_at <= now() THEN 1 ELSE counter.hits + 1 END,
|
|
2589
|
+
resets_at = CASE WHEN counter.resets_at <= now() THEN excluded.resets_at ELSE counter.resets_at END
|
|
2590
|
+
RETURNING hits AS current, ceil(extract(epoch FROM resets_at - now()) * 1000)::integer AS ttl`.then(([row]) => {
|
|
2591
|
+
clearTimeout(timer);
|
|
2592
|
+
if (row === void 0) reject(/* @__PURE__ */ new Error("the count returned no row"));
|
|
2593
|
+
else resolve(row);
|
|
2594
|
+
}, (error) => {
|
|
2595
|
+
clearTimeout(timer);
|
|
2596
|
+
reject(error);
|
|
2597
|
+
}).finally(done);
|
|
2598
|
+
};
|
|
2599
|
+
const timer = setTimeout(() => {
|
|
2600
|
+
const queued = waiting.indexOf(send);
|
|
2601
|
+
if (queued !== -1) waiting.splice(queued, 1);
|
|
2602
|
+
reject(late());
|
|
2603
|
+
}, timeoutMs);
|
|
2604
|
+
if (running < 32) send();
|
|
2605
|
+
else waiting.push(send);
|
|
2606
|
+
});
|
|
2607
|
+
}
|
|
2608
|
+
/** On the back of a count, at most once per interval, one at a time, and never waited for. */
|
|
2609
|
+
function sweep() {
|
|
2610
|
+
const now = Date.now();
|
|
2611
|
+
if (sweeping || running >= 32 || now - sweptAt < sweepEveryMs) return;
|
|
2612
|
+
sweptAt = now;
|
|
2613
|
+
sweeping = true;
|
|
2614
|
+
running++;
|
|
2615
|
+
sql.begin(async (tx) => {
|
|
2616
|
+
await tx`SELECT set_config('statement_timeout', '0', true)`;
|
|
2617
|
+
await tx`DELETE FROM rate_limit_counters WHERE resets_at <= now()`;
|
|
2618
|
+
}).then(() => {}, (error) => log?.error({
|
|
2619
|
+
event: "ratelimit.sweep.failed",
|
|
2620
|
+
err: error
|
|
2621
|
+
})).finally(() => {
|
|
2622
|
+
sweeping = false;
|
|
2623
|
+
done();
|
|
2624
|
+
});
|
|
2625
|
+
}
|
|
2626
|
+
class PostgresRateLimitStore {
|
|
2627
|
+
#route;
|
|
2628
|
+
constructor(_options, route = "") {
|
|
2629
|
+
this.#route = route;
|
|
2630
|
+
}
|
|
2631
|
+
incr(key, callback, timeWindow) {
|
|
2632
|
+
count(`${this.#route}${key}`, timeWindow).then((result) => {
|
|
2633
|
+
if (!answering) {
|
|
2634
|
+
answering = true;
|
|
2635
|
+
log?.error({ event: "ratelimit.answering" });
|
|
2636
|
+
}
|
|
2637
|
+
callback(null, result);
|
|
2638
|
+
}, (error) => {
|
|
2639
|
+
if (answering) {
|
|
2640
|
+
answering = false;
|
|
2641
|
+
log?.error({
|
|
2642
|
+
event: "ratelimit.unanswered",
|
|
2643
|
+
err: error
|
|
2644
|
+
});
|
|
2645
|
+
}
|
|
2646
|
+
callback(uncounted());
|
|
2647
|
+
});
|
|
2648
|
+
sweep();
|
|
2649
|
+
}
|
|
2650
|
+
child(routeOptions) {
|
|
2651
|
+
const { method, url } = routeOptions.routeInfo;
|
|
2652
|
+
return new PostgresRateLimitStore(void 0, `${[method].flat().join(",")} ${url} `);
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
return {
|
|
2656
|
+
store: PostgresRateLimitStore,
|
|
2657
|
+
end: () => sql.end({ timeout: timeoutMs / 1e3 })
|
|
2658
|
+
};
|
|
2659
|
+
}
|
|
2660
|
+
/** What a route that refuses without a count answers: a 503, and nothing about the database. */
|
|
2661
|
+
function uncounted() {
|
|
2662
|
+
return Object.assign(/* @__PURE__ */ new Error("This request could not be counted against its limit, so it was not taken. Try again in a moment."), {
|
|
2663
|
+
statusCode: 503,
|
|
2664
|
+
code: "RATE_LIMIT_UNAVAILABLE"
|
|
2665
|
+
});
|
|
2666
|
+
}
|
|
2667
|
+
/**
|
|
2668
|
+
* A count not answered within the bound. Its own code, because the log writes an error's kind
|
|
2669
|
+
* and code and nothing else: without one, a database too slow to count — a lock on the table, a
|
|
2670
|
+
* flood — reads the same as any other failure, and the operator cannot tell which to look for.
|
|
2671
|
+
*/
|
|
2672
|
+
function late() {
|
|
2673
|
+
return Object.assign(/* @__PURE__ */ new Error("The count was not answered within its bound."), { code: "RATE_LIMIT_TIMEOUT" });
|
|
2674
|
+
}
|
|
2675
|
+
//#endregion
|
|
2676
|
+
//#region src/file-store.ts
|
|
2677
|
+
/**
|
|
2678
|
+
* Bytes under one directory, one file per key.
|
|
2679
|
+
*
|
|
2680
|
+
* **Keys are checked, not trusted.** A key is minted by the server from a form
|
|
2681
|
+
* id and a file id, so it cannot contain a traversal — but this resolves it
|
|
2682
|
+
* and refuses anything landing outside the root regardless. A path check that
|
|
2683
|
+
* only runs when the input is untrusted is a path check that stops running the
|
|
2684
|
+
* day somebody adds a caller.
|
|
2685
|
+
*/
|
|
2686
|
+
function createLocalFileStore(root) {
|
|
2687
|
+
const base = resolve(root);
|
|
2688
|
+
const pathFor = (key) => {
|
|
2689
|
+
if (key.includes("\\")) throw new Error(`Refusing a storage key that could resolve outside the store: ${key}`);
|
|
2690
|
+
const full = resolve(join(base, key));
|
|
2691
|
+
if (full !== base && !full.startsWith(base + sep)) throw new Error(`Refusing a storage key that resolves outside the store: ${key}`);
|
|
2692
|
+
return full;
|
|
2693
|
+
};
|
|
2694
|
+
return {
|
|
2695
|
+
async put(key, bytes) {
|
|
2696
|
+
const full = pathFor(key);
|
|
2697
|
+
await mkdir(dirname(full), { recursive: true });
|
|
2698
|
+
await writeFile(full, bytes);
|
|
2699
|
+
},
|
|
2700
|
+
async open(key) {
|
|
2701
|
+
const full = pathFor(key);
|
|
2702
|
+
try {
|
|
2703
|
+
await stat(full);
|
|
2704
|
+
} catch {
|
|
2705
|
+
return;
|
|
2706
|
+
}
|
|
2707
|
+
return createReadStream(full);
|
|
2708
|
+
},
|
|
2709
|
+
async remove(key) {
|
|
2710
|
+
await rm(pathFor(key), { force: true });
|
|
2711
|
+
},
|
|
2712
|
+
async sizeOf(key) {
|
|
2713
|
+
try {
|
|
2714
|
+
return (await stat(pathFor(key))).size;
|
|
2715
|
+
} catch {
|
|
2716
|
+
return;
|
|
2717
|
+
}
|
|
2718
|
+
}
|
|
2719
|
+
};
|
|
2720
|
+
}
|
|
2721
|
+
/**
|
|
2722
|
+
* The checksum a receiver can compare against.
|
|
2723
|
+
*
|
|
2724
|
+
* Returned when the bytes are stored and never asked of the client: a checksum
|
|
2725
|
+
* the uploader computed is a checksum the uploader can lie about, so this one
|
|
2726
|
+
* is only useful for telling a *later* reader that the bytes have not changed.
|
|
2727
|
+
*/
|
|
2728
|
+
function sha256(bytes) {
|
|
2729
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
2730
|
+
}
|
|
2731
|
+
//#endregion
|
|
2732
|
+
//#region src/file-collector.ts
|
|
2733
|
+
function startFileCollector(storage, store, options = {}) {
|
|
2734
|
+
const interval = options.intervalMs ?? 9e5;
|
|
2735
|
+
const log = workerLog(options.log);
|
|
2736
|
+
const afterHours = options.afterHours ?? 24;
|
|
2737
|
+
let stopped = false;
|
|
2738
|
+
let running = false;
|
|
2739
|
+
const tick = async () => {
|
|
2740
|
+
if (running || stopped) return;
|
|
2741
|
+
running = true;
|
|
2742
|
+
try {
|
|
2743
|
+
const abandoned = await collectAbandonedFiles(storage, /* @__PURE__ */ new Date(), afterHours);
|
|
2744
|
+
if (abandoned.length === 0) return;
|
|
2745
|
+
const removed = [];
|
|
2746
|
+
for (const file of abandoned) {
|
|
2747
|
+
await store.remove(file.storageKey);
|
|
2748
|
+
removed.push(file.id);
|
|
2749
|
+
}
|
|
2750
|
+
await storage.deleteFiles(removed);
|
|
2751
|
+
} catch (error) {
|
|
2752
|
+
log.error({
|
|
2753
|
+
event: "collector.failed",
|
|
2754
|
+
err: error
|
|
2755
|
+
});
|
|
2756
|
+
} finally {
|
|
2757
|
+
running = false;
|
|
2758
|
+
}
|
|
2759
|
+
};
|
|
2760
|
+
const timer = setInterval(() => void tick(), interval);
|
|
2761
|
+
timer.unref?.();
|
|
2762
|
+
return { stop: () => {
|
|
2763
|
+
stopped = true;
|
|
2764
|
+
clearInterval(timer);
|
|
2765
|
+
} };
|
|
2766
|
+
}
|
|
2767
|
+
//#endregion
|
|
2768
|
+
//#region src/model-settings.ts
|
|
2769
|
+
/**
|
|
2770
|
+
* Which model the environment names for the builders: none, or a provider, its key and
|
|
2771
|
+
* the model.
|
|
2772
|
+
*
|
|
2773
|
+
* A function of the environment rather than lines in `main.ts`, as `fileStoreSettings` is:
|
|
2774
|
+
* the rules are where a deployment goes wrong, and `main.ts` is a composition root no test
|
|
2775
|
+
* imports. Constructing the adapter stays in `main.ts`; this only says which
|
|
2776
|
+
* ([0165](../../../docs/decisions/0165-a-deployments-model-is-asked-through-its-server.md)).
|
|
2777
|
+
*/
|
|
2778
|
+
const MODEL_PROVIDERS = [
|
|
2779
|
+
"anthropic",
|
|
2780
|
+
"openai",
|
|
2781
|
+
"xai"
|
|
2782
|
+
];
|
|
2783
|
+
/**
|
|
2784
|
+
* The variable each provider's SDK reads for headers of its own to add to every request.
|
|
2785
|
+
*
|
|
2786
|
+
* The adapters name the base URL, the organisation, the project, the credential and the log
|
|
2787
|
+
* level rather than leave them to the environment; these headers no option undoes. So a
|
|
2788
|
+
* server with a model refuses to start while the variable its provider's SDK reads is set.
|
|
2789
|
+
* xAI is asked through OpenAI's client, so it is OpenAI's variable.
|
|
2790
|
+
*/
|
|
2791
|
+
const SDK_HEADERS_VARIABLE = {
|
|
2792
|
+
anthropic: "ANTHROPIC_CUSTOM_HEADERS",
|
|
2793
|
+
openai: "OPENAI_CUSTOM_HEADERS",
|
|
2794
|
+
xai: "OPENAI_CUSTOM_HEADERS"
|
|
2795
|
+
};
|
|
2796
|
+
/**
|
|
2797
|
+
* Absent — all three unset or blank — is a supported state, and the default: this
|
|
2798
|
+
* deployment has no model, the server says so to the admin, and no prompt pane is drawn.
|
|
2799
|
+
*
|
|
2800
|
+
* Throws on anything in between, so the server refuses to start rather than starting
|
|
2801
|
+
* without the model the operator thinks they configured: a key or a model with no
|
|
2802
|
+
* provider, a provider formancy has no adapter for, or a provider without its key or
|
|
2803
|
+
* its model. Throws too when the environment would add headers to every request the
|
|
2804
|
+
* provider's SDK sends. **No provider has a default model.** A model id goes stale when
|
|
2805
|
+
* the provider retires it, and then every turn fails as a model that could not be
|
|
2806
|
+
* reached, after an upgrade nobody made; and which model runs is what the operator pays
|
|
2807
|
+
* for. Neither is this file's to decide. The message never repeats a value, because the
|
|
2808
|
+
* key is one.
|
|
2809
|
+
*/
|
|
2810
|
+
function modelSettings(env) {
|
|
2811
|
+
const provider = setting(env, "FORMANCY_MODEL_PROVIDER");
|
|
2812
|
+
const apiKey = setting(env, "FORMANCY_MODEL_API_KEY");
|
|
2813
|
+
const model = setting(env, "FORMANCY_MODEL");
|
|
2814
|
+
if (provider === void 0) {
|
|
2815
|
+
if (apiKey === void 0 && model === void 0) return { kind: "none" };
|
|
2816
|
+
throw new Error("FORMANCY_MODEL_API_KEY or FORMANCY_MODEL is set without FORMANCY_MODEL_PROVIDER: name the provider (anthropic, openai or xai), or unset both to run without a model.");
|
|
2817
|
+
}
|
|
2818
|
+
if (!isProvider(provider)) throw new Error("FORMANCY_MODEL_PROVIDER must be anthropic, openai or xai.");
|
|
2819
|
+
if (apiKey === void 0) throw new Error("FORMANCY_MODEL_API_KEY is required when FORMANCY_MODEL_PROVIDER is set.");
|
|
2820
|
+
if (model === void 0) throw new Error("FORMANCY_MODEL is required when FORMANCY_MODEL_PROVIDER is set: name the model as the provider's documentation does. There is no default.");
|
|
2821
|
+
const headers = SDK_HEADERS_VARIABLE[provider];
|
|
2822
|
+
if (setting(env, headers) !== void 0) throw new Error(`${headers} is set, and the SDK formancy asks ${provider} through would send each line of it as a header with every form. Unset it to run with a model.`);
|
|
2823
|
+
return {
|
|
2824
|
+
kind: "configured",
|
|
2825
|
+
provider,
|
|
2826
|
+
apiKey,
|
|
2827
|
+
model
|
|
2828
|
+
};
|
|
2829
|
+
}
|
|
2830
|
+
function isProvider(value) {
|
|
2831
|
+
return MODEL_PROVIDERS.includes(value);
|
|
2832
|
+
}
|
|
2833
|
+
/** A setting, with the empty string read as unset — compose blanks a variable, it cannot drop one. */
|
|
2834
|
+
function setting(env, name) {
|
|
2835
|
+
const value = env[name];
|
|
2836
|
+
return value === void 0 || value.trim() === "" ? void 0 : value;
|
|
2837
|
+
}
|
|
2838
|
+
//#endregion
|
|
2839
|
+
//#region src/anthropic-completer.ts
|
|
2840
|
+
/**
|
|
2841
|
+
* Where the request goes. Named rather than left to the SDK, which would otherwise take
|
|
2842
|
+
* `ANTHROPIC_BASE_URL` from the environment: the deployment view says which host a form is
|
|
2843
|
+
* sent to, and an ambient variable could change it without anybody reading this file.
|
|
2844
|
+
*/
|
|
2845
|
+
const ANTHROPIC_BASE_URL = "https://api.anthropic.com";
|
|
2846
|
+
/**
|
|
2847
|
+
* The most a turn may write, thinking included.
|
|
2848
|
+
*
|
|
2849
|
+
* Generous on purpose: an answer is a whole form document, and one cut off at the limit is
|
|
2850
|
+
* not an answer at all (`truncated`), so a limit too low wastes the turn it stopped.
|
|
2851
|
+
* Streamed, so a long answer is not cut off by the HTTP timeout either. It is the ceiling
|
|
2852
|
+
* on what one turn can cost — the operator pays per token written, not per token allowed.
|
|
2853
|
+
*/
|
|
2854
|
+
const ANTHROPIC_MAX_OUTPUT_TOKENS = 64e3;
|
|
2855
|
+
/** What a refused turn says when Anthropic gave no explanation. */
|
|
2856
|
+
const DECLINED = "The model service declined this request.";
|
|
2857
|
+
/**
|
|
2858
|
+
* Claude, through Anthropic's official SDK (0165).
|
|
2859
|
+
*
|
|
2860
|
+
* One system prompt and one user message, with adaptive thinking — the model decides how
|
|
2861
|
+
* much to think — streamed and read with the SDK's final-message helper. **The stop reason
|
|
2862
|
+
* is read before the content.** A refusal ends the person's run as a decline, with
|
|
2863
|
+
* Anthropic's explanation; an answer stopped at the output limit, or by the context
|
|
2864
|
+
* window, is `truncated` and never handed over as the first half of a document. Thinking
|
|
2865
|
+
* blocks are not part of the answer.
|
|
2866
|
+
*
|
|
2867
|
+
* Adaptive thinking needs a model that has it — Claude Opus 4.6, Sonnet 4.6 and later. A
|
|
2868
|
+
* model without it is refused by Anthropic with a 400, and every turn ends `unavailable`.
|
|
2869
|
+
*
|
|
2870
|
+
* The SDK retries what it calls temporary failures (a 429, a 5xx, a dropped connection)
|
|
2871
|
+
* twice, by its own default.
|
|
2872
|
+
*/
|
|
2873
|
+
function createAnthropicCompleter(options) {
|
|
2874
|
+
const client = new Anthropic({
|
|
2875
|
+
apiKey: options.apiKey,
|
|
2876
|
+
authToken: null,
|
|
2877
|
+
logLevel: "off",
|
|
2878
|
+
baseURL: ANTHROPIC_BASE_URL,
|
|
2879
|
+
...options.fetch === void 0 ? {} : { fetch: options.fetch }
|
|
2880
|
+
});
|
|
2881
|
+
return { async complete(prompt, cancellation) {
|
|
2882
|
+
const controller = new AbortController();
|
|
2883
|
+
cancellation.onCancel(() => controller.abort());
|
|
2884
|
+
if (controller.signal.aborted) return {
|
|
2885
|
+
ok: false,
|
|
2886
|
+
failure: "cancelled"
|
|
2887
|
+
};
|
|
2888
|
+
let message;
|
|
2889
|
+
try {
|
|
2890
|
+
message = await client.messages.stream({
|
|
2891
|
+
model: options.model,
|
|
2892
|
+
max_tokens: ANTHROPIC_MAX_OUTPUT_TOKENS,
|
|
2893
|
+
thinking: { type: "adaptive" },
|
|
2894
|
+
system: prompt.system,
|
|
2895
|
+
messages: [{
|
|
2896
|
+
role: "user",
|
|
2897
|
+
content: prompt.user
|
|
2898
|
+
}]
|
|
2899
|
+
}, { signal: controller.signal }).finalMessage();
|
|
2900
|
+
} catch (error) {
|
|
2901
|
+
if (controller.signal.aborted) return {
|
|
2902
|
+
ok: false,
|
|
2903
|
+
failure: "cancelled"
|
|
2904
|
+
};
|
|
2905
|
+
return unavailable$1(error);
|
|
2906
|
+
}
|
|
2907
|
+
return read$1(message);
|
|
2908
|
+
} };
|
|
2909
|
+
}
|
|
2910
|
+
/** A finished message, by its stop reason first. */
|
|
2911
|
+
function read$1(message) {
|
|
2912
|
+
switch (message.stop_reason) {
|
|
2913
|
+
case "refusal": {
|
|
2914
|
+
const explanation = message.stop_details?.explanation;
|
|
2915
|
+
return {
|
|
2916
|
+
ok: false,
|
|
2917
|
+
failure: "refused",
|
|
2918
|
+
reason: typeof explanation === "string" && explanation.trim() !== "" ? explanation : DECLINED
|
|
2919
|
+
};
|
|
2920
|
+
}
|
|
2921
|
+
case "max_tokens":
|
|
2922
|
+
case "model_context_window_exceeded": return {
|
|
2923
|
+
ok: false,
|
|
2924
|
+
failure: "truncated"
|
|
2925
|
+
};
|
|
2926
|
+
case "end_turn":
|
|
2927
|
+
case "stop_sequence": return {
|
|
2928
|
+
ok: true,
|
|
2929
|
+
text: message.content.flatMap((block) => block.type === "text" ? [block.text] : []).join("")
|
|
2930
|
+
};
|
|
2931
|
+
default: return {
|
|
2932
|
+
ok: false,
|
|
2933
|
+
failure: "unavailable",
|
|
2934
|
+
cause: `Anthropic stopped for ${String(message.stop_reason)}.`
|
|
2935
|
+
};
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
/** An error from the SDK, as a failure: the status when Anthropic answered, the message for the log. */
|
|
2939
|
+
function unavailable$1(error) {
|
|
2940
|
+
const status = error instanceof APIError && typeof error.status === "number" ? error.status : void 0;
|
|
2941
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
2942
|
+
return {
|
|
2943
|
+
ok: false,
|
|
2944
|
+
failure: "unavailable",
|
|
2945
|
+
...status === void 0 ? {} : { status },
|
|
2946
|
+
cause
|
|
2947
|
+
};
|
|
2948
|
+
}
|
|
2949
|
+
//#endregion
|
|
2950
|
+
//#region src/openai-completer.ts
|
|
2951
|
+
/**
|
|
2952
|
+
* Where each provider's requests go. Named rather than left to the SDK, which would take
|
|
2953
|
+
* `OPENAI_BASE_URL` from the environment and send every form wherever it pointed.
|
|
2954
|
+
*/
|
|
2955
|
+
const OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
2956
|
+
/**
|
|
2957
|
+
* xAI's API, which xAI documents as compatible with OpenAI's SDKs and their Responses API at
|
|
2958
|
+
* this base URL. Taken from xAI's documentation as a web search summarised it on
|
|
2959
|
+
* 2026-10-10; the pages themselves refused the fetch, and no test here reaches xAI (0165).
|
|
2960
|
+
*/
|
|
2961
|
+
const XAI_BASE_URL = "https://api.x.ai/v1";
|
|
2962
|
+
/**
|
|
2963
|
+
* The most a turn may write, reasoning included — as generous as Anthropic's, and for the
|
|
2964
|
+
* same reason: an answer cut off at the limit is not an answer (`truncated`).
|
|
2965
|
+
*/
|
|
2966
|
+
const OPENAI_MAX_OUTPUT_TOKENS = 64e3;
|
|
2967
|
+
/** What a withheld answer says: the provider gives no words of its own for a filter. */
|
|
2968
|
+
const FILTERED = "The model service withheld its answer to this request.";
|
|
2969
|
+
/**
|
|
2970
|
+
* OpenAI's models, or xAI's Grok, through OpenAI's official SDK and the Responses API (0165).
|
|
2971
|
+
*
|
|
2972
|
+
* The briefing as a system message and the request as a user message, streamed and read
|
|
2973
|
+
* with the SDK's final-response helper. `store: false`, so the provider keeps no copy of
|
|
2974
|
+
* the form for later retrieval; what it keeps under its own terms is not this adapter's
|
|
2975
|
+
* to say. **The response's status is read before its text.** Incomplete at the output
|
|
2976
|
+
* limit is `truncated`, never half a document; withheld by the provider's filter, or a
|
|
2977
|
+
* refusal in the output, ends the person's run as a decline.
|
|
2978
|
+
*
|
|
2979
|
+
* The SDK would take an organisation and a project from the environment and send them as
|
|
2980
|
+
* headers — to xAI as well — so both are set to nothing here. It would take a log level
|
|
2981
|
+
* too, and with OPENAI_LOG=debug write a line for every request and response to the
|
|
2982
|
+
* console — a second request log, outside the rule the server's own is built by
|
|
2983
|
+
* (SAFETY-ANALYSIS C3) — so the level is set. OPENAI_CUSTOM_HEADERS cannot be undone here, so `modelSettings` refuses to start
|
|
2984
|
+
* with it.
|
|
2985
|
+
*/
|
|
2986
|
+
function createOpenAiCompleter(options) {
|
|
2987
|
+
const client = new OpenAI({
|
|
2988
|
+
apiKey: options.apiKey,
|
|
2989
|
+
baseURL: options.baseURL,
|
|
2990
|
+
organization: null,
|
|
2991
|
+
project: null,
|
|
2992
|
+
logLevel: "off",
|
|
2993
|
+
...options.fetch === void 0 ? {} : { fetch: options.fetch }
|
|
2994
|
+
});
|
|
2995
|
+
return { async complete(prompt, cancellation) {
|
|
2996
|
+
const controller = new AbortController();
|
|
2997
|
+
cancellation.onCancel(() => controller.abort());
|
|
2998
|
+
if (controller.signal.aborted) return {
|
|
2999
|
+
ok: false,
|
|
3000
|
+
failure: "cancelled"
|
|
3001
|
+
};
|
|
3002
|
+
let response;
|
|
3003
|
+
try {
|
|
3004
|
+
response = await client.responses.stream({
|
|
3005
|
+
model: options.model,
|
|
3006
|
+
input: [{
|
|
3007
|
+
role: "system",
|
|
3008
|
+
content: prompt.system
|
|
3009
|
+
}, {
|
|
3010
|
+
role: "user",
|
|
3011
|
+
content: prompt.user
|
|
3012
|
+
}],
|
|
3013
|
+
max_output_tokens: OPENAI_MAX_OUTPUT_TOKENS,
|
|
3014
|
+
store: false
|
|
3015
|
+
}, { signal: controller.signal }).finalResponse();
|
|
3016
|
+
} catch (error) {
|
|
3017
|
+
if (controller.signal.aborted) return {
|
|
3018
|
+
ok: false,
|
|
3019
|
+
failure: "cancelled"
|
|
3020
|
+
};
|
|
3021
|
+
return unavailable(error);
|
|
3022
|
+
}
|
|
3023
|
+
return read(response);
|
|
3024
|
+
} };
|
|
3025
|
+
}
|
|
3026
|
+
/** A finished response, by its status first. */
|
|
3027
|
+
function read(response) {
|
|
3028
|
+
if (response.status === "incomplete") {
|
|
3029
|
+
const reason = response.incomplete_details?.reason;
|
|
3030
|
+
if (reason === "max_output_tokens") return {
|
|
3031
|
+
ok: false,
|
|
3032
|
+
failure: "truncated"
|
|
3033
|
+
};
|
|
3034
|
+
if (reason === "content_filter") return {
|
|
3035
|
+
ok: false,
|
|
3036
|
+
failure: "refused",
|
|
3037
|
+
reason: FILTERED
|
|
3038
|
+
};
|
|
3039
|
+
return {
|
|
3040
|
+
ok: false,
|
|
3041
|
+
failure: "unavailable",
|
|
3042
|
+
cause: `The response was incomplete: ${String(reason)}.`
|
|
3043
|
+
};
|
|
3044
|
+
}
|
|
3045
|
+
if (response.status !== "completed") {
|
|
3046
|
+
const said = response.error?.message;
|
|
3047
|
+
return {
|
|
3048
|
+
ok: false,
|
|
3049
|
+
failure: "unavailable",
|
|
3050
|
+
cause: `The response ended ${String(response.status)}${said === void 0 ? "" : `: ${said}`}.`
|
|
3051
|
+
};
|
|
3052
|
+
}
|
|
3053
|
+
const parts = response.output.flatMap((item) => item.type === "message" ? item.content : []);
|
|
3054
|
+
const refusal = parts.find((part) => part.type === "refusal");
|
|
3055
|
+
if (refusal !== void 0) return {
|
|
3056
|
+
ok: false,
|
|
3057
|
+
failure: "refused",
|
|
3058
|
+
reason: refusal.refusal.trim() === "" ? FILTERED : refusal.refusal
|
|
3059
|
+
};
|
|
3060
|
+
return {
|
|
3061
|
+
ok: true,
|
|
3062
|
+
text: parts.flatMap((part) => part.type === "output_text" ? [part.text] : []).join("")
|
|
3063
|
+
};
|
|
3064
|
+
}
|
|
3065
|
+
/** An error from the SDK, as a failure: the status when the provider answered, the message for the log. */
|
|
3066
|
+
function unavailable(error) {
|
|
3067
|
+
const status = error instanceof APIError$1 && typeof error.status === "number" ? error.status : void 0;
|
|
3068
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
3069
|
+
return {
|
|
3070
|
+
ok: false,
|
|
3071
|
+
failure: "unavailable",
|
|
3072
|
+
...status === void 0 ? {} : { status },
|
|
3073
|
+
cause
|
|
3074
|
+
};
|
|
3075
|
+
}
|
|
3076
|
+
//#endregion
|
|
3077
|
+
//#region src/completers.ts
|
|
3078
|
+
/**
|
|
3079
|
+
* The adapter for the provider the operator named (0165): Anthropic through its own SDK,
|
|
3080
|
+
* OpenAI and xAI through OpenAI's, at each one's base URL.
|
|
3081
|
+
*
|
|
3082
|
+
* `fetch` is for the tests, which check where each provider's requests go.
|
|
3083
|
+
*/
|
|
3084
|
+
function createCompleter(settings, fetch) {
|
|
3085
|
+
const { apiKey, model } = settings;
|
|
3086
|
+
const transport = fetch === void 0 ? {} : { fetch };
|
|
3087
|
+
switch (settings.provider) {
|
|
3088
|
+
case "anthropic": return createAnthropicCompleter({
|
|
3089
|
+
apiKey,
|
|
3090
|
+
model,
|
|
3091
|
+
...transport
|
|
3092
|
+
});
|
|
3093
|
+
case "openai": return createOpenAiCompleter({
|
|
3094
|
+
apiKey,
|
|
3095
|
+
model,
|
|
3096
|
+
baseURL: OPENAI_BASE_URL,
|
|
3097
|
+
...transport
|
|
3098
|
+
});
|
|
3099
|
+
case "xai": return createOpenAiCompleter({
|
|
3100
|
+
apiKey,
|
|
3101
|
+
model,
|
|
3102
|
+
baseURL: XAI_BASE_URL,
|
|
3103
|
+
...transport
|
|
3104
|
+
});
|
|
3105
|
+
}
|
|
3106
|
+
}
|
|
3107
|
+
//#endregion
|
|
3108
|
+
export { SUBMISSION_TOKEN_HEADER as _, createLocalFileStore as a, createPostgresStorage as c, LOG_LEVELS as d, createServerLog as f, SCHEMA_HASH_HEADER as g, maxFileBytesFrom as h, startFileCollector as i, bootstrapSchema as l, workerLog as m, MODEL_PROVIDERS as n, sha256 as o, databaseNotices as p, modelSettings as r, createPostgresRateLimitStore as s, createCompleter as t, createApp as u };
|
|
3109
|
+
|
|
3110
|
+
//# sourceMappingURL=completers-DCFoZpo3.mjs.map
|