domma-cms 1.47.3 → 1.47.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,7 +29,7 @@ import {deleteForm, ensureCollectionForForm, ensureFormsDir, listForms, maxSubmi
29
29
  import {executeAction} from '../../services/actions.js';
30
30
  // The same rules the browser draws the buttons by (a Cancel link must be safe).
31
31
  import {buttonSettingsProblems, formClassProblems, publicButtonSettings} from '../../../public/js/form-submit.mjs';
32
- import {cleanUploadName, countRefusal, fileRefusal, submissionTooLarge, fileRules, fileValueText, isFileValue} from '../../../public/js/form-files.mjs';
32
+ import {cleanUploadName, countRefusal, fileRefs, fileRefusal, submissionTooLarge, fileRules, fileValueText, isFileValue} from '../../../public/js/form-files.mjs';
33
33
  import {sanitiseCsv} from '../../../public/js/collection-export.mjs';
34
34
  import {originOf, trustedOrigin} from '../../services/passwordReset.js';
35
35
  import {createTransport, mailFailure, sendFormEmail, TEST_EMAIL_FAILED} from '../../services/email.js';
@@ -47,9 +47,10 @@ import {
47
47
  import {getConfig} from '../../config.js';
48
48
  import {authenticate, requireAdmin, requirePermission} from '../../middleware/auth.js';
49
49
  import {hooks} from '../../services/hooks.js';
50
- import {parseCookies, SESSION_COOKIE} from '../../services/viewerSession.js';
50
+ import {parseCookies, readSessionToken, SESSION_COOKIE} from '../../services/viewerSession.js';
51
+ import {getUserById} from '../../services/users.js';
51
52
  import {pluginDenialFor} from '../../services/collectionAccess.js';
52
- import {deleteMedia, saveMedia} from '../../services/content.js';
53
+ import {deleteUpload, disposition, openUpload, parseUploadUrl, saveUpload, servedAs, statUpload, uploadUrl, validParts} from '../../services/formUploads.js';
53
54
  import {formMayCollect, servedType} from '../../services/mediaTypes.js';
54
55
  import {v4 as uuidv4} from 'uuid';
55
56
  import {ago, clip, plural} from '../../services/sidebarBadges.js';
@@ -252,7 +253,12 @@ function fileFieldsOnly(form, body, pending) {
252
253
  * Write the files of a submission that passed every check, into the reference objects
253
254
  * parseMultipartForm() left in `body`. Only the fields that will be kept are written -
254
255
  * nothing a bot, a rate-limited or refused submission or a switched-off field sent
255
- * reaches content/media/.
256
+ * is stored.
257
+ *
258
+ * They go to the private store (services/formUploads.js), one folder per submission:
259
+ * content/form-uploads/<form slug>/<upload id>/<8 hex>-<name> - never content/media/,
260
+ * so no visitor's file is public or in the Media library. Each reference's url is the
261
+ * admin-only route that serves it.
256
262
  *
257
263
  * If one cannot be written, the ones already written for this submission are removed
258
264
  * and the error is thrown - a submission keeps all its files or none.
@@ -260,10 +266,12 @@ function fileFieldsOnly(form, body, pending) {
260
266
  * @param {Map<string, {buffer: Buffer, filename: string}[]>} pending
261
267
  * @param {object} body
262
268
  * @param {Set<string>} keep - field names the submission keeps
263
- * @returns {Promise<string[]>} the media file names written (for removeFormFiles)
269
+ * @param {string} slug - the form's
270
+ * @returns {Promise<{slug: string, id: string, name: string}[]>} what was written (for removeFormFiles)
264
271
  */
265
- async function saveFormFiles(pending, body, keep) {
272
+ async function saveFormFiles(pending, body, keep, slug) {
266
273
  const written = [];
274
+ const uploadId = uuidv4();
267
275
  try {
268
276
  for (const [name, files] of pending) {
269
277
  if (!keep.has(name)) {
@@ -272,8 +280,8 @@ async function saveFormFiles(pending, body, keep) {
272
280
  }
273
281
  const refs = Array.isArray(body[name]) ? body[name] : [body[name]];
274
282
  for (let i = 0; i < files.length; i++) {
275
- const saved = await saveNewMedia(files[i]);
276
- written.push(saved.name);
283
+ const saved = await saveNewUpload(slug, uploadId, files[i]);
284
+ written.push(saved);
277
285
  refs[i].url = saved.url;
278
286
  }
279
287
  }
@@ -288,10 +296,10 @@ async function saveFormFiles(pending, body, keep) {
288
296
  * Write one visitor file under a new name. Never over another file: a clash of the random
289
297
  * prefix picks another, and a write that fails part way removes what it wrote.
290
298
  */
291
- async function saveNewMedia(file) {
299
+ async function saveNewUpload(slug, uploadId, file) {
292
300
  for (let tries = 0; ; tries++) {
293
301
  try {
294
- return await saveMedia(safeFilename(file.filename), file.buffer, {exclusive: true});
302
+ return await saveUpload(slug, uploadId, safeFilename(file.filename), file.buffer);
295
303
  } catch (err) {
296
304
  if (err.code !== 'EEXIST' || tries >= 4) throw err;
297
305
  }
@@ -299,9 +307,71 @@ async function saveNewMedia(file) {
299
307
  }
300
308
 
301
309
  /** Remove a submission's written files again (it was not stored after all). Never throws. */
302
- async function removeFormFiles(names) {
303
- for (const name of names || []) {
304
- try { await deleteMedia(name); } catch { /* already gone */ }
310
+ async function removeFormFiles(written) {
311
+ for (const f of written || []) {
312
+ try { await deleteUpload(f.slug, f.id, f.name); } catch { /* already gone */ }
313
+ }
314
+ }
315
+
316
+ /**
317
+ * The files a form saved for one submission: the references in its file fields
318
+ * to the private store (services/formUploads.js) under THIS form's slug.
319
+ * Nothing else is ever the form's to delete - not another address, a path,
320
+ * another form's folder, a field the form does not define as a file field,
321
+ * and never anything under /media/. A /media/ reference may be an old upload
322
+ * (form-uploads-migration.js moves the real ones into the store at boot) or a
323
+ * reference planted through a public create, a token or an editor pointing at
324
+ * a file a page uses, so it is left alone. Each name is checked again at the
325
+ * file (fileIn, in deleteUpload), so a tampered reference never reaches outside
326
+ * the store.
327
+ *
328
+ * @param {object} form
329
+ * @param {object} entry
330
+ * @returns {{url: string, slug: string, id: string, name: string}[]}
331
+ */
332
+ export function submissionUploads(form, entry) {
333
+ const out = new Map();
334
+ for (const f of form?.fields || []) {
335
+ if (!f || f.type !== 'file' || !f.name) continue;
336
+ for (const ref of fileRefs(entry?.data?.[f.name])) {
337
+ const up = parseUploadUrl(ref.url);
338
+ if (!up || up.slug !== form.slug) continue;
339
+ out.set(ref.url, {url: ref.url, ...up});
340
+ }
341
+ }
342
+ return [...out.values()];
343
+ }
344
+
345
+ /**
346
+ * After submissions are deleted, delete the files the form saved for them - but
347
+ * not a file another entry left in the collection still links to (an imported
348
+ * or copied entry). If the rest of the collection cannot be read, the files stay:
349
+ * an orphaned upload is better than a broken link. Never throws.
350
+ *
351
+ * @param {object} form
352
+ * @param {object[]} removed - the entries that were deleted
353
+ * @param {{warn: Function}} [log]
354
+ */
355
+ async function removeSubmissionUploads(form, removed, log = console) {
356
+ const files = new Map((removed || []).flatMap(e => submissionUploads(form, e)).map(u => [u.url, u]));
357
+ if (!files.size) return;
358
+ try {
359
+ const left = await listEntries(submissionsCollectionFor(form), {limit: 0});
360
+ for (const e of left.entries || []) {
361
+ for (const v of Object.values(e?.data || {})) {
362
+ for (const ref of fileRefs(v)) files.delete(ref.url);
363
+ }
364
+ }
365
+ } catch (err) {
366
+ log.warn(`[forms] Kept the files of deleted "${form?.slug}" submissions: the collection could not be read (${err.message})`);
367
+ return;
368
+ }
369
+ for (const up of files.values()) {
370
+ try {
371
+ await deleteUpload(up.slug, up.id, up.name);
372
+ } catch (err) {
373
+ log.warn(`[forms] Could not delete the upload "${up.name}" of a "${form?.slug}" submission: ${err.message}`);
374
+ }
305
375
  }
306
376
  }
307
377
 
@@ -819,17 +889,23 @@ export async function formsRoutes(fastify) {
819
889
  if (!(await canSeeForm(request.user, form))) return reply.status(404).send({ error: 'Form not found.' });
820
890
  if (await refusedPluginTarget(request, reply, form)) return;
821
891
  const target = submissionsCollectionFor(form);
892
+ const removed = [];
822
893
  try {
823
894
  // Its own collection: clear it. A shared one: delete only this form's
824
895
  // entries (one at a time - parallel writes to a file collection lose updates).
896
+ const mine = await submissionsOf(form);
825
897
  if (target === form.slug) {
826
898
  await clearEntries(target);
899
+ removed.push(...mine);
827
900
  } else {
828
- for (const e of await submissionsOf(form)) await deleteEntry(target, e.id);
901
+ for (const e of mine) { await deleteEntry(target, e.id); removed.push(e); }
829
902
  }
830
903
  } catch {
904
+ await removeSubmissionUploads(form, removed, fastify.log);
831
905
  return reply.status(404).send({ error: 'Collection not found for this form.' });
832
906
  }
907
+ // Their uploads go with them (only the files the form saved for them).
908
+ await removeSubmissionUploads(form, removed, fastify.log);
833
909
  forgetStats(slug);
834
910
  return { ok: true };
835
911
  });
@@ -850,16 +926,85 @@ export async function formsRoutes(fastify) {
850
926
  if (!(await canSeeForm(request.user, form))) return reply.status(404).send({ error: 'Form not found.' });
851
927
  if (await refusedPluginTarget(request, reply, form)) return;
852
928
  const target = submissionsCollectionFor(form);
929
+ let entry;
853
930
  try {
854
- if (!isSubmissionOf(form, await getEntry(target, id))) return reply.status(404).send({ error: 'Submission not found.' });
931
+ entry = await getEntry(target, id);
932
+ if (!isSubmissionOf(form, entry)) return reply.status(404).send({ error: 'Submission not found.' });
855
933
  await deleteEntry(target, id);
856
934
  } catch {
857
935
  return reply.status(404).send({ error: 'Submission not found.' });
858
936
  }
937
+ // Its uploads go with it (only the files the form saved for it).
938
+ await removeSubmissionUploads(form, [entry], fastify.log);
859
939
  forgetStats(slug);
860
940
  return { ok: true };
861
941
  });
862
942
 
943
+ // -----------------------------------------------------------------------
944
+ // GET /forms/:slug/uploads/:id/:name - one visitor upload, to a signed-in
945
+ // user who may read the form's submissions (the inbox's own permission),
946
+ // and only while a submission of this form links to it.
947
+ //
948
+ // Signed in means the Bearer token the admin sends, or - for a link opened
949
+ // from the notification email or the entry screen, which a browser follows
950
+ // with no header - the dm_session cookie login sets (services/viewerSession.js).
951
+ // This is a read with no side effects, the one kind of route that cookie
952
+ // may serve. Sent inline only as PNG, JPEG, GIF, WebP or PDF; anything else
953
+ // is an attachment of type application/octet-stream. Always nosniff,
954
+ // private/no-store, and a CSP under which nothing in the file can run.
955
+ // -----------------------------------------------------------------------
956
+ const signedInForFile = async (request, reply) => {
957
+ if (request.headers.authorization) return authenticate(request, reply);
958
+ let decoded = null;
959
+ const raw = readSessionToken(request);
960
+ if (raw) { try { decoded = fastify.jwt.verify(raw); } catch { decoded = null; } }
961
+ if (!decoded || decoded.type !== 'access') {
962
+ const html = /text\/html/i.test(String(request.headers.accept || ''));
963
+ reply.code(401).header('Cache-Control', 'private, no-store').header('X-Content-Type-Options', 'nosniff');
964
+ if (!html) return reply.send({statusCode: 401, error: 'Unauthorised', message: 'Sign in to view this file.'});
965
+ return reply.type('text/html; charset=utf-8').header('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'")
966
+ .send('<!doctype html><meta charset="utf-8"><title>Sign in to view this file</title>'
967
+ + '<body style="font-family:sans-serif;max-width:32rem;margin:3rem auto;padding:0 1rem">'
968
+ + '<h1 style="font-size:1.3rem">Sign in to view this file</h1>'
969
+ + '<p>Files sent with a form are private. <a href="/admin/">Sign in to the admin</a>, then open this link again.</p>');
970
+ }
971
+ request.user = decoded;
972
+ const user = decoded.id ? await getUserById(decoded.id) : null;
973
+ request.user.projects = Array.isArray(user?.projects) ? user.projects : [];
974
+ };
975
+
976
+ fastify.get('/forms/:slug/uploads/:id/:name', {preHandler: [signedInForFile, requirePermission('collections', 'read')]}, async (request, reply) => {
977
+ const {slug, id, name} = request.params;
978
+ reply.header('X-Content-Type-Options', 'nosniff').header('Cache-Control', 'private, no-store');
979
+ const notFound = () => reply.status(404).send({error: 'File not found.'});
980
+ if (!validParts(slug, id, name)) return notFound();
981
+ let form;
982
+ try { form = await readForm(slug); } catch { return notFound(); }
983
+ if (!(await canSeeForm(request.user, form))) return notFound();
984
+ if (await refusedPluginTarget(request, reply, form)) return;
985
+ const url = uploadUrl(slug, id, name);
986
+ let ref = null;
987
+ try {
988
+ for (const e of await submissionsOf(form)) {
989
+ if (!submissionUploads(form, e).some(u => u.url === url)) continue;
990
+ for (const v of Object.values(e.data || {})) ref = ref || fileRefs(v).find(r => r.url === url) || null;
991
+ break;
992
+ }
993
+ } catch { return notFound(); }
994
+ if (!ref) return notFound();
995
+ const st = await statUpload(slug, id, name);
996
+ if (!st) return notFound();
997
+ const {type, inline} = servedAs(name);
998
+ reply.header('Content-Type', type)
999
+ .header('Content-Length', st.size)
1000
+ .header('Content-Disposition', disposition(inline, ref.name || name))
1001
+ .header('Content-Security-Policy', inline && type === 'application/pdf'
1002
+ ? "default-src 'none'; style-src 'unsafe-inline'; img-src 'self' data: blob:; object-src 'self'; frame-ancestors 'none'"
1003
+ : "default-src 'none'; img-src 'self' data:; style-src 'unsafe-inline'; frame-ancestors 'none'; sandbox")
1004
+ .header('Cross-Origin-Resource-Policy', 'same-origin');
1005
+ return reply.send(openUpload(st.file));
1006
+ });
1007
+
863
1008
  // -----------------------------------------------------------------------
864
1009
  // POST /forms/:slug/submissions/:id/spam - flag or unflag one submission.
865
1010
  // Merged server-side (updateEntry replaces `data` wholesale), in the field
@@ -1034,7 +1179,7 @@ export async function formsRoutes(fastify) {
1034
1179
  let writtenFiles = [];
1035
1180
  if (pendingFiles?.size) {
1036
1181
  try {
1037
- writtenFiles = await saveFormFiles(pendingFiles, body, record ? visibleFieldNames : new Set());
1182
+ writtenFiles = await saveFormFiles(pendingFiles, body, record ? visibleFieldNames : new Set(), slug);
1038
1183
  } catch (err) {
1039
1184
  fastify.log.warn(`[forms] Saving the files of a "${slug}" submission failed: ${err.message}`);
1040
1185
  return reply.status(500).send({error: 'Your files could not be saved, so the submission was not sent. Please try again.'});
package/server/server.js CHANGED
@@ -366,6 +366,18 @@ try {
366
366
  app.log.warn(`[collections] Number-field repair skipped: ${err.message}`);
367
367
  }
368
368
 
369
+ // Visitors' form uploads out of content/media/ into the private store - one-shot, resumable.
370
+ try {
371
+ const {migrateFormUploads} = await import('./services/form-uploads-migration.js');
372
+ const r = await migrateFormUploads();
373
+ if (r.ran && (r.entries || r.missing || r.skipped)) {
374
+ app.log.info(`[forms] Form uploads made private: ${r.entries} submission(s) updated, ${r.moved} file(s) moved, ${r.copied} copied (also used elsewhere, so left in Media), ${r.missing} missing, ${r.skipped} skipped`);
375
+ }
376
+ if (r.pending.length) app.log.warn(`[forms] Form-uploads migration will retry next start for: ${r.pending.slice(0, 20).join(', ')}`);
377
+ } catch (err) {
378
+ app.log.warn(`[forms] Form-uploads migration skipped: ${err.message}`);
379
+ }
380
+
369
381
  /*
370
382
  * An admin change clears the response cache.
371
383
  *
@@ -492,7 +504,9 @@ app.addHook('onSend', async (request, reply) => {
492
504
  const url = request.raw.url || '';
493
505
  if (!url.startsWith('/api/')) return;
494
506
  if (url.startsWith('/api/v1/') || url.startsWith('/api/x/') || url.startsWith('/api/content/v1/')) return;
495
- reply.header('Cache-Control', 'no-store, must-revalidate');
507
+ // A route that already said private + no-store (a visitor's form upload) keeps it.
508
+ const own = String(reply.getHeader('cache-control') || '');
509
+ reply.header('Cache-Control', /\bprivate\b/.test(own) && /\bno-store\b/.test(own) ? own : 'no-store, must-revalidate');
496
510
  reply.header('Pragma', 'no-cache');
497
511
  reply.header('Expires', '0');
498
512
  if (RUNNING_CMS_VERSION && request.headers.authorization) reply.header('X-Domma-CMS-Version', RUNNING_CMS_VERSION);
@@ -6,6 +6,13 @@ import nodemailer from 'nodemailer';
6
6
  import {signedOnly} from './codePolicy.js';
7
7
  import {normaliseMailHost, resolveMailHost} from './mailHost.js';
8
8
  import {fileRefs, fileUrl, fileValueText} from '../../public/js/form-files.mjs';
9
+ import {answerLabel} from '../../public/js/form-answers.mjs';
10
+
11
+ /** Form field types that lay a form out rather than ask anything - no row in the email. */
12
+ const LAYOUT_FIELD_TYPES = new Set(['page-break', 'heading', 'paragraph', 'html', 'divider', 'spacer', 'banner']);
13
+
14
+ /** Text for a plain-text email part: line breaks kept, every other control character a space. */
15
+ const plainText = (s) => String(s ?? '').replace(/\r\n?/g, '\n').replace(/[\u0000-\u0008\u000b-\u001f\u007f]/g, ' ');
9
16
 
10
17
  /**
11
18
  * What a person is told when mail could not be sent. The mail server's own
@@ -283,25 +290,31 @@ export async function sendEmail(transport, {from, fromName, to, subject, html, t
283
290
  * Send an HTML + plain-text form submission notification email.
284
291
  * Builds a generic table of field→value pairs from the submitted data.
285
292
  *
286
- * @param {import('nodemailer').Transporter} transport
287
- * A file field's value (one reference, or a list from a `multiple` field) is shown
288
- * as a link per file, made absolute with `baseUrl` (site.json) when the site has one.
293
+ * Each answer reads as the admin's submission screen shows it (answerLabel,
294
+ * public/js/form-answers.mjs): an option's label rather than its stored value,
295
+ * Yes/No for a single checkbox, dd/mm/yyyy dates. A file field's value (one
296
+ * reference, or a list from a `multiple` field) is shown as a link per file,
297
+ * made absolute with `baseUrl` (site.json) when the site has one; the link needs
298
+ * sign-in, which the email says. Files are never attached. Everything a
299
+ * visitor or an author wrote is escaped in the HTML part, and the plain-text
300
+ * part carries no control characters but line breaks.
289
301
  *
290
302
  * @param {import('nodemailer').Transporter} transport
291
- * @param {{ from: string, fromName: string, to: string, subject: string, formTitle: string, fields: Array<{name: string, label: string, type?: string}>, data: Record<string, unknown>, baseUrl?: string }} opts
303
+ * @param {{ from: string, fromName: string, to: string, subject: string, formTitle: string, fields: Array<{name: string, label: string, type?: string, options?: object[]}>, data: Record<string, unknown>, baseUrl?: string }} opts
292
304
  * @returns {Promise<void>}
293
305
  * @throws {Error} If sending the email fails.
294
306
  */
295
307
  export async function sendFormEmail(transport, { from, fromName, to, subject, formTitle, fields, data, baseUrl = '' }) {
308
+ const asked = (fields || []).filter(f => f && f.name && !LAYOUT_FIELD_TYPES.has(f.type));
296
309
  const isFile = (field) => field.type === 'file' && fileRefs(data[field.name]).length > 0;
297
310
  const fileLinks = (val) => fileRefs(val).map((f) => {
298
311
  const url = fileUrl(f, baseUrl);
299
312
  const name = escapeHtml(f.name || url);
300
313
  return /^(https?:\/\/|\/)/i.test(url) ? `<a href="${escapeHtml(url)}">${name}</a>` : name;
301
314
  }).join('<br>');
302
- const rows = fields.map(field => {
315
+ const rows = asked.map(field => {
303
316
  const val = data[field.name] ?? '';
304
- const safe = isFile(field) ? fileLinks(val) : escapeHtml(String(val)).replace(/\n/g, '<br>');
317
+ const safe = isFile(field) ? fileLinks(val) : escapeHtml(plainText(answerLabel(field, val))).replace(/\n/g, '<br>');
305
318
  const safeLabel = escapeHtml(field.label || field.name);
306
319
  return `
307
320
  <tr>
@@ -310,24 +323,30 @@ export async function sendFormEmail(transport, { from, fromName, to, subject, fo
310
323
  </tr>`.trim();
311
324
  }).join('\n');
312
325
 
313
- const plainRows = fields.map(field => {
326
+ const plainRows = asked.map(field => {
314
327
  const val = data[field.name] ?? '';
315
- return `${field.label || field.name}: ${isFile(field) ? fileValueText(val, {base: baseUrl}) : val}`;
328
+ const shown = isFile(field) ? fileValueText(val, {base: baseUrl}) : answerLabel(field, val);
329
+ return `${plainText(field.label || field.name)}: ${plainText(shown)}`;
316
330
  }).join('\n');
317
331
 
332
+ // Visitors' files are private (services/formUploads.js): their links open for a signed-in admin only.
333
+ const hasFiles = asked.some(isFile);
334
+ const FILES_NOTE = 'Files sent with the form are private: sign in to the admin to view them.';
335
+
318
336
  const html = `
319
337
  <!DOCTYPE html>
320
338
  <html>
321
339
  <body style="font-family:sans-serif;max-width:640px;margin:0 auto;padding:20px;">
322
340
  <h2 style="color:#333;margin-bottom:4px;">New Form Submission</h2>
323
- <p style="color:#888;margin-top:0;font-size:.9rem;">${escapeHtml(formTitle)}</p>
341
+ <p style="color:#888;margin-top:0;font-size:.9rem;">${escapeHtml(formTitle ?? '')}</p>
324
342
  <table style="width:100%;border-collapse:collapse;margin-top:16px;">
325
343
  ${rows}
326
- </table>
344
+ </table>${hasFiles ? `
345
+ <p style="color:#888;font-size:.85rem;margin-top:12px;">${escapeHtml(FILES_NOTE)}</p>` : ''}
327
346
  </body>
328
347
  </html>`.trim();
329
348
 
330
- const text = `New form submission: ${formTitle}\n\n${plainRows}`;
349
+ const text = `New form submission: ${plainText(formTitle ?? '')}\n\n${plainRows}${hasFiles ? `\n\n${FILES_NOTE}` : ''}`;
331
350
 
332
351
  try {
333
352
  const info = await transport.sendMail({
@@ -0,0 +1,254 @@
1
+ /**
2
+ * Form-uploads migration - one-off move of visitors' form uploads out of
3
+ * content/media/ into the private store (services/formUploads.js).
4
+ *
5
+ * Before uploads were private, saveFormFiles() wrote each file to
6
+ * content/media/<8 hex>-<name>, served to anyone at /media/ and listed in the
7
+ * Media library. This finds every file a form SUBMISSION (an entry stamped
8
+ * `meta.source: 'form:<slug>'`) references that way - in one of the form's file
9
+ * fields, with exactly that shape, resolved through fileIn inside the media root,
10
+ * and a regular file (never a link or a folder) - and nothing else in Media:
11
+ *
12
+ * 1. copies it to content/form-uploads/<form slug>/<entry id>/<name>
13
+ * (never over a file already there: one there is a previous run's copy);
14
+ * 2. rewrites the reference to the admin-only route (uploadUrl);
15
+ * 3. removes the media copy - unless the file's name appears anywhere else in
16
+ * the site's content (a page, block, menu, view, config, another
17
+ * collection's data, an entry that is not a submission), where removing it
18
+ * would break a link. Then the media copy is left: COPIED, not moved.
19
+ *
20
+ * The "used elsewhere" test is a text search for the stored name (it carries a
21
+ * random 8-hex prefix, so a match is not a coincidence) over content/ and
22
+ * config/ text files, minus the submissions themselves. Collections kept in
23
+ * MongoDB are searched only for the forms' own collections.
24
+ *
25
+ * Idempotent and resumable: a reference already private is skipped; a missing
26
+ * file is skipped (and counted) with its reference left as it was; the media
27
+ * copies still to remove are recorded in content/form-uploads/.migration.json
28
+ * as each entry is rewritten, and are removed only once no submission still
29
+ * links to them. The marker (content/.migrations.json) is written when a run
30
+ * finishes with nothing left to retry.
31
+ */
32
+ import fs from 'fs/promises';
33
+ import {existsSync} from 'fs';
34
+ import path from 'path';
35
+ import {config} from '../config.js';
36
+ import {fileRefs} from '../../public/js/form-files.mjs';
37
+ import {fileIn} from './safePath.js';
38
+ import {formUploadsRoot, keepUploadsOutOfGit, parseUploadUrl, uploadPath, uploadUrl, validParts} from './formUploads.js';
39
+
40
+ export const MIGRATION_ID = 'form-uploads-private';
41
+
42
+ /** What saveFormFiles() stored before uploads were private. */
43
+ const LEGACY_URL = /^\/media\/([0-9a-f]{8}-[\w.-]{1,200})$/;
44
+ const TEXT_FILE = /\.(md|markdown|json|html?|txt|css|js|mjs|ya?ml|xml|csv)$/i;
45
+ const MAX_TEXT_BYTES = 8 * 1024 * 1024;
46
+
47
+ const contentRoot = () => path.dirname(path.resolve(config.content.collectionsDir));
48
+ const defaultMarkerPath = () => path.join(contentRoot(), '.migrations.json');
49
+
50
+ async function readJson(file, fallback) {
51
+ try { return JSON.parse(await fs.readFile(file, 'utf8')) ?? fallback; } catch { return fallback; }
52
+ }
53
+
54
+ /** The collection a form's submissions are in, and whether an entry is one of them (as routes/api/forms.js). */
55
+ const collectionOf = (form) => (form?.actions?.collection?.enabled && form.actions.collection.slug) ? form.actions.collection.slug : form?.slug;
56
+ /**
57
+ * A real submission of the form: stamped `meta.source: 'form:<slug>'` by the submit
58
+ * route. Only these are migrated - an entry made any other way (the admin, a token,
59
+ * a public create, an import) may carry a planted /media/ reference, and must not
60
+ * pull a Media file out of public view.
61
+ */
62
+ const isSubmissionOf = (form, entry) => entry?.meta?.source === `form:${form.slug}`;
63
+
64
+ /** The legacy media name a reference points at, or null. */
65
+ export function legacyName(ref) {
66
+ const m = LEGACY_URL.exec(String(ref?.url || ''));
67
+ return m && m[1] !== '.' && m[1] !== '..' ? m[1] : null;
68
+ }
69
+
70
+ /** Every text file under dir, skipping the folders in `skip` (absolute paths). */
71
+ async function* textFiles(dir, skip) {
72
+ let list;
73
+ try { list = await fs.readdir(dir, {withFileTypes: true}); } catch { return; }
74
+ for (const d of list) {
75
+ const p = path.join(dir, d.name);
76
+ if (skip.has(p)) continue;
77
+ if (d.isDirectory()) yield* textFiles(p, skip);
78
+ else if (d.isFile() && TEXT_FILE.test(d.name)) yield p;
79
+ }
80
+ }
81
+
82
+ /**
83
+ * Run the migration.
84
+ *
85
+ * @param {{markerPath?: string, mediaDir?: string, uploadsRoot?: string, searchRoots?: string[], force?: boolean, log?: {info: Function, warn: Function}}} [opts]
86
+ * @returns {Promise<{ran: boolean, entries: number, moved: number, copied: number, missing: number, skipped: number, pending: string[]}>}
87
+ */
88
+ export async function migrateFormUploads({
89
+ markerPath = defaultMarkerPath(),
90
+ mediaDir = path.resolve(config.content.mediaDir),
91
+ uploadsRoot = formUploadsRoot(),
92
+ searchRoots = [contentRoot(), path.resolve(contentRoot(), '..', 'config')],
93
+ force = false
94
+ } = {}) {
95
+ const result = {ran: false, entries: 0, moved: 0, copied: 0, missing: 0, skipped: 0, pending: []};
96
+ const marker = await readJson(markerPath, {});
97
+ if (marker[MIGRATION_ID] && !force) return result;
98
+ result.ran = true;
99
+
100
+ const {listForms} = await import('./forms.js');
101
+ const {getAdapter} = await import('./adapterRegistry.js');
102
+ const cache = await import('./cache/index.js');
103
+
104
+ const progressFile = path.join(uploadsRoot, '.migration.json');
105
+ const progress = await readJson(progressFile, {remove: []});
106
+ const toRemove = new Set(Array.isArray(progress.remove) ? progress.remove : []);
107
+ const saveProgress = async () => {
108
+ keepUploadsOutOfGit(uploadsRoot);
109
+ await fs.writeFile(progressFile, JSON.stringify({remove: [...toRemove]}, null, 2) + '\n');
110
+ };
111
+
112
+ // 1. The forms, their submissions, and the legacy files those reference.
113
+ const forms = (await listForms()).filter(f => f && f.slug);
114
+ const work = []; // {form, collection, adapter, entries}
115
+ const submissionFiles = new Set(); // data.json files of the forms' collections - searched entry by entry instead
116
+ let haystack = ''; // text of everything that is not a submission
117
+ for (const form of forms) {
118
+ const slug = collectionOf(form);
119
+ if (!form.fields?.some(f => f?.type === 'file')) continue;
120
+ try {
121
+ const adapter = await getAdapter(slug);
122
+ const entries = await adapter.all(slug);
123
+ work.push({form, slug, adapter, entries});
124
+ submissionFiles.add(path.join(path.resolve(config.content.collectionsDir), slug));
125
+ } catch {
126
+ result.pending.push(form.slug);
127
+ }
128
+ }
129
+ // A collection's entries that are no form's submission (several forms may share one) are "elsewhere".
130
+ for (const slug of new Set(work.map(w => w.slug))) {
131
+ const sharing = forms.filter(f => collectionOf(f) === slug);
132
+ const {entries} = work.find(w => w.slug === slug);
133
+ for (const e of entries) if (!sharing.some(f => isSubmissionOf(f, e))) haystack += JSON.stringify(e.data ?? {}) + '\n';
134
+ }
135
+ const work2 = work.filter(w => w.entries.some(e => isSubmissionOf(w.form, e)
136
+ && w.form.fields.some(f => f?.type === 'file' && fileRefs(e.data?.[f.name]).some(legacyName))));
137
+
138
+ if (work2.length || toRemove.size) {
139
+ // 2. Where else a name appears: content/ and config/ text, minus media, the store,
140
+ // and the forms' own collection folders (their other entries were added above).
141
+ const skip = new Set([mediaDir, uploadsRoot, ...submissionFiles]);
142
+ for (const root of searchRoots) {
143
+ for await (const file of textFiles(root, skip)) {
144
+ try {
145
+ const st = await fs.stat(file);
146
+ if (st.size <= MAX_TEXT_BYTES) haystack += await fs.readFile(file, 'utf8') + '\n';
147
+ } catch { /* unreadable: not a use */ }
148
+ }
149
+ }
150
+ }
151
+ const usedElsewhere = (name) => haystack.includes(name);
152
+
153
+ // 3. Copy, rewrite, record.
154
+ for (const {form, slug, adapter, entries} of work2) {
155
+ let changed = 0;
156
+ for (const entry of entries) {
157
+ if (!isSubmissionOf(form, entry) || !entry?.data) continue;
158
+ let touched = false;
159
+ const data = {...entry.data};
160
+ const moved = [];
161
+ try { for (const f of form.fields) {
162
+ if (f?.type !== 'file' || !f.name || data[f.name] == null) continue;
163
+ const one = !Array.isArray(data[f.name]);
164
+ const list = (one ? [data[f.name]] : data[f.name]).map(ref => {
165
+ const name = legacyName(ref);
166
+ if (!name) return ref;
167
+ if (!validParts(form.slug, String(entry.id ?? ''), name)) { result.skipped++; return ref; }
168
+ let src;
169
+ try { src = fileIn(mediaDir, name); } catch { result.skipped++; return ref; }
170
+ return {ref, name, src};
171
+ });
172
+ const next = [];
173
+ for (const item of list) {
174
+ if (!item || !item.src) { next.push(item); continue; }
175
+ const {ref, name, src} = item;
176
+ const dest = uploadPath(form.slug, String(entry.id), name, uploadsRoot);
177
+ // Only a regular file: lstat does not follow a link, and copyFile would.
178
+ const kind = await fs.lstat(src).then(st => (st.isFile() ? 'file' : 'other'), () => 'none');
179
+ if (kind === 'other') { result.skipped++; next.push(ref); continue; }
180
+ if (kind === 'none' && !existsSync(dest)) { result.missing++; next.push(ref); continue; }
181
+ if (!existsSync(dest)) {
182
+ await fs.mkdir(path.dirname(dest), {recursive: true});
183
+ keepUploadsOutOfGit(uploadsRoot);
184
+ await fs.copyFile(src, dest, fs.constants.COPYFILE_EXCL).catch((err) => { if (err.code !== 'EEXIST') throw err; });
185
+ }
186
+ next.push({...ref, url: uploadUrl(form.slug, String(entry.id), name)});
187
+ moved.push(name);
188
+ touched = true;
189
+ }
190
+ data[f.name] = one ? next[0] : next;
191
+ } } catch {
192
+ // A copy that failed (a full disk): this entry is left as it was, for the next start.
193
+ result.pending.push(`${form.slug}/${entry.id}`);
194
+ continue;
195
+ }
196
+ if (!touched) continue;
197
+ // Recorded before the rewrite: a run cut short after it still removes the media copy next time.
198
+ const removing = moved.filter(name => !usedElsewhere(name));
199
+ for (const name of removing) toRemove.add(name);
200
+ try {
201
+ await saveProgress();
202
+ await adapter.update(slug, entry.id, {...entry, data});
203
+ } catch {
204
+ result.pending.push(`${form.slug}/${entry.id}`);
205
+ continue;
206
+ }
207
+ result.copied += moved.length - removing.length;
208
+ result.moved += removing.length;
209
+ entry.data = data;
210
+ changed++;
211
+ }
212
+ result.entries += changed;
213
+ if (changed) await cache.invalidateTags([`collection:${slug}`]).catch(() => {});
214
+ }
215
+
216
+ // 4. Remove the media copies no submission links to any more (and nothing else uses).
217
+ // Only a name a submission now links to in the store, with the copy there - so
218
+ // whatever the progress file says, nothing else in Media is ever removed.
219
+ const stillLinked = new Set();
220
+ const migrated = new Set();
221
+ for (const {form, entries} of work) {
222
+ for (const e of entries) {
223
+ if (!isSubmissionOf(form, e)) continue;
224
+ for (const f of form.fields) {
225
+ if (f?.type !== 'file' || !f.name) continue;
226
+ for (const ref of fileRefs(e.data?.[f.name])) {
227
+ const n = legacyName(ref);
228
+ if (n) { stillLinked.add(n); continue; }
229
+ const priv = parseUploadUrl(ref.url);
230
+ if (priv && priv.slug === form.slug && existsSync(uploadPath(priv.slug, priv.id, priv.name, uploadsRoot))) migrated.add(priv.name);
231
+ }
232
+ }
233
+ }
234
+ }
235
+ for (const name of [...toRemove]) {
236
+ if (!migrated.has(name) || stillLinked.has(name) || usedElsewhere(name)) { toRemove.delete(name); continue; }
237
+ try {
238
+ const file = fileIn(mediaDir, name);
239
+ if (!(await fs.lstat(file)).isFile()) { toRemove.delete(name); continue; }
240
+ await fs.unlink(file);
241
+ } catch (err) {
242
+ if (err.code !== 'ENOENT') { result.pending.push(name); continue; }
243
+ }
244
+ toRemove.delete(name);
245
+ }
246
+ if (toRemove.size) await saveProgress();
247
+ else await fs.rm(progressFile, {force: true});
248
+
249
+ if (!result.pending.length) {
250
+ const latest = await readJson(markerPath, {});
251
+ await fs.writeFile(markerPath, JSON.stringify({...latest, [MIGRATION_ID]: new Date().toISOString()}, null, 2) + '\n');
252
+ }
253
+ return result;
254
+ }