@materializecms/server 0.2.0 → 0.4.0

package/src/mail.js

@@ -1,3 +1,6 @@
+import { Resolver } from 'node:dns/promises';
+import { connect } from 'node:net';
+import { COMMON_DKIM, providerByHost, providerById } from '@materializecms/core/mail';
 import nodemailer from 'nodemailer';
 export const EMPTY_MAIL = {
     host: '',
@@ -34,3 +37,170 @@ export function submissionText(page, data) {
     const lines = Object.entries(data).map(([k, v]) => `${k}:\n${v}\n`);
     return `Nuovo messaggio dal modulo del sito${page ? ` (pagina ${page})` : ''}.\n\n${lines.join('\n')}`;
 }
+// ---------- why sending failed, in words ----------
+/** Whether the server answers on a port at all: the way to tell a blocked port from a wrong setting. */
+export function portOpen(host, port, timeout = 4000) {
+    return new Promise((resolve) => {
+        const socket = connect({ host, port, timeout });
+        const done = (ok) => {
+            socket.destroy();
+            resolve(ok);
+        };
+        socket.once('connect', () => done(true));
+        socket.once('timeout', () => done(false));
+        socket.once('error', () => done(false));
+    });
+}
+/**
+ * A sending error from nodemailer, explained: wrong server name, a port the hosting blocks (then
+ * the ports that do answer), security that does not match the port, credentials refused, a sender
+ * the provider has not verified, a limit reached.
+ */
+export async function explainMailError(error, m) {
+    const e = error;
+    const text = [e.message, e.response, ...(e.errors ?? []).flatMap((x) => [x.code, x.message])].join(' ');
+    const provider = providerById(m.provider) ?? providerByHost(m.host);
+    if (e.code === 'EDNS' || /ENOTFOUND|EAI_AGAIN/.test(text))
+        return {
+            message: `Il server «${m.host}» non esiste`,
+            hint: 'Controlla il nome del server: un errore di battitura basta.',
+        };
+    if (/wrong version number|ssl3_get_record|packet length too long|unknown protocol/i.test(text))
+        return {
+            message: 'La sicurezza non corrisponde alla porta',
+            hint: 'Di solito: porta 587 con STARTTLS, porta 465 con TLS.',
+        };
+    if (/certificate|self.signed|CERT_/i.test(text))
+        return {
+            message: 'Il certificato del server di posta non è valido',
+            hint: 'Controlla il nome del server: deve essere esattamente quello indicato dal fornitore.',
+        };
+    if (e.code === 'ETIMEDOUT' ||
+        e.code === 'ECONNECTION' ||
+        (e.code === 'ESOCKET' && e.command === 'CONN') ||
+        /ECONNREFUSED|ETIMEDOUT|ECONNRESET|EHOSTUNREACH|ENETUNREACH|Greeting never received/i.test(text)) {
+        const others = [587, 465, 2525].filter((p) => p !== m.port);
+        const open = [];
+        for (const p of others)
+            if (await portOpen(m.host, p))
+                open.push(p);
+        if (open.length)
+            return {
+                message: `Il server non risponde sulla porta ${m.port}, ma risponde sulla ${open.join(' e sulla ')}`,
+                hint: `Prova la porta ${open[0]}${open[0] === 465 ? ' con TLS' : ' con STARTTLS'}: la ${m.port} potrebbe essere bloccata dal fornitore del server.`,
+            };
+        return {
+            message: `Il server di posta non risponde (porta ${m.port})`,
+            hint: 'Molti fornitori di VPS bloccano le porte della posta in uscita, soprattutto per gli account nuovi: chiedi al loro supporto di sbloccarle, oppure usa un servizio che accetta la porta 2525.',
+        };
+    }
+    if (e.code === 'EAUTH' || e.responseCode === 535 || e.responseCode === 534)
+        return {
+            message: 'Utente o password rifiutati dal server di posta',
+            hint: provider ? `${provider.name}: ${provider.userHint} ${provider.passHint}` : 'Controlla utente e password.',
+        };
+    if (e.code === 'EENVELOPE' || (e.responseCode && e.responseCode >= 550 && e.responseCode <= 554)) {
+        const sender = e.command === 'MAIL FROM' || /sender|from address|not verified|not allowed to send/i.test(text);
+        return sender
+            ? {
+                message: 'Il server di posta non accetta questo mittente',
+                hint: `${provider?.note ?? 'Il mittente deve essere un indirizzo o un dominio verificato presso il fornitore.'}`,
+            }
+            : { message: 'Il destinatario è stato rifiutato', hint: 'Controlla l’indirizzo del destinatario dei moduli.' };
+    }
+    if (e.responseCode && e.responseCode >= 421 && e.responseCode <= 452)
+        return {
+            message: 'Il server di posta ha rimandato l’invio',
+            hint: 'Spesso è un limite di messaggi raggiunto: riprova più tardi o controlla il piano del fornitore.',
+        };
+    return { message: e.message || text.trim() || 'Invio non riuscito' };
+}
+const txt = async (resolver, name) => (await resolver.resolveTxt(name).catch(() => [])).map((parts) => parts.join(''));
+/**
+ * What mail servers check before trusting a message from this domain. Without SPF and DKIM the
+ * messages tend to end in spam; Gmail and Yahoo also ask for DMARC from whoever sends a lot.
+ */
+export async function checkSenderDomain(domain, providerId) {
+    const resolver = new Resolver({ timeout: 3000, tries: 2 });
+    const provider = providerById(providerId);
+    const checks = [];
+    const spf = (await txt(resolver, domain)).filter((r) => /^v=spf1\b/i.test(r));
+    if (!spf.length)
+        checks.push({
+            id: 'spf',
+            state: 'bad',
+            title: 'Manca il record SPF',
+            detail: `Senza SPF le email da ${domain} finiscono spesso nello spam. ${provider?.spf ? `Aggiungi nel DNS un record TXT su ${domain}: v=spf1 include:${provider.spf[0]} ~all` : 'Il fornitore della posta indica quale record TXT aggiungere nel DNS.'}`,
+        });
+    else if (spf.length > 1)
+        checks.push({
+            id: 'spf',
+            state: 'bad',
+            title: 'Ci sono due record SPF',
+            detail: 'I server di posta li scartano entrambi: uniscili in un solo record TXT che inizia con v=spf1.',
+        });
+    else if (!/\b(include:|a\b|a:|mx\b|mx:|ip4:|ip6:|exists:|redirect=)/i.test(spf[0]))
+        checks.push({
+            id: 'spf',
+            state: 'bad',
+            title: 'Il record SPF non autorizza nessun server',
+            detail: `${spf[0]}: così il dominio dichiara di non inviare email, e i messaggi vengono rifiutati. Aggiungi il server del tuo fornitore${provider?.spf ? ` (include:${provider.spf[0]})` : ''}.`,
+        });
+    else if (provider?.spf && !provider.spf.some((h) => spf[0].includes(`include:${h}`)))
+        checks.push({
+            id: 'spf',
+            state: 'warn',
+            title: `Il record SPF non autorizza ${provider.name}`,
+            detail: `Aggiungi include:${provider.spf[0]} al record attuale, prima di ~all o -all: ${spf[0]}`,
+        });
+    else
+        checks.push({ id: 'spf', state: 'ok', title: 'Record SPF presente', detail: spf[0] });
+    const selectors = provider ? provider.dkim : COMMON_DKIM;
+    if (provider && !selectors.length)
+        checks.push({
+            id: 'dkim',
+            state: 'info',
+            title: 'Firma DKIM: da controllare nel pannello del fornitore',
+            detail: `${provider.name} dà a ogni account nomi propri per la firma DKIM: nel suo pannello vedi se il dominio è verificato.`,
+        });
+    else {
+        let found = null;
+        // An empty key (p=) is a revoked one: it means "no signature".
+        for (const s of selectors)
+            if ((await txt(resolver, `${s}._domainkey.${domain}`)).some((r) => /\bp=[A-Za-z0-9+/]/.test(r))) {
+                found = s;
+                break;
+            }
+        checks.push(found
+            ? {
+                id: 'dkim',
+                state: 'ok',
+                title: 'Firma DKIM presente',
+                detail: `Trovata con il nome ${found}._domainkey.${domain}.`,
+            }
+            : {
+                id: 'dkim',
+                state: 'warn',
+                title: 'Firma DKIM non trovata',
+                detail: `Non l’ho trovata con i nomi più comuni${provider ? ` di ${provider.name}` : ''}. Nel pannello del fornitore attiva la firma del dominio: ti dà i record da aggiungere nel DNS.`,
+            });
+    }
+    const dmarc = (await txt(resolver, `_dmarc.${domain}`)).find((r) => /^v=DMARC1\b/i.test(r));
+    if (!dmarc)
+        checks.push({
+            id: 'dmarc',
+            state: 'warn',
+            title: 'Manca il record DMARC',
+            detail: `Gmail e Yahoo lo chiedono a chi invia molte email. Un inizio sicuro, che osserva senza bloccare niente: record TXT su _dmarc.${domain} con v=DMARC1; p=none`,
+        });
+    else {
+        const policy = /\bp=(none|quarantine|reject)/i.exec(dmarc)?.[1]?.toLowerCase();
+        checks.push({
+            id: 'dmarc',
+            state: 'ok',
+            title: 'Record DMARC presente',
+            detail: policy === 'none' ? `Osserva soltanto (p=none): ${dmarc}` : dmarc,
+        });
+    }
+    return checks;
+}

package/src/offsite.d.ts

@@ -0,0 +1,52 @@
+import type { Backups } from './backups.ts';
+import { type RemoteStorage, type S3Target, type WebDavTarget } from './remote-storage.ts';
+/**
+ * A copy of the backups outside the server, encrypted before it leaves: every night the newest
+ * database copy, and the media and fonts not sent yet (uploaded files never change, so they go
+ * once). On the storage the media are only added; database copies beyond `keep` are deleted.
+ *
+ * Encryption: AES-256-GCM with a random key made by Materialize, the "recovery key", shown once
+ * to the administrator. Whoever reads the bucket or the WebDAV folder sees noise. Files are
+ * encrypted in blocks of 1 MiB, each authenticated, the last one marked: a file cut short (an
+ * upload that stopped) is refused instead of giving back half a database.
+ */
+export interface OffsiteSettings {
+    kind: 'off' | 's3' | 'webdav';
+    s3?: S3Target & {
+        prefix?: string;
+    };
+    webdav?: WebDavTarget;
+    /** How many database copies stay on the storage; older ones are deleted. */
+    keep: number;
+    /** The recovery key: base64url of 32 random bytes, with the "mzk1-" mark. */
+    key?: string;
+}
+export interface OffsiteStatus {
+    at: string;
+    ok: boolean;
+    message?: string;
+    database?: string;
+    media?: number;
+    fonts?: number;
+}
+export declare const DEFAULT_OFFSITE: OffsiteSettings;
+/** Encrypted files end with this, so they are told apart from anything else in the folder. */
+export declare const SUFFIX = ".mzb";
+export declare function newRecoveryKey(): string;
+export declare function parseRecoveryKey(text: string): Buffer;
+/** Encrypts `source` into `target` (block by block: large videos never sit in memory). */
+export declare function encryptFile(source: string, target: string, key: Buffer): Promise<void>;
+/** The other way; throws on a wrong key, a changed file or one cut short. */
+export declare function decryptFile(source: string, target: string, key: Buffer): Promise<void>;
+export declare function storageOf(s: OffsiteSettings): RemoteStorage | null;
+/**
+ * One run: what is missing goes up, old database copies go away. `fresh` makes a new database
+ * copy first (the "Esegui ora" button); otherwise the newest daily copy is sent.
+ */
+export declare function runOffsite(s: OffsiteSettings, backups: Backups, mediaDir: string, fresh?: boolean, 
+/** For tests: a storage in memory. */
+storage?: RemoteStorage | null): Promise<OffsiteStatus>;
+/** "Prova il collegamento": writes, reads back, lists and deletes a small file. */
+export declare function testOffsite(s: OffsiteSettings): Promise<void>;
+/** Every encrypted file under a folder (or one file), decrypted next to it without the suffix. */
+export declare function decryptTree(source: string, key: Buffer): Promise<string[]>;

package/src/offsite.js

@@ -0,0 +1,230 @@
+import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto';
+import { mkdtemp, open, readdir, rm, stat } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join, relative, resolve } from 'node:path';
+import { createS3Storage, createWebDavStorage, } from './remote-storage.js';
+export const DEFAULT_OFFSITE = { kind: 'off', keep: 30 };
+const KEY_MARK = 'mzk1-';
+const MAGIC = Buffer.from('MZB1');
+const BLOCK = 1024 * 1024;
+/** Encrypted files end with this, so they are told apart from anything else in the folder. */
+export const SUFFIX = '.mzb';
+export function newRecoveryKey() {
+    return KEY_MARK + randomBytes(32).toString('base64url');
+}
+export function parseRecoveryKey(text) {
+    const clean = text.trim().replace(/\s+/g, '');
+    const raw = clean.startsWith(KEY_MARK) ? clean.slice(KEY_MARK.length) : clean;
+    const key = Buffer.from(raw, 'base64url');
+    if (key.length !== 32)
+        throw new Error('La chiave di ripristino non è valida: deve iniziare con mzk1-');
+    return key;
+}
+const nonceOf = (prefix, index) => {
+    const n = Buffer.alloc(12);
+    prefix.copy(n, 0);
+    n.writeUInt32BE(index, 8);
+    return n;
+};
+/** Encrypts `source` into `target` (block by block: large videos never sit in memory). */
+export async function encryptFile(source, target, key) {
+    const input = await open(source, 'r');
+    const output = await open(target, 'w');
+    try {
+        const { size } = await input.stat();
+        const blocks = Math.max(1, Math.ceil(size / BLOCK));
+        const prefix = randomBytes(8);
+        await output.write(Buffer.concat([MAGIC, prefix]));
+        const buffer = Buffer.alloc(BLOCK);
+        for (let i = 0; i < blocks; i++) {
+            const { bytesRead } = await input.read(buffer, 0, BLOCK, i * BLOCK);
+            const cipher = createCipheriv('aes-256-gcm', key, nonceOf(prefix, i));
+            cipher.setAAD(Buffer.from([i === blocks - 1 ? 1 : 0]));
+            const sealed = Buffer.concat([cipher.update(buffer.subarray(0, bytesRead)), cipher.final(), cipher.getAuthTag()]);
+            const length = Buffer.alloc(4);
+            length.writeUInt32BE(bytesRead);
+            await output.write(Buffer.concat([length, sealed]));
+        }
+    }
+    finally {
+        await input.close();
+        await output.close();
+    }
+}
+/** The other way; throws on a wrong key, a changed file or one cut short. */
+export async function decryptFile(source, target, key) {
+    const input = await open(source, 'r');
+    const output = await open(target, 'w');
+    let ok = false;
+    try {
+        const head = Buffer.alloc(12);
+        await input.read(head, 0, 12, 0);
+        if (!head.subarray(0, 4).equals(MAGIC))
+            throw new Error('Non è un file di backup cifrato da Materialize');
+        const prefix = head.subarray(4, 12);
+        let position = 12;
+        for (let i = 0;; i++) {
+            const length = Buffer.alloc(4);
+            if ((await input.read(length, 0, 4, position)).bytesRead < 4)
+                throw new Error('Il file è incompleto');
+            const size = length.readUInt32BE();
+            if (size > BLOCK)
+                throw new Error('Il file è danneggiato');
+            const sealed = Buffer.alloc(size + 16);
+            if ((await input.read(sealed, 0, size + 16, position + 4)).bytesRead < size + 16)
+                throw new Error('Il file è incompleto');
+            position += 4 + size + 16;
+            const end = position >= (await input.stat()).size;
+            const decipher = createDecipheriv('aes-256-gcm', key, nonceOf(prefix, i));
+            decipher.setAAD(Buffer.from([end ? 1 : 0]));
+            decipher.setAuthTag(sealed.subarray(size));
+            try {
+                await output.write(Buffer.concat([decipher.update(sealed.subarray(0, size)), decipher.final()]));
+            }
+            catch {
+                throw new Error(end ? 'Chiave sbagliata, oppure il file è stato modificato' : 'Il file è incompleto o è stato modificato');
+            }
+            if (end)
+                break;
+        }
+        ok = true;
+    }
+    finally {
+        await input.close();
+        await output.close();
+        if (!ok)
+            await rm(target, { force: true });
+    }
+}
+export function storageOf(s) {
+    if (s.kind === 's3' && s.s3)
+        return createS3Storage(s.s3, s.s3.prefix ?? '');
+    if (s.kind === 'webdav' && s.webdav)
+        return createWebDavStorage(s.webdav);
+    return null;
+}
+const README = `Backup di Materialize
+
+Questi file sono cifrati (AES-256-GCM): senza la chiave di ripristino non si leggono.
+
+  database/   le copie del database, dalla più recente
+  media/      le immagini e i file caricati
+  fonts/      i caratteri del sito
+
+Per rimetterli su un server:
+  1. scarica la cartella (con lo strumento del servizio, per esempio rclone);
+  2. nella cartella del sito: npx materialize decrypt <cartella scaricata> --key <chiave>
+  3. npx materialize restore <cartella decifrata>/database/content-….db
+     e copia media/ e fonts/ in .materialize/
+
+https://github.com/materializecms/materialize/blob/main/docs/guida/12-aggiornamenti-e-backup.md
+`;
+/** Every file under a folder, as paths relative to it; hidden files and the media cache left out. */
+async function filesUnder(dir) {
+    const out = [];
+    const walk = async (current) => {
+        for (const item of await readdir(current, { withFileTypes: true }).catch(() => [])) {
+            if (item.name.startsWith('.'))
+                continue;
+            const path = join(current, item.name);
+            if (item.isDirectory())
+                await walk(path);
+            else if (item.isFile())
+                out.push(relative(dir, path).split('\\').join('/'));
+        }
+    };
+    await walk(dir);
+    return out;
+}
+/**
+ * One run: what is missing goes up, old database copies go away. `fresh` makes a new database
+ * copy first (the "Esegui ora" button); otherwise the newest daily copy is sent.
+ */
+export async function runOffsite(s, backups, mediaDir, fresh = false, 
+/** For tests: a storage in memory. */
+storage = storageOf(s)) {
+    if (!storage || !s.key)
+        throw new Error('La copia fuori dal server non è configurata');
+    const key = parseRecoveryKey(s.key);
+    const work = await mkdtemp(join(tmpdir(), 'materialize-offsite-'));
+    try {
+        const remote = new Set((await storage.list('')).map((i) => i.key));
+        if (!remote.has('LEGGIMI.txt'))
+            await storage.putText('LEGGIMI.txt', README);
+        const send = async (local, remoteKey) => {
+            const sealed = join(work, 'file');
+            await encryptFile(local, sealed, key);
+            await storage.put(remoteKey, sealed);
+            await rm(sealed, { force: true });
+        };
+        let database;
+        if (backups.available()) {
+            const latest = fresh ? await backups.create('esterno') : (await backups.list())[0];
+            if (latest) {
+                database = latest.name;
+                const target = `database/${latest.name}${SUFFIX}`;
+                if (!remote.has(target))
+                    await send(join(backups.folder(), latest.name), target);
+            }
+        }
+        const media = resolve(mediaDir);
+        let mediaSent = 0;
+        for (const name of await filesUnder(media)) {
+            const target = `media/${name}${SUFFIX}`;
+            if (remote.has(target))
+                continue;
+            await send(join(media, name), target);
+            mediaSent++;
+        }
+        const fonts = join(resolve(mediaDir), '..', 'fonts');
+        let fontsSent = 0;
+        for (const name of await filesUnder(fonts)) {
+            const target = `fonts/${name}${SUFFIX}`;
+            if (remote.has(target))
+                continue;
+            await send(join(fonts, name), target);
+            fontsSent++;
+        }
+        // The newest copies stay; the media are never deleted (an old copy of the database may use them).
+        const copies = (await storage.list('database/'))
+            .filter((i) => i.key.endsWith(SUFFIX))
+            .sort((a, b) => b.key.localeCompare(a.key));
+        for (const old of copies.slice(Math.max(1, s.keep)))
+            await storage.delete(old.key);
+        return { at: new Date().toISOString(), ok: true, database, media: mediaSent, fonts: fontsSent };
+    }
+    finally {
+        await rm(work, { recursive: true, force: true });
+    }
+}
+/** "Prova il collegamento": writes, reads back, lists and deletes a small file. */
+export async function testOffsite(s) {
+    const storage = storageOf(s);
+    if (!storage)
+        throw new Error('Scegli S3 o WebDAV e compila i campi');
+    const key = `.materialize-prova-${randomBytes(4).toString('hex')}.txt`;
+    const text = `Prova di Materialize, ${new Date().toISOString()}`;
+    await storage.putText(key, text);
+    try {
+        if ((await storage.getText(key)) !== text)
+            throw new Error('Il file di prova è tornato diverso');
+        await storage.list('');
+    }
+    finally {
+        await storage.delete(key);
+    }
+}
+/** Every encrypted file under a folder (or one file), decrypted next to it without the suffix. */
+export async function decryptTree(source, key) {
+    const info = await stat(source);
+    const files = info.isDirectory()
+        ? (await filesUnder(source)).filter((f) => f.endsWith(SUFFIX)).map((f) => join(source, f))
+        : [source];
+    const done = [];
+    for (const file of files) {
+        const target = file.endsWith(SUFFIX) ? file.slice(0, -SUFFIX.length) : `${file}.decifrato`;
+        await decryptFile(file, target, key);
+        done.push(target);
+    }
+    return done;
+}

package/src/remote-storage.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Storage outside the server for the backups: any S3-compatible service (Amazon S3, Cloudflare
+ * R2, Backblaze B2, Hetzner, Scaleway, OVH, Wasabi, MinIO…) or a WebDAV folder (Nextcloud,
+ * ownCloud, Synology and QNAP, pCloud, kDrive…). Written with fetch and node:crypto only.
+ */
+export interface S3Target {
+    /** https://… of the service, without the bucket: https://<account>.r2.cloudflarestorage.com */
+    endpoint: string;
+    region: string;
+    bucket: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+}
+export interface WebDavTarget {
+    /** The folder the backups go in: https://cloud.example.com/remote.php/dav/files/nome/backup/ */
+    url: string;
+    user: string;
+    password: string;
+}
+export interface RemoteItem {
+    key: string;
+    size: number;
+    modified: Date;
+}
+export interface RemoteStorage {
+    put(key: string, file: string): Promise<void>;
+    putText(key: string, text: string): Promise<void>;
+    get(key: string, file: string): Promise<void>;
+    getText(key: string): Promise<string>;
+    /** Everything under a folder of the storage ("database/", "media/"), with sizes. */
+    list(prefix: string): Promise<RemoteItem[]>;
+    delete(key: string): Promise<void>;
+}
+export declare class RemoteError extends Error {
+    readonly status?: number;
+    constructor(message: string, status?: number);
+}
+/**
+ * A request signed with AWS Signature Version 4, which every S3-compatible service accepts. The
+ * body is not hashed (UNSIGNED-PAYLOAD): files are streamed over HTTPS, as large as they are.
+ */
+export declare function signS3(target: S3Target, method: string, key: string, query?: Record<string, string>, headers?: Record<string, string>, now?: Date, payloadHash?: string): {
+    url: string;
+    headers: Record<string, string>;
+};
+export declare function createS3Storage(target: S3Target, prefix?: string): RemoteStorage;
+export declare function createWebDavStorage(target: WebDavTarget): RemoteStorage;