domma-cms 1.8.33 → 1.8.35

package/plugins/docs/lib/pdf.js

@@ -0,0 +1,222 @@
+/**
+ * PDF files as documents (2.13): what Docs checks before it keeps one, and
+ * the text it reads out of one for search and the card excerpt.
+ *
+ * An uploaded file is hostile until shown otherwise. `inspectPdf` refuses
+ * (with a plain reason, never by stripping) a file that is not a PDF, is too
+ * large, or names anything that runs or reaches out - JavaScript, Launch,
+ * attached files, rich media, XFA forms - looking inside its FlateDecode
+ * streams too, with `#xx` name escapes undone first, because that is where a
+ * name hides. Nothing here uses a regular expression that can backtrack over
+ * the file.
+ *
+ * The text extract is pdfjs-dist's (an optional dependency of the CMS), run
+ * in a worker thread with a time budget: a file that will not read in time
+ * is kept without an extract, never left to stall the server. No pdfjs = no
+ * extract, and Docs says so in its settings.
+ *
+ * @module docs/lib/pdf
+ */
+
+import zlib from 'zlib';
+import crypto from 'crypto';
+import {Worker} from 'worker_threads';
+import {escapeHtml} from '../admin/lib/convert.js';
+
+export const MAX_PDF_BYTES = 20 * 1024 * 1024;
+export const MAX_EXTRACT_CHARS = 200_000;
+export const MAX_EXTRACT_PAGES = 300;
+export const EXTRACT_TIMEOUT_MS = 20_000;
+
+// Each inflated stream is capped, and so is the total: a tiny file can inflate into gigabytes. A stream
+// over the cap is not skipped but REFUSED (a name could hide past the cap) - `searchableText` says so.
+const STREAM_INFLATE_CAP = 32 * 1024 * 1024;
+const TOTAL_INFLATE_CAP = 160 * 1024 * 1024;
+const NESTED_INFLATE = 3;   // /Filter [/FlateDecode /FlateDecode]: inflated again while the output looks like zlib
+
+/**
+ * Names that make a PDF do something other than show pages. Each with the
+ * reason the person sees. /OpenAction is not here: on its own it is "open at
+ * page 1" (every LaTeX export has one), and a script behind it is caught by
+ * /JS or /JavaScript anyway.
+ */
+export const REFUSED_NAMES = {
+    JavaScript: 'contains JavaScript',
+    JS: 'contains JavaScript',
+    Launch: 'can start programs (a Launch action)',
+    EmbeddedFile: 'has files attached inside it',
+    EmbeddedFiles: 'has files attached inside it',
+    RichMedia: 'contains rich media (Flash, video or 3D)',
+    XFA: 'is an XFA form',
+    GoToE: 'links into embedded files',
+    SubmitForm: 'submits a form to a web address',
+    ImportData: 'imports data from another file'
+};
+const NAME_RE = new RegExp(`/(${Object.keys(REFUSED_NAMES).join('|')})(?![A-Za-z0-9_.])`, 'g');
+
+/** `/J#61vaScript` -> `/JavaScript`: PDF names may spell any character as #xx. */
+function unescapeNames(text) {
+    return text.replace(/#([0-9A-Fa-f]{2})/g, (m, h) => String.fromCharCode(parseInt(h, 16)));
+}
+
+/** zlib's own header: 0x78 then a byte that makes the pair a multiple of 31. */
+const looksDeflated = (b) => b.length > 2 && b[0] === 0x78 && ((b[0] << 8) | b[1]) % 31 === 0;
+
+/**
+ * The file's own bytes plus every stream that inflates, as one latin1 string
+ * to search. No dictionary is parsed to decide (the first build read the
+ * innermost `<<` before `stream`, so a stream whose dictionary held a nested
+ * one - every object stream with /DecodeParms - was never inflated); every
+ * stream body is simply tried, and what will not inflate was not Flate. A
+ * stream whose output looks deflated again is inflated again (a double
+ * /FlateDecode filter). One over the cap means the file cannot be checked:
+ * `{unchecked: true}`, and the caller refuses it.
+ */
+function searchableText(buf) {
+    const raw = buf.toString('latin1');
+    const parts = [raw];
+    let total = 0;
+    let at = 0;
+    while (at < raw.length) {
+        const s = raw.indexOf('stream', at);
+        if (s === -1) break;
+        // 'endstream' also contains 'stream': only a keyword that starts a stream counts.
+        if (raw.startsWith('endstream', s - 3)) { at = s + 6; continue; }
+        let dataStart = s + 6;
+        if (raw[dataStart] === '\r') dataStart++;
+        if (raw[dataStart] === '\n') dataStart++;
+        const e = raw.indexOf('endstream', dataStart);
+        if (e === -1) break;
+        at = e + 9;
+        let bytes = buf.subarray(dataStart, e);
+        for (let depth = 0; depth < NESTED_INFLATE && looksDeflated(bytes); depth++) {
+            let out;
+            try {
+                out = zlib.inflateSync(bytes, {maxOutputLength: Math.min(STREAM_INFLATE_CAP, Math.max(1, TOTAL_INFLATE_CAP - total)), finishFlush: zlib.constants.Z_SYNC_FLUSH});
+            } catch (err) {
+                if (err && (err.code === 'ERR_BUFFER_TOO_LARGE' || /maxOutputLength|too large/i.test(String(err.message)))) return {unchecked: true};
+                break;   // not Flate after all: the raw bytes were searched
+            }
+            total += out.length;
+            if (total > TOTAL_INFLATE_CAP) return {unchecked: true};
+            parts.push(out.toString('latin1'));
+            bytes = out;
+        }
+    }
+    return {text: unescapeNames(parts.join('\n'))};
+}
+
+/**
+ * Check a file before it is kept. Never throws.
+ *
+ * @param   {Buffer} buf
+ * @param   {{maxBytes?: number}} [opts]
+ * @returns {{ok: true, pages: number, sha256: string, encrypted: boolean} | {ok: false, reason: string}}
+ */
+export function inspectPdf(buf, {maxBytes = MAX_PDF_BYTES} = {}) {
+    if (!Buffer.isBuffer(buf) || !buf.length) return {ok: false, reason: 'The file is empty.'};
+    if (buf.length > maxBytes) return {ok: false, reason: `The file is over ${Math.round(maxBytes / 1024 / 1024)} MB.`};
+    if (buf.subarray(0, 1024).toString('latin1').indexOf('%PDF-') === -1) return {ok: false, reason: 'This is not a PDF file.'};
+    const searched = searchableText(buf);
+    if (searched.unchecked) return {ok: false, reason: 'This PDF holds a compressed part too large to check, so Docs will not keep it.'};
+    const text = searched.text;
+    const hit = NAME_RE.exec(text);
+    NAME_RE.lastIndex = 0;
+    if (hit) return {ok: false, reason: `This PDF ${REFUSED_NAMES[hit[1]]}, so Docs will not keep it.`};
+    // An approximate page count from the page objects; pdfjs' number replaces it when there is one.
+    const pages = (text.match(/\/Type\s*\/Page(?![a-zA-Z])/g) || []).length;
+    return {ok: true, pages, sha256: crypto.createHash('sha256').update(buf).digest('hex'), encrypted: /\/Encrypt(?![a-zA-Z])/.test(text)};
+}
+
+let capability = null;
+
+/**
+ * Whether this server can read the text out of a PDF (pdfjs-dist present).
+ * Resolved without loading it.
+ *
+ * @returns {{available: boolean, reason: string|null}}
+ */
+export function extractCapability() {
+    if (capability) return capability;
+    try {
+        import.meta.resolve('pdfjs-dist/legacy/build/pdf.mjs');
+        capability = {available: true, reason: null};
+    } catch {
+        capability = {available: false, reason: 'Searching inside PDF files needs the optional pdfjs-dist package on this server.'};
+    }
+    return capability;
+}
+
+/** Tests only. */
+export function _resetCapability(value = null) { capability = value; }
+
+/**
+ * The text of a PDF, read in a worker thread within a time budget.
+ *
+ * @param   {Buffer} buf
+ * @param   {{timeoutMs?: number}} [opts]
+ * @returns {Promise<{pages: number, text: string}|null>} null when it cannot be read (no pdfjs, broken, encrypted, out of time)
+ */
+export function extractPdfText(buf, {timeoutMs = EXTRACT_TIMEOUT_MS} = {}) {
+    if (!extractCapability().available) return Promise.resolve(null);
+    return new Promise((resolve) => {
+        let done = false;
+        const finish = (v) => { if (!done) { done = true; clearTimeout(timer); resolve(v); } };
+        let worker;
+        try {
+            // A copy: the worker owns its bytes, the caller keeps theirs.
+            const copy = new Uint8Array(buf.byteLength);
+            copy.set(buf);
+            worker = new Worker(new URL('./pdf-worker.js', import.meta.url), {
+                workerData: {data: copy.buffer, maxPages: MAX_EXTRACT_PAGES, maxChars: MAX_EXTRACT_CHARS},
+                transferList: [copy.buffer],
+                resourceLimits: {maxOldGenerationSizeMb: 512}
+            });
+        } catch { finish(null); return; }
+        const timer = setTimeout(() => { finish(null); worker.terminate().catch(() => {}); }, timeoutMs);
+        worker.once('message', (m) => { finish(m && typeof m.text === 'string' ? {pages: Number(m.pages) || 0, text: m.text} : null); worker.terminate().catch(() => {}); });
+        worker.once('error', () => finish(null));
+        worker.once('exit', () => finish(null));
+    });
+}
+
+/**
+ * The extract as a document body: a paragraph per line group, escaped, so
+ * `content` holds what search and the card excerpt read, as HTML like any
+ * other document's.
+ *
+ * @param   {string} text
+ * @returns {string}
+ */
+export function extractToHtml(text) {
+    const t = String(text || '').replace(/\r\n?/g, '\n').slice(0, MAX_EXTRACT_CHARS);
+    return t.split(/\n{2,}/).map(p => p.replace(/[ \t]+/g, ' ').trim()).filter(Boolean)
+        .map(p => `<p>${escapeHtml(p).replace(/\n/g, '<br>')}</p>`).join('');
+}
+
+/**
+ * Content-Disposition with a UTF-8 file name and an ASCII fallback.
+ *
+ * @param {string}  name
+ * @param {boolean} [inline=false]  shown in the browser rather than saved
+ */
+export function contentDisposition(name, inline = false) {
+    const ascii = name.replace(/[^\x20-\x7e]/g, '_').replace(/"/g, '');
+    // RFC 5987: encodeURIComponent leaves ' ( ) * alone, and ' ends the charset part.
+    const ext = encodeURIComponent(name).replace(/['()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
+    return `${inline ? 'inline' : 'attachment'}; filename="${ascii}"; filename*=UTF-8''${ext}`;
+}
+
+/** "1.2 MB" / "640 KB". */
+export function sizeText(bytes) {
+    const n = Number(bytes) || 0;
+    if (n >= 1024 * 1024) return `${(n / 1024 / 1024).toFixed(n >= 10 * 1024 * 1024 ? 0 : 1)} MB`;
+    if (n >= 1024) return `${Math.round(n / 1024)} KB`;
+    return `${n} B`;
+}
+
+/** A display name for a file: the uploaded name with path parts and control characters gone. */
+export function cleanFileName(name) {
+    const base = String(name || '').split(/[\\/]/).pop().replace(/[\u0000-\u001f\u007f]/g, '').trim().slice(0, 160);
+    return base || 'document.pdf';
+}