@intentic/fileq 1.317.0 → 1.318.0

package/src/lib/check/package.ts

@@ -0,0 +1,199 @@
+import { posix } from "node:path";
+import { unzipSync, type UnzipFileInfo } from "fflate";
+import { error, type Finding } from "./finding.js";
+import { childrenNamed, parseXml, type XmlElement } from "./xml-tree.js";
+
+// An OOXML package (docx, pptx, xlsx) opened for checking: the zip's names, its XML parts parsed on demand, and the
+// relationships that tie them together. A missing target or content type is the damage Office meets with "found a
+// problem with content… repair?", so both are read off here once for every format.
+// Only XML parts are inflated up front, each under a cap: a check must not be the thing a zip bomb takes down, and the
+// media it needs (an image's pixel size) is inflated by name later.
+
+const MAX_XML_PART_BYTES = 64 * 1024 * 1024;
+const MAX_XML_TOTAL_BYTES = 256 * 1024 * 1024;
+
+export interface Relationship {
+    readonly id: string;
+    /** The relationship type's last segment: `image`, `slide`, `hyperlink`, `slideLayout`. */
+    readonly kind: string;
+    /** A part name for an internal target (`ppt/media/image1.png`), the raw target for an external one. */
+    readonly target: string;
+    readonly external: boolean;
+}
+
+export interface OoxmlPackage {
+    readonly names: ReadonlySet<string>;
+    /** Parts left unread because they are over the cap, for the report's notes. */
+    readonly skipped: readonly string[];
+    readonly has: (part: string) => boolean;
+    /** A part's parsed tree; undefined when absent or skipped. */
+    readonly xml: (part: string) => XmlElement | undefined;
+    /** A part's text; undefined when absent or skipped. */
+    readonly text: (part: string) => string | undefined;
+    /** The relationships of a part, by id (`""` for the package's own). */
+    readonly rels: (part: string) => ReadonlyMap<string, Relationship>;
+    /** Inflates the named binary parts (media), each under the part cap. */
+    readonly binaries: (parts: readonly string[]) => Readonly<Record<string, Uint8Array>>;
+}
+
+export class NotAPackage extends Error {}
+
+const isXmlPart = (name: string): boolean => name.endsWith(".xml") || name.endsWith(".rels") || name.endsWith(".vml");
+
+export const openPackage = (bytes: Uint8Array): OoxmlPackage => {
+    const names = new Set<string>();
+    const skipped: string[] = [];
+    let budget = MAX_XML_TOTAL_BYTES;
+    let entries: Record<string, Uint8Array>;
+    try {
+        entries = unzipSync(bytes, {
+            filter: (file: UnzipFileInfo) => {
+                names.add(file.name);
+                if (!isXmlPart(file.name)) {
+                    return false;
+                }
+                if (file.originalSize > MAX_XML_PART_BYTES || file.originalSize > budget) {
+                    skipped.push(file.name);
+                    return false;
+                }
+                budget -= file.originalSize;
+                return true;
+            },
+        });
+    } catch (cause) {
+        throw new NotAPackage(`not a readable zip package (${cause instanceof Error ? cause.message : String(cause)})`);
+    }
+    const decoder = new TextDecoder();
+    const texts = new Map<string, string>();
+    const trees = new Map<string, XmlElement>();
+    const relsCache = new Map<string, ReadonlyMap<string, Relationship>>();
+    const text = (part: string): string | undefined => {
+        const cached = texts.get(part);
+        if (cached !== undefined) {
+            return cached;
+        }
+        const raw = entries[part];
+        if (raw === undefined) {
+            return undefined;
+        }
+        const decoded = decoder.decode(raw);
+        texts.set(part, decoded);
+        return decoded;
+    };
+    const xml = (part: string): XmlElement | undefined => {
+        const cached = trees.get(part);
+        if (cached !== undefined) {
+            return cached;
+        }
+        const source = text(part);
+        if (source === undefined) {
+            return undefined;
+        }
+        const tree = parseXml(source);
+        trees.set(part, tree);
+        return tree;
+    };
+    const has = (part: string): boolean => names.has(part);
+    const rels = (part: string): ReadonlyMap<string, Relationship> => {
+        const cached = relsCache.get(part);
+        if (cached !== undefined) {
+            return cached;
+        }
+        const parsed = new Map<string, Relationship>();
+        const tree = xml(relsPartOf(part));
+        for (const relationship of tree === undefined ? [] : childrenNamed(tree, "rel:Relationship")) {
+            const id = relationship.attrs["Id"];
+            const rawTarget = relationship.attrs["Target"];
+            if (id === undefined || rawTarget === undefined) {
+                continue;
+            }
+            const external = relationship.attrs["TargetMode"] === "External";
+            parsed.set(id, {
+                id,
+                kind: (relationship.attrs["Type"] ?? "").split("/").at(-1) ?? "",
+                target: external ? rawTarget : resolveTarget(names, part, rawTarget),
+                external,
+            });
+        }
+        relsCache.set(part, parsed);
+        return parsed;
+    };
+    const binaries = (parts: readonly string[]): Readonly<Record<string, Uint8Array>> => {
+        const wanted = new Set(parts);
+        if (wanted.size === 0) {
+            return {};
+        }
+        return unzipSync(bytes, { filter: (file) => wanted.has(file.name) && file.originalSize <= MAX_XML_PART_BYTES });
+    };
+    return { names, skipped, has, xml, text, rels, binaries };
+};
+
+/** The package's main part, as its root relationships name it; `fallback` when they do not. */
+export const mainPartOf = (pkg: OoxmlPackage, fallback: string): string =>
+    [...pkg.rels("").values()].find((relationship) => relationship.kind === "officeDocument" && !relationship.external)?.target ?? fallback;
+
+/** Where a part's relationships live: `ppt/slides/slide1.xml` → `ppt/slides/_rels/slide1.xml.rels`. */
+export const relsPartOf = (part: string): string => (part === "" ? "_rels/.rels" : `${posix.dirname(part)}/_rels/${posix.basename(part)}.rels`.replace(/^\.\//, ""));
+
+// A target is relative to its source part's folder, or absolute from the package root; percent-encoding is how a
+// writer spells a space, and the zip may hold either spelling.
+const resolveTarget = (names: ReadonlySet<string>, source: string, target: string): string => {
+    const base = target.startsWith("/") ? target.slice(1) : posix.normalize(posix.join(posix.dirname(source), target)).replace(/^\.\//, "");
+    if (names.has(base)) {
+        return base;
+    }
+    try {
+        return decodeURIComponent(base);
+    } catch {
+        return base;
+    }
+};
+
+const DAMAGED = "Office will offer to repair the file, and may drop what it cannot place";
+
+/** Internal relationships of `part` whose target is not in the package, as errors. */
+export const missingTargetFindings = (pkg: OoxmlPackage, part: string, where: string, ignore: ReadonlySet<string> = new Set()): Finding[] =>
+    [...pkg.rels(part).values()]
+        .filter((relationship) => !relationship.external && !ignore.has(relationship.id) && !pkg.has(relationship.target))
+        .map((relationship) =>
+            error(
+                "missing-part",
+                where,
+                `${relationship.kind} ${relationship.id} points to ${relationship.target}, which is not in the file: ${DAMAGED}`,
+            ),
+        );
+
+// A part's extension as [Content_Types].xml keys it: `_rels/.rels` is "rels", where path.extname would say "".
+const extensionOf = (name: string): string => {
+    const base = posix.basename(name);
+    const dot = base.lastIndexOf(".");
+    return dot === -1 ? "" : base.slice(dot + 1).toLowerCase();
+};
+
+/** Package-level damage: no content type for a part, or no main part where the package root says there is one. */
+export const packageFindings = (pkg: OoxmlPackage, mainPart: string): Finding[] => {
+    const findings: Finding[] = [];
+    const types = pkg.xml("[Content_Types].xml");
+    if (types === undefined) {
+        return [error("missing-part", "", `the file has no [Content_Types].xml: ${DAMAGED}`)];
+    }
+    if (!pkg.has(mainPart)) {
+        findings.push(error("missing-part", "", `the file has no ${mainPart}, so nothing can open it as this format`));
+    }
+    const defaults = new Set(childrenNamed(types, "Default").map((entry) => (entry.attrs["Extension"] ?? "").toLowerCase()));
+    const overrides = new Set(childrenNamed(types, "Override").map((entry) => (entry.attrs["PartName"] ?? "").replace(/^\//, "")));
+    const untyped = [...pkg.names].filter(
+        (name) => !name.endsWith("/") && name !== "[Content_Types].xml" && !overrides.has(name) && !defaults.has(extensionOf(name)),
+    );
+    if (untyped.length > 0) {
+        const shown = untyped.slice(0, 3).join(", ");
+        findings.push(
+            error(
+                "content-type",
+                "",
+                `no content type for ${untyped.length === 1 ? "part" : `${untyped.length} parts`} ${shown}${untyped.length > 3 ? ", …" : ""}: add a Default for the extension in [Content_Types].xml; ${DAMAGED}`,
+            ),
+        );
+    }
+    return findings;
+};

package/src/lib/check/pdf.ts

@@ -0,0 +1,228 @@
+// Structural checks over a PDF: blank pages, text set in fonts the file does not carry, internal links to nowhere, a
+// page size that changes partway, and a document with no text layer at all. Read through pdf.js (unpdf), which the
+// deriver already loads: a page's operator list says whether anything is drawn on it and in which fonts.
+//
+// The blank-page and unembedded-font rules are adapted from SurfSense's PDF artifact verification
+// (surfsense_backend/app/artifacts/verification/formats/pdf.py, Copyright (c) SurfSense, Apache-2.0,
+// https://github.com/MODSetter/SurfSense; see this package's NOTICE). Changed here: a page is blank only when nothing
+// at all is drawn on it (a cover image with no text is not), and the fourteen standard PDF fonts, which every reader
+// carries, are never reported as missing.
+import { getDocumentProxy, getResolvedPDFJS } from "unpdf";
+import { error, plural, warning, type CheckReport, type Finding } from "./finding.js";
+
+// Past this, the rest of a long PDF is left unchecked and the notes say so.
+const MAX_PAGES = 500;
+const POINTS_PER_INCH = 72;
+
+const STANDARD_FONTS = new Set([
+    "Helvetica",
+    "Helvetica-Bold",
+    "Helvetica-Oblique",
+    "Helvetica-BoldOblique",
+    "Times-Roman",
+    "Times-Bold",
+    "Times-Italic",
+    "Times-BoldItalic",
+    "Courier",
+    "Courier-Bold",
+    "Courier-Oblique",
+    "Courier-BoldOblique",
+    "Symbol",
+    "ZapfDingbats",
+]);
+
+type Operators = Awaited<ReturnType<typeof getResolvedPDFJS>>["OPS"];
+
+// The operator codes that draw: text, and paths and images. pdf.js is resolved when a PDF is checked, never at load:
+// every fileq run imports this module, and pdf.js is the largest thing it could pull in.
+interface Painters {
+    readonly setFont: number;
+    readonly constructPath: number;
+    readonly text: ReadonlySet<number>;
+    readonly paint: ReadonlySet<number>;
+}
+
+const paintersOf = (OPS: Operators): Painters => ({
+    setFont: OPS.setFont,
+    constructPath: OPS.constructPath,
+    text: new Set([OPS.showText, OPS.showSpacedText, OPS.nextLineShowText, OPS.nextLineSetSpacingShowText]),
+    paint: new Set([
+        OPS.stroke,
+        OPS.closeStroke,
+        OPS.fill,
+        OPS.eoFill,
+        OPS.fillStroke,
+        OPS.eoFillStroke,
+        OPS.closeFillStroke,
+        OPS.closeEOFillStroke,
+        OPS.shadingFill,
+        OPS.rawFillPath,
+        OPS.paintImageXObject,
+        OPS.paintInlineImageXObject,
+        OPS.paintInlineImageXObjectGroup,
+        OPS.paintImageXObjectRepeat,
+        OPS.paintImageMaskXObject,
+        OPS.paintImageMaskXObjectGroup,
+        OPS.paintImageMaskXObjectRepeat,
+        OPS.paintSolidColorImageMask,
+    ]),
+});
+
+type PdfDocument = Awaited<ReturnType<typeof getDocumentProxy>>;
+type PdfPage = Awaited<ReturnType<PdfDocument["getPage"]>>;
+
+interface PageScan {
+    readonly characters: number;
+    readonly painted: boolean;
+    /** Base names of the fonts that drew visible text and are not in the file. */
+    readonly missingFonts: readonly string[];
+}
+
+// What this module reads of pdf.js's untyped operator arguments, fonts and annotations, declared once at the boundary.
+// A glyph run is glyph objects with numbers (spacing adjustments) between them.
+interface Glyph {
+    readonly unicode?: string;
+}
+type Operand = string | number | null | readonly (Glyph | number)[];
+interface PdfFont {
+    readonly name?: string;
+    /** pdf.js found no font program in the file and fell back to a system font. */
+    readonly missingFile?: boolean;
+}
+interface PdfAnnotation {
+    readonly subtype?: string;
+    /** A named destination, an explicit one (an array), or none. */
+    readonly dest?: string | readonly Operand[] | null;
+}
+
+const isGlyph = (item: Glyph | number): item is Glyph => typeof item !== "number";
+
+const visibleCharacters = (run: readonly (Glyph | number)[]): number => run.filter((item) => isGlyph(item) && /\S/.test(item.unicode ?? "")).length;
+
+interface UsedFont {
+    /** The base font name, a subset's six-letter prefix removed. */
+    readonly name: string;
+    readonly missing: boolean;
+}
+
+const fontOf = (page: PdfPage, id: string | undefined): UsedFont | undefined => {
+    if (id === undefined) {
+        return undefined;
+    }
+    let font: PdfFont | undefined;
+    try {
+        font = page.commonObjs.get(id);
+    } catch {
+        // allow(silent-catch): pdf.js throws for a font it never resolved, and a font never loaded drew nothing.
+        return undefined;
+    }
+    return { name: (font?.name ?? "").replace(/^[A-Z]{6}\+/, ""), missing: font?.missingFile === true };
+};
+
+const scanPage = async (page: PdfPage, ops: Painters): Promise<PageScan> => {
+    const list = await page.getOperatorList();
+    const operands: readonly (readonly Operand[] | null)[] = list.argsArray;
+    let characters = 0;
+    let painted = false;
+    let currentFont: string | undefined;
+    const missing = new Set<string>();
+    list.fnArray.forEach((op, index) => {
+        const first = operands[index]?.[0];
+        if (op === ops.setFont) {
+            currentFont = first === undefined || first === null ? undefined : String(first);
+        } else if (ops.text.has(op)) {
+            const count = Array.isArray(first) ? visibleCharacters(first) : 0;
+            characters += count;
+            const font = count > 0 ? fontOf(page, currentFont) : undefined;
+            if (font?.missing === true && font.name !== "" && !STANDARD_FONTS.has(font.name)) {
+                missing.add(font.name);
+            }
+        } else if (ops.paint.has(op) || (op === ops.constructPath && ops.paint.has(Number(first)))) {
+            painted = true;
+        }
+    });
+    return { characters, painted: painted || characters > 0, missingFonts: [...missing] };
+};
+
+// A link whose named destination the file does not define goes nowhere when clicked.
+const linkFindings = async (pdf: PdfDocument, page: PdfPage, number: number): Promise<Finding[]> => {
+    const annotations: readonly PdfAnnotation[] = await page.getAnnotations();
+    const findings: Finding[] = [];
+    for (const annotation of annotations) {
+        const dest = annotation.dest;
+        if (annotation.subtype !== "Link" || dest === undefined || dest === null || Array.isArray(dest)) {
+            continue;
+        }
+        const named = String(dest);
+        // allow(silent-catch): pdf.js throws where the file's destination tree is damaged, and then the link goes nowhere too.
+        const target = await pdf.getDestination(named).catch(() => null);
+        if (target === null) {
+            findings.push(error("broken-link", `page ${number}`, `a link goes to "${named}", a destination the file does not define, so clicking it does nothing`));
+        }
+    }
+    return findings;
+};
+
+const sizeLabel = (width: number, height: number): string =>
+    `${Number((width / POINTS_PER_INCH).toFixed(2))} × ${Number((height / POINTS_PER_INCH).toFixed(2))} in`;
+
+const pageList = (numbers: readonly number[]): string => (numbers.length > 6 ? `${numbers.slice(0, 6).join(", ")}, …` : numbers.join(", "));
+
+const openPdf = async (bytes: Uint8Array): Promise<PdfDocument | CheckReport> => {
+    try {
+        // Errors only: pdf.js otherwise narrates its recovery of a damaged file on the console, into the report.
+        return await getDocumentProxy(bytes, { verbosity: 0 });
+    } catch (cause) {
+        const reason = cause instanceof Error && cause.name === "PasswordException" ? "it is password-protected, so nothing past the password can be checked" : `it is not a readable PDF (${cause instanceof Error ? cause.message : String(cause)})`;
+        return { format: "pdf", findings: [error("unreadable", "", reason)], notes: [] };
+    }
+};
+
+export const checkPdf = async (bytes: Uint8Array): Promise<CheckReport> => {
+    const pdf = await openPdf(bytes);
+    if ("findings" in pdf) {
+        return pdf;
+    }
+    const ops = paintersOf((await getResolvedPDFJS()).OPS);
+    const findings: Finding[] = [];
+    const notes: string[] = [];
+    const total = pdf.numPages;
+    if (total > MAX_PAGES) {
+        notes.push(`checked the first ${MAX_PAGES} of ${total} pages`);
+    }
+    const sizes = new Map<string, number[]>();
+    const unembedded = new Map<string, number[]>();
+    let characters = 0;
+    for (let number = 1; number <= Math.min(total, MAX_PAGES); number += 1) {
+        const page = await pdf.getPage(number);
+        const scan = await scanPage(page, ops);
+        characters += scan.characters;
+        if (!scan.painted) {
+            findings.push(error("blank-page", `page ${number}`, "the page is blank: nothing is drawn on it"));
+        }
+        for (const font of scan.missingFonts) {
+            unembedded.set(font, [...(unembedded.get(font) ?? []), number]);
+        }
+        const [x0 = 0, y0 = 0, x1 = 0, y1 = 0] = page.view;
+        const turned = page.rotate % 180 !== 0;
+        const label = turned ? sizeLabel(y1 - y0, x1 - x0) : sizeLabel(x1 - x0, y1 - y0);
+        sizes.set(label, [...(sizes.get(label) ?? []), number]);
+        findings.push(...(await linkFindings(pdf, page, number)));
+    }
+    for (const [font, pages] of unembedded) {
+        findings.push(
+            warning("font-not-embedded", `page ${pageList(pages)}`, `text is set in ${font}, which the file does not carry: a reader without that font substitutes another, and lines can break differently; embed it`),
+        );
+    }
+    if (sizes.size > 1) {
+        const [common] = [...sizes.entries()].toSorted((a, b) => b[1].length - a[1].length);
+        const odd = [...sizes.entries()].filter(([label]) => label !== common?.[0]);
+        for (const [label, pages] of odd) {
+            findings.push(warning("page-size", `page ${pageList(pages)}`, `${plural(pages.length, "page is", "pages are")} ${label}, the rest ${common?.[0] ?? "another size"}`));
+        }
+    }
+    if (characters === 0 && total > 0) {
+        findings.push(warning("no-text-layer", "", "no page has a text layer, so its words cannot be searched, copied or read aloud"));
+    }
+    return { format: "pdf", extent: plural(total, "page"), findings, notes };
+};