@intentic/fileq 1.317.0

package/src/lib/archives.ts

@@ -0,0 +1,209 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { unzipSync } from "fflate";
+import { onPath } from "./tools.js";
+
+// Reading an archive's SHAPE without unpacking it: which members are inside and how big each is, never their bytes.
+// Three containers are read in process — zip's central directory, tar's header blocks, gzip's trailer — because those
+// are the ones whose index is readable without a decompressor this package carries.
+// Everything else (xz, bzip2, zstd) is listed by GNU tar when the image carries it, since the codecs are on the image
+// but not in node_modules; a container nothing here can open is named as unlistable rather than guessed at.
+
+const run = promisify(execFile);
+
+export interface ArchiveEntry {
+    readonly path: string;
+    /** Unpacked bytes; undefined where the container records none (a tar listing's directory, a streamed zip entry). */
+    readonly size: number | undefined;
+    /** Bytes the member occupies inside the archive; zip alone records both, so it alone can state a ratio. */
+    readonly packed: number | undefined;
+    readonly directory: boolean;
+}
+
+/** Every member of a zip, from the central directory alone. */
+export const zipEntries = (bytes: Uint8Array): ArchiveEntry[] => {
+    const entries: ArchiveEntry[] = [];
+    unzipSync(bytes, {
+        filter: (file) => {
+            entries.push({ path: file.name, size: file.originalSize, packed: file.size, directory: file.name.endsWith("/") });
+            return false;
+        },
+    });
+    return entries;
+};
+
+const BLOCK = 512;
+const decoder = new TextDecoder();
+
+// A NUL-terminated ASCII field, which is how every string in a tar header is written.
+const field = (bytes: Uint8Array, offset: number, length: number): string => {
+    const raw = bytes.subarray(offset, offset + length);
+    const end = raw.indexOf(0);
+    return decoder.decode(end === -1 ? raw : raw.subarray(0, end)).trim();
+};
+
+// Numeric fields are octal text. A base-256 field (the >8GB encoding) reads as 0, which only a file far past the
+// derivation size cap could carry.
+const octal = (bytes: Uint8Array, offset: number, length: number): number => {
+    const text = field(bytes, offset, length).replaceAll(/[^0-7]/g, "");
+    return text === "" ? 0 : Number.parseInt(text, 8);
+};
+
+// The header's own checksum, with the checksum field itself read as spaces. The one test that recognizes a pre-POSIX
+// tar, which carries no `ustar` magic at all.
+const headerChecks = (block: Uint8Array): boolean => {
+    if (block.length < BLOCK) {
+        return false;
+    }
+    let sum = 0;
+    for (let index = 0; index < BLOCK; index++) {
+        sum += index >= 148 && index < 156 ? 0x20 : (block[index] ?? 0);
+    }
+    return sum === octal(block, 148, 8);
+};
+
+/** Whether these bytes open with a tar header, by magic or by checksum. */
+export const looksLikeTar = (bytes: Uint8Array): boolean => bytes.length >= BLOCK && (field(bytes, 257, 6).startsWith("ustar") || headerChecks(bytes));
+
+// A pax record block: `<length> <key>=<value>\n`, repeated. Only `path` matters here, and it overrides the ustar name.
+const paxPath = (data: Uint8Array): string | undefined => {
+    for (const record of decoder.decode(data).split("\n")) {
+        const match = /^\d+ path=(.*)$/.exec(record);
+        if (match !== null) {
+            return match[1];
+        }
+    }
+    return undefined;
+};
+
+interface TarBlock {
+    readonly header: Uint8Array;
+    /** One of tar's typeflags: "" or "0" a file, "5" a directory, "L"/"x" a name carrier, "g" archive defaults. */
+    readonly type: string;
+    readonly size: number;
+    readonly data: Uint8Array;
+    /** Offset of the block after this one's data, already rounded up to the 512-byte grid. */
+    readonly next: number;
+}
+
+// The block at `offset`, or undefined at the zero block that ends every tar (and at a truncated tail).
+const blockAt = (bytes: Uint8Array, offset: number): TarBlock | undefined => {
+    if (offset + BLOCK > bytes.length) {
+        return undefined;
+    }
+    const header = bytes.subarray(offset, offset + BLOCK);
+    if (header.every((byte) => byte === 0)) {
+        return undefined;
+    }
+    const size = octal(header, 124, 12);
+    return {
+        header,
+        type: field(header, 156, 1),
+        size,
+        data: bytes.subarray(offset + BLOCK, offset + BLOCK + size),
+        next: offset + BLOCK + Math.ceil(size / BLOCK) * BLOCK,
+    };
+};
+
+// Blocks that name the entry after them instead of being one; a path longer than the 100-byte name field arrives here.
+const NAME_CARRIERS: ReadonlySet<string> = new Set(["L", "x", "X", "g"]);
+
+const carriedName = (block: TarBlock, current: string | undefined): string | undefined => {
+    if (block.type === "L") {
+        return field(block.data, 0, block.data.length);
+    }
+    // "g" carries defaults for the whole archive rather than a name, so it leaves the pending one alone.
+    return block.type === "g" ? current : (paxPath(block.data) ?? current);
+};
+
+const entryOf = (block: TarBlock, carried: string | undefined): ArchiveEntry => {
+    const prefix = field(block.header, 345, 155);
+    const name = field(block.header, 0, 100);
+    const path = carried ?? (prefix === "" ? name : `${prefix}/${name}`);
+    const directory = block.type === "5" || path.endsWith("/");
+    return { path, size: directory ? undefined : block.size, packed: undefined, directory };
+};
+
+/**
+ * Every member of a tar, by walking its header blocks and skipping the data between them.
+ * Throws when the first block is not a header, which is how a file lying about being a tar becomes a loud skip.
+ */
+export const tarEntries = (bytes: Uint8Array): ArchiveEntry[] => {
+    if (!looksLikeTar(bytes)) {
+        throw new Error("not a tar archive: no header block at the start");
+    }
+    const entries: ArchiveEntry[] = [];
+    // Set by a name-carrying block for the entry that follows it, cleared the moment that entry consumes it.
+    let carried: string | undefined;
+    let offset = 0;
+    for (let block = blockAt(bytes, offset); block !== undefined; block = blockAt(bytes, offset)) {
+        offset = block.next;
+        if (NAME_CARRIERS.has(block.type)) {
+            carried = carriedName(block, carried);
+            continue;
+        }
+        entries.push(entryOf(block, carried));
+        carried = undefined;
+    }
+    return entries;
+};
+
+export interface GzipHead {
+    /** The member name gzip stored, when it stored one; a file compressed from stdin carries none. */
+    readonly name: string | undefined;
+    /** Unpacked size from the trailer, exact below 4 GiB and wrapped above it, which nothing derived here reaches. */
+    readonly unpackedBytes: number;
+}
+
+// FNAME, the NUL-terminated original name, sits after the fixed header and after FEXTRA's own length-prefixed block.
+const gzipName = (bytes: Uint8Array, flags: number): string | undefined => {
+    if ((flags & 0x08) === 0) {
+        return undefined;
+    }
+    const extra = (flags & 0x04) === 0 ? 0 : 2 + ((bytes[10] ?? 0) | ((bytes[11] ?? 0) << 8));
+    const end = bytes.indexOf(0, 10 + extra);
+    const name = end === -1 ? "" : decoder.decode(bytes.subarray(10 + extra, end));
+    return name === "" ? undefined : name;
+};
+
+/** A gzip member's header and trailer facts, without inflating a byte; undefined when the bytes are not gzip. */
+export const gzipHead = (bytes: Uint8Array): GzipHead | undefined => {
+    if (bytes.length < 18 || bytes[0] !== 0x1f || bytes[1] !== 0x8b) {
+        return undefined;
+    }
+    const view = new DataView(bytes.buffer, bytes.byteOffset + bytes.length - 4, 4);
+    return { name: gzipName(bytes, bytes[3] ?? 0), unpackedBytes: view.getUint32(0, true) };
+};
+
+/** Whether GNU tar is on PATH, which is what lets this tier list the codecs node_modules cannot decompress. */
+export const tarOnPath = (): boolean => onPath("tar");
+
+// `tar -tvf` prints `<mode> <owner/group> <size> <date> <time> <path>`; the path is the rest of the line, so a name
+// with spaces in it survives.
+const LISTING_LINE = /^(\S+)\s+\S+\s+(\d+)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}(?::\d{2})?\s+(.+)$/;
+
+/** Parses `tar -tvf` output; split out from the spawn so the shape of a listing is testable without a tar on PATH. */
+export const parseTarListing = (stdout: string): ArchiveEntry[] => {
+    const entries: ArchiveEntry[] = [];
+    for (const line of stdout.split("\n")) {
+        const match = LISTING_LINE.exec(line);
+        if (match === null) {
+            continue;
+        }
+        const [, mode = "", size = "0", path = ""] = match;
+        // A link prints as `target -> source`; the member is the name on the left of the arrow.
+        const directory = mode.startsWith("d");
+        entries.push({ path: path.split(" -> ")[0] ?? path, size: directory ? undefined : Number(size), packed: undefined, directory });
+    }
+    return entries;
+};
+
+// Long enough for a large archive's index, short enough that a hung tar cannot hold the sweep behind it.
+const TAR_TIMEOUT_MS = 60_000;
+const TAR_MAX_BUFFER = 16 * 1024 * 1024;
+
+/** Lists an archive through GNU tar, which auto-detects gzip, bzip2, xz and zstd on read. */
+export const listWithTar = async (absPath: string): Promise<ArchiveEntry[]> => {
+    const { stdout } = await run("tar", ["-tvf", absPath], { timeout: TAR_TIMEOUT_MS, maxBuffer: TAR_MAX_BUFFER });
+    return parseTarListing(stdout);
+};

package/src/lib/derive.ts

@@ -0,0 +1,132 @@
+import type { Stats } from "node:fs";
+import { stat } from "node:fs/promises";
+import { isAbsolute, relative, sep } from "node:path";
+import { estimateTokens } from "@intentic/base/format";
+import { errnoCode, errorMessage, isMissing } from "@intentic/base/errors";
+import { IGNORED_DIRS, isAgentWorktreePath, isReferencePath } from "@intentic/workspace-ignore";
+import { STATE_DIR } from "@intentic/constants";
+import { detectFormat, type Format } from "./formats.js";
+import { deriverStamp, neutralizeDoc, type DerivedDoc, type Deriver } from "./derivers/deriver.js";
+import { archiveDeriver } from "./derivers/archive.js";
+import { docxDeriver } from "./derivers/docx.js";
+import { epubDeriver } from "./derivers/epub.js";
+import { htmlDeriver } from "./derivers/html.js";
+import { imageDeriver } from "./derivers/image.js";
+import { ipynbDeriver } from "./derivers/ipynb.js";
+import { mediaDeriver } from "./derivers/media.js";
+import { odpDeriver } from "./derivers/odp.js";
+import { odsDeriver } from "./derivers/ods.js";
+import { odtDeriver } from "./derivers/odt.js";
+import { pdfDeriver } from "./derivers/pdf.js";
+import { pptxDeriver } from "./derivers/pptx.js";
+import { profileDeriver } from "./derivers/profile.js";
+import { rtfDeriver } from "./derivers/rtf.js";
+import { xlsxDeriver } from "./derivers/xlsx.js";
+import { isFresh, readSidecar, removeSidecar, sidecarBody, sidecarPathFor, sha256OfFile, writeSidecar } from "./sidecar.js";
+
+// Pipeline both commands and the daemon's sweep run: place the file, recognize it, route it, keep its shadow honest.
+// One module, so `read` and `derive` cannot disagree about a file's markdown.
+
+export const DERIVERS: Record<Format, Deriver> = {
+    docx: docxDeriver,
+    xlsx: xlsxDeriver,
+    pptx: pptxDeriver,
+    pdf: pdfDeriver,
+    image: imageDeriver,
+    media: mediaDeriver,
+    html: htmlDeriver,
+    ipynb: ipynbDeriver,
+    odt: odtDeriver,
+    ods: odsDeriver,
+    odp: odpDeriver,
+    rtf: rtfDeriver,
+    epub: epubDeriver,
+    archive: archiveDeriver,
+    profile: profileDeriver,
+};
+
+// Above this, a derivation isn't background-cheap and the file is data to process, not shadow; skipped loudly.
+export const MAX_SOURCE_BYTES = 200 * 1024 * 1024;
+
+/** Workspace-relative path when `abs` sits under `root`; undefined outside it (no sidecar can exist there). */
+export const relPathIn = (root: string, abs: string): string | undefined => {
+    const rel = relative(root, abs);
+    return rel === "" || rel.startsWith("..") || isAbsolute(rel) ? undefined : rel.split(sep).join("/");
+};
+
+// What the shadow tree refuses: machine subtrees, the daemon's own state, the reference shelf, agent worktrees.
+export const isDeriveIgnored = (relPath: string): boolean => {
+    const segments = relPath.split("/");
+    return segments.some((segment) => IGNORED_DIRS.has(segment) || segment === STATE_DIR) || isReferencePath(relPath) || isAgentWorktreePath(relPath);
+};
+
+export type Outcome =
+    | {
+          readonly kind: "derived";
+          readonly relPath: string;
+          readonly format: Format;
+          readonly sidecarPath: string;
+          readonly body: string;
+          readonly doc: DerivedDoc;
+          readonly tokens: number;
+      }
+    | {
+          readonly kind: "fresh";
+          readonly relPath: string;
+          readonly format: Format;
+          readonly sidecarPath: string;
+          readonly body: string;
+          readonly tokens: number;
+      }
+    | { readonly kind: "removed"; readonly relPath: string; readonly sidecarPath: string }
+    | { readonly kind: "skipped"; readonly relPath: string; readonly reason: string };
+
+/** Converges a workspace file's sidecar with its source: derive when stale, reuse fresh, remove when gone. */
+export const ensureSidecar = async (workspaceRoot: string, absPath: string, now: () => Date = () => new Date()): Promise<Outcome> => {
+    const relPath = relPathIn(workspaceRoot, absPath);
+    if (relPath === undefined) {
+        return { kind: "skipped", relPath: absPath, reason: "outside-workspace" };
+    }
+    if (isDeriveIgnored(relPath)) {
+        return { kind: "skipped", relPath, reason: "ignored-path" };
+    }
+    const sidecarPath = sidecarPathFor(workspaceRoot, relPath);
+    let source: Stats | undefined;
+    try {
+        source = await stat(absPath);
+    } catch (error) {
+        // Only a source that is gone takes its sidecar with it; one this process cannot stat is still there.
+        if (!isMissing(error)) {
+            return { kind: "skipped", relPath, reason: `unreadable (${errnoCode(error) ?? errorMessage(error)})` };
+        }
+    }
+    if (source === undefined || !source.isFile()) {
+        const removed = await removeSidecar(workspaceRoot, relPath);
+        return removed ? { kind: "removed", relPath, sidecarPath } : { kind: "skipped", relPath, reason: "missing" };
+    }
+    if (source.size > MAX_SOURCE_BYTES) {
+        return { kind: "skipped", relPath, reason: `too-large (${Math.round(source.size / 1024 / 1024)} MB)` };
+    }
+    const format = await detectFormat(absPath);
+    if (format === undefined) {
+        return { kind: "skipped", relPath, reason: "unsupported" };
+    }
+    const deriver = DERIVERS[format];
+    const stamp = deriverStamp(deriver);
+    const sourceSha = await sha256OfFile(absPath);
+    const existing = await readSidecar(sidecarPath);
+    if (isFresh(existing, sourceSha, stamp)) {
+        const body = sidecarBody(existing ?? "");
+        return { kind: "fresh", relPath, format, sidecarPath, body, tokens: estimateTokens(body) };
+    }
+    // Neutralized once here so every consumer gets folded text; a corrupt file's failure is a loud skip, not fatal.
+    let doc: DerivedDoc;
+    try {
+        doc = neutralizeDoc(await deriver.derive(absPath));
+    } catch (error) {
+        const detail = errorMessage(error).split("\n")[0];
+        return { kind: "skipped", relPath, reason: `derive-failed (${format}): ${detail}` };
+    }
+    const written = await writeSidecar(workspaceRoot, { relPath, sourceSha, deriverStamp: stamp, doc, derivedAt: now() });
+    return { kind: "derived", relPath, format, sidecarPath: written.path, body: written.body, doc, tokens: written.tokens };
+};

package/src/lib/derivers/archive.ts

@@ -0,0 +1,168 @@
+import { readFile } from "node:fs/promises";
+import { basename } from "node:path";
+import { gunzipSync } from "fflate";
+import { plural } from "@intentic/base/format";
+import { gzipHead, listWithTar, looksLikeTar, tarEntries, tarOnPath, zipEntries, type ArchiveEntry } from "../archives.js";
+import type { DerivedDoc, Deriver } from "./deriver.js";
+
+/* Archives: what is inside, not what it says. */
+
+// Members listed before the table is more index than shadow; the count above it still states the whole truth.
+const MAX_ENTRIES = 500;
+// A gzip member worth inflating into memory during a background sweep; above it the trailer's facts are the shadow.
+const MAX_UNPACKED_BYTES = 128 * 1024 * 1024;
+// Text of a single compressed file carried into the shadow; the rest stays one `zcat` away.
+const MAX_MEMBER_TEXT_BYTES = 256 * 1024;
+
+// What the container is, as a reader would name it; drives both the capsule's first line and which reader runs.
+type Container = "zip" | "tar" | "gzip" | "xz" | "bzip2" | "zstd" | "7z" | "rar";
+
+// Leading bytes that name a container. Checked before tar, whose own recognition is a checksum over the first block.
+const MAGIC: readonly (readonly [Container, readonly number[]])[] = [
+    ["zip", [0x50, 0x4b]],
+    ["gzip", [0x1f, 0x8b]],
+    ["xz", [0xfd, 0x37, 0x7a, 0x58, 0x5a]],
+    ["bzip2", [0x42, 0x5a, 0x68]],
+    ["zstd", [0x28, 0xb5, 0x2f, 0xfd]],
+    ["7z", [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]],
+    ["rar", [0x52, 0x61, 0x72, 0x21]],
+];
+
+const containerOf = (bytes: Uint8Array): Container | undefined => {
+    const magic = MAGIC.find(([, signature]) => signature.every((byte, index) => bytes[index] === byte));
+    return magic?.[0] ?? (looksLikeTar(bytes) ? "tar" : undefined);
+};
+
+// Sizes read by a person, not a machine: three significant figures at most, since an exact byte count of a member
+// nobody has unpacked yet is noise.
+const formatBytes = (bytes: number): string => {
+    const units = ["B", "kB", "MB", "GB", "TB"];
+    let value = bytes;
+    let unit = 0;
+    while (value >= 1000 && unit < units.length - 1) {
+        value /= 1000;
+        unit += 1;
+    }
+    return `${unit === 0 ? value : value.toFixed(value < 10 ? 1 : 0)} ${units[unit]}`;
+};
+
+// The capsule above the table: what the container is and how much is in it, so a reader knows the shape before the list.
+const summaryOf = (container: Container, entries: readonly ArchiveEntry[]): string[] => {
+    const files = entries.filter((entry) => !entry.directory);
+    const unpacked = files.reduce((sum, entry) => sum + (entry.size ?? 0), 0);
+    const packed = files.reduce((sum, entry) => sum + (entry.packed ?? 0), 0);
+    const folders = entries.length - files.length;
+    const lines = [`- Archive: ${container}`, `- Members: ${plural(files.length, "file")}${folders === 0 ? "" : `, ${plural(folders, "folder")}`}`];
+    // Ratio only where the container records both sides; a tar header knows nothing about what compressed it.
+    const ratio = packed > 0 && unpacked > 0 ? ` · packed ${formatBytes(packed)} (${Math.round((1 - packed / unpacked) * 100)}% smaller)` : "";
+    if (unpacked > 0) {
+        lines.push(`- Unpacked: ${formatBytes(unpacked)}${ratio}`);
+    }
+    return lines;
+};
+
+const tableRow = (cells: readonly string[]): string => `| ${cells.map((cell) => cell.replaceAll("|", "\\|")).join(" | ")} |`;
+
+// Files only: a folder row carries no size and says nothing the paths beside it don't already.
+const entryTable = (entries: readonly ArchiveEntry[]): string => {
+    const files = entries.filter((entry) => !entry.directory).slice(0, MAX_ENTRIES);
+    if (files.length === 0) {
+        return "";
+    }
+    return [
+        tableRow(["Path", "Size"]),
+        tableRow(["---", "---"]),
+        ...files.map((entry) => tableRow([entry.path, entry.size === undefined ? "" : formatBytes(entry.size)])),
+    ].join("\n");
+};
+
+const listingDoc = (container: Container, entries: readonly ArchiveEntry[], notes: readonly string[]): DerivedDoc => {
+    const files = entries.filter((entry) => !entry.directory).length;
+    const table = entryTable(entries);
+    return {
+        markdown: [summaryOf(container, entries).join("\n"), table].filter((part) => part !== "").join("\n\n"),
+        notes: [
+            ...notes,
+            ...(files > MAX_ENTRIES ? [`listing ${MAX_ENTRIES} of ${files} files: the rest are in the archive, not here`] : []),
+            ...(entries.length === 0 ? ["this archive holds nothing"] : []),
+            "member listing only: what the files inside SAY is not derived — unpack the archive to read them",
+        ],
+    };
+};
+
+// A NUL in the first pages is how every other surface here decides text from bytes; the same rule, so they agree.
+const isText = (bytes: Uint8Array): boolean => !bytes.subarray(0, 8192).includes(0);
+
+// One compressed file, where the archive is a wrapper rather than a container: the member's own text is the shadow.
+const memberDoc = (name: string, unpacked: Uint8Array): DerivedDoc => {
+    const facts = [`- Archive: gzip`, `- Member: ${name}`, `- Unpacked: ${formatBytes(unpacked.length)}`].join("\n");
+    if (!isText(unpacked)) {
+        return { markdown: facts, notes: [`${name} is not text: nothing inside this archive can be shown as prose`] };
+    }
+    const clipped = unpacked.length > MAX_MEMBER_TEXT_BYTES;
+    const text = new TextDecoder().decode(unpacked.subarray(0, MAX_MEMBER_TEXT_BYTES));
+    return {
+        markdown: `${facts}\n\n## ${name}\n\n${clipped ? text.slice(0, text.lastIndexOf("\n") + 1) : text}`,
+        notes: clipped ? [`showing the first ${formatBytes(MAX_MEMBER_TEXT_BYTES)} of ${formatBytes(unpacked.length)}: the rest is in the archive`] : [],
+    };
+};
+
+const gzipDoc = (absPath: string, bytes: Uint8Array): DerivedDoc => {
+    const head = gzipHead(bytes);
+    const name = head?.name ?? basename(absPath).replace(/\.(gz|tgz)$/i, "");
+    if (head !== undefined && head.unpackedBytes > MAX_UNPACKED_BYTES) {
+        return {
+            markdown: [`- Archive: gzip`, `- Member: ${name}`, `- Unpacked: ${formatBytes(head.unpackedBytes)}`].join("\n"),
+            notes: [`too large to open in the background (${formatBytes(head.unpackedBytes)} unpacked): the header's facts are all this shadow carries`],
+        };
+    }
+    const unpacked = gunzipSync(bytes);
+    // A .tgz is a tar wearing gzip: its shadow is the manifest, the same one a plain .tar gets.
+    return looksLikeTar(unpacked) ? listingDoc("gzip", tarEntries(unpacked), ["gzip-compressed tar"]) : memberDoc(name, unpacked);
+};
+
+// Codecs the image carries as binaries but node_modules cannot decompress; GNU tar reads all three on the way in.
+const externalDoc = async (container: Container, absPath: string): Promise<DerivedDoc> => {
+    if (!tarOnPath()) {
+        return { markdown: `- Archive: ${container}`, notes: [`no listing: reading a ${container} archive needs tar, which is not on PATH here`] };
+    }
+    try {
+        return listingDoc(container, await listWithTar(absPath), [`listed through tar`]);
+    } catch {
+        // tar refuses anything that is not a tar inside: a bare `notes.xz` is one compressed file, and this tier
+        // has no decompressor for it.
+        return {
+            markdown: `- Archive: ${container}`,
+            notes: [`no listing: this is a single ${container}-compressed file rather than an archive, and ${container} is not decompressed by this tier`],
+        };
+    }
+};
+
+// 7z and rar are the two containers nothing on the image opens; named rather than silently empty.
+const UNREADABLE: ReadonlySet<Container> = new Set(["7z", "rar"]);
+
+export const archiveDeriver: Deriver = {
+    // The stamp names the listing capability for the same reason the pdf deriver's names OCR: a shadow written
+    // without tar on PATH must read as stale once an image carries it.
+    get name(): string {
+        return tarOnPath() ? "archive+tar" : "archive";
+    },
+    version: 1,
+    derive: async (absPath): Promise<DerivedDoc> => {
+        const bytes = new Uint8Array(await readFile(absPath));
+        const container = containerOf(bytes);
+        if (container === undefined) {
+            throw new Error("not an archive: no container signature in the first bytes");
+        }
+        if (UNREADABLE.has(container)) {
+            return { markdown: `- Archive: ${container}`, notes: [`no listing: nothing in this image opens a ${container} archive`] };
+        }
+        if (container === "zip") {
+            return listingDoc(container, zipEntries(bytes), []);
+        }
+        if (container === "tar") {
+            return listingDoc(container, tarEntries(bytes), []);
+        }
+        return container === "gzip" ? gzipDoc(absPath, bytes) : externalDoc(container, absPath);
+    },
+};

package/src/lib/derivers/deriver.ts

@@ -0,0 +1,33 @@
+import { neutralizeOutsideText } from "@intentic/base/outside-text";
+
+// Contract every format handler meets: markdown body (no front matter) plus a note for every cap or degradation hit,
+// since a truncated document reading as complete is a lie.
+// `version` is the staleness lever, stamped into sidecar front matter; a stamp mismatch re-derives on the next touch.
+// Bump it whenever the output's shape changes, including a bug fix that changes what the markdown says.
+
+export interface DerivedDoc {
+    /** Empty means nothing to say, and still gets a sidecar so freshness has somewhere to live. */
+    readonly markdown: string;
+    /** Human title when the format carries one (metadata, first heading); the capsule leads with it. */
+    readonly title?: string | undefined;
+    /** Degradations and caps, one line each; printed in the capsule and stored in sidecar front matter. */
+    readonly notes: string[];
+}
+
+export interface Deriver {
+    readonly name: string;
+    readonly version: number;
+    readonly derive: (absPath: string) => Promise<DerivedDoc>;
+}
+
+/** `name v<version>`, the exact string stamped into and compared against sidecar front matter. */
+export const deriverStamp = (deriver: Deriver): string => `${deriver.name} v${deriver.version}`;
+
+// Any field can carry forged text (a pdf's Title as easily as its body), so the whole doc is neutralized at the
+// pipeline boundary before it reaches stdout or disk.
+// The sidecar writer neutralizes again independently; the fold is idempotent, so the second pass costs nothing.
+export const neutralizeDoc = (doc: DerivedDoc): DerivedDoc => ({
+    markdown: neutralizeOutsideText(doc.markdown),
+    title: doc.title === undefined ? undefined : neutralizeOutsideText(doc.title),
+    notes: doc.notes.map((note) => neutralizeOutsideText(note)),
+});

package/src/lib/derivers/docx.test.ts

@@ -0,0 +1,38 @@
+import { notesOf } from "./docx.js";
+
+// mammoth's messages as they reach the notes: a template's worth of style warnings folds to one line, everything
+// else keeps its own, and the fold counts style names so two saves of one template agree.
+const message = (text: string): { message: string } => ({ message: text });
+
+describe("notesOf", () => {
+    test("style warnings fold to one line counting distinct style names, whatever their ids", () => {
+        const notes = notesOf([
+            message("Unrecognised run style: 'Tekst treści (4) Exact' (Style ID: Teksttreci4Exact)"),
+            message("Unrecognised run style: 'Tekst treści (4) Exact' (Style ID: 1009)"),
+            message("Unrecognised paragraph style: 'Body Text 2' (Style ID: Tekstpodstawowy2)"),
+            message("Unrecognised paragraph style: 'Body Text 2' (Style ID: Tekstpodstawowy2)"),
+            message("Unrecognised table style: 'Grid' (Style ID: Grid)"),
+        ]);
+        expect(notes).toEqual(["docx conversion: 3 Word styles without a markdown equivalent, read as plain text"]);
+    });
+
+    test("one style reads in the singular", () => {
+        expect(notesOf([message("Unrecognised paragraph style: 'Note' (Style ID: Note)")])).toEqual([
+            "docx conversion: 1 Word style without a markdown equivalent, read as plain text",
+        ]);
+    });
+
+    test("content the markdown lacks keeps a line each, capped, and comes before the style line", () => {
+        const dropped = Array.from({ length: 7 }, (_, index) => message(`An unrecognised element was ignored: w:custom${index}`));
+        const notes = notesOf([...dropped, message("Unrecognised run style: 'Emphasis' (Style ID: Emphasis)")]);
+        expect(notes).toEqual([
+            ...dropped.slice(0, 5).map((warning) => `docx conversion: ${warning.message}`),
+            "docx conversion: 2 more warnings of the same kind",
+            "docx conversion: 1 Word style without a markdown equivalent, read as plain text",
+        ]);
+    });
+
+    test("no messages, no notes", () => {
+        expect(notesOf([])).toEqual([]);
+    });
+});

package/src/lib/derivers/docx.ts

@@ -0,0 +1,44 @@
+import { bodyOf, parseHtml } from "@intentic/webq/dom";
+import { renderMarkdown } from "@intentic/webq/markdown";
+import type { DerivedDoc, Deriver } from "./deriver.js";
+
+/* Word documents: mammoth maps the docx to semantic HTML (headings, lists, tables — its whole reason to exist). */
+
+// mammoth is 0.26s to load, and only a Word document needs it; see xlsx.ts for why that is loaded inside derive().
+
+// A style mammoth's map does not name still yields its text, as a plain paragraph or run; what the reader loses is
+// the heading or emphasis that style stood for, the same loss for every such style. One line says it; the style
+// names would not, and a corporate template carries dozens, renumbered by every application that re-saves the file.
+const STYLE_WARNING = /^Unrecognised (?:paragraph|run|table) style: '(.*)' \(Style ID: /;
+
+// Every other warning is a piece of content the markdown lacks (an element it dropped, an image it could not carry)
+// and stays a line of its own, up to this many.
+const MAX_NOTES = 5;
+
+/** The notes of a conversion, from mammoth's messages; exported for tests. */
+export const notesOf = (messages: readonly { readonly message: string }[]): string[] => {
+    const distinct = [...new Set(messages.map((message) => message.message))];
+    // Counted by name, not id: the same template re-saved by another application renumbers every id.
+    const styles = new Set(distinct.flatMap((warning) => STYLE_WARNING.exec(warning)?.[1] ?? []));
+    const others = distinct.filter((warning) => !STYLE_WARNING.test(warning));
+    const notes = others.slice(0, MAX_NOTES).map((warning) => `docx conversion: ${warning}`);
+    if (others.length > MAX_NOTES) {
+        notes.push(`docx conversion: ${others.length - MAX_NOTES} more warnings of the same kind`);
+    }
+    if (styles.size > 0) {
+        notes.push(`docx conversion: ${styles.size} Word style${styles.size === 1 ? "" : "s"} without a markdown equivalent, read as plain text`);
+    }
+    return notes;
+};
+
+export const docxDeriver: Deriver = {
+    name: "docx",
+    version: 2,
+    derive: async (absPath): Promise<DerivedDoc> => {
+        const { default: mammoth } = await import("mammoth");
+        const converted = await mammoth.convertToHtml({ path: absPath });
+        const body = bodyOf(parseHtml(converted.value));
+        const markdown = body === undefined ? "" : renderMarkdown(body);
+        return { markdown, notes: notesOf(converted.messages) };
+    },
+};