@intentic/fileq 1.317.0

package/dist/lib/derivers/ipynb.js

@@ -0,0 +1,75 @@
+import { readFile } from "node:fs/promises";
+import { plural } from "@intentic/base/format";
+const MAX_OUTPUT_LINES = 40;
+const joined = (value) => (Array.isArray(value) ? value.join("") : (value ?? ""));
+const linesOf = (value) => joined(value).trimEnd().split("\n");
+const outputLines = (output, counters) => {
+    if (output.output_type === "stream") {
+        return linesOf(output.text);
+    }
+    if (output.output_type === "error") {
+        return [`${output.ename ?? "Error"}: ${output.evalue ?? ""}`.trimEnd()];
+    }
+    if (output.data === undefined) {
+        return [];
+    }
+    const plain = output.data["text/plain"];
+    if (plain !== undefined) {
+        return linesOf(plain);
+    }
+    counters.rich += 1;
+    return [`[${Object.keys(output.data).join(", ")} output omitted]`];
+};
+const codeCell = (cell, source, language, counters) => {
+    const block = [`\`\`\`${language}`, source, "```"];
+    const lines = (cell.outputs ?? []).flatMap((output) => outputLines(output, counters));
+    if (lines.length === 0) {
+        return block.join("\n");
+    }
+    const kept = lines.slice(0, MAX_OUTPUT_LINES);
+    if (lines.length > MAX_OUTPUT_LINES) {
+        counters.clipped += 1;
+        kept.push(`… ${lines.length - MAX_OUTPUT_LINES} more output lines`);
+    }
+    return [...block, "", "```text", ...kept, "```"].join("\n");
+};
+const sectionOf = (cell, language, counters) => {
+    const source = joined(cell.source).trimEnd();
+    if (cell.cell_type === "code") {
+        return codeCell(cell, source, language, counters);
+    }
+    if (source === "") {
+        return undefined;
+    }
+    return cell.cell_type === "markdown" ? source : ["```text", source, "```"].join("\n");
+};
+const notesOf = (cellCount, counters) => {
+    const notes = [];
+    if (cellCount === 0) {
+        notes.push("no cells in this notebook");
+    }
+    if (counters.clipped > 0) {
+        notes.push(`${plural(counters.clipped, "cell")}: output cut at ${MAX_OUTPUT_LINES} lines`);
+    }
+    if (counters.rich > 0) {
+        notes.push(`${plural(counters.rich, "rich output")} (images, HTML) omitted: a notebook's pictures need the notebook itself`);
+    }
+    return notes;
+};
+export const ipynbDeriver = {
+    name: "ipynb",
+    version: 1,
+    derive: async (absPath) => {
+        const notebook = JSON.parse(await readFile(absPath, "utf8"));
+        const language = notebook.metadata?.language_info?.name ?? notebook.metadata?.kernelspec?.language ?? "python";
+        const cells = Array.isArray(notebook.cells) ? notebook.cells : [];
+        const counters = { rich: 0, clipped: 0 };
+        const sections = cells.map((cell) => sectionOf(cell, language, counters)).filter((section) => section !== undefined);
+        const title = cells
+            .filter((cell) => cell.cell_type === "markdown")
+            .map((cell) => /^#\s+(.+)$/m.exec(joined(cell.source))?.[1]?.trim())
+            .find((heading) => heading !== undefined);
+        return { markdown: sections.join("\n\n"), title, notes: notesOf(cells.length, counters) };
+    },
+};
+//# sourceMappingURL=ipynb.js.map

package/dist/lib/derivers/ipynb.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ipynb.js","sourceRoot":"","sources":["../../../src/lib/derivers/ipynb.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAK/C,MAAM,gBAAgB,GAAG,EAAE,CAAC;AA2B5B,MAAM,MAAM,GAAG,CAAC,KAAoC,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AAEzH,MAAM,OAAO,GAAG,CAAC,KAAoC,EAAY,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAExG,MAAM,WAAW,GAAG,CAAC,MAAc,EAAE,QAAkB,EAAY,EAAE;IACjE,IAAI,MAAM,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,MAAM,CAAC,KAAK,IAAI,OAAO,KAAK,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACxC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IACD,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;AACvE,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,IAAU,EAAE,MAAc,EAAE,QAAgB,EAAE,QAAkB,EAAU,EAAE;IAC1F,MAAM,KAAK,GAAG,CAAC,SAAS,QAAQ,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACtF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,IAAI,KAAK,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;QAClC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,gBAAgB,oBAAoB,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChE,CAAC,CAAC;AAGF,MAAM,SAAS,GAAG,CAAC,IAAU,EAAE,QAAgB,EAAE,QAAkB,EAAsB,EAAE;IACvF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7C,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;QAC5B,OAAO,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAChB,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1F,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,SAAiB,EAAE,QAAkB,EAAY,EAAE;IAChE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,QAAQ,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,mBAAmB,gBAAgB,QAAQ,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,yEAAyE,CAAC,CAAC;IACjI,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAY;IACjC,IAAI,EAAE,OAAO;IACb,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,KAAK,EAAE,OAAO,EAAuB,EAAE;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAa,CAAC;QACzE,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,IAAI,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,IAAI,QAAQ,CAAC;QAC/G,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAa,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAqB,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;QAExI,MAAM,KAAK,GAAG,KAAK;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,UAAU,CAAC;aAC/C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;aACnE,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;QAC9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC9F,CAAC;CACJ,CAAC"}

package/dist/lib/derivers/media.d.ts

@@ -0,0 +1,3 @@
+import type { Deriver } from "./deriver.js";
+export declare const mediaDeriver: Deriver;
+//# sourceMappingURL=media.d.ts.map

package/dist/lib/derivers/media.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"media.d.ts","sourceRoot":"","sources":["../../../src/lib/derivers/media.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,OAAO,EAAE,MAAM,cAAc,CAAC;AAYxD,eAAO,MAAM,YAAY,EAAE,OAsC1B,CAAC"}

package/dist/lib/derivers/media.js

@@ -0,0 +1,45 @@
+const formatDuration = (seconds) => {
+    const whole = Math.round(seconds);
+    const minutes = Math.floor(whole / 60);
+    return minutes === 0 ? `${whole}s` : `${minutes}m ${String(whole % 60).padStart(2, "0")}s`;
+};
+export const mediaDeriver = {
+    name: "media",
+    version: 1,
+    derive: async (absPath) => {
+        const { parseFile } = await import("music-metadata");
+        const meta = await parseFile(absPath, { duration: true });
+        const lines = [];
+        const { format, common } = meta;
+        if (format.container !== undefined) {
+            lines.push(`- Container: ${format.container}${format.codec === undefined ? "" : ` (${format.codec})`}`);
+        }
+        if (format.duration !== undefined) {
+            lines.push(`- Duration: ${formatDuration(format.duration)}`);
+        }
+        if (format.sampleRate !== undefined) {
+            lines.push(`- Sample rate: ${format.sampleRate} Hz${format.numberOfChannels === undefined ? "" : `, ${format.numberOfChannels}ch`}`);
+        }
+        if (format.bitrate !== undefined) {
+            lines.push(`- Bitrate: ${Math.round(format.bitrate / 1000)} kbit/s`);
+        }
+        const tags = [
+            ["Title", common.title],
+            ["Artist", common.artist],
+            ["Album", common.album],
+            ["Year", common.year],
+            ["Genre", common.genre?.join(", ")],
+        ];
+        for (const [label, value] of tags) {
+            if (value !== undefined && value !== "") {
+                lines.push(`- ${label}: ${value}`);
+            }
+        }
+        return {
+            markdown: lines.join("\n"),
+            title: common.title,
+            notes: ["no transcript: audio transcription is a later, whisper-backed tier"],
+        };
+    },
+};
+//# sourceMappingURL=media.js.map

package/dist/lib/derivers/media.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"media.js","sourceRoot":"","sources":["../../../src/lib/derivers/media.ts"],"names":[],"mappings":"AAMA,MAAM,cAAc,GAAG,CAAC,OAAe,EAAU,EAAE;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACvC,OAAO,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,MAAM,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;AAC/F,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAY;IACjC,IAAI,EAAE,OAAO;IACb,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,KAAK,EAAE,OAAO,EAAuB,EAAE;QAC3C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAChC,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;QAC5G,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,eAAe,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;QACzI,CAAC;QACD,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,IAAI,GAAgE;YACtE,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC;YACvB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC;YACzB,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC;YACvB,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC;YACrB,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;SACtC,CAAC;QACF,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;YAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QACD,OAAO;YACH,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,CAAC,oEAAoE,CAAC;SAChF,CAAC;IACN,CAAC;CACJ,CAAC"}

package/dist/lib/derivers/odp.d.ts

@@ -0,0 +1,3 @@
+import type { Deriver } from "./deriver.js";
+export declare const odpDeriver: Deriver;
+//# sourceMappingURL=odp.d.ts.map

package/dist/lib/derivers/odp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"odp.d.ts","sourceRoot":"","sources":["../../../src/lib/derivers/odp.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAc,OAAO,EAAE,MAAM,cAAc,CAAC;AAuBxD,eAAO,MAAM,UAAU,EAAE,OA0BxB,CAAC"}

package/dist/lib/derivers/odp.js

@@ -0,0 +1,41 @@
+import { readFile } from "node:fs/promises";
+import { unzipSync } from "fflate";
+import { attributeOf, decodeEntities } from "../xml.js";
+const PAGE = /<draw:page\b([^>]*)>([\s\S]*?)<\/draw:page>/g;
+const NOTES = /<presentation:notes\b[^>]*>[\s\S]*?<\/presentation:notes>/g;
+const PARAGRAPH = /<text:(p|h)\b[^>]*(?:\/>|>([\s\S]*?)<\/text:\1>)/g;
+const lineOf = (inner) => decodeEntities(inner
+    .replaceAll(/<text:s\b[^>]*\/>/g, " ")
+    .replaceAll(/<text:tab\b[^>]*\/>/g, " ")
+    .replaceAll(/<text:line-break\b[^>]*\/>/g, " ")
+    .replaceAll(/<[^>]+>/g, ""))
+    .replaceAll(/\s+/g, " ")
+    .trim();
+const linesOf = (xml) => [...xml.matchAll(PARAGRAPH)].map((match) => lineOf(match[2] ?? "")).filter((line) => line !== "");
+export const odpDeriver = {
+    name: "odp",
+    version: 1,
+    derive: async (absPath) => {
+        const zip = unzipSync(new Uint8Array(await readFile(absPath)), { filter: (file) => file.name === "content.xml" });
+        const content = zip["content.xml"];
+        if (content === undefined) {
+            return { markdown: "", notes: ["no content.xml in this OpenDocument container"] };
+        }
+        const xml = new TextDecoder().decode(content);
+        const sections = [];
+        for (const page of xml.matchAll(PAGE)) {
+            const body = page[2] ?? "";
+            const slide = body.replaceAll(NOTES, "");
+            const notes = (body.match(NOTES) ?? []).flatMap((part) => linesOf(part)).filter((line) => !/^\d+$/.test(line));
+            const name = attributeOf(page[1] ?? "", "draw:name");
+            const heading = `## Slide ${sections.length + 1}${name === undefined || name === "" ? "" : `: ${name}`}`;
+            const section = [heading, ...linesOf(slide)];
+            if (notes.length > 0) {
+                section.push("", "Notes:", ...notes.map((line) => `> ${line}`));
+            }
+            sections.push(section.join("\n"));
+        }
+        return { markdown: sections.join("\n\n"), notes: sections.length === 0 ? ["no slides in this presentation"] : [] };
+    },
+};
+//# sourceMappingURL=odp.js.map

package/dist/lib/derivers/odp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"odp.js","sourceRoot":"","sources":["../../../src/lib/derivers/odp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAIxD,MAAM,IAAI,GAAG,8CAA8C,CAAC;AAC5D,MAAM,KAAK,GAAG,4DAA4D,CAAC;AAC3E,MAAM,SAAS,GAAG,mDAAmD,CAAC;AAGtE,MAAM,MAAM,GAAG,CAAC,KAAa,EAAU,EAAE,CACrC,cAAc,CACV,KAAK;KACA,UAAU,CAAC,oBAAoB,EAAE,GAAG,CAAC;KACrC,UAAU,CAAC,sBAAsB,EAAE,GAAG,CAAC;KACvC,UAAU,CAAC,6BAA6B,EAAE,GAAG,CAAC;KAC9C,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC,CAClC;KACI,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;KACvB,IAAI,EAAE,CAAC;AAEhB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAY,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AAE7I,MAAM,CAAC,MAAM,UAAU,GAAY;IAC/B,IAAI,EAAE,KAAK;IACX,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,KAAK,EAAE,OAAO,EAAuB,EAAE;QAC3C,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC,CAAC;QAClH,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,+CAA+C,CAAC,EAAE,CAAC;QACtF,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE3B,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACzC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/G,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,YAAY,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACzG,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;YACpE,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACvH,CAAC;CACJ,CAAC"}

package/dist/lib/derivers/ods.d.ts

@@ -0,0 +1,3 @@
+import type { Deriver } from "./deriver.js";
+export declare const odsDeriver: Deriver;
+//# sourceMappingURL=ods.d.ts.map

package/dist/lib/derivers/ods.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ods.d.ts","sourceRoot":"","sources":["../../../src/lib/derivers/ods.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAc,OAAO,EAAE,MAAM,cAAc,CAAC;AAmFxD,eAAO,MAAM,UAAU,EAAE,OAkBxB,CAAC"}

package/dist/lib/derivers/ods.js

@@ -0,0 +1,86 @@
+import { readFile } from "node:fs/promises";
+import { unzipSync } from "fflate";
+import { attributeOf, decodeEntities } from "../xml.js";
+const MAX_ROWS_PER_SHEET = 200;
+const MAX_COLUMNS = 30;
+const TABLE = /<table:table\b([^>]*)>([\s\S]*?)<\/table:table>/g;
+const ROW = /<table:table-row\b([^>]*?)(?:\/>|>([\s\S]*?)<\/table:table-row>)/g;
+const CELL = /<table:(covered-table-cell|table-cell)\b([^>]*?)(?:\/>|>([\s\S]*?)<\/table:\1>)/g;
+const repeatOf = (attributes, name, limit) => {
+    const value = Number.parseInt(attributeOf(attributes, name) ?? "1", 10);
+    return Number.isFinite(value) && value > 0 ? Math.min(value, limit) : 1;
+};
+const cellText = (attributes, inner) => {
+    const shown = decodeEntities(inner
+        .replaceAll(/<text:s\b[^>]*\/>/g, " ")
+        .replaceAll(/<text:tab\b[^>]*\/>/g, " ")
+        .replaceAll(/<\/text:p>/g, " ")
+        .replaceAll(/<[^>]+>/g, ""))
+        .replaceAll(/\s+/g, " ")
+        .trim();
+    return shown === "" ? (attributeOf(attributes, "office:value") ?? "") : shown;
+};
+const trimEnd = (items, empty) => {
+    let end = items.length;
+    while (end > 0 && empty(items[end - 1])) {
+        end -= 1;
+    }
+    return items.slice(0, end);
+};
+const cellsOf = (xml) => {
+    const cells = [];
+    for (const cell of xml.matchAll(CELL)) {
+        const attributes = cell[2] ?? "";
+        const text = cell[1] === "covered-table-cell" ? "" : cellText(attributes, cell[3] ?? "");
+        for (let repeat = 0; repeat < repeatOf(attributes, "table:number-columns-repeated", MAX_COLUMNS); repeat += 1) {
+            cells.push(text);
+        }
+    }
+    return trimEnd(cells, (cell) => cell === "");
+};
+const rowsOf = (xml) => {
+    const rows = [];
+    for (const row of xml.matchAll(ROW)) {
+        const cells = cellsOf(row[2] ?? "");
+        for (let repeat = 0; repeat < repeatOf(row[1] ?? "", "table:number-rows-repeated", MAX_ROWS_PER_SHEET); repeat += 1) {
+            rows.push(cells);
+        }
+    }
+    return trimEnd(rows, (row) => row.length === 0);
+};
+const tableRow = (cells) => `| ${cells.map((cell) => cell.replaceAll("|", "\\|")).join(" | ")} |`;
+const sectionOf = (name, rows, notes) => {
+    if (rows.length === 0) {
+        return `## ${name}\n\n(empty sheet)`;
+    }
+    const width = Math.min(MAX_COLUMNS, Math.max(...rows.map((row) => row.length)));
+    if (Math.max(...rows.map((row) => row.length)) > MAX_COLUMNS) {
+        notes.push(`sheet "${name}": showing ${MAX_COLUMNS} of ${Math.max(...rows.map((row) => row.length))} columns`);
+    }
+    if (rows.length > MAX_ROWS_PER_SHEET) {
+        notes.push(`sheet "${name}": showing ${MAX_ROWS_PER_SHEET} of ${rows.length} rows`);
+    }
+    const kept = rows.slice(0, MAX_ROWS_PER_SHEET).map((row) => Array.from({ length: width }, (_, index) => row[index] ?? ""));
+    const [header, ...body] = kept;
+    return `## ${name}\n\n${[tableRow(header), tableRow(header.map(() => "---")), ...body.map(tableRow)].join("\n")}`;
+};
+export const odsDeriver = {
+    name: "ods",
+    version: 1,
+    derive: async (absPath) => {
+        const zip = unzipSync(new Uint8Array(await readFile(absPath)), { filter: (file) => file.name === "content.xml" });
+        const content = zip["content.xml"];
+        if (content === undefined) {
+            return { markdown: "", notes: ["no content.xml in this OpenDocument container"] };
+        }
+        const xml = new TextDecoder().decode(content);
+        const notes = [];
+        const sections = [];
+        for (const table of xml.matchAll(TABLE)) {
+            const name = attributeOf(table[1] ?? "", "table:name") ?? `Sheet ${sections.length + 1}`;
+            sections.push(sectionOf(name, rowsOf(table[2] ?? ""), notes));
+        }
+        return { markdown: sections.join("\n\n"), notes: sections.length === 0 ? [...notes, "no sheets in this spreadsheet"] : notes };
+    },
+};
+//# sourceMappingURL=ods.js.map

package/dist/lib/derivers/ods.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ods.js","sourceRoot":"","sources":["../../../src/lib/derivers/ods.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAKxD,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,WAAW,GAAG,EAAE,CAAC;AAEvB,MAAM,KAAK,GAAG,kDAAkD,CAAC;AACjE,MAAM,GAAG,GAAG,mEAAmE,CAAC;AAEhF,MAAM,IAAI,GAAG,kFAAkF,CAAC;AAEhG,MAAM,QAAQ,GAAG,CAAC,UAAkB,EAAE,IAAY,EAAE,KAAa,EAAU,EAAE;IACzE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IACxE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC,CAAC;AAGF,MAAM,QAAQ,GAAG,CAAC,UAAkB,EAAE,KAAa,EAAU,EAAE;IAC3D,MAAM,KAAK,GAAG,cAAc,CACxB,KAAK;SACA,UAAU,CAAC,oBAAoB,EAAE,GAAG,CAAC;SACrC,UAAU,CAAC,sBAAsB,EAAE,GAAG,CAAC;SACvC,UAAU,CAAC,aAAa,EAAE,GAAG,CAAC;SAC9B,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC,CAClC;SACI,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,IAAI,EAAE,CAAC;IACZ,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAClF,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAI,KAAU,EAAE,KAA2B,EAAO,EAAE;IAChE,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,OAAO,GAAG,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAM,CAAC,EAAE,CAAC;QAC3C,GAAG,IAAI,CAAC,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,GAAW,EAAY,EAAE;IACtC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzF,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,+BAA+B,EAAE,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5G,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC;AAGF,MAAM,MAAM,GAAG,CAAC,GAAW,EAAc,EAAE;IACvC,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,4BAA4B,EAAE,kBAAkB,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;YAClH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,KAAe,EAAU,EAAE,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;AAEpH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,IAAgB,EAAE,KAAe,EAAU,EAAE;IAC1E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,MAAM,IAAI,mBAAmB,CAAC;IACzC,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,WAAW,EAAE,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,cAAc,WAAW,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;IACnH,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,cAAc,kBAAkB,OAAO,IAAI,CAAC,MAAM,OAAO,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3H,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,IAAiC,CAAC;IAC5D,OAAO,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AACtH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAY;IAC/B,IAAI,EAAE,KAAK;IACX,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,KAAK,EAAE,OAAO,EAAuB,EAAE;QAC3C,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC,CAAC;QAClH,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,+CAA+C,CAAC,EAAE,CAAC;QACtF,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,YAAY,CAAC,IAAI,SAAS,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzF,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,EAAE,+BAA+B,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IACnI,CAAC;CACJ,CAAC"}

package/dist/lib/derivers/odt.d.ts

@@ -0,0 +1,4 @@
+import type { Deriver } from "./deriver.js";
+export declare const odfToHtml: (xml: string) => string;
+export declare const odtDeriver: Deriver;
+//# sourceMappingURL=odt.d.ts.map

package/dist/lib/derivers/odt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"odt.d.ts","sourceRoot":"","sources":["../../../src/lib/derivers/odt.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAc,OAAO,EAAE,MAAM,cAAc,CAAC;AAyCxD,eAAO,MAAM,SAAS,QAAS,MAAM,KAAG,MAqCvC,CAAC;AAEF,eAAO,MAAM,UAAU,EAAE,OAgBxB,CAAC"}

package/dist/lib/derivers/odt.js

@@ -0,0 +1,77 @@
+import { readFile } from "node:fs/promises";
+import { unzipSync } from "fflate";
+import { bodyOf, parseHtml } from "@intentic/webq/dom";
+import { renderMarkdown } from "@intentic/webq/markdown";
+import { attributeOf, decodeEntities } from "../xml.js";
+const TAG = {
+    "text:p": "p",
+    "text:list": "ul",
+    "text:list-item": "li",
+    "text:list-header": "li",
+    "text:span": "span",
+    "text:a": "a",
+    "text:line-break": "br",
+    "text:note-citation": "sup",
+    "table:table": "table",
+    "table:table-header-rows": "thead",
+    "table:table-row": "tr",
+    "table:table-cell": "td",
+    "table:covered-table-cell": "td",
+    "table:table-column": "col",
+    "table:table-columns": "colgroup",
+};
+const DROP_SUBTREE = [
+    "office:annotation",
+    "text:tracked-changes",
+    "draw:frame",
+    "text:table-of-content-source",
+    "text:index-title-template",
+    "text:alphabetical-index-source",
+];
+const dropSubtrees = (xml) => DROP_SUBTREE.reduce((text, tag) => text.replaceAll(new RegExp(`<${tag}\\b[^>]*?(?:/>|>[\\s\\S]*?</${tag}>)`, "g"), ""), xml);
+export const odfToHtml = (xml) => {
+    const start = xml.indexOf("<office:text");
+    const end = xml.lastIndexOf("</office:text>");
+    const body = dropSubtrees(start === -1 || end === -1 ? xml : xml.slice(start, end));
+    return (body
+        .replaceAll(/<text:h\b([^>]*)>([\s\S]*?)<\/text:h>/g, (_, attributes, inner) => {
+        const level = Math.min(6, Math.max(1, Number(attributeOf(attributes, "text:outline-level") ?? "1") || 1));
+        return `<h${level}>${inner}</h${level}>`;
+    })
+        .replaceAll(/<text:s\b([^>]*)\/>/g, (_, attributes) => " ".repeat(Number(attributeOf(attributes, "text:c") ?? "1") || 1))
+        .replaceAll(/<text:tab\b[^>]*\/>/g, " ")
+        .replaceAll(/<(\/?)([a-zA-Z0-9]+:[a-zA-Z0-9-]+)((?:\s[^>]*?)?)(\/?)>/g, (_, close, name, attributes, selfClosing) => {
+        const mapped = TAG[name];
+        if (mapped === undefined) {
+            return "";
+        }
+        if (close === "/") {
+            return `</${mapped}>`;
+        }
+        if (mapped === "a") {
+            const href = attributeOf(attributes, "xlink:href");
+            return href === undefined ? "<a>" : `<a href="${href.replaceAll('"', "&quot;")}">`;
+        }
+        return selfClosing === "/" && mapped !== "br" && mapped !== "col" ? `<${mapped}></${mapped}>` : `<${mapped}>`;
+    })
+        .replaceAll(/<li>\s*<p>/g, "<li>")
+        .replaceAll(/<\/p>\s*<\/li>/g, "</li>"));
+};
+export const odtDeriver = {
+    name: "odt",
+    version: 1,
+    derive: async (absPath) => {
+        const zip = unzipSync(new Uint8Array(await readFile(absPath)));
+        const decoder = new TextDecoder();
+        const content = zip["content.xml"];
+        if (content === undefined) {
+            return { markdown: "", notes: ["no content.xml in this OpenDocument container"] };
+        }
+        const meta = zip["meta.xml"];
+        const titleMatch = meta === undefined ? null : /<dc:title>([\s\S]*?)<\/dc:title>/.exec(decoder.decode(meta));
+        const title = titleMatch?.[1] === undefined || titleMatch[1].trim() === "" ? undefined : decodeEntities(titleMatch[1].trim());
+        const body = bodyOf(parseHtml(`<html><body>${odfToHtml(decoder.decode(content))}</body></html>`));
+        return { markdown: body === undefined ? "" : renderMarkdown(body), title, notes: [] };
+    },
+};
+//# sourceMappingURL=odt.js.map

package/dist/lib/derivers/odt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"odt.js","sourceRoot":"","sources":["../../../src/lib/derivers/odt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAQxD,MAAM,GAAG,GAA2B;IAChC,QAAQ,EAAE,GAAG;IACb,WAAW,EAAE,IAAI;IACjB,gBAAgB,EAAE,IAAI;IACtB,kBAAkB,EAAE,IAAI;IACxB,WAAW,EAAE,MAAM;IACnB,QAAQ,EAAE,GAAG;IACb,iBAAiB,EAAE,IAAI;IACvB,oBAAoB,EAAE,KAAK;IAC3B,aAAa,EAAE,OAAO;IACtB,yBAAyB,EAAE,OAAO;IAClC,iBAAiB,EAAE,IAAI;IACvB,kBAAkB,EAAE,IAAI;IACxB,0BAA0B,EAAE,IAAI;IAChC,oBAAoB,EAAE,KAAK;IAC3B,qBAAqB,EAAE,UAAU;CACpC,CAAC;AAGF,MAAM,YAAY,GAAG;IACjB,mBAAmB;IACnB,sBAAsB;IACtB,YAAY;IACZ,8BAA8B;IAC9B,2BAA2B;IAC3B,gCAAgC;CACnC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,GAAW,EAAU,EAAE,CACzC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,+BAA+B,GAAG,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;AAGjI,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAU,EAAE;IAC7C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IACpF,OAAO,CACH,IAAI;SAEC,UAAU,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE,UAAkB,EAAE,KAAa,EAAE,EAAE;QAC3F,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,oBAAoB,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1G,OAAO,KAAK,KAAK,IAAI,KAAK,MAAM,KAAK,GAAG,CAAC;IAC7C,CAAC,CAAC;SAED,UAAU,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,UAAkB,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAChI,UAAU,CAAC,sBAAsB,EAAE,GAAG,CAAC;SAEvC,UAAU,CACP,0DAA0D,EAC1D,CAAC,CAAC,EAAE,KAAa,EAAE,IAAY,EAAE,UAAkB,EAAE,WAAmB,EAAE,EAAE;QACxE,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACd,CAAC;QACD,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;YAChB,OAAO,KAAK,MAAM,GAAG,CAAC;QAC1B,CAAC;QACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,WAAW,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;YACnD,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;QACvF,CAAC;QACD,OAAO,WAAW,KAAK,GAAG,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,GAAG,CAAC;IAClH,CAAC,CACJ;SAGA,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC;SACjC,UAAU,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAC9C,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAY;IAC/B,IAAI,EAAE,KAAK;IACX,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,KAAK,EAAE,OAAO,EAAuB,EAAE;QAC3C,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,+CAA+C,CAAC,EAAE,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kCAAkC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7G,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9H,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,eAAe,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAClG,OAAO,EAAE,QAAQ,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAC1F,CAAC;CACJ,CAAC"}

package/dist/lib/derivers/pdf-furniture.d.ts

@@ -0,0 +1,7 @@
+export interface Destripped {
+    readonly pages: string[];
+    readonly removed: number;
+    readonly example: string | undefined;
+}
+export declare const stripRunningFurniture: (pages: readonly string[]) => Destripped;
+//# sourceMappingURL=pdf-furniture.d.ts.map

package/dist/lib/derivers/pdf-furniture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-furniture.d.ts","sourceRoot":"","sources":["../../../src/lib/derivers/pdf-furniture.ts"],"names":[],"mappings":"AAoBA,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAEzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC;AA2DD,eAAO,MAAM,qBAAqB,UAAW,SAAS,MAAM,EAAE,KAAG,UAmChE,CAAC"}

package/dist/lib/derivers/pdf-furniture.js

@@ -0,0 +1,76 @@
+const EDGE_LINES = 3;
+const RECUR_SHARE = 0.6;
+const MIN_PAGES = 4;
+const PAGE_NUMBER = [
+    /^page\s+\d{1,4}(?:\s+of\s+\d{1,4})?$/i,
+    /^\d{1,4}\s*(?:of|\/)\s*\d{1,4}$/i,
+    /^[-–—]\s*\d{1,4}\s*[-–—]$/,
+];
+const signaturesOf = (line, page) => {
+    const exact = line.replaceAll(/\s+/g, " ").trim().toLowerCase();
+    return [`=${exact}`, `~${exact.replaceAll(/\d+/g, (n) => `#${Number(n) - page}`)}`];
+};
+const edgesOf = (lines) => {
+    const filled = lines.flatMap((line, index) => (line.trim() === "" ? [] : [index]));
+    const edge = Math.min(EDGE_LINES, Math.floor(filled.length / 2));
+    return edge === 0 ? { top: [], bottom: [] } : { top: filled.slice(0, edge), bottom: filled.slice(-edge) };
+};
+const recurringAt = (split, side, needed) => {
+    const recurrence = new Map();
+    for (const [page, lines] of split.entries()) {
+        for (const signature of new Set(edgesOf(lines)[side].flatMap((index) => signaturesOf(lines[index] ?? "", page)))) {
+            recurrence.set(signature, (recurrence.get(signature) ?? 0) + 1);
+        }
+    }
+    return new Set([...recurrence].filter(([signature, count]) => count >= needed && !/^[=~]#?-?\d+$/.test(signature)).map(([signature]) => signature));
+};
+const BARE_NUMBER = /^\d{1,4}$/;
+const folioOffset = (split, needed) => {
+    const votes = new Map();
+    split.forEach((lines, page) => {
+        const offsets = new Set([...edgesOf(lines).top, ...edgesOf(lines).bottom]
+            .map((index) => (lines[index] ?? "").trim())
+            .filter((line) => BARE_NUMBER.test(line))
+            .map((line) => Number(line) - page));
+        for (const offset of offsets) {
+            votes.set(offset, (votes.get(offset) ?? 0) + 1);
+        }
+    });
+    const [offset, count] = [...votes].toSorted((a, b) => b[1] - a[1])[0] ?? [0, 0];
+    return count >= needed ? offset : undefined;
+};
+export const stripRunningFurniture = (pages) => {
+    if (pages.length < MIN_PAGES) {
+        return { pages: [...pages], removed: 0, example: undefined };
+    }
+    const split = pages.map((page) => page.split("\n"));
+    const needed = Math.ceil(pages.length * RECUR_SHARE);
+    const headers = recurringAt(split, "top", needed);
+    const footers = recurringAt(split, "bottom", needed);
+    const offset = folioOffset(split, needed);
+    const seen = new Map();
+    let removed = 0;
+    const out = split.map((lines, page) => {
+        const { top, bottom } = edgesOf(lines);
+        const furniture = (running) => (index) => {
+            const line = (lines[index] ?? "").trim();
+            const folio = offset !== undefined && BARE_NUMBER.test(line) && Number(line) - page === offset;
+            return folio || signaturesOf(line, page).some((signature) => running.has(signature)) || PAGE_NUMBER.some((shape) => shape.test(line));
+        };
+        const drop = new Set([...top.filter(furniture(headers)), ...bottom.filter(furniture(footers))]);
+        for (const index of drop) {
+            const line = (lines[index] ?? "").trim();
+            if (!/^\d+$/.test(line)) {
+                seen.set(line, (seen.get(line) ?? 0) + 1);
+            }
+        }
+        removed += drop.size;
+        return lines
+            .filter((_, index) => !drop.has(index))
+            .join("\n")
+            .trim();
+    });
+    const example = [...seen].toSorted((a, b) => b[1] - a[1])[0]?.[0];
+    return { pages: out, removed, example };
+};
+//# sourceMappingURL=pdf-furniture.js.map

package/dist/lib/derivers/pdf-furniture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-furniture.js","sourceRoot":"","sources":["../../../src/lib/derivers/pdf-furniture.ts"],"names":[],"mappings":"AAQA,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,SAAS,GAAG,CAAC,CAAC;AAGpB,MAAM,WAAW,GAAG;IAChB,uCAAuC;IACvC,kCAAkC;IAClC,2BAA2B;CAC9B,CAAC;AAaF,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,IAAY,EAAY,EAAE;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAChE,OAAO,CAAC,IAAI,KAAK,EAAE,EAAE,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;AACxF,CAAC,CAAC;AAOF,MAAM,OAAO,GAAG,CAAC,KAAwB,EAAS,EAAE;IAChD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEnF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;AAC9G,CAAC,CAAC;AAGF,MAAM,WAAW,GAAG,CAAC,KAA0B,EAAE,IAAiB,EAAE,MAAc,EAAe,EAAE;IAC/F,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAE1C,KAAK,MAAM,SAAS,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/G,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,CAAC;IACL,CAAC;IAED,OAAO,IAAI,GAAG,CACV,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CACtI,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,WAAW,CAAC;AAIhC,MAAM,WAAW,GAAG,CAAC,KAA0B,EAAE,MAAc,EAAsB,EAAE;IACnF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,IAAI,GAAG,CACnB,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;aAC5C,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;aAC3C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAC1C,CAAC;QACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,OAAO,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC,CAAC;AAGF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,KAAwB,EAAc,EAAE;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IACjE,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAClC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,SAAS,GACX,CAAC,OAAoB,EAAE,EAAE,CACzB,CAAC,KAAa,EAAW,EAAE;YACvB,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,MAAM,CAAC;YAC/F,OAAO,KAAK,IAAI,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1I,CAAC,CAAC;QACN,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAChG,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,CAAC;QACL,CAAC;QACD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC;QACrB,OAAO,KAAK;aACP,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;aACtC,IAAI,CAAC,IAAI,CAAC;aACV,IAAI,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAClE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC5C,CAAC,CAAC"}

package/dist/lib/derivers/pdf.d.ts

@@ -0,0 +1,4 @@
+import type { Deriver } from "./deriver.js";
+export declare const ocrAvailable: () => boolean;
+export declare const pdfDeriver: Deriver;
+//# sourceMappingURL=pdf.d.ts.map

package/dist/lib/derivers/pdf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../../src/lib/derivers/pdf.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAc,OAAO,EAAE,MAAM,cAAc,CAAC;AAmBxD,eAAO,MAAM,YAAY,QAAO,OAAoD,CAAC;AAsCrF,eAAO,MAAM,UAAU,EAAE,OAqCxB,CAAC"}

package/dist/lib/derivers/pdf.js

@@ -0,0 +1,79 @@
+import { execFile } from "node:child_process";
+import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { extractText, getDocumentProxy, getMeta } from "unpdf";
+import { onPath } from "../tools.js";
+import { stripRunningFurniture } from "./pdf-furniture.js";
+const SCAN_THRESHOLD_CHARS_PER_PAGE = 24;
+const MAX_OCR_PAGES = 20;
+const OCR_DPI = "200";
+const run = promisify(execFile);
+export const ocrAvailable = () => onPath("tesseract") && onPath("pdftoppm");
+const recognisePages = async (absPath, totalPages) => {
+    const dir = await mkdtemp(join(tmpdir(), "fileq-ocr-"));
+    try {
+        await run("pdftoppm", ["-r", OCR_DPI, "-png", "-f", "1", "-l", String(Math.min(totalPages, MAX_OCR_PAGES)), absPath, join(dir, "page")], {
+            timeout: 120_000,
+        });
+        const images = (await readdir(dir)).filter((name) => name.endsWith(".png")).toSorted();
+        const pages = [];
+        for (const image of images) {
+            const { stdout } = await run("tesseract", [join(dir, image), "stdout", "-l", "eng"], { timeout: 120_000, maxBuffer: 16 * 1024 * 1024 });
+            pages.push(stdout.replaceAll(/[ \t]+/g, " ").trim());
+        }
+        return pages;
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+};
+const pagesToMarkdown = (pages) => pages.length === 1
+    ? (pages[0] ?? "")
+    : pages.map((page, index) => `## Page ${index + 1}\n\n${page === "" ? "(no text on this page)" : page}`).join("\n\n");
+const furnitureNotes = (removed, example) => removed === 0
+    ? []
+    : [
+        `removed ${removed} running header, footer and page-number line${removed === 1 ? "" : "s"}${example === undefined ? "" : ` (such as "${example}")`}`,
+    ];
+const withoutFurniture = (pages) => {
+    const { pages: kept, removed, example } = stripRunningFurniture(pages);
+    return { markdown: pagesToMarkdown(kept), notes: furnitureNotes(removed, example) };
+};
+export const pdfDeriver = {
+    get name() {
+        return ocrAvailable() ? "pdf+ocr" : "pdf";
+    },
+    version: 2,
+    derive: async (absPath) => {
+        const pdf = await getDocumentProxy(new Uint8Array(await readFile(absPath)));
+        const { totalPages, text } = await extractText(pdf, { mergePages: false });
+        const meta = await getMeta(pdf).catch(() => undefined);
+        const info = meta?.info;
+        const title = typeof info?.["Title"] === "string" && info["Title"] !== "" ? info["Title"] : undefined;
+        const pages = text.map((page) => page.replaceAll(/[ \t]+/g, " ").trim());
+        const totalChars = pages.reduce((sum, page) => sum + page.length, 0);
+        if (totalChars < SCAN_THRESHOLD_CHARS_PER_PAGE * totalPages) {
+            const plural = totalPages === 1 ? "" : "s";
+            if (!ocrAvailable()) {
+                return {
+                    markdown: "",
+                    title,
+                    notes: [`no usable text layer across ${totalPages} page${plural} (a scan?): OCR is not part of this tier`],
+                };
+            }
+            const recognised = await recognisePages(absPath, totalPages);
+            const notes = [
+                `scanned: text recognised by OCR (tesseract) on ${recognised.length} of ${totalPages} page${plural} — recognised, not exact; check figures against the page image`,
+            ];
+            if (totalPages > MAX_OCR_PAGES) {
+                notes.push(`OCR stops at ${MAX_OCR_PAGES} pages: run tesseract over the rest yourself if they matter`);
+            }
+            const { markdown, notes: stripped } = withoutFurniture(recognised);
+            return { markdown, title, notes: [...notes, ...stripped] };
+        }
+        return { title, ...withoutFurniture(pages) };
+    },
+};
+//# sourceMappingURL=pdf.js.map

package/dist/lib/derivers/pdf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../src/lib/derivers/pdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAS3D,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAGzC,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,OAAO,GAAG,KAAK,CAAC;AAEtB,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAGhC,MAAM,CAAC,MAAM,YAAY,GAAG,GAAY,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;AAErF,MAAM,cAAc,GAAG,KAAK,EAAE,OAAe,EAAE,UAAkB,EAAqB,EAAE;IACpF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;IACxD,IAAI,CAAC;QACD,MAAM,GAAG,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE;YACrI,OAAO,EAAE,OAAO;SACnB,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACvF,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC;YACxI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;YAAS,CAAC;QACP,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,KAAwB,EAAU,EAAE,CACzD,KAAK,CAAC,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,WAAW,KAAK,GAAG,CAAC,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAG9H,MAAM,cAAc,GAAG,CAAC,OAAe,EAAE,OAA2B,EAAY,EAAE,CAC9E,OAAO,KAAK,CAAC;IACT,CAAC,CAAC,EAAE;IACJ,CAAC,CAAC;QACI,WAAW,OAAO,+CAA+C,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,OAAO,IAAI,EAAE;KACvJ,CAAC;AAEZ,MAAM,gBAAgB,GAAG,CAAC,KAAwB,EAAyC,EAAE;IACzF,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,EAAE,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC;AACxF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAY;IAI/B,IAAI,IAAI;QACJ,OAAO,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;IAC9C,CAAC;IACD,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,KAAK,EAAE,OAAO,EAAuB,EAAE;QAC3C,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5E,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACvD,MAAM,IAAI,GAAG,IAAI,EAAE,IAA2C,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtG,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzE,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACrE,IAAI,UAAU,GAAG,6BAA6B,GAAG,UAAU,EAAE,CAAC;YAC1D,MAAM,MAAM,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAC3C,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;gBAClB,OAAO;oBACH,QAAQ,EAAE,EAAE;oBACZ,KAAK;oBACL,KAAK,EAAE,CAAC,+BAA+B,UAAU,QAAQ,MAAM,0CAA0C,CAAC;iBAC7G,CAAC;YACN,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YAC7D,MAAM,KAAK,GAAG;gBACV,kDAAkD,UAAU,CAAC,MAAM,OAAO,UAAU,QAAQ,MAAM,gEAAgE;aACrK,CAAC;YACF,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,gBAAgB,aAAa,6DAA6D,CAAC,CAAC;YAC3G,CAAC;YACD,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;YACnE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC/D,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,GAAG,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;IACjD,CAAC;CACJ,CAAC"}

package/dist/lib/derivers/pptx.d.ts

@@ -0,0 +1,4 @@
+import type { Deriver } from "./deriver.js";
+export declare const slideParagraphs: (xml: string) => string[];
+export declare const pptxDeriver: Deriver;
+//# sourceMappingURL=pptx.d.ts.map

package/dist/lib/derivers/pptx.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pptx.d.ts","sourceRoot":"","sources":["../../../src/lib/derivers/pptx.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAc,OAAO,EAAE,MAAM,cAAc,CAAC;AASxD,eAAO,MAAM,eAAe,QAAS,MAAM,KAAG,MAAM,EAQd,CAAC;AAavC,eAAO,MAAM,WAAW,EAAE,OA4BzB,CAAC"}