@intentic/fileq 1.337.0 → 1.339.0

package/src/lib/check/check-pdf.ts

@@ -7,6 +7,7 @@
 // https://github.com/MODSetter/SurfSense; see this package's NOTICE). Changed here: a page is blank only when nothing
 // at all is drawn on it (a cover image with no text is not), and the fourteen standard PDF fonts, which every reader
 // carries, are never reported as missing.
+import { errorMessage } from "@intentic/base/errors";
 import { getDocumentProxy, getResolvedPDFJS } from "unpdf";
 import { error, plural, warning, type CheckReport, type Finding } from "./finding.js";
 
@@ -182,7 +183,7 @@ const openPdf = async (bytes: Uint8Array): Promise<PdfDocument | CheckReport> =>
         const reason =
             cause instanceof Error && cause.name === "PasswordException"
                 ? "it is password-protected, so nothing past the password can be checked"
-                : `it is not a readable PDF (${cause instanceof Error ? cause.message : String(cause)})`;
+                : `it is not a readable PDF (${errorMessage(cause)})`;
         return { format: "pdf", findings: [error("unreadable", "", reason)], notes: [] };
     }
 };

package/src/lib/check/package.ts

@@ -1,4 +1,5 @@
 import { posix } from "node:path";
+import { errorMessage } from "@intentic/base/errors";
 import { unzipSync, type UnzipFileInfo } from "fflate";
 import { error, type Finding } from "./finding.js";
 import { childrenNamed, parseXml, type XmlElement } from "./xml-tree.js";
@@ -61,7 +62,7 @@ export const openPackage = (bytes: Uint8Array): OoxmlPackage => {
             },
         });
     } catch (cause) {
-        throw new NotAPackage(`not a readable zip package (${cause instanceof Error ? cause.message : String(cause)})`);
+        throw new NotAPackage(`not a readable zip package (${errorMessage(cause)})`);
     }
     const decoder = new TextDecoder();
     const texts = new Map<string, string>();

package/src/lib/derive.test.ts

@@ -1,4 +1,7 @@
-import { STATE_DIR } from "@intentic/constants";
+import { SHADOWED_DOCUMENT_EXTENSIONS, STATE_DIR } from "@intentic/constants";
+import { documentExtensions } from "./formats.js";
+import { ocrPageCap } from "./derivers/pdf.js";
+import { ocrCapOf } from "./sidecar.js";
 import { isDeriveIgnored } from "./derive.js";
 
 describe("the derive floor (what never gets a shadow, whoever asks)", () => {
@@ -19,3 +22,26 @@ describe("the derive floor (what never gets a shadow, whoever asks)", () => {
         }
     });
 });
+
+describe("the OCR page cap (interactive 20, the background keeper more)", () => {
+    afterEach(() => {
+        delete process.env["FILEQ_OCR_PAGES"];
+    });
+
+    test("FILEQ_OCR_PAGES raises it; nonsense leaves the default", () => {
+        expect(ocrPageCap()).toBe(20);
+        process.env["FILEQ_OCR_PAGES"] = "500";
+        expect(ocrPageCap()).toBe(500);
+        process.env["FILEQ_OCR_PAGES"] = "lots";
+        expect(ocrPageCap()).toBe(20);
+    });
+
+    test("a rendering's cap is read back off its note; an uncapped one has none", () => {
+        expect(ocrCapOf(["scanned: text recognised by OCR", "OCR stops at 20 pages: draw the rest with pdftoppm"])).toBe(20);
+        expect(ocrCapOf(["scanned: text recognised by OCR"])).toBeUndefined();
+    });
+});
+
+test("fileq's document formats are the extensions iq and the preprocessor treat as documents", () => {
+    expect(documentExtensions()).toEqual([...SHADOWED_DOCUMENT_EXTENSIONS].toSorted());
+});

package/src/lib/derive.ts

@@ -17,13 +17,26 @@ import { mediaDeriver } from "./derivers/media.js";
 import { odpDeriver } from "./derivers/odp.js";
 import { odsDeriver } from "./derivers/ods.js";
 import { odtDeriver } from "./derivers/odt.js";
-import { pdfDeriver } from "./derivers/pdf.js";
+import { ocrPageCap, pdfDeriver } from "./derivers/pdf.js";
 import { pptxDeriver } from "./derivers/pptx.js";
 import { profileDeriver } from "./derivers/profile.js";
 import { rtfDeriver } from "./derivers/rtf.js";
 import { xlsxDeriver } from "./derivers/xlsx.js";
 import { cacheRendering, readCachedRendering } from "./content-cache.js";
-import { isFresh, readSidecar, removeSidecar, sidecarBody, sidecarPathFor, sha256OfFile, writeSidecar } from "./sidecar.js";
+import {
+    isFresh,
+    ocrCapOf,
+    parseSidecarFront,
+    readSidecar,
+    readSourceStat,
+    removeSidecar,
+    sidecarBody,
+    sidecarPathFor,
+    sha256OfFile,
+    sourceUnchanged,
+    writeSidecar,
+    type SourceStat,
+} from "./sidecar.js";
 
 // Pipeline `read` and `derive` both run: place the file, recognize it, route it, keep its shadow honest.
 // One module, so `read` and `derive` cannot disagree about a file's markdown.
@@ -90,6 +103,10 @@ interface Rendered {
     readonly cachedPath: string | undefined;
 }
 
+// A rendering whose OCR stopped short of the page cap this run asks for (a 20-page interactive read, met by the shadow
+// keeper's 500) is not good enough for it; one that went deeper serves a shallower ask as it is.
+const deepEnough = (notes: readonly string[]): boolean => (ocrCapOf(notes) ?? Number.POSITIVE_INFINITY) >= ocrPageCap();
+
 // These bytes as this deriver renders them: from the content cache when rendered before, else derived and stored there.
 // Throws only what the deriver throws.
 const renderCached = async (
@@ -101,7 +118,7 @@ const renderCached = async (
 ): Promise<Rendered> => {
     const stamp = deriverStamp(deriver);
     const cached = await readCachedRendering(workspaceRoot, sourceSha, stamp);
-    if (cached !== undefined) {
+    if (cached !== undefined && deepEnough(cached.doc.notes)) {
         return { kind: "fresh", doc: cached.doc, derivedAt: cached.derivedAt, cachedPath: cached.path };
     }
     // Neutralized once here so every consumer gets folded text.
@@ -110,6 +127,40 @@ const renderCached = async (
     return { kind: "derived", doc, derivedAt, cachedPath: await cacheRendering(workspaceRoot, { sourceSha, deriverStamp: stamp, doc, derivedAt }) };
 };
 
+interface ReuseAsk {
+    readonly relPath: string;
+    readonly sourceSha: string;
+    readonly stamp: string;
+    readonly sourceStat: SourceStat | undefined;
+}
+
+// The sidecar's body when it is fresh for these bytes and deep enough for this run; undefined when it must be derived.
+// Same bytes under a new stat (a touch, a checkout, a sidecar from before stats were kept) get the stat rewritten, so
+// hash-free readers find the shadow current again.
+const reuseFresh = async (workspaceRoot: string, ask: ReuseAsk, now: () => Date): Promise<string | undefined> => {
+    const existing = await readSidecar(sidecarPathFor(workspaceRoot, ask.relPath));
+    if (existing === undefined || !isFresh(existing, ask.sourceSha, ask.stamp)) {
+        return undefined;
+    }
+    const front = parseSidecarFront(existing);
+    if (!deepEnough(front.notes)) {
+        return undefined;
+    }
+    const body = sidecarBody(existing);
+    if (!sourceUnchanged(front, ask.sourceStat)) {
+        const derivedAt = new Date(front.derivedAt ?? Number.NaN);
+        await writeSidecar(workspaceRoot, {
+            relPath: ask.relPath,
+            sourceSha: ask.sourceSha,
+            deriverStamp: ask.stamp,
+            sourceStat: ask.sourceStat,
+            doc: { markdown: body.endsWith("\n") ? body.slice(0, -1) : body, title: front.title, notes: [...front.notes] },
+            derivedAt: Number.isNaN(derivedAt.getTime()) ? now() : derivedAt,
+        });
+    }
+    return body;
+};
+
 /** Converges a workspace file's sidecar with its source: derive when stale, reuse fresh, remove when gone. */
 export const ensureSidecar = async (workspaceRoot: string, absPath: string, now: () => Date = () => new Date()): Promise<Outcome> => {
     const relPath = relPathIn(workspaceRoot, absPath);
@@ -142,11 +193,13 @@ export const ensureSidecar = async (workspaceRoot: string, absPath: string, now:
     }
     const deriver = DERIVERS[format];
     const stamp = deriverStamp(deriver);
+    // Taken before the hash, so a write racing the read leaves a stat that no longer matches, never one that vouches
+    // for bytes it did not see.
+    const sourceStat = await readSourceStat(absPath);
     const sourceSha = await sha256OfFile(absPath);
-    const existing = await readSidecar(sidecarPath);
-    if (isFresh(existing, sourceSha, stamp)) {
-        const body = sidecarBody(existing ?? "");
-        return { kind: "fresh", relPath, format, sidecarPath, body, tokens: estimateTokens(body) };
+    const fresh = await reuseFresh(workspaceRoot, { relPath, sourceSha, stamp, sourceStat }, now);
+    if (fresh !== undefined) {
+        return { kind: "fresh", relPath, format, sidecarPath, body: fresh, tokens: estimateTokens(fresh) };
     }
     // A corrupt file's failure is a loud skip, not fatal.
     let rendered: Rendered;
@@ -156,7 +209,14 @@ export const ensureSidecar = async (workspaceRoot: string, absPath: string, now:
         const detail = errorMessage(error).split("\n")[0];
         return { kind: "skipped", relPath, reason: `derive-failed (${format}): ${detail}` };
     }
-    const written = await writeSidecar(workspaceRoot, { relPath, sourceSha, deriverStamp: stamp, doc: rendered.doc, derivedAt: rendered.derivedAt });
+    const written = await writeSidecar(workspaceRoot, {
+        relPath,
+        sourceSha,
+        deriverStamp: stamp,
+        sourceStat,
+        doc: rendered.doc,
+        derivedAt: rendered.derivedAt,
+    });
     return rendered.kind === "fresh"
         ? { kind: "fresh", relPath, format, sidecarPath: written.path, body: written.body, tokens: written.tokens }
         : { kind: "derived", relPath, format, sidecarPath: written.path, body: written.body, doc: rendered.doc, tokens: written.tokens };

package/src/lib/derivers/pdf.ts

@@ -18,11 +18,24 @@ import { stripRunningFurniture } from "./pdf-furniture.js";
 // Below this many chars per page, the text layer is furniture, not content: a scan with a vestigial layer.
 const SCAN_THRESHOLD_CHARS_PER_PAGE = 24;
 
-// OCR costs seconds per page, so a long scan gets its first pages and a note.
-const MAX_OCR_PAGES = 20;
+// OCR costs seconds per page, so a long scan gets its first pages and a note. Someone reading now gets 20; the daemon's
+// shadow keeper, working in the background with nobody waiting, asks for more (`fileq derive --ocr-pages 500`, which
+// sets FILEQ_OCR_PAGES for the run).
+const DEFAULT_OCR_PAGES = 20;
+const MAX_OCR_PAGES = 5000;
 const OCR_DPI = "200";
+
+/** How many pages of a scan this run recognises: FILEQ_OCR_PAGES when it is a sane count, else 20. */
+export const ocrPageCap = (): number => {
+    const asked = Number(process.env["FILEQ_OCR_PAGES"]);
+    return Number.isInteger(asked) && asked > 0 ? Math.min(asked, MAX_OCR_PAGES) : DEFAULT_OCR_PAGES;
+};
+
+// The note a capped recognition leaves; ocrCapOf (sidecar.ts) reads the cap back off it.
+const capNote = (cap: number): string => `OCR stops at ${cap} pages: draw the rest with pdftoppm and read them with \`ocr <page.png>...\` if they matter`;
+
 // The reader loads its models once per run and then takes a second or few per page.
-const OCR_TIMEOUT_MS = 60_000 + MAX_OCR_PAGES * 15_000;
+const ocrTimeoutMs = (pages: number): number => 60_000 + pages * 15_000;
 
 // Asked once per PATH: the answer is a fact of the image, and a stamp is read on every derive.
 const checked = new Map<string, boolean>();
@@ -46,18 +59,19 @@ export const ocrAvailable = (): boolean => {
 // could not read leaves its page empty (exit 1) rather than shifting the ones after it. The pages are split on those
 // form feeds, so the cap stays well past what twenty pages of text come to: a cut middle would shift every page after it.
 const readPages = async (images: readonly string[]): Promise<string> => {
-    const result = await runBounded("ocr", images, { timeoutMs: OCR_TIMEOUT_MS, maxOutput: 16 * 1024 * 1024 });
+    const result = await runBounded("ocr", images, { timeoutMs: ocrTimeoutMs(images.length), maxOutput: 64 * 1024 * 1024 });
     if (!result.timedOut && (result.exitCode === 0 || result.exitCode === 1)) {
         return result.stdout.text;
     }
     throw new ProcessError("ocr", images, result);
 };
 
-const recognisePages = async (absPath: string, totalPages: number): Promise<string[]> => {
+const recognisePages = async (absPath: string, totalPages: number, cap: number): Promise<string[]> => {
     const dir = await mkdtemp(join(tmpdir(), "fileq-ocr-"));
+    const last = Math.min(totalPages, cap);
     try {
-        await execBounded("pdftoppm", ["-r", OCR_DPI, "-png", "-f", "1", "-l", String(Math.min(totalPages, MAX_OCR_PAGES)), absPath, join(dir, "page")], {
-            timeoutMs: 120_000,
+        await execBounded("pdftoppm", ["-r", OCR_DPI, "-png", "-f", "1", "-l", String(last), absPath, join(dir, "page")], {
+            timeoutMs: 120_000 + last * 2_000,
         });
         const images = (await readdir(dir)).filter((name) => name.endsWith(".png")).toSorted();
         if (images.length === 0) {
@@ -114,12 +128,13 @@ export const pdfDeriver: Deriver = {
                     notes: [`no usable text layer across ${totalPages} page${plural} (a scan?): OCR is not part of this tier`],
                 };
             }
-            const recognised = await recognisePages(absPath, totalPages);
+            const cap = ocrPageCap();
+            const recognised = await recognisePages(absPath, totalPages, cap);
             const notes = [
                 `scanned: text recognised by OCR (PaddleOCR) on ${recognised.length} of ${totalPages} page${plural} — recognised, not exact; check figures against the page image`,
             ];
-            if (totalPages > MAX_OCR_PAGES) {
-                notes.push(`OCR stops at ${MAX_OCR_PAGES} pages: draw the rest with pdftoppm and read them with \`ocr <page.png>...\` if they matter`);
+            if (totalPages > cap) {
+                notes.push(capNote(cap));
             }
             const { markdown, notes: stripped } = withoutFurniture(recognised);
             return { markdown, title, notes: [...notes, ...stripped] };

package/src/lib/formats.ts

@@ -129,6 +129,38 @@ export const detectFormat = async (absPath: string): Promise<Format | undefined>
  */
 export const claimedFormat = async (absPath: string): Promise<Format | undefined> => (await detectFormat(absPath)) ?? EXTENSION_FORMAT[extname(absPath).toLowerCase()];
 
+// Documents: formats whose text is what a reader searches for, and cheap to derive (no model is asked anything, unless a
+// scanned pdf finds the OCR pack). The daemon's shadow keeper keeps a shadow of every one of these in the workspace, and
+// ripgrep reads them through fileq's preprocessor; pictures, recordings, archives and notebooks are left to be read
+// when someone asks.
+export const DOCUMENT_FORMATS: ReadonlySet<Format> = new Set(["pdf", "docx", "xlsx", "pptx", "odt", "ods", "odp", "rtf", "epub"]);
+
+/** Every extension (lower case, with its dot) a document format is recognised by. */
+export const documentExtensions = (): string[] =>
+    Object.entries(EXTENSION_FORMAT)
+        .filter(([, format]) => DOCUMENT_FORMATS.has(format))
+        .map(([extension]) => extension)
+        .toSorted();
+
+/** The format a path's extension names when it is a document; undefined for anything else. */
+export const documentFormatOf = (path: string): Format | undefined => {
+    const format = EXTENSION_FORMAT[extname(path).toLowerCase()];
+    return format !== undefined && DOCUMENT_FORMATS.has(format) ? format : undefined;
+};
+
+// Both spellings of each extension, since ripgrep's globs are case-sensitive and a scanner names files `SCAN.PDF`.
+const caseGlob = (extension: string): string => `*.{${extension.slice(1)},${extension.slice(1).toUpperCase()}}`;
+
+/**
+ * A ripgrep config (RIPGREP_CONFIG_PATH) routing every document through `pre`, fileq's preprocessor: one flag per
+ * line, as ripgrep reads it. Code and every other file are untouched, since only these globs reach the preprocessor.
+ */
+export const ripgrepConfigLines = (pre: string): string[] => [
+    "# fileq: ripgrep searches the text of documents, cited by page (`[p. 12] `); see `fileq ripgrep-config --help`.",
+    `--pre=${pre}`,
+    ...documentExtensions().map((extension) => `--pre-glob=${caseGlob(extension)}`),
+];
+
 // Formats whose text diff is already the better reading: a textconv over them would replace a line diff with a
 // rendering of the same text.
 const TEXT_UNDERNEATH: ReadonlySet<Format> = new Set(["html"]);

package/src/lib/render/render.ts

@@ -2,6 +2,8 @@ import { createHash } from "node:crypto";
 import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, join } from "node:path";
+import { parseJsonOr } from "@intentic/base/json";
+import { readTextIfExists } from "@intentic/base/fs";
 import { toolOutDir } from "@intentic/agent-cli/env";
 import { STATE_DIR } from "@intentic/constants";
 import { getDocumentProxy } from "unpdf";
@@ -75,13 +77,11 @@ export const defaultOutDir = (absPath: string, workspaceRoot: string | undefined
 };
 
 const readStamp = async (dir: string): Promise<Stamp | undefined> => {
-    try {
-        const parsed: Partial<Stamp> = JSON.parse(await readFile(join(dir, STAMP_FILE), "utf8"));
-        return parsed.sha256 === undefined || parsed.size === undefined || parsed.total === undefined ? undefined : { sha256: parsed.sha256, size: parsed.size, total: parsed.total };
-    } catch {
-        // allow(silent-catch): no earlier render, or a stamp someone edited; either way the answer is to render afresh.
-        return undefined;
-    }
+    // No earlier render, or a stamp someone edited; either way the answer is to render afresh.
+    const text = await readTextIfExists(join(dir, STAMP_FILE));
+    // SAFETY: only this module writes the stamp, and each field is checked for presence before the stamp is believed.
+    const parsed = text === undefined ? undefined : (parseJsonOr(text, undefined) as Partial<Stamp> | null | undefined);
+    return parsed?.sha256 === undefined || parsed.size === undefined || parsed.total === undefined ? undefined : { sha256: parsed.sha256, size: parsed.size, total: parsed.total };
 };
 
 // Removes the images an earlier render of different bytes left, and nothing else in the folder.

package/src/lib/sidecar.test.ts

@@ -1,4 +1,4 @@
-import { isFresh, parseSidecarFront, sidecarBody, sidecarPathFor } from "./sidecar.js";
+import { isFresh, parseSidecarFront, sidecarBody, sidecarPathFor, sourceStatOf, sourceUnchanged } from "./sidecar.js";
 
 const SIDECAR = `---
 source: docs/spec.docx
@@ -24,6 +24,8 @@ describe("sidecar front matter", () => {
             derivedAt: "2026-08-29T10:00:00.000Z",
             // JSON-encoded on the way in, so a colon in a title cannot split the line it rides on.
             title: "Quarterly: the plan",
+            sourceSize: undefined,
+            sourceMtime: undefined,
             notes: ["sheet cut", "showing 200 of 4,000 rows"],
         });
     });
@@ -39,6 +41,8 @@ describe("sidecar front matter", () => {
             deriver: undefined,
             derivedAt: undefined,
             title: undefined,
+            sourceSize: undefined,
+            sourceMtime: undefined,
             notes: [],
         });
         expect(sidecarBody("just text")).toBe("just text");
@@ -58,3 +62,30 @@ describe("freshness", () => {
 test("the shadow of a path is the path, mirrored under the derived tree", () => {
     expect(sidecarPathFor("/work", "docs/spec.docx")).toBe("/work/.intentic/local/cache/derived/docs/spec.docx.md");
 });
+
+describe("source stat (the hash-free test)", () => {
+    const WITH_STAT = SIDECAR.replace("deriver: docx v1\n", "deriver: docx v1\nsource_size: 4096\nsource_mtime: 1791910037.034237706\n");
+
+    test("spelled as GNU stat -c '%s %.9Y' prints it, nanoseconds zero-padded", () => {
+        expect(sourceStatOf({ size: 4096n, mtimeNs: 1_791_910_037_034_237_706n })).toEqual({ size: "4096", mtime: "1791910037.034237706" });
+        expect(sourceStatOf({ size: 0n, mtimeNs: 5_000_000_007n })).toEqual({ size: "0", mtime: "5.000000007" });
+    });
+
+    test("a sidecar carrying it matches only the same size and mtime", () => {
+        const front = parseSidecarFront(WITH_STAT);
+        expect(front.sourceSize).toBe("4096");
+        expect(sourceUnchanged(front, { size: "4096", mtime: "1791910037.034237706" })).toBe(true);
+        expect(sourceUnchanged(front, { size: "4096", mtime: "1791910038.034237706" })).toBe(false);
+        expect(sourceUnchanged(front, { size: "4097", mtime: "1791910037.034237706" })).toBe(false);
+        expect(sourceUnchanged(front, undefined)).toBe(false);
+        // The stat is a hint beside the hash, never part of freshness itself.
+        expect(isFresh(WITH_STAT, "abc123", "docx v1")).toBe(true);
+    });
+
+    test("an older sidecar without it still reads, still counts as fresh by hash, and never as unchanged", () => {
+        const front = parseSidecarFront(SIDECAR);
+        expect(front.sha256).toBe("abc123");
+        expect(isFresh(SIDECAR, "abc123", "docx v1")).toBe(true);
+        expect(sourceUnchanged(front, { size: "4096", mtime: "1791910037.034237706" })).toBe(false);
+    });
+});

package/src/lib/sidecar.ts

@@ -1,11 +1,11 @@
 import { createHash } from "node:crypto";
 import { createReadStream } from "node:fs";
-import { readFile, rm, writeFile } from "node:fs/promises";
+import { readFile, rm, stat, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { estimateTokens } from "@intentic/base/format";
 import { writeFileAtomic } from "@intentic/base/fs";
 import { neutralizeOutsideText } from "@intentic/base/outside-text";
-import { STATE_DIR } from "@intentic/constants";
+import { DERIVED_SHADOWS_DIR } from "@intentic/constants";
 import type { DerivedDoc } from "./derivers/deriver.js";
 
 // One markdown shadow per derivable file, at a predictable mirrored path under .intentic/local/cache/derived (portable,
@@ -14,8 +14,12 @@ import type { DerivedDoc } from "./derivers/deriver.js";
 // deriver stamp both still match, never by mtime.
 // Writes (sidecars and content-cache entries alike) go through exactly one function, since either is read back by a
 // plain `Read` with no untrusted-content wrapper, so neutralization must happen at write time.
+// A sidecar also records the source's size and mtime (`source_size`, `source_mtime`). They are not what freshness
+// means here, and fileq never trusts them over the hash; they let a reader that cannot afford a hash (the ripgrep
+// preprocessor, a POSIX shell script; the daemon's shadow keeper, over thousands of files) tell an untouched source
+// with one `stat`. A sidecar written before they existed simply lacks them, and such a reader takes its slow path.
 
-export const DERIVED_DIR = `${STATE_DIR}/local/cache/derived`;
+export const DERIVED_DIR = DERIVED_SHADOWS_DIR;
 
 export const sidecarPathFor = (workspaceRoot: string, relPath: string): string => join(workspaceRoot, DERIVED_DIR, `${relPath}.md`);
 
@@ -35,11 +39,65 @@ export interface SidecarFront {
     readonly deriver: string | undefined;
     readonly derivedAt: string | undefined;
     readonly title: string | undefined;
+    /** The source's byte size and mtime when this was written (sourceStatOf's spelling); absent on older sidecars. */
+    readonly sourceSize: string | undefined;
+    readonly sourceMtime: string | undefined;
     /** Every cap and degradation the derivation hit, in the order it wrote them. */
     readonly notes: readonly string[];
 }
 
-const EMPTY_FRONT: SidecarFront = { source: undefined, sha256: undefined, deriver: undefined, derivedAt: undefined, title: undefined, notes: [] };
+const EMPTY_FRONT: SidecarFront = {
+    source: undefined,
+    sha256: undefined,
+    deriver: undefined,
+    derivedAt: undefined,
+    title: undefined,
+    sourceSize: undefined,
+    sourceMtime: undefined,
+    notes: [],
+};
+
+/** A source's size and mtime as a sidecar records them: what GNU `stat -c '%s %.9Y'` prints, so a shell can compare. */
+export interface SourceStat {
+    readonly size: string;
+    readonly mtime: string;
+}
+
+export const sourceStatOf = (stats: { readonly size: bigint; readonly mtimeNs: bigint }): SourceStat => {
+    const seconds = stats.mtimeNs / 1_000_000_000n;
+    const nanos = stats.mtimeNs % 1_000_000_000n;
+    return { size: stats.size.toString(), mtime: `${seconds.toString()}.${nanos.toString().padStart(9, "0")}` };
+};
+
+/** The file's SourceStat now; undefined when it cannot be stat'd. */
+export const readSourceStat = async (absPath: string): Promise<SourceStat | undefined> => {
+    try {
+        return sourceStatOf(await stat(absPath, { bigint: true }));
+    } catch {
+        // allow(silent-catch): no stat means no hash-free answer; every caller then takes the path that hashes.
+        return undefined;
+    }
+};
+
+// pdf.ts's note for a scan whose OCR stopped at a page cap (interactive reads recognise 20 pages, the daemon's
+// background keeper more). Read here, beside the front matter, so a caller that never loads the pdf reader (the
+// daemon) can tell a shadow cut shorter than it wants.
+const CAP_NOTE = /^OCR stops at (\d+) pages/;
+
+/** The page cap a rendering's OCR stopped at, off its notes; undefined when it recognised every page (or none). */
+export const ocrCapOf = (notes: readonly string[]): number | undefined => {
+    for (const note of notes) {
+        const match = CAP_NOTE.exec(note);
+        if (match !== null) {
+            return Number(match[1]);
+        }
+    }
+    return undefined;
+};
+
+/** Whether a sidecar's recorded size and mtime are this source's: the hash-free test, false when either is missing. */
+export const sourceUnchanged = (front: SidecarFront, now: SourceStat | undefined): boolean =>
+    now !== undefined && front.sourceSize === now.size && front.sourceMtime === now.mtime;
 
 // A line scan rather than a YAML parse, so an edited-by-hand sidecar simply reads as stale instead of throwing.
 // Title and notes are JSON-encoded on the way in (writeSidecar), so they decode back rather than splitting on a colon.
@@ -67,6 +125,8 @@ export const parseSidecarFront = (content: string): SidecarFront => {
         deriver: field("deriver"),
         derivedAt: field("derived_at"),
         title: decode(field("title")),
+        sourceSize: field("source_size"),
+        sourceMtime: field("source_mtime"),
         notes: [...head.matchAll(/^note: (.+)$/gm)].map((match) => decode(match[1]) ?? ""),
     };
 };
@@ -103,6 +163,8 @@ export interface DerivedEntry {
     readonly deriverStamp: string;
     readonly doc: DerivedDoc;
     readonly derivedAt: Date;
+    /** The source's stat when it was read, for a sidecar only (see SourceStat). */
+    readonly sourceStat?: SourceStat | undefined;
 }
 
 export interface WriteSidecarInput extends Omit<DerivedEntry, "source"> {
@@ -123,6 +185,7 @@ export const writeDerived = async (path: string, entry: DerivedEntry): Promise<{
         "---",
         ...(entry.source === undefined ? [] : [`source: ${entry.source}`]),
         `sha256: ${entry.sourceSha}`,
+        ...(entry.sourceStat === undefined ? [] : [`source_size: ${entry.sourceStat.size}`, `source_mtime: ${entry.sourceStat.mtime}`]),
         `deriver: ${entry.deriverStamp}`,
         `derived_at: ${entry.derivedAt.toISOString()}`,
         ...(title === undefined ? [] : [`title: ${JSON.stringify(title)}`]),

package/src/rg-preprocessor.integration.test.ts

@@ -0,0 +1,139 @@
+// The ripgrep preprocessor (bin/fileq-rg-pre) against a temp workspace: a fresh shadow is printed with page marks
+// and no fileq run, a stale or pre-stat shadow falls back to `fileq read --plain`, a non-document passes through, and
+// plain `rg` under the generated config cites documents by page while code reads exactly as without it.
+import { spawnSync } from "node:child_process";
+import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { markPages } from "@intentic/base/shadow-text";
+import { captureCli } from "@intentic/agent-cli/testing";
+import { app } from "./app.js";
+import { documentExtensions, ripgrepConfigLines } from "./lib/formats.js";
+import { DERIVED_DIR, sidecarBody } from "./lib/sidecar.js";
+import { pdfPagesBytes } from "./testing.js";
+
+const PRE = fileURLToPath(new URL("../bin/fileq-rg-pre", import.meta.url));
+const hasRg = spawnSync("rg", ["--version"]).status === 0;
+
+let root: string;
+let bin: string;
+let calls: string;
+
+const page = (text: string) => ({ content: `BT /F1 12 Tf 72 700 Td (${text}) Tj ET` });
+
+beforeAll(() => {
+    root = mkdtempSync(join(tmpdir(), "fileq-rgpre-"));
+    bin = mkdtempSync(join(tmpdir(), "fileq-rgpre-bin-"));
+    calls = join(bin, "calls");
+    // A stand-in fileq on PATH: records that the slow path ran, and answers as `read --plain` would.
+    writeFileSync(join(bin, "fileq"), `#!/bin/sh\necho "$@" >> '${calls}'\nprintf '## Page 1\\n\\nslow first\\n\\n## Page 2\\n\\nslow second\\n'\n`);
+    chmodSync(join(bin, "fileq"), 0o755);
+    process.env["WORKSPACE_ROOT"] = root;
+    mkdirSync(join(root, "docs"), { recursive: true });
+    writeFileSync(join(root, "docs/terms.pdf"), pdfPagesBytes([page("Introduction to the terms of service here"), page("Each wniosek must be filed in writing today")]));
+});
+afterAll(() => {
+    delete process.env["WORKSPACE_ROOT"];
+    rmSync(root, { recursive: true, force: true });
+    rmSync(bin, { recursive: true, force: true });
+});
+
+const pre = (file: string, cwd = root) => {
+    rmSync(calls, { force: true });
+    const result = spawnSync("sh", [PRE, file], { cwd, encoding: "utf8", env: { PATH: `${bin}:${process.env["PATH"] ?? ""}`, WORKSPACE_ROOT: root } });
+    expect(result.status).toBe(0);
+    return { out: result.stdout, slow: existsSync(calls) };
+};
+
+const shadowOf = (relPath: string): string => join(root, DERIVED_DIR, `${relPath}.md`);
+
+describe("fast path: a shadow whose recorded stat is the file's", () => {
+    test("prints the body with page marks, without running fileq", async () => {
+        expect((await captureCli(app, ["derive", "docs/terms.pdf"])).exitCode).toBe(0);
+        const shadow = readFileSync(shadowOf("docs/terms.pdf"), "utf8");
+        expect(shadow).toMatch(/^source_size: \d+$/m);
+        expect(shadow).toMatch(/^source_mtime: \d+\.\d{9}$/m);
+        const { out, slow } = pre("./docs/terms.pdf");
+        expect(slow).toBe(false);
+        // The shell's marks are exactly the TypeScript ones iq indexes with.
+        expect(out).toBe(markPages(sidecarBody(shadow)));
+        expect(out).toContain("[p. 2] Each wniosek must be filed in writing today");
+        expect(out.split("\n")[0]).toBe("[p. 1] ## Page 1");
+    });
+
+    test("an absolute path and a path from a subdirectory find the same shadow", () => {
+        expect(pre(join(root, "docs/terms.pdf")).slow).toBe(false);
+        expect(pre("terms.pdf", join(root, "docs")).slow).toBe(false);
+        expect(pre("../docs/terms.pdf", join(root, "docs")).slow).toBe(false);
+    });
+});
+
+describe("slow path: fileq read --plain, marked the same way", () => {
+    test("a file touched since its shadow was written", () => {
+        const later = new Date(Date.now() + 60_000);
+        utimesSync(join(root, "docs/terms.pdf"), later, later);
+        const { out, slow } = pre("docs/terms.pdf");
+        expect(slow).toBe(true);
+        expect(readFileSync(calls, "utf8")).toContain(`read --plain ${join(root, "docs/terms.pdf")}`);
+        expect(out).toBe("[p. 1] ## Page 1\n[p. 1] \n[p. 1] slow first\n[p. 1] \n[p. 2] ## Page 2\n[p. 2] \n[p. 2] slow second\n");
+    });
+
+    test("fileq rewrites the stat of an unchanged file, and the next run is fast again", async () => {
+        expect((await captureCli(app, ["derive", "docs/terms.pdf"])).out).toContain("fresh docs/terms.pdf");
+        expect(pre("docs/terms.pdf").slow).toBe(false);
+    });
+
+    test("a sidecar written before stats were recorded still reads, through the slow path", () => {
+        const path = shadowOf("docs/terms.pdf");
+        const old = readFileSync(path, "utf8").replaceAll(/^source_(size|mtime): .*\n/gm, "");
+        writeFileSync(path, old);
+        expect(pre("docs/terms.pdf").slow).toBe(true);
+    });
+
+    test("a document with no shadow at all", () => {
+        writeFileSync(join(root, "docs/fresh.pdf"), pdfPagesBytes([page("No shadow yet for this one")]));
+        expect(pre("docs/fresh.pdf").slow).toBe(true);
+    });
+});
+
+test("a file that is not a document passes through untouched", () => {
+    writeFileSync(join(root, "notes.ts"), "export const wniosek = 1;\n");
+    const { out, slow } = pre("notes.ts");
+    expect(out).toBe("export const wniosek = 1;\n");
+    expect(slow).toBe(false);
+});
+
+test("the script's extension list is fileq's document list", () => {
+    const script = readFileSync(PRE, "utf8");
+    for (const extension of documentExtensions()) {
+        const bracketed = Array.from(extension.slice(1), (char) => `[${char}${char.toUpperCase()}]`).join("");
+        expect(script, extension).toContain(`*.${bracketed}`);
+    }
+});
+
+describe.skipIf(!hasRg)("plain rg under the generated config", () => {
+    const rg = (args: string[], config: boolean): string => {
+        const configPath = join(bin, "ripgreprc");
+        // rg execs `--pre` directly, and git records bin/ scripts without the executable bit (the image's Dockerfile
+        // adds it), so a fresh checkout's copy cannot start: an executable copy stands in for the image's.
+        const executable = join(bin, "fileq-rg-pre");
+        copyFileSync(PRE, executable);
+        chmodSync(executable, 0o755);
+        writeFileSync(configPath, `${ripgrepConfigLines(executable).join("\n")}\n`);
+        // ripgrep reads an empty RIPGREP_CONFIG_PATH as unset.
+        const env = { PATH: process.env["PATH"] ?? "", WORKSPACE_ROOT: root, RIPGREP_CONFIG_PATH: config ? configPath : "" };
+        return spawnSync("rg", args, { cwd: root, encoding: "utf8", env }).stdout;
+    };
+
+    test("a document hit is cited by path, line and page", async () => {
+        await captureCli(app, ["derive", "docs/terms.pdf"]);
+        expect(rg(["-n", "wniosek", "docs"], true)).toMatch(/^docs\/terms\.pdf:\d+:\[p\. 2\] Each wniosek must be filed/m);
+    });
+
+    test("code reads exactly as it does without the config", () => {
+        expect(rg(["-n", "wniosek", "notes.ts"], true)).toBe(rg(["-n", "wniosek", "notes.ts"], false));
+        // Without it, rg sees the file's raw bytes: this fixture's stream happens to be uncompressed, and still no page.
+        expect(rg(["-n", "Each wniosek must", "docs"], false)).not.toContain("[p. ");
+    });
+});