@stratta/mcp 0.10.0 → 0.12.0

package/scripts/ingest-prepass.py

@@ -2,10 +2,13 @@
 """Stratta — ingest pre-pass.
 
 Reads a norm PDF with PyMuPDF, builds the hierarchical TreeRAG skeleton (chapters
-from PDF bookmarks + sub-sections via heading regex), extracts per-section raw
-text, and rasterizes each page that contains a figure caption. Output is a
-single JSON consumed by the `ingest-norm` skill, which then enriches sections
-(formulas, tables, cross-refs, summaries) and uploads the figures.
+from PDF bookmarks + sub-sections via heading regex, down to X.Y.Z.W), gives
+each node the text between its heading and the next one on cleaned pages (no
+running headers, no page numbers, hyphenation resolved, clause numbers joined
+to their paragraph), and rasterizes each page that contains a figure caption.
+Output is a single JSON consumed by the `ingest-norm` skill, which then
+enriches sections (formulas, tables, cross-refs, summaries) and uploads the
+figures. `warnings[]` lists what the pre-pass could not decide on its own.
 
 Usage:
     python scripts/ingest-prepass.py --pdf <path> --output <dir> [--language fr]
@@ -54,23 +57,137 @@ def clean(s: str) -> str:
     return re.sub(r"\s+", " ", CLEAN_DOTS.sub("", s)).strip()
 
 
+# --- page lines ----------------------------------------------------------------
+#
+# PyMuPDF's plain text follows the PDF's own block order. On a SIA norm the
+# clause numbers sit in a column of their own, and on an OCR'd copy (the form
+# a bureau actually holds: 60 of the 80 PDFs of the first client corpus are
+# scans) that column is read as one block BEFORE the paragraphs, so a page
+# arrives as "4.4 / 4.4.1 / 4.4.1.1 / ... / Résistances du terrain /
+# Généralités / Sont considérés ..." and no heading regex can see a heading.
+#
+# Lines are therefore rebuilt from the words and their positions: words on the
+# same baseline form a line, left to right. That puts "4.4 Résistances du
+# terrain" back together on both the native and the OCR'd copy. A genuine
+# two-column page (two text columns, not a numbering column) would be
+# scrambled by that rule, so such pages are detected and read in block order.
+
+_LINES_CACHE: dict[tuple[int, int], list[str]] = {}
+NUMBER_TOKEN_RE = re.compile(r"^(?:\d{1,2}(?:\.\d{1,3}){0,4}\.?|[A-Z](?:\.\d{1,2}){1,3}|—|-|•)$")
+OCR_DOTTED_NUMBER_RE = re.compile(r"^(\d+(?:\.\d+)*) \.(\d)")
+
+
+def _rows(page: "fitz.Page") -> list[list[tuple]]:
+    words = page.get_text("words")  # x0, y0, x1, y1, text, block, line, word
+    if not words:
+        return []
+    heights = sorted(w[3] - w[1] for w in words)
+    median = heights[len(heights) // 2]
+    # A word set vertically (a library watermark such as "Ecole Polytechnique
+    # Fédérale de Lausanne" running up the margin) has a box far taller than
+    # the text. Left in, its words land on every line they cross.
+    words = [w for w in words if (w[3] - w[1]) <= 2.5 * median]
+    if not words:
+        return []
+    tol = max(2.0, 0.45 * median)
+    rows: list[list[tuple]] = []
+    for w in sorted(words, key=lambda w: (w[1], w[0])):
+        if rows and abs(rows[-1][0][1] - w[1]) <= tol:
+            rows[-1].append(w)
+        else:
+            rows.append([w])
+    for row in rows:
+        row.sort(key=lambda w: w[0])
+    return rows
+
+
+def _segments(row: list[tuple], gap: float) -> list[list[tuple]]:
+    """Split a row where the horizontal gap between words is a column gutter."""
+    out: list[list[tuple]] = [[row[0]]]
+    for prev, w in zip(row, row[1:]):
+        if w[0] - prev[2] > gap:
+            out.append([w])
+        else:
+            out[-1].append(w)
+    return out
+
+
+def _two_column(rows: list[list[list[tuple]]]) -> bool:
+    """Two text columns: many rows carry two segments that both read as prose."""
+    if len(rows) < 12:
+        return False
+    both = 0
+    for segs in rows:
+        texts = [" ".join(w[4] for w in s) for s in segs]
+        prose = [t for t in texts if len(t) > 24 and not NUMBER_TOKEN_RE.match(t.split()[0])]
+        if len(prose) >= 2:
+            both += 1
+    return both / len(rows) > 0.3
+
+
+def page_lines(doc: "fitz.Document", pageno: int) -> list[str]:
+    """The lines of a page, in reading order, numbers joined to their text."""
+    key = (id(doc), pageno)
+    cached = _LINES_CACHE.get(key)
+    if cached is not None:
+        return cached
+    page = doc[pageno]
+    rows = _rows(page)
+    gap = 0.12 * page.rect.width
+    segmented = [_segments(row, gap) for row in rows]
+    if _two_column(segmented):
+        lines = join_split_headings(page.get_text("text")).splitlines()
+    else:
+        lines = []
+        for segs in segmented:
+            text = " ".join(" ".join(w[4] for w in s) for s in segs)
+            text = OCR_DOTTED_NUMBER_RE.sub(r"\1.\2", text)
+            lines.append(text)
+    lines = [l.rstrip() for l in lines]
+    _LINES_CACHE[key] = lines
+    return lines
+
+
+def page_text(doc: "fitz.Document", pageno: int) -> str:
+    return "\n".join(page_lines(doc, pageno))
+
+
 def detect_toc_pages(doc: "fitz.Document") -> set[int]:
     out = set()
     for i in range(doc.page_count):
-        lines = [l for l in doc[i].get_text("text").splitlines() if l.strip()]
+        lines = [l for l in page_lines(doc, i) if l.strip()]
         leader = sum(1 for l in lines if TOC_DOT_RE.search(l))
         if leader >= 5:
             out.add(i)
     return out
 
 
+# Standalone integers only: the "9" and "1" of a heading "9.1 Délimitation"
+# stay, or every "Généralités" heading would read as one running line.
+FURNITURE_NUMBER_RE = re.compile(r"(?<![\d.])\d{1,4}(?![\d.])")
+FURNITURE_EDGE_RE = re.compile(r"^[\s/|\-–—·.]+|[\s/|\-–—·.]+$")
+
+
+def furniture_key(line: str) -> str:
+    """A running line without the page number it carries.
+
+    `SIA 267, Copyright © 2013 by SIA Zurich 59` on an odd page and
+    `64 SIA 267, Copyright © 2013 by SIA Zurich` on an even one are the same
+    footer; so are the copies a watermark decorates with a stray slash.
+    """
+    s = FURNITURE_NUMBER_RE.sub(" ", line)
+    s = re.sub(r"\s+", " ", s).strip()
+    return FURNITURE_EDGE_RE.sub("", s).strip()
+
+
 def detect_running_text(doc: "fitz.Document") -> set[str]:
-    """Lines appearing on >=5 pages near the top/bottom (running headers/footers)."""
+    """Lines appearing on >=5 pages near the top/bottom (running headers/footers),
+    keyed without their page number."""
     counter: dict[str, int] = defaultdict(int)
     for i in range(doc.page_count):
-        lines = [l.strip() for l in doc[i].get_text("text").splitlines() if l.strip()]
+        lines = [l.strip() for l in page_lines(doc, i) if l.strip()]
         for l in lines[:3] + lines[-3:]:
-            counter[l] += 1
+            counter[furniture_key(l)] += 1
     return {l for l, c in counter.items() if c >= 5 and len(l) > 8}
 
 
@@ -124,8 +241,8 @@ def join_split_headings(text: str) -> str:
 
 
 def heading_text(doc: "fitz.Document", pageno: int) -> str:
-    """Page text prepared for heading detection (never for section content)."""
-    return join_split_headings(doc[pageno].get_text("text"))
+    """Page text prepared for heading detection."""
+    return join_split_headings(page_text(doc, pageno))
 
 
 NOT_A_TITLE_RE = re.compile(
@@ -212,12 +329,14 @@ def prune_chapters(chapters: dict[str, dict]) -> dict[str, dict]:
         idx = prev[idx]
 
     # Missing a heading or two leaves a small gap; doubling (10 → 20 → 30) means
-    # the numbers stopped being chapters and started being table rows.
+    # the numbers stopped being chapters and started being table rows. Only
+    # from 5 upwards: an OCR'd copy that lost chapters 3 to 6 jumps from 2 to
+    # 7, and that is a hole, not a doubling.
     kept: dict[str, dict] = {}
     previous = None
     for key in reversed(chain):
         n = int(key)
-        if previous is not None and n - previous > 3 and n >= 2 * previous:
+        if previous is not None and previous >= 5 and n - previous > 3 and n >= 2 * previous:
             break
         kept[key] = chapters[key]
         previous = n
@@ -301,7 +420,106 @@ def extract_chapters(
         if len([k for k in chapters if k.isdigit()]) >= 3:
             break
 
-    return prune_chapters(chapters)
+    for num, meta in infer_unnumbered_chapters(doc, skip).items():
+        chapters.setdefault(num, meta)
+    chapters = prune_chapters(chapters)
+    return recover_missing_chapters(doc, chapters, skip)
+
+
+SCOPE_TITLE_RE = re.compile(
+    r"^(DOMAINE D.APPLICATION|GELTUNGSBEREICH|CAMPO D.APPLICAZIONE|SCOPE)\b", re.I
+)
+
+
+def _page_top_titles(doc: "fitz.Document", pages: range, skip: set[int]) -> list[tuple[int, str]]:
+    """Uppercase, plausible titles among the first lines of each page: where a
+    chapter of a SIA norm starts. Deeper in a page a capitalised line is a
+    table header or a note."""
+    out: list[tuple[int, str]] = []
+    for pageno in pages:
+        if pageno in skip:
+            continue
+        for line in [l.strip() for l in page_lines(doc, pageno)][:3]:
+            m = UNNUMBERED_UPPER_RE.match(line)
+            if m and plausible_title(m.group(1)) and not re.match(r"^(TABLEAU|TABELLE|TABELLA|TABLE|FIGURE|FIG\.|BILD|ANNEXE|ANHANG|ANNEX)\b", line):
+                out.append((pageno + 1, clean(m.group(1))))
+                break
+    return out
+
+
+def recover_missing_chapters(doc: "fitz.Document", chapters: dict[str, dict], skip: set[int]) -> dict[str, dict]:
+    """Fill holes in the chapter sequence by position.
+
+    On the OCR'd copies the numbering column is sometimes lost on a whole run
+    of pages: the chapter title survives, and so does the first sub-heading,
+    but neither carries its number, so `infer_unnumbered_chapters` has nothing
+    to read. When the sequence goes 1, 2, 6 and exactly three title-only pages
+    lie between chapter 2 and chapter 6, those are chapters 3, 4 and 5, in
+    page order. Anything less certain is left as a hole for the agent.
+
+    A leading "Domaine d'application" before chapter 1 is chapter 0, which is
+    how SIA numbers it.
+    """
+    numbered = sorted((int(k), k) for k in chapters if k.isdigit())
+    if not numbered:
+        return chapters
+    used = {meta["title"].upper() for meta in chapters.values()}
+    recovered: dict[str, dict] = {}
+
+    for (a, ka), (b, kb) in zip(numbered, numbered[1:]):
+        holes = list(range(a + 1, b))
+        if not holes:
+            continue
+        start, end = chapters[ka]["pageStart"], chapters[kb]["pageStart"]
+        candidates = [
+            (page, title)
+            for page, title in _page_top_titles(doc, range(start, end - 1), skip)
+            if title.upper() not in used
+        ]
+        if len(candidates) != len(holes):
+            continue
+        for n, (page, title) in zip(holes, candidates):
+            recovered[str(n)] = {"title": title, "pageStart": page, "recovered": True}
+
+    first_n, first_k = numbered[0]
+    if first_n == 1 and "0" not in chapters:
+        before = _page_top_titles(doc, range(0, chapters[first_k]["pageStart"] - 1), skip)
+        scope = [(p, t) for p, t in before if SCOPE_TITLE_RE.match(t)]
+        if len(scope) == 1:
+            recovered["0"] = {"title": scope[0][1], "pageStart": scope[0][0], "recovered": True}
+
+    return {**chapters, **recovered}
+
+
+UNNUMBERED_UPPER_RE = re.compile(r"^([A-ZÉÈÀÂÔÎÛÇ][^a-z\n]{5,120})$")
+FIRST_SUB_RE = re.compile(r"^(\d{1,2})\.\d{1,2}\b")
+
+
+def infer_unnumbered_chapters(doc: "fitz.Document", skip: set[int]) -> dict[str, dict]:
+    """Chapters whose number the OCR lost.
+
+    On a scanned copy the large chapter number is often the one glyph the OCR
+    does not recognise, so the page reads "FONDATIONS SUR PIEUX" followed by
+    "9.1 Délimitation". The number is then taken from the first sub-heading
+    under the title. Only fills holes: a chapter found with its number wins.
+    """
+    found: dict[str, dict] = {}
+    for pageno in range(doc.page_count):
+        if pageno in skip:
+            continue
+        lines = [l.strip() for l in page_lines(doc, pageno)]
+        for i, line in enumerate(lines):
+            m = UNNUMBERED_UPPER_RE.match(line)
+            if not m or not plausible_title(m.group(1)):
+                continue
+            for follow in lines[i + 1 : i + 9]:
+                sub = FIRST_SUB_RE.match(follow)
+                if sub:
+                    num = sub.group(1)
+                    if num not in found:
+                        found[num] = {"title": clean(m.group(1)), "pageStart": pageno + 1}
+                    break
+    return found
 
 
 def chapter_page_spans(chapters: dict[str, dict], n_pages: int) -> dict[str, tuple[int, int]]:
@@ -340,6 +558,11 @@ def extract_subsections(
             top = num.split(".", 1)[0]
             if top not in chap_prefixes:
                 continue
+            # "9.522" is "9.5.2.2" with a dot the OCR dropped. No norm has a
+            # fortieth sub-section; keeping it would hang a page of 9.5 under
+            # a sibling of 9.7 and knock the real 9.6 and 9.7 out of sequence.
+            if any(int(part) > 40 for part in num.split(".")[1:]):
+                continue
             # A sub-section sits inside its chapter. "1.1" found sixty pages
             # after chapter 1 ended is a numbered line in an annexe, and keeping
             # it hangs an unrelated page under the wrong parent.
@@ -403,51 +626,241 @@ def slugify(path: str) -> str:
     return re.sub(r"[^a-zA-Z0-9]+", "-", path).strip("-").lower() or "root"
 
 
-def extract_section_text(doc: "fitz.Document", nodes: list[dict]) -> None:
-    """Attach rawText to each node = concat of pages it spans."""
-    page_texts = [doc[i].get_text("text") for i in range(doc.page_count)]
-    for node in nodes:
-        ps, pe = node["pageStart"], node["pageEnd"]
-        chunks = page_texts[ps - 1 : pe]
-        node["rawText"] = "\n".join(chunks).strip()
+# --- section text ------------------------------------------------------------
+#
+# Until 2026-09-04 a node's text was the concatenation of the pages it spanned,
+# which meant three things the audit measured on the production corpus
+# (ADR 22 § 2.1): a chapter carried the full text of all its sub-sections
+# (61 000 characters twice for SIA 267.153 § 11 and § 11.3), the running header
+# and the page number of every page sat in the middle of the prose, and words
+# hyphenated at a line end stayed broken ("ter- rain"). The text of a node is
+# now what lies between its heading and the next heading, on cleaned pages.
 
-    trim_shared_pages(nodes, page_texts)
+PAGE_NUMBER_RE = re.compile(r"^\s*\d{1,4}\s*$")
+CLAUSE_NUMBER_RE = re.compile(r"^\s*(\d{1,2}(?:\.\d{1,3}){1,4})\s*$")
+HYPHEN_BREAK_RE = re.compile(r"([a-zà-ÿ])[-—]\n([a-zà-ÿ])")
 
 
-def trim_shared_pages(nodes: list[dict], page_texts: list[str]) -> None:
-    """Cut the opening page where several sections start on it.
+def dehyphenate(text: str) -> str:
+    """Join a word broken at a line end: `tra-\\nvaux` becomes `travaux`.
 
-    Page granularity is the unit everywhere else, but two sections beginning on
-    the same page would otherwise carry byte-identical text — so a summary reads
-    like its neighbour's, and the table of contents misleads before anything is
-    even opened.
+    A compound word split at its hyphen loses it too (`pieux-\\nradier` reads
+    `pieuxradier`); syllable breaks outnumber compounds at line ends by far,
+    and a broken word is worse for search than a missing hyphen.
     """
-    by_page: dict[int, list[dict]] = defaultdict(list)
-    for node in nodes:
-        by_page[node["pageStart"]].append(node)
+    return HYPHEN_BREAK_RE.sub(r"\1\2", text)
+
+
+def clean_page(text: str, running: set[str], page_count: int) -> str:
+    """Drop what is page furniture rather than norm text.
 
-    for page, group in by_page.items():
-        if len(group) < 2:
+    - the running header and footer lines `detect_running_text` found (they
+      were detected before, and never removed);
+    - a bare page number in the three first or three last lines of the page;
+    - trailing whitespace on every line.
+    """
+    lines = [l.rstrip() for l in text.splitlines()]
+    kept: list[str] = []
+    n = len(lines)
+    for i, line in enumerate(lines):
+        s = line.strip()
+        if s and (s in running or furniture_key(s) in running):
             continue
-        text = page_texts[page - 1]
-        # Where each section's heading sits on that page, in reading order.
-        marks: list[tuple[int, dict]] = []
-        for node in group:
-            title = node["title"].split(" — ")[-1].strip()
-            at = text.find(title) if len(title) >= 4 else -1
-            if at == -1:
-                at = text.find(f"{node['path']} ")
-            marks.append((at, node))
-        if any(at == -1 for at, _ in marks):
-            continue  # a heading we cannot locate: leave the whole page in place
-        marks.sort(key=lambda m: m[0])
-        for i, (at, node) in enumerate(marks):
-            end = marks[i + 1][0] if i + 1 < len(marks) else len(text)
-            head = text[at:end].strip()
-            if not head:
-                continue
-            rest = "\n".join(page_texts[node["pageStart"] : node["pageEnd"]]).strip()
-            node["rawText"] = f"{head}\n{rest}".strip() if rest else head
+        # Printed page numbers run a little past the PDF's page count when
+        # the front matter is numbered separately; well past it, a bare
+        # number in the margin is a table value.
+        near_edge = i < 3 or i >= n - 3
+        if near_edge and PAGE_NUMBER_RE.match(s) and int(s) <= page_count + 20:
+            continue
+        kept.append(line)
+    return "\n".join(kept)
+
+
+def join_clause_numbers(text: str) -> str:
+    """Rewrite `9.5.1.1\\nLes pieux…` as `9.5.1.1 Les pieux…`.
+
+    PyMuPDF reads the numbering column before the paragraph, so every numbered
+    clause of a SIA norm arrives as a number on its own line. Joined, the
+    number becomes an anchor the reader can search for; alone, it is noise.
+    Content only: heading detection keeps its stricter rule
+    (`join_split_headings`), because a clause number followed by a sentence is
+    exactly what must NOT become a heading.
+    """
+    lines = text.splitlines()
+    out: list[str] = []
+    i = 0
+    while i < len(lines):
+        m = CLAUSE_NUMBER_RE.match(lines[i])
+        if m and i + 1 < len(lines) and lines[i + 1].strip():
+            out.append(f"{m.group(1)} {lines[i + 1].strip()}")
+            i += 2
+            continue
+        out.append(lines[i])
+        i += 1
+    return "\n".join(out)
+
+
+def tidy(text: str) -> str:
+    """Collapse runs of blank lines; strip the ends."""
+    return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def locate_heading(lines: list[str], node: dict) -> int:
+    """Index of the line carrying this node's heading on its start page, or -1.
+
+    Matches on the number first (`9.5 ` at the start of a line), then on the
+    first words of the title, so a title PyMuPDF wrapped differently from the
+    contents page is still found.
+    """
+    path = node["path"]
+    title = node["title"].split(" — ")[-1].strip()
+    stem = re.sub(r"\s+", " ", title[:24]).lower()
+    if path.startswith("Annexe"):
+        needle = path.lower()
+        for i, line in enumerate(lines):
+            if line.strip().lower().startswith(needle):
+                return i
+        return -1
+    prefix = path + " "
+    for i, line in enumerate(lines):
+        s = line.strip()
+        if not s.startswith(prefix):
+            continue
+        rest = re.sub(r"\s+", " ", s[len(prefix):]).lower()
+        if not stem or rest.startswith(stem[: min(len(stem), 12)]):
+            return i
+    for i, line in enumerate(lines):
+        if stem and len(stem) >= 8 and re.sub(r"\s+", " ", line.strip()).lower().startswith(stem):
+            return i
+    return -1
+
+
+def extract_section_text(
+    doc: "fitz.Document", nodes: list[dict], running: set[str]
+) -> list[str]:
+    """Give each node the text between its heading and the next one.
+
+    Pages are cleaned first (`clean_page`), then read as one stream. Every
+    node's heading is located on its start page; the node's text runs from the
+    line after its heading to the next located heading, in document order,
+    whatever the depth. A heading that cannot be located falls back to the
+    start of its page, which is the previous behaviour for that node only, and
+    is reported in the warnings so the skill can look at it.
+
+    Returns the warnings.
+    """
+    warnings: list[str] = []
+    pages = [
+        clean_page(join_split_headings(page_text(doc, i)), running, doc.page_count)
+        for i in range(doc.page_count)
+    ]
+    lines_per_page = [p.split("\n") for p in pages]
+
+    # Absolute line index of the first line of each page in the stream.
+    page_offsets: list[int] = []
+    total = 0
+    for lines in lines_per_page:
+        page_offsets.append(total)
+        total += len(lines)
+    stream = [line for lines in lines_per_page for line in lines]
+
+    located: list[tuple[int, int, dict]] = []  # (start_line, body_line, node)
+    for node in nodes:
+        pageno = node["pageStart"] - 1
+        at = locate_heading(lines_per_page[pageno], node)
+        if at == -1:
+            warnings.append(f"heading not located: {node['path']} {node['title'][:40]!r} (p. {node['pageStart']})")
+            start = page_offsets[pageno]
+            located.append((start, start, node))
+        else:
+            start = page_offsets[pageno] + at
+            located.append((start, start + 1, node))
+
+    # Document order, not path order: annexes sort last by path but sit last
+    # in the document anyway; a heading not located sorts at its page start.
+    located.sort(key=lambda t: (t[0], -t[2]["depth"]))
+    for idx, (start, body, node) in enumerate(located):
+        end = located[idx + 1][0] if idx + 1 < len(located) else len(stream)
+        text = "\n".join(stream[body:end])
+        node["rawText"] = tidy(dehyphenate(join_clause_numbers(text)))
+        node["textSource"] = "heading" if body > start else "page-start"
+
+    # A chapter whose text lives in its sections is normal; a leaf with
+    # nothing under its heading is a heading the text extraction lost.
+    parents = {n["parentNodeId"] for n in nodes if n.get("parentNodeId")}
+    empty = [n["path"] for n in nodes if len(n["rawText"]) < 40 and n["nodeId"] not in parents]
+    if empty:
+        warnings.append(f"{len(empty)} leaf node(s) with no text: {', '.join(empty[:8])}")
+    return warnings
+
+
+def missing_chapter_numbers(chapters: dict[str, dict]) -> list[str]:
+    """Holes in the chapter sequence, for the skill to look at the contents page."""
+    numbers = sorted(int(k) for k in chapters if k.isdigit())
+    if len(numbers) < 2:
+        return []
+    return [str(n) for n in range(numbers[0], numbers[-1]) if n not in set(numbers)]
+
+
+def prune_subsections(
+    subsections: dict[str, tuple[str, int]], chapters: dict[str, dict]
+) -> tuple[dict[str, tuple[str, int]], list[str]]:
+    """Keep, under each parent, the children whose numbers advance with the pages.
+
+    The same idea as `prune_chapters`, one level down: a titled "4.2" found on
+    page 63 inside chapter 9 is a table row, and keeping it would hang a page
+    of chapter 9 under chapter 4. Children are checked against their parent's
+    first page and against each other's order.
+    """
+    dropped: list[str] = []
+    by_parent: dict[str, list[str]] = defaultdict(list)
+    for path in subsections:
+        parent = path.rsplit(".", 1)[0]
+        by_parent[parent].append(path)
+
+    kept: dict[str, tuple[str, int]] = {}
+    for parent, children in by_parent.items():
+        parent_page = (
+            chapters.get(parent, {}).get("pageStart")
+            if parent in chapters
+            else (subsections[parent][1] if parent in subsections else None)
+        ) or 0
+        ordered = [
+            p
+            for p in sorted(children, key=lambda p: [int(x) for x in p.split(".")])
+            if subsections[p][1] >= parent_page
+        ]
+        for path in children:
+            if path not in ordered:
+                title, page = subsections[path]
+                dropped.append(f"{path} {title[:30]!r} (p. {page}, before its parent)")
+
+        # Longest run whose pages never go backwards, like `prune_chapters`:
+        # a greedy walk would keep the one stray row and drop every real
+        # child after it.
+        pages = [subsections[p][1] for p in ordered]
+        best = [1] * len(ordered)
+        prev = [-1] * len(ordered)
+        for i in range(len(ordered)):
+            for j in range(i):
+                if pages[j] <= pages[i] and best[j] + 1 > best[i]:
+                    best[i], prev[i] = best[j] + 1, j
+        chain: set[int] = set()
+        if ordered:
+            # On a tie, the run that ends earliest in the document: a stray
+            # table row sits far past the real children.
+            top = max(best)
+            idx = min((i for i in range(len(ordered)) if best[i] == top), key=lambda i: pages[i])
+            while idx != -1:
+                chain.add(idx)
+                idx = prev[idx]
+        for i, path in enumerate(ordered):
+            title, page = subsections[path]
+            if i in chain:
+                kept[path] = (title, page)
+            else:
+                dropped.append(f"{path} {title[:30]!r} (p. {page}, out of sequence)")
+    return kept, dropped
 
 
 def extract_figures(
@@ -464,7 +877,7 @@ def extract_figures(
     for pageno in range(doc.page_count):
         if pageno in toc_pages:
             continue
-        text = doc[pageno].get_text("text")
+        text = page_text(doc, pageno)
         rendered = False
         page_png: Path | None = None
         for m in CAPTION_RE.finditer(text):
@@ -492,7 +905,7 @@ def extract_figures(
 
 def guess_language(doc: "fitz.Document") -> str:
     """Best-effort language guess based on common French/German/Italian markers."""
-    sample = " ".join(doc[i].get_text("text") for i in range(min(5, doc.page_count))).lower()
+    sample = " ".join(page_text(doc, i) for i in range(min(5, doc.page_count))).lower()
     scores = {
         "fr": sum(sample.count(w) for w in (" la ", " les ", " sont ", " avec ", " selon ")),
         "de": sum(sample.count(w) for w in (" der ", " die ", " sind ", " mit ", " nach ")),
@@ -528,8 +941,22 @@ def main() -> None:
     chap_prefixes = {k for k in chapters if k.isdigit()}
     spans = chapter_page_spans(chapters, doc.page_count)
     subsections = extract_subsections(doc, chap_prefixes, toc_pages, running, spans)
+    subsections, pruned = prune_subsections(subsections, chapters)
     nodes = build_tree(chapters, subsections, doc.page_count)
-    extract_section_text(doc, nodes)
+    warnings = [f"sub-section dropped, out of sequence: {p}" for p in pruned]
+    holes = missing_chapter_numbers(chapters)
+    if holes:
+        warnings.append(
+            f"chapters missing from the sequence: {', '.join(holes)} (check the contents page; an OCR'd copy often loses the chapter number)"
+        )
+    recovered = sorted(
+        (int(k) for k, meta in chapters.items() if k.isdigit() and meta.get("recovered"))
+    )
+    if recovered:
+        warnings.append(
+            f"chapter numbers recovered by position: {', '.join(map(str, recovered))} (their sub-headings are unnumbered in the text; check the contents page)"
+        )
+    warnings += extract_section_text(doc, nodes, running)
     figures = extract_figures(doc, toc_pages, out_dir, dpi=args.dpi)
 
     lang = args.language or guess_language(doc)
@@ -537,6 +964,13 @@ def main() -> None:
     by_depth: dict[int, int] = defaultdict(int)
     for n in nodes:
         by_depth[n["depth"]] += 1
+    chars = sum(len(n["rawText"]) for n in nodes)
+    parents = {n["parentNodeId"] for n in nodes if n.get("parentNodeId")}
+    empty_nodes = sum(
+        1 for n in nodes if len(n["rawText"]) < 40 and n["nodeId"] not in parents
+    )
+    if not chapters:
+        warnings.append("no chapter detected: neither bookmarks nor headings")
 
     manifest = {
         "doc": {
@@ -552,7 +986,16 @@ def main() -> None:
             "sectionCount": len(nodes),
             "byDepth": dict(by_depth),
             "figureCount": len(figures),
+            # What the server's coverage score will see (`corpusQuality.ts`),
+            # so the skill can judge the pre-pass before writing anything.
+            "chars": chars,
+            "charsPerPage": round(chars / max(1, doc.page_count)),
+            "emptyLeaves": empty_nodes,
         },
+        # Anything the pre-pass could not decide on its own. The skill reads
+        # this list and looks at the pages it names; an empty list is the
+        # normal case on a native SIA norm.
+        "warnings": warnings,
         "sections": nodes,
         "figures": figures,
     }
@@ -568,6 +1011,8 @@ def main() -> None:
                 "sectionCount": len(nodes),
                 "byDepth": dict(by_depth),
                 "figureCount": len(figures),
+                "charsPerPage": round(chars / max(1, doc.page_count)),
+                "warnings": len(warnings),
                 "language": lang,
                 "pageCount": doc.page_count,
             },