@stratta/mcp 0.9.7 → 0.9.8

package/README.md

@@ -20,6 +20,25 @@ MCP server exposing Swiss engineering norms (SIA / Eurocodes) to Claude clients
 > a shell environment variable. If a key leaks, revoke it immediately at
 > https://stratta.ch/api-keys.
 
+## Do you need this package?
+
+Often not. Stratta also runs as a **remote connector** — one address, a browser
+sign-in, no key and no Node:
+
+```
+https://stratta.ch/mcp
+```
+
+That is the shorter path, and the only one that works in an agent running in the
+cloud (claude.ai). See https://stratta.ch/docs/en/guides/connect-remote.
+
+This package is what you want when:
+
+- you are **ingesting a norm** — it reads a PDF from your disk and runs a Python
+  pre-pass, neither of which a remote connector can reach;
+- your agent **cannot open a browser** — CI, a scheduled task, a server;
+- you would simply rather run the server yourself.
+
 ## Install
 
 ### Claude Code (recommended)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@stratta/mcp",
   "mcpName": "ch.stratta/mcp",
-  "version": "0.9.7",
+  "version": "0.9.8",
   "description": "MCP server exposing the engineering norms your firm is licensed for (SIA / Eurocodes) to any MCP client, via Stratta TreeRAG.",
   "license": "UNLICENSED",
   "author": "SmartFlow <hello@stratta.ch>",

package/scripts/ingest-prepass.py

@@ -77,10 +77,163 @@ def detect_running_text(doc: "fitz.Document") -> set[str]:
 CHAP_UPPER_RE = re.compile(
     r"^(\d+)\s+([A-ZÉÈÀÂÔÎÛÇ][^a-z\n]{3,120})\s*$", re.MULTILINE
 )
+# Norms adopted from CEN (SIA 262.6xx, SIA 267.1xx) set their headings in
+# sentence case, which CHAP_UPPER_RE rejects by design. Used only as a second
+# pass, when the uppercase form found almost nothing — on a native SIA norm it
+# would promote body sentences to chapters.
+CHAP_MIXED_RE = re.compile(
+    r"^(\d{1,2})\s+([A-ZÀ-Þ][A-Za-zÀ-ÿ][^\n]{2,90})\s*$", re.MULTILINE
+)
 ANNEX_BODY_RE = re.compile(
     r"^(?:ANNEXE|Annexe)\s+([A-Z])(?:\s*\(([^)]+)\))?\s*(.*?)$", re.MULTILINE
 )
 
+# A heading split across two lines: the number alone, the title underneath.
+# Comes from the numbering column of CEN-style layouts, where PyMuPDF reads the
+# column before the text. Left as-is, every regex below misses the heading.
+SPLIT_NUM_RE = re.compile(r"^\s*(\d{1,2}(?:\.\d{1,2}){0,3})\s*$")
+SPLIT_TITLE_RE = re.compile(r"^\s*([A-ZÀ-Þ][A-Za-zÀ-ÿ'’\-][^\n]{2,90})\s*$")
+# Words that open a continuing sentence, never a heading.
+SPLIT_STOP_RE = re.compile(
+    r"^(Le |La |Les |Il |Elle |Dans |Pour |Selon |Cette |Ce |Ces |Si |En |Au |Aux |De |Des |Du |Un |Une )",
+)
+
+
+def join_split_headings(text: str) -> str:
+    """Rewrite `12\\nTitre` as `12 Titre` so the heading regexes can see it.
+
+    Conservative on purpose: the title line must look like a title (starts
+    uppercase, no sentence-ending punctuation, not a sentence opener). A false
+    join invents a chapter, which is worse than missing one.
+    """
+    lines = text.splitlines()
+    out: list[str] = []
+    i = 0
+    while i < len(lines):
+        m = SPLIT_NUM_RE.match(lines[i])
+        if m and i + 1 < len(lines):
+            nxt = lines[i + 1]
+            t = SPLIT_TITLE_RE.match(nxt)
+            if t and not nxt.rstrip().endswith((".", ",", ";", ":")) and not SPLIT_STOP_RE.match(t.group(1)):
+                out.append(f"{m.group(1)} {t.group(1).strip()}")
+                i += 2
+                continue
+        out.append(lines[i])
+        i += 1
+    return "\n".join(out)
+
+
+def heading_text(doc: "fitz.Document", pageno: int) -> str:
+    """Page text prepared for heading detection (never for section content)."""
+    return join_split_headings(doc[pageno].get_text("text"))
+
+
+NOT_A_TITLE_RE = re.compile(
+    r"^(?:EN|SN|ISO|DIN|SIA|NOTE|Tableau|Figure|Table|Bild)\b|^\W|\.$", re.IGNORECASE
+)
+# A heading cut mid-phrase by the column break: "Résistance à la flexion au".
+TRUNCATED_RE = re.compile(
+    r"\b(?:ou|et|au|aux|de|des|du|le|la|les|un|une|dans|pour|par|sur|avec|sans|selon|entre)$",
+    re.IGNORECASE,
+)
+
+
+def plausible_title(title: str) -> bool:
+    """Reject what a numbered line can be other than a heading.
+
+    A stray table cell ("I ~ ~"), a normative reference ("EN 12063:2024 (F)") or
+    a wrapped sentence all match the heading shape. Promoting one invents a
+    chapter, and an invented chapter swallows the page range of a real one.
+    """
+    t = title.strip()
+    if not 3 <= len(t) <= 90:
+        return False
+    # A normative heading is capitalised. A lowercase one is a table row that
+    # happens to sit behind a number ("2.3 retrait des obstacles").
+    if not (t[0].isupper() or t[0].isdigit()):
+        return False
+    if NOT_A_TITLE_RE.search(t):
+        return False
+    if SPLIT_STOP_RE.match(t) or TRUNCATED_RE.search(t):
+        return False
+    letters = sum(1 for c in t if c.isalpha() or c.isspace())
+    return letters / len(t) >= 0.7
+
+
+def dense_heading_pages(doc: "fitz.Document", toc_pages: set[int]) -> set[int]:
+    """Pages listing many numbered headings: a contents page, whatever its
+    typography. Detecting it by leader dots alone misses the ones set without
+    them, and every heading read there points at the contents page instead of
+    the section it names."""
+    dense: set[int] = set()
+    for i in range(doc.page_count):
+        if i in toc_pages:
+            continue
+        found = {m.group(1) for m in SUB_RE.finditer(heading_text(doc, i))}
+        # A page of the body drills into one chapter; a contents page walks
+        # across several. Counting headings alone would drop a dense page of
+        # definitions (3.1 … 3.8), which is real content.
+        if len(found) >= 5 and len({n.split(".", 1)[0] for n in found}) >= 3:
+            dense.add(i)
+    return dense
+
+
+def prune_chapters(chapters: dict[str, dict]) -> dict[str, dict]:
+    """Keep the run of chapter numbers that reads like a table of contents.
+
+    Real chapters are consecutive and move forward through the document. A lone
+    "25" between chapters 1 and 2, or a chapter that starts thirty pages before
+    the one preceding it, is a detection artefact.
+    """
+    numbered = sorted(((int(k), k) for k in chapters if k.isdigit()))
+    if not numbered:
+        return chapters
+
+    # A norm with N detected chapters does not have a chapter 40. Missing a few
+    # headings is normal; a number far past the count is a table cell.
+    ceiling = 2 * len(numbered) + 3
+    numbered = [(n, k) for n, k in numbered if n <= ceiling]
+    if not numbered:
+        return chapters
+
+    # Longest run whose pages move forward, so one bad page does not discard
+    # every chapter after it.
+    best = [1] * len(numbered)
+    prev = [-1] * len(numbered)
+    for i in range(len(numbered)):
+        page_i = chapters[numbered[i][1]]["pageStart"]
+        for j in range(i):
+            if chapters[numbered[j][1]]["pageStart"] <= page_i and best[j] + 1 > best[i]:
+                best[i], prev[i] = best[j] + 1, j
+    idx = best.index(max(best))
+    chain = []
+    while idx != -1:
+        chain.append(numbered[idx][1])
+        idx = prev[idx]
+
+    # Missing a heading or two leaves a small gap; doubling (10 → 20 → 30) means
+    # the numbers stopped being chapters and started being table rows.
+    kept: dict[str, dict] = {}
+    previous = None
+    for key in reversed(chain):
+        n = int(key)
+        if previous is not None and n - previous > 3 and n >= 2 * previous:
+            break
+        kept[key] = chapters[key]
+        previous = n
+
+    # Annexes close a norm, in order. One that lands before the last chapter was
+    # read off the contents page, and its page range would swallow the document.
+    last_page = max((m["pageStart"] for m in kept.values()), default=0)
+    for key, meta in sorted(
+        ((k, m) for k, m in chapters.items() if not k.isdigit()), key=lambda kv: kv[0]
+    ):
+        if meta["pageStart"] < last_page:
+            continue
+        kept[key] = meta
+        last_page = meta["pageStart"]
+    return kept
+
 
 def extract_chapters(
     doc: "fitz.Document", toc_pages: set[int] | None = None
@@ -105,32 +258,64 @@ def extract_chapters(
     if chapters:
         return chapters
 
-    # Fallback: no bookmarks → scan body for uppercase chapter headings.
-    skip = toc_pages or set()
-    for pageno in range(doc.page_count):
-        if pageno in skip:
-            continue
-        text = doc[pageno].get_text("text")
-        for m in CHAP_UPPER_RE.finditer(text):
-            num = m.group(1)
-            title = clean(m.group(2))
-            if int(num) > 50:
-                continue
-            if num in chapters:
-                continue
-            chapters[num] = {"title": title, "pageStart": pageno + 1}
-        for m in ANNEX_BODY_RE.finditer(text):
-            letter = m.group(1)
-            kind = m.group(2) or ""
-            rest = clean(m.group(3) or "")
-            path = f"Annexe {letter}"
-            if path in chapters:
+    # Fallback: no bookmarks → scan body for chapter headings.
+    skip = set(toc_pages or set())
+
+    def scan(pattern: "re.Pattern[str]", skip_pages: set[int]) -> dict[str, dict]:
+        found: dict[str, dict] = {}
+        for pageno in range(doc.page_count):
+            if pageno in skip_pages:
                 continue
-            t = path + (f" ({kind})" if kind else "")
-            if rest and rest.lower() != path.lower():
-                t += f" — {rest}"
-            chapters[path] = {"title": t, "pageStart": pageno + 1}
-    return chapters
+            text = heading_text(doc, pageno)
+            for m in pattern.finditer(text):
+                num, title = m.group(1), clean(m.group(2))
+                if int(num) > 50 or num in found or not plausible_title(title):
+                    continue
+                found[num] = {"title": title, "pageStart": pageno + 1}
+            for m in ANNEX_BODY_RE.finditer(text):
+                letter, kind = m.group(1), m.group(2) or ""
+                rest = clean(m.group(3) or "")
+                path = f"Annexe {letter}"
+                if path in found:
+                    continue
+                t = path + (f" ({kind})" if kind else "")
+                if rest and rest.lower() != path.lower():
+                    t += f" — {rest}"
+                found[path] = {"title": t, "pageStart": pageno + 1}
+        return found
+
+    def toc_like_pages(found: dict[str, dict]) -> set[int]:
+        """Pages holding three or more chapter headings are the table of
+        contents, whatever the typography. Without this every chapter of such a
+        norm starts on the contents page, and every section body is wrong."""
+        per_page: dict[int, int] = defaultdict(int)
+        for meta in found.values():
+            per_page[meta["pageStart"]] += 1
+        return {p - 1 for p, n in per_page.items() if n >= 3}
+
+    for pattern in (CHAP_UPPER_RE, CHAP_MIXED_RE):
+        chapters = scan(pattern, skip)
+        extra = toc_like_pages(chapters)
+        if extra:
+            chapters = scan(pattern, skip | extra)
+        if len([k for k in chapters if k.isdigit()]) >= 3:
+            break
+
+    return prune_chapters(chapters)
+
+
+def chapter_page_spans(chapters: dict[str, dict], n_pages: int) -> dict[str, tuple[int, int]]:
+    """First and last page of each numbered chapter, from where the next starts."""
+    ordered = sorted(
+        ((k, v["pageStart"]) for k, v in chapters.items()),
+        key=lambda kv: kv[1],
+    )
+    spans: dict[str, tuple[int, int]] = {}
+    for idx, (key, start) in enumerate(ordered):
+        end = ordered[idx + 1][1] - 1 if idx + 1 < len(ordered) else n_pages
+        if key.isdigit():
+            spans[key] = (start, max(start, end))
+    return spans
 
 
 def extract_subsections(
@@ -138,13 +323,15 @@ def extract_subsections(
     chap_prefixes: set[str],
     toc_pages: set[int],
     running: set[str],
+    chapter_spans: dict[str, tuple[int, int]] | None = None,
 ) -> dict[str, tuple[str, int]]:
     """Numeric sub-sections (depths 2-4) detected in body pages."""
     out: dict[str, tuple[str, int]] = {}
+    skip = set(toc_pages) | dense_heading_pages(doc, toc_pages)
     for i in range(doc.page_count):
-        if i in toc_pages:
+        if i in skip:
             continue
-        text = doc[i].get_text("text")
+        text = heading_text(doc, i)
         for m in SUB_RE.finditer(text):
             num = m.group(1)
             title = clean(m.group(2))
@@ -153,7 +340,13 @@ def extract_subsections(
             top = num.split(".", 1)[0]
             if top not in chap_prefixes:
                 continue
-            if len(title) < 3 or re.fullmatch(r"[^A-Za-zÀ-ÿ]+", title):
+            # A sub-section sits inside its chapter. "1.1" found sixty pages
+            # after chapter 1 ended is a numbered line in an annexe, and keeping
+            # it hangs an unrelated page under the wrong parent.
+            span = chapter_spans.get(top) if chapter_spans else None
+            if span and not span[0] <= i + 1 <= span[1]:
+                continue
+            if not plausible_title(title):
                 continue
             if num in out:
                 continue
@@ -218,6 +411,44 @@ def extract_section_text(doc: "fitz.Document", nodes: list[dict]) -> None:
         chunks = page_texts[ps - 1 : pe]
         node["rawText"] = "\n".join(chunks).strip()
 
+    trim_shared_pages(nodes, page_texts)
+
+
+def trim_shared_pages(nodes: list[dict], page_texts: list[str]) -> None:
+    """Cut the opening page where several sections start on it.
+
+    Page granularity is the unit everywhere else, but two sections beginning on
+    the same page would otherwise carry byte-identical text — so a summary reads
+    like its neighbour's, and the table of contents misleads before anything is
+    even opened.
+    """
+    by_page: dict[int, list[dict]] = defaultdict(list)
+    for node in nodes:
+        by_page[node["pageStart"]].append(node)
+
+    for page, group in by_page.items():
+        if len(group) < 2:
+            continue
+        text = page_texts[page - 1]
+        # Where each section's heading sits on that page, in reading order.
+        marks: list[tuple[int, dict]] = []
+        for node in group:
+            title = node["title"].split(" — ")[-1].strip()
+            at = text.find(title) if len(title) >= 4 else -1
+            if at == -1:
+                at = text.find(f"{node['path']} ")
+            marks.append((at, node))
+        if any(at == -1 for at, _ in marks):
+            continue  # a heading we cannot locate: leave the whole page in place
+        marks.sort(key=lambda m: m[0])
+        for i, (at, node) in enumerate(marks):
+            end = marks[i + 1][0] if i + 1 < len(marks) else len(text)
+            head = text[at:end].strip()
+            if not head:
+                continue
+            rest = "\n".join(page_texts[node["pageStart"] : node["pageEnd"]]).strip()
+            node["rawText"] = f"{head}\n{rest}".strip() if rest else head
+
 
 def extract_figures(
     doc: "fitz.Document",
@@ -295,7 +526,8 @@ def main() -> None:
     running = detect_running_text(doc)
     chapters = extract_chapters(doc, toc_pages)
     chap_prefixes = {k for k in chapters if k.isdigit()}
-    subsections = extract_subsections(doc, chap_prefixes, toc_pages, running)
+    spans = chapter_page_spans(chapters, doc.page_count)
+    subsections = extract_subsections(doc, chap_prefixes, toc_pages, running, spans)
     nodes = build_tree(chapters, subsections, doc.page_count)
     extract_section_text(doc, nodes)
     figures = extract_figures(doc, toc_pages, out_dir, dpi=args.dpi)